mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-08 04:00:15 +00:00
Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 BingEdward
|
c449209168 | ||
|
Sameer KadamandSameer Kadam
|
7be1582786 | ||
|
Sakshi JainandMohd Kaif
|
3a69721abf | ||
|
Sameer Kadam
|
07c74cc544 | ||
|
Kevin
|
9c38fd49e5 | ||
|
BingEdward
|
0babf0d787 | ||
|
Zohaib Hassnain
|
82fd1b8d88 | ||
|
|
97840da51b | ||
|
Mohd KaifandClaude Sonnet 5
|
b8011eb44a |
+1
-1
@@ -19,7 +19,7 @@
|
||||
.github
|
||||
.github/**
|
||||
!.github/requirements/
|
||||
!.github/requirements/explorer-extra-py314.txt
|
||||
!.github/requirements/explorer-extra-py313.txt
|
||||
!.github/requirements/pep517-build.txt
|
||||
.claude
|
||||
.claude/**
|
||||
|
||||
@@ -8,7 +8,7 @@ These exist because OpenSSF Scorecard's Pinned-Dependencies check flags any
|
||||
the project's own local-source install.
|
||||
|
||||
Each `.txt` was generated from the adjacent `.in` (or, for `explorer-extra-py311.txt`,
|
||||
`explorer-extra-py314.txt`, and `base-deps.txt`, from `pyproject.toml` directly) with:
|
||||
`explorer-extra-py313.txt`, and `base-deps.txt`, from `pyproject.toml` directly) with:
|
||||
|
||||
```
|
||||
uv pip compile <input> --python-version 3.11 --python-platform linux \
|
||||
@@ -29,7 +29,7 @@ each file's own autogenerated header comment for its exact command).
|
||||
| `bootstrap.txt` | security-scan.yml, benchmark.yml | pip, setuptools (upgrade before anything else) |
|
||||
| `pep517-build.txt` | ci.yml, benchmark.yml, Dockerfile | exact `[build-system] requires` from `pyproject.toml` (setuptools, wheel) - installed with `--no-build-isolation` before any `pip install -e .` / `pip install .`, since `--no-deps` alone doesn't stop pip's PEP 517 build isolation from fetching those two *unhashed* |
|
||||
| `explorer-extra-py311.txt` | ci.yml | semantica's base deps + the `explorer` extra, resolved for python 3.11 |
|
||||
| `explorer-extra-py314.txt` | Dockerfile | the same, resolved for python 3.14 (the image's actual interpreter) |
|
||||
| `explorer-extra-py313.txt` | Dockerfile | the same, resolved for python 3.13 (the image's actual interpreter) |
|
||||
| `pytest-tool.txt` | ci.yml | pytest, for the pre-all-extras deterministic test |
|
||||
| `uv-tool.txt` | ci.yml | uv, to verify requirements-ci.txt is current |
|
||||
| `build-tools.txt` | ci.yml, release.yml | build, wheel |
|
||||
@@ -39,19 +39,19 @@ each file's own autogenerated header comment for its exact command).
|
||||
| `base-deps.txt` | benchmark.yml | semantica's base deps (no extras) |
|
||||
| `benchmark-extra.txt` | benchmark.yml | the benchmark-only libs (neo4j, pdfplumber, etc.) |
|
||||
|
||||
`explorer-extra-py311.txt` and `explorer-extra-py314.txt` are large (they
|
||||
mirror most of `requirements-ci.txt`) because semantica's `dependencies`
|
||||
list in `pyproject.toml` isn't extras-gated - installing the package at all
|
||||
pulls the full base set. That's expected, not a mistake.
|
||||
`explorer-extra-py31{1,3}.txt` and `base-deps.txt` are large (they mirror
|
||||
most of `requirements-ci.txt`) because semantica's `dependencies` list in
|
||||
`pyproject.toml` isn't extras-gated - installing the package at all pulls
|
||||
the full base set. That's expected, not a mistake.
|
||||
|
||||
`explorer-extra-py311.txt` and `explorer-extra-py314.txt` are **not**
|
||||
`explorer-extra-py311.txt` and `explorer-extra-py313.txt` are **not**
|
||||
interchangeable, and can't be collapsed into one file compiled for either
|
||||
version: `librosa`'s `audioread` dependency needs `standard-aifc` /
|
||||
`standard-sunau` only under `python_version >= "3.13"` (Python 3.13 dropped
|
||||
`aifc`/`sunau` from stdlib). A file resolved for 3.11 simply omits those
|
||||
packages' hashes, so installing it with `--require-hashes` on a real 3.14
|
||||
packages' hashes, so installing it with `--require-hashes` on a real 3.13
|
||||
interpreter (the Dockerfile's base image) fails outright rather than
|
||||
silently under-pinning. Any other file shared across a 3.11 and 3.14
|
||||
silently under-pinning. Any other file shared across a 3.11 and 3.13
|
||||
consumer would need the same split if it hits a similar stdlib-removal
|
||||
edge case - check for `ERROR: In --require-hashes mode, all requirements
|
||||
must have their versions pinned` on the *other* Python version before
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
# This file was autogenerated by uv via the following command:
|
||||
# uv pip compile pyproject.toml --extra explorer --python-version 3.14 --python-platform linux --generate-hashes --constraint requirements-ci.txt -o .github/requirements/explorer-extra-py314.txt
|
||||
# uv pip compile pyproject.toml --extra explorer --python-version 3.13 --python-platform linux --generate-hashes --constraint requirements-ci.txt -o .github/requirements/explorer-extra-py313.txt
|
||||
annotated-doc==0.0.5 \
|
||||
--hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \
|
||||
--hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb
|
||||
@@ -112,8 +112,8 @@ jobs:
|
||||
# `uv pip compile pyproject.toml --extra explorer --python-version 3.11 --constraint requirements-ci.txt --generate-hashes`
|
||||
# - regenerate it the same way if pyproject.toml's base/explorer
|
||||
# deps change. Resolved specifically for this job's python 3.11
|
||||
# (see the Dockerfile's explorer-extra-py314.txt for why this
|
||||
# can't be shared with python 3.14: audioread needs extra
|
||||
# (see the Dockerfile's explorer-extra-py313.txt for why this
|
||||
# can't be shared with python 3.13: audioread needs extra
|
||||
# standard-aifc/standard-sunau hashes only on 3.13+).
|
||||
#
|
||||
# --no-deps only skips *runtime* dependency resolution - `-e .`
|
||||
|
||||
@@ -12,7 +12,7 @@ on:
|
||||
- 'README.md'
|
||||
- 'LICENSE'
|
||||
- 'MANIFEST.in'
|
||||
- '.github/requirements/explorer-extra-py314.txt'
|
||||
- '.github/requirements/explorer-extra-py313.txt'
|
||||
- '.github/requirements/pep517-build.txt'
|
||||
- 'semantica/**'
|
||||
- 'integrations/**'
|
||||
|
||||
+18
-7
@@ -20,7 +20,18 @@ RUN mkdir -p /app/semantica && npm run build
|
||||
# .github/dependabot.yml opens a PR bumping the digest pin above. Also: this
|
||||
# image only serves plain HTTP via uvicorn and never opens a QUIC listener,
|
||||
# so the bug isn't reachable here regardless.
|
||||
FROM python:3.14-slim@sha256:cae66f2ef0ec51a9891263eeee7f987dacf0a9879e8aa9353d5606e0530619a5 AS runtime
|
||||
#
|
||||
# Pinned to 3.13, NOT 3.14: #1290 bumped this to python:3.14-slim and broke
|
||||
# the build outright (Container Security Scan, every run since) - gensim
|
||||
# (a base, non-extras-gated dependency) ships no cp314 wheel on PyPI yet, so
|
||||
# pip falls back to building it from source, which needs a C compiler this
|
||||
# slim image doesn't carry ("error: [Errno 2] No such file or directory:
|
||||
# 'gcc'"). Revisit the 3.14 bump once gensim (and anything else pulled in
|
||||
# transitively) publishes cp314 wheels - check with
|
||||
# `pip index versions gensim` / the project's PyPI files page, not just
|
||||
# whether `uv pip compile` resolves (resolution only reads sdist metadata,
|
||||
# it doesn't attempt the build that fails here).
|
||||
FROM python:3.13-slim@sha256:7ce4b6dfe35e55397b7cda544f8a13f191b7ae28dc5aad71fe664dbc9bc2623f AS runtime
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
@@ -34,18 +45,18 @@ RUN groupadd --system semantica \
|
||||
&& useradd --system --gid semantica --home-dir /app --shell /usr/sbin/nologin semantica
|
||||
|
||||
COPY pyproject.toml README.md LICENSE MANIFEST.in \
|
||||
.github/requirements/explorer-extra-py314.txt .github/requirements/pep517-build.txt ./
|
||||
.github/requirements/explorer-extra-py313.txt .github/requirements/pep517-build.txt ./
|
||||
COPY semantica/ ./semantica/
|
||||
COPY integrations/ ./integrations/
|
||||
COPY --from=frontend-builder /app/semantica/static ./semantica/static
|
||||
|
||||
# explorer-extra-py314.txt is `uv pip compile pyproject.toml --extra explorer
|
||||
# --python-version 3.14 --constraint requirements-ci.txt --generate-hashes`
|
||||
# explorer-extra-py313.txt is `uv pip compile pyproject.toml --extra explorer
|
||||
# --python-version 3.13 --constraint requirements-ci.txt --generate-hashes`
|
||||
# (see ci.yml's explorer-extra-py311.txt for the CI counterpart, resolved
|
||||
# for CI's python 3.11 instead - the two aren't interchangeable: audioread
|
||||
# (via librosa) needs standard-aifc/standard-sunau only on python>=3.13,
|
||||
# since aifc/sunau left stdlib there, so a 3.11-resolved lockfile is
|
||||
# missing hashes pip needs on this image's actual 3.14 interpreter and
|
||||
# missing hashes pip needs on this image's actual 3.13 interpreter and
|
||||
# --require-hashes fails outright rather than silently under-pinning).
|
||||
# Every fetched package is hash-verified (Scorecard Pinned-Dependencies)
|
||||
# and pinned to the same versions CI audited, e.g. msgpack==1.2.1 and
|
||||
@@ -59,9 +70,9 @@ COPY --from=frontend-builder /app/semantica/static ./semantica/static
|
||||
# build-system.requires; installing it first and passing
|
||||
# --no-build-isolation makes pip reuse those hash-verified copies instead
|
||||
# of fetching its own.
|
||||
RUN pip install --no-cache-dir -r explorer-extra-py314.txt -r pep517-build.txt --require-hashes \
|
||||
RUN pip install --no-cache-dir -r explorer-extra-py313.txt -r pep517-build.txt --require-hashes \
|
||||
&& pip install --no-cache-dir --no-deps --no-build-isolation . \
|
||||
&& rm -f explorer-extra-py314.txt pep517-build.txt \
|
||||
&& rm -f explorer-extra-py313.txt pep517-build.txt \
|
||||
&& chown -R semantica:semantica /app
|
||||
|
||||
USER semantica
|
||||
|
||||
+43
-21
@@ -64,7 +64,7 @@ Whether you're running your first pipeline or deploying Semantica in production,
|
||||
[Temporal Graphs notebook](https://github.com/semantica-agi/semantica/blob/main/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb): `valid_from`/`valid_until`, Allen interval algebra, point-in-time queries.
|
||||
</Step>
|
||||
<Step title="Ontology-driven knowledge bases">
|
||||
[Ontology notebook](https://github.com/semantica-agi/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb): auto-generation, SHACL validation, Ontology Hub (v0.5.0).
|
||||
[Ontology notebook](https://github.com/semantica-agi/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb): auto-generation, SHACL validation, Ontology Hub.
|
||||
</Step>
|
||||
<Step title="Advanced visualization">
|
||||
[Complete Visualization Suite notebook](https://github.com/semantica-agi/semantica/blob/main/cookbook/advanced/03_Complete_Visualization_Suite.ipynb): UMAP, t-SNE, community layouts, embedding projections.
|
||||
@@ -86,10 +86,10 @@ All settings can be overridden with environment variables: no code changes neede
|
||||
| OpenAI API Key | `OPENAI_API_KEY` | `None` |
|
||||
| Groq API Key | `GROQ_API_KEY` | `None` |
|
||||
| Anthropic API Key | `ANTHROPIC_API_KEY` | `None` |
|
||||
| Embedding Provider | `SEMANTICA_EMBEDDING_PROVIDER` | `"openai"` |
|
||||
| Graph Backend | `SEMANTICA_GRAPH_BACKEND` | `"networkx"` |
|
||||
| Log Level | `SEMANTICA_LOG_LEVEL` | `"INFO"` |
|
||||
| Log Format | `SEMANTICA_LOG_FORMAT` | `"text"` |
|
||||
| Graph Store Backend | `GRAPH_STORE_DEFAULT_BACKEND` | `"neo4j"` |
|
||||
| Vector Store Backend | `VECTOR_STORE_DEFAULT_BACKEND` | `"faiss"` |
|
||||
| Server Host | `SEMANTICA_HOST` | `"127.0.0.1"` |
|
||||
| Server API Key | `SEMANTICA_API_KEY` | `None` |
|
||||
|
||||
|
||||
## Troubleshooting
|
||||
@@ -146,10 +146,15 @@ Also reduce batch sizes and enable streaming ingestion for large corpora.
|
||||
Enable parallel execution and GPU acceleration:
|
||||
|
||||
```python
|
||||
from semantica.pipeline import Pipeline
|
||||
from semantica.pipeline import ParallelismManager, Task
|
||||
|
||||
pipeline = Pipeline(workers=8, batch_size=32)
|
||||
pipeline.run(sources)
|
||||
# Run pipeline tasks concurrently across worker threads
|
||||
manager = ParallelismManager(max_workers=8)
|
||||
tasks = [
|
||||
Task("task_1", lambda: "process part 1"),
|
||||
Task("task_2", lambda: "process part 2"),
|
||||
]
|
||||
results = manager.execute_parallel(tasks)
|
||||
```
|
||||
|
||||
```bash
|
||||
@@ -160,19 +165,19 @@ pip install "semantica[gpu]" # CUDA-backed embeddings
|
||||
|
||||
<Accordion title="Windows [all] installation fails" icon="windows">
|
||||
|
||||
Fixed in **v0.5.0**. Upgrade:
|
||||
Upgrade to the latest release:
|
||||
|
||||
```bash
|
||||
pip install --upgrade semantica
|
||||
```
|
||||
|
||||
Or install extras individually: `pip install "semantica[core]"`, then add `[llm-openai]`, `[gpu]`, etc. as needed.
|
||||
Or install extras individually: `pip install semantica`, then add `[llm-openai]`, `[gpu]`, etc. as needed.
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="cp1252 encoding crash on Windows" icon="windows">
|
||||
|
||||
Fixed in **v0.5.0**. For earlier versions, set the encoding environment variable:
|
||||
Set the encoding environment variable:
|
||||
|
||||
```bash
|
||||
set PYTHONIOENCODING=utf-8
|
||||
@@ -202,27 +207,44 @@ Use NetworkX for local development and prototyping. Switch to a persistent backe
|
||||
|
||||
<Accordion title="Batch processing for large corpora" icon="layer-group">
|
||||
|
||||
Process documents in batches rather than one at a time. Configure `chunk_size` based on available RAM: a good starting point is 1,000 documents per batch on a 16 GB machine.
|
||||
Process documents in batches rather than one at a time. Split large texts into chunks and extract entities in batches:
|
||||
|
||||
```python
|
||||
from semantica.pipeline import Pipeline
|
||||
from semantica.split import TextSplitter
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
|
||||
pipeline = Pipeline(workers=8, batch_size=32)
|
||||
pipeline.run(sources)
|
||||
document_text = "Acme Corp announced record revenue in Seattle. CEO Jane Doe presented results."
|
||||
splitter = TextSplitter(chunk_size=1000, chunk_overlap=100)
|
||||
chunks = splitter.split(document_text)
|
||||
|
||||
extractor = NERExtractor()
|
||||
batch_entities = extractor.extract_entities_batch([c.text for c in chunks])
|
||||
```
|
||||
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Deduplication v2: up to 7× faster" icon="bolt">
|
||||
|
||||
If deduplication is a bottleneck, switch from v1 strategies to the v2 engine:
|
||||
If deduplication is a bottleneck, use candidate blocking to reduce O(n²) comparisons before similarity scoring:
|
||||
|
||||
```python
|
||||
resolver = EntityResolver()
|
||||
merged = resolver.resolve(entities, strategy="semantic_v2") # up to 7x faster
|
||||
from semantica.deduplication import DuplicateDetector, EntityMerger
|
||||
|
||||
entities = [
|
||||
{"id": "1", "name": "Acme Corp", "type": "Company"},
|
||||
{"id": "2", "name": "Acme Corporation", "type": "Company"},
|
||||
{"id": "3", "name": "Globex", "type": "Company"},
|
||||
]
|
||||
|
||||
# Fast candidate blocking for large entity sets
|
||||
detector = DuplicateDetector(similarity_threshold=0.8)
|
||||
duplicates = detector.detect_duplicates(entities, candidate_strategy="blocking_v2")
|
||||
|
||||
merger = EntityMerger()
|
||||
merged = merger.merge_duplicates(entities, strategy="keep_most_complete")
|
||||
```
|
||||
|
||||
The `blocking_v2`, `hybrid_v2`, and `semantic_v2` strategies reduce O(n²) comparisons via candidate blocking before similarity scoring.
|
||||
The `blocking_v2` and `hybrid_v2` candidate strategies filter candidate pairs before calculating fine-grained similarity.
|
||||
|
||||
</Accordion>
|
||||
|
||||
@@ -233,8 +255,8 @@ The `blocking_v2`, `hybrid_v2`, and `semantic_v2` strategies reduce O(n²) compa
|
||||
|
||||
- **API keys**: store in environment variables or a secrets manager; never commit them to version control; rotate on a schedule
|
||||
- **Sensitive data**: use local embedding models (Ollama, HuggingFace) for PII or classified content; avoid sending sensitive data to external APIs without data handling agreements
|
||||
- **Graph exports**: encrypt sensitive exports at rest; use the v0.5.0 SSRF-safe `base_url` validation when configuring custom LLM gateways
|
||||
- **XML ingestion**: always use `XMLIngestor` (v0.5.0), which uses the XXE-safe lxml backend; never parse untrusted XML with the standard library parser
|
||||
- **Graph exports**: encrypt sensitive exports at rest; use SSRF-safe `base_url` validation when configuring custom LLM gateways
|
||||
- **XML ingestion**: always use `XMLIngestor`, which uses the XXE-safe lxml backend; never parse untrusted XML with the standard library parser
|
||||
|
||||
- [Cookbook](/cookbook): interactive Jupyter notebooks from beginner to advanced.
|
||||
- [FAQ](/faq): common questions answered.
|
||||
|
||||
@@ -160,7 +160,7 @@ No installation or API key required. FAISS requires `pip install faiss-cpu`.
|
||||
<Tab title="Pinecone">
|
||||
|
||||
```bash
|
||||
pip install "semantica[pinecone]"
|
||||
pip install "semantica[vectorstore-pinecone]"
|
||||
```
|
||||
|
||||
```python
|
||||
@@ -178,7 +178,7 @@ store = VectorStore(
|
||||
<Tab title="Weaviate">
|
||||
|
||||
```bash
|
||||
pip install "semantica[weaviate]"
|
||||
pip install "semantica[vectorstore-weaviate]"
|
||||
```
|
||||
|
||||
```python
|
||||
@@ -194,7 +194,7 @@ store = VectorStore(
|
||||
<Tab title="Qdrant">
|
||||
|
||||
```bash
|
||||
pip install "semantica[qdrant]"
|
||||
pip install "semantica[vectorstore-qdrant]"
|
||||
```
|
||||
|
||||
```python
|
||||
@@ -210,7 +210,7 @@ store = VectorStore(
|
||||
<Tab title="PgVector">
|
||||
|
||||
```bash
|
||||
pip install "semantica[pgvector]"
|
||||
pip install "semantica[vectorstore-pgvector]"
|
||||
```
|
||||
|
||||
```python
|
||||
|
||||
+9
-9
@@ -4409,15 +4409,15 @@ pillow==12.3.0 \
|
||||
# python-pptx
|
||||
# rapidocr
|
||||
# torchvision
|
||||
pinecone==9.1.0 \
|
||||
--hash=sha256:461632bb07919da32b943100b8a047c74be53a6aa15c8b7679bff7a0f834c939 \
|
||||
--hash=sha256:6c3a6dfa577dc11aed3197e1b221e65522603e9e1f6bd27a1b504a0909b3559f \
|
||||
--hash=sha256:d3871bd3f39cb430ae8470158dc9c5dcffbac5ae31d144d9a7c3b351ac51755f \
|
||||
--hash=sha256:d53fe6f4978ab0642eb2d3a0ee3b2576ccfeebaa11e0690b18e67dac4e057047 \
|
||||
--hash=sha256:e930ba819f5b7e20aac688d04c840a8b6fbc6d12630d71303bb2130881a9d169 \
|
||||
--hash=sha256:fc71ec431108de2df1a1978d3a24ac16f74ba3d8f3265c3760f969386e8742b8 \
|
||||
--hash=sha256:fe6aeaf6515e9021984755ebc162f643c79d98056059aab2e765962a7538818c \
|
||||
--hash=sha256:ffae8fb7cbb4056b920586629f15b08107350be4802a5637d10b31e2ad841f9c
|
||||
pinecone==10.0.0 \
|
||||
--hash=sha256:0994270c514b16c72ec94dd6c29ff2708b81d30ff8467e19de192a28a7c86b7e \
|
||||
--hash=sha256:0e05956a3201b1fbb54a1861277df919318a4941797f2d87fd558ac5ec232151 \
|
||||
--hash=sha256:2f4e3200ee3562d195802b363487dd7fe6039a8c13630fc25fa3e8726c7a8654 \
|
||||
--hash=sha256:3ab0c843b4fb04fbac22f1b8455e389063208a50f6a95d7a90627939968198de \
|
||||
--hash=sha256:6066bbe9a7ae1d667cde08d262deb6fbea6feb35deb9177dd47141b55bbd9833 \
|
||||
--hash=sha256:94d4c64779f3213a5cc538d3bd10a873da192cb9b0039db56690e556ba00b55c \
|
||||
--hash=sha256:995c06e905940b10bb2aefe653225340b5a3f56f3efb373fe07f4e57b5043705 \
|
||||
--hash=sha256:d482ed27a805cbd4aca2660da212008dd6e41d87d255279f405ff13b725970e2
|
||||
# via semantica (pyproject.toml)
|
||||
platformdirs==4.11.7 \
|
||||
--hash=sha256:4f41487eeeeeb07f3a6625e61d9bc0ae6809f92d3386dbd74392fbb76108104d \
|
||||
|
||||
+32
-15
@@ -95,7 +95,26 @@ _ERROR_HINTS: Dict[type, str] = {
|
||||
}
|
||||
|
||||
|
||||
def _json_error_mode() -> bool:
|
||||
"""True when this invocation promised machine-readable stdout.
|
||||
|
||||
Covers both the global ``--json`` flag (stored on the CLI context) and a
|
||||
subcommand's local ``--json`` flag (uniformly named ``local_json``).
|
||||
"""
|
||||
ctx = click.get_current_context(silent=True)
|
||||
if ctx is None:
|
||||
return False
|
||||
if ctx.params.get("local_json"):
|
||||
return True
|
||||
return isinstance(ctx.obj, CLIContext) and ctx.obj.json_output
|
||||
|
||||
|
||||
def _show_error_card(title: str, detail: str, hint: Optional[str] = None) -> None:
|
||||
if _json_error_mode():
|
||||
# --json promises machine-readable stdout with errors on stderr, so
|
||||
# emit a structured error line there instead of a Rich panel.
|
||||
click.echo(json.dumps({"error": detail, "type": title}), err=True)
|
||||
return
|
||||
body = f"[bold]{title}[/bold]\n[{_DIM}]{detail}[/{_DIM}]"
|
||||
if hint:
|
||||
body += f"\n\n[{_KEY}]→[/{_KEY}] [{_DIM}]{hint}[/{_DIM}]"
|
||||
@@ -105,7 +124,7 @@ def _show_error_card(title: str, detail: str, hint: Optional[str] = None) -> Non
|
||||
|
||||
|
||||
def _run_with_error_handling(action: Callable[[], None]) -> None:
|
||||
"""Run a CLI action with Rich error cards on failure."""
|
||||
"""Run a CLI action with error cards (or JSON-mode stderr errors) on failure."""
|
||||
try:
|
||||
action()
|
||||
except click.ClickException as exc:
|
||||
@@ -4745,15 +4764,10 @@ def mcp_list_tools(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
cli_ctx = _require_ctx(cli_ctx)
|
||||
|
||||
def _action() -> None:
|
||||
try:
|
||||
from semantica_mcp.mcp.tools import __all__ as tools
|
||||
except ImportError:
|
||||
tools = [
|
||||
"extract_entities", "extract_relations", "build_graph",
|
||||
"query_graph", "get_graph_analytics", "run_reasoning",
|
||||
"record_decision", "get_decisions", "export_graph",
|
||||
"validate_shacl", "get_provenance", "embed_and_search",
|
||||
]
|
||||
# Same catalog the server exposes via tools/list, so `list-tools`
|
||||
# and `mcp start` can't drift (issue #1355).
|
||||
from semantica_mcp.mcp.tools import TOOL_DEFINITIONS
|
||||
tools = [t["name"] for t in TOOL_DEFINITIONS]
|
||||
if _is_json(cli_ctx, local_json):
|
||||
_jecho({"tools": list(tools)})
|
||||
else:
|
||||
@@ -4786,12 +4800,15 @@ def mcp_call(cli_ctx: CLIContext, tool_name: str, args: str, local_json: bool) -
|
||||
tool_args = json.loads(args)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise click.ClickException(f"Invalid JSON in --args: {exc}") from exc
|
||||
if not isinstance(tool_args, dict):
|
||||
raise click.ClickException("--args must be a JSON object")
|
||||
# Dispatch through the same server `mcp start` spawns; its session
|
||||
# module never defined MCPSession (issue #1355).
|
||||
from semantica_mcp.mcp.server import UnknownToolError, call_tool
|
||||
try:
|
||||
from semantica_mcp.mcp.session import MCPSession
|
||||
session = MCPSession(config=cli_ctx.config.to_dict())
|
||||
result = session.call_tool(tool_name, **tool_args)
|
||||
except ImportError as exc:
|
||||
raise click.ClickException(f"MCP module not available: {exc}") from exc
|
||||
result = call_tool(tool_name, tool_args)
|
||||
except UnknownToolError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
if _is_json(cli_ctx, local_json):
|
||||
_jecho(result if isinstance(result, (dict, list)) else {"result": str(result)})
|
||||
else:
|
||||
|
||||
@@ -14,13 +14,15 @@ not. It *composes* the existing public APIs; nothing in ``context_graph.py`` or
|
||||
``agent_memory.py`` changes, and ``ContextGraph`` keeps its graph-scope
|
||||
contract.
|
||||
|
||||
The property that matters is honest partial reporting. FAISS exposes no delete
|
||||
at all -- a flat FAISS index cannot remove individual vectors without a full
|
||||
rebuild -- so erasure is genuinely not completable on it today. Milvus and
|
||||
Weaviate now expose ``delete_vectors`` and are fully supported. The receipt
|
||||
says ``unsupported`` for FAISS rather than reporting a success it did not
|
||||
achieve -- a receipt that reads
|
||||
"graph: erased, memory: 14 erased, vectors: unsupported on faiss" is
|
||||
The property that matters is honest partial reporting. FAISS Flat indices now
|
||||
expose ``delete_vectors`` backed by native ``remove_ids``, so erasure is
|
||||
completable on them. FAISS IVF indices explicitly reject deletion because
|
||||
their internal labels are not compacted after ``remove_ids``, which would
|
||||
desynchronize search results from the ``vector_ids`` mapping. HNSW does not
|
||||
implement ``remove_ids`` at all. Both IVF and HNSW report ``unsupported``.
|
||||
Milvus and Weaviate are also fully supported. The receipt says ``unsupported``
|
||||
rather than reporting a success it did not achieve -- a receipt that reads
|
||||
"graph: erased, memory: 14 erased, vectors: unsupported on faiss/hnsw" is
|
||||
actionable; a bare ``True`` is a compliance liability.
|
||||
|
||||
Example:
|
||||
@@ -379,8 +381,9 @@ class ErasureCoordinator:
|
||||
|
||||
method_name, target = _vector_delete_capability(self.vector_store)
|
||||
if method_name is None:
|
||||
# FAISS exposes no delete at all; it cannot remove vectors from a
|
||||
# flat index without a full rebuild.
|
||||
# FAISS HNSW does not implement remove_ids and FAISS IVF
|
||||
# does not compact labels after remove_ids. Only Flat indices
|
||||
# currently support deletion via this code path.
|
||||
self.logger.warning(
|
||||
"Vector backend %r exposes no delete; %d vector id(s) for %r "
|
||||
"were not erased",
|
||||
|
||||
@@ -128,6 +128,11 @@ class FAISSIndex:
|
||||
self.index_type = index_type
|
||||
self.vector_ids: List[str] = []
|
||||
self.metadata: Dict[str, Dict[str, Any]] = {}
|
||||
# Monotonic counter for default ID generation, mirroring FAISSStore._next_id.
|
||||
# Persisted in the .meta.json sidecar so that load_index restores the
|
||||
# correct value rather than deriving it from ntotal (which underestimates
|
||||
# when vectors have been deleted and sparse gaps exist).
|
||||
self.next_id: int = 0
|
||||
|
||||
def add_vectors(self, vectors: np.ndarray, ids: Optional[List[str]] = None):
|
||||
"""
|
||||
@@ -199,6 +204,98 @@ class FAISSIndex:
|
||||
"""Get metadata by ID."""
|
||||
return self.metadata.get(vector_id)
|
||||
|
||||
def delete_vectors(self, vector_ids_to_delete: List[str]) -> Dict[str, Any]:
|
||||
"""Remove vectors by their external string IDs.
|
||||
|
||||
Translates each requested external ID to its sequential internal FAISS
|
||||
position, calls ``index.remove_ids`` with an ``IDSelectorBatch`` of
|
||||
those positions, then updates ``vector_ids`` and ``metadata`` to match
|
||||
the compacted index. The invariant ``len(self.vector_ids) ==
|
||||
self.index.ntotal`` is re-checked after the operation.
|
||||
|
||||
**Persistence:** the deletion is in-memory only. Call
|
||||
:meth:`FAISSStore.save_index` afterwards to write the updated state to
|
||||
disk; without that call the deleted vectors will reappear on the next
|
||||
process restart.
|
||||
|
||||
Args:
|
||||
vector_ids_to_delete: External string IDs to remove. Unknown IDs
|
||||
are silently ignored. Duplicate entries are deduplicated.
|
||||
|
||||
Returns:
|
||||
``{"delete_count": N}`` where *N* is the number of vectors
|
||||
actually removed from the FAISS index (0 if none existed).
|
||||
|
||||
Raises:
|
||||
NotImplementedError: If the underlying FAISS index type does not
|
||||
support ``remove_ids`` (e.g. ``IndexHNSWFlat``). No state is
|
||||
mutated before this is raised.
|
||||
ProcessingError: For any other unexpected FAISS error.
|
||||
"""
|
||||
if not vector_ids_to_delete:
|
||||
return {"delete_count": 0}
|
||||
|
||||
delete_set = set(vector_ids_to_delete)
|
||||
|
||||
# Map external string IDs to sequential internal FAISS positions.
|
||||
positions = [
|
||||
pos
|
||||
for pos, vid in enumerate(self.vector_ids)
|
||||
if vid in delete_set
|
||||
]
|
||||
if not positions:
|
||||
return {"delete_count": 0}
|
||||
|
||||
# IVF-family indices (IndexIVFFlat, etc.) do NOT compact their internal
|
||||
# labels after remove_ids: the surviving vectors keep their original
|
||||
# sequential labels. The current architecture interprets search-result
|
||||
# labels as offsets into vector_ids, so a non-compacting removal would
|
||||
# silently return wrong external IDs and cause IndexError on labels
|
||||
# beyond the compacted list length. Raise NotImplementedError here so
|
||||
# callers get STATUS_UNSUPPORTED rather than silent data corruption.
|
||||
# (Flat and PQ indices DO compact labels, so they are safe.)
|
||||
if FAISS_AVAILABLE and isinstance(self.index, faiss.IndexIVF):
|
||||
raise NotImplementedError(
|
||||
f"The underlying FAISS index type ({type(self.index).__name__}) "
|
||||
"does not compact internal labels after remove_ids, which would "
|
||||
"desynchronize search labels from the vector_ids mapping. Use a "
|
||||
"Flat index for deletion support, or rebuild the IVF index without "
|
||||
"the deleted vectors."
|
||||
)
|
||||
|
||||
sel = faiss.IDSelectorBatch(np.array(positions, dtype=np.int64))
|
||||
try:
|
||||
removed = self.index.remove_ids(sel)
|
||||
except RuntimeError as exc:
|
||||
if "not implemented" in str(exc).lower():
|
||||
# HNSW and a handful of other index types do not implement
|
||||
# remove_ids. Raise NotImplementedError so callers (and the
|
||||
# ErasureCoordinator) can distinguish "unsupported" from a
|
||||
# transient failure worth retrying.
|
||||
raise NotImplementedError(
|
||||
f"The underlying FAISS index type "
|
||||
f"({type(self.index).__name__}) does not support "
|
||||
"remove_ids(). Use a Flat index for deletion support, "
|
||||
"or rebuild the index without the deleted vectors."
|
||||
) from exc
|
||||
raise ProcessingError(f"FAISS remove_ids failed: {exc}") from exc
|
||||
|
||||
# Keep state consistent: update the Python-side list and metadata
|
||||
# dict to mirror the now-compacted FAISS array. The list comprehension
|
||||
# cannot raise, so the index and its metadata are always updated
|
||||
# together (no partial-mutation window).
|
||||
self.vector_ids = [vid for vid in self.vector_ids if vid not in delete_set]
|
||||
for vid in delete_set:
|
||||
self.metadata.pop(vid, None)
|
||||
|
||||
if len(self.vector_ids) != self.index.ntotal:
|
||||
raise ProcessingError(
|
||||
f"FAISSIndex invariant broken after delete_vectors: "
|
||||
f"vector_ids={len(self.vector_ids)}, ntotal={self.index.ntotal}. "
|
||||
"This indicates a bug in FAISS remove_ids or the deletion logic."
|
||||
)
|
||||
return {"delete_count": removed}
|
||||
|
||||
def save(self, path: Union[str, Path]):
|
||||
"""Save index to disk.
|
||||
|
||||
@@ -223,6 +320,7 @@ class FAISSIndex:
|
||||
"metadata": self.metadata,
|
||||
"dimension": self.dimension,
|
||||
"index_type": self.index_type,
|
||||
"next_id": self.next_id,
|
||||
},
|
||||
cls=_LosslessJSONEncoder,
|
||||
)
|
||||
@@ -262,6 +360,12 @@ class FAISSIndex:
|
||||
if persisted_index_type is not None:
|
||||
index_type = persisted_index_type
|
||||
|
||||
# Restore the monotonic ID counter. Older sidecar files written
|
||||
# before this field was added will not have the key; fall back to
|
||||
# ntotal, which equals the counter value for stores that have never
|
||||
# had a deletion (no gaps in label space).
|
||||
persisted_next_id = data.get("next_id")
|
||||
|
||||
# Check for vector count vs sidecar ID count mismatch
|
||||
if len(vector_ids) != index.ntotal:
|
||||
raise ProcessingError(
|
||||
@@ -279,10 +383,28 @@ class FAISSIndex:
|
||||
)
|
||||
vector_ids = []
|
||||
metadata = {}
|
||||
persisted_next_id = None
|
||||
|
||||
obj = cls(index, dimension, index_type)
|
||||
obj.vector_ids = vector_ids
|
||||
obj.metadata = metadata
|
||||
# Restore the monotonic counter. Always clamp to at least the
|
||||
# highest inferred vec_N ID, so a stale or corrupted persisted value
|
||||
# (e.g. written before a deletion that shifted the gap) cannot cause
|
||||
# future default IDs to collide with existing vector IDs.
|
||||
_vec_nums = [
|
||||
int(v[4:]) + 1
|
||||
for v in vector_ids
|
||||
if v.startswith("vec_") and v[4:].isdigit()
|
||||
]
|
||||
_inferred = max(_vec_nums) if _vec_nums else index.ntotal
|
||||
if persisted_next_id is not None:
|
||||
# Trust the persisted value but never go below the inferred minimum
|
||||
# (guards against stale/corrupted sidecars).
|
||||
obj.next_id = max(int(persisted_next_id), _inferred)
|
||||
else:
|
||||
# Older sidecar files lack this field. Use the inferred value.
|
||||
obj.next_id = _inferred
|
||||
return obj
|
||||
|
||||
|
||||
@@ -315,7 +437,7 @@ class FAISSSearch:
|
||||
|
||||
results = []
|
||||
for i, (dist, idx) in enumerate(zip(distances[0], indices[0])):
|
||||
if idx < len(self.index.vector_ids):
|
||||
if idx < len(self.index.vector_ids) and idx >= 0:
|
||||
vector_id = self.index.vector_ids[idx]
|
||||
dist_val = float(dist)
|
||||
|
||||
@@ -422,6 +544,12 @@ class FAISSStore:
|
||||
self.index: Optional[FAISSIndex] = None
|
||||
self.index_builder = FAISSIndexBuilder(dimension)
|
||||
self.search_engine: Optional[FAISSSearch] = None
|
||||
# Path remembered by load_index so delete_vectors can auto-save.
|
||||
self._index_path: Optional[Path] = None
|
||||
# Monotonic counter for default ID generation. Incremented on every
|
||||
# successful add, never decremented on deletion, so ids generated by
|
||||
# consecutive add_vectors calls can never collide with surviving IDs.
|
||||
self._next_id: int = 0
|
||||
|
||||
# Check FAISS availability
|
||||
if not FAISS_AVAILABLE:
|
||||
@@ -498,13 +626,25 @@ class FAISSStore:
|
||||
|
||||
vectors = vectors.astype(np.float32)
|
||||
|
||||
# Generate IDs if not provided
|
||||
# Generate IDs if not provided. Use a monotonic counter so
|
||||
# that default IDs never collide with surviving IDs after a
|
||||
# deletion (len(vector_ids) would decrease, potentially reusing
|
||||
# a label that still exists in the index).
|
||||
if ids is None:
|
||||
ids = [
|
||||
f"vec_{len(self.index.vector_ids) + i}" for i in range(len(vectors))
|
||||
]
|
||||
_existing = set(self.index.vector_ids)
|
||||
generated: List[str] = []
|
||||
while len(generated) < len(vectors):
|
||||
cand = f"vec_{self._next_id}"
|
||||
self._next_id += 1
|
||||
if cand not in _existing:
|
||||
generated.append(cand)
|
||||
_existing.add(cand)
|
||||
ids = generated
|
||||
# Sync FAISSIndex.next_id so save() persists the correct value.
|
||||
self.index.next_id = self._next_id
|
||||
|
||||
# Store metadata
|
||||
# Assign metadata before the duplicate-skip filter so callers
|
||||
# always get up-to-date metadata even for already-present ids.
|
||||
if metadata:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Storing metadata..."
|
||||
@@ -625,6 +765,12 @@ class FAISSStore:
|
||||
|
||||
self.index = FAISSIndex.load(path, self.dimension, index_type)
|
||||
self.search_engine = FAISSSearch(self.index)
|
||||
# Remember the path so delete_vectors can auto-save to the same location.
|
||||
self._index_path = path
|
||||
# Restore the monotonic counter from the sidecar (via FAISSIndex.next_id)
|
||||
# rather than using ntotal. After a deletion ntotal is smaller than the
|
||||
# highest generated ID, so ntotal would cause ID collisions on the next add.
|
||||
self._next_id = self.index.next_id
|
||||
|
||||
self.logger.info(f"Loaded FAISS index from {path}")
|
||||
return self.index
|
||||
@@ -735,10 +881,62 @@ class FAISSStore:
|
||||
"""Return the number of vectors currently tracked in this store.
|
||||
|
||||
Returns the length of the ``vector_ids`` list maintained by
|
||||
``FAISSIndex``. FAISSStore does not implement vector deletion, so
|
||||
this list is strictly append-only and is always consistent with the
|
||||
underlying FAISS index (``index.ntotal``).
|
||||
``FAISSIndex``. This list is always kept consistent with the
|
||||
underlying FAISS index (``index.ntotal``), including after deletions.
|
||||
"""
|
||||
if self.index is None:
|
||||
return 0
|
||||
return len(self.index.vector_ids)
|
||||
|
||||
def delete_vectors(self, vector_ids: List[str], **options) -> Dict[str, Any]:
|
||||
"""Delete vectors by their external string IDs.
|
||||
|
||||
Delegates to :meth:`FAISSIndex.delete_vectors`. When the store was
|
||||
loaded from disk via :meth:`load_index`, the updated index and sidecar
|
||||
are written back to disk before this method returns, so the deletion is
|
||||
durable across process restarts without the caller needing a separate
|
||||
:meth:`save_index` call. Note: only the ``.meta.json`` sidecar write
|
||||
is atomic (temp-file + rename); the ``.faiss`` binary is written in
|
||||
place. A process crash between those two writes would leave the files
|
||||
inconsistent, but the mismatch guard in :meth:`FAISSIndex.load` would
|
||||
detect it on the next load rather than silently returning wrong data.
|
||||
|
||||
No-op deletions (all requested IDs unknown, or empty input) do not
|
||||
trigger a disk write.
|
||||
|
||||
When the store was created in memory (no :meth:`load_index` call), the
|
||||
deletion is in-memory only and the caller must invoke
|
||||
:meth:`save_index` to persist it.
|
||||
|
||||
IVF indices do not support deletion because their internal labels do
|
||||
not compact after ``remove_ids``, which would desynchronize search
|
||||
labels from the ``vector_ids`` mapping. HNSW indices also do not
|
||||
support ``remove_ids``. Both raise ``NotImplementedError``, which the
|
||||
:class:`ErasureCoordinator` translates to ``STATUS_UNSUPPORTED``.
|
||||
|
||||
Args:
|
||||
vector_ids: External string IDs to delete. Unknown IDs are
|
||||
silently ignored. Duplicates are deduplicated.
|
||||
**options: Accepted for API parity with other backends; unused.
|
||||
|
||||
Returns:
|
||||
``{"delete_count": N}``
|
||||
|
||||
Raises:
|
||||
ProcessingError: If no index has been initialized.
|
||||
NotImplementedError: If the underlying index type (IVF or HNSW)
|
||||
does not support safe deletion.
|
||||
"""
|
||||
if self.index is None:
|
||||
raise ProcessingError(
|
||||
"Index not initialized. Call create_index() first."
|
||||
)
|
||||
result = self.index.delete_vectors(vector_ids)
|
||||
# If the store was loaded from disk (load_index recorded the path),
|
||||
# persist the deletion so that the vectors cannot be resurrected by a
|
||||
# process restart. Only write when something was actually removed:
|
||||
# a no-op deletion (all IDs unknown or empty list) must not trigger
|
||||
# a full index rewrite.
|
||||
if self._index_path is not None and result.get("delete_count", 0) > 0:
|
||||
self.index.save(self._index_path)
|
||||
return result
|
||||
|
||||
@@ -289,4 +289,122 @@ GET_ANALYTICS = {
|
||||
},
|
||||
}
|
||||
|
||||
STORE_DOCUMENT = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {
|
||||
"type": "string",
|
||||
"description": "Document text to chunk and store for semantic retrieval",
|
||||
},
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "Provenance identifier, e.g. 'policy_manual_v2#page12'",
|
||||
},
|
||||
"authority": {
|
||||
"type": "string",
|
||||
"description": "Authority level of the content, e.g. 'official', 'draft', 'external'",
|
||||
},
|
||||
"version": {
|
||||
"type": "string",
|
||||
"description": "Document version tag used together with source as the upsert key (default: 'v1')",
|
||||
},
|
||||
"project": {
|
||||
"type": "string",
|
||||
"description": "Optional project namespace for later filtering",
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"description": "Additional key-value properties stored on every chunk",
|
||||
},
|
||||
"chunk_size": {
|
||||
"type": "integer",
|
||||
"minimum": 100,
|
||||
"description": "Chunk window in characters (default: 1000)",
|
||||
},
|
||||
"chunk_overlap": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Overlap between consecutive chunks in characters (default: 200)",
|
||||
},
|
||||
},
|
||||
"required": ["content", "source", "authority"],
|
||||
}
|
||||
|
||||
RETRIEVE_CONTEXT = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Natural language query to embed and search for",
|
||||
},
|
||||
"top_k": {
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"maximum": 10,
|
||||
"description": "Maximum number of chunks to return (default: 5, capped at 10)",
|
||||
},
|
||||
"project": {
|
||||
"type": "string",
|
||||
"description": "Only return chunks stored under this project namespace (optional)",
|
||||
},
|
||||
},
|
||||
"required": ["query"],
|
||||
}
|
||||
|
||||
UPDATE_DOCUMENT = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {
|
||||
"type": "string",
|
||||
"description": "New document text replacing the stored version",
|
||||
},
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "Provenance identifier of the document to update",
|
||||
},
|
||||
"version": {
|
||||
"type": "string",
|
||||
"description": "Version tag identifying which stored version to replace (default: 'v1')",
|
||||
},
|
||||
"authority": {
|
||||
"type": "string",
|
||||
"description": "Updated authority level (defaults to the stored value)",
|
||||
},
|
||||
"project": {
|
||||
"type": "string",
|
||||
"description": "Updated project namespace (defaults to the stored value)",
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"description": "Additional key-value properties merged into chunk metadata",
|
||||
},
|
||||
"chunk_size": {
|
||||
"type": "integer",
|
||||
"minimum": 100,
|
||||
"description": "Chunk window in characters (default: 1000)",
|
||||
},
|
||||
"chunk_overlap": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"description": "Overlap between consecutive chunks in characters (default: 200)",
|
||||
},
|
||||
},
|
||||
"required": ["content", "source"],
|
||||
}
|
||||
|
||||
REMOVE_DOCUMENT = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"source": {
|
||||
"type": "string",
|
||||
"description": "Provenance identifier of the document to remove",
|
||||
},
|
||||
"version": {
|
||||
"type": "string",
|
||||
"description": "Version tag identifying which stored version to remove (default: 'v1')",
|
||||
},
|
||||
},
|
||||
"required": ["source"],
|
||||
}
|
||||
|
||||
EMPTY = {"type": "object", "properties": {}}
|
||||
|
||||
@@ -51,6 +51,27 @@ _INTERNAL_ERROR = -32603
|
||||
_TOOL_INDEX: dict[str, dict] = {t["name"]: t for t in TOOL_DEFINITIONS}
|
||||
|
||||
|
||||
class UnknownToolError(Exception):
|
||||
"""Raised by :func:`call_tool` when the tool name is not in the catalog.
|
||||
|
||||
A dedicated type (rather than ``KeyError``) so callers can distinguish
|
||||
a bad tool name from a ``KeyError`` raised inside a handler indexing a
|
||||
required argument (e.g. ``args["category"]``).
|
||||
"""
|
||||
|
||||
|
||||
def call_tool(name: str, arguments: dict) -> dict:
|
||||
"""Invoke a tool in-process by name and return its raw result dict.
|
||||
|
||||
Shared by the JSON-RPC ``tools/call`` handler and ``semantica mcp call``
|
||||
(issue #1355), so both expose exactly the same tool set.
|
||||
"""
|
||||
tool = _TOOL_INDEX.get(name)
|
||||
if tool is None:
|
||||
raise UnknownToolError(f"Unknown tool: {name}")
|
||||
return tool["_handler"](arguments)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request handlers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -85,12 +106,10 @@ def _handle_tools_call(req_id: Any, params: dict) -> dict:
|
||||
name = params.get("name", "")
|
||||
args = params.get("arguments", {}) or {}
|
||||
|
||||
tool = _TOOL_INDEX.get(name)
|
||||
if tool is None:
|
||||
return _err(req_id, _METHOD_NOT_FOUND, f"Unknown tool: {name}")
|
||||
|
||||
try:
|
||||
result = tool["_handler"](args)
|
||||
result = call_tool(name, args)
|
||||
except UnknownToolError as exc:
|
||||
return _err(req_id, _METHOD_NOT_FOUND, str(exc))
|
||||
except Exception as exc:
|
||||
log.exception("Tool %s raised an exception", name)
|
||||
# The exception's class name (e.g. "ValidationError", "TimeoutError")
|
||||
|
||||
@@ -14,7 +14,15 @@ from typing import Any, Optional
|
||||
|
||||
log = logging.getLogger("semantica.mcp.session")
|
||||
|
||||
# Backends the retrieval tools can actually support end to end. faiss
|
||||
# and pgvector have no metadata-scoped delete, so update_document and
|
||||
# remove_document cannot work on them; selecting them fails fast here
|
||||
# instead of blowing up mid-update.
|
||||
SUPPORTED_VECTOR_BACKENDS = ("inmemory", "sqlite")
|
||||
|
||||
_graph: Optional[Any] = None
|
||||
_embedder: Optional[Any] = None
|
||||
_vector_store: Optional[Any] = None
|
||||
|
||||
# Tracks whether the last graph initialisation successfully loaded the
|
||||
# configured SEMANTICA_KG_PATH file. When True (or no path was configured)
|
||||
@@ -59,6 +67,103 @@ def get_graph() -> Any:
|
||||
return _graph
|
||||
|
||||
|
||||
def get_embedder() -> Any:
|
||||
"""
|
||||
Return the shared EmbeddingGenerator instance, creating it on first call.
|
||||
|
||||
Used by the semantic retrieval tools (#1235) to embed documents and
|
||||
queries with one consistent model, so stored vectors and query
|
||||
vectors always share the same dimensionality.
|
||||
"""
|
||||
global _embedder
|
||||
if _embedder is None:
|
||||
from semantica.embeddings import EmbeddingGenerator
|
||||
|
||||
_embedder = EmbeddingGenerator()
|
||||
log.info(
|
||||
"Embedding generator initialised (method=%s)",
|
||||
_embedder.get_text_method(),
|
||||
)
|
||||
return _embedder
|
||||
|
||||
|
||||
def get_vector_store() -> Any:
|
||||
"""
|
||||
Return the shared VectorStore instance, creating it on first call.
|
||||
|
||||
Backend selection:
|
||||
|
||||
• ``SEMANTICA_VECTOR_BACKEND`` — ``inmemory`` (default) or ``sqlite``.
|
||||
The ``sqlite`` backend additionally requires
|
||||
``SEMANTICA_VECTOR_DB_PATH``. Other VectorStore backends (faiss,
|
||||
pgvector) are rejected: they lack the metadata-scoped delete the
|
||||
update/remove tools need.
|
||||
• ``SEMANTICA_VECTOR_PATH`` — a *directory* previously written by
|
||||
``VectorStore.save()``. If it exists, the store is loaded from it
|
||||
on start. Note this is a directory, unlike SEMANTICA_KG_PATH which
|
||||
is a single JSON file. The persisted dimension must match the
|
||||
active embedder or startup fails — otherwise queries would either
|
||||
error on shape mismatch or silently rank across incompatible
|
||||
embedding spaces.
|
||||
"""
|
||||
global _vector_store
|
||||
if _vector_store is None:
|
||||
from semantica.vector_store import VectorStore
|
||||
|
||||
backend = os.environ.get("SEMANTICA_VECTOR_BACKEND", "inmemory").strip().lower()
|
||||
if backend not in SUPPORTED_VECTOR_BACKENDS:
|
||||
raise ValueError(
|
||||
f"SEMANTICA_VECTOR_BACKEND={backend!r} is not supported by the "
|
||||
"MCP retrieval tools; supported backends: "
|
||||
+ ", ".join(SUPPORTED_VECTOR_BACKENDS)
|
||||
)
|
||||
config: dict = {}
|
||||
if backend == "sqlite":
|
||||
db_path = os.environ.get("SEMANTICA_VECTOR_DB_PATH", "").strip()
|
||||
if not db_path:
|
||||
raise ValueError(
|
||||
"SEMANTICA_VECTOR_BACKEND=sqlite requires "
|
||||
"SEMANTICA_VECTOR_DB_PATH to point at the database file"
|
||||
)
|
||||
config["db_path"] = db_path
|
||||
# VectorStore defaults to dimension 768, which does not match the
|
||||
# default embedding model (all-MiniLM-L6-v2 = 384, hash fallback
|
||||
# = 128). Always derive it from the embedder so store and
|
||||
# queries stay consistent.
|
||||
embedder = get_embedder()
|
||||
config["dimension"] = embedder.text_embedder.get_embedding_dimension()
|
||||
|
||||
store = VectorStore(backend=backend, config=config)
|
||||
|
||||
vector_path = os.environ.get("SEMANTICA_VECTOR_PATH", "").strip()
|
||||
if vector_path and os.path.isdir(vector_path):
|
||||
try:
|
||||
store.load(vector_path)
|
||||
log.info("Vector store loaded from %s", vector_path)
|
||||
except Exception as exc:
|
||||
raise ValueError(
|
||||
f"Could not load vector store from {vector_path}: {exc}"
|
||||
) from exc
|
||||
loaded_dim = getattr(store, "dimension", None)
|
||||
if loaded_dim and loaded_dim != config["dimension"]:
|
||||
raise ValueError(
|
||||
f"Persisted vector store at {vector_path} has dimension "
|
||||
f"{loaded_dim}, but the active embedder produces "
|
||||
f"{config['dimension']}. Re-embed the corpus or point "
|
||||
"SEMANTICA_VECTOR_PATH at a store built with the same model."
|
||||
)
|
||||
|
||||
_vector_store = store
|
||||
log.info("Vector store initialised (backend=%s)", backend)
|
||||
return _vector_store
|
||||
|
||||
|
||||
def reset_vector_store() -> None:
|
||||
"""Reset the vector store singleton (mainly useful in tests)."""
|
||||
global _vector_store
|
||||
_vector_store = None
|
||||
|
||||
|
||||
def is_persistence_safe() -> bool:
|
||||
"""Return True when it is safe to write mutations back to SEMANTICA_KG_PATH.
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ from .export import EXPORT_TOOLS
|
||||
from .extraction import EXTRACTION_TOOLS
|
||||
from .graph import GRAPH_TOOLS
|
||||
from .reasoning import REASONING_TOOLS
|
||||
from .retrieval import RETRIEVAL_TOOLS
|
||||
|
||||
# Ordered list — exposed to the MCP client via tools/list
|
||||
TOOL_DEFINITIONS = (
|
||||
@@ -17,6 +18,7 @@ TOOL_DEFINITIONS = (
|
||||
+ GRAPH_TOOLS
|
||||
+ REASONING_TOOLS
|
||||
+ EXPORT_TOOLS
|
||||
+ RETRIEVAL_TOOLS
|
||||
)
|
||||
|
||||
__all__ = ["TOOL_DEFINITIONS"]
|
||||
|
||||
@@ -0,0 +1,533 @@
|
||||
"""
|
||||
Semantic retrieval tools — store, retrieve, update and remove documents
|
||||
in a vector store, combined with knowledge-graph context (#1235).
|
||||
|
||||
Design notes:
|
||||
|
||||
• Documents are chunked with a fixed sliding window (default 1000 chars,
|
||||
200 overlap) and every chunk carries full provenance metadata:
|
||||
chunk_id, source, authority, version, project, content hash, status
|
||||
and character offsets.
|
||||
• (source, version) is the upsert key. The content hash only decides
|
||||
whether a re-store can be skipped as a no-op.
|
||||
• Updates and removals on the in-memory backend rebuild the store from
|
||||
scratch (read everything, filter, clear, re-store) instead of calling
|
||||
delete_vectors. In-memory ids are derived from ``len(self.vectors)``
|
||||
and fall back after a delete, so deleting then writing can overwrite
|
||||
live data (#1029). Rebuilding from an empty dict starts the counter
|
||||
at zero — nothing to collide with. The real fix for #1029 (ids that
|
||||
never get reused) belongs in its own PR.
|
||||
• Retrieval results are combined with related graph nodes: for each hit
|
||||
source we look up ContextGraph nodes tagged with the same
|
||||
``metadata.source`` and attach their 1-hop neighbours.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ..schemas import (
|
||||
REMOVE_DOCUMENT,
|
||||
RETRIEVE_CONTEXT,
|
||||
STORE_DOCUMENT,
|
||||
UPDATE_DOCUMENT,
|
||||
)
|
||||
from ..session import get_embedder, get_graph, get_vector_store
|
||||
|
||||
log = logging.getLogger("semantica.mcp.tools.retrieval")
|
||||
|
||||
DEFAULT_CHUNK_SIZE = 1000
|
||||
DEFAULT_CHUNK_OVERLAP = 200
|
||||
MAX_TOP_K = 10
|
||||
FILTER_OVERFETCH = 3
|
||||
MAX_FILTER_MATCHES = 10_000
|
||||
MAX_CHUNKS_PER_DOC = 10_000
|
||||
|
||||
# Metadata fields owned by the upsert logic. Caller-supplied metadata
|
||||
# can add extra context but must not rewrite provenance: overwriting
|
||||
# source/version/hash/status would break the (source, version) upsert
|
||||
# key, the idempotent no-op check, and retrieval filters.
|
||||
PROTECTED_META_KEYS = frozenset(
|
||||
{
|
||||
"chunk_id",
|
||||
"text",
|
||||
"source",
|
||||
"authority",
|
||||
"version",
|
||||
"hash",
|
||||
"status",
|
||||
"chunk_index",
|
||||
"char_start",
|
||||
"char_end",
|
||||
"project",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _chunk_text(text: str, chunk_size: int, chunk_overlap: int) -> List[Tuple[int, int, str]]:
|
||||
"""Split text into (char_start, char_end, chunk) windows."""
|
||||
if chunk_overlap >= chunk_size:
|
||||
raise ValueError("chunk_overlap must be smaller than chunk_size")
|
||||
chunks: List[Tuple[int, int, str]] = []
|
||||
start = 0
|
||||
n = len(text)
|
||||
while start < n:
|
||||
end = min(start + chunk_size, n)
|
||||
chunks.append((start, end, text[start:end]))
|
||||
if end >= n:
|
||||
break
|
||||
start = end - chunk_overlap
|
||||
return chunks
|
||||
|
||||
|
||||
def _chunk_id(source: str, version: str, index: int, text: str) -> str:
|
||||
"""Stable chunk id derived from the location key and chunk content."""
|
||||
digest = hashlib.sha256(
|
||||
f"{source}|{version}|{index}|{text}".encode("utf-8")
|
||||
).hexdigest()
|
||||
return f"chk_{digest[:16]}"
|
||||
|
||||
|
||||
def _doc_hash(content: str) -> str:
|
||||
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _find_matching_rows(store: Any, source: str, version: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Return rows (``{id, vector, metadata}``) matching (source, version).
|
||||
|
||||
The persistent branch pulls whole rows (vector included) into memory;
|
||||
the limit keeps the scan bounded. Documents beyond MAX_CHUNKS_PER_DOC
|
||||
chunks are rejected at ingestion, so the cap cannot leave stale
|
||||
chunks behind on update/remove.
|
||||
"""
|
||||
if getattr(store, "backend", "") == "inmemory":
|
||||
rows = []
|
||||
for vid, vec in getattr(store, "vectors", {}).items():
|
||||
meta = getattr(store, "metadata", {}).get(vid) or {}
|
||||
if meta.get("source") == source and meta.get("version") == version:
|
||||
rows.append({"id": vid, "vector": vec, "metadata": meta})
|
||||
return rows
|
||||
backend_store = getattr(store, "_backend_store", None)
|
||||
if backend_store is not None and hasattr(backend_store, "filter_by_metadata"):
|
||||
return backend_store.filter_by_metadata(
|
||||
{"source": source, "version": version}, limit=MAX_FILTER_MATCHES
|
||||
)
|
||||
raise NotImplementedError(
|
||||
f"Backend {type(backend_store).__name__} does not support metadata lookup; "
|
||||
"cannot locate chunks for update/remove"
|
||||
)
|
||||
|
||||
|
||||
def _find_matching_ids(store: Any, source: str, version: str) -> List[str]:
|
||||
"""Return every vector id whose metadata matches (source, version)."""
|
||||
return [row["id"] for row in _find_matching_rows(store, source, version)]
|
||||
|
||||
|
||||
def _remove_ids(store: Any, remove_ids: List[str]) -> None:
|
||||
"""Remove vectors by id, avoiding the #1029 in-memory id collision."""
|
||||
if getattr(store, "backend", "") == "inmemory":
|
||||
# Full rebuild: read all, filter in memory, clear, re-store once.
|
||||
# store_vectors derives ids from len(self.vectors), and the dicts
|
||||
# are empty here, so the counter restarts at zero — no reuse of
|
||||
# ids that are still referenced anywhere.
|
||||
remove = set(remove_ids)
|
||||
vectors = getattr(store, "vectors", {})
|
||||
metadata = getattr(store, "metadata", {})
|
||||
saved_vectors = dict(vectors)
|
||||
saved_metadata = dict(metadata)
|
||||
keep_vectors = []
|
||||
keep_meta = []
|
||||
for vid, vec in list(vectors.items()):
|
||||
if vid in remove:
|
||||
continue
|
||||
keep_vectors.append(vec)
|
||||
keep_meta.append(metadata.get(vid, {}))
|
||||
vectors.clear()
|
||||
metadata.clear()
|
||||
try:
|
||||
if keep_vectors:
|
||||
store.store_vectors(keep_vectors, keep_meta)
|
||||
except Exception:
|
||||
# Restore the pre-rebuild state so a failed re-store does not
|
||||
# silently drop every surviving document.
|
||||
vectors.update(saved_vectors)
|
||||
metadata.update(saved_metadata)
|
||||
store.indexer.create_index(
|
||||
list(vectors.values()), list(vectors.keys())
|
||||
)
|
||||
raise
|
||||
return
|
||||
# Persistent backends do not have the len-based id collision, so a
|
||||
# direct delete is safe there.
|
||||
store.delete_vectors(remove_ids)
|
||||
|
||||
|
||||
def _persist(store: Any) -> Any:
|
||||
"""
|
||||
Persist the store when SEMANTICA_VECTOR_PATH is configured.
|
||||
|
||||
Returns ``None`` when no path is configured, ``True`` on success and
|
||||
``False`` when saving failed — surfaced in tool results so a caller
|
||||
can tell an in-memory-only write from a durable one.
|
||||
"""
|
||||
path = os.environ.get("SEMANTICA_VECTOR_PATH", "").strip()
|
||||
if not path:
|
||||
return None
|
||||
try:
|
||||
store.save(path)
|
||||
except Exception as exc:
|
||||
log.warning("Could not persist vector store to %s: %s", path, exc)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _node_source(meta: Any) -> str:
|
||||
"""
|
||||
Extract a node's source tag from its metadata.
|
||||
|
||||
ContextGraph.add_node nests the caller-supplied metadata dict one
|
||||
level down (``{'label': ..., 'metadata': {...}}``), while nodes added
|
||||
through other paths may carry ``source`` directly. Check both.
|
||||
"""
|
||||
if not isinstance(meta, dict):
|
||||
return ""
|
||||
direct = meta.get("source")
|
||||
if direct:
|
||||
return str(direct)
|
||||
nested = meta.get("metadata")
|
||||
if isinstance(nested, dict):
|
||||
return str(nested.get("source", "") or "")
|
||||
return ""
|
||||
|
||||
|
||||
def _graph_relationships(sources: List[str], max_per_source: int = 3) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Collect 1-hop graph neighbours for nodes tagged with the hit sources.
|
||||
|
||||
Node lookup matches the node's source tag against the stored document
|
||||
sources. Failures degrade to an empty list — graph context is a
|
||||
bonus, never a hard dependency of retrieval.
|
||||
"""
|
||||
if not sources:
|
||||
return []
|
||||
try:
|
||||
graph = get_graph()
|
||||
nodes = list(graph.find_nodes())
|
||||
except Exception as exc:
|
||||
log.debug("Graph context unavailable: %s", exc)
|
||||
return []
|
||||
|
||||
relationships: List[Dict[str, Any]] = []
|
||||
seen: set = set()
|
||||
for source in sources:
|
||||
anchor = None
|
||||
for n in nodes:
|
||||
if _node_source(n.get("metadata")) == source:
|
||||
anchor = n
|
||||
break
|
||||
if anchor is None:
|
||||
continue
|
||||
try:
|
||||
neighbors = graph.get_neighbors(anchor["id"], hops=1)
|
||||
except Exception as exc:
|
||||
log.debug("get_neighbors failed for %s: %s", anchor.get("id"), exc)
|
||||
continue
|
||||
added = 0
|
||||
for nb in neighbors:
|
||||
key = (anchor.get("id"), nb.get("id"), nb.get("relationship"))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
relationships.append(
|
||||
{
|
||||
"node": {
|
||||
"id": anchor.get("id"),
|
||||
"type": anchor.get("type"),
|
||||
"content": str(anchor.get("content") or "")[:200],
|
||||
"source": source,
|
||||
},
|
||||
"related": {
|
||||
"id": nb.get("id"),
|
||||
"type": nb.get("type"),
|
||||
"content": str(nb.get("content") or "")[:200],
|
||||
},
|
||||
"relationship": nb.get("relationship"),
|
||||
}
|
||||
)
|
||||
added += 1
|
||||
if added >= max_per_source:
|
||||
break
|
||||
return relationships
|
||||
|
||||
|
||||
def _upsert(args: dict, action: str) -> dict:
|
||||
"""Shared implementation for store_document and update_document."""
|
||||
content = args.get("content", "")
|
||||
source = str(args.get("source", "")).strip()
|
||||
if not content or not source:
|
||||
return {"error": "content and source are required"}
|
||||
authority = str(args.get("authority", "")).strip()
|
||||
if action == "store" and not authority:
|
||||
return {"error": "authority is required"}
|
||||
version = str(args.get("version", "")).strip() or "v1"
|
||||
project = str(args.get("project", "")).strip() or None
|
||||
chunk_size = int(args.get("chunk_size", DEFAULT_CHUNK_SIZE))
|
||||
chunk_overlap = int(args.get("chunk_overlap", DEFAULT_CHUNK_OVERLAP))
|
||||
if chunk_overlap >= chunk_size:
|
||||
return {"error": "chunk_overlap must be smaller than chunk_size"}
|
||||
extra = args.get("metadata") or {}
|
||||
if not isinstance(extra, dict):
|
||||
return {"error": "metadata must be an object"}
|
||||
doc_hash = _doc_hash(content)
|
||||
|
||||
try:
|
||||
store = get_vector_store()
|
||||
embedder = get_embedder()
|
||||
|
||||
existing_ids = _find_matching_ids(store, source, version)
|
||||
if action == "update" and not existing_ids:
|
||||
return {"status": "not_found", "source": source, "version": version}
|
||||
existing_first: Dict[str, Any] = {}
|
||||
if existing_ids:
|
||||
existing_first = store.get_metadata(existing_ids[0]) or {}
|
||||
if action == "store" and existing_first.get("hash") == doc_hash:
|
||||
# Same content already stored under (source, version) —
|
||||
# skip re-embedding entirely.
|
||||
return {
|
||||
"status": "unchanged",
|
||||
"source": source,
|
||||
"version": version,
|
||||
"chunk_ids": [
|
||||
(store.get_metadata(vid) or {}).get("chunk_id")
|
||||
for vid in existing_ids
|
||||
],
|
||||
}
|
||||
|
||||
chunks = _chunk_text(content, chunk_size, chunk_overlap)
|
||||
if len(chunks) > MAX_CHUNKS_PER_DOC:
|
||||
return {
|
||||
"error": (
|
||||
f"document produces {len(chunks)} chunks, above the "
|
||||
f"{MAX_CHUNKS_PER_DOC}-chunk limit; split it into smaller "
|
||||
"documents or raise chunk_size"
|
||||
)
|
||||
}
|
||||
vectors = np.asarray(
|
||||
embedder.generate_embeddings([c_text for _, _, c_text in chunks])
|
||||
)
|
||||
if vectors.ndim == 1:
|
||||
vectors = vectors.reshape(1, -1)
|
||||
if vectors.shape[0] != len(chunks):
|
||||
return {
|
||||
"error": (
|
||||
f"embedder returned {vectors.shape[0]} vectors "
|
||||
f"for {len(chunks)} chunks"
|
||||
)
|
||||
}
|
||||
|
||||
final_authority = authority or existing_first.get("authority") or "unknown"
|
||||
final_project = project or existing_first.get("project")
|
||||
|
||||
old_rows: List[Dict[str, Any]] = []
|
||||
if existing_ids:
|
||||
# Snapshot the rows being replaced so a failed write of the
|
||||
# new chunks can put the old document back instead of leaving
|
||||
# (source, version) silently empty.
|
||||
old_rows = _find_matching_rows(store, source, version)
|
||||
_remove_ids(store, existing_ids)
|
||||
|
||||
metas = []
|
||||
chunk_ids = []
|
||||
for idx, (start, end, c_text) in enumerate(chunks):
|
||||
cid = _chunk_id(source, version, idx, c_text)
|
||||
chunk_ids.append(cid)
|
||||
meta: Dict[str, Any] = {
|
||||
"chunk_id": cid,
|
||||
"text": c_text,
|
||||
"source": source,
|
||||
"authority": final_authority,
|
||||
"version": version,
|
||||
"hash": doc_hash,
|
||||
"status": "active",
|
||||
"chunk_index": idx,
|
||||
"char_start": start,
|
||||
"char_end": end,
|
||||
}
|
||||
if final_project:
|
||||
meta["project"] = final_project
|
||||
for key in extra:
|
||||
if key in PROTECTED_META_KEYS:
|
||||
log.debug(
|
||||
"Ignoring caller metadata key %r: provenance field is "
|
||||
"managed by the tool",
|
||||
key,
|
||||
)
|
||||
else:
|
||||
meta[key] = extra[key]
|
||||
metas.append(meta)
|
||||
|
||||
try:
|
||||
store.store_vectors(list(vectors), metas)
|
||||
except Exception:
|
||||
if old_rows:
|
||||
log.warning(
|
||||
"Storing new chunks failed for (%s, %s); restoring the "
|
||||
"previous document",
|
||||
source,
|
||||
version,
|
||||
)
|
||||
store.store_vectors(
|
||||
[row["vector"] for row in old_rows],
|
||||
[row["metadata"] for row in old_rows],
|
||||
)
|
||||
raise
|
||||
persisted = _persist(store)
|
||||
return {
|
||||
"status": "stored" if action == "store" else "updated",
|
||||
"source": source,
|
||||
"version": version,
|
||||
"chunk_ids": chunk_ids,
|
||||
"chunk_count": len(chunk_ids),
|
||||
"hash": doc_hash,
|
||||
"persisted": persisted,
|
||||
}
|
||||
except Exception as exc:
|
||||
log.exception("%s_document failed", action)
|
||||
return {"error": str(exc)}
|
||||
|
||||
|
||||
def handle_store_document(args: dict) -> dict:
|
||||
"""Chunk a document, embed it, and store it for semantic retrieval."""
|
||||
return _upsert(args, "store")
|
||||
|
||||
|
||||
def handle_update_document(args: dict) -> dict:
|
||||
"""Replace the stored content of a (source, version) document."""
|
||||
return _upsert(args, "update")
|
||||
|
||||
|
||||
def handle_retrieve_context(args: dict) -> dict:
|
||||
"""Embed a query and return the most relevant stored chunks."""
|
||||
query = str(args.get("query", "")).strip()
|
||||
if not query:
|
||||
return {"error": "query is required", "results": []}
|
||||
try:
|
||||
top_k = max(1, min(int(args.get("top_k", 5)), MAX_TOP_K))
|
||||
except (TypeError, ValueError):
|
||||
top_k = 5
|
||||
project = str(args.get("project", "")).strip() or None
|
||||
|
||||
try:
|
||||
store = get_vector_store()
|
||||
query_vector = np.asarray(get_embedder().generate_embeddings([query]))[0]
|
||||
# Over-fetch so a project filter can drop hits without starving
|
||||
# the result list.
|
||||
fetch_k = top_k * FILTER_OVERFETCH if project else top_k
|
||||
raw = store.search_vectors(query_vector, k=fetch_k)
|
||||
|
||||
results = []
|
||||
for hit in raw:
|
||||
meta = hit.get("metadata") or {}
|
||||
if project and meta.get("project") != project:
|
||||
continue
|
||||
results.append(
|
||||
{
|
||||
"chunk_id": meta.get("chunk_id", hit.get("id")),
|
||||
"text": meta.get("text", ""),
|
||||
"score": hit.get("score"),
|
||||
"source": meta.get("source"),
|
||||
"authority": meta.get("authority"),
|
||||
"version": meta.get("version"),
|
||||
"project": meta.get("project"),
|
||||
"status": meta.get("status"),
|
||||
"hash": meta.get("hash"),
|
||||
}
|
||||
)
|
||||
if len(results) >= top_k:
|
||||
break
|
||||
|
||||
sources = list(dict.fromkeys(r["source"] for r in results if r["source"]))
|
||||
return {
|
||||
"query": query,
|
||||
"results": results,
|
||||
"count": len(results),
|
||||
"graph_context": _graph_relationships(sources),
|
||||
}
|
||||
except Exception as exc:
|
||||
log.exception("retrieve_context failed")
|
||||
return {"error": str(exc), "results": []}
|
||||
|
||||
|
||||
def handle_remove_document(args: dict) -> dict:
|
||||
"""Remove every chunk stored under (source, version)."""
|
||||
source = str(args.get("source", "")).strip()
|
||||
if not source:
|
||||
return {"error": "source is required"}
|
||||
version = str(args.get("version", "")).strip() or "v1"
|
||||
try:
|
||||
store = get_vector_store()
|
||||
existing_ids = _find_matching_ids(store, source, version)
|
||||
if not existing_ids:
|
||||
return {"status": "not_found", "source": source, "version": version}
|
||||
_remove_ids(store, existing_ids)
|
||||
persisted = _persist(store)
|
||||
return {
|
||||
"status": "removed",
|
||||
"source": source,
|
||||
"version": version,
|
||||
"removed_chunks": len(existing_ids),
|
||||
"persisted": persisted,
|
||||
}
|
||||
except Exception as exc:
|
||||
log.exception("remove_document failed")
|
||||
return {"error": str(exc)}
|
||||
|
||||
|
||||
RETRIEVAL_TOOLS = [
|
||||
{
|
||||
"name": "store_document",
|
||||
"description": (
|
||||
"Chunk a document, embed the chunks, and store them for semantic "
|
||||
"retrieval. Keyed on (source, version); storing identical content "
|
||||
"again is a no-op."
|
||||
),
|
||||
"inputSchema": STORE_DOCUMENT,
|
||||
"_handler": handle_store_document,
|
||||
},
|
||||
{
|
||||
"name": "retrieve_context",
|
||||
"description": (
|
||||
"Embed a natural-language query and return the most relevant "
|
||||
"stored chunks with scores and provenance, combined with related "
|
||||
"knowledge-graph relationships."
|
||||
),
|
||||
"inputSchema": RETRIEVE_CONTEXT,
|
||||
"_handler": handle_retrieve_context,
|
||||
},
|
||||
{
|
||||
"name": "update_document",
|
||||
"description": (
|
||||
"Replace the stored content of a document identified by "
|
||||
"(source, version). Old chunks are removed and the new content "
|
||||
"is re-chunked and re-embedded. Returns not_found when no "
|
||||
"stored document matches (source, version)."
|
||||
),
|
||||
"inputSchema": UPDATE_DOCUMENT,
|
||||
"_handler": handle_update_document,
|
||||
},
|
||||
{
|
||||
"name": "remove_document",
|
||||
"description": (
|
||||
"Remove every chunk stored under (source, version) from the "
|
||||
"vector store."
|
||||
),
|
||||
"inputSchema": REMOVE_DOCUMENT,
|
||||
"_handler": handle_remove_document,
|
||||
},
|
||||
]
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Regression tests for the legacy-SDK model-instance cache on ``GeminiProvider``.
|
||||
|
||||
When the new ``google.genai`` package is unavailable, ``GeminiProvider`` falls
|
||||
back to the legacy ``google.generativeai`` package, whose ``GenerativeModel``
|
||||
binds its model name at construction time and whose API key lives in
|
||||
module-level state (``genai.configure()``).
|
||||
|
||||
``GeminiProvider._legacy_client_for()`` therefore keeps a per-instance cache
|
||||
keyed by model name, so a repeated per-call ``model=`` override reuses one
|
||||
``GenerativeModel`` instead of rebuilding it on every request, and re-asserts
|
||||
``genai.configure(api_key=...)`` with this provider's own key before each use.
|
||||
|
||||
PR #1488 (issue #1268) locked in *which* model a per-call override resolves to.
|
||||
These tests cover what it did not: that the resolved instance is built once and
|
||||
cached, and that the cache and credentials stay isolated per provider instance
|
||||
(issue #1269).
|
||||
"""
|
||||
|
||||
import sys
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.semantic_extract.providers import GeminiProvider
|
||||
|
||||
CONSTRUCTION_MODEL = "gemini-pro"
|
||||
OVERRIDE_MODEL = "gemini-1.5-flash"
|
||||
OTHER_MODEL = "gemini-1.5-pro"
|
||||
JSON_TEXT = '{"answer": 42}'
|
||||
|
||||
|
||||
def _make_provider(api_key="fake-key", model=CONSTRUCTION_MODEL):
|
||||
"""A GeminiProvider on the legacy path with the real SDK bootstrap skipped."""
|
||||
with patch.object(GeminiProvider, "_init_client", return_value=None):
|
||||
provider = GeminiProvider(api_key=api_key, model=model)
|
||||
provider._use_new_genai = False
|
||||
provider.client = MagicMock(name="construction client")
|
||||
return provider
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_legacy_genai(monkeypatch):
|
||||
"""Install a stand-in ``google.generativeai`` module.
|
||||
|
||||
Unlike the fake in ``test_gemini_model_override``, ``GenerativeModel`` here
|
||||
does **not** cache internally: it returns a fresh mock every call and records
|
||||
every model name it was asked to build, so a test can tell whether the
|
||||
provider rebuilt a model or served it from its own cache. ``configure`` is a
|
||||
plain mock so credential re-assertion is observable.
|
||||
"""
|
||||
module = MagicMock()
|
||||
module.build_calls = []
|
||||
|
||||
def build_model(name):
|
||||
module.build_calls.append(name)
|
||||
model = MagicMock(name=f"GenerativeModel({name})#{len(module.build_calls)}")
|
||||
response = MagicMock()
|
||||
response.text = JSON_TEXT
|
||||
model.generate_content.return_value = response
|
||||
return model
|
||||
|
||||
module.GenerativeModel.side_effect = build_model
|
||||
monkeypatch.setitem(sys.modules, "google.generativeai", module)
|
||||
return module
|
||||
|
||||
|
||||
class TestLegacyModelCacheReuse:
|
||||
"""``_legacy_client_for()`` builds each per-call model once, then caches it."""
|
||||
|
||||
def test_repeated_override_builds_one_generative_model(self, fake_legacy_genai):
|
||||
provider = _make_provider()
|
||||
|
||||
provider.generate("hello", model=OVERRIDE_MODEL)
|
||||
provider.generate("hello", model=OVERRIDE_MODEL)
|
||||
provider.generate_structured("hello", model=OVERRIDE_MODEL)
|
||||
|
||||
assert fake_legacy_genai.build_calls == [OVERRIDE_MODEL]
|
||||
assert list(provider._legacy_model_cache) == [OVERRIDE_MODEL]
|
||||
|
||||
def test_cache_hit_returns_the_same_instance(self, fake_legacy_genai):
|
||||
provider = _make_provider()
|
||||
|
||||
first = provider._legacy_client_for(OVERRIDE_MODEL)
|
||||
second = provider._legacy_client_for(OVERRIDE_MODEL)
|
||||
|
||||
assert first is second
|
||||
assert first is provider._legacy_model_cache[OVERRIDE_MODEL]
|
||||
assert fake_legacy_genai.build_calls == [OVERRIDE_MODEL]
|
||||
|
||||
def test_distinct_overrides_are_cached_separately(self, fake_legacy_genai):
|
||||
provider = _make_provider()
|
||||
|
||||
provider.generate("hello", model=OVERRIDE_MODEL)
|
||||
provider.generate("hello", model=OTHER_MODEL)
|
||||
provider.generate("hello", model=OVERRIDE_MODEL)
|
||||
|
||||
assert fake_legacy_genai.build_calls == [OVERRIDE_MODEL, OTHER_MODEL]
|
||||
assert set(provider._legacy_model_cache) == {OVERRIDE_MODEL, OTHER_MODEL}
|
||||
assert (
|
||||
provider._legacy_model_cache[OVERRIDE_MODEL]
|
||||
is not provider._legacy_model_cache[OTHER_MODEL]
|
||||
)
|
||||
|
||||
def test_default_model_is_not_cached_or_rebuilt(self, fake_legacy_genai):
|
||||
provider = _make_provider(model=CONSTRUCTION_MODEL)
|
||||
construction_client = provider.client
|
||||
|
||||
provider.generate("hello")
|
||||
provider.generate("hello", model=CONSTRUCTION_MODEL)
|
||||
|
||||
assert fake_legacy_genai.build_calls == []
|
||||
assert provider._legacy_model_cache == {}
|
||||
assert construction_client.generate_content.call_count == 2
|
||||
|
||||
|
||||
class TestLegacyModelCacheIsolation:
|
||||
"""The cache and the legacy SDK's module-level key stay per-instance."""
|
||||
|
||||
def test_configure_reasserted_with_this_key_before_every_call(
|
||||
self, fake_legacy_genai
|
||||
):
|
||||
provider = _make_provider(api_key="key-A")
|
||||
|
||||
provider.generate("hello", model=OVERRIDE_MODEL)
|
||||
provider.generate("hello", model=OVERRIDE_MODEL) # cache hit still re-asserts
|
||||
|
||||
assert fake_legacy_genai.configure.call_count == 2
|
||||
for call in fake_legacy_genai.configure.call_args_list:
|
||||
assert call.kwargs == {"api_key": "key-A"}
|
||||
|
||||
def test_two_instances_keep_separate_caches_and_keys(self, fake_legacy_genai):
|
||||
provider_a = _make_provider(api_key="key-A")
|
||||
provider_b = _make_provider(api_key="key-B")
|
||||
|
||||
provider_a.generate("hello", model=OVERRIDE_MODEL)
|
||||
provider_b.generate("hello", model=OVERRIDE_MODEL)
|
||||
|
||||
# Same model name, but each instance built and cached its own object.
|
||||
assert fake_legacy_genai.build_calls == [OVERRIDE_MODEL, OVERRIDE_MODEL]
|
||||
assert (
|
||||
provider_a._legacy_model_cache[OVERRIDE_MODEL]
|
||||
is not provider_b._legacy_model_cache[OVERRIDE_MODEL]
|
||||
)
|
||||
assert fake_legacy_genai.configure.call_args_list[-2].kwargs == {
|
||||
"api_key": "key-A"
|
||||
}
|
||||
assert fake_legacy_genai.configure.call_args_list[-1].kwargs == {
|
||||
"api_key": "key-B"
|
||||
}
|
||||
+63
-10
@@ -2090,12 +2090,16 @@ class TestMCP:
|
||||
# Table renders correctly — at minimum the column header is present
|
||||
assert "Tool" in result.output or "tool" in result.output.lower()
|
||||
|
||||
def test_list_tools_with_mock_shows_known_tools(self, runner, monkeypatch):
|
||||
fake_tools = _fake_module(__all__=["extract_entities", "query_graph"])
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica_mcp.mcp.tools", fake_tools)
|
||||
def test_list_tools_reads_server_catalog(self, runner, monkeypatch):
|
||||
"""list-tools must read TOOL_DEFINITIONS (what the server serves via
|
||||
tools/list), not the module's ``__all__`` (issue #1355)."""
|
||||
import semantica_mcp.mcp.tools as tools_mod
|
||||
fake = [{"name": "fake_tool_from_catalog", "description": "", "inputSchema": {},
|
||||
"_handler": lambda a: {}}]
|
||||
monkeypatch.setattr(tools_mod, "TOOL_DEFINITIONS", fake)
|
||||
result = runner.invoke(cli_module.main, ["mcp", "list-tools"])
|
||||
_ok(result)
|
||||
assert "extract_entities" in result.output
|
||||
assert "fake_tool_from_catalog" in result.output
|
||||
|
||||
def test_list_tools_json(self, runner):
|
||||
result = runner.invoke(cli_module.main, ["mcp", "list-tools", "--json"])
|
||||
@@ -2114,14 +2118,63 @@ class TestMCP:
|
||||
assert "Traceback" not in result.output
|
||||
assert "Invalid JSON" in result.output
|
||||
|
||||
def test_call_import_error_is_clean(self, runner):
|
||||
with patch("builtins.__import__", side_effect=lambda n, *a, **k: (
|
||||
(_ for _ in ()).throw(ImportError(n))
|
||||
if n.startswith("mcp") else __import__(n, *a, **k)
|
||||
)):
|
||||
result = runner.invoke(cli_module.main, ["mcp", "call", "extract_entities"])
|
||||
def test_call_failure_global_json_mode_keeps_stdout_clean(self, runner):
|
||||
"""Under global --json, stdout must stay machine-readable: failures are
|
||||
emitted as structured JSON on stderr, never as a Rich panel on stdout."""
|
||||
result = runner.invoke(
|
||||
cli_module.main,
|
||||
["--json", "mcp", "call", "some_tool", "--args", "{bad json}"],
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert result.stdout == ""
|
||||
err = json.loads(result.stderr)
|
||||
assert err["error"].startswith("Invalid JSON in --args")
|
||||
assert err["type"] == "ClickException"
|
||||
|
||||
def test_call_failure_local_json_mode_keeps_stdout_clean(self, runner):
|
||||
"""The subcommand's own --json flag promises the same stream contract."""
|
||||
result = runner.invoke(
|
||||
cli_module.main,
|
||||
["mcp", "call", "some_tool", "--args", "{bad json}", "--json"],
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert result.stdout == ""
|
||||
err = json.loads(result.stderr)
|
||||
assert err["error"].startswith("Invalid JSON in --args")
|
||||
|
||||
def test_call_dispatches_through_packaged_server(self, runner):
|
||||
"""Regression for issue #1355: ``mcp call`` dispatches in-process through
|
||||
``semantica_mcp.mcp.server`` (the server ``mcp start`` spawns) instead
|
||||
of importing the nonexistent ``MCPSession``."""
|
||||
result = runner.invoke(
|
||||
cli_module.main, ["--json", "mcp", "call", "extract_entities"]
|
||||
)
|
||||
_ok(result)
|
||||
# Empty args short-circuit before heavy imports; reaching the
|
||||
# handler's own validation proves the dispatch path works.
|
||||
assert "text is required" in result.output
|
||||
|
||||
def test_call_unknown_tool_fails_cleanly(self, runner):
|
||||
result = runner.invoke(cli_module.main, ["mcp", "call", "no_such_tool"])
|
||||
assert result.exit_code != 0
|
||||
assert "Traceback" not in result.output
|
||||
assert "Unknown tool" in result.output
|
||||
|
||||
def test_call_non_object_args_rejected(self, runner):
|
||||
result = runner.invoke(
|
||||
cli_module.main, ["mcp", "call", "extract_entities", "--args", "[1, 2]"]
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "Traceback" not in result.output
|
||||
assert "--args must be a JSON object" in result.output
|
||||
|
||||
def test_list_tools_json_matches_server_catalog(self, runner):
|
||||
"""The CLI catalog and the MCP server catalog must be the same list."""
|
||||
from semantica_mcp.mcp.tools import TOOL_DEFINITIONS
|
||||
result = runner.invoke(cli_module.main, ["mcp", "list-tools", "--json"])
|
||||
_ok(result)
|
||||
data = _json_output(result)
|
||||
assert data["tools"] == [t["name"] for t in TOOL_DEFINITIONS]
|
||||
|
||||
|
||||
# ─── services group (backward-compat wrapper) ─────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Tests for the shared in-process tool entry point (issue #1355).
|
||||
|
||||
``semantica_mcp.mcp.server.call_tool`` is the dispatch used by both the
|
||||
JSON-RPC ``tools/call`` handler and the ``semantica mcp call`` CLI command,
|
||||
so the two surfaces cannot expose different tool sets.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
from semantica_mcp.mcp import server
|
||||
from semantica_mcp.mcp.server import UnknownToolError, _handle_tools_call, call_tool
|
||||
|
||||
|
||||
class TestCallTool(unittest.TestCase):
|
||||
|
||||
def test_known_tool_dispatches_to_handler(self):
|
||||
# Empty args hit extract_entities' own validation before any heavy
|
||||
# imports, which is enough to prove dispatch reached the handler.
|
||||
result = call_tool("extract_entities", {})
|
||||
self.assertEqual(result["error"], "text is required")
|
||||
|
||||
def test_unknown_tool_raises_unknown_tool_error(self):
|
||||
with self.assertRaises(UnknownToolError):
|
||||
call_tool("no_such_tool", {})
|
||||
|
||||
def test_unknown_tool_error_is_not_a_key_error(self):
|
||||
"""A handler's own KeyError (missing required arg) must remain
|
||||
distinguishable from an unknown tool name."""
|
||||
self.assertFalse(issubclass(UnknownToolError, KeyError))
|
||||
|
||||
|
||||
class TestToolsCallDispatch(unittest.TestCase):
|
||||
|
||||
@staticmethod
|
||||
def _tools_call(name, arguments):
|
||||
return _handle_tools_call(1, {"name": name, "arguments": arguments})
|
||||
|
||||
def test_unknown_tool_returns_method_not_found(self):
|
||||
response = self._tools_call("no_such_tool", {})
|
||||
self.assertEqual(response["error"]["code"], -32601)
|
||||
self.assertEqual(response["error"]["message"], "Unknown tool: no_such_tool")
|
||||
|
||||
def test_handler_key_error_is_internal_error_not_unknown_tool(self):
|
||||
def _boom(args):
|
||||
raise KeyError("category")
|
||||
|
||||
fake = {"name": "boom", "description": "", "inputSchema": {}, "_handler": _boom}
|
||||
with patch.dict(server._TOOL_INDEX, {"boom": fake}):
|
||||
response = self._tools_call("boom", {})
|
||||
self.assertEqual(response["error"]["code"], -32603)
|
||||
|
||||
def test_known_tool_returns_result_content(self):
|
||||
response = self._tools_call("extract_entities", {})
|
||||
self.assertIn("content", response["result"])
|
||||
self.assertTrue(response["result"]["isError"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,787 @@
|
||||
"""
|
||||
Tests for the MCP semantic retrieval tools (#1235).
|
||||
|
||||
Covers the six acceptance behaviours proposed in the issue:
|
||||
|
||||
1. store_document chunks content and stores it in a real supported
|
||||
vector backend with provenance metadata (status / version / hash).
|
||||
2. retrieve_context returns semantically relevant chunks with scores
|
||||
and provenance, combined with related graph relationships.
|
||||
3. update_document replaces stored content under (source, version).
|
||||
4. remove_document deletes every chunk of a document.
|
||||
5. Remove-then-store does not collide with surviving in-memory ids
|
||||
(regression guard for the #1029 interaction).
|
||||
6. The same tool set works against the sqlite backend (real persistent
|
||||
store, skipped when the sqlite_vec extension is missing).
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zlib
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
import semantica_mcp.mcp.session as session
|
||||
import semantica.embeddings as _embeddings_pkg
|
||||
import semantica.vector_store.vector_store as _vs_module
|
||||
from semantica_mcp.mcp.session import get_vector_store, reset_vector_store
|
||||
from semantica_mcp.mcp.tools import TOOL_DEFINITIONS
|
||||
from semantica_mcp.mcp.tools.retrieval import (
|
||||
_chunk_id,
|
||||
_chunk_text,
|
||||
handle_remove_document,
|
||||
handle_retrieve_context,
|
||||
handle_store_document,
|
||||
handle_update_document,
|
||||
)
|
||||
|
||||
|
||||
class FakeTextEmbedder:
|
||||
def __init__(self, dim: int = 64):
|
||||
self.dim = dim
|
||||
|
||||
def get_embedding_dimension(self) -> int:
|
||||
return self.dim
|
||||
|
||||
|
||||
class FakeEmbedder:
|
||||
"""
|
||||
Deterministic keyword-bag embedder on a fixed dimension.
|
||||
|
||||
Same words land on the same dimensions, so a query sharing vocabulary
|
||||
with a chunk scores higher than one that does not — enough signal for
|
||||
ranking assertions without any model download. crc32 keeps the
|
||||
word-to-dimension mapping stable across processes (unlike builtin
|
||||
hash(), whose per-process salt would make collisions flaky), and 64
|
||||
dims keep the test keywords collision-free.
|
||||
"""
|
||||
|
||||
def __init__(self, dim: int = 64):
|
||||
self.dim = dim
|
||||
self.text_embedder = FakeTextEmbedder(dim)
|
||||
|
||||
def get_text_method(self) -> str:
|
||||
return "fake"
|
||||
|
||||
def generate_embeddings(self, texts):
|
||||
out = []
|
||||
for t in texts:
|
||||
v = np.zeros(self.dim, dtype=float)
|
||||
for w in str(t).lower().split():
|
||||
if w == "x":
|
||||
# "x" is the filler make_doc pads with; treating it
|
||||
# as a stopword keeps vectors keyword-driven instead
|
||||
# of filler-dominated.
|
||||
continue
|
||||
v[zlib.crc32(w.encode("utf-8")) % self.dim] += 1.0
|
||||
norm = np.linalg.norm(v)
|
||||
if norm:
|
||||
v /= norm
|
||||
out.append(v)
|
||||
return np.array(out)
|
||||
|
||||
|
||||
def make_doc(*keywords) -> str:
|
||||
"""
|
||||
Build filler text with exactly one keyword per chunk.
|
||||
|
||||
With the default 1000 window / 200 overlap, chunk i covers
|
||||
[800*i, 800*i+1000). Keyword i is placed at 800*i + 300, which sits
|
||||
inside chunk i only — clear of both neighbouring overlap zones.
|
||||
Filler is spaced "x " tokens, which the fake embedder treats as a
|
||||
stopword, so chunk vectors are keyword-driven.
|
||||
"""
|
||||
filler = "x "
|
||||
parts = []
|
||||
pos = 0
|
||||
for i, word in enumerate(keywords):
|
||||
target = 800 * i + 300
|
||||
parts.append(filler * ((target - pos) // 2))
|
||||
parts.append(word + " ")
|
||||
pos = target + len(word) + 1
|
||||
parts.append(filler * 30)
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def patch_embedding_generators():
|
||||
"""
|
||||
Patch every EmbeddingGenerator construction site with FakeEmbedder.
|
||||
|
||||
The real EmbeddingGenerator probes FastEmbed / sentence-transformers
|
||||
on init; where those packages are installed but the model is not
|
||||
cached, the probe blocks on a full TCP connect timeout (~30s each).
|
||||
VectorStore's in-memory branch builds one internally, so tests patch
|
||||
both import sites to keep the suite fast and network-free.
|
||||
"""
|
||||
return (
|
||||
patch.object(_embeddings_pkg, "EmbeddingGenerator", FakeEmbedder),
|
||||
patch.object(_vs_module, "EmbeddingGenerator", FakeEmbedder),
|
||||
)
|
||||
|
||||
|
||||
def _clear_retrieval_env():
|
||||
for var in ("SEMANTICA_VECTOR_PATH", "SEMANTICA_VECTOR_BACKEND", "SEMANTICA_VECTOR_DB_PATH"):
|
||||
os.environ.pop(var, None)
|
||||
|
||||
|
||||
class InmemoryBackendTestBase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
_clear_retrieval_env()
|
||||
session._embedder = FakeEmbedder()
|
||||
session._vector_store = None
|
||||
session._graph = None
|
||||
self._patches = patch_embedding_generators()
|
||||
for p in self._patches:
|
||||
p.start()
|
||||
|
||||
def tearDown(self):
|
||||
for p in self._patches:
|
||||
p.stop()
|
||||
session._embedder = None
|
||||
reset_vector_store()
|
||||
session._graph = None
|
||||
_clear_retrieval_env()
|
||||
|
||||
|
||||
class TestChunking(InmemoryBackendTestBase):
|
||||
def test_fixed_window_with_overlap(self):
|
||||
text = "a" * 2600
|
||||
chunks = _chunk_text(text, 1000, 200)
|
||||
self.assertEqual([c[:2] for c in chunks], [(0, 1000), (800, 1800), (1600, 2600)])
|
||||
self.assertTrue(all(c == text[s:e] for s, e, c in chunks))
|
||||
|
||||
def test_short_text_single_chunk(self):
|
||||
chunks = _chunk_text("short", 1000, 200)
|
||||
self.assertEqual(chunks, [(0, 5, "short")])
|
||||
|
||||
def test_overlap_must_be_smaller_than_window(self):
|
||||
with self.assertRaises(ValueError):
|
||||
_chunk_text("abc", 200, 200)
|
||||
|
||||
def test_chunk_id_is_stable_and_position_sensitive(self):
|
||||
a = _chunk_id("src", "v1", 0, "hello")
|
||||
b = _chunk_id("src", "v1", 0, "hello")
|
||||
c = _chunk_id("src", "v1", 1, "hello")
|
||||
self.assertEqual(a, b)
|
||||
self.assertNotEqual(a, c)
|
||||
|
||||
|
||||
class TestStoreDocument(InmemoryBackendTestBase):
|
||||
def test_chunks_carry_provenance_metadata(self):
|
||||
result = handle_store_document(
|
||||
{
|
||||
"content": make_doc("alpha", "beta"),
|
||||
"source": "policy_manual#p12",
|
||||
"authority": "official",
|
||||
"version": "v2",
|
||||
"project": "lending",
|
||||
}
|
||||
)
|
||||
self.assertNotIn("error", result)
|
||||
self.assertEqual(result["status"], "stored")
|
||||
self.assertEqual(result["chunk_count"], len(result["chunk_ids"]))
|
||||
|
||||
store = get_vector_store()
|
||||
first = next(
|
||||
m
|
||||
for m in store.metadata.values()
|
||||
if m.get("source") == "policy_manual#p12" and m.get("chunk_index") == 0
|
||||
)
|
||||
self.assertEqual(first["chunk_id"], result["chunk_ids"][0])
|
||||
self.assertEqual(first["authority"], "official")
|
||||
self.assertEqual(first["version"], "v2")
|
||||
self.assertEqual(first["project"], "lending")
|
||||
self.assertEqual(first["status"], "active")
|
||||
self.assertEqual(first["hash"], result["hash"])
|
||||
self.assertEqual(first["char_start"], 0)
|
||||
|
||||
def test_identical_content_is_a_noop(self):
|
||||
args = {"content": "same content", "source": "doc", "authority": "official"}
|
||||
first = handle_store_document(args)
|
||||
second = handle_store_document(args)
|
||||
self.assertEqual(second["status"], "unchanged")
|
||||
self.assertEqual(second["chunk_ids"], first["chunk_ids"])
|
||||
self.assertEqual(get_vector_store().count(), first["chunk_count"])
|
||||
|
||||
def test_missing_authority_rejected(self):
|
||||
result = handle_store_document({"content": "text", "source": "doc"})
|
||||
self.assertIn("error", result)
|
||||
|
||||
def test_caller_metadata_cannot_override_provenance(self):
|
||||
result = handle_store_document(
|
||||
{
|
||||
"content": make_doc("alpha"),
|
||||
"source": "real_source",
|
||||
"authority": "official",
|
||||
"metadata": {
|
||||
"source": "spoofed_source",
|
||||
"authority": "backdated",
|
||||
"status": "tombstone",
|
||||
"hash": "deadbeef",
|
||||
"version": "v99",
|
||||
"project": "shadow_project",
|
||||
"dept": "risk",
|
||||
},
|
||||
}
|
||||
)
|
||||
self.assertNotIn("error", result)
|
||||
|
||||
store = get_vector_store()
|
||||
meta = next(
|
||||
m
|
||||
for m in store.metadata.values()
|
||||
if m.get("chunk_id") == result["chunk_ids"][0]
|
||||
)
|
||||
self.assertEqual(meta["source"], "real_source")
|
||||
self.assertEqual(meta["authority"], "official")
|
||||
self.assertEqual(meta["status"], "active")
|
||||
self.assertEqual(meta["hash"], result["hash"])
|
||||
self.assertEqual(meta["version"], "v1")
|
||||
self.assertNotIn("project", meta)
|
||||
# Non-provenance keys still land.
|
||||
self.assertEqual(meta["dept"], "risk")
|
||||
|
||||
# Provenance stays intact, so the idempotent no-op still works.
|
||||
again = handle_store_document(
|
||||
{
|
||||
"content": make_doc("alpha"),
|
||||
"source": "real_source",
|
||||
"authority": "official",
|
||||
}
|
||||
)
|
||||
self.assertEqual(again["status"], "unchanged")
|
||||
|
||||
def test_non_dict_metadata_rejected(self):
|
||||
result = handle_store_document(
|
||||
{"content": "text", "source": "doc", "authority": "official", "metadata": ["bad"]}
|
||||
)
|
||||
self.assertIn("error", result)
|
||||
|
||||
|
||||
class TestRetrieveContext(InmemoryBackendTestBase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
handle_store_document(
|
||||
{
|
||||
"content": make_doc("approval", "collateral", "interest"),
|
||||
"source": "lending_policy",
|
||||
"authority": "official",
|
||||
"project": "lending",
|
||||
}
|
||||
)
|
||||
handle_store_document(
|
||||
{
|
||||
"content": make_doc("payment", "refund"),
|
||||
"source": "billing_faq",
|
||||
"authority": "draft",
|
||||
"project": "billing",
|
||||
}
|
||||
)
|
||||
|
||||
def test_relevant_chunks_ranked_with_provenance(self):
|
||||
result = handle_retrieve_context({"query": "collateral", "top_k": 3})
|
||||
self.assertNotIn("error", result)
|
||||
self.assertGreater(result["count"], 0)
|
||||
relevant = [r for r in result["results"] if r["score"] and r["score"] > 0]
|
||||
self.assertTrue(relevant)
|
||||
top = relevant[0]
|
||||
self.assertIn("collateral", top["text"])
|
||||
self.assertEqual(top["source"], "lending_policy")
|
||||
self.assertEqual(top["authority"], "official")
|
||||
self.assertEqual(top["version"], "v1")
|
||||
self.assertEqual(top["status"], "active")
|
||||
self.assertTrue(top["hash"])
|
||||
self.assertIsInstance(top["score"], float)
|
||||
|
||||
def test_top_k_is_capped_at_ten(self):
|
||||
# 13 chunks (one keyword per chunk) so the cap is actually hit;
|
||||
# with fewer stored chunks the assertion would pass trivially.
|
||||
handle_store_document(
|
||||
{
|
||||
"content": make_doc(*["k%02d" % i for i in range(1, 14)]),
|
||||
"source": "capdoc",
|
||||
"authority": "official",
|
||||
}
|
||||
)
|
||||
result = handle_retrieve_context({"query": "k01", "top_k": 99})
|
||||
self.assertEqual(result["count"], 10)
|
||||
|
||||
def test_project_filter_narrows_results(self):
|
||||
result = handle_retrieve_context({"query": "collateral", "project": "billing"})
|
||||
for r in result["results"]:
|
||||
self.assertEqual(r["project"], "billing")
|
||||
|
||||
def test_graph_relationships_attached(self):
|
||||
graph = session.get_graph()
|
||||
graph.add_node(
|
||||
node_id="policy_doc_lending_policy",
|
||||
label="Lending policy doc",
|
||||
node_type="Document",
|
||||
metadata={"source": "lending_policy"},
|
||||
)
|
||||
graph.add_node(node_id="risk_team", label="Risk team", node_type="Team")
|
||||
graph.add_edge(
|
||||
source_id="policy_doc_lending_policy",
|
||||
target_id="risk_team",
|
||||
edge_type="OWNED_BY",
|
||||
)
|
||||
result = handle_retrieve_context({"query": "collateral"})
|
||||
self.assertGreaterEqual(len(result["graph_context"]), 1)
|
||||
rel = result["graph_context"][0]
|
||||
self.assertEqual(rel["node"]["source"], "lending_policy")
|
||||
self.assertEqual(rel["related"]["id"], "risk_team")
|
||||
self.assertEqual(rel["relationship"], "OWNED_BY")
|
||||
|
||||
def test_empty_query_rejected(self):
|
||||
result = handle_retrieve_context({"query": " "})
|
||||
self.assertIn("error", result)
|
||||
|
||||
|
||||
class TestUpdateDocument(InmemoryBackendTestBase):
|
||||
def test_update_replaces_chunks(self):
|
||||
handle_store_document(
|
||||
{
|
||||
"content": make_doc("oldterm", "legacy"),
|
||||
"source": "handbook",
|
||||
"authority": "official",
|
||||
}
|
||||
)
|
||||
result = handle_update_document(
|
||||
{
|
||||
"content": make_doc("newterm"),
|
||||
"source": "handbook",
|
||||
"version": "v1",
|
||||
}
|
||||
)
|
||||
self.assertEqual(result["status"], "updated")
|
||||
self.assertEqual(result["chunk_count"], 1)
|
||||
|
||||
hits = handle_retrieve_context({"query": "newterm"})["results"]
|
||||
hits = [h for h in hits if h["score"] and h["score"] > 0]
|
||||
self.assertTrue(hits and "newterm" in hits[0]["text"])
|
||||
stale = handle_retrieve_context({"query": "oldterm"})["results"]
|
||||
stale = [h for h in stale if h["score"] and h["score"] > 0]
|
||||
self.assertEqual(stale, [])
|
||||
# Authority is inherited from the stored version when omitted.
|
||||
self.assertEqual(hits[0]["authority"], "official")
|
||||
self.assertEqual(get_vector_store().count(), 1)
|
||||
|
||||
def test_update_rolls_back_when_new_write_fails(self):
|
||||
handle_store_document(
|
||||
{
|
||||
"content": make_doc("oldterm", "legacy"),
|
||||
"source": "handbook",
|
||||
"authority": "official",
|
||||
}
|
||||
)
|
||||
store = get_vector_store()
|
||||
real_store_vectors = store.store_vectors
|
||||
|
||||
def failing_write(vectors, metas):
|
||||
if any("phoenix" in (m.get("text") or "") for m in metas):
|
||||
raise RuntimeError("simulated write failure")
|
||||
return real_store_vectors(vectors, metas)
|
||||
|
||||
with patch.object(store, "store_vectors", side_effect=failing_write):
|
||||
result = handle_update_document(
|
||||
{"content": make_doc("phoenix"), "source": "handbook"}
|
||||
)
|
||||
self.assertIn("error", result)
|
||||
self.assertIn("simulated write failure", result["error"])
|
||||
|
||||
# The old document must survive the failed replacement, with no
|
||||
# trace of the new content.
|
||||
store = get_vector_store()
|
||||
self.assertEqual(store.count(), 2)
|
||||
old = [
|
||||
h
|
||||
for h in handle_retrieve_context({"query": "oldterm"})["results"]
|
||||
if h["score"] and h["score"] > 0
|
||||
]
|
||||
self.assertTrue(old and "oldterm" in old[0]["text"])
|
||||
self.assertEqual(old[0]["source"], "handbook")
|
||||
self.assertEqual(old[0]["authority"], "official")
|
||||
phoenix = [
|
||||
h
|
||||
for h in handle_retrieve_context({"query": "phoenix"})["results"]
|
||||
if h["score"] and h["score"] > 0
|
||||
]
|
||||
self.assertEqual(phoenix, [])
|
||||
|
||||
def test_update_missing_document_reports_not_found(self):
|
||||
result = handle_update_document(
|
||||
{
|
||||
"content": make_doc("neverseen"),
|
||||
"source": "never_stored",
|
||||
"version": "v1",
|
||||
}
|
||||
)
|
||||
self.assertEqual(result["status"], "not_found")
|
||||
self.assertEqual(result["source"], "never_stored")
|
||||
self.assertEqual(result["version"], "v1")
|
||||
self.assertEqual(get_vector_store().count(), 0)
|
||||
|
||||
|
||||
class TestRemoveDocument(InmemoryBackendTestBase):
|
||||
def test_remove_deletes_every_chunk(self):
|
||||
handle_store_document(
|
||||
{
|
||||
"content": make_doc("alpha", "beta", "gamma"),
|
||||
"source": "docA",
|
||||
"authority": "official",
|
||||
}
|
||||
)
|
||||
result = handle_remove_document({"source": "docA"})
|
||||
self.assertEqual(result["status"], "removed")
|
||||
self.assertEqual(result["removed_chunks"], 3)
|
||||
self.assertEqual(get_vector_store().count(), 0)
|
||||
again = handle_remove_document({"source": "docA"})
|
||||
self.assertEqual(again["status"], "not_found")
|
||||
|
||||
def test_remove_missing_document_reports_not_found(self):
|
||||
result = handle_remove_document({"source": "never_stored"})
|
||||
self.assertEqual(result["status"], "not_found")
|
||||
|
||||
|
||||
class TestInMemoryIdCollisionRegression(InmemoryBackendTestBase):
|
||||
"""
|
||||
#1029 interaction guard.
|
||||
|
||||
In-memory vector ids are ``vec_{len(self.vectors) + i}``. Deleting a
|
||||
document that is NOT a suffix makes len() fall below surviving ids, so
|
||||
the next plain write overwrites live data. Our rebuild path must
|
||||
prevent that: store a 1-chunk doc, then a 3-chunk doc, remove the
|
||||
1-chunk one, then store another doc. Without the rebuild the last
|
||||
store lands on the surviving document's third chunk id and destroys
|
||||
it.
|
||||
"""
|
||||
|
||||
def test_remove_then_store_keeps_surviving_chunks_intact(self):
|
||||
handle_store_document(
|
||||
{"content": make_doc("alpha"), "source": "docA", "authority": "official"}
|
||||
)
|
||||
handle_store_document(
|
||||
{
|
||||
"content": make_doc("bravo", "charlie", "delta"),
|
||||
"source": "docB",
|
||||
"authority": "official",
|
||||
}
|
||||
)
|
||||
self.assertEqual(get_vector_store().count(), 4)
|
||||
|
||||
removed = handle_remove_document({"source": "docA"})
|
||||
self.assertEqual(removed["status"], "removed")
|
||||
|
||||
stored = handle_store_document(
|
||||
{"content": make_doc("echo"), "source": "docC", "authority": "official"}
|
||||
)
|
||||
self.assertEqual(stored["status"], "stored")
|
||||
|
||||
store = get_vector_store()
|
||||
self.assertEqual(store.count(), 4)
|
||||
|
||||
delta_hits = handle_retrieve_context({"query": "delta"})["results"]
|
||||
delta_hits = [h for h in delta_hits if h["score"] and h["score"] > 0]
|
||||
self.assertTrue(delta_hits, "docB's third chunk was destroyed by an id collision")
|
||||
self.assertIn("delta", delta_hits[0]["text"])
|
||||
self.assertEqual(delta_hits[0]["source"], "docB")
|
||||
|
||||
for keyword, expected_source in (
|
||||
("bravo", "docB"),
|
||||
("charlie", "docB"),
|
||||
("echo", "docC"),
|
||||
):
|
||||
hits = [
|
||||
h
|
||||
for h in handle_retrieve_context({"query": keyword})["results"]
|
||||
if h["score"] and h["score"] > 0
|
||||
]
|
||||
self.assertTrue(hits, f"expected a hit for {keyword}")
|
||||
self.assertEqual(hits[0]["source"], expected_source)
|
||||
|
||||
|
||||
class TestBackendPolicy(InmemoryBackendTestBase):
|
||||
def test_unsupported_backend_fails_fast(self):
|
||||
# faiss/pgvector lack a metadata-scoped delete, so update/remove
|
||||
# cannot work on them; selecting them must fail at startup, not
|
||||
# mid-update.
|
||||
for backend in ("faiss", "pgvector"):
|
||||
with self.subTest(backend=backend):
|
||||
os.environ["SEMANTICA_VECTOR_BACKEND"] = backend
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
get_vector_store()
|
||||
self.assertIn("not supported", str(ctx.exception))
|
||||
|
||||
def test_oversized_document_rejected_before_embedding(self):
|
||||
# chunk_size=1 turns a 12k-char body into 12k chunks, crossing
|
||||
# the ingestion cap without any expensive embedding work.
|
||||
result = handle_store_document(
|
||||
{
|
||||
"content": "ab" * 6000,
|
||||
"source": "bigdoc",
|
||||
"authority": "official",
|
||||
"chunk_size": 1,
|
||||
"chunk_overlap": 0,
|
||||
}
|
||||
)
|
||||
self.assertIn("error", result)
|
||||
self.assertIn("chunks", result["error"])
|
||||
self.assertEqual(get_vector_store().count(), 0)
|
||||
|
||||
|
||||
class TestToolRegistration(unittest.TestCase):
|
||||
def test_retrieval_tools_are_registered(self):
|
||||
retrieval = {
|
||||
t["name"]: t
|
||||
for t in TOOL_DEFINITIONS
|
||||
if t["name"] in ("store_document", "retrieve_context", "update_document", "remove_document")
|
||||
}
|
||||
self.assertEqual(len(retrieval), 4)
|
||||
for name, t in retrieval.items():
|
||||
self.assertTrue(callable(t["_handler"]))
|
||||
self.assertIn("required", t["inputSchema"])
|
||||
|
||||
|
||||
class TestSqliteBackend(unittest.TestCase):
|
||||
def setUp(self):
|
||||
try:
|
||||
import sqlite_vec # noqa: F401
|
||||
except ImportError:
|
||||
self.skipTest("sqlite_vec extension not installed")
|
||||
self.tmpdir = tempfile.mkdtemp(prefix="semantica_sqlite_test_")
|
||||
self.patches = patch_embedding_generators()
|
||||
for p in self.patches:
|
||||
p.start()
|
||||
_clear_retrieval_env()
|
||||
os.environ["SEMANTICA_VECTOR_BACKEND"] = "sqlite"
|
||||
os.environ["SEMANTICA_VECTOR_DB_PATH"] = os.path.join(self.tmpdir, "vectors.db")
|
||||
session._embedder = FakeEmbedder()
|
||||
session._vector_store = None
|
||||
|
||||
def tearDown(self):
|
||||
for p in self.patches:
|
||||
p.stop()
|
||||
session._embedder = None
|
||||
reset_vector_store()
|
||||
_clear_retrieval_env()
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(self.tmpdir, ignore_errors=True)
|
||||
|
||||
def test_sqlite_backend_roundtrip(self):
|
||||
handle_store_document(
|
||||
{
|
||||
"content": make_doc("alpha", "beta"),
|
||||
"source": "docS",
|
||||
"authority": "official",
|
||||
}
|
||||
)
|
||||
hits = handle_retrieve_context({"query": "beta"})["results"]
|
||||
self.assertTrue(hits and "beta" in hits[0]["text"])
|
||||
self.assertEqual(hits[0]["source"], "docS")
|
||||
|
||||
updated = handle_update_document(
|
||||
{"content": make_doc("gamma"), "source": "docS"}
|
||||
)
|
||||
self.assertEqual(updated["status"], "updated")
|
||||
# NB: score scales differ across backends (sqlite maps distance
|
||||
# through 1/(1+d), so an orthogonal chunk still scores 0.5).
|
||||
# Assert on text, the only backend-independent signal.
|
||||
stale = [
|
||||
h
|
||||
for h in handle_retrieve_context({"query": "beta"})["results"]
|
||||
if "beta" in (h.get("text") or "")
|
||||
]
|
||||
self.assertEqual(stale, [])
|
||||
self.assertTrue(handle_retrieve_context({"query": "gamma"})["results"])
|
||||
|
||||
removed = handle_remove_document({"source": "docS"})
|
||||
self.assertEqual(removed["status"], "removed")
|
||||
self.assertEqual(get_vector_store().count(), 0)
|
||||
|
||||
def test_sqlite_multi_document_isolation(self):
|
||||
handle_store_document(
|
||||
{
|
||||
"content": make_doc("harbor", "vessel"),
|
||||
"source": "nav_docs",
|
||||
"authority": "official",
|
||||
}
|
||||
)
|
||||
handle_store_document(
|
||||
{
|
||||
"content": make_doc("ledger", "invoice"),
|
||||
"source": "fin_docs",
|
||||
"authority": "draft",
|
||||
"version": "v2",
|
||||
}
|
||||
)
|
||||
|
||||
nav = [
|
||||
h
|
||||
for h in handle_retrieve_context({"query": "vessel"})["results"]
|
||||
if "vessel" in (h.get("text") or "")
|
||||
]
|
||||
self.assertTrue(nav)
|
||||
self.assertEqual(nav[0]["source"], "nav_docs")
|
||||
self.assertEqual(nav[0]["authority"], "official")
|
||||
self.assertEqual(nav[0]["status"], "active")
|
||||
self.assertTrue(nav[0]["hash"])
|
||||
|
||||
# Updating one document must leave the other untouched.
|
||||
updated = handle_update_document(
|
||||
{"content": make_doc("anchor"), "source": "nav_docs"}
|
||||
)
|
||||
self.assertEqual(updated["status"], "updated")
|
||||
fin = [
|
||||
h
|
||||
for h in handle_retrieve_context({"query": "invoice"})["results"]
|
||||
if "invoice" in (h.get("text") or "")
|
||||
]
|
||||
self.assertTrue(fin)
|
||||
self.assertEqual(fin[0]["source"], "fin_docs")
|
||||
self.assertEqual(fin[0]["authority"], "draft")
|
||||
vessel_stale = [
|
||||
h
|
||||
for h in handle_retrieve_context({"query": "vessel"})["results"]
|
||||
if "vessel" in (h.get("text") or "")
|
||||
]
|
||||
self.assertEqual(vessel_stale, [])
|
||||
|
||||
# Removing the other document must leave the first intact.
|
||||
removed = handle_remove_document({"source": "fin_docs", "version": "v2"})
|
||||
self.assertEqual(removed["status"], "removed")
|
||||
anchor = [
|
||||
h
|
||||
for h in handle_retrieve_context({"query": "anchor"})["results"]
|
||||
if "anchor" in (h.get("text") or "")
|
||||
]
|
||||
self.assertTrue(anchor and anchor[0]["source"] == "nav_docs")
|
||||
ledger_stale = [
|
||||
h
|
||||
for h in handle_retrieve_context({"query": "ledger"})["results"]
|
||||
if "ledger" in (h.get("text") or "")
|
||||
]
|
||||
self.assertEqual(ledger_stale, [])
|
||||
|
||||
def test_sqlite_update_rolls_back_on_write_failure(self):
|
||||
# Persistent path: removal is a direct delete_vectors, so the
|
||||
# rollback has to re-store the snapshotted rows (plain lists,
|
||||
# not arrays) when the new write fails.
|
||||
handle_store_document(
|
||||
{
|
||||
"content": make_doc("oldterm", "legacy"),
|
||||
"source": "handbook",
|
||||
"authority": "official",
|
||||
}
|
||||
)
|
||||
store = get_vector_store()
|
||||
real_store_vectors = store.store_vectors
|
||||
|
||||
def failing_write(vectors, metas):
|
||||
if any("phoenix" in (m.get("text") or "") for m in metas):
|
||||
raise RuntimeError("simulated write failure")
|
||||
return real_store_vectors(vectors, metas)
|
||||
|
||||
with patch.object(store, "store_vectors", side_effect=failing_write):
|
||||
result = handle_update_document(
|
||||
{"content": make_doc("phoenix"), "source": "handbook"}
|
||||
)
|
||||
self.assertIn("error", result)
|
||||
self.assertEqual(get_vector_store().count(), 2)
|
||||
old = [
|
||||
h
|
||||
for h in handle_retrieve_context({"query": "oldterm"})["results"]
|
||||
if "oldterm" in (h.get("text") or "")
|
||||
]
|
||||
self.assertTrue(old and old[0]["source"] == "handbook")
|
||||
|
||||
def test_sqlite_without_db_path_raises(self):
|
||||
os.environ.pop("SEMANTICA_VECTOR_DB_PATH", None)
|
||||
with self.assertRaises(ValueError):
|
||||
get_vector_store()
|
||||
|
||||
|
||||
class TestPersistence(InmemoryBackendTestBase):
|
||||
def test_store_persists_and_reloads(self):
|
||||
tmpdir = tempfile.mkdtemp(prefix="semantica_vec_test_")
|
||||
try:
|
||||
os.environ["SEMANTICA_VECTOR_PATH"] = tmpdir
|
||||
result = handle_store_document(
|
||||
{"content": make_doc("persist"), "source": "docP", "authority": "official"}
|
||||
)
|
||||
self.assertTrue(result["persisted"])
|
||||
self.assertTrue(os.path.isfile(os.path.join(tmpdir, "store_data.json")))
|
||||
|
||||
# Fresh session state: the store must reload from disk.
|
||||
reset_vector_store()
|
||||
hits = handle_retrieve_context({"query": "persist"})["results"]
|
||||
self.assertTrue(hits and "persist" in hits[0]["text"])
|
||||
self.assertEqual(hits[0]["source"], "docP")
|
||||
finally:
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
def test_persist_failure_is_reported_not_silent(self):
|
||||
tmpdir = tempfile.mkdtemp(prefix="semantica_vec_test_")
|
||||
try:
|
||||
os.environ["SEMANTICA_VECTOR_PATH"] = tmpdir
|
||||
store = get_vector_store()
|
||||
with patch.object(store, "save", side_effect=RuntimeError("disk full")):
|
||||
result = handle_store_document(
|
||||
{"content": make_doc("volatile"), "source": "docV", "authority": "official"}
|
||||
)
|
||||
# The write itself succeeded; only the durable copy failed.
|
||||
self.assertEqual(result["status"], "stored")
|
||||
self.assertFalse(result["persisted"])
|
||||
finally:
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
def test_reload_dimension_mismatch_rejected(self):
|
||||
tmpdir = tempfile.mkdtemp(prefix="semantica_vec_test_")
|
||||
try:
|
||||
os.environ["SEMANTICA_VECTOR_PATH"] = tmpdir
|
||||
handle_store_document(
|
||||
{"content": make_doc("persist"), "source": "docP", "authority": "official"}
|
||||
)
|
||||
# A different embedder dimension must not silently rank
|
||||
# vectors from an incompatible embedding space.
|
||||
session._embedder = FakeEmbedder(32)
|
||||
reset_vector_store()
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
get_vector_store()
|
||||
self.assertIn("dimension", str(ctx.exception))
|
||||
finally:
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
def test_corrupt_store_fails_on_startup(self):
|
||||
"""A broken persisted store must raise immediately, not silently
|
||||
fall back to an empty store that would overwrite the user's data
|
||||
on the first persist."""
|
||||
tmpdir = tempfile.mkdtemp(prefix="semantica_vec_test_")
|
||||
try:
|
||||
# Drop a file that looks like a store directory but won't load.
|
||||
with open(os.path.join(tmpdir, "store_data.json"), "w") as f:
|
||||
f.write("{not valid json")
|
||||
os.environ["SEMANTICA_VECTOR_PATH"] = tmpdir
|
||||
reset_vector_store()
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
get_vector_store()
|
||||
self.assertIn("Could not load", str(ctx.exception))
|
||||
finally:
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -91,7 +91,7 @@ _INIT_REQUEST = _jsonrpc("initialize", 1, {
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMCPStdioFramingContract(unittest.TestCase):
|
||||
"""Run 'python -m mcp' exactly as an MCP client would, over a real pipe.
|
||||
"""Run 'python -m semantica_mcp.mcp' exactly as an MCP client would, over a real pipe.
|
||||
|
||||
Each test sends a complete JSON-RPC session through stdin and asserts that
|
||||
every byte on stdout is valid JSON — catching the exact failure mode from
|
||||
@@ -102,7 +102,7 @@ class TestMCPStdioFramingContract(unittest.TestCase):
|
||||
|
||||
def _run(self, *requests: bytes) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
[sys.executable, "-m", "mcp"],
|
||||
[sys.executable, "-m", "semantica_mcp.mcp"],
|
||||
input=b"".join(requests),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
|
||||
@@ -0,0 +1,662 @@
|
||||
"""Tests for FAISSIndex.delete_vectors and FAISSStore.delete_vectors (#1374)."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from semantica.context.erasure import (
|
||||
STATUS_ERASED,
|
||||
STATUS_UNSUPPORTED,
|
||||
ErasureCoordinator,
|
||||
)
|
||||
from semantica.utils.exceptions import ProcessingError
|
||||
from semantica.vector_store import VectorStore
|
||||
from semantica.vector_store.faiss_store import FAISSIndex, FAISSStore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures and helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _flat_index(dim: int = 3) -> "faiss.IndexFlatL2": # noqa: F821
|
||||
faiss = pytest.importorskip("faiss")
|
||||
return faiss.IndexFlatL2(dim)
|
||||
|
||||
|
||||
def _populated_store(
|
||||
dim: int = 3,
|
||||
ids=("a", "b", "c", "d", "e"),
|
||||
meta=None,
|
||||
):
|
||||
"""Return an FAISSStore with *ids* already inserted (random unit vectors)."""
|
||||
faiss = pytest.importorskip("faiss")
|
||||
store = FAISSStore(dimension=dim)
|
||||
n = len(ids)
|
||||
rng = np.random.default_rng(seed=42)
|
||||
vectors = rng.random((n, dim)).astype(np.float32)
|
||||
metadata = meta or [{} for _ in ids]
|
||||
store.add_vectors(vectors, ids=list(ids), metadata=metadata)
|
||||
return store
|
||||
|
||||
|
||||
def _populated_index(dim: int = 3, ids=("a", "b", "c", "d", "e")):
|
||||
"""Return a bare FAISSIndex with *ids* inserted (random unit vectors)."""
|
||||
faiss = pytest.importorskip("faiss")
|
||||
idx = FAISSIndex(faiss.IndexFlatL2(dim), dimension=dim)
|
||||
n = len(ids)
|
||||
rng = np.random.default_rng(seed=42)
|
||||
vectors = rng.random((n, dim)).astype(np.float32)
|
||||
idx.add_vectors(vectors, ids=list(ids))
|
||||
return idx
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FAISSIndex-level unit tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFAISSIndexDeleteVectors:
|
||||
def test_delete_single_existing_id(self):
|
||||
idx = _populated_index()
|
||||
result = idx.delete_vectors(["b"])
|
||||
assert result == {"delete_count": 1}
|
||||
assert "b" not in idx.vector_ids
|
||||
assert idx.index.ntotal == len(idx.vector_ids) == 4
|
||||
|
||||
def test_delete_multiple_existing_ids(self):
|
||||
idx = _populated_index()
|
||||
result = idx.delete_vectors(["b", "d"])
|
||||
assert result == {"delete_count": 2}
|
||||
assert "b" not in idx.vector_ids
|
||||
assert "d" not in idx.vector_ids
|
||||
assert sorted(idx.vector_ids) == ["a", "c", "e"]
|
||||
assert idx.index.ntotal == 3
|
||||
|
||||
def test_delete_nonexistent_id_is_noop(self):
|
||||
idx = _populated_index()
|
||||
result = idx.delete_vectors(["z"])
|
||||
assert result == {"delete_count": 0}
|
||||
assert len(idx.vector_ids) == 5
|
||||
assert idx.index.ntotal == 5
|
||||
|
||||
def test_delete_empty_list_is_noop(self):
|
||||
idx = _populated_index()
|
||||
result = idx.delete_vectors([])
|
||||
assert result == {"delete_count": 0}
|
||||
assert len(idx.vector_ids) == 5
|
||||
|
||||
def test_delete_duplicate_ids_in_request_only_removes_once(self):
|
||||
idx = _populated_index()
|
||||
result = idx.delete_vectors(["b", "b", "b"])
|
||||
assert result == {"delete_count": 1}
|
||||
assert "b" not in idx.vector_ids
|
||||
assert len(idx.vector_ids) == 4
|
||||
|
||||
def test_delete_count_reflects_actual_removal(self):
|
||||
idx = _populated_index()
|
||||
# "z" doesn't exist; only "a" and "c" do
|
||||
result = idx.delete_vectors(["a", "c", "z"])
|
||||
assert result == {"delete_count": 2}
|
||||
|
||||
def test_metadata_removed_for_deleted_id(self):
|
||||
faiss = pytest.importorskip("faiss")
|
||||
idx = FAISSIndex(faiss.IndexFlatL2(3), dimension=3)
|
||||
vectors = np.eye(3, dtype=np.float32)[:2]
|
||||
idx.add_vectors(vectors, ids=["x", "y"])
|
||||
idx.metadata = {"x": {"val": 1}, "y": {"val": 2}}
|
||||
idx.delete_vectors(["x"])
|
||||
assert "x" not in idx.metadata
|
||||
assert "y" in idx.metadata
|
||||
|
||||
def test_vector_ids_list_stays_parallel_to_faiss_ntotal(self):
|
||||
idx = _populated_index(ids=["a", "b", "c"])
|
||||
idx.delete_vectors(["b"])
|
||||
assert len(idx.vector_ids) == idx.index.ntotal == 2
|
||||
|
||||
def test_search_does_not_return_deleted_id(self):
|
||||
"""After deletion, similarity search must not return the deleted ID."""
|
||||
faiss = pytest.importorskip("faiss")
|
||||
idx = FAISSIndex(faiss.IndexFlatL2(3), dimension=3)
|
||||
vectors = np.array(
|
||||
[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], dtype=np.float32
|
||||
)
|
||||
idx.add_vectors(vectors, ids=["a", "b", "c"])
|
||||
idx.delete_vectors(["b"])
|
||||
|
||||
query = np.array([[0.0, 1.0, 0.0]], dtype=np.float32)
|
||||
distances, indices = idx.search(query, k=3)
|
||||
# Filter both negative sentinels (-1) and out-of-range indices.
|
||||
returned_ids = [
|
||||
idx.vector_ids[i]
|
||||
for i in indices[0]
|
||||
if 0 <= i < len(idx.vector_ids)
|
||||
]
|
||||
assert "b" not in returned_ids
|
||||
|
||||
def test_get_vector_returns_none_after_deletion(self):
|
||||
faiss = pytest.importorskip("faiss")
|
||||
idx = FAISSIndex(faiss.IndexFlatL2(3), dimension=3)
|
||||
vectors = np.eye(3, dtype=np.float32)
|
||||
idx.add_vectors(vectors, ids=["a", "b", "c"])
|
||||
idx.delete_vectors(["b"])
|
||||
assert idx.get_vector("b") is None
|
||||
|
||||
def test_get_metadata_returns_none_after_deletion(self):
|
||||
faiss = pytest.importorskip("faiss")
|
||||
idx = FAISSIndex(faiss.IndexFlatL2(3), dimension=3)
|
||||
idx.add_vectors(np.eye(3, dtype=np.float32)[:2], ids=["a", "b"])
|
||||
idx.metadata = {"a": {"k": 1}, "b": {"k": 2}}
|
||||
idx.delete_vectors(["b"])
|
||||
assert idx.get_metadata("b") is None
|
||||
|
||||
def test_add_vectors_after_deletion_works(self):
|
||||
"""Inserting new vectors after deletion maintains correct position mapping."""
|
||||
idx = _populated_index(ids=["a", "b", "c"])
|
||||
idx.delete_vectors(["b"])
|
||||
new_vecs = np.array([[0.5, 0.5, 0.0]], dtype=np.float32)
|
||||
idx.add_vectors(new_vecs, ids=["new"])
|
||||
assert "new" in idx.vector_ids
|
||||
assert len(idx.vector_ids) == idx.index.ntotal == 3
|
||||
|
||||
def test_save_load_after_deletion_preserves_state(self, tmp_path):
|
||||
"""Deletion persists correctly through save/load round-trip."""
|
||||
_ = pytest.importorskip("faiss")
|
||||
idx = _populated_index(ids=["a", "b", "c"])
|
||||
idx.delete_vectors(["b"])
|
||||
|
||||
path = tmp_path / "idx.faiss"
|
||||
idx.save(path)
|
||||
loaded = FAISSIndex.load(path, dimension=3)
|
||||
|
||||
assert "b" not in loaded.vector_ids
|
||||
assert sorted(loaded.vector_ids) == ["a", "c"]
|
||||
assert loaded.index.ntotal == 2
|
||||
|
||||
def test_hnsw_delete_raises_not_implemented(self):
|
||||
"""HNSW does not support remove_ids; must raise NotImplementedError."""
|
||||
faiss = pytest.importorskip("faiss")
|
||||
hnsw = FAISSIndex(faiss.IndexHNSWFlat(4, 16), dimension=4)
|
||||
vecs = np.random.rand(5, 4).astype(np.float32)
|
||||
hnsw.add_vectors(vecs, ids=["a", "b", "c", "d", "e"])
|
||||
with pytest.raises(NotImplementedError):
|
||||
hnsw.delete_vectors(["a"])
|
||||
# Python-side state must be untouched
|
||||
assert len(hnsw.vector_ids) == 5
|
||||
|
||||
def test_ivf_delete_raises_not_implemented(self):
|
||||
"""IVF does not compact labels after remove_ids; raise NotImplementedError.
|
||||
|
||||
IVF surviving labels stay sparse (0,2,4 not 0,1,2), so the list-compact
|
||||
approach used by Flat would desynchronize search labels from vector_ids.
|
||||
"""
|
||||
faiss = pytest.importorskip("faiss")
|
||||
dim = 4
|
||||
train = np.random.rand(80, dim).astype(np.float32)
|
||||
q = faiss.IndexFlatL2(dim)
|
||||
ivf = faiss.IndexIVFFlat(q, dim, 2)
|
||||
ivf.train(train)
|
||||
idx = FAISSIndex(ivf, dimension=dim)
|
||||
vecs = np.random.rand(5, dim).astype(np.float32)
|
||||
idx.add_vectors(vecs, ids=["a", "b", "c", "d", "e"])
|
||||
with pytest.raises(NotImplementedError):
|
||||
idx.delete_vectors(["b"])
|
||||
# Python-side state must be completely untouched
|
||||
assert idx.vector_ids == ["a", "b", "c", "d", "e"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FAISSStore-level unit tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFAISSStoreDeleteVectors:
|
||||
def test_delete_uninitialized_index_raises_processing_error(self):
|
||||
store = FAISSStore(dimension=3)
|
||||
with pytest.raises(ProcessingError, match="Index not initialized"):
|
||||
store.delete_vectors(["a"])
|
||||
|
||||
def test_delete_existing_id_returns_dict(self):
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(ids=["a", "b", "c"])
|
||||
result = store.delete_vectors(["b"])
|
||||
assert result == {"delete_count": 1}
|
||||
|
||||
def test_delete_reduces_count(self):
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(ids=["a", "b", "c"])
|
||||
assert store.count() == 3
|
||||
store.delete_vectors(["b"])
|
||||
assert store.count() == 2
|
||||
|
||||
def test_delete_multiple_ids(self):
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(ids=["a", "b", "c", "d"])
|
||||
result = store.delete_vectors(["a", "c"])
|
||||
assert result == {"delete_count": 2}
|
||||
assert store.count() == 2
|
||||
|
||||
def test_delete_empty_input_is_noop(self):
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(ids=["a", "b"])
|
||||
result = store.delete_vectors([])
|
||||
assert result == {"delete_count": 0}
|
||||
assert store.count() == 2
|
||||
|
||||
def test_delete_nonexistent_id_is_zero(self):
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(ids=["a", "b"])
|
||||
result = store.delete_vectors(["z"])
|
||||
assert result == {"delete_count": 0}
|
||||
assert store.count() == 2
|
||||
|
||||
def test_duplicate_ids_in_request(self):
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(ids=["a", "b"])
|
||||
result = store.delete_vectors(["a", "a"])
|
||||
assert result == {"delete_count": 1}
|
||||
assert store.count() == 1
|
||||
|
||||
def test_metadata_cleaned_up(self):
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(
|
||||
ids=["a", "b"],
|
||||
meta=[{"owner": "alice"}, {"owner": "bob"}],
|
||||
)
|
||||
store.delete_vectors(["a"])
|
||||
assert store.get_metadata("a") is None
|
||||
assert store.get_metadata("b") == {"owner": "bob"}
|
||||
|
||||
def test_get_vector_returns_none_after_deletion(self):
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(ids=["a", "b"])
|
||||
store.delete_vectors(["a"])
|
||||
assert store.get_vector("a") is None
|
||||
|
||||
def test_search_excludes_deleted_vector(self):
|
||||
"""search_similar must not return a deleted vector's ID."""
|
||||
faiss = pytest.importorskip("faiss")
|
||||
store = FAISSStore(dimension=3)
|
||||
vectors = np.array(
|
||||
[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], dtype=np.float32
|
||||
)
|
||||
store.add_vectors(vectors, ids=["a", "b", "c"])
|
||||
store.delete_vectors(["b"])
|
||||
query = np.array([0.0, 1.0, 0.0], dtype=np.float32)
|
||||
results = store.search_similar(query, k=3)
|
||||
returned_ids = [r["id"] for r in results]
|
||||
assert "b" not in returned_ids
|
||||
|
||||
def test_add_vectors_after_deletion(self):
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(ids=["a", "b", "c"])
|
||||
store.delete_vectors(["b"])
|
||||
vecs = np.array([[0.5, 0.5, 0.0]], dtype=np.float32)
|
||||
store.add_vectors(vecs, ids=["new"])
|
||||
assert store.count() == 3
|
||||
assert store.get_vector("new") is not None
|
||||
|
||||
def test_save_load_after_deletion(self, tmp_path):
|
||||
"""Deleted vectors do not reappear after save/load."""
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(ids=["a", "b", "c"])
|
||||
store.delete_vectors(["b"])
|
||||
|
||||
path = tmp_path / "store.faiss"
|
||||
store.save_index(path)
|
||||
|
||||
fresh = FAISSStore(dimension=3)
|
||||
fresh.load_index(path)
|
||||
|
||||
assert fresh.count() == 2
|
||||
assert "b" not in fresh.index.vector_ids
|
||||
assert fresh.get_vector("b") is None
|
||||
|
||||
def test_options_kwarg_is_accepted_and_ignored(self):
|
||||
"""delete_vectors(**options) must not crash even with extra kwargs."""
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(ids=["a"])
|
||||
result = store.delete_vectors(["a"], unused_option=True)
|
||||
assert result["delete_count"] == 1
|
||||
|
||||
def test_hnsw_raises_not_implemented(self):
|
||||
"""FAISSStore.delete_vectors on HNSW must propagate NotImplementedError."""
|
||||
faiss = pytest.importorskip("faiss")
|
||||
store = FAISSStore(dimension=4)
|
||||
store.create_index(index_type="hnsw", metric="L2")
|
||||
vecs = np.random.rand(5, 4).astype(np.float32)
|
||||
store.add_vectors(vecs, ids=["a", "b", "c", "d", "e"])
|
||||
with pytest.raises(NotImplementedError):
|
||||
store.delete_vectors(["a"])
|
||||
# Count must be unchanged
|
||||
assert store.count() == 5
|
||||
|
||||
def test_ivf_raises_not_implemented(self):
|
||||
"""FAISSStore.delete_vectors on IVF must raise NotImplementedError.
|
||||
|
||||
IVF remove_ids preserves original labels rather than compacting them,
|
||||
which would desynchronize search labels from vector_ids.
|
||||
"""
|
||||
faiss = pytest.importorskip("faiss")
|
||||
store = FAISSStore(dimension=4)
|
||||
# nlist=2 so we only need >= 2*39 = 78 training points
|
||||
store.create_index(index_type="ivf", metric="L2", nlist=2)
|
||||
train = np.random.rand(80, 4).astype(np.float32)
|
||||
store.index.index.train(train)
|
||||
store.add_vectors(train[:5], ids=["a", "b", "c", "d", "e"])
|
||||
with pytest.raises(NotImplementedError):
|
||||
store.delete_vectors(["a"])
|
||||
# State must be completely unchanged
|
||||
assert store.count() == 5
|
||||
|
||||
def test_delete_with_loaded_index_auto_saves(self, tmp_path):
|
||||
"""Deletion on a store loaded from disk auto-saves without explicit save_index."""
|
||||
_ = pytest.importorskip("faiss")
|
||||
# Create, populate, save
|
||||
store = _populated_store(ids=["a", "b", "c"])
|
||||
path = tmp_path / "store.faiss"
|
||||
store.save_index(path)
|
||||
|
||||
# Load into a fresh store and delete
|
||||
loaded = FAISSStore(dimension=3)
|
||||
loaded.load_index(path)
|
||||
loaded.delete_vectors(["b"])
|
||||
|
||||
# Reload without any additional save call — deletion must have persisted
|
||||
reloaded = FAISSStore(dimension=3)
|
||||
reloaded.load_index(path)
|
||||
assert reloaded.count() == 2
|
||||
assert "b" not in reloaded.index.vector_ids
|
||||
|
||||
def test_default_id_no_collision_after_deletion(self):
|
||||
"""Default vec_N IDs must not reuse a surviving ID after deletion."""
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(ids=["vec_0", "vec_1", "vec_2"])
|
||||
# Delete the middle one; len(vector_ids) drops to 2
|
||||
store.delete_vectors(["vec_1"])
|
||||
assert store.count() == 2
|
||||
|
||||
# Add a new vector — without the monotonic counter, the default ID
|
||||
# would be vec_2 which already exists and would be silently skipped.
|
||||
new_vecs = np.random.rand(1, 3).astype(np.float32)
|
||||
returned_ids = store.add_vectors(new_vecs)
|
||||
# The returned ID must not be an existing one
|
||||
assert returned_ids[0] not in {"vec_0", "vec_2"}, (
|
||||
f"Default ID {returned_ids[0]} collides with a surviving ID"
|
||||
)
|
||||
# And the vector must actually have been inserted
|
||||
assert store.count() == 3
|
||||
|
||||
def test_default_id_skip_past_explicit_id(self):
|
||||
"""Blocker: default IDs must skip over explicit IDs already in the store.
|
||||
|
||||
If a user inserts an explicit ``"vec_N"`` and then adds two vectors
|
||||
without IDs, the generator must skip ``"vec_N"`` rather than
|
||||
producing it and losing the second vector silently.
|
||||
"""
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = FAISSStore(dimension=3)
|
||||
|
||||
# Explicit vec_1 first
|
||||
store.add_vectors(np.ones((1, 3), dtype=np.float32), ids=["vec_1"])
|
||||
|
||||
# 2 default vectors — one would collide with vec_1 if not skipped
|
||||
store.add_vectors(np.ones((2, 3), dtype=np.float32))
|
||||
|
||||
# 1 more default vector — must get a fresh ID, not re-generate a used one
|
||||
original_meta = {vid: {"original": vid} for vid in store.index.vector_ids}
|
||||
for vid, m in original_meta.items():
|
||||
store.index.metadata[vid] = m
|
||||
count_before = store.count()
|
||||
|
||||
ret = store.add_vectors(
|
||||
np.ones((1, 3), dtype=np.float32), metadata=[{"new": True}]
|
||||
)
|
||||
new_id = ret[0]
|
||||
|
||||
assert store.count() == count_before + 1, (
|
||||
f"Vector was silently skipped; count stayed {store.count()}"
|
||||
)
|
||||
assert new_id not in original_meta, (
|
||||
f"Generated ID {new_id!r} collides with an already-existing ID"
|
||||
)
|
||||
# Surviving IDs' metadata must not be overwritten
|
||||
for vid, m in original_meta.items():
|
||||
assert store.index.metadata.get(vid) == m, (
|
||||
f"Metadata for surviving {vid!r} was overwritten"
|
||||
)
|
||||
|
||||
def test_stale_persisted_next_id_is_clamped_to_inferred_minimum(self, tmp_path):
|
||||
"""Regression: a stale ``next_id`` in the sidecar must be clamped to
|
||||
at least ``max(vec_N)+1`` so that auto-save after deletion cannot
|
||||
propagate the stale value and cause future ID collisions.
|
||||
"""
|
||||
import json as _json
|
||||
_ = pytest.importorskip("faiss")
|
||||
rng = np.random.default_rng(seed=3)
|
||||
store = FAISSStore(dimension=3)
|
||||
store.add_vectors(rng.random((5, 3)).astype(np.float32))
|
||||
# IDs are vec_0..vec_4, next_id=5
|
||||
path = tmp_path / "s.faiss"
|
||||
store.save_index(path)
|
||||
|
||||
# Corrupt the sidecar: set next_id to a stale low value
|
||||
meta = _json.loads((tmp_path / "s.faiss.meta.json").read_text())
|
||||
meta["next_id"] = 2 # stale — vec_2, vec_3, vec_4 still exist
|
||||
(tmp_path / "s.faiss.meta.json").write_text(_json.dumps(meta))
|
||||
|
||||
# Load and immediately delete one vector (auto-save fires)
|
||||
s2 = FAISSStore(dimension=3)
|
||||
s2.load_index(path)
|
||||
assert s2._next_id == 5, f"Stale next_id should be clamped to 5, got {s2._next_id}"
|
||||
s2.delete_vectors(["vec_3"]) # triggers auto-save
|
||||
|
||||
# The sidecar must not carry the stale value forward
|
||||
persisted = _json.loads((tmp_path / "s.faiss.meta.json").read_text())
|
||||
assert persisted["next_id"] >= 5, (
|
||||
f"Auto-save propagated stale next_id={persisted['next_id']} (expected >= 5)"
|
||||
)
|
||||
|
||||
def test_search_does_not_return_phantom_id_when_k_exceeds_ntotal(self):
|
||||
"""Regression: when k > ntotal, FAISS returns -1 sentinel values.
|
||||
``-1 < len(vector_ids)`` is always True in Python, so without an
|
||||
explicit non-negative guard ``-1`` maps to ``vector_ids[-1]``,
|
||||
making the last vector appear as a spurious extra result.
|
||||
"""
|
||||
faiss = pytest.importorskip("faiss")
|
||||
store = FAISSStore(dimension=3)
|
||||
vecs = np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]], dtype=np.float32)
|
||||
store.add_vectors(vecs, ids=["only_a", "only_b"])
|
||||
|
||||
# Ask for 10 neighbors but only 2 exist
|
||||
results = store.search_similar(
|
||||
np.array([0.0, 1.0, 0.0], dtype=np.float32), k=10
|
||||
)
|
||||
returned_ids = [r["id"] for r in results]
|
||||
assert len(results) == 2, (
|
||||
f"Expected exactly 2 results, got {len(results)}: {returned_ids}"
|
||||
)
|
||||
assert returned_ids.count("only_b") == 1, (
|
||||
f"only_b appears {returned_ids.count('only_b')} time(s) — "
|
||||
"sentinel -1 is mapping to vector_ids[-1]"
|
||||
)
|
||||
|
||||
def test_next_id_persisted_across_delete_save_reload(self, tmp_path):
|
||||
"""Regression test for critical bug: delete → auto-save → reload → add.
|
||||
|
||||
Without persisting ``next_id`` in the sidecar, ``load_index`` would
|
||||
set ``_next_id = ntotal`` (4 after one deletion from 5 vectors), which
|
||||
would generate ``"vec_4"`` as the next default ID. That ID is still
|
||||
present in the surviving vector list, so the insertion would be
|
||||
silently skipped, the count would not increase, and the old vector's
|
||||
metadata would be overwritten by the new metadata.
|
||||
|
||||
This test pins the full lifecycle so any regression is caught
|
||||
immediately.
|
||||
"""
|
||||
_ = pytest.importorskip("faiss")
|
||||
rng = np.random.default_rng(seed=7)
|
||||
dim = 4
|
||||
|
||||
# Step 1: create vec_0 .. vec_4, record their embeddings
|
||||
store1 = FAISSStore(dimension=dim)
|
||||
vecs = rng.random((5, dim)).astype(np.float32)
|
||||
store1.add_vectors(vecs)
|
||||
for vid in store1.index.vector_ids:
|
||||
store1.index.metadata[vid] = {"original": vid}
|
||||
path = tmp_path / "idx.faiss"
|
||||
store1.save_index(path)
|
||||
|
||||
# Step 2: reload → delete vec_2 (auto-saves) → reload again
|
||||
store2 = FAISSStore(dimension=dim)
|
||||
store2.load_index(path)
|
||||
store2.delete_vectors(["vec_2"]) # ntotal drops to 4; auto-save triggered
|
||||
|
||||
store3 = FAISSStore(dimension=dim)
|
||||
store3.load_index(path)
|
||||
|
||||
# Step 3: add a new vector without an explicit ID
|
||||
new_vec = rng.random((1, dim)).astype(np.float32)
|
||||
count_before = store3.count()
|
||||
returned_ids = store3.add_vectors(new_vec, metadata=[{"new": True}])
|
||||
|
||||
# The generated ID must not collide with any surviving ID
|
||||
surviving = set(store3.index.vector_ids[:count_before])
|
||||
new_id = returned_ids[0]
|
||||
assert new_id not in surviving, (
|
||||
f"Generated ID {new_id!r} collides with surviving ID "
|
||||
f"(surviving={sorted(surviving)})"
|
||||
)
|
||||
|
||||
# The new vector must actually have been inserted
|
||||
assert store3.count() == count_before + 1, (
|
||||
f"Count did not increase: was {count_before}, still {store3.count()}"
|
||||
)
|
||||
|
||||
# The new vector must be retrievable
|
||||
assert store3.get_vector(new_id) is not None, (
|
||||
f"New vector with ID {new_id!r} is not retrievable"
|
||||
)
|
||||
|
||||
# The surviving vec_4's embedding must be unchanged
|
||||
original_vec4 = vecs[4]
|
||||
loaded_vec4 = store3.get_vector("vec_4")
|
||||
assert loaded_vec4 is not None
|
||||
np.testing.assert_allclose(loaded_vec4, original_vec4, atol=1e-5,
|
||||
err_msg="vec_4 embedding was corrupted by the new add")
|
||||
|
||||
# The surviving vec_4's metadata must be unchanged
|
||||
assert store3.index.metadata.get("vec_4") == {"original": "vec_4"}, (
|
||||
f"vec_4 metadata was overwritten: {store3.index.metadata.get('vec_4')}"
|
||||
)
|
||||
|
||||
# The new vector's metadata must be the new value
|
||||
assert store3.index.metadata.get(new_id) == {"new": True}
|
||||
|
||||
def test_no_op_delete_does_not_rewrite_disk(self, tmp_path):
|
||||
"""A deletion of only nonexistent IDs must not call FAISSIndex.save().
|
||||
|
||||
Uses a spy on ``FAISSIndex.save`` rather than filesystem mtime so the
|
||||
assertion is deterministic regardless of filesystem timestamp resolution.
|
||||
"""
|
||||
from unittest.mock import patch
|
||||
_ = pytest.importorskip("faiss")
|
||||
store = _populated_store(ids=["a", "b", "c"])
|
||||
path = tmp_path / "idx.faiss"
|
||||
store.save_index(path)
|
||||
|
||||
loaded = FAISSStore(dimension=3)
|
||||
loaded.load_index(path)
|
||||
|
||||
with patch.object(loaded.index, "save", wraps=loaded.index.save) as mock_save:
|
||||
loaded.delete_vectors(["z"]) # nonexistent → delete_count 0
|
||||
loaded.delete_vectors([]) # empty list → delete_count 0
|
||||
assert mock_save.call_count == 0, (
|
||||
f"save() called {mock_save.call_count} time(s) for a no-op deletion"
|
||||
)
|
||||
|
||||
# A real deletion must still trigger save()
|
||||
loaded.delete_vectors(["b"])
|
||||
assert mock_save.call_count == 1, (
|
||||
f"save() was not called after a real deletion (calls={mock_save.call_count})"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Facade delegation test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFAISSFacadeDelegation:
|
||||
def test_vector_store_facade_delegates_to_faiss_store(self):
|
||||
"""VectorStore(backend='faiss').delete_vectors() must call FAISSStore."""
|
||||
_ = pytest.importorskip("faiss")
|
||||
vs = VectorStore(backend="faiss", config={"dimension": 3})
|
||||
vecs = np.eye(3, dtype=np.float32)
|
||||
vs.store_vectors(list(vecs), metadata=[{}, {}, {}])
|
||||
# Count before
|
||||
assert vs._backend_store.count() == 3
|
||||
|
||||
result = vs.delete_vectors(["vec_0"])
|
||||
|
||||
assert result == {"delete_count": 1}
|
||||
assert vs._backend_store.count() == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ErasureCoordinator integration tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestFAISSErasureCoordinator:
|
||||
def _faiss_vector_store(self, dim: int = 3) -> VectorStore:
|
||||
_ = pytest.importorskip("faiss")
|
||||
vs = VectorStore(backend="faiss", config={"dimension": dim})
|
||||
vecs = np.eye(dim, dtype=np.float32)
|
||||
vs.store_vectors(list(vecs), metadata=[{}, {}, {}])
|
||||
return vs
|
||||
|
||||
def test_erasure_reports_status_erased(self):
|
||||
vs = self._faiss_vector_store()
|
||||
# store_vectors assigns ids "vec_0", "vec_1", "vec_2"
|
||||
vector_ids = vs._backend_store.index.vector_ids
|
||||
coord = ErasureCoordinator(vector_store=vs)
|
||||
receipt = coord.erase_entity(vector_ids[0], vector_ids=[vector_ids[0]])
|
||||
assert receipt.stores["vectors"]["status"] == STATUS_ERASED
|
||||
|
||||
def test_erasure_backend_name_is_faiss(self):
|
||||
vs = self._faiss_vector_store()
|
||||
coord = ErasureCoordinator(vector_store=vs)
|
||||
receipt = coord.erase_entity("vec_0", vector_ids=["vec_0"])
|
||||
assert receipt.stores["vectors"]["backend"] == "faiss"
|
||||
|
||||
def test_erasure_receipt_is_complete_after_deletion(self):
|
||||
vs = self._faiss_vector_store()
|
||||
coord = ErasureCoordinator(vector_store=vs)
|
||||
receipt = coord.erase_entity("vec_0", vector_ids=["vec_0"])
|
||||
assert receipt.complete
|
||||
|
||||
def test_erasure_hnsw_reports_unsupported(self):
|
||||
"""HNSW deletion raises NotImplementedError; coordinator must report unsupported."""
|
||||
faiss = pytest.importorskip("faiss")
|
||||
vs = VectorStore(backend="faiss", config={"dimension": 4})
|
||||
vs._backend_store.create_index(index_type="hnsw", metric="L2")
|
||||
vecs = np.random.rand(5, 4).astype(np.float32)
|
||||
vs._backend_store.add_vectors(vecs, ids=["a", "b", "c", "d", "e"])
|
||||
coord = ErasureCoordinator(vector_store=vs)
|
||||
receipt = coord.erase_entity("a", vector_ids=["a"])
|
||||
assert receipt.stores["vectors"]["status"] == STATUS_UNSUPPORTED
|
||||
assert not receipt.complete
|
||||
|
||||
def test_erasure_ivf_reports_unsupported(self):
|
||||
"""IVF deletion raises NotImplementedError; coordinator must report unsupported."""
|
||||
faiss = pytest.importorskip("faiss")
|
||||
dim = 4
|
||||
vs = VectorStore(backend="faiss", config={"dimension": dim})
|
||||
vs._backend_store.create_index(index_type="ivf", metric="L2", nlist=2)
|
||||
train = np.random.rand(80, dim).astype(np.float32)
|
||||
vs._backend_store.index.index.train(train)
|
||||
vs._backend_store.add_vectors(train[:5], ids=["a", "b", "c", "d", "e"])
|
||||
coord = ErasureCoordinator(vector_store=vs)
|
||||
receipt = coord.erase_entity("a", vector_ids=["a"])
|
||||
assert receipt.stores["vectors"]["status"] == STATUS_UNSUPPORTED
|
||||
assert not receipt.complete
|
||||
Reference in New Issue
Block a user