mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-10 04:00:35 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Sameer Kadam
|
d79695cd42 |
+2694
-256
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+18
-26
@@ -97,10 +97,24 @@ jobs:
|
||||
- name: Build Explorer frontend
|
||||
working-directory: explorer
|
||||
run: npm run build
|
||||
- name: Install core package and base dependencies
|
||||
- name: Install Explorer backend test dependencies
|
||||
run: |
|
||||
# Verify that core semantica installs cleanly with only its base dependencies
|
||||
# (no optional extras) and that core imports and lazy missing-dependency hints work.
|
||||
# Run the deterministic backend path before the all-extras CI
|
||||
# environment is installed. The Explorer extra supplies the
|
||||
# production API dependencies without importing optional vector
|
||||
# providers such as Pinecone during test collection.
|
||||
#
|
||||
# --no-deps + a separate hash-pinned install (rather than the old
|
||||
# `pip install -e ".[explorer]" pytest==9.1.1`) so every fetched
|
||||
# package is hash-verified (Scorecard Pinned-Dependencies); the
|
||||
# local editable install itself has nothing to hash.
|
||||
# .github/requirements/explorer-extra-py311.txt is
|
||||
# `uv pip compile pyproject.toml --extra explorer --python-version 3.11 --constraint requirements-ci.txt --generate-hashes`
|
||||
# - regenerate it the same way if pyproject.toml's base/explorer
|
||||
# deps change. Resolved specifically for this job's python 3.11
|
||||
# (see the Dockerfile's explorer-extra-py313.txt for why this
|
||||
# can't be shared with python 3.13: audioread needs extra
|
||||
# standard-aifc/standard-sunau hashes only on 3.13+).
|
||||
#
|
||||
# --no-deps only skips *runtime* dependency resolution - `-e .`
|
||||
# still does a PEP 517 build, which by default creates an isolated
|
||||
@@ -111,30 +125,8 @@ jobs:
|
||||
# copies instead of fetching its own.
|
||||
pip install -r .github/requirements/pep517-build.txt --require-hashes
|
||||
pip install --no-deps --no-build-isolation -e .
|
||||
pip install -r .github/requirements/base-deps.txt --require-hashes
|
||||
pip install -r .github/requirements/pytest-tool.txt --require-hashes
|
||||
- name: Verify core-only package importability and slim behavior
|
||||
run: |
|
||||
python -c "
|
||||
import semantica
|
||||
print('semantica', semantica.__version__, 'core installed and importable')
|
||||
"
|
||||
pytest -q tests/test_issue_1513_slim_core.py
|
||||
- name: Install Explorer backend test dependencies
|
||||
run: |
|
||||
# Run the deterministic backend path before the all-extras CI
|
||||
# environment is installed. The Explorer extra supplies the
|
||||
# production API dependencies without importing optional vector
|
||||
# providers such as Pinecone during test collection.
|
||||
#
|
||||
# .github/requirements/explorer-extra-py311.txt is
|
||||
# `uv pip compile pyproject.toml --extra explorer --python-version 3.11 --constraint requirements-ci.txt --generate-hashes`
|
||||
# - regenerate it the same way if pyproject.toml's base/explorer
|
||||
# deps change. Resolved specifically for this job's python 3.11
|
||||
# (see the Dockerfile's explorer-extra-py313.txt for why this
|
||||
# can't be shared with python 3.13: audioread needs extra
|
||||
# standard-aifc/standard-sunau hashes only on 3.13+).
|
||||
pip install -r .github/requirements/explorer-extra-py311.txt --require-hashes
|
||||
pip install -r .github/requirements/pytest-tool.txt --require-hashes
|
||||
- name: Test deterministic Explorer backend path
|
||||
run: |
|
||||
pytest -q tests/explorer/test_explorer_deterministic_rendering_e2e.py
|
||||
|
||||
@@ -154,21 +154,13 @@ jobs:
|
||||
fi
|
||||
|
||||
# Vulnerability IDs reviewed and accepted as non-actionable for this
|
||||
# project:
|
||||
# - GHSA-4j2p-28q2-5m79 (aka CVE-2026-69112): accelerate<=1.14.0
|
||||
# (transitive via docling-slim). Path traversal in sharded checkpoint
|
||||
# index loading (load_checkpoint_in_model). 1.14.0 is the latest
|
||||
# available PyPI release; no upstream patch exists yet. Semantica does
|
||||
# not load arbitrary user checkpoints. Re-evaluate once accelerate
|
||||
# releases a fixed version.
|
||||
# NOTE: pip-audit's OSV-backed report may surface either identifier as
|
||||
# the primary `id` (with the other listed under `aliases`) depending on
|
||||
# which alias the backing database picks as canonical, so both need to
|
||||
# be listed here and the matching below checks aliases too - see
|
||||
# https://github.com/semantica-agi/semantica/actions/runs/34296586683
|
||||
# where this ignore list had only the GHSA id but the report's `id`
|
||||
# was the CVE, so the gate still failed.
|
||||
IGNORED_VULN_IDS="GHSA-4j2p-28q2-5m79,CVE-2026-69112"
|
||||
# project. Empty for now: pip-audit's OSV-backed database doesn't
|
||||
# currently carry either of the findings Safety used to flag here
|
||||
# (cuda-toolkit CVE-2025-33228, torchvision CVE-2026-65918), so
|
||||
# there's nothing to exclude. Left in place so a future finding can
|
||||
# be added the same way without restructuring this step - see git
|
||||
# history on this file for the reasoning behind past entries.
|
||||
IGNORED_VULN_IDS=""
|
||||
|
||||
# Exported so the "Comment PR with Security Results" step below can
|
||||
# apply the same exclusion list to the raw report - it reads
|
||||
@@ -184,16 +176,9 @@ jobs:
|
||||
# no vulns field at all (see the skip_reason handling above) -
|
||||
# without the fallback, iterating `null[]` raises inside jq and
|
||||
# this whole computation silently evaluates to empty.
|
||||
#
|
||||
# Matching checks `.id` AND `.aliases` (pip-audit includes aliases by
|
||||
# default for JSON output): the OSV-backed report can surface either
|
||||
# the GHSA or the CVE identifier as the canonical `id` for the same
|
||||
# advisory, with the other one demoted to an alias, so matching on
|
||||
# `.id` alone is not reliable.
|
||||
VULNS=$(jq --arg ignored "$IGNORED_VULN_IDS" '
|
||||
($ignored | split(",") | map(select(length > 0))) as $ignore_list
|
||||
| [.dependencies[] | (.vulns // [])[]
|
||||
| select(([.id] + (.aliases // [])) as $ids | ($ids - $ignore_list | length) == ($ids | length))]
|
||||
| [.dependencies[] | (.vulns // [])[] | select(.id as $id | ($ignore_list | index($id)) | not)]
|
||||
| length
|
||||
' pip-audit-report.json 2>/dev/null)
|
||||
|
||||
@@ -214,8 +199,7 @@ jobs:
|
||||
jq --arg ignored "$IGNORED_VULN_IDS" -r '
|
||||
($ignored | split(",") | map(select(length > 0))) as $ignore_list
|
||||
| .dependencies[] as $dependency
|
||||
| ($dependency.vulns // [])[]
|
||||
| select(([.id] + (.aliases // [])) as $ids | ($ids - $ignore_list | length) == ($ids | length))
|
||||
| ($dependency.vulns // [])[] | select(.id as $id | ($ignore_list | index($id)) | not)
|
||||
| "- \($dependency.name)==\($dependency.version): \(.id)"
|
||||
' pip-audit-report.json || true
|
||||
exit 1
|
||||
@@ -361,16 +345,9 @@ jobs:
|
||||
return null;
|
||||
}
|
||||
|
||||
// A vuln's canonical `id` and its `aliases` (e.g. GHSA vs. CVE
|
||||
// for the same advisory) are checked together - mirrors the
|
||||
// shell gate above, which needs the same fallback because
|
||||
// pip-audit's OSV-backed report doesn't consistently pick the
|
||||
// same identifier as canonical across advisories.
|
||||
return data.dependencies.flatMap((dependency) =>
|
||||
(dependency.vulns || [])
|
||||
.filter((vulnerability) =>
|
||||
![vulnerability.id, ...(vulnerability.aliases || [])].some((id) => ignoredVulnIds.includes(id))
|
||||
)
|
||||
.filter((vulnerability) => !ignoredVulnIds.includes(vulnerability.id))
|
||||
.map(
|
||||
(vulnerability) => `- \`${dependency.name}==${dependency.version}\`: ${vulnerability.id}` +
|
||||
(vulnerability.fix_versions?.length ? ` (fixed by ${vulnerability.fix_versions.join(', ')})` : '')
|
||||
|
||||
BIN
Binary file not shown.
@@ -9,41 +9,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- **Schema-guided extraction validation** (#1510) by @Besokus
|
||||
- New `SchemaValidator` (`semantica.semantic_extract`, lazy export): a deterministic sibling of `ExtractionValidator` that checks extraction output for *conformance to a domain ontology* — an axis orthogonal to `ExtractionValidator`'s confidence checks. It mirrors the same interface (`validate_entities()` / `validate_relations()` returning `ValidationResult`, batch-aware), so the two compose back-to-back
|
||||
- Entity labels must be concepts in the schema; relation predicates must be in the schema and satisfy their `domain` / `range`. Violations are reported in `ValidationResult.errors` with counts in `metrics` and `score` = conformance ratio; `filter_by_schema()` / `filter_relations_by_schema()` return the conforming subset (mirroring `filter_by_confidence`). No LLM required
|
||||
- New `ExtractionSchema` (`semantica.semantic_extract`, lazy export): a lightweight, read-only view over a domain ontology (allowed concepts + predicates with optional `domain` / `range`). Reuses the project's existing OWL ontology representation rather than a parallel type — build one from a `generate_ontology`-style dict (`ExtractionSchema.from_ontology`) or an OWL/Turtle file/string (`ExtractionSchema.from_owl`, via the existing `rdflib` dependency). An empty `domain`/`range` means unconstrained, matching OWL
|
||||
- Implements the deterministic core of ontology-based information extraction (OBIE; Wimalasuriya & Dou, 2010). No new runtime dependencies
|
||||
- New `tests/semantic_extract/test_schema_validator.py`
|
||||
|
||||
## [0.7.0] - 2026-09-07
|
||||
|
||||
### Changed
|
||||
|
||||
- **Slim core dependencies: moved ~22 heavy packages to optional extras** (#1513)
|
||||
- Core dependencies in `pyproject.toml` are now reduced to exactly 22 direct packages: `numpy`, `pandas`, `scipy`, `scikit-learn`, `rdflib`, `networkx`, `requests`, `chardet`, `protobuf`, `grpcio`, `pillow`, `pydantic`, `click`, `rich`, `tqdm`, `pyyaml`, `toml`, `python-dotenv`, `loguru`, `structlog`, `httpx`, and `pyarrow`.
|
||||
- Heavy ML/NLP, visualization, document parsing, and ingestion packages moved into granular optional extras:
|
||||
- `models-huggingface`: `torch`, `transformers`
|
||||
- `embeddings-local`: `sentence-transformers`, `fastembed`, `onnxruntime`, `tokenizers`
|
||||
- `nlp-spacy`: `spacy`
|
||||
- `viz`: expanded to include `matplotlib`, `seaborn`, `plotly`, `ipywidgets`, `umap-learn`, alongside `pyvis`, `graphviz`, and `d3blocks`
|
||||
- `media`: `librosa`, `opencv-python`
|
||||
- `vectorstore-faiss`: `faiss-cpu` (also included in `vectorstore-all`)
|
||||
- `documents`: `python-docx`, `openpyxl`, `lxml`, `beautifulsoup4`
|
||||
- `ingest-git`: `GitPython`
|
||||
- `graph-embeddings`: `gensim` (also included in `graph-all`)
|
||||
- Full bundled behavior preserved via `pip install "semantica[all]"`, which includes all optional extras. Pinning `semantica<0.7.0` remains a permanent escape hatch for legacy workflows.
|
||||
- Safe lazy construction across parsers and visualizers:
|
||||
- `DOCXParser`, `ExcelParser`, `HTMLParser`, and `XMLParser` remain constructible without error on `__init__()`. They fail only upon calling `.parse()` with actionable error messages directing users to install `semantica[documents]`.
|
||||
- `XMLParser` automatically falls back to standard library `xml.etree` (`_parse_with_etree`) when `lxml` is not installed, preserving XML parsing capabilities without extra dependencies.
|
||||
- `EmbeddingVisualizer` and `OntologyVisualizer` safely guard `matplotlib` and optional reduction packages, advising `pip install 'semantica[viz]'`.
|
||||
- `RepoIngestor` guards `GitPython` with a clear error pointing to `semantica[ingest-git]`.
|
||||
- `PublicAPIIngestor` guards `lxml` and `_SAFE_XML_PARSER`.
|
||||
- Updated user-facing installation hints across CLI doctor commands, node embeddings (`NodeEmbedder`), vector stores (`FAISSStore`), and model loaders.
|
||||
- Recompiled CI lockfiles (`requirements-ci.txt`, `.github/requirements/explorer-extra-py311.txt`, `.github/requirements/explorer-extra-py313.txt`, and `.github/requirements/base-deps.txt`).
|
||||
|
||||
## [0.6.8] - 2026-09-05
|
||||
|
||||
### Added
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ authors:
|
||||
repository-code: "https://github.com/semantica-agi/semantica"
|
||||
url: "https://getsemantica.ai"
|
||||
license: MIT
|
||||
version: 0.7.0
|
||||
version: 0.6.8
|
||||
date-released: 2026-09-05
|
||||
keywords:
|
||||
- knowledge-graph
|
||||
|
||||
+1
-1
@@ -31,7 +31,7 @@ RUN mkdir -p /app/semantica && npm run build
|
||||
# `pip index versions gensim` / the project's PyPI files page, not just
|
||||
# whether `uv pip compile` resolves (resolution only reads sdist metadata,
|
||||
# it doesn't attempt the build that fails here).
|
||||
FROM python:3.14-slim@sha256:cad9a2c871761c413caa6fdd6441c783451e740a48aaeba60ae62a8b53525ef6 AS runtime
|
||||
FROM python:3.13-slim@sha256:7ce4b6dfe35e55397b7cda544f8a13f191b7ae28dc5aad71fe664dbc9bc2623f AS runtime
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
|
||||
@@ -1480,14 +1480,6 @@ app = create_app(session=GraphSession(graph), agent_memory=memory)
|
||||
The Memories workspace is shown only when `agent_memory` is provided. Apply
|
||||
updates the supplied runtime object; it does not add disk persistence.
|
||||
|
||||
## What's New in v0.7.0
|
||||
|
||||
**Slim core dependencies: lightweight base install with granular optional extras** — `pip install semantica` now installs only 22 essential core dependencies, moving heavy packages into dedicated optional extras:
|
||||
- **Dramatically lighter and faster installation**: Core installation no longer pulls heavy machine learning or visualization packages by default.
|
||||
- **Granular extras**: Install only what your workload requires (`documents`, `embeddings-local`, `models-huggingface`, `nlp-spacy`, `viz`, `media`, `vectorstore-faiss`, `graph-embeddings`, `ingest-git`).
|
||||
- **Full backward compatibility**: `pip install "semantica[all]"` preserves the full bundled suite, while `semantica<0.7.0` remains a permanent escape hatch.
|
||||
- **Lazy parser construction & graceful fallbacks**: Document parsers can be constructed without extras and only raise actionable error hints upon calling `.parse()`; `XMLParser` automatically falls back to Python's standard library `xml.etree`.
|
||||
|
||||
---
|
||||
|
||||
## What's New in v0.6.8
|
||||
@@ -1526,42 +1518,32 @@ Semantica is designed for environments where AI outputs must be explainable, aud
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install semantica # lightweight core (22 essential dependencies)
|
||||
pip install "semantica[all]" # full bundled behavior with all extras
|
||||
pip install semantica # core
|
||||
pip install semantica[all] # everything
|
||||
```
|
||||
|
||||
> **Note for upgrades from <0.7.0**: In Semantica 0.7.0+, heavy machine learning, NLP, visualization, and document dependencies were moved into optional extras to make core installation significantly lighter and faster. If you want the previous bundled installation, install with `pip install "semantica[all]"` or pin `semantica<0.7.0`.
|
||||
|
||||
```bash
|
||||
# Granular Extras
|
||||
pip install "semantica[documents]" # Document parsing (docx, openpyxl, lxml, beautifulsoup4)
|
||||
pip install "semantica[embeddings-local]" # Local embeddings (sentence-transformers, fastembed, onnxruntime)
|
||||
pip install "semantica[models-huggingface]" # HuggingFace models (transformers, torch)
|
||||
pip install "semantica[nlp-spacy]" # spaCy NLP pipelines (spacy)
|
||||
pip install "semantica[viz]" # Visualization (matplotlib, seaborn, plotly, pyvis, graphviz)
|
||||
pip install "semantica[media]" # Audio & computer vision (librosa, opencv-python)
|
||||
pip install "semantica[graph-embeddings]" # Knowledge graph embeddings (gensim / Node2Vec)
|
||||
pip install "semantica[ingest-git]" # Git repository ingestor (GitPython)
|
||||
pip install "semantica[vectorstore-faiss]" # FAISS vector store
|
||||
pip install "semantica[vectorstore-all]" # All vector stores (Qdrant, Pinecone, Weaviate, FAISS, PgVector, SQLite)
|
||||
pip install "semantica[agno]" # Agno multi-agent integration
|
||||
pip install "semantica[crewai]" # CrewAI integration
|
||||
pip install "semantica[langchain]" # LangChain / LangGraph integration
|
||||
pip install "semantica[llm-all]" # All LLM provider clients
|
||||
pip install "semantica[graph-neo4j]" # Neo4j graph store (LPG)
|
||||
pip install "semantica[graph-falkordb]" # FalkorDB graph store (LPG)
|
||||
pip install "semantica[graph-apache-age]" # Apache AGE graph store (LPG)
|
||||
pip install "semantica[graph-amazon-neptune]" # AWS Neptune graph store (LPG)
|
||||
pip install "semantica[tripletstore-oxigraph]" # Embedded in-memory/on-disk RDF store
|
||||
pip install semantica[agno] # Agno multi-agent integration
|
||||
pip install semantica[crewai] # CrewAI integration
|
||||
pip install semantica[langchain] # LangChain / LangGraph integration
|
||||
pip install semantica[llm-litellm] # OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Bedrock, Ollama, DeepSeek, and more
|
||||
pip install semantica[graph-neo4j] # Neo4j graph store (LPG)
|
||||
pip install semantica[graph-falkordb] # FalkorDB graph store (LPG)
|
||||
pip install semantica[graph-apache-age] # Apache AGE graph store (LPG)
|
||||
pip install semantica[graph-amazon-neptune] # AWS Neptune graph store (LPG)
|
||||
pip install semantica[tripletstore-oxigraph] # Embedded in-memory/on-disk RDF store
|
||||
# RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J) need no extra:
|
||||
# semantica.triplet_store talks SPARQL over HTTP using the core `requests` dependency
|
||||
pip install "semantica[db-snowflake]" # Snowflake
|
||||
pip install "semantica[db-databricks]" # Databricks (SDK + SQL connector)
|
||||
pip install "semantica[ingest-sap]" # SAP OData
|
||||
pip install "semantica[ingest-parquet]" # Parquet / PyArrow
|
||||
pip install "semantica[ingest-arrow]" # Apache Arrow, Feather, IPC
|
||||
pip install "semantica[watch]" # Directory file watcher
|
||||
pip install "semantica[explorer]" # Knowledge Explorer dashboard
|
||||
pip install semantica[vectorstore-qdrant] # Qdrant vector store
|
||||
pip install semantica[vectorstore-pinecone] # Pinecone vector store
|
||||
pip install semantica[db-snowflake] # Snowflake
|
||||
pip install semantica[db-databricks] # Databricks (SDK + SQL connector)
|
||||
pip install semantica[ingest-sap] # SAP OData
|
||||
pip install semantica[ingest-parquet] # Parquet / PyArrow
|
||||
pip install semantica[ingest-arrow] # Apache Arrow, Feather, IPC
|
||||
pip install semantica[viz] # HTML interactive visualization
|
||||
pip install semantica[watch] # Directory file watcher
|
||||
pip install semantica[explorer] # Knowledge Explorer dashboard
|
||||
```
|
||||
|
||||
For production deployments, use Docker or Kubernetes rather than a local `pip install`. Set `SEMANTICA_API_KEY`, configure a persistent LPG graph store (Neo4j / FalkorDB / Apache AGE / AWS Neptune) and/or RDF triple store (Blazegraph / Apache Jena / Eclipse RDF4J), and point the vector store at a hosted backend (Qdrant / Pinecone). See [ARCHITECTURE.md](ARCHITECTURE.md) for the full deployment topology.
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview",
|
||||
"test:graph-store": "node --test tests/graphStore.multi-edge.test.mjs",
|
||||
"test:graph-workspace": "node --import tsx --test tests/markdownContentViewer.test.ts tests/markdownEditorInteraction.test.tsx tests/markdownEditorState.test.ts tests/nodeMarkdownSync.test.ts tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts tests/temporalScrubberBounds.test.ts tests/deterministicExplorerRendering.test.ts tests/explorerCapabilities.test.tsx tests/smallGraphLayout.test.ts tests/realtimeGraphAttributes.test.ts tests/ontologyEditorModel.test.ts tests/graphColorLegend.test.ts",
|
||||
"test:graph-workspace": "node --import tsx --test tests/markdownContentViewer.test.ts tests/markdownEditorInteraction.test.tsx tests/markdownEditorState.test.ts tests/nodeMarkdownSync.test.ts tests/graphSceneState.display.test.ts tests/temporalLifecycle.test.ts tests/deterministicExplorerRendering.test.ts tests/explorerCapabilities.test.tsx tests/smallGraphLayout.test.ts tests/realtimeGraphAttributes.test.ts tests/ontologyEditorModel.test.ts tests/graphColorLegend.test.ts",
|
||||
"test:deterministic-e2e": "node --import tsx --test tests/deterministicExplorerRendering.e2e.ts",
|
||||
"test:graph-legend-e2e": "node --import tsx --test tests/graphColorLegend.e2e.ts",
|
||||
"test:plugin-registry": "node --import tsx --test tests/pluginRegistry.temporal.test.mjs"
|
||||
|
||||
@@ -4,7 +4,6 @@ import { Timeline } from "vis-timeline";
|
||||
import type { TimelineOptions } from "vis-timeline";
|
||||
import "vis-timeline/styles/vis-timeline-graph2d.css";
|
||||
import { GRAPH_THEME } from "./graphTheme";
|
||||
import { DEFAULT_MIN_DATE, resolvePlayStepMs, resolveScrubberBounds } from "./temporalScrubberBounds";
|
||||
|
||||
export interface TimelinePanelProps {
|
||||
onTimeChange: (time: Date) => void;
|
||||
@@ -12,9 +11,11 @@ export interface TimelinePanelProps {
|
||||
maxDate?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_MIN_DATE = new Date("1970-01-01T00:00:00Z");
|
||||
const DEFAULT_MAX_DATE = new Date("2030-01-01T00:00:00Z");
|
||||
const PLAYHEAD_ID = "playhead";
|
||||
const PLAY_INTERVAL_MS = 500;
|
||||
const ONE_DAY_MS = 1000 * 60 * 60 * 24;
|
||||
const PLAY_STEP_MONTHS = 6;
|
||||
|
||||
const VIS_OVERRIDE_CSS = `
|
||||
.sem-timeline-wrap .vis-timeline { border: none !important; background: transparent !important; overflow: visible !important; }
|
||||
@@ -53,6 +54,12 @@ const VIS_OVERRIDE_CSS = `
|
||||
.sem-timeline-wrap .vis-panel.vis-left { display: none !important; }
|
||||
`;
|
||||
|
||||
function safeDate(value: string | undefined, fallback: Date): Date {
|
||||
if (!value) return fallback;
|
||||
const parsed = new Date(value);
|
||||
return Number.isNaN(parsed.getTime()) ? fallback : parsed;
|
||||
}
|
||||
|
||||
function formatPlayheadLabel(value: Date): string {
|
||||
return `${value.getFullYear()}/${String(value.getMonth() + 1).padStart(2, "0")}`;
|
||||
}
|
||||
@@ -65,13 +72,9 @@ export function TimelinePanel({ onTimeChange, minDate, maxDate }: TimelinePanelP
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [displayDate, setDisplayDate] = useState(formatPlayheadLabel(DEFAULT_MIN_DATE));
|
||||
|
||||
// Captured once per mount so re-renders keep the same reference and do not
|
||||
// retrigger the timeline effect below.
|
||||
const now = useMemo(() => new Date(), []);
|
||||
const { minBound, maxBound, defaultTime } = useMemo(
|
||||
() => resolveScrubberBounds({ minDate, maxDate, now }),
|
||||
[maxDate, minDate, now],
|
||||
);
|
||||
const minBound = useMemo(() => safeDate(minDate, DEFAULT_MIN_DATE), [minDate]);
|
||||
const maxBound = useMemo(() => safeDate(maxDate, DEFAULT_MAX_DATE), [maxDate]);
|
||||
const defaultTime = useMemo(() => new Date(Math.round((minBound.getTime() + maxBound.getTime()) / 2)), [maxBound, minBound]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!containerRef.current) return;
|
||||
@@ -88,10 +91,12 @@ export function TimelinePanel({ onTimeChange, minDate, maxDate }: TimelinePanelP
|
||||
showCurrentTime: false,
|
||||
zoomable: true,
|
||||
moveable: true,
|
||||
zoomMin: ONE_DAY_MS,
|
||||
zoomMin: 1000 * 60 * 60 * 24 * 365,
|
||||
zoomMax: 1000 * 60 * 60 * 24 * 365 * 80,
|
||||
showMajorLabels: true,
|
||||
showMinorLabels: true,
|
||||
timeAxis: { scale: "year", step: 5 },
|
||||
format: { minorLabels: { year: "YYYY" }, majorLabels: { year: "YYYY" } },
|
||||
orientation: { axis: "bottom" },
|
||||
margin: { item: 0, axis: 0 },
|
||||
selectable: false,
|
||||
@@ -129,7 +134,8 @@ export function TimelinePanel({ onTimeChange, minDate, maxDate }: TimelinePanelP
|
||||
playIntervalRef.current = setInterval(() => {
|
||||
const timeline = timelineRef.current;
|
||||
if (!timeline) return;
|
||||
const next = new Date(playheadRef.current.getTime() + resolvePlayStepMs(minBound, maxBound));
|
||||
const next = new Date(playheadRef.current);
|
||||
next.setMonth(next.getMonth() + PLAY_STEP_MONTHS);
|
||||
if (next >= maxBound) {
|
||||
next.setTime(minBound.getTime());
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ export const temporalOverlayPlugin: GraphPlugin = {
|
||||
<div style={detailRowStyle}>
|
||||
<span style={detailLabelStyle}>Bounds</span>
|
||||
<span style={detailValueStyle}>
|
||||
{(temporal?.minDate ?? "1970")} → {(temporal?.maxDate ?? "now")}
|
||||
{(temporal?.minDate ?? "1970")} → {(temporal?.maxDate ?? "2030")}
|
||||
</span>
|
||||
</div>
|
||||
<div style={detailRowStyle}>
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
export interface ScrubberBoundsInput {
|
||||
minDate?: string;
|
||||
maxDate?: string;
|
||||
now: Date;
|
||||
}
|
||||
|
||||
export interface ScrubberBounds {
|
||||
minBound: Date;
|
||||
maxBound: Date;
|
||||
defaultTime: Date;
|
||||
}
|
||||
|
||||
export const DEFAULT_MIN_DATE = new Date("1970-01-01T00:00:00Z");
|
||||
const ONE_DAY_MS = 1000 * 60 * 60 * 24;
|
||||
const PLAY_FRAMES = 60;
|
||||
|
||||
function parseBound(value: string | undefined, fallback: Date): Date {
|
||||
if (!value) return fallback;
|
||||
const parsed = new Date(value);
|
||||
return Number.isNaN(parsed.getTime()) ? fallback : parsed;
|
||||
}
|
||||
|
||||
function clamp(value: Date, minBound: Date, maxBound: Date): Date {
|
||||
if (value < minBound) return minBound;
|
||||
if (value > maxBound) return maxBound;
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* `/api/temporal/bounds` reports `max: null` for graphs whose nodes carry
|
||||
* `valid_from` instants and no `valid_until`, which is the common case rather
|
||||
* than malformed data. Such a graph is known up to the present and no further,
|
||||
* so `now` is the honest upper bound and the honest starting playhead.
|
||||
*/
|
||||
export function resolveScrubberBounds({ minDate, maxDate, now }: ScrubberBoundsInput): ScrubberBounds {
|
||||
const minBound = parseBound(minDate, DEFAULT_MIN_DATE);
|
||||
const maxBound = parseBound(maxDate, now);
|
||||
const orderedMax = maxBound > minBound ? maxBound : minBound;
|
||||
return { minBound, maxBound: orderedMax, defaultTime: clamp(now, minBound, orderedMax) };
|
||||
}
|
||||
|
||||
/** Keeps a play-through at ~PLAY_FRAMES steps whatever the span, with a one-day floor. */
|
||||
export function resolvePlayStepMs(minBound: Date, maxBound: Date): number {
|
||||
const span = maxBound.getTime() - minBound.getTime();
|
||||
return Math.max(ONE_DAY_MS, Math.round(span / PLAY_FRAMES));
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
DEFAULT_MIN_DATE,
|
||||
resolvePlayStepMs,
|
||||
resolveScrubberBounds,
|
||||
} from "../src/workspaces/GraphWorkspace/temporalScrubberBounds.ts";
|
||||
|
||||
const NOW = new Date("2026-09-09T10:30:00Z");
|
||||
|
||||
// ── resolveScrubberBounds ────────────────────────────────────────────────────
|
||||
|
||||
test("scrubber bounds: open max ends the window at now, not at a future year", () => {
|
||||
const { minBound, maxBound } = resolveScrubberBounds({ minDate: "2026-01-15T00:00:00Z", now: NOW });
|
||||
|
||||
assert.equal(minBound.toISOString(), "2026-01-15T00:00:00.000Z");
|
||||
assert.equal(
|
||||
maxBound.getTime(),
|
||||
NOW.getTime(),
|
||||
"a graph carrying only valid_from instants is known up to the present and no further",
|
||||
);
|
||||
});
|
||||
|
||||
test("scrubber bounds: playhead starts at now so the first snapshot describes the present", () => {
|
||||
const { defaultTime } = resolveScrubberBounds({ minDate: "2026-01-15T00:00:00Z", now: NOW });
|
||||
|
||||
assert.equal(defaultTime.getTime(), NOW.getTime());
|
||||
});
|
||||
|
||||
test("scrubber bounds: playhead is not the midpoint of the range", () => {
|
||||
const { minBound, maxBound, defaultTime } = resolveScrubberBounds({
|
||||
minDate: "2020-01-01T00:00:00Z",
|
||||
maxDate: "2030-01-01T00:00:00Z",
|
||||
now: NOW,
|
||||
});
|
||||
const midpoint = Math.round((minBound.getTime() + maxBound.getTime()) / 2);
|
||||
|
||||
assert.notEqual(defaultTime.getTime(), midpoint, "the midpoint was the source of the future start time");
|
||||
assert.equal(defaultTime.getTime(), NOW.getTime());
|
||||
});
|
||||
|
||||
test("scrubber bounds: reported max is honoured when the data supplies one", () => {
|
||||
const { maxBound } = resolveScrubberBounds({
|
||||
minDate: "2020-01-01T00:00:00Z",
|
||||
maxDate: "2030-06-01T00:00:00Z",
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
assert.equal(maxBound.toISOString(), "2030-06-01T00:00:00.000Z");
|
||||
});
|
||||
|
||||
test("scrubber bounds: playhead clamps into a range that ends before now", () => {
|
||||
const { maxBound, defaultTime } = resolveScrubberBounds({
|
||||
minDate: "2019-01-01T00:00:00Z",
|
||||
maxDate: "2020-01-01T00:00:00Z",
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
assert.equal(defaultTime.getTime(), maxBound.getTime());
|
||||
});
|
||||
|
||||
test("scrubber bounds: playhead clamps into a range that starts after now", () => {
|
||||
const { minBound, defaultTime } = resolveScrubberBounds({
|
||||
minDate: "2030-01-01T00:00:00Z",
|
||||
maxDate: "2031-01-01T00:00:00Z",
|
||||
now: NOW,
|
||||
});
|
||||
|
||||
assert.equal(defaultTime.getTime(), minBound.getTime());
|
||||
});
|
||||
|
||||
test("scrubber bounds: min ahead of an open max keeps the window ordered", () => {
|
||||
const { minBound, maxBound } = resolveScrubberBounds({ minDate: "2031-01-01T00:00:00Z", now: NOW });
|
||||
|
||||
assert.ok(maxBound >= minBound, "vis-timeline requires min <= max");
|
||||
});
|
||||
|
||||
test("scrubber bounds: malformed and missing dates fall back without producing Invalid Date", () => {
|
||||
const { minBound, maxBound } = resolveScrubberBounds({ minDate: "not-a-date", maxDate: "also-bad", now: NOW });
|
||||
|
||||
assert.equal(minBound.getTime(), DEFAULT_MIN_DATE.getTime());
|
||||
assert.equal(maxBound.getTime(), NOW.getTime());
|
||||
});
|
||||
|
||||
// ── resolvePlayStepMs ────────────────────────────────────────────────────────
|
||||
|
||||
test("play step: a one-year span advances in ~60 frames, not 2", () => {
|
||||
const minBound = new Date("2026-01-01T00:00:00Z");
|
||||
const maxBound = new Date("2027-01-01T00:00:00Z");
|
||||
const span = maxBound.getTime() - minBound.getTime();
|
||||
|
||||
const frames = span / resolvePlayStepMs(minBound, maxBound);
|
||||
|
||||
assert.ok(frames > 50 && frames < 70, `expected ~60 frames, got ${frames}`);
|
||||
});
|
||||
|
||||
test("play step: a decade-long span also advances in ~60 frames", () => {
|
||||
const minBound = new Date("2016-01-01T00:00:00Z");
|
||||
const maxBound = new Date("2026-01-01T00:00:00Z");
|
||||
const span = maxBound.getTime() - minBound.getTime();
|
||||
|
||||
const frames = span / resolvePlayStepMs(minBound, maxBound);
|
||||
|
||||
assert.ok(frames > 50 && frames < 70, `expected ~60 frames, got ${frames}`);
|
||||
});
|
||||
|
||||
test("play step: a span of hours still advances by at least a day", () => {
|
||||
const minBound = new Date("2026-09-09T00:00:00Z");
|
||||
const maxBound = new Date("2026-09-09T06:00:00Z");
|
||||
|
||||
assert.equal(resolvePlayStepMs(minBound, maxBound), 1000 * 60 * 60 * 24);
|
||||
});
|
||||
@@ -21,7 +21,6 @@ from typing import Any, List, Optional
|
||||
try:
|
||||
from google.adk.events import Event
|
||||
from google.adk.sessions import BaseSessionService, Session
|
||||
|
||||
try:
|
||||
from google.adk.sessions import ListSessionsResponse
|
||||
except ImportError:
|
||||
@@ -34,7 +33,7 @@ try:
|
||||
except ImportError:
|
||||
from google.adk.sessions.base_session_service import GetSessionConfig
|
||||
ADK_AVAILABLE = True
|
||||
except (ImportError, OSError):
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
ADK_AVAILABLE = False
|
||||
BaseSessionService = object
|
||||
Session = Any
|
||||
@@ -163,10 +162,10 @@ class SemanticaSessionService(BaseSessionService):
|
||||
return {}
|
||||
|
||||
def _find_session_node(
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
) -> Optional[Any]:
|
||||
"""Find a session node by its logical ADK session ID."""
|
||||
expected_node_id = self._node_id(app_name, user_id, session_id)
|
||||
@@ -183,18 +182,18 @@ class SemanticaSessionService(BaseSessionService):
|
||||
metadata = node.get("metadata")
|
||||
|
||||
if (
|
||||
isinstance(metadata, dict)
|
||||
and str(metadata.get("session_id")) == str(session_id)
|
||||
and str(metadata.get("app_name")) == str(app_name)
|
||||
and str(metadata.get("user_id")) == str(user_id)
|
||||
isinstance(metadata, dict)
|
||||
and str(metadata.get("session_id")) == str(session_id)
|
||||
and str(metadata.get("app_name")) == str(app_name)
|
||||
and str(metadata.get("user_id")) == str(user_id)
|
||||
):
|
||||
return node
|
||||
|
||||
return None
|
||||
|
||||
def _find_node_by_id(
|
||||
self,
|
||||
node_id: str,
|
||||
self,
|
||||
node_id: str,
|
||||
) -> Optional[Any]:
|
||||
"""Find a ContextGraph node by graph node ID."""
|
||||
for node in self.graph.find_nodes() or []:
|
||||
@@ -213,13 +212,13 @@ class SemanticaSessionService(BaseSessionService):
|
||||
data = SemanticaSessionService._safe_dict(event)
|
||||
|
||||
for field in (
|
||||
"id",
|
||||
"invocation_id",
|
||||
"author",
|
||||
"timestamp",
|
||||
"partial",
|
||||
"turn_complete",
|
||||
"branch",
|
||||
"id",
|
||||
"invocation_id",
|
||||
"author",
|
||||
"timestamp",
|
||||
"partial",
|
||||
"turn_complete",
|
||||
"branch",
|
||||
):
|
||||
if field not in data and hasattr(event, field):
|
||||
value = getattr(event, field)
|
||||
@@ -232,10 +231,10 @@ class SemanticaSessionService(BaseSessionService):
|
||||
return data
|
||||
|
||||
def _event_nodes(
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
) -> List[Any]:
|
||||
"""Return all event nodes connected to a session."""
|
||||
session_node_id = self._node_id(app_name, user_id, session_id)
|
||||
@@ -276,8 +275,8 @@ class SemanticaSessionService(BaseSessionService):
|
||||
return str(timestamp)
|
||||
|
||||
def _event_from_node(
|
||||
self,
|
||||
node: Any,
|
||||
self,
|
||||
node: Any,
|
||||
) -> Any:
|
||||
"""
|
||||
Reconstruct an ADK Event from its stored metadata.
|
||||
@@ -290,7 +289,7 @@ class SemanticaSessionService(BaseSessionService):
|
||||
if not event_id and graph_node_id:
|
||||
graph_node_id = str(graph_node_id)
|
||||
if graph_node_id.startswith("adk-event:"):
|
||||
event_id = graph_node_id[len("adk-event:") :]
|
||||
event_id = graph_node_id[len("adk-event:"):]
|
||||
|
||||
if event_id:
|
||||
properties["id"] = event_id
|
||||
@@ -311,11 +310,11 @@ class SemanticaSessionService(BaseSessionService):
|
||||
|
||||
@staticmethod
|
||||
def _session_kwargs(
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
state: Optional[dict],
|
||||
events: Optional[List[Any]],
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
state: Optional[dict],
|
||||
events: Optional[List[Any]],
|
||||
) -> dict:
|
||||
"""Build kwargs for the ADK Session model."""
|
||||
return {
|
||||
@@ -327,8 +326,8 @@ class SemanticaSessionService(BaseSessionService):
|
||||
}
|
||||
|
||||
def _session_from_node(
|
||||
self,
|
||||
node: Any,
|
||||
self,
|
||||
node: Any,
|
||||
) -> Session:
|
||||
"""Reconstruct an ADK Session from a ContextGraph node."""
|
||||
properties = self._node_properties(node)
|
||||
@@ -348,14 +347,14 @@ class SemanticaSessionService(BaseSessionService):
|
||||
# splitting on ':' after the prefix always yields
|
||||
# exactly 3 parts regardless of what characters the
|
||||
# original app_name/user_id/session_id contained.
|
||||
parts = graph_node_id[len("adk-session:") :].split(":")
|
||||
parts = graph_node_id[len("adk-session:"):].split(":")
|
||||
if len(parts) == 3:
|
||||
decoded = [urllib.parse.unquote(part) for part in parts]
|
||||
app_name = app_name or decoded[0]
|
||||
user_id = user_id or decoded[1]
|
||||
session_id = decoded[2]
|
||||
else:
|
||||
session_id = graph_node_id[len("adk-session:") :]
|
||||
session_id = graph_node_id[len("adk-session:"):]
|
||||
else:
|
||||
session_id = graph_node_id
|
||||
|
||||
@@ -384,12 +383,12 @@ class SemanticaSessionService(BaseSessionService):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def create_session(
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
state: Optional[dict[str, Any]] = None,
|
||||
session_id: Optional[str] = None,
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
state: Optional[dict[str, Any]] = None,
|
||||
session_id: Optional[str] = None,
|
||||
) -> Session:
|
||||
"""Create and persist an ADK session."""
|
||||
return await asyncio.to_thread(
|
||||
@@ -397,11 +396,11 @@ class SemanticaSessionService(BaseSessionService):
|
||||
)
|
||||
|
||||
def _create_session_sync(
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
state: Optional[dict[str, Any]],
|
||||
session_id: Optional[str],
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
state: Optional[dict[str, Any]],
|
||||
session_id: Optional[str],
|
||||
) -> Session:
|
||||
with self._lock:
|
||||
session_id = session_id or str(uuid.uuid4())
|
||||
@@ -431,12 +430,12 @@ class SemanticaSessionService(BaseSessionService):
|
||||
)
|
||||
|
||||
async def get_session(
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
config: Optional[GetSessionConfig] = None,
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
config: Optional[GetSessionConfig] = None,
|
||||
) -> Optional[Session]:
|
||||
"""Retrieve an ADK session from ContextGraph."""
|
||||
return await asyncio.to_thread(
|
||||
@@ -444,11 +443,11 @@ class SemanticaSessionService(BaseSessionService):
|
||||
)
|
||||
|
||||
def _get_session_sync(
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
config: Optional[GetSessionConfig],
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
config: Optional[GetSessionConfig],
|
||||
) -> Optional[Session]:
|
||||
with self._lock:
|
||||
node = self._find_session_node(app_name, user_id, session_id)
|
||||
@@ -469,7 +468,7 @@ class SemanticaSessionService(BaseSessionService):
|
||||
# trims the already-built Session object.
|
||||
if config:
|
||||
if config.num_recent_events:
|
||||
session.events = session.events[-config.num_recent_events :]
|
||||
session.events = session.events[-config.num_recent_events:]
|
||||
if config.after_timestamp:
|
||||
i = len(session.events) - 1
|
||||
while i >= 0:
|
||||
@@ -477,14 +476,14 @@ class SemanticaSessionService(BaseSessionService):
|
||||
break
|
||||
i -= 1
|
||||
if i >= 0:
|
||||
session.events = session.events[i + 1 :]
|
||||
session.events = session.events[i + 1:]
|
||||
|
||||
return session
|
||||
|
||||
async def append_event(
|
||||
self,
|
||||
session: Session,
|
||||
event: Event,
|
||||
self,
|
||||
session: Session,
|
||||
event: Event,
|
||||
) -> Event:
|
||||
"""Persist an ADK event and associate it with a session."""
|
||||
# ADK's own base implementation is a no-op for partial/streaming
|
||||
@@ -511,13 +510,8 @@ class SemanticaSessionService(BaseSessionService):
|
||||
|
||||
# Verify cross-tenant security
|
||||
properties = self._node_properties(session_node)
|
||||
if (
|
||||
properties.get("app_name") != app_name
|
||||
or properties.get("user_id") != user_id
|
||||
):
|
||||
raise ValueError(
|
||||
"Cross-tenant session write denied: app_name or user_id mismatch."
|
||||
)
|
||||
if properties.get("app_name") != app_name or properties.get("user_id") != user_id:
|
||||
raise ValueError("Cross-tenant session write denied: app_name or user_id mismatch.")
|
||||
|
||||
# Apply ADK in-memory event and state delta semantics. This
|
||||
# runs inside asyncio.to_thread's worker thread, which has no
|
||||
@@ -559,11 +553,11 @@ class SemanticaSessionService(BaseSessionService):
|
||||
)
|
||||
|
||||
async def delete_session(
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
) -> None:
|
||||
"""Delete a session and all of its graph-backed events."""
|
||||
await asyncio.to_thread(
|
||||
@@ -571,10 +565,10 @@ class SemanticaSessionService(BaseSessionService):
|
||||
)
|
||||
|
||||
def _delete_session_sync(
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
) -> None:
|
||||
with self._lock:
|
||||
session_node = self._find_session_node(app_name, user_id, session_id)
|
||||
@@ -596,9 +590,9 @@ class SemanticaSessionService(BaseSessionService):
|
||||
continue
|
||||
|
||||
if (
|
||||
edge.get("source") == session_node_id
|
||||
and edge.get("type") == "HAS_EVENT"
|
||||
and edge.get("target")
|
||||
edge.get("source") == session_node_id
|
||||
and edge.get("type") == "HAS_EVENT"
|
||||
and edge.get("target")
|
||||
):
|
||||
event_node_ids.append(str(edge["target"]))
|
||||
|
||||
@@ -608,18 +602,18 @@ class SemanticaSessionService(BaseSessionService):
|
||||
self.graph.purge_node(session_node_id)
|
||||
|
||||
async def list_sessions(
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: Optional[str] = None,
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: Optional[str] = None,
|
||||
) -> ListSessionsResponse:
|
||||
"""List sessions for an app, optionally scoped to one user."""
|
||||
return await asyncio.to_thread(self._list_sessions_sync, app_name, user_id)
|
||||
|
||||
def _list_sessions_sync(
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: Optional[str],
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: Optional[str],
|
||||
) -> ListSessionsResponse:
|
||||
with self._lock:
|
||||
sessions: List[Session] = []
|
||||
@@ -649,4 +643,4 @@ class SemanticaSessionService(BaseSessionService):
|
||||
__all__ = [
|
||||
"ADK_AVAILABLE",
|
||||
"SemanticaSessionService",
|
||||
]
|
||||
]
|
||||
@@ -1,20 +0,0 @@
|
||||
# OSV-Scanner ignore config (also consumed by OpenSSF Scorecard's
|
||||
# "Vulnerabilities" check, which reports advisories found in this repo's
|
||||
# dependency manifests via https://osv.dev).
|
||||
#
|
||||
# See https://github.com/google/osv-scanner#ignore-vulnerabilities-by-id for
|
||||
# the file format.
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-4j2p-28q2-5m79"
|
||||
reason = """
|
||||
accelerate<=1.14.0 (transitive dependency via docling-slim, pinned in
|
||||
requirements-ci.txt) has an open path traversal / DoS advisory (also tracked
|
||||
as CVE-2026-69112) in load_checkpoint_in_model / load_checkpoint_and_dispatch,
|
||||
which fail to sanitize weight_map entries from sharded checkpoint indexes.
|
||||
1.14.0 is the latest release on PyPI; no patched version exists yet.
|
||||
Semantica does not call either function or load arbitrary/untrusted sharded
|
||||
checkpoints, so the vulnerable code path is not reachable. Re-evaluate once
|
||||
accelerate ships a fix - see .github/workflows/security-scan.yml for the
|
||||
matching pip-audit exclusion.
|
||||
"""
|
||||
+52
-88
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "semantica"
|
||||
version = "0.7.0"
|
||||
version = "0.6.8"
|
||||
description = "Graph-Native Infrastructure for Context and Accountable AI Systems: context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable."
|
||||
readme = "README.md"
|
||||
license = { text = "MIT" }
|
||||
@@ -12,11 +12,7 @@ license = { text = "MIT" }
|
||||
authors = [{ name = "Semantica", email = "kaif@getsemantica.ai" }]
|
||||
maintainers = [{ name = "Semantica", email = "kaif@getsemantica.ai" }]
|
||||
|
||||
# 3.8 is already unsatisfiable in practice (numpy>=2.0.2 requires >=3.9) and is
|
||||
# not exercised by the Install Matrix (3.9-3.12). The floor is 3.9.2 rather
|
||||
# than 3.9.0 because cryptography (db-snowflake) excludes 3.9.0/3.9.1 from
|
||||
# every release's requires-python, so those patch levels can never resolve.
|
||||
requires-python = ">=3.9.2"
|
||||
requires-python = ">=3.8"
|
||||
|
||||
classifiers = [
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
@@ -26,6 +22,7 @@ classifiers = [
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
@@ -55,13 +52,30 @@ dependencies = [
|
||||
# last 3.9-compatible release line; 3.10+ is left unconstrained.
|
||||
"scikit-learn>=1.6.1,<1.7.0; python_version < '3.10'",
|
||||
"scikit-learn>=1.7.2; python_version >= '3.10'",
|
||||
"umap-learn>=0.5.12",
|
||||
# thinc (spacy's core dep) dropped Python 3.9 wheels at 8.3.10, and later
|
||||
# spacy patch releases (3.8.8+) require thinc>=8.3.9-only-on-3.10+ ranges,
|
||||
# which forces a source build that fails outright on 3.9 (see Install
|
||||
# Matrix run history). Capping both keeps 3.9 on the last wheel-compatible
|
||||
# pair; 3.10+ is left unconstrained to always get the latest spacy/thinc.
|
||||
"spacy>=3.4.0,<3.8.8; python_version < '3.10'",
|
||||
"spacy>=3.4.0; python_version >= '3.10'",
|
||||
"thinc<8.3.5; python_version < '3.10'",
|
||||
"transformers>=4.20.0",
|
||||
"torch>=1.13.1",
|
||||
"sentence-transformers>=2.2.0",
|
||||
"rdflib>=6.2.0",
|
||||
"networkx>=2.8.0",
|
||||
"matplotlib>=3.9.4",
|
||||
"seaborn>=0.13.2",
|
||||
"plotly>=6.8.0",
|
||||
"ipywidgets>=8.0.0",
|
||||
# requests dropped Python 3.9 support at 2.33.0 (requires_python >=3.10),
|
||||
# so an unqualified >=2.34.2 floor is unsatisfiable on 3.9. Cap 3.9 to the
|
||||
# last 3.9-compatible release; 3.10+ is left unconstrained.
|
||||
"requests>=2.32.5,<2.33.0; python_version < '3.10'",
|
||||
"requests>=2.34.2; python_version >= '3.10'",
|
||||
"GitPython>=3.1.58",
|
||||
# chardet dropped Python 3.9 support at 6.0.0 (requires_python >=3.10), so
|
||||
# an unqualified >=7.4.3 floor is unsatisfiable on 3.9. Cap 3.9 to the last
|
||||
# 3.9-compatible release; 3.10+ is left unconstrained.
|
||||
@@ -73,11 +87,26 @@ dependencies = [
|
||||
# 3.9-compatible release; 3.10+ is left unconstrained.
|
||||
"grpcio>=1.80.0,<1.81.0; python_version < '3.10'",
|
||||
"grpcio>=1.81.1; python_version >= '3.10'",
|
||||
"beautifulsoup4>=4.15.0",
|
||||
"lxml>=6.1.1",
|
||||
"python-docx>=1.2.0",
|
||||
"openpyxl>=3.1.5",
|
||||
# pillow dropped Python 3.9 support at 12.0.0 (requires_python >=3.10), so
|
||||
# an unqualified >=12.2.0 floor is unsatisfiable on 3.9. Cap 3.9 to the last
|
||||
# 3.9-compatible release; 3.10+ is left unconstrained.
|
||||
"pillow>=11.3.0,<12.0.0; python_version < '3.10'",
|
||||
"pillow>=12.2.0; python_version >= '3.10'",
|
||||
"librosa>=0.9.0",
|
||||
"opencv-python>=4.13.0.92",
|
||||
"faiss-cpu>=1.7.0",
|
||||
"fastembed>=0.2.0",
|
||||
# onnxruntime stopped shipping cp39 wheels at 1.20.0 (its PyPI metadata
|
||||
# still claims requires_python >=3.9, but no matching wheel exists), so an
|
||||
# unqualified >=1.20.1 floor is unsatisfiable on 3.9. Cap 3.9 to the last
|
||||
# release with a cp39 wheel; 3.10+ is left unconstrained.
|
||||
"onnxruntime>=1.19.2,<1.20.0; python_version < '3.10'",
|
||||
"onnxruntime>=1.20.1; python_version >= '3.10'",
|
||||
"tokenizers>=0.15.0",
|
||||
"pydantic>=2.13.4",
|
||||
# click dropped Python 3.9 support at 8.2.0 (requires_python >=3.10), so an
|
||||
# unqualified >=8.4.2 floor is unsatisfiable on 3.9. Cap 3.9 to the last
|
||||
@@ -91,6 +120,7 @@ dependencies = [
|
||||
"python-dotenv>=1.2.1",
|
||||
"loguru>=0.7.3",
|
||||
"structlog>=22.1.0",
|
||||
"gensim>=4.4.0",
|
||||
"httpx<0.29.0",
|
||||
"pyarrow>=14.0.0"
|
||||
]
|
||||
@@ -114,10 +144,7 @@ llm-anthropic = ["anthropic>=0.122.0"]
|
||||
llm-ollama = ["ollama>=0.1.0"]
|
||||
llm-deepseek = ["openai>=1.0.0"]
|
||||
llm-novita = ["openai>=1.0.0"]
|
||||
# litellm>=1.83.10 requires Python>=3.10, and the last 3.9-compatible release
|
||||
# (1.83.9) pins python-dotenv==1.0.1, which conflicts with our >=1.2.1 core
|
||||
# floor — so no litellm satisfies 3.9 at all. Gate it to 3.10+.
|
||||
llm-litellm = ["litellm>=1.83.9; python_version >= '3.10'"]
|
||||
llm-litellm = ["litellm>=1.83.9"]
|
||||
llm-instructor = ["instructor>=1.15.3"]
|
||||
|
||||
llm-all = [
|
||||
@@ -125,49 +152,19 @@ llm-all = [
|
||||
]
|
||||
|
||||
# ---- Document Parsing ----
|
||||
documents = [
|
||||
"python-docx>=1.2.0",
|
||||
"openpyxl>=3.1.5",
|
||||
"lxml>=6.1.1",
|
||||
"beautifulsoup4>=4.15.0"
|
||||
]
|
||||
# every docling release requires Python>=3.10 (no 3.9-compatible version
|
||||
# exists to cap to), so gate it like google-adk below rather than split it.
|
||||
parse-docling = ["docling>=2.107.0; python_version >= '3.10'"]
|
||||
# pdfplumber powers the default PDFParser; not pulled in by any other extra.
|
||||
parse-pdf = ["pdfplumber>=0.10.0"]
|
||||
parse-docling = ["docling>=2.107.0"]
|
||||
|
||||
# ---- SHACL Validation ----
|
||||
shacl = ["pyshacl>=0.25.0"]
|
||||
|
||||
# ---- Database Connectors ----
|
||||
# snowflake-connector-python dropped Python 3.9 support at 4.6.0
|
||||
# (requires_python >=3.10), so an unqualified >=4.6.0 floor is unsatisfiable
|
||||
# on 3.9. Cap 3.9 below it; 3.10+ keeps the newer floor.
|
||||
db-snowflake = [
|
||||
"snowflake-connector-python>=4.6.0; python_version >= '3.10'",
|
||||
"snowflake-connector-python>=3.13.0,<4.6.0; python_version < '3.10'",
|
||||
"cryptography>=49.0.0"
|
||||
]
|
||||
db-snowflake = ["snowflake-connector-python>=4.6.0", "cryptography>=49.0.0"]
|
||||
db-databricks = ["databricks-sdk>=0.60.0", "databricks-sql-connector>=4.0.0"]
|
||||
# pyarrow dropped Python 3.9 support at 24.0.0 (requires_python >=3.10), so an
|
||||
# unqualified >=24.0.0 floor is unsatisfiable on 3.9. Cap 3.9 below the last
|
||||
# 3.9-compatible release line; 3.10+ is left unconstrained.
|
||||
db-arrow = [
|
||||
"pyarrow>=24.0.0; python_version >= '3.10'",
|
||||
"pyarrow>=14.0.0,<24.0.0; python_version < '3.10'"
|
||||
]
|
||||
db-arrow = ["pyarrow>=24.0.0"]
|
||||
db-salesforce = ["simple-salesforce>=1.12.0"]
|
||||
ingest-parquet = [
|
||||
"pyarrow>=24.0.0; python_version >= '3.10'",
|
||||
"pyarrow>=14.0.0,<24.0.0; python_version < '3.10'"
|
||||
]
|
||||
ingest-arrow = [
|
||||
"pyarrow>=24.0.0; python_version >= '3.10'",
|
||||
"pyarrow>=14.0.0,<24.0.0; python_version < '3.10'"
|
||||
]
|
||||
ingest-parquet = ["pyarrow>=24.0.0"]
|
||||
ingest-arrow = ["pyarrow>=24.0.0"]
|
||||
ingest-sap = ["requests>=2.28.0"]
|
||||
ingest-git = ["GitPython>=3.1.58"]
|
||||
|
||||
db-all = [
|
||||
"semantica[db-snowflake,db-databricks,db-salesforce,db-arrow]"
|
||||
@@ -178,35 +175,22 @@ models-huggingface = [
|
||||
"transformers>=4.20.0",
|
||||
"torch>=1.13.1"
|
||||
]
|
||||
embeddings-local = [
|
||||
"sentence-transformers>=2.2.0",
|
||||
"fastembed>=0.2.0",
|
||||
"onnxruntime>=1.19.2,<1.20.0; python_version < '3.10'",
|
||||
"onnxruntime>=1.20.1; python_version >= '3.10'",
|
||||
"tokenizers>=0.15.0"
|
||||
]
|
||||
nlp-spacy = [
|
||||
"spacy>=3.4.0,<3.8.8; python_version < '3.10'",
|
||||
"spacy>=3.4.0; python_version >= '3.10'"
|
||||
]
|
||||
|
||||
# ---- Graph Backends ----
|
||||
graph-neo4j = ["neo4j>=5.0.0"]
|
||||
graph-falkordb = ["falkordb>=1.0.0", "redis>=4.3.0"]
|
||||
graph-amazon-neptune = ["boto3>=1.24.0", "neo4j>=5.0.0"]
|
||||
graph-apache-age = ["psycopg2-binary>=2.9.0"]
|
||||
graph-embeddings = ["gensim>=4.4.0"]
|
||||
|
||||
graph-all = [
|
||||
"semantica[graph-neo4j,graph-falkordb,graph-amazon-neptune,graph-apache-age,graph-embeddings]"
|
||||
"semantica[graph-neo4j,graph-falkordb,graph-amazon-neptune,graph-apache-age]"
|
||||
]
|
||||
|
||||
# ---- Triplet Store Backends ----
|
||||
tripletstore-oxigraph = ["pyoxigraph>=0.5.0"]
|
||||
|
||||
# ---- Vector Store Backends ----
|
||||
vectorstore-faiss = ["faiss-cpu>=1.7.0"]
|
||||
vectorstore-qdrant = ["qdrant-client>=1.10.0"]
|
||||
vectorstore-qdrant = ["qdrant-client>=1.0.0"]
|
||||
vectorstore-weaviate = ["weaviate-client>=4.0.0"]
|
||||
vectorstore-pinecone = ["pinecone>=3.0.0"]
|
||||
vectorstore-milvus = ["pymilvus>=2.0.0"]
|
||||
@@ -214,7 +198,7 @@ vectorstore-pgvector = ["psycopg[binary,pool]>=3.0.0", "pgvector>=0.2.0"]
|
||||
vectorstore-sqlite = ["sqlite-vec>=0.1.1"]
|
||||
|
||||
vectorstore-all = [
|
||||
"semantica[vectorstore-qdrant,vectorstore-weaviate,vectorstore-pinecone,vectorstore-milvus,vectorstore-pgvector,vectorstore-sqlite,vectorstore-faiss]"
|
||||
"semantica[vectorstore-qdrant,vectorstore-weaviate,vectorstore-pinecone,vectorstore-milvus,vectorstore-pgvector,vectorstore-sqlite]"
|
||||
]
|
||||
|
||||
# ---- Infra / Queues / Workers ----
|
||||
@@ -246,18 +230,7 @@ monitoring = [
|
||||
viz = [
|
||||
"pyvis>=0.3.0",
|
||||
"graphviz>=0.21",
|
||||
"d3blocks>=1.0.0",
|
||||
"matplotlib>=3.9.4",
|
||||
"seaborn>=0.13.2",
|
||||
"plotly>=6.8.0",
|
||||
"ipywidgets>=8.0.0",
|
||||
"umap-learn>=0.5.12"
|
||||
]
|
||||
|
||||
# ---- Media ----
|
||||
media = [
|
||||
"librosa>=0.9.0",
|
||||
"opencv-python>=4.13.0.92"
|
||||
"d3blocks>=1.0.0"
|
||||
]
|
||||
|
||||
# ---- GPU ----
|
||||
@@ -271,9 +244,7 @@ agno = ["agno>=1.0.0"]
|
||||
# crewai core provides BaseTool and BaseKnowledgeSource; crewai-tools is not
|
||||
# needed (it pulls vulnerable transitive deps like chromadb) and would only
|
||||
# duplicate the prebuilt tooling users can install separately.
|
||||
# No un-yanked crewai release supports Python 3.9 (all require >=3.10), so
|
||||
# gate the extra to 3.10+.
|
||||
crewai = ["crewai>=0.80.0; python_version >= '3.10'"]
|
||||
crewai = ["crewai>=0.80.0"]
|
||||
langchain = ["langchain-core>=0.3.0"]
|
||||
google-adk = ["google-adk>=1.27.0; python_version >= '3.10'"]
|
||||
|
||||
@@ -298,23 +269,15 @@ dev = [
|
||||
"isort>=6.1.0",
|
||||
"flake8>=4.0.0",
|
||||
"mypy>=0.971",
|
||||
# pre-commit dropped Python 3.9 support at 4.6.0 (requires_python >=3.10).
|
||||
# Cap 3.9 below it; 3.10+ keeps the >=4.6.0 floor.
|
||||
"pre-commit>=4.0.0,<4.6.0; python_version < '3.10'",
|
||||
"pre-commit>=4.6.0; python_version >= '3.10'",
|
||||
"pre-commit>=4.6.0",
|
||||
"jupyter>=1.0.0",
|
||||
"ipykernel>=6.15.0"
|
||||
]
|
||||
|
||||
# Explorer Dashboard
|
||||
# fastapi dropped Python 3.9 support at 0.129.0 (requires_python >=3.10), and
|
||||
# every fastapi below that caps starlette<0.53.0 — so the 3.10+ starlette
|
||||
# floor is unsatisfiable on 3.9. Cap both on 3.9; 3.10+ keeps the newer floors.
|
||||
explorer = [
|
||||
"fastapi>=0.109.2,<0.129.0; python_version < '3.10'",
|
||||
"fastapi>=0.109.2; python_version >= '3.10'",
|
||||
"starlette>=0.36.3,<0.53.0; python_version < '3.10'",
|
||||
"starlette>=0.53.0; python_version >= '3.10'",
|
||||
"fastapi>=0.109.2",
|
||||
"starlette>=0.53.0",
|
||||
"uvicorn[standard]>=0.22.0",
|
||||
"websockets>=15.0.1",
|
||||
"python-multipart>=0.0.7",
|
||||
@@ -331,7 +294,8 @@ explorer-lite = [
|
||||
# (CVE-2026-45829) with no fixed release — including it here would fail the CI
|
||||
# dependency-audit/security gates. Install it explicitly via ``semantica[crewai]``.
|
||||
all = [
|
||||
"semantica[dev,viz,media,infra,cloud,monitoring,watch,llm-all,models-huggingface,embeddings-local,nlp-spacy,documents,ingest-git,graph-embeddings,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,parse-pdf,ingest-parquet,ingest-arrow,shacl,explorer,agno,langchain,google-adk]"
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]",
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno,langchain,google-adk]"
|
||||
]
|
||||
|
||||
# ---------------- ENTRYPOINTS ----------------
|
||||
|
||||
+2
-17
@@ -181,7 +181,6 @@ anyio==4.14.2 \
|
||||
# jupyter-server
|
||||
# langsmith
|
||||
# openai
|
||||
# pinecone
|
||||
# starlette
|
||||
# watchfiles
|
||||
argon2-cffi==25.1.0 \
|
||||
@@ -787,9 +786,7 @@ charset-normalizer==3.5.1 \
|
||||
--hash=sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f \
|
||||
--hash=sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0 \
|
||||
--hash=sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f
|
||||
# via
|
||||
# pdfminer-six
|
||||
# requests
|
||||
# via requests
|
||||
click==8.5.0 \
|
||||
--hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \
|
||||
--hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34
|
||||
@@ -1100,7 +1097,6 @@ cryptography==50.0.1 \
|
||||
# azure-storage-blob
|
||||
# google-auth
|
||||
# joserfc
|
||||
# pdfminer-six
|
||||
cuda-bindings==13.3.1 \
|
||||
--hash=sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708 \
|
||||
--hash=sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86 \
|
||||
@@ -4302,14 +4298,6 @@ patsy==1.0.3 \
|
||||
--hash=sha256:79ebf4c93ff4d296e58a9d5be2b2ee31bd49d737cf11d70ffbd8a44b2de42e65 \
|
||||
--hash=sha256:d3dbebe8fd5f46e29912d030b63c6268647b59bf788a99e2af28a30234cf357c
|
||||
# via statsmodels
|
||||
pdfminer-six==20260107 \
|
||||
--hash=sha256:366585ba97e80dffa8f00cebe303d2f381884d8637af4ce422f1df3ef38111a9 \
|
||||
--hash=sha256:96bfd431e3577a55a0efd25676968ca4ce8fd5b53f14565f85716ff363889602
|
||||
# via pdfplumber
|
||||
pdfplumber==0.11.10 \
|
||||
--hash=sha256:7741ea81bf165b474b153e6789d10d18e06b6ddcf3ec84289c3ef2fed6802580 \
|
||||
--hash=sha256:b95b2d28c66efb0a794a83b88c6c6aea5987532a445d20a1cbcfa657022e6e57
|
||||
# via semantica (pyproject.toml)
|
||||
pexpect==4.9.0 \
|
||||
--hash=sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523 \
|
||||
--hash=sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f
|
||||
@@ -4418,7 +4406,6 @@ pillow==12.3.0 \
|
||||
# docling-slim
|
||||
# fastembed
|
||||
# matplotlib
|
||||
# pdfplumber
|
||||
# python-pptx
|
||||
# rapidocr
|
||||
# torchvision
|
||||
@@ -5306,9 +5293,7 @@ pypdfium2==5.13.0 \
|
||||
--hash=sha256:d96929bde3bd64c771ab3558ca1ffd7704cc4d872ab92cd9f8f8b8a20f7f36b8 \
|
||||
--hash=sha256:d96beb7f379e6c76d874ca93fcd182ac3168dd499056407070f9927fb1061b8e \
|
||||
--hash=sha256:da5c7b74eebf40b5c1fbe1de01aa1edc8827a79fb1efd999616bc20dcaf77ba4
|
||||
# via
|
||||
# docling-slim
|
||||
# pdfplumber
|
||||
# via docling-slim
|
||||
pypickle==2.0.2 \
|
||||
--hash=sha256:d3307127314465fe3dc8f0162e11777d5e8284f3a29dc48b0f770d364a85d998 \
|
||||
--hash=sha256:d577e39cf501c7c80b1387f6d7dc885cf4efeba65f213df41226d1f24881b1e8
|
||||
|
||||
@@ -10,7 +10,7 @@ Main exports:
|
||||
- Config: Configuration management
|
||||
"""
|
||||
|
||||
__version__ = "0.7.0"
|
||||
__version__ = "0.6.8"
|
||||
__author__ = "Semantica Contributors"
|
||||
__license__ = "MIT"
|
||||
|
||||
|
||||
+22
-26
@@ -465,7 +465,7 @@ def _show_startup(cli_ctx: CLIContext) -> None:
|
||||
return
|
||||
cfg = cli_ctx.config.to_dict()
|
||||
graph_store = (
|
||||
cli_ctx.store_backend or cfg.get("graph_db", {}).get("backend", "neo4j")
|
||||
cli_ctx.store_backend or cfg.get("graph_db", {}).get("backend", "memory")
|
||||
)
|
||||
vector_store = (
|
||||
cli_ctx.vector_store_backend
|
||||
@@ -851,7 +851,9 @@ def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None
|
||||
# Graph store reachability
|
||||
def _graph() -> str:
|
||||
cfg = cli_ctx.config.to_dict()
|
||||
backend = cli_ctx.store_backend or cfg.get("graph_db", {}).get("backend", "neo4j")
|
||||
backend = cli_ctx.store_backend or cfg.get("graph_db", {}).get("backend", "memory")
|
||||
if backend == "memory":
|
||||
return "memory (always available)"
|
||||
gs = _get_graph_store(cli_ctx)
|
||||
gs.ping() if hasattr(gs, "ping") else gs.connect()
|
||||
return f"{backend} reachable"
|
||||
@@ -878,18 +880,10 @@ def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None
|
||||
def _embedding_backend(method: str) -> str:
|
||||
if method == "sentence_transformers":
|
||||
import sentence_transformers # noqa: F401
|
||||
try:
|
||||
ver = importlib.metadata.version("sentence-transformers")
|
||||
except Exception:
|
||||
ver = getattr(sentence_transformers, "__version__", "installed")
|
||||
note = f"importable ({ver})"
|
||||
note = f"importable ({importlib.metadata.version('sentence-transformers')})"
|
||||
else:
|
||||
import fastembed # noqa: F401
|
||||
try:
|
||||
ver = importlib.metadata.version("fastembed")
|
||||
except Exception:
|
||||
ver = getattr(fastembed, "__version__", "installed")
|
||||
note = f"importable ({ver})"
|
||||
note = f"importable ({importlib.metadata.version('fastembed')})"
|
||||
if not deep:
|
||||
return note
|
||||
try:
|
||||
@@ -910,12 +904,12 @@ def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None
|
||||
checks.append(_check(
|
||||
"Embeddings (sentence-transformers)",
|
||||
lambda: _embedding_backend("sentence_transformers"),
|
||||
hint="pip install 'semantica[embeddings-local]'",
|
||||
hint="pip install sentence-transformers",
|
||||
))
|
||||
checks.append(_check(
|
||||
"Embeddings (fastembed)",
|
||||
lambda: _embedding_backend("fastembed"),
|
||||
hint="pip install 'semantica[embeddings-local]'",
|
||||
hint="pip install fastembed",
|
||||
))
|
||||
|
||||
# LLM provider keys
|
||||
@@ -4770,10 +4764,15 @@ def mcp_list_tools(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
cli_ctx = _require_ctx(cli_ctx)
|
||||
|
||||
def _action() -> None:
|
||||
# Same catalog the server exposes via tools/list, so `list-tools`
|
||||
# and `mcp start` can't drift (issue #1355).
|
||||
from semantica_mcp.mcp.tools import TOOL_DEFINITIONS
|
||||
tools = [t["name"] for t in TOOL_DEFINITIONS]
|
||||
try:
|
||||
from semantica_mcp.mcp.tools import __all__ as tools
|
||||
except ImportError:
|
||||
tools = [
|
||||
"extract_entities", "extract_relations", "build_graph",
|
||||
"query_graph", "get_graph_analytics", "run_reasoning",
|
||||
"record_decision", "get_decisions", "export_graph",
|
||||
"validate_shacl", "get_provenance", "embed_and_search",
|
||||
]
|
||||
if _is_json(cli_ctx, local_json):
|
||||
_jecho({"tools": list(tools)})
|
||||
else:
|
||||
@@ -4806,15 +4805,12 @@ def mcp_call(cli_ctx: CLIContext, tool_name: str, args: str, local_json: bool) -
|
||||
tool_args = json.loads(args)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise click.ClickException(f"Invalid JSON in --args: {exc}") from exc
|
||||
if not isinstance(tool_args, dict):
|
||||
raise click.ClickException("--args must be a JSON object")
|
||||
# Dispatch through the same server `mcp start` spawns; its session
|
||||
# module never defined MCPSession (issue #1355).
|
||||
from semantica_mcp.mcp.server import UnknownToolError, call_tool
|
||||
try:
|
||||
result = call_tool(tool_name, tool_args)
|
||||
except UnknownToolError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
from semantica_mcp.mcp.session import MCPSession
|
||||
session = MCPSession(config=cli_ctx.config.to_dict())
|
||||
result = session.call_tool(tool_name, **tool_args)
|
||||
except ImportError as exc:
|
||||
raise click.ClickException(f"MCP module not available: {exc}") from exc
|
||||
if _is_json(cli_ctx, local_json):
|
||||
_jecho(result if isinstance(result, (dict, list)) else {"result": str(result)})
|
||||
else:
|
||||
|
||||
@@ -212,7 +212,7 @@ class FastEmbedStore(ProviderStore):
|
||||
self.logger.info(f"Loaded FastEmbed model: {self.model_name}")
|
||||
except (ImportError, OSError):
|
||||
self.logger.warning(
|
||||
"fastembed not available. Install with: pip install 'semantica[embeddings-local]'"
|
||||
"fastembed not available. Install with: pip install fastembed"
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to load FastEmbed model: {e}")
|
||||
|
||||
@@ -35,7 +35,6 @@ try:
|
||||
|
||||
SENTENCE_TRANSFORMERS_AVAILABLE = True
|
||||
except (ImportError, OSError):
|
||||
SentenceTransformer = None
|
||||
SENTENCE_TRANSFORMERS_AVAILABLE = False
|
||||
|
||||
try:
|
||||
@@ -43,7 +42,6 @@ try:
|
||||
|
||||
FASTEMBED_AVAILABLE = True
|
||||
except (ImportError, OSError):
|
||||
TextEmbedding = None
|
||||
FASTEMBED_AVAILABLE = False
|
||||
|
||||
|
||||
@@ -158,7 +156,7 @@ class TextEmbedder:
|
||||
else:
|
||||
self.logger.warning(
|
||||
"fastembed not available. "
|
||||
"Install with: pip install 'semantica[embeddings-local]'. "
|
||||
"Install with: pip install fastembed. "
|
||||
"Using fallback embedding method."
|
||||
)
|
||||
else:
|
||||
@@ -180,7 +178,7 @@ class TextEmbedder:
|
||||
else:
|
||||
self.logger.warning(
|
||||
"sentence-transformers not available. "
|
||||
"Install with: pip install 'semantica[embeddings-local]'. "
|
||||
"Install with: pip install sentence-transformers. "
|
||||
"Using fallback embedding method."
|
||||
)
|
||||
|
||||
|
||||
@@ -52,23 +52,6 @@ async def list_decisions(
|
||||
return [_node_to_decision(node) for node in nodes[skip : skip + limit]]
|
||||
|
||||
|
||||
@router.get("/causal-distance", response_model=CausalDistanceReport)
|
||||
async def causal_distance(
|
||||
source: str = Query(..., description="Source node/decision ID"),
|
||||
target: str = Query(..., description="Target node/decision ID"),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""FR-8 — Compute causal distance between two decisions via causal-edge-only traversal."""
|
||||
from ...context.causal_analyzer import CausalChainAnalyzer
|
||||
|
||||
analyzer = CausalChainAnalyzer(session.graph)
|
||||
report = await asyncio.to_thread(analyzer.interpret_causal_distance, source, target)
|
||||
return CausalDistanceReport(**report)
|
||||
|
||||
|
||||
# NOTE: static routes (e.g. /causal-distance above) must stay above this
|
||||
# dynamic route — Starlette matches in definition order, otherwise the static
|
||||
# path is captured as decision_id (see issue #1531).
|
||||
@router.get("/{decision_id}", response_model=DecisionResponse)
|
||||
async def get_decision(
|
||||
decision_id: str,
|
||||
@@ -142,6 +125,20 @@ async def get_precedents(
|
||||
return [_node_to_decision(decision) for _, decision in scored[:limit]]
|
||||
|
||||
|
||||
@router.get("/causal-distance", response_model=CausalDistanceReport)
|
||||
async def causal_distance(
|
||||
source: str = Query(..., description="Source node/decision ID"),
|
||||
target: str = Query(..., description="Target node/decision ID"),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""FR-8 — Compute causal distance between two decisions via causal-edge-only traversal."""
|
||||
from ...context.causal_analyzer import CausalChainAnalyzer
|
||||
|
||||
analyzer = CausalChainAnalyzer(session.graph)
|
||||
report = await asyncio.to_thread(analyzer.interpret_causal_distance, source, target)
|
||||
return CausalDistanceReport(**report)
|
||||
|
||||
|
||||
@router.get("/{decision_id}/compliance", response_model=ComplianceResponse)
|
||||
async def check_compliance(
|
||||
decision_id: str,
|
||||
|
||||
@@ -329,27 +329,12 @@ class GraphExporter:
|
||||
lines.append(' http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">')
|
||||
lines.append("")
|
||||
|
||||
# Define attribute keys.
|
||||
# GraphML requires every key id referenced by a <data> element to be
|
||||
# declared here with a matching <key> element.
|
||||
#
|
||||
# for="all" — key is valid on both nodes and edges
|
||||
# for="node" — key is valid on nodes only
|
||||
# for="edge" — key is valid on edges only
|
||||
#
|
||||
# label : used on <node> (human-readable label) and <edge> (type).
|
||||
# Declared for="all" so both uses are schema-valid.
|
||||
# type : node entity type; only written on nodes.
|
||||
# confidence: written on both nodes and edges when include_attributes
|
||||
# is True; declared for="all" so edge confidence is valid.
|
||||
lines.append(
|
||||
' <key id="label" for="all" attr.name="label" attr.type="string"/>'
|
||||
)
|
||||
# Define attribute keys
|
||||
lines.append(
|
||||
' <key id="type" for="node" attr.name="type" attr.type="string"/>'
|
||||
)
|
||||
lines.append(
|
||||
' <key id="confidence" for="all" attr.name="confidence" attr.type="double"/>'
|
||||
' <key id="confidence" for="node" attr.name="confidence" attr.type="double"/>'
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
@@ -357,31 +342,23 @@ class GraphExporter:
|
||||
lines.append(' <graph id="G" edgedefault="directed">')
|
||||
lines.append("")
|
||||
|
||||
# xml.sax.saxutils helpers:
|
||||
# escape(v) – escapes & < > in text node content
|
||||
# quoteattr(v) – escapes & < > " ' and wraps in the
|
||||
# appropriate quote character for use as an
|
||||
# XML attribute value (including the quotes)
|
||||
from xml.sax.saxutils import escape, quoteattr
|
||||
|
||||
# Export nodes
|
||||
nodes = graph_data.get("nodes", [])
|
||||
for node in nodes:
|
||||
node_id = str(node.get("id") or "")
|
||||
label = str(node.get("label") or "")
|
||||
node_type = str(node.get("type") or "")
|
||||
node_id = node.get("id", "")
|
||||
label = node.get("label", "")
|
||||
node_type = node.get("type", "")
|
||||
|
||||
# quoteattr produces the surrounding quotes; do NOT add extra "…"
|
||||
lines.append(f" <node id={quoteattr(node_id)}>")
|
||||
lines.append(f" <data key=\"label\">{escape(label)}</data>")
|
||||
lines.append(f" <data key=\"type\">{escape(node_type)}</data>")
|
||||
lines.append(f' <node id="{node_id}">')
|
||||
lines.append(f' <data key="label">{label}</data>')
|
||||
lines.append(f' <data key="type">{node_type}</data>')
|
||||
|
||||
# Add attributes if requested
|
||||
if self.include_attributes and "attributes" in node:
|
||||
attrs = node["attributes"]
|
||||
if "confidence" in attrs:
|
||||
lines.append(
|
||||
f" <data key=\"confidence\">{escape(str(attrs['confidence']))}</data>"
|
||||
f' <data key="confidence">{attrs["confidence"]}</data>'
|
||||
)
|
||||
|
||||
lines.append(" </node>")
|
||||
@@ -391,19 +368,19 @@ class GraphExporter:
|
||||
# Export edges
|
||||
edges = graph_data.get("edges", [])
|
||||
for edge in edges:
|
||||
source = str(edge.get("source") or "")
|
||||
target = str(edge.get("target") or "")
|
||||
edge_type = str(edge.get("type") or "")
|
||||
source = edge.get("source", "")
|
||||
target = edge.get("target", "")
|
||||
edge_type = edge.get("type", "")
|
||||
|
||||
lines.append(f" <edge source={quoteattr(source)} target={quoteattr(target)}>")
|
||||
lines.append(f" <data key=\"label\">{escape(edge_type)}</data>")
|
||||
lines.append(f' <edge source="{source}" target="{target}">')
|
||||
lines.append(f' <data key="label">{edge_type}</data>')
|
||||
|
||||
# Add attributes if requested
|
||||
if self.include_attributes and "attributes" in edge:
|
||||
attrs = edge["attributes"]
|
||||
if "confidence" in attrs:
|
||||
lines.append(
|
||||
f" <data key=\"confidence\">{escape(str(attrs['confidence']))}</data>"
|
||||
f' <data key="confidence">{attrs["confidence"]}</data>'
|
||||
)
|
||||
|
||||
lines.append(" </edge>")
|
||||
|
||||
@@ -421,7 +421,7 @@ class VectorExporter:
|
||||
import numpy as np
|
||||
except (ImportError, OSError):
|
||||
raise ImportError(
|
||||
"FAISS not installed. Install with: pip install 'semantica[vectorstore-faiss]' (or 'semantica[gpu]' for CUDA)"
|
||||
"FAISS not installed. Install with: pip install faiss-cpu or faiss-gpu"
|
||||
)
|
||||
|
||||
# Extract vectors and IDs
|
||||
|
||||
@@ -133,11 +133,7 @@ import importlib
|
||||
from typing import TYPE_CHECKING, Any, Dict, Tuple
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .salesforce_ingestor import (
|
||||
SalesforceConnector,
|
||||
SalesforceData,
|
||||
SalesforceIngestor,
|
||||
)
|
||||
from .salesforce_ingestor import SalesforceConnector, SalesforceData, SalesforceIngestor
|
||||
|
||||
from .config import IngestConfig, ingest_config
|
||||
from .file_ingestor import (
|
||||
@@ -252,42 +248,32 @@ _LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
|
||||
_OPTIONAL_DEPENDENCY_MESSAGES = {
|
||||
".repo_ingestor": (
|
||||
"Repository ingestion requires optional dependency 'GitPython'. "
|
||||
"Install it before importing RepoIngestor or using ingest_repository(). "
|
||||
"Install it with: pip install 'semantica[ingest-git]'"
|
||||
"Install it before importing RepoIngestor or using ingest_repository()."
|
||||
),
|
||||
".web_ingestor": (
|
||||
"Web ingestion requires optional dependency 'beautifulsoup4'. "
|
||||
"Install it before importing WebIngestor or using ingest_web(). "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
"Install it before importing WebIngestor or using ingest_web()."
|
||||
),
|
||||
".feed_ingestor": (
|
||||
"Feed ingestion requires optional dependency 'beautifulsoup4'. "
|
||||
"Install it before importing FeedIngestor or using ingest_feed(). "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
"Install it before importing FeedIngestor or using ingest_feed()."
|
||||
),
|
||||
".email_ingestor": (
|
||||
"Email ingestion requires optional dependency 'beautifulsoup4'. "
|
||||
"Install it before importing EmailIngestor or using ingest_email(). "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
),
|
||||
".xml_ingestor": (
|
||||
"XML ingestion requires optional dependency 'lxml'. "
|
||||
"Install it before importing XMLIngestor or using ingest_xml(). "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
"Install it before importing EmailIngestor or using ingest_email()."
|
||||
),
|
||||
".parquet_ingestor": (
|
||||
"Parquet ingestion requires optional dependency 'pyarrow'. "
|
||||
"Install it before importing ParquetIngestor or using ingest_parquet(). "
|
||||
"Install it with: pip install 'semantica[ingest-parquet]'"
|
||||
"Install it before importing ParquetIngestor or using ingest_parquet()."
|
||||
),
|
||||
".arrow_ingestor": (
|
||||
"Arrow ingestion requires optional dependency 'pyarrow'. "
|
||||
"Install it before importing ArrowIngestor or using ingest_arrow(). "
|
||||
"Install it with: pip install 'semantica[ingest-arrow]'"
|
||||
"Install it before importing ArrowIngestor or using ingest_arrow()."
|
||||
),
|
||||
".salesforce_ingestor": (
|
||||
"Salesforce ingestion requires optional dependency 'simple-salesforce'. "
|
||||
"Install it with: pip install 'semantica[db-salesforce]'"
|
||||
"Install it with: pip install \"semantica[db-salesforce]\" "
|
||||
"or: pip install simple-salesforce>=1.12.0"
|
||||
),
|
||||
}
|
||||
|
||||
@@ -300,55 +286,13 @@ def __getattr__(name: str) -> Any:
|
||||
module_name, attr_name = _LAZY_EXPORTS[name]
|
||||
try:
|
||||
module = importlib.import_module(module_name, __name__)
|
||||
except (ImportError, OSError) as exc:
|
||||
except ModuleNotFoundError as exc:
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
missing_name = getattr(exc, "name", None)
|
||||
if message and (
|
||||
missing_name is None
|
||||
or any(
|
||||
pkg in missing_name
|
||||
for pkg in ("git", "bs4", "pyarrow", "simple_salesforce", "lxml")
|
||||
)
|
||||
):
|
||||
if message and missing_name in {"git", "bs4", "pyarrow", "simple_salesforce"}:
|
||||
raise ImportError(message) from exc
|
||||
raise
|
||||
|
||||
# Guard against backends whose modules imported cleanly with dependencies
|
||||
# set to None; ensure probe imports (e.g. try: from semantica.ingest import ...)
|
||||
# fail at import time rather than postponing failure to construction time.
|
||||
if module_name == ".repo_ingestor" and name in {"RepoIngestor"}:
|
||||
if getattr(module, "git", None) is None:
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
if message:
|
||||
raise ImportError(message)
|
||||
|
||||
if module_name == ".xml_ingestor" and name in {"XMLIngestor"}:
|
||||
if getattr(module, "etree", None) is None:
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
if message:
|
||||
raise ImportError(message)
|
||||
|
||||
if module_name == ".parquet_ingestor" and name in {"ParquetIngestor"}:
|
||||
if not getattr(module, "PARQUET_AVAILABLE", True):
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
if message:
|
||||
raise ImportError(message)
|
||||
|
||||
if module_name == ".arrow_ingestor" and name in {"ArrowIngestor"}:
|
||||
if not getattr(module, "ARROW_AVAILABLE", True):
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
if message:
|
||||
raise ImportError(message)
|
||||
|
||||
if module_name == ".salesforce_ingestor" and name in {
|
||||
"SalesforceIngestor",
|
||||
"SalesforceConnector",
|
||||
}:
|
||||
if not getattr(module, "SALESFORCE_AVAILABLE", True):
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
if message:
|
||||
raise ImportError(message)
|
||||
|
||||
value = getattr(module, attr_name)
|
||||
globals()[name] = value
|
||||
return value
|
||||
|
||||
+129
-200
@@ -193,7 +193,6 @@ def _is_scp_like_repo_source(source: str) -> bool:
|
||||
"""Return True for scp-like SSH remotes (``user@host:path``)."""
|
||||
return bool(_SCP_LIKE_REPO_URL_RE.match(source.strip()))
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .api_ingestor import APIData
|
||||
from .arrow_ingestor import ArrowData
|
||||
@@ -253,12 +252,7 @@ def ingest_file(
|
||||
if custom_method and custom_method != ingest_file:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -324,18 +318,21 @@ def ingest_parquet(
|
||||
if custom_method and custom_method != ingest_parquet:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .parquet_ingestor import ParquetIngestor
|
||||
try:
|
||||
from .parquet_ingestor import ParquetIngestor
|
||||
except ModuleNotFoundError as exc:
|
||||
if _is_missing_dependency(exc, "pyarrow"):
|
||||
raise _missing_optional_dependency(
|
||||
"Parquet ingestion",
|
||||
"pyarrow",
|
||||
) from exc
|
||||
raise
|
||||
|
||||
config = ingest_config.get_method_config("parquet")
|
||||
config.update(kwargs)
|
||||
@@ -400,18 +397,21 @@ def ingest_arrow(
|
||||
if custom_method and custom_method != ingest_arrow:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .arrow_ingestor import ArrowIngestor
|
||||
try:
|
||||
from .arrow_ingestor import ArrowIngestor
|
||||
except ModuleNotFoundError as exc:
|
||||
if _is_missing_dependency(exc, "pyarrow"):
|
||||
raise _missing_optional_dependency(
|
||||
"Arrow ingestion",
|
||||
"pyarrow",
|
||||
) from exc
|
||||
raise
|
||||
|
||||
config = ingest_config.get_method_config("arrow")
|
||||
config.update(kwargs)
|
||||
@@ -481,12 +481,7 @@ def ingest_xml(
|
||||
if custom_method and custom_method != ingest_xml:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -496,10 +491,7 @@ def ingest_xml(
|
||||
|
||||
config = ingest_config.get_method_config("xml")
|
||||
config.update(kwargs)
|
||||
try:
|
||||
ingestor = XMLIngestor(**config)
|
||||
except ImportError as exc:
|
||||
raise _missing_optional_dependency("XML ingestion", "lxml") from exc
|
||||
ingestor = XMLIngestor(**config)
|
||||
|
||||
def _run_single(
|
||||
path: Union[str, Path],
|
||||
@@ -519,8 +511,6 @@ def ingest_xml(
|
||||
|
||||
return _run_single(source_path)
|
||||
|
||||
except ConfigurationError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to ingest XML: {e}")
|
||||
raise
|
||||
@@ -555,12 +545,7 @@ def ingest_web(
|
||||
if custom_method and custom_method != ingest_web:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -650,12 +635,7 @@ def ingest_public_api(
|
||||
if custom_method and custom_method != ingest_public_api:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -742,12 +722,7 @@ def ingest_feed(
|
||||
if custom_method and custom_method != ingest_feed:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -816,12 +791,7 @@ def ingest_stream(
|
||||
if custom_method and custom_method != ingest_stream:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -898,29 +868,26 @@ def ingest_repository(
|
||||
if custom_method and custom_method != ingest_repository:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .repo_ingestor import RepoIngestor
|
||||
try:
|
||||
from .repo_ingestor import RepoIngestor
|
||||
except ModuleNotFoundError as exc:
|
||||
if _is_missing_dependency(exc, "git"):
|
||||
raise _missing_optional_dependency(
|
||||
"Repository ingestion", "GitPython"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
# Get config
|
||||
config = ingest_config.get_method_config("repo")
|
||||
config.update(kwargs)
|
||||
|
||||
try:
|
||||
ingestor = RepoIngestor(**config)
|
||||
except ImportError as exc:
|
||||
raise _missing_optional_dependency(
|
||||
"Repository ingestion", "GitPython"
|
||||
) from exc
|
||||
ingestor = RepoIngestor(**config)
|
||||
|
||||
if method == "clone" or (
|
||||
isinstance(source, str)
|
||||
@@ -973,12 +940,7 @@ def ingest_email(
|
||||
if custom_method and custom_method != ingest_email:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -1057,12 +1019,7 @@ def ingest_ontology(
|
||||
if custom_method and custom_method != ingest_ontology:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -1128,12 +1085,7 @@ def ingest_database(
|
||||
if custom_method and custom_method != ingest_database:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -1281,122 +1233,105 @@ def ingest_salesforce(
|
||||
if custom_method and custom_method != ingest_salesforce:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source,
|
||||
fallback_on_custom_error=fallback, **kwargs,
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .salesforce_ingestor import SalesforceIngestor
|
||||
|
||||
# Unpack credential dict (if given); everything else stays in kwargs.
|
||||
creds: Dict[str, Any] = {}
|
||||
if source is not None:
|
||||
if not isinstance(source, dict):
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() source must be a credential dict or None. "
|
||||
"Pass sobject_name / soql as keyword arguments."
|
||||
)
|
||||
creds = dict(source)
|
||||
|
||||
# Merge any ingest_config method config under "salesforce".
|
||||
# get_method_config() now returns a copy, so this dict is safe to mutate.
|
||||
# We build the final connector config in order of increasing priority:
|
||||
# 1. base method config (lowest — global defaults set by operator)
|
||||
# 2. per-call credential dict supplied via `source`
|
||||
# 3. per-call connector params supplied as kwargs
|
||||
# Credentials are extracted from kwargs and removed so they don't also
|
||||
# flow into the ingest method call (which doesn't understand them).
|
||||
_CONNECTOR_PARAMS = frozenset(
|
||||
{
|
||||
"username",
|
||||
"password",
|
||||
"security_token",
|
||||
"domain",
|
||||
"instance_url",
|
||||
"session_id",
|
||||
"api_version",
|
||||
}
|
||||
)
|
||||
connector_kwargs = {k: v for k, v in kwargs.items() if k in _CONNECTOR_PARAMS}
|
||||
for k in _CONNECTOR_PARAMS:
|
||||
kwargs.pop(k, None)
|
||||
|
||||
# Build a fresh per-call config dict — never mutate the global store.
|
||||
config: Dict[str, Any] = {
|
||||
**ingest_config.get_method_config("salesforce"), # base (already a copy)
|
||||
**creds, # source dict credentials
|
||||
**connector_kwargs, # kwarg credentials
|
||||
}
|
||||
|
||||
try:
|
||||
ingestor = SalesforceIngestor(**config)
|
||||
except ImportError as exc:
|
||||
except ModuleNotFoundError as exc:
|
||||
if _is_missing_dependency(exc, "simple_salesforce"):
|
||||
raise _missing_optional_dependency(
|
||||
"Salesforce ingestion", "simple-salesforce"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
if method == "sobject":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='sobject' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
|
||||
elif method == "query":
|
||||
soql = kwargs.pop("soql", None)
|
||||
if not soql:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='query' requires "
|
||||
"soql keyword argument."
|
||||
)
|
||||
return ingestor.ingest_query(soql, **kwargs)
|
||||
|
||||
elif method == "list_sobjects":
|
||||
return ingestor.list_sobjects()
|
||||
|
||||
elif method == "schema":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='schema' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.get_sobject_schema(sobject_name)
|
||||
|
||||
elif method == "documents":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='documents' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
id_field = kwargs.pop("id_field", "Id")
|
||||
text_fields = kwargs.pop("text_fields", None)
|
||||
data = ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
return ingestor.export_as_documents(
|
||||
data, id_field=id_field, text_fields=text_fields
|
||||
)
|
||||
|
||||
else:
|
||||
# Unpack credential dict (if given); everything else stays in kwargs.
|
||||
creds: Dict[str, Any] = {}
|
||||
if source is not None:
|
||||
if not isinstance(source, dict):
|
||||
raise ProcessingError(
|
||||
f"Unknown ingest_salesforce method: {method!r}. "
|
||||
"Valid methods: 'sobject', 'query', 'list_sobjects', 'schema', "
|
||||
"'documents'."
|
||||
"ingest_salesforce() source must be a credential dict or None. "
|
||||
"Pass sobject_name / soql as keyword arguments."
|
||||
)
|
||||
creds = dict(source)
|
||||
|
||||
except ConfigurationError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to ingest salesforce: {e}")
|
||||
raise
|
||||
# Merge any ingest_config method config under "salesforce".
|
||||
# get_method_config() now returns a copy, so this dict is safe to mutate.
|
||||
# We build the final connector config in order of increasing priority:
|
||||
# 1. base method config (lowest — global defaults set by operator)
|
||||
# 2. per-call credential dict supplied via `source`
|
||||
# 3. per-call connector params supplied as kwargs
|
||||
# Credentials are extracted from kwargs and removed so they don't also
|
||||
# flow into the ingest method call (which doesn't understand them).
|
||||
_CONNECTOR_PARAMS = frozenset({
|
||||
"username", "password", "security_token", "domain",
|
||||
"instance_url", "session_id", "api_version",
|
||||
})
|
||||
connector_kwargs = {k: v for k, v in kwargs.items() if k in _CONNECTOR_PARAMS}
|
||||
for k in _CONNECTOR_PARAMS:
|
||||
kwargs.pop(k, None)
|
||||
|
||||
# Build a fresh per-call config dict — never mutate the global store.
|
||||
config: Dict[str, Any] = {
|
||||
**ingest_config.get_method_config("salesforce"), # base (already a copy)
|
||||
**creds, # source dict credentials
|
||||
**connector_kwargs, # kwarg credentials
|
||||
}
|
||||
|
||||
ingestor = SalesforceIngestor(**config)
|
||||
|
||||
if method == "sobject":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='sobject' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
|
||||
elif method == "query":
|
||||
soql = kwargs.pop("soql", None)
|
||||
if not soql:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='query' requires "
|
||||
"soql keyword argument."
|
||||
)
|
||||
return ingestor.ingest_query(soql, **kwargs)
|
||||
|
||||
elif method == "list_sobjects":
|
||||
return ingestor.list_sobjects()
|
||||
|
||||
elif method == "schema":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='schema' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.get_sobject_schema(sobject_name)
|
||||
|
||||
elif method == "documents":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='documents' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
id_field = kwargs.pop("id_field", "Id")
|
||||
text_fields = kwargs.pop("text_fields", None)
|
||||
data = ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
return ingestor.export_as_documents(data, id_field=id_field,
|
||||
text_fields=text_fields)
|
||||
|
||||
else:
|
||||
raise ProcessingError(
|
||||
f"Unknown ingest_salesforce method: {method!r}. "
|
||||
"Valid methods: 'sobject', 'query', 'list_sobjects', 'schema', "
|
||||
"'documents'."
|
||||
)
|
||||
|
||||
|
||||
def ingest_mcp(
|
||||
@@ -1464,12 +1399,7 @@ def ingest_mcp(
|
||||
if custom_method and custom_method != ingest_mcp:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -1708,9 +1638,8 @@ def ingest(
|
||||
elif source_type == "mcp":
|
||||
return {"data": ingest_mcp(sources, method=method or "resources", **kwargs)}
|
||||
elif source_type == "salesforce":
|
||||
return {
|
||||
"data": ingest_salesforce(sources, method=method or "sobject", **kwargs)
|
||||
}
|
||||
return {"data": ingest_salesforce(sources,
|
||||
method=method or "sobject", **kwargs)}
|
||||
else:
|
||||
raise ProcessingError(f"Unknown source type: {source_type}")
|
||||
|
||||
|
||||
@@ -28,29 +28,12 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import requests
|
||||
|
||||
try:
|
||||
from lxml import etree as lxml_etree
|
||||
|
||||
_SAFE_XML_PARSER = lxml_etree.XMLParser(
|
||||
resolve_entities=False,
|
||||
no_network=True,
|
||||
recover=False,
|
||||
huge_tree=False,
|
||||
load_dtd=False,
|
||||
remove_comments=True,
|
||||
remove_pis=True,
|
||||
)
|
||||
_LXML_SYNTAX_ERRORS: Tuple[type, ...] = (lxml_etree.XMLSyntaxError,)
|
||||
except (ImportError, OSError):
|
||||
lxml_etree = None
|
||||
_SAFE_XML_PARSER = None
|
||||
_LXML_SYNTAX_ERRORS = ()
|
||||
from lxml import etree as lxml_etree
|
||||
|
||||
try:
|
||||
from defusedxml import ElementTree as safe_xml_etree
|
||||
from defusedxml.common import DefusedXmlException
|
||||
except (ImportError, OSError): # pragma: no cover - fallback for minimal installs
|
||||
except ModuleNotFoundError: # pragma: no cover - fallback for minimal installs
|
||||
safe_xml_etree = None
|
||||
|
||||
class DefusedXmlException(Exception):
|
||||
@@ -88,6 +71,14 @@ AUTH_PARAM_NAMES = {
|
||||
"subscription-key",
|
||||
}
|
||||
|
||||
_SAFE_XML_PARSER = lxml_etree.XMLParser(
|
||||
resolve_entities=False,
|
||||
no_network=True,
|
||||
recover=False,
|
||||
huge_tree=False,
|
||||
load_dtd=False,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PublicAPIExample:
|
||||
@@ -452,9 +443,7 @@ class PublicAPIIngestor(RESTIngestor):
|
||||
APIData: Normalized public API response and metadata
|
||||
"""
|
||||
self._validate_endpoint(endpoint)
|
||||
self._validate_no_auth_request(
|
||||
headers=headers, params=params, options=options, endpoint=endpoint
|
||||
)
|
||||
self._validate_no_auth_request(headers=headers, params=params, options=options, endpoint=endpoint)
|
||||
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=endpoint,
|
||||
@@ -615,9 +604,7 @@ class PublicAPIIngestor(RESTIngestor):
|
||||
for endpoint in endpoints:
|
||||
try:
|
||||
results.append(
|
||||
self.ingest_public_api(
|
||||
endpoint, method=method, **copy.deepcopy(options)
|
||||
)
|
||||
self.ingest_public_api(endpoint, method=method, **copy.deepcopy(options))
|
||||
)
|
||||
except Exception as exc:
|
||||
self.logger.warning(f"Failed to fetch public API {endpoint}: {exc}")
|
||||
@@ -743,7 +730,7 @@ class PublicAPIIngestor(RESTIngestor):
|
||||
raise ProcessingError(
|
||||
f"Failed to parse {detected_format.upper()} public API response"
|
||||
) from exc
|
||||
except (DefusedXmlException, *_LXML_SYNTAX_ERRORS) as exc:
|
||||
except (DefusedXmlException, lxml_etree.XMLSyntaxError) as exc:
|
||||
raise ProcessingError("Failed to parse XML public API response") from exc
|
||||
|
||||
def _detect_response_format(
|
||||
@@ -783,40 +770,15 @@ class PublicAPIIngestor(RESTIngestor):
|
||||
def _parse_xml(self, xml_text: str) -> Dict[str, Any]:
|
||||
if safe_xml_etree is not None:
|
||||
root = safe_xml_etree.fromstring(xml_text)
|
||||
elif lxml_etree is not None and _SAFE_XML_PARSER is not None:
|
||||
else:
|
||||
root = lxml_etree.fromstring(
|
||||
xml_text.encode("utf-8"),
|
||||
parser=_SAFE_XML_PARSER,
|
||||
)
|
||||
for elem in root.iter():
|
||||
if (
|
||||
elem.tag is lxml_etree.Comment
|
||||
or elem.tag is lxml_etree.PI
|
||||
or getattr(elem.tag, "__name__", "")
|
||||
in ("Comment", "ProcessingInstruction", "PI")
|
||||
):
|
||||
continue
|
||||
if callable(elem.tag) or not isinstance(elem.tag, str):
|
||||
raise ProcessingError("Failed to parse XML public API response")
|
||||
else:
|
||||
raise ProcessingError(
|
||||
"XML parsing requires 'defusedxml' or 'lxml'. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
return self._element_to_dict(root)
|
||||
|
||||
def _element_to_dict(self, element: Any) -> Dict[str, Any]:
|
||||
children = [
|
||||
self._element_to_dict(child)
|
||||
for child in list(element)
|
||||
if not (
|
||||
callable(child.tag)
|
||||
or (
|
||||
lxml_etree is not None
|
||||
and (child.tag is lxml_etree.Comment or child.tag is lxml_etree.PI)
|
||||
)
|
||||
)
|
||||
]
|
||||
children = [self._element_to_dict(child) for child in list(element)]
|
||||
return {
|
||||
"tag": self._strip_namespace(element.tag),
|
||||
"attributes": {
|
||||
@@ -827,9 +789,7 @@ class PublicAPIIngestor(RESTIngestor):
|
||||
"children": children,
|
||||
}
|
||||
|
||||
def _strip_namespace(self, value: Any) -> str:
|
||||
if not isinstance(value, str):
|
||||
return str(value)
|
||||
def _strip_namespace(self, value: str) -> str:
|
||||
if value.startswith("{") and "}" in value:
|
||||
return value.split("}", 1)[1]
|
||||
return value
|
||||
|
||||
@@ -29,8 +29,6 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import os
|
||||
import re
|
||||
@@ -46,10 +44,7 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple, Union
|
||||
from urllib.parse import urlparse
|
||||
|
||||
try:
|
||||
import git
|
||||
except (ImportError, OSError):
|
||||
git = None
|
||||
import git
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -62,7 +57,9 @@ ALLOWED_CLONE_OPTIONS: Set[str] = {"depth", "branch", "single_branch", "no_tags"
|
||||
ALLOWED_REPO_URL_SCHEMES = frozenset({"https", "http", "git", "ssh"})
|
||||
# SCP-like SSH remotes: user@host:path/to/repo.git (no scheme)
|
||||
_SCP_LIKE_REPO_URL_RE = re.compile(r"^[^@\s]+@[^:\s]+:.+$")
|
||||
_ENV_VAR_TOKEN_RE = re.compile(r"\$(\{[A-Za-z_][A-Za-z0-9_]*\}|[A-Za-z_][A-Za-z0-9_]*)")
|
||||
_ENV_VAR_TOKEN_RE = re.compile(
|
||||
r"\$(\{[A-Za-z_][A-Za-z0-9_]*\}|[A-Za-z_][A-Za-z0-9_]*)"
|
||||
)
|
||||
# Short-lived DNS cache for host validation. This reduces repeated lookups but
|
||||
# does not eliminate DNS-rebinding / TOCTOU races between validate and clone —
|
||||
# network egress controls remain recommended.
|
||||
@@ -528,11 +525,6 @@ class RepoIngestor:
|
||||
**kwargs: Additional configuration parameters (merged into config)
|
||||
"""
|
||||
self.logger = get_logger("repo_ingestor")
|
||||
if git is None:
|
||||
raise ImportError(
|
||||
"GitPython is required for repository ingestion. "
|
||||
"Install it with: pip install 'semantica[ingest-git]'"
|
||||
)
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
|
||||
@@ -598,7 +590,10 @@ class RepoIngestor:
|
||||
networks). Those addresses are not SSRF-sensitive.
|
||||
"""
|
||||
return bool(
|
||||
ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_unspecified
|
||||
ip.is_private
|
||||
or ip.is_loopback
|
||||
or ip.is_link_local
|
||||
or ip.is_unspecified
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -625,7 +620,9 @@ class RepoIngestor:
|
||||
# or hanging lookup for one host cannot stall cache access for
|
||||
# concurrent lookups of other hosts.
|
||||
try:
|
||||
addrinfos = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)
|
||||
addrinfos = socket.getaddrinfo(
|
||||
host, None, type=socket.SOCK_STREAM
|
||||
)
|
||||
except socket.gaierror as exc:
|
||||
raise ValidationError(
|
||||
f"Cannot resolve repository host {host!r}: {exc}"
|
||||
@@ -798,7 +795,9 @@ class RepoIngestor:
|
||||
f"Allowed schemes: {sorted(ALLOWED_REPO_URL_SCHEMES)}"
|
||||
)
|
||||
if not parsed.netloc or not host:
|
||||
raise ValidationError(f"Repository URL must include a host: {repo_url}")
|
||||
raise ValidationError(
|
||||
f"Repository URL must include a host: {repo_url}"
|
||||
)
|
||||
|
||||
RepoIngestor._validate_repo_host(host)
|
||||
|
||||
@@ -813,7 +812,9 @@ class RepoIngestor:
|
||||
"include_extensions",
|
||||
"max_depth",
|
||||
}
|
||||
candidate = {k: v for k, v in options.items() if k not in non_git_options}
|
||||
candidate = {
|
||||
k: v for k, v in options.items() if k not in non_git_options
|
||||
}
|
||||
unsafe = set(candidate) - ALLOWED_CLONE_OPTIONS
|
||||
if unsafe:
|
||||
raise ValidationError(
|
||||
@@ -892,7 +893,9 @@ class RepoIngestor:
|
||||
if "include_extensions" in options:
|
||||
# Normalize extensions to include dot prefix
|
||||
exts = options["include_extensions"]
|
||||
normalized_exts = [e if e.startswith(".") else f".{e}" for e in exts]
|
||||
normalized_exts = [
|
||||
e if e.startswith(".") else f".{e}" for e in exts
|
||||
]
|
||||
file_filters["extensions"] = normalized_exts
|
||||
|
||||
# Process code files
|
||||
@@ -1059,14 +1062,14 @@ class RepoIngestor:
|
||||
return code_files
|
||||
|
||||
def get_repository_info(
|
||||
self, repo_url: str, repo: Optional[Any] = None
|
||||
self, repo_url: str, repo: Optional[git.Repo] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Get repository metadata and information.
|
||||
|
||||
Args:
|
||||
repo_url: Repository URL
|
||||
repo: Git repository object (git.Repo, optional)
|
||||
repo: Git repository object (optional)
|
||||
|
||||
Returns:
|
||||
dict: Repository information
|
||||
@@ -1108,7 +1111,6 @@ class RepoIngestor:
|
||||
def cleanup(self):
|
||||
"""Cleanup temporary repository files."""
|
||||
if self.temp_dir and os.path.exists(self.temp_dir):
|
||||
|
||||
def onexc(func, path, exc_info):
|
||||
"""
|
||||
Error handler for shutil.rmtree.
|
||||
@@ -1121,7 +1123,6 @@ class RepoIngestor:
|
||||
Usage : shutil.rmtree(path, onerror=onexc)
|
||||
"""
|
||||
import stat
|
||||
|
||||
if not os.access(path, os.W_OK):
|
||||
# Is the error an access error ?
|
||||
os.chmod(path, stat.S_IWUSR)
|
||||
|
||||
@@ -24,10 +24,7 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
try:
|
||||
from lxml import etree
|
||||
except (ImportError, OSError):
|
||||
etree = None
|
||||
from lxml import etree
|
||||
|
||||
from ..utils.constants import FILE_SIZE_LIMITS
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
@@ -75,11 +72,6 @@ class XMLIngestor:
|
||||
**kwargs: Additional configuration values
|
||||
"""
|
||||
self.logger = get_logger("xml_ingestor")
|
||||
if etree is None:
|
||||
raise ImportError(
|
||||
"lxml is required for XMLIngestor. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
@@ -792,8 +784,8 @@ class XMLIngestor:
|
||||
first_error = errors[0] if errors else "No detailed validation error available."
|
||||
return f"{prefix} for {source}: {first_error}"
|
||||
|
||||
def _format_xml_error(self, exc: Any) -> str:
|
||||
if hasattr(exc, "error_log") and exc.error_log:
|
||||
def _format_xml_error(self, exc: etree.XMLSyntaxError) -> str:
|
||||
if exc.error_log:
|
||||
return str(exc.error_log.last_error)
|
||||
return str(exc)
|
||||
|
||||
|
||||
@@ -141,7 +141,7 @@ class NodeEmbedder:
|
||||
|
||||
if method == "node2vec" and not GENSIM_AVAILABLE:
|
||||
raise ImportError(
|
||||
"gensim is required for Node2Vec. Install with: pip install 'semantica[graph-embeddings]'"
|
||||
"gensim is required for Node2Vec. Install with: pip install gensim"
|
||||
)
|
||||
|
||||
def compute_embeddings(
|
||||
|
||||
@@ -32,20 +32,12 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
try:
|
||||
from docx import Document
|
||||
from docx.document import Document as DocxDocument
|
||||
from docx.oxml.table import CT_Tbl
|
||||
from docx.oxml.text.paragraph import CT_P
|
||||
from docx.table import Table
|
||||
from docx.text.paragraph import Paragraph
|
||||
except (ImportError, OSError):
|
||||
Document = None
|
||||
DocxDocument = None
|
||||
CT_Tbl = None
|
||||
CT_P = None
|
||||
Table = None
|
||||
Paragraph = None
|
||||
from docx import Document
|
||||
from docx.document import Document as DocxDocument
|
||||
from docx.oxml.table import CT_Tbl
|
||||
from docx.oxml.text.paragraph import CT_P
|
||||
from docx.table import Table
|
||||
from docx.text.paragraph import Paragraph
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -92,9 +84,7 @@ class DOCXParser:
|
||||
self.config = config
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
def parse(
|
||||
self, file_path: Union[str, Path], pipeline_id: Optional[str] = None, **options
|
||||
) -> Dict[str, Any]:
|
||||
def parse(self, file_path: Union[str, Path], pipeline_id: Optional[str] = None, **options) -> Dict[str, Any]:
|
||||
"""
|
||||
Parse DOCX document.
|
||||
|
||||
@@ -109,12 +99,6 @@ class DOCXParser:
|
||||
Returns:
|
||||
dict: Parsed document data
|
||||
"""
|
||||
if Document is None:
|
||||
raise ProcessingError(
|
||||
"python-docx is required to parse DOCX files. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
|
||||
file_path = Path(file_path)
|
||||
|
||||
# Track DOCX parsing
|
||||
|
||||
@@ -33,11 +33,7 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import pandas as pd
|
||||
|
||||
try:
|
||||
from openpyxl import load_workbook
|
||||
except (ImportError, OSError):
|
||||
load_workbook = None
|
||||
from openpyxl import load_workbook
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -96,12 +92,6 @@ class ExcelParser:
|
||||
Returns:
|
||||
ExcelData or ExcelSheet: Parsed Excel data
|
||||
"""
|
||||
if load_workbook is None:
|
||||
raise ProcessingError(
|
||||
"openpyxl is required to parse Excel files. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
|
||||
file_path = Path(file_path)
|
||||
|
||||
# Track Excel parsing
|
||||
|
||||
@@ -33,10 +33,7 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from urllib.parse import urljoin
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except (ImportError, OSError):
|
||||
BeautifulSoup = None
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -116,12 +113,6 @@ class HTMLParser:
|
||||
Returns:
|
||||
HTMLData: Parsed HTML data
|
||||
"""
|
||||
if BeautifulSoup is None:
|
||||
raise ProcessingError(
|
||||
"beautifulsoup4 is required to parse HTML files. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
|
||||
# Track HTML parsing
|
||||
file_path = None
|
||||
if isinstance(html_content, Path) or (
|
||||
|
||||
+33
-99
@@ -123,6 +123,7 @@ Example Usage:
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ConfigurationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
|
||||
from .code_parser import CodeParser
|
||||
@@ -177,16 +178,10 @@ def parse_document(
|
||||
>>> text = parse_document("document.pdf", method="default", extract_text=True)
|
||||
"""
|
||||
custom_method = method_registry.get("document", method)
|
||||
if custom_method and custom_method != parse_document:
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
file_type,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, file_type, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -253,8 +248,7 @@ def parse_document_docling(
|
||||
|
||||
# Register Docling method
|
||||
try:
|
||||
from . import docling_parser # noqa: F401
|
||||
|
||||
from .docling_parser import DoclingParser
|
||||
method_registry.register("document", "docling", parse_document_docling)
|
||||
except (ImportError, OSError):
|
||||
# Docling not available, skip registration
|
||||
@@ -295,17 +289,10 @@ def parse_web_content(
|
||||
>>> html = parse_web_content("page.html", content_type="html", method="default")
|
||||
"""
|
||||
custom_method = method_registry.get("web", method)
|
||||
if custom_method and custom_method != parse_web_content:
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
content,
|
||||
content_type,
|
||||
base_url,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, content, content_type, base_url, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -358,16 +345,10 @@ def parse_structured_data(
|
||||
>>> csv_data = parse_structured_data("data.csv", data_format="csv", method="default")
|
||||
"""
|
||||
custom_method = method_registry.get("structured", method)
|
||||
if custom_method and custom_method != parse_structured_data:
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
data,
|
||||
data_format,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, data, data_format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -411,15 +392,10 @@ def parse_email(
|
||||
>>> headers = parse_email("email.eml", method="headers")
|
||||
"""
|
||||
custom_method = method_registry.get("email", method)
|
||||
if custom_method and custom_method != parse_email:
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
email_content,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, email_content, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -466,16 +442,10 @@ def parse_code(
|
||||
>>> structure = parse_code("script.py", method="ast")
|
||||
"""
|
||||
custom_method = method_registry.get("code", method)
|
||||
if custom_method and custom_method != parse_code:
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
language,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, language, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -525,16 +495,10 @@ def parse_media(
|
||||
>>> video = parse_media("video.mp4", method="default")
|
||||
"""
|
||||
custom_method = method_registry.get("media", method)
|
||||
if custom_method and custom_method != parse_media:
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
media_type,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, media_type, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -577,15 +541,10 @@ def parse_pdf(
|
||||
>>> pages = parse_pdf("document.pdf", method="default", pages=[1, 2, 3])
|
||||
"""
|
||||
custom_method = method_registry.get("document", method)
|
||||
if custom_method and custom_method not in (parse_pdf, parse_document):
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -626,15 +585,10 @@ def parse_docx(
|
||||
>>> docx = parse_docx("document.docx", method="default")
|
||||
"""
|
||||
custom_method = method_registry.get("document", method)
|
||||
if custom_method and custom_method not in (parse_docx, parse_document):
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -674,15 +628,10 @@ def parse_json(file_path: Union[str, Path], method: str = "default", **kwargs) -
|
||||
>>> flattened = parse_json("data.json", method="default", flatten=True)
|
||||
"""
|
||||
custom_method = method_registry.get("structured", method)
|
||||
if custom_method and custom_method not in (parse_json, parse_structured_data):
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -726,16 +675,10 @@ def parse_csv(
|
||||
>>> tab_separated = parse_csv("data.tsv", delimiter="\t", method="default")
|
||||
"""
|
||||
custom_method = method_registry.get("structured", method)
|
||||
if custom_method and custom_method not in (parse_csv, parse_structured_data):
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
delimiter,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, delimiter, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -771,15 +714,10 @@ def parse_xml(file_path: Union[str, Path], method: str = "default", **kwargs) ->
|
||||
>>> xml_data = parse_xml("data.xml", method="default")
|
||||
"""
|
||||
custom_method = method_registry.get("structured", method)
|
||||
if custom_method and custom_method not in (parse_xml, parse_structured_data):
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -824,15 +762,10 @@ def parse_image(
|
||||
>>> ocr_text = image.get("ocr_result", {}).get("text", "")
|
||||
"""
|
||||
custom_method = method_registry.get("media", method)
|
||||
if custom_method and custom_method not in (parse_image, parse_media):
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -859,9 +792,10 @@ def list_available_methods(task: Optional[str] = None) -> Dict[str, List[str]]:
|
||||
return method_registry.list_all(task)
|
||||
|
||||
|
||||
# NOTE: the built-in dispatchers (parse_document, parse_web_content, ...) must
|
||||
# NOT be registered under their own task's "default" method name. Each
|
||||
# dispatcher starts with method_registry.get(<task>, method) and would find
|
||||
# itself, re-entering infinitely until RecursionError. "default" is the
|
||||
# built-in code path and stays unregistered; users can still register their
|
||||
# own "default" (or any other name) to override it.
|
||||
# Register default methods
|
||||
method_registry.register("document", "default", parse_document)
|
||||
method_registry.register("web", "default", parse_web_content)
|
||||
method_registry.register("structured", "default", parse_structured_data)
|
||||
method_registry.register("email", "default", parse_email)
|
||||
method_registry.register("code", "default", parse_code)
|
||||
method_registry.register("media", "default", parse_media)
|
||||
|
||||
@@ -33,10 +33,7 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except (ImportError, OSError):
|
||||
BeautifulSoup = None
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -190,12 +187,6 @@ class HTMLContentParser(HTMLParser):
|
||||
}
|
||||
|
||||
# Load HTML for structure extraction
|
||||
if BeautifulSoup is None:
|
||||
raise ProcessingError(
|
||||
"beautifulsoup4 is required for HTML structure extraction. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
|
||||
if isinstance(html_content, Path) or (
|
||||
isinstance(html_content, str) and Path(html_content).exists()
|
||||
):
|
||||
@@ -252,12 +243,6 @@ class HTMLContentParser(HTMLParser):
|
||||
else:
|
||||
html_string = html_content
|
||||
|
||||
if BeautifulSoup is None:
|
||||
raise ProcessingError(
|
||||
"beautifulsoup4 is required for HTML cleaning. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
|
||||
soup = BeautifulSoup(html_string, "html.parser")
|
||||
|
||||
# Remove scripts and styles
|
||||
|
||||
@@ -34,10 +34,7 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
try:
|
||||
from lxml import etree
|
||||
except (ImportError, OSError):
|
||||
etree = None
|
||||
from lxml import etree
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -108,13 +105,7 @@ class XMLParser:
|
||||
)
|
||||
|
||||
try:
|
||||
explicit_engine = options.get("engine") or self.config.get("engine")
|
||||
engine = explicit_engine or ("lxml" if etree is not None else "etree")
|
||||
if engine == "lxml" and etree is None:
|
||||
raise ProcessingError(
|
||||
"lxml is required to parse XML with engine='lxml'. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
engine = options.get("engine", "lxml")
|
||||
|
||||
# Load XML content
|
||||
if file_path_obj:
|
||||
@@ -156,7 +147,7 @@ class XMLParser:
|
||||
self, xml_string: str, source: str, options: Dict[str, Any]
|
||||
) -> XMLData:
|
||||
"""Parse XML using lxml."""
|
||||
parser = etree.XMLParser(remove_blank_text=True, remove_comments=True)
|
||||
parser = etree.XMLParser(remove_blank_text=True)
|
||||
root = etree.fromstring(xml_string.encode("utf-8"), parser)
|
||||
|
||||
# Extract namespaces
|
||||
@@ -197,10 +188,8 @@ class XMLParser:
|
||||
metadata={"source": source, "engine": "etree"},
|
||||
)
|
||||
|
||||
def _element_to_xml_element(self, element) -> Optional[XMLElement]:
|
||||
def _element_to_xml_element(self, element) -> XMLElement:
|
||||
"""Convert lxml element to XMLElement."""
|
||||
if not hasattr(element, "tag") or not isinstance(element.tag, str):
|
||||
return None
|
||||
tag = element.tag
|
||||
if "}" in tag:
|
||||
namespace, tag = tag.split("}", 1)
|
||||
@@ -217,18 +206,12 @@ class XMLParser:
|
||||
|
||||
# Process children
|
||||
for child in element:
|
||||
child_elem = self._element_to_xml_element(child)
|
||||
if child_elem is not None:
|
||||
xml_elem.children.append(child_elem)
|
||||
xml_elem.children.append(self._element_to_xml_element(child))
|
||||
|
||||
return xml_elem
|
||||
|
||||
def _etree_element_to_xml_element(
|
||||
self, element: ET.Element
|
||||
) -> Optional[XMLElement]:
|
||||
def _etree_element_to_xml_element(self, element: ET.Element) -> XMLElement:
|
||||
"""Convert ElementTree element to XMLElement."""
|
||||
if not hasattr(element, "tag") or not isinstance(element.tag, str):
|
||||
return None
|
||||
tag = element.tag
|
||||
if "}" in tag:
|
||||
namespace, tag = tag.split("}", 1)
|
||||
@@ -245,9 +228,7 @@ class XMLParser:
|
||||
|
||||
# Process children
|
||||
for child in element:
|
||||
child_elem = self._etree_element_to_xml_element(child)
|
||||
if child_elem is not None:
|
||||
xml_elem.children.append(child_elem)
|
||||
xml_elem.children.append(self._etree_element_to_xml_element(child))
|
||||
|
||||
return xml_elem
|
||||
|
||||
@@ -268,30 +249,18 @@ class XMLParser:
|
||||
xml_data = self.parse(file_path, **options)
|
||||
|
||||
# Use lxml for XPath queries
|
||||
if etree is None:
|
||||
raise ProcessingError(
|
||||
"lxml is required for find_elements (XPath queries). "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
|
||||
xml_string = (
|
||||
file_path
|
||||
if isinstance(file_path, str) and not Path(file_path).exists()
|
||||
else Path(file_path).read_text(encoding="utf-8")
|
||||
else Path(file_path).read_text()
|
||||
)
|
||||
parser = etree.XMLParser(remove_blank_text=True, remove_comments=True)
|
||||
root = etree.fromstring(xml_string.encode("utf-8"), parser=parser)
|
||||
root = etree.fromstring(xml_string.encode("utf-8"))
|
||||
|
||||
# Register namespaces for XPath
|
||||
namespaces = xml_data.namespaces
|
||||
elements = root.xpath(xpath, namespaces=namespaces)
|
||||
|
||||
results = []
|
||||
for elem in elements:
|
||||
xml_elem = self._element_to_xml_element(elem)
|
||||
if xml_elem is not None:
|
||||
results.append(xml_elem)
|
||||
return results
|
||||
return [self._element_to_xml_element(elem) for elem in elements]
|
||||
|
||||
def extract_by_tag(
|
||||
self, file_path: Union[str, Path], tag_name: str, **options
|
||||
|
||||
@@ -34,11 +34,11 @@ Example Usage:
|
||||
>>> from semantica.semantic_extract import NamedEntityRecognizer
|
||||
>>> ner = NamedEntityRecognizer(confidence_threshold=0.7)
|
||||
>>> entities = ner.extract_entities("Steve Jobs founded Apple.")
|
||||
|
||||
|
||||
>>> from semantica.semantic_extract import RelationExtractor
|
||||
>>> rel_extractor = RelationExtractor(confidence_threshold=0.6)
|
||||
>>> relations = rel_extractor.extract_relations(text, entities=entities)
|
||||
|
||||
|
||||
>>> from semantica.semantic_extract import TripletExtractor
|
||||
>>> triplet_extractor = TripletExtractor(include_temporal=True)
|
||||
>>> triplets = triplet_extractor.extract_triplets(text)
|
||||
@@ -52,6 +52,7 @@ from __future__ import annotations
|
||||
import importlib
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
|
||||
_LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
|
||||
# Named Entity Recognition
|
||||
"NamedEntityRecognizer": (".named_entity_recognizer", "NamedEntityRecognizer"),
|
||||
@@ -91,10 +92,7 @@ _LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
|
||||
"RoleLabeler": (".semantic_analyzer", "RoleLabeler"),
|
||||
"SemanticClusterer": (".semantic_analyzer", "SemanticClusterer"),
|
||||
# Semantic Network
|
||||
"SemanticNetworkExtractor": (
|
||||
".semantic_network_extractor",
|
||||
"SemanticNetworkExtractor",
|
||||
),
|
||||
"SemanticNetworkExtractor": (".semantic_network_extractor", "SemanticNetworkExtractor"),
|
||||
"SemanticNode": (".semantic_network_extractor", "SemanticNode"),
|
||||
"SemanticEdge": (".semantic_network_extractor", "SemanticEdge"),
|
||||
"SemanticNetwork": (".semantic_network_extractor", "SemanticNetwork"),
|
||||
@@ -105,10 +103,6 @@ _LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
|
||||
# Validation
|
||||
"ExtractionValidator": (".extraction_validator", "ExtractionValidator"),
|
||||
"ValidationResult": (".extraction_validator", "ValidationResult"),
|
||||
# Schema-guided validation
|
||||
"ExtractionSchema": (".schema", "ExtractionSchema"),
|
||||
"Predicate": (".schema", "Predicate"),
|
||||
"SchemaValidator": (".schema_validator", "SchemaValidator"),
|
||||
# Providers
|
||||
"BaseProvider": (".providers", "BaseProvider"),
|
||||
"OpenAIProvider": (".providers", "OpenAIProvider"),
|
||||
@@ -145,7 +139,6 @@ def __getattr__(name: str) -> Any:
|
||||
globals()[name] = value
|
||||
return value
|
||||
|
||||
|
||||
__all__ = [
|
||||
# Named Entity Recognition
|
||||
"NamedEntityRecognizer",
|
||||
@@ -195,10 +188,6 @@ __all__ = [
|
||||
# Validation
|
||||
"ExtractionValidator",
|
||||
"ValidationResult",
|
||||
# Schema-guided validation
|
||||
"ExtractionSchema",
|
||||
"Predicate",
|
||||
"SchemaValidator",
|
||||
# Providers
|
||||
"BaseProvider",
|
||||
"OpenAIProvider",
|
||||
|
||||
@@ -209,11 +209,6 @@ def load_spacy_model(name: str):
|
||||
Raises whatever ``spacy.load`` raises (``OSError`` for a missing model), so
|
||||
callers keep their existing fallback behavior.
|
||||
"""
|
||||
if spacy is None:
|
||||
raise ImportError(
|
||||
"spaCy is not installed. Install with: pip install 'semantica[nlp-spacy]'"
|
||||
)
|
||||
|
||||
cached = _spacy_model_cache.get(name)
|
||||
if cached is not None and cached[0] is spacy:
|
||||
return cached[1]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,234 +0,0 @@
|
||||
"""Domain schema view over an ontology, for schema-guided extraction.
|
||||
|
||||
An :class:`ExtractionSchema` is a lightweight, read-only view over a domain
|
||||
ontology: the set of allowed *concept* names (entity labels) and the allowed
|
||||
*predicates* with optional ``domain`` / ``range`` constraints. It is what
|
||||
:class:`~semantica.semantic_extract.schema_validator.SchemaValidator` checks
|
||||
extraction output against.
|
||||
|
||||
The schema deliberately **reuses the project's existing OWL ontology
|
||||
representation** instead of introducing a parallel "template" type. Build one
|
||||
from the dict produced by :func:`semantica.ontology.generate_ontology`
|
||||
(:meth:`ExtractionSchema.from_ontology`), or from an OWL / Turtle file or string
|
||||
(:meth:`ExtractionSchema.from_owl`).
|
||||
|
||||
An empty ``domain`` / ``range`` means "unconstrained", matching OWL's convention
|
||||
that an object property with no ``rdfs:domain`` / ``rdfs:range`` places no
|
||||
restriction on its subjects / objects.
|
||||
|
||||
Reference
|
||||
---------
|
||||
Using an ontology to constrain what may be extracted is the defining idea of
|
||||
ontology-based information extraction (OBIE): Wimalasuriya & Dou, "Ontology-Based
|
||||
Information Extraction: An Introduction and a Survey" (2010).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, FrozenSet, Iterable, Mapping, Optional, Set
|
||||
|
||||
|
||||
def _as_name_set(value: Any) -> Set[str]:
|
||||
"""Coerce a ``domain`` / ``range`` value to a set of concept names.
|
||||
|
||||
Accepts a string, an iterable of strings / mappings, a mapping (reads its
|
||||
``name`` / ``label``), or ``None``. ``None`` / empty yields an empty set,
|
||||
interpreted downstream as "unconstrained".
|
||||
"""
|
||||
if value is None:
|
||||
return set()
|
||||
if isinstance(value, str):
|
||||
return {value}
|
||||
if isinstance(value, Mapping):
|
||||
name = value.get("name") or value.get("label")
|
||||
return {str(name)} if name else {str(k) for k in value}
|
||||
if isinstance(value, Iterable):
|
||||
out: Set[str] = set()
|
||||
for item in value:
|
||||
out |= _as_name_set(item)
|
||||
return out
|
||||
return {str(value)}
|
||||
|
||||
|
||||
_OWL_THING = {
|
||||
"owl:Thing",
|
||||
"Thing",
|
||||
"http://www.w3.org/2002/07/owl#Thing",
|
||||
}
|
||||
|
||||
|
||||
def _drop_thing(names: Set[str]) -> Set[str]:
|
||||
"""Collapse an ``owl:Thing`` domain / range to *unconstrained* (empty set).
|
||||
|
||||
``owl:Thing`` is the universal class, so a property whose ``domain`` / ``range``
|
||||
is ``owl:Thing`` places no restriction. ``OntologyGenerator`` emits it as the
|
||||
fallback when it cannot resolve endpoint types; keeping it as a literal
|
||||
``{"Thing"}`` constraint would reject every real endpoint, so we treat its
|
||||
presence as "any concept".
|
||||
"""
|
||||
return set() if names & _OWL_THING else names
|
||||
|
||||
|
||||
def _constraint_set(value: Any) -> Set[str]:
|
||||
"""A ``domain`` / ``range`` constraint set, with ``owl:Thing`` meaning unconstrained."""
|
||||
return _drop_thing(_as_name_set(value))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Predicate:
|
||||
"""An allowed predicate with optional ``domain`` / ``range`` constraints.
|
||||
|
||||
Empty ``domain`` / ``range`` means any concept is allowed in that position.
|
||||
"""
|
||||
|
||||
name: str
|
||||
domain: FrozenSet[str] = frozenset()
|
||||
range: FrozenSet[str] = frozenset()
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractionSchema:
|
||||
"""Read-only view over a domain ontology used to gate extraction.
|
||||
|
||||
Names are matched **exactly**: the schema vocabulary and the extraction labels
|
||||
must share a normalization convention. ``OntologyGenerator`` normalizes concept
|
||||
names to PascalCase and predicate names to camelCase, so entity labels /
|
||||
relation predicates validated against a generated schema should follow the same
|
||||
convention (e.g. label entities ``Person`` rather than ``person``).
|
||||
"""
|
||||
|
||||
concepts: FrozenSet[str] = field(default_factory=frozenset)
|
||||
predicates: Dict[str, Predicate] = field(default_factory=dict)
|
||||
|
||||
# ---- constructors -------------------------------------------------
|
||||
|
||||
@classmethod
|
||||
def from_ontology(cls, ontology: Any) -> "ExtractionSchema":
|
||||
"""Build a schema from a ``generate_ontology``-style ontology.
|
||||
|
||||
Accepts the mapping returned by :func:`semantica.ontology.generate_ontology`,
|
||||
or an object exposing such a mapping via a ``.data`` attribute — e.g. the
|
||||
``OntologyData`` returned by ``semantica.ingest.OntologyIngestor``.
|
||||
|
||||
Reads ``ontology["classes"]`` (each carrying a ``name`` / ``label``) as
|
||||
concepts and ``ontology["properties"]`` (each carrying a ``name`` and
|
||||
optional ``domain`` / ``range``) as predicates. Missing ``domain`` /
|
||||
``range`` means unconstrained; unrecognised keys are ignored.
|
||||
|
||||
Endpoint types named in a property's ``domain`` / ``range`` are also folded
|
||||
into the concept set (consistent with :meth:`from_owl`), so a type referenced
|
||||
only as an endpoint — e.g. one that didn't clear the class-frequency gate
|
||||
during induction — is still a known concept.
|
||||
"""
|
||||
if not isinstance(ontology, Mapping) and hasattr(ontology, "data"):
|
||||
ontology = ontology.data # unwrap OntologyData-like objects
|
||||
|
||||
concepts: Set[str] = set()
|
||||
for c in ontology.get("classes", []) or []:
|
||||
name = (c.get("name") or c.get("label")) if isinstance(c, Mapping) else c
|
||||
if name:
|
||||
concepts.add(str(name))
|
||||
|
||||
predicates: Dict[str, Predicate] = {}
|
||||
for p in ontology.get("properties", []) or []:
|
||||
if not isinstance(p, Mapping):
|
||||
continue
|
||||
name = p.get("name") or p.get("label")
|
||||
if not name:
|
||||
continue
|
||||
domain = _constraint_set(p.get("domain"))
|
||||
rng = _constraint_set(p.get("range"))
|
||||
predicates[str(name)] = Predicate(
|
||||
name=str(name),
|
||||
domain=frozenset(domain),
|
||||
range=frozenset(rng),
|
||||
)
|
||||
concepts |= domain | rng
|
||||
return cls(concepts=frozenset(concepts), predicates=predicates)
|
||||
|
||||
@classmethod
|
||||
def from_owl(
|
||||
cls, source: str, *, format: Optional[str] = None
|
||||
) -> "ExtractionSchema":
|
||||
"""Build a schema from an OWL / RDF file path or serialized string.
|
||||
|
||||
``owl:Class`` / ``rdfs:Class`` become concepts; ``owl:ObjectProperty`` with
|
||||
``rdfs:domain`` / ``rdfs:range`` becomes a predicate (its domain / range
|
||||
names are folded into the concept set, with ``owl:Thing`` treated as
|
||||
unconstrained). Names prefer an explicit ``rdfs:label``, falling back to the
|
||||
URI's local name, so the vocabulary matches :meth:`from_ontology`. Requires
|
||||
``rdflib`` (an existing project dependency).
|
||||
"""
|
||||
from rdflib import OWL, RDF, RDFS, Graph, URIRef
|
||||
|
||||
graph = Graph()
|
||||
if os.path.exists(source):
|
||||
graph.parse(source, format=format)
|
||||
else:
|
||||
graph.parse(data=source, format=format or "turtle")
|
||||
|
||||
def _local(term: Any) -> str:
|
||||
text = str(term)
|
||||
for sep in ("#", "/"):
|
||||
if sep in text:
|
||||
text = text.rsplit(sep, 1)[-1]
|
||||
return text
|
||||
|
||||
def _name_of(term: Any) -> str:
|
||||
label = graph.value(term, RDFS.label)
|
||||
return str(label) if label is not None else _local(term)
|
||||
|
||||
concepts: Set[str] = {
|
||||
_name_of(c)
|
||||
for class_type in (OWL.Class, RDFS.Class)
|
||||
for c in graph.subjects(RDF.type, class_type)
|
||||
if isinstance(c, URIRef)
|
||||
}
|
||||
predicates: Dict[str, Predicate] = {}
|
||||
for prop in graph.subjects(RDF.type, OWL.ObjectProperty):
|
||||
name = _name_of(prop)
|
||||
domain = _drop_thing(
|
||||
{_name_of(d) for d in graph.objects(prop, RDFS.domain)}
|
||||
)
|
||||
rng = _drop_thing({_name_of(r) for r in graph.objects(prop, RDFS.range)})
|
||||
predicates[name] = Predicate(
|
||||
name=name, domain=frozenset(domain), range=frozenset(rng)
|
||||
)
|
||||
concepts |= domain | rng
|
||||
return cls(concepts=frozenset(concepts), predicates=predicates)
|
||||
|
||||
# ---- queries ------------------------------------------------------
|
||||
|
||||
def has_concept(self, name: str) -> bool:
|
||||
"""Whether ``name`` is an allowed concept (entity label)."""
|
||||
return name in self.concepts
|
||||
|
||||
def has_predicate(self, name: str) -> bool:
|
||||
"""Whether ``name`` is an allowed predicate."""
|
||||
return name in self.predicates
|
||||
|
||||
def allows_relation(
|
||||
self, subject_label: str, predicate: str, object_label: str
|
||||
) -> bool:
|
||||
"""Whether a relation conforms to the schema.
|
||||
|
||||
True iff subject and object are known concepts, the predicate is known,
|
||||
and subject / object satisfy the predicate's ``domain`` / ``range``
|
||||
(an empty ``domain`` / ``range`` allows any concept).
|
||||
|
||||
Membership is exact; ``subClassOf`` hierarchies are not traversed, so a
|
||||
subclass endpoint is not accepted for a superclass ``domain`` / ``range``
|
||||
(subclass-aware validation is a possible follow-up).
|
||||
"""
|
||||
if subject_label not in self.concepts or object_label not in self.concepts:
|
||||
return False
|
||||
pred = self.predicates.get(predicate)
|
||||
if pred is None:
|
||||
return False
|
||||
if pred.domain and subject_label not in pred.domain:
|
||||
return False
|
||||
if pred.range and object_label not in pred.range:
|
||||
return False
|
||||
return True
|
||||
@@ -1,196 +0,0 @@
|
||||
"""Schema-guided validation for semantic extractions.
|
||||
|
||||
:class:`SchemaValidator` is a sibling of
|
||||
:class:`~semantica.semantic_extract.extraction_validator.ExtractionValidator`:
|
||||
same two entry points (:meth:`validate_entities` / :meth:`validate_relations`)
|
||||
returning the same :class:`ValidationResult`, so the two compose back-to-back.
|
||||
|
||||
The two validators check **orthogonal** axes. ``ExtractionValidator`` checks
|
||||
*confidence* and structural sanity; ``SchemaValidator`` checks *conformance to a
|
||||
domain ontology*:
|
||||
|
||||
* every entity label must be a concept in the schema;
|
||||
* every relation predicate must be in the schema and satisfy its ``domain`` /
|
||||
``range``.
|
||||
|
||||
This is the deterministic core of ontology-based information extraction (OBIE,
|
||||
Wimalasuriya & Dou 2010): it needs no LLM and is fully unit-testable.
|
||||
:meth:`validate_entities` / :meth:`validate_relations` *report* conformance;
|
||||
:meth:`filter_by_schema` / :meth:`filter_relations_by_schema` return the
|
||||
conforming subset (mirroring ``ExtractionValidator.filter_by_confidence``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from .extraction_validator import ValidationResult
|
||||
from .ner_extractor import Entity
|
||||
from .relation_extractor import Relation
|
||||
from .schema import ExtractionSchema
|
||||
|
||||
|
||||
class SchemaValidator:
|
||||
"""Validate extractions against a domain ontology (:class:`ExtractionSchema`)."""
|
||||
|
||||
def __init__(
|
||||
self, schema: ExtractionSchema, method: Optional[str] = None, **config: Any
|
||||
) -> None:
|
||||
"""Initialize the validator.
|
||||
|
||||
Args:
|
||||
schema: The domain ontology view to validate against.
|
||||
method: Reserved for future method-specific validation (unused),
|
||||
mirroring ``ExtractionValidator``.
|
||||
**config: Reserved configuration options.
|
||||
"""
|
||||
self.schema = schema
|
||||
self.method = method
|
||||
self.config = config
|
||||
|
||||
def validate_entities(
|
||||
self, entities: Union[List[Entity], List[List[Entity]]], **options: Any
|
||||
) -> Union[ValidationResult, List[ValidationResult]]:
|
||||
"""Validate that entity labels are concepts in the schema.
|
||||
|
||||
Handles both a single list and a batch (list of lists), like
|
||||
``ExtractionValidator.validate_entities``.
|
||||
"""
|
||||
if entities and isinstance(entities, list) and isinstance(entities[0], list):
|
||||
results = []
|
||||
for idx, batch in enumerate(entities):
|
||||
res = self.validate_entities(batch, **options)
|
||||
if "batch_index" not in res.metadata:
|
||||
res.metadata["batch_index"] = idx
|
||||
results.append(res)
|
||||
return results
|
||||
|
||||
errors: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
out_of_vocab = [e for e in entities if not self.schema.has_concept(e.label)]
|
||||
unknown_labels = sorted({e.label for e in out_of_vocab})
|
||||
if out_of_vocab:
|
||||
errors.append(
|
||||
f"{len(out_of_vocab)} entities with labels outside the schema: "
|
||||
f"{', '.join(unknown_labels)}"
|
||||
)
|
||||
|
||||
total = len(entities)
|
||||
conforming = total - len(out_of_vocab)
|
||||
metrics = {
|
||||
"total_entities": total,
|
||||
"in_vocabulary": conforming,
|
||||
"out_of_vocabulary": len(out_of_vocab),
|
||||
"unknown_labels": unknown_labels,
|
||||
"schema_concepts": len(self.schema.concepts),
|
||||
}
|
||||
score = conforming / total if total else 1.0
|
||||
|
||||
return ValidationResult(
|
||||
valid=not errors,
|
||||
score=score,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
metrics=metrics,
|
||||
metadata=self._metadata(entities),
|
||||
)
|
||||
|
||||
def validate_relations(
|
||||
self, relations: Union[List[Relation], List[List[Relation]]], **options: Any
|
||||
) -> Union[ValidationResult, List[ValidationResult]]:
|
||||
"""Validate relation predicates and ``domain`` / ``range`` against the schema.
|
||||
|
||||
Handles both a single list and a batch (list of lists), like
|
||||
``ExtractionValidator.validate_relations``.
|
||||
"""
|
||||
if relations and isinstance(relations, list) and isinstance(relations[0], list):
|
||||
results = []
|
||||
for idx, batch in enumerate(relations):
|
||||
res = self.validate_relations(batch, **options)
|
||||
if "batch_index" not in res.metadata:
|
||||
res.metadata["batch_index"] = idx
|
||||
results.append(res)
|
||||
return results
|
||||
|
||||
errors: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
# Guard malformed relations (missing subject/object) before dereferencing
|
||||
# their endpoints, matching ExtractionValidator's own leniency.
|
||||
malformed = [r for r in relations if not r.subject or not r.object]
|
||||
well_formed = [r for r in relations if r.subject and r.object]
|
||||
|
||||
unknown_predicate = [
|
||||
r for r in well_formed if not self.schema.has_predicate(r.predicate)
|
||||
]
|
||||
dr_violation = [
|
||||
r
|
||||
for r in well_formed
|
||||
if self.schema.has_predicate(r.predicate)
|
||||
and not self.schema.allows_relation(
|
||||
r.subject.label, r.predicate, r.object.label
|
||||
)
|
||||
]
|
||||
if malformed:
|
||||
errors.append(f"{len(malformed)} relations missing a subject or object")
|
||||
if unknown_predicate:
|
||||
preds = sorted({r.predicate for r in unknown_predicate})
|
||||
errors.append(
|
||||
f"{len(unknown_predicate)} relations with predicates outside the "
|
||||
f"schema: {', '.join(preds)}"
|
||||
)
|
||||
if dr_violation:
|
||||
errors.append(
|
||||
f"{len(dr_violation)} relations violating domain/range constraints"
|
||||
)
|
||||
|
||||
total = len(relations)
|
||||
conforming = total - len(malformed) - len(unknown_predicate) - len(dr_violation)
|
||||
metrics = {
|
||||
"total_relations": total,
|
||||
"conforming": conforming,
|
||||
"malformed": len(malformed),
|
||||
"unknown_predicate": len(unknown_predicate),
|
||||
"domain_range_violation": len(dr_violation),
|
||||
"schema_predicates": len(self.schema.predicates),
|
||||
}
|
||||
score = conforming / total if total else 1.0
|
||||
|
||||
return ValidationResult(
|
||||
valid=not errors,
|
||||
score=score,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
metrics=metrics,
|
||||
metadata=self._metadata(relations),
|
||||
)
|
||||
|
||||
def filter_by_schema(self, entities: List[Entity]) -> List[Entity]:
|
||||
"""Return only entities whose label is a concept in the schema."""
|
||||
return [e for e in entities if self.schema.has_concept(e.label)]
|
||||
|
||||
def filter_relations_by_schema(self, relations: List[Relation]) -> List[Relation]:
|
||||
"""Return only relations that fully conform to the schema."""
|
||||
return [
|
||||
r
|
||||
for r in relations
|
||||
if r.subject
|
||||
and r.object
|
||||
and self.schema.has_predicate(r.predicate)
|
||||
and self.schema.allows_relation(
|
||||
r.subject.label, r.predicate, r.object.label
|
||||
)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _metadata(items: List[Any]) -> Dict[str, Any]:
|
||||
"""Carry ``batch_index`` / ``document_id`` through, like ``ExtractionValidator``."""
|
||||
metadata: Dict[str, Any] = {}
|
||||
if items:
|
||||
first = items[0]
|
||||
if getattr(first, "metadata", None):
|
||||
for key in ("batch_index", "document_id"):
|
||||
if key in first.metadata:
|
||||
metadata[key] = first.metadata[key]
|
||||
return metadata
|
||||
+13
-21
@@ -38,15 +38,15 @@ Example Usage:
|
||||
>>> from semantica.utils import clean_text, normalize_entities
|
||||
>>> cleaned = clean_text(" Hello World ")
|
||||
>>> entities = normalize_entities([{"id": "e1", "text": "John", "type": "PERSON"}])
|
||||
>>>
|
||||
>>>
|
||||
>>> from semantica.utils import hash_data, safe_filename
|
||||
>>> data_hash = hash_data({"key": "value"})
|
||||
>>> safe_name = safe_filename("my file.txt")
|
||||
>>>
|
||||
>>>
|
||||
>>> from semantica.utils import merge_dicts, get_nested_value
|
||||
>>> merged = merge_dicts({"a": 1}, {"b": 2}, deep=True)
|
||||
>>> value = get_nested_value(config, "database.host", default="localhost")
|
||||
>>>
|
||||
>>>
|
||||
>>> from semantica.utils import retry_on_error
|
||||
>>> @retry_on_error(max_retries=3, delay=1.0)
|
||||
... def fetch_data():
|
||||
@@ -457,9 +457,7 @@ def chunk_list(items: List[Any], chunk_size: int) -> List[List[Any]]:
|
||||
Returns:
|
||||
List of chunks
|
||||
"""
|
||||
return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)]
|
||||
|
||||
|
||||
return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)]
|
||||
def flatten_dict(
|
||||
d: Dict[str, Any], parent_key: str = "", sep: str = "."
|
||||
) -> Dict[str, Any]:
|
||||
@@ -558,27 +556,27 @@ def safe_import(
|
||||
) -> Tuple[Any, bool]:
|
||||
"""
|
||||
Safely import an optional module, handling both ImportError and OSError.
|
||||
|
||||
|
||||
This is useful for optional dependencies that may fail to import due to:
|
||||
- Missing package (ImportError)
|
||||
- DLL loading failures on Windows, e.g., PyTorch (OSError)
|
||||
|
||||
|
||||
Args:
|
||||
module_name: Name of the module to import (e.g., "spacy", "docling.document_converter")
|
||||
package: Optional package name for relative imports
|
||||
default: Default value to return if import fails
|
||||
error_message: Optional custom error message for logging
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (module_or_default, success_flag):
|
||||
- If import succeeds: (imported_module, True)
|
||||
- If import fails: (default, False)
|
||||
|
||||
|
||||
Example:
|
||||
>>> spacy, available = safe_import("spacy")
|
||||
>>> if available:
|
||||
... doc = spacy.load("en_core_web_sm")
|
||||
>>>
|
||||
>>>
|
||||
>>> converter, available = safe_import("docling.document_converter", default=None)
|
||||
>>> if available:
|
||||
... converter = converter()
|
||||
@@ -589,13 +587,11 @@ def safe_import(
|
||||
else:
|
||||
module = importlib.import_module(module_name)
|
||||
return module, True
|
||||
except (ImportError, OSError) as e:
|
||||
except (ImportError, ModuleNotFoundError, OSError) as e:
|
||||
if error_message:
|
||||
import sys
|
||||
|
||||
if "logging" in sys.modules:
|
||||
from .logging import get_logger
|
||||
|
||||
logger = get_logger("utils.helpers")
|
||||
logger.debug(f"{error_message}: {e}")
|
||||
return default, False
|
||||
@@ -811,13 +807,9 @@ def _is_record(value: Any) -> bool:
|
||||
exporters rather than a ``ValidationError`` at the boundary where the
|
||||
problem is visible.
|
||||
"""
|
||||
return (
|
||||
isinstance(value, Mapping)
|
||||
or is_dataclass(value)
|
||||
or (
|
||||
hasattr(value, "__dict__")
|
||||
and not isinstance(value, (types.ModuleType, type))
|
||||
)
|
||||
return isinstance(value, Mapping) or is_dataclass(value) or (
|
||||
hasattr(value, "__dict__")
|
||||
and not isinstance(value, (types.ModuleType, type))
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -343,9 +343,7 @@ class FAISSIndex:
|
||||
was originally saved.
|
||||
"""
|
||||
if not FAISS_AVAILABLE:
|
||||
raise ProcessingError(
|
||||
"FAISS not available. Install with: pip install 'semantica[vectorstore-faiss]' (or 'semantica[gpu]' for CUDA)"
|
||||
)
|
||||
raise ProcessingError("FAISS not available")
|
||||
|
||||
path = Path(path)
|
||||
index = faiss.read_index(str(path))
|
||||
@@ -484,7 +482,7 @@ class FAISSIndexBuilder:
|
||||
"""
|
||||
if not FAISS_AVAILABLE:
|
||||
raise ProcessingError(
|
||||
"FAISS is not available. Install it with: pip install 'semantica[vectorstore-faiss]' (or 'semantica[gpu]' for CUDA)"
|
||||
"FAISS is not available. Install it with: pip install faiss-cpu or faiss-gpu"
|
||||
)
|
||||
|
||||
# Create index based on type
|
||||
@@ -556,7 +554,7 @@ class FAISSStore:
|
||||
# Check FAISS availability
|
||||
if not FAISS_AVAILABLE:
|
||||
self.logger.warning(
|
||||
"FAISS not available. Install with: pip install 'semantica[vectorstore-faiss]' (or 'semantica[gpu]' for CUDA)"
|
||||
"FAISS not available. Install with: pip install faiss-cpu or faiss-gpu"
|
||||
)
|
||||
|
||||
def create_index(
|
||||
@@ -755,9 +753,7 @@ class FAISSStore:
|
||||
FAISSIndex instance
|
||||
"""
|
||||
if not FAISS_AVAILABLE:
|
||||
raise ProcessingError(
|
||||
"FAISS not available. Install with: pip install 'semantica[vectorstore-faiss]' (or 'semantica[gpu]' for CUDA)"
|
||||
)
|
||||
raise ProcessingError("FAISS not available")
|
||||
|
||||
path = Path(path)
|
||||
if path.exists() and not _metadata_path(path).exists():
|
||||
|
||||
@@ -153,12 +153,9 @@ class QdrantCollection:
|
||||
raise ProcessingError("Qdrant not available")
|
||||
|
||||
try:
|
||||
# qdrant-client >=1.10.0: query_points() supersedes the removed search().
|
||||
# It returns a QueryResponse whose .points attribute is a list of
|
||||
# ScoredPoint objects (id, score, payload, …).
|
||||
response = self.client.query_points(
|
||||
search_results = self.client.search(
|
||||
collection_name=self.collection_name,
|
||||
query=query_vector.tolist(),
|
||||
query_vector=query_vector.tolist(),
|
||||
limit=limit,
|
||||
query_filter=query_filter,
|
||||
with_payload=True,
|
||||
@@ -167,19 +164,19 @@ class QdrantCollection:
|
||||
)
|
||||
|
||||
results = []
|
||||
for point in response.points:
|
||||
for result in search_results:
|
||||
results.append(
|
||||
{
|
||||
"id": point.id,
|
||||
"id": result.id,
|
||||
# See pinecone_store.py PineconeIndex.search_vectors for why
|
||||
# this uses x/(1+|x|) rather than clamping distance-to-zero:
|
||||
# Qdrant's Dot distance metric is unbounded, and the old
|
||||
# clamped formula collapsed every score >= 1.0 to 1.0.
|
||||
"score": (
|
||||
float(point.score) / (1.0 + abs(float(point.score))) + 1.0
|
||||
float(result.score) / (1.0 + abs(float(result.score))) + 1.0
|
||||
)
|
||||
/ 2.0,
|
||||
"metadata": point.payload or {},
|
||||
"metadata": result.payload or {},
|
||||
"vector": None,
|
||||
"distance": None,
|
||||
}
|
||||
@@ -384,27 +381,6 @@ class QdrantStore:
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to get collection: {str(e)}")
|
||||
|
||||
def _ensure_default_collection(self, dim: int = 384) -> QdrantCollection:
|
||||
"""Lazily attach the configured collection, creating it on first use.
|
||||
|
||||
Mirrors FAISSStore's automatic index creation so the VectorStore
|
||||
facade can read/write without an explicit create_collection() call.
|
||||
Reuses the existing collection if a previous process created it.
|
||||
"""
|
||||
# ``collection_name`` is the option the VectorStore facade and the
|
||||
# docs pass through; accept the legacy ``collection`` spelling too.
|
||||
name = (
|
||||
self.config.get("collection_name")
|
||||
or self.config.get("collection")
|
||||
or "semantica_default"
|
||||
)
|
||||
try:
|
||||
self.create_collection(name, vector_size=dim)
|
||||
except ProcessingError:
|
||||
self.get_collection(name)
|
||||
self.logger.info(f"Auto-initialized Qdrant collection '{name}' (dim={dim})")
|
||||
return self.collection
|
||||
|
||||
def insert_vectors(
|
||||
self,
|
||||
vectors: List[Union[np.ndarray, List[float]]],
|
||||
@@ -424,14 +400,6 @@ class QdrantStore:
|
||||
Returns:
|
||||
Insert response
|
||||
"""
|
||||
if len(ids) != len(vectors):
|
||||
# Points are paired with zip(vectors, ids), so a mismatched ID
|
||||
# list would silently drop the unpaired vectors while the
|
||||
# completion message still reports the full batch as inserted.
|
||||
raise ValidationError(
|
||||
f"Number of ids ({len(ids)}) must match number of vectors ({len(vectors)})"
|
||||
)
|
||||
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="vector_store",
|
||||
submodule="QdrantStore",
|
||||
@@ -440,10 +408,12 @@ class QdrantStore:
|
||||
|
||||
try:
|
||||
if self.collection is None:
|
||||
# len() not truthiness: vectors may be a 2-D ndarray, whose
|
||||
# truth value is ambiguous.
|
||||
dim = int(len(vectors[0])) if len(vectors) else 384
|
||||
self._ensure_default_collection(dim)
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message="Collection not initialized"
|
||||
)
|
||||
raise ProcessingError(
|
||||
"Collection not initialized. Call create_collection() or get_collection() first."
|
||||
)
|
||||
|
||||
if not QDRANT_AVAILABLE:
|
||||
self.progress_tracker.stop_tracking(
|
||||
@@ -508,7 +478,12 @@ class QdrantStore:
|
||||
|
||||
try:
|
||||
if self.search_engine is None:
|
||||
self._ensure_default_collection(int(len(query_vector)))
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message="Collection not initialized"
|
||||
)
|
||||
raise ProcessingError(
|
||||
"Collection not initialized. Call create_collection() or get_collection() first."
|
||||
)
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Performing similarity search..."
|
||||
@@ -720,31 +695,9 @@ class QdrantStore:
|
||||
collection_info = self.client.get_collection(
|
||||
self.collection.collection_name
|
||||
)
|
||||
# vectors_count was removed in qdrant-client 1.16.0.
|
||||
# When it is absent, only infer the total from points_count if we
|
||||
# can confirm the collection uses a single unnamed vector per point
|
||||
# (VectorParams). Named/multi-vector collections (dict of VectorParams)
|
||||
# have an unknown multiplier, so return None rather than a wrong value.
|
||||
# get_collection() accepts externally-created collections without schema
|
||||
# validation, so the schema must be inspected at stats time.
|
||||
vectors_count_fallback: Optional[int]
|
||||
try:
|
||||
vectors_cfg = collection_info.config.params.vectors
|
||||
vectors_count_fallback = (
|
||||
collection_info.points_count
|
||||
if QDRANT_AVAILABLE and isinstance(vectors_cfg, VectorParams)
|
||||
else None
|
||||
)
|
||||
except Exception:
|
||||
vectors_count_fallback = None
|
||||
|
||||
return {
|
||||
"points_count": collection_info.points_count,
|
||||
"vectors_count": getattr(
|
||||
collection_info,
|
||||
"vectors_count",
|
||||
vectors_count_fallback,
|
||||
),
|
||||
"vectors_count": collection_info.vectors_count,
|
||||
"status": str(collection_info.status)
|
||||
if hasattr(collection_info, "status")
|
||||
else "unknown",
|
||||
|
||||
@@ -68,8 +68,6 @@ License: MIT
|
||||
from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union, cast
|
||||
import concurrent.futures
|
||||
import inspect
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
import numpy as np
|
||||
|
||||
@@ -143,14 +141,6 @@ class VectorStore:
|
||||
if self.backend == "inmemory":
|
||||
self.vectors: Dict[str, np.ndarray] = {}
|
||||
self.metadata: Dict[str, Dict[str, Any]] = {}
|
||||
# Monotonic counter for default ID generation. Never decremented
|
||||
# on deletion, so IDs generated by consecutive store_vectors calls
|
||||
# can never collide with surviving IDs (fixes #1029).
|
||||
self._next_id: int = 0
|
||||
# Reentrant lock protecting all in-memory state mutations:
|
||||
# _next_id, vectors, metadata, and index rebuilds. Matches the
|
||||
# threading model used by SQLiteVecStore and AgentMemory.
|
||||
self._inmemory_lock = threading.RLock()
|
||||
|
||||
# Initialize backend-specific indexer
|
||||
# Avoid duplicate dimension argument
|
||||
@@ -492,11 +482,7 @@ class VectorStore:
|
||||
doc_meta = doc.metadata
|
||||
elif isinstance(doc, dict):
|
||||
doc_meta = doc.get("metadata", {})
|
||||
elif isinstance(doc, str):
|
||||
# Plain-text documents: keep the text itself in the
|
||||
# payload, otherwise it is silently dropped.
|
||||
doc_meta = {"document": doc}
|
||||
|
||||
|
||||
final_metadata[i].update(doc_meta)
|
||||
|
||||
return self.store_vectors(vectors, metadata=final_metadata, **options)
|
||||
@@ -539,16 +525,6 @@ class VectorStore:
|
||||
if supports_metadata:
|
||||
return self._backend_store.add_vectors(vectors, metadata=metadata, **options)
|
||||
return self._backend_store.add_vectors(vectors, **options)
|
||||
elif hasattr(self._backend_store, 'insert_vectors'):
|
||||
# QdrantStore: insert_vectors(vectors, ids, payloads=None)
|
||||
# upserts the points but returns the client's status dict,
|
||||
# while this facade promises callers the stored vector IDs
|
||||
# (decision storage indexes the result at position 0).
|
||||
# metadata entries already carry the source document (folded
|
||||
# in by store()), so they map directly to Qdrant payloads.
|
||||
ids = options.pop('ids', None) or [str(uuid.uuid4()) for _ in range(len(vectors))]
|
||||
self._backend_store.insert_vectors(vectors, ids, payloads=metadata, **options)
|
||||
return ids
|
||||
else:
|
||||
raise NotImplementedError(f"Backend store {type(self._backend_store).__name__} does not have add or add_vectors method")
|
||||
|
||||
@@ -566,47 +542,18 @@ class VectorStore:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Storing vectors..."
|
||||
)
|
||||
# Hold the lock for the entire ID-allocation → dict-write →
|
||||
# index-rebuild sequence so concurrent callers cannot observe
|
||||
# half-written state or generate the same candidate ID.
|
||||
with self._inmemory_lock:
|
||||
# Snapshot the live key-set at lock-entry so the generator
|
||||
# and the within-batch de-dupe use a consistent view.
|
||||
pre_existing = set(self.vectors)
|
||||
# within_batch tracks IDs chosen during *this* call so the
|
||||
# same candidate is never returned twice in one batch.
|
||||
within_batch: set = set()
|
||||
start_idx = len(self.vectors)
|
||||
for i, (vector, meta) in enumerate(zip(vectors, metadata)):
|
||||
vector_id = f"vec_{start_idx + i}"
|
||||
self.vectors[vector_id] = vector
|
||||
self.metadata[vector_id] = meta
|
||||
vector_ids.append(vector_id)
|
||||
|
||||
for vector, meta in zip(vectors, metadata):
|
||||
# Advance the monotonic counter until we find a candidate
|
||||
# that is free both in the live store and in this batch.
|
||||
#
|
||||
# The counter is never decremented on deletion, so under
|
||||
# normal operation every candidate it produces is genuinely
|
||||
# fresh. The only reason a candidate can be occupied is
|
||||
# that a caller pre-inserted a ``vec_N`` key ahead of the
|
||||
# counter (e.g. manually writing to self.vectors). Skipping
|
||||
# over such keys is intentional and matches FAISSStore's
|
||||
# identical behaviour. Nothing is overwritten: the loop
|
||||
# breaks only on a candidate that is absent from both
|
||||
# pre_existing and within_batch.
|
||||
while True:
|
||||
candidate = f"vec_{self._next_id}"
|
||||
self._next_id += 1
|
||||
if candidate not in pre_existing and candidate not in within_batch:
|
||||
break
|
||||
|
||||
within_batch.add(candidate)
|
||||
self.vectors[candidate] = vector
|
||||
self.metadata[candidate] = meta
|
||||
vector_ids.append(candidate)
|
||||
|
||||
# Update index inside the lock so readers always see a
|
||||
# consistent (vectors, index) pair.
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Updating vector index..."
|
||||
)
|
||||
self.indexer.create_index(list(self.vectors.values()), list(self.vectors.keys()))
|
||||
# Update index
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Updating vector index..."
|
||||
)
|
||||
self.indexer.create_index(list(self.vectors.values()), vector_ids)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
@@ -648,11 +595,7 @@ class VectorStore:
|
||||
"metadata": getattr(self, "metadata", {}),
|
||||
"config": self.config,
|
||||
"backend": self.backend,
|
||||
"dimension": self.dimension,
|
||||
# Persist the monotonic counter so that load() can restore it
|
||||
# rather than re-deriving it from len(vectors), which would be
|
||||
# too small after a deletion and cause ID collisions (issue #1029).
|
||||
"next_id": getattr(self, "_next_id", None),
|
||||
"dimension": self.dimension
|
||||
}
|
||||
|
||||
with open(os.path.join(path, "store_data.json"), "w", encoding="utf-8") as f:
|
||||
@@ -699,25 +642,6 @@ class VectorStore:
|
||||
self.config = data.get("config", {})
|
||||
self.backend = data.get("backend", "faiss")
|
||||
self.dimension = data.get("dimension", 768)
|
||||
|
||||
# Restore the monotonic ID counter. Always clamp to at least
|
||||
# max(vec_N suffix)+1 so a stale or missing persisted value (e.g.
|
||||
# written before this field was added, or written before a deletion
|
||||
# that lowered the count) cannot produce IDs that collide with
|
||||
# existing vectors (issue #1029).
|
||||
if self.backend == "inmemory":
|
||||
_vec_nums = [
|
||||
int(v[4:]) + 1
|
||||
for v in self.vectors
|
||||
if v.startswith("vec_") and v[4:].isdigit()
|
||||
]
|
||||
_inferred = max(_vec_nums) if _vec_nums else 0
|
||||
persisted_next_id = data.get("next_id")
|
||||
if persisted_next_id is not None:
|
||||
self._next_id = max(int(persisted_next_id), _inferred)
|
||||
else:
|
||||
# Older store files lack this field; use the safe inferred value.
|
||||
self._next_id = _inferred
|
||||
|
||||
# Restore backend-specific index
|
||||
indexer = getattr(self, "indexer", None)
|
||||
@@ -798,25 +722,14 @@ class VectorStore:
|
||||
)
|
||||
return []
|
||||
|
||||
# Snapshot vectors and metadata together under the lock so a
|
||||
# concurrent delete_vectors / store_vectors cannot cause
|
||||
# "RuntimeError: dictionary changed size during iteration" and
|
||||
# cannot produce an inconsistent (values, keys) pair where one
|
||||
# list is shorter than the other. The lock is released before
|
||||
# the (potentially slow) similarity computation.
|
||||
with self._inmemory_lock:
|
||||
snapshot_vectors = list(self.vectors.values())
|
||||
snapshot_keys = list(self.vectors.keys())
|
||||
snapshot_metadata = dict(self.metadata)
|
||||
|
||||
# Use retriever for similarity search
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Performing similarity search..."
|
||||
)
|
||||
results = self.retriever.search_similar(
|
||||
query_vector,
|
||||
snapshot_vectors,
|
||||
snapshot_keys,
|
||||
list(self.vectors.values()),
|
||||
list(self.vectors.keys()),
|
||||
k=k,
|
||||
**options,
|
||||
)
|
||||
@@ -824,8 +737,8 @@ class VectorStore:
|
||||
# Add metadata to results; guarantee the key always exists.
|
||||
for result in results:
|
||||
vector_id = result.get("id")
|
||||
if vector_id and vector_id in snapshot_metadata:
|
||||
result["metadata"] = snapshot_metadata[vector_id]
|
||||
if vector_id and vector_id in self.metadata:
|
||||
result["metadata"] = self.metadata[vector_id]
|
||||
elif "metadata" not in result:
|
||||
result["metadata"] = {}
|
||||
|
||||
@@ -854,21 +767,19 @@ class VectorStore:
|
||||
else:
|
||||
raise NotImplementedError(f"Backend store {type(self._backend_store).__name__} does not have update or update_vectors method")
|
||||
|
||||
with self._inmemory_lock:
|
||||
for vec_id, new_vec in zip(vector_ids, new_vectors):
|
||||
if vec_id in self.vectors:
|
||||
self.vectors[vec_id] = new_vec
|
||||
for vec_id, new_vec in zip(vector_ids, new_vectors):
|
||||
if vec_id in self.vectors:
|
||||
self.vectors[vec_id] = new_vec
|
||||
|
||||
if metadata:
|
||||
for vec_id, meta in zip(vector_ids, metadata):
|
||||
if vec_id in self.metadata:
|
||||
self.metadata[vec_id] = meta
|
||||
if metadata:
|
||||
for vec_id, meta in zip(vector_ids, metadata):
|
||||
if vec_id in self.metadata:
|
||||
self.metadata[vec_id] = meta
|
||||
|
||||
# Rebuild index under the lock so readers see a consistent
|
||||
# (vectors, index) pair, matching store_vectors and delete_vectors.
|
||||
self.indexer.create_index(
|
||||
list(self.vectors.values()), list(self.vectors.keys())
|
||||
)
|
||||
# Rebuild index
|
||||
self.indexer.create_index(
|
||||
list(self.vectors.values()), list(self.vectors.keys())
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
@@ -883,17 +794,15 @@ class VectorStore:
|
||||
else:
|
||||
raise NotImplementedError(f"Backend store {type(self._backend_store).__name__} does not have delete or delete_vectors method")
|
||||
|
||||
with self._inmemory_lock:
|
||||
for vec_id in vector_ids:
|
||||
self.vectors.pop(vec_id, None)
|
||||
self.metadata.pop(vec_id, None)
|
||||
for vec_id in vector_ids:
|
||||
self.vectors.pop(vec_id, None)
|
||||
self.metadata.pop(vec_id, None)
|
||||
|
||||
# Rebuild index under the lock so a concurrent search cannot
|
||||
# see vectors without a corresponding index entry.
|
||||
if self.vectors:
|
||||
self.indexer.create_index(
|
||||
list(self.vectors.values()), list(self.vectors.keys())
|
||||
)
|
||||
# Rebuild index
|
||||
if self.vectors:
|
||||
self.indexer.create_index(
|
||||
list(self.vectors.values()), list(self.vectors.keys())
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
@@ -85,7 +85,7 @@ class AnalyticsVisualizer:
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for analytics visualization. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
if np is None:
|
||||
raise ProcessingError(
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
"""
|
||||
Embedding Visualizer Module
|
||||
|
||||
This module provides comprehensive visualization capabilities for vector
|
||||
embeddings in the Semantica framework, including 2D/3D dimensionality
|
||||
reduction projections, similarity heatmaps, clustering visualizations,
|
||||
multi-modal comparisons, and quality metrics analysis.
|
||||
This module provides comprehensive visualization capabilities for vector embeddings in the
|
||||
Semantica framework, including 2D/3D dimensionality reduction projections, similarity heatmaps,
|
||||
clustering visualizations, multi-modal comparisons, and quality metrics analysis.
|
||||
|
||||
Key Features:
|
||||
- 2D and 3D dimensionality reduction (UMAP, t-SNE, PCA)
|
||||
@@ -32,8 +31,9 @@ License: MIT
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
@@ -45,12 +45,8 @@ except (ImportError, OSError):
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
try:
|
||||
from sklearn.decomposition import PCA
|
||||
from sklearn.manifold import TSNE
|
||||
except (ImportError, OSError):
|
||||
PCA = None
|
||||
TSNE = None
|
||||
from sklearn.decomposition import PCA
|
||||
from sklearn.manifold import TSNE
|
||||
|
||||
try:
|
||||
import umap
|
||||
@@ -61,7 +57,7 @@ from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .utils.color_schemes import ColorPalette, ColorScheme
|
||||
from .utils.export_formats import export_plotly_figure
|
||||
from .utils.export_formats import export_matplotlib_figure, export_plotly_figure
|
||||
|
||||
|
||||
class EmbeddingVisualizer:
|
||||
@@ -98,17 +94,12 @@ class EmbeddingVisualizer:
|
||||
self.color_scheme = ColorScheme.DEFAULT
|
||||
self.point_size = config.get("point_size", 5)
|
||||
|
||||
def _check_dependencies(self, require_sklearn: bool = False):
|
||||
def _check_dependencies(self):
|
||||
"""Check if dependencies are available."""
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for embedding visualization. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
)
|
||||
if require_sklearn and (PCA is None or TSNE is None):
|
||||
raise ProcessingError(
|
||||
"scikit-learn is required for dimensionality reduction. "
|
||||
"Reinstall scikit-learn or install dependencies."
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_2d_projection(
|
||||
@@ -159,12 +150,10 @@ class EmbeddingVisualizer:
|
||||
|
||||
try:
|
||||
self.logger.info(f"Visualizing 2D projection using {method}")
|
||||
|
||||
|
||||
# Step 2: Data Analysis
|
||||
n_samples, n_features = embeddings.shape
|
||||
self.logger.info(
|
||||
f"Embedding Analysis: {n_samples} samples, {n_features} dimensions"
|
||||
)
|
||||
self.logger.info(f"Embedding Analysis: {n_samples} samples, {n_features} dimensions")
|
||||
|
||||
if embeddings.shape[1] <= 2:
|
||||
# Already 2D or less, use directly
|
||||
@@ -176,32 +165,28 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message=f"Reducing dimensions using {method}..."
|
||||
)
|
||||
dim_options = dict(options)
|
||||
n_comp = dim_options.pop("n_components", 2)
|
||||
projected = self._reduce_dimensions(
|
||||
embeddings, method=method, n_components=n_comp, **dim_options
|
||||
embeddings, method=method, n_components=2, **options
|
||||
)
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Generating visualization..."
|
||||
)
|
||||
result = self._visualize_2d_plotly(
|
||||
projected,
|
||||
labels,
|
||||
output,
|
||||
file_path,
|
||||
projected,
|
||||
labels,
|
||||
output,
|
||||
file_path,
|
||||
color_by=color_by,
|
||||
size_by=size_by,
|
||||
hover_data=hover_data,
|
||||
**options,
|
||||
**options
|
||||
)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=(
|
||||
f"2D projection visualization generated: {len(projected)} points"
|
||||
),
|
||||
message=f"2D projection visualization generated: {len(projected)} points",
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
@@ -252,10 +237,8 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message=f"Reducing dimensions using {method}..."
|
||||
)
|
||||
dim_options = dict(options)
|
||||
n_comp = dim_options.pop("n_components", 3)
|
||||
projected = self._reduce_dimensions(
|
||||
embeddings, method=method, n_components=n_comp, **dim_options
|
||||
embeddings, method=method, n_components=3, **options
|
||||
)
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
@@ -268,9 +251,7 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=(
|
||||
f"3D projection visualization generated: {len(projected)} points"
|
||||
),
|
||||
message=f"3D projection visualization generated: {len(projected)} points",
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
@@ -361,10 +342,7 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=(
|
||||
f"Similarity heatmap generated: "
|
||||
f"{len(embeddings)}x{len(embeddings)} matrix"
|
||||
),
|
||||
message=f"Similarity heatmap generated: {len(embeddings)}x{len(embeddings)} matrix",
|
||||
)
|
||||
return fig
|
||||
elif file_path:
|
||||
@@ -426,10 +404,8 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message=f"Reducing dimensions using {method}..."
|
||||
)
|
||||
dim_options = dict(options)
|
||||
n_comp = dim_options.pop("n_components", 2)
|
||||
projected = self._reduce_dimensions(
|
||||
embeddings, method=method, n_components=n_comp, **dim_options
|
||||
embeddings, method=method, n_components=2, **options
|
||||
)
|
||||
|
||||
num_clusters = len(set(cluster_labels))
|
||||
@@ -469,10 +445,7 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=(
|
||||
f"Clustering visualization generated: "
|
||||
f"{num_clusters} clusters, {len(embeddings)} points"
|
||||
),
|
||||
message=f"Clustering visualization generated: {num_clusters} clusters, {len(embeddings)} points",
|
||||
)
|
||||
return fig
|
||||
elif file_path:
|
||||
@@ -568,13 +541,8 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message=f"Reducing dimensions using {method}..."
|
||||
)
|
||||
dim_options = dict(options)
|
||||
n_comp = dim_options.pop("n_components", 2)
|
||||
projected = self._reduce_dimensions(
|
||||
combined_embeddings,
|
||||
method=method,
|
||||
n_components=n_comp,
|
||||
**dim_options,
|
||||
combined_embeddings, method=method, n_components=2, **options
|
||||
)
|
||||
|
||||
# Color by type
|
||||
@@ -611,10 +579,7 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=(
|
||||
f"Multi-modal comparison generated: "
|
||||
f"{len(combined_embeddings)} embeddings"
|
||||
),
|
||||
message=f"Multi-modal comparison generated: {len(combined_embeddings)} embeddings",
|
||||
)
|
||||
return fig
|
||||
elif file_path:
|
||||
@@ -633,6 +598,8 @@ class EmbeddingVisualizer:
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
|
||||
def _reduce_dimensions(
|
||||
self,
|
||||
embeddings: np.ndarray,
|
||||
@@ -641,56 +608,39 @@ class EmbeddingVisualizer:
|
||||
**options,
|
||||
) -> np.ndarray:
|
||||
"""Reduce embedding dimensions using specified method."""
|
||||
opts = dict(options)
|
||||
opts.pop("n_components", None)
|
||||
|
||||
if method == "pca":
|
||||
if PCA is None:
|
||||
raise ProcessingError(
|
||||
"scikit-learn is required for dimensionality reduction. "
|
||||
"Reinstall scikit-learn or install dependencies."
|
||||
)
|
||||
pca = PCA(n_components=n_components, **opts)
|
||||
pca = PCA(n_components=n_components, **options)
|
||||
return pca.fit_transform(embeddings)
|
||||
|
||||
elif method == "tsne":
|
||||
if TSNE is None:
|
||||
raise ProcessingError(
|
||||
"scikit-learn is required for dimensionality reduction. "
|
||||
"Reinstall scikit-learn or install dependencies."
|
||||
)
|
||||
perplexity = opts.pop("perplexity", min(30, len(embeddings) - 1))
|
||||
random_state = opts.pop("random_state", 42)
|
||||
perplexity = options.get("perplexity", min(30, len(embeddings) - 1))
|
||||
tsne = TSNE(
|
||||
n_components=n_components,
|
||||
perplexity=perplexity,
|
||||
random_state=random_state,
|
||||
**opts,
|
||||
random_state=42,
|
||||
**options,
|
||||
)
|
||||
return tsne.fit_transform(embeddings)
|
||||
|
||||
elif method == "umap":
|
||||
if umap is not None:
|
||||
n_neighbors = opts.pop("n_neighbors", min(15, len(embeddings) - 1))
|
||||
n_neighbors = options.get("n_neighbors", min(15, len(embeddings) - 1))
|
||||
reducer = umap.UMAP(
|
||||
n_components=n_components, n_neighbors=n_neighbors, **opts
|
||||
n_components=n_components, n_neighbors=n_neighbors, **options
|
||||
)
|
||||
return reducer.fit_transform(embeddings)
|
||||
else:
|
||||
raise ProcessingError(
|
||||
"UMAP is required for UMAP dimensionality reduction. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
# Fallback to PCA if UMAP not available
|
||||
self.logger.warning(
|
||||
"UMAP not available, using PCA. Install with: pip install umap-learn"
|
||||
)
|
||||
pca = PCA(n_components=n_components)
|
||||
return pca.fit_transform(embeddings)
|
||||
|
||||
else:
|
||||
if PCA is None:
|
||||
raise ProcessingError(
|
||||
"scikit-learn is required for dimensionality reduction. "
|
||||
"Reinstall scikit-learn or install dependencies."
|
||||
)
|
||||
# Fallback to PCA
|
||||
self.logger.warning(f"Method {method} not available, using PCA")
|
||||
pca = PCA(n_components=n_components, **opts)
|
||||
pca = PCA(n_components=n_components)
|
||||
return pca.fit_transform(embeddings)
|
||||
|
||||
def _visualize_2d_plotly(
|
||||
|
||||
@@ -124,7 +124,7 @@ class KGVisualizer:
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for KG visualization. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def _convert_knowledge_graph(self, kg: Any) -> Dict[str, Any]:
|
||||
|
||||
@@ -35,14 +35,8 @@ License: MIT
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
try:
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import FancyBboxPatch
|
||||
except (ImportError, OSError):
|
||||
mpatches = None
|
||||
plt = None
|
||||
FancyBboxPatch = None
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
try:
|
||||
import plotly.express as px
|
||||
@@ -53,6 +47,8 @@ except (ImportError, OSError):
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
from matplotlib.patches import FancyBboxPatch
|
||||
|
||||
try:
|
||||
import graphviz
|
||||
except (ImportError, OSError):
|
||||
@@ -110,13 +106,13 @@ class OntologyVisualizer:
|
||||
if graphviz is None:
|
||||
raise ProcessingError(
|
||||
"Graphviz is required for DOT export. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
"Install with: pip install graphviz"
|
||||
)
|
||||
else:
|
||||
if go is None:
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for ontology visualization. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_hierarchy(
|
||||
@@ -896,7 +892,7 @@ class OntologyVisualizer:
|
||||
"""Create Graphviz hierarchy visualization."""
|
||||
if graphviz is None:
|
||||
raise ProcessingError(
|
||||
"Graphviz not available. Install with: pip install 'semantica[viz]'"
|
||||
"Graphviz not available. Install with: pip install graphviz"
|
||||
)
|
||||
|
||||
dot = graphviz.Digraph(comment="Ontology Hierarchy")
|
||||
|
||||
@@ -74,7 +74,7 @@ class SemanticNetworkVisualizer:
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for semantic network visualization. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_network(
|
||||
|
||||
@@ -83,7 +83,7 @@ class TemporalVisualizer:
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for temporal visualization. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_temporal_dashboard(
|
||||
|
||||
@@ -51,27 +51,6 @@ _INTERNAL_ERROR = -32603
|
||||
_TOOL_INDEX: dict[str, dict] = {t["name"]: t for t in TOOL_DEFINITIONS}
|
||||
|
||||
|
||||
class UnknownToolError(Exception):
|
||||
"""Raised by :func:`call_tool` when the tool name is not in the catalog.
|
||||
|
||||
A dedicated type (rather than ``KeyError``) so callers can distinguish
|
||||
a bad tool name from a ``KeyError`` raised inside a handler indexing a
|
||||
required argument (e.g. ``args["category"]``).
|
||||
"""
|
||||
|
||||
|
||||
def call_tool(name: str, arguments: dict) -> dict:
|
||||
"""Invoke a tool in-process by name and return its raw result dict.
|
||||
|
||||
Shared by the JSON-RPC ``tools/call`` handler and ``semantica mcp call``
|
||||
(issue #1355), so both expose exactly the same tool set.
|
||||
"""
|
||||
tool = _TOOL_INDEX.get(name)
|
||||
if tool is None:
|
||||
raise UnknownToolError(f"Unknown tool: {name}")
|
||||
return tool["_handler"](arguments)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request handlers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -106,10 +85,12 @@ def _handle_tools_call(req_id: Any, params: dict) -> dict:
|
||||
name = params.get("name", "")
|
||||
args = params.get("arguments", {}) or {}
|
||||
|
||||
tool = _TOOL_INDEX.get(name)
|
||||
if tool is None:
|
||||
return _err(req_id, _METHOD_NOT_FOUND, f"Unknown tool: {name}")
|
||||
|
||||
try:
|
||||
result = call_tool(name, args)
|
||||
except UnknownToolError as exc:
|
||||
return _err(req_id, _METHOD_NOT_FOUND, str(exc))
|
||||
result = tool["_handler"](args)
|
||||
except Exception as exc:
|
||||
log.exception("Tool %s raised an exception", name)
|
||||
# The exception's class name (e.g. "ValidationError", "TimeoutError")
|
||||
|
||||
@@ -58,69 +58,21 @@ def handle_export_graph(args: dict) -> dict:
|
||||
return {"format": "csv", "data": header + "\n" + "\n".join(rows)}
|
||||
|
||||
if fmt in ("graphml",):
|
||||
# GraphMLExporter does not exist. GraphExporter is the correct
|
||||
# class; it writes to a file and returns None, so we need a
|
||||
# temporary directory for safe cleanup regardless of success or
|
||||
# failure.
|
||||
#
|
||||
# export_knowledge_graph() is the required entry point because
|
||||
# to_kg_dict() returns {"entities": [...], "relationships": [...]}
|
||||
# while export() / _export_graphml() only consumes {"nodes": [...],
|
||||
# "edges": [...]}. Calling export() directly therefore produces a
|
||||
# structurally-valid but empty GraphML document (zero nodes, zero
|
||||
# edges). export_knowledge_graph() runs _convert_kg_to_graph()
|
||||
# first, which maps entities→nodes and relationships→edges.
|
||||
try:
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from semantica.export import GraphExporter
|
||||
kg_dict = graph.to_kg_dict()
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_path = Path(tmpdir) / "export.graphml"
|
||||
exporter = GraphExporter(format="graphml")
|
||||
exporter.export_knowledge_graph(kg_dict, file_path=tmp_path)
|
||||
data = tmp_path.read_text(encoding="utf-8")
|
||||
from semantica.export import GraphMLExporter
|
||||
exporter = GraphMLExporter()
|
||||
data = exporter.export(graph)
|
||||
return {"format": "graphml", "data": data}
|
||||
except ImportError as exc:
|
||||
log.warning("GraphML export unavailable: %s", exc)
|
||||
return {"error": f"GraphML export unavailable: {exc}"}
|
||||
except Exception as exc:
|
||||
log.exception("GraphML export failed")
|
||||
return {"error": f"GraphML export failed: {exc}"}
|
||||
|
||||
if fmt in ("parquet",):
|
||||
# ParquetExporter.export() writes to file(s) and returns None;
|
||||
# passing the ContextGraph object or using the return value as
|
||||
# data were both wrong. export_knowledge_graph() is the correct
|
||||
# entry point for a full KG: it splits into _entities.parquet and
|
||||
# _relationships.parquet under a provided base path. We read both
|
||||
# files and return them base64-encoded so the MCP client can
|
||||
# reconstruct them without a shared filesystem.
|
||||
try:
|
||||
import base64
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
try:
|
||||
from semantica.export import ParquetExporter
|
||||
exporter = ParquetExporter()
|
||||
except ImportError as exc:
|
||||
log.warning("Parquet export unavailable: %s", exc)
|
||||
return {"error": f"Parquet export unavailable (pyarrow not installed): {exc}"}
|
||||
kg_dict = graph.to_kg_dict()
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
base_path = Path(tmpdir) / "kg"
|
||||
exporter.export_knowledge_graph(kg_dict, base_path)
|
||||
# Collect the produced files and return them as base64
|
||||
files = {}
|
||||
for p in sorted(Path(tmpdir).glob("*.parquet")):
|
||||
files[p.name] = base64.b64encode(p.read_bytes()).decode("ascii")
|
||||
if not files:
|
||||
return {"error": "Parquet export produced no files"}
|
||||
return {"format": "parquet", "data": files,
|
||||
"encoding": "base64",
|
||||
"note": "Each value is a base64-encoded Parquet file."}
|
||||
from semantica.export import ParquetExporter
|
||||
exporter = ParquetExporter()
|
||||
data = exporter.export(graph, include_metadata)
|
||||
return {"format": "parquet", "data": str(data)}
|
||||
except Exception as exc:
|
||||
log.exception("Parquet export failed")
|
||||
return {"error": f"Parquet export failed: {exc}"}
|
||||
|
||||
# RDF formats
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
"""Regression tests for issue #1531.
|
||||
|
||||
``GET /api/decisions/causal-distance`` was unreachable because the dynamic
|
||||
route ``GET /api/decisions/{decision_id}`` is registered before the static
|
||||
``/causal-distance`` route in ``semantica/explorer/routes/decisions.py``.
|
||||
Starlette matches routes in definition order, so a request to
|
||||
``causal-distance`` was bound as ``decision_id="causal-distance"`` and
|
||||
returned ``404 Decision 'causal-distance' not found``.
|
||||
|
||||
Each test below fails while the static route sits after ``/{decision_id}``
|
||||
and passes once it is moved above it.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
|
||||
from semantica.context.context_graph import ContextGraph # noqa: E402
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
|
||||
def _client() -> TestClient:
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
return TestClient(create_app(session=GraphSession(graph)))
|
||||
|
||||
|
||||
def test_causal_distance_route_is_not_shadowed_by_decision_id():
|
||||
"""GET /api/decisions/causal-distance must reach the distance handler.
|
||||
|
||||
Fails before the fix with: 404 {"detail": "Decision 'causal-distance' not found"}.
|
||||
After the fix the analyzer returns an unreachable-distance report (200).
|
||||
"""
|
||||
with _client() as client:
|
||||
response = client.get(
|
||||
"/api/decisions/causal-distance", params={"source": "x", "target": "y"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
payload = response.json()
|
||||
assert payload["source_id"] == "x"
|
||||
assert payload["target_id"] == "y"
|
||||
|
||||
|
||||
def test_causal_distance_between_linked_decisions():
|
||||
"""Two causally linked decisions must report a 1-hop path."""
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
first = graph.record_decision(
|
||||
category="credit_application",
|
||||
scenario="Personal loan review",
|
||||
reasoning="Income meets threshold",
|
||||
outcome="proceed_to_underwriting",
|
||||
confidence=0.88,
|
||||
)
|
||||
second = graph.record_decision(
|
||||
category="loan_underwriting",
|
||||
scenario="Underwriting review",
|
||||
reasoning="DTI within policy",
|
||||
outcome="approved",
|
||||
confidence=0.94,
|
||||
)
|
||||
graph.add_causal_relationship(first, second, relationship_type="CAUSED")
|
||||
|
||||
with TestClient(create_app(session=GraphSession(graph))) as client:
|
||||
response = client.get(
|
||||
"/api/decisions/causal-distance",
|
||||
params={"source": first, "target": second},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["causal_hop_count"] == 1
|
||||
|
||||
|
||||
def test_unknown_decision_id_still_404s():
|
||||
"""Moving the static route must not change param-route 404 behavior."""
|
||||
with _client() as client:
|
||||
response = client.get("/api/decisions/does-not-exist")
|
||||
|
||||
assert response.status_code == 404
|
||||
assert response.json() == {"detail": "Decision 'does-not-exist' not found"}
|
||||
@@ -141,134 +141,3 @@ else:
|
||||
assert "ConfigurationError" in result.stdout
|
||||
assert "Parquet ingestion" in result.stdout
|
||||
assert "pyarrow" in result.stdout
|
||||
|
||||
|
||||
def test_repo_ingestor_probe_fails_without_gitpython() -> None:
|
||||
result = _run_python_with_blocked_modules(
|
||||
"""
|
||||
try:
|
||||
from semantica.ingest import RepoIngestor
|
||||
has_git = True
|
||||
except ImportError:
|
||||
has_git = False
|
||||
|
||||
assert not has_git, "Expected RepoIngestor import to fail without GitPython"
|
||||
print("RepoIngestor probe passed")
|
||||
""",
|
||||
("git",),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "RepoIngestor probe passed" in result.stdout
|
||||
|
||||
|
||||
def test_xml_ingestor_probe_fails_without_lxml() -> None:
|
||||
result = _run_python_with_blocked_modules(
|
||||
"""
|
||||
try:
|
||||
from semantica.ingest import XMLIngestor
|
||||
has_lxml = True
|
||||
except ImportError:
|
||||
has_lxml = False
|
||||
|
||||
assert not has_lxml, "Expected XMLIngestor import to fail without lxml"
|
||||
print("XMLIngestor probe passed")
|
||||
""",
|
||||
("lxml",),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "XMLIngestor probe passed" in result.stdout
|
||||
|
||||
|
||||
def test_xml_ingestion_reports_missing_lxml_when_used() -> None:
|
||||
result = _run_python_with_blocked_modules(
|
||||
"""
|
||||
from semantica.ingest import ingest_xml
|
||||
|
||||
try:
|
||||
ingest_xml("catalog.xml")
|
||||
except Exception as exc:
|
||||
print(type(exc).__name__, exc)
|
||||
else:
|
||||
raise SystemExit("expected XML ingestion to fail without lxml")
|
||||
""",
|
||||
("lxml",),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "ConfigurationError" in result.stdout
|
||||
assert "XML ingestion" in result.stdout
|
||||
assert "lxml" in result.stdout
|
||||
|
||||
|
||||
def test_sibling_imports_succeed_without_optional_backends() -> None:
|
||||
result = _run_python_with_blocked_modules(
|
||||
"""
|
||||
from semantica.ingest import (
|
||||
CodeExtractor,
|
||||
CodeFile,
|
||||
CommitInfo,
|
||||
GitAnalyzer,
|
||||
XMLIngestionData,
|
||||
SalesforceData,
|
||||
)
|
||||
print(
|
||||
CodeExtractor.__name__,
|
||||
CodeFile.__name__,
|
||||
CommitInfo.__name__,
|
||||
GitAnalyzer.__name__,
|
||||
XMLIngestionData.__name__,
|
||||
SalesforceData.__name__,
|
||||
)
|
||||
""",
|
||||
("git", "lxml", "simple_salesforce"),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert (
|
||||
"CodeExtractor CodeFile CommitInfo GitAnalyzer XMLIngestionData SalesforceData"
|
||||
in result.stdout
|
||||
)
|
||||
|
||||
|
||||
def test_salesforce_ingestor_probe_fails_without_simple_salesforce() -> None:
|
||||
result = _run_python_with_blocked_modules(
|
||||
"""
|
||||
try:
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
has_salesforce = True
|
||||
except ImportError:
|
||||
has_salesforce = False
|
||||
|
||||
assert not has_salesforce, (
|
||||
"Expected SalesforceIngestor import to fail without simple-salesforce"
|
||||
)
|
||||
print("SalesforceIngestor probe passed")
|
||||
""",
|
||||
("simple_salesforce",),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "SalesforceIngestor probe passed" in result.stdout
|
||||
|
||||
|
||||
def test_salesforce_ingestion_reports_missing_dep_when_used() -> None:
|
||||
result = _run_python_with_blocked_modules(
|
||||
"""
|
||||
from semantica.ingest import ingest_salesforce
|
||||
|
||||
try:
|
||||
ingest_salesforce()
|
||||
except Exception as exc:
|
||||
print(type(exc).__name__, exc)
|
||||
else:
|
||||
raise SystemExit("expected Salesforce ingestion to fail without simple-salesforce")
|
||||
""",
|
||||
("simple_salesforce",),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "ConfigurationError" in result.stdout
|
||||
assert "Salesforce ingestion" in result.stdout
|
||||
assert "simple-salesforce" in result.stdout
|
||||
|
||||
@@ -1,282 +0,0 @@
|
||||
"""Unit tests for ExtractionSchema and SchemaValidator (schema-guided validation)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from semantica.semantic_extract import (
|
||||
Entity,
|
||||
ExtractionSchema,
|
||||
ExtractionValidator,
|
||||
Relation,
|
||||
SchemaValidator,
|
||||
ValidationResult,
|
||||
)
|
||||
|
||||
ONTOLOGY = {
|
||||
"classes": [{"name": "Person"}, {"name": "Organization"}, {"label": "City"}],
|
||||
"properties": [
|
||||
{"name": "worksAt", "domain": ["Person"], "range": ["Organization"]},
|
||||
{"name": "locatedIn", "domain": "Organization", "range": "City"},
|
||||
{"name": "knows"}, # unconstrained domain / range
|
||||
],
|
||||
}
|
||||
|
||||
TTL = """
|
||||
@prefix : <https://example.org/> .
|
||||
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
|
||||
:Person a owl:Class .
|
||||
:Organization a owl:Class .
|
||||
:worksAt a owl:ObjectProperty ;
|
||||
rdfs:domain :Person ;
|
||||
rdfs:range :Organization .
|
||||
"""
|
||||
|
||||
|
||||
def _schema() -> ExtractionSchema:
|
||||
return ExtractionSchema.from_ontology(ONTOLOGY)
|
||||
|
||||
|
||||
def _person() -> Entity:
|
||||
return Entity(text="Alice", label="Person", start_char=0, end_char=5)
|
||||
|
||||
|
||||
def _org() -> Entity:
|
||||
return Entity(text="Acme", label="Organization", start_char=0, end_char=4)
|
||||
|
||||
|
||||
def _city() -> Entity:
|
||||
return Entity(text="Paris", label="City", start_char=0, end_char=5)
|
||||
|
||||
|
||||
def _product() -> Entity:
|
||||
return Entity(text="Widget", label="Product", start_char=0, end_char=6)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# ExtractionSchema
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_from_ontology_parses_concepts_and_predicates() -> None:
|
||||
schema = _schema()
|
||||
assert schema.concepts == frozenset({"Person", "Organization", "City"})
|
||||
assert set(schema.predicates) == {"worksAt", "locatedIn", "knows"}
|
||||
assert schema.predicates["worksAt"].domain == frozenset({"Person"})
|
||||
assert schema.predicates["worksAt"].range == frozenset({"Organization"})
|
||||
# Missing domain / range means unconstrained.
|
||||
assert schema.predicates["knows"].domain == frozenset()
|
||||
assert schema.predicates["knows"].range == frozenset()
|
||||
|
||||
|
||||
def test_allows_relation_respects_domain_range() -> None:
|
||||
schema = _schema()
|
||||
assert schema.allows_relation("Person", "worksAt", "Organization")
|
||||
assert not schema.allows_relation("Person", "worksAt", "City") # range violation
|
||||
assert not schema.allows_relation("Person", "unknownPred", "Organization")
|
||||
assert not schema.allows_relation("Product", "worksAt", "Organization") # off-vocab
|
||||
# Unconstrained predicate accepts any known concepts.
|
||||
assert schema.allows_relation("Person", "knows", "City")
|
||||
|
||||
|
||||
def test_from_owl_parses_turtle() -> None:
|
||||
schema = ExtractionSchema.from_owl(TTL, format="turtle")
|
||||
assert {"Person", "Organization"} <= schema.concepts
|
||||
assert schema.predicates["worksAt"].domain == frozenset({"Person"})
|
||||
assert schema.predicates["worksAt"].range == frozenset({"Organization"})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# SchemaValidator — entities
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_validate_entities_all_conforming() -> None:
|
||||
result = SchemaValidator(_schema()).validate_entities([_person(), _org(), _city()])
|
||||
assert isinstance(result, ValidationResult)
|
||||
assert result.valid
|
||||
assert result.score == 1.0
|
||||
assert result.metrics["out_of_vocabulary"] == 0
|
||||
|
||||
|
||||
def test_validate_entities_flags_out_of_vocabulary() -> None:
|
||||
result = SchemaValidator(_schema()).validate_entities([_person(), _product()])
|
||||
assert not result.valid
|
||||
assert result.metrics["out_of_vocabulary"] == 1
|
||||
assert result.metrics["unknown_labels"] == ["Product"]
|
||||
assert result.score == 0.5
|
||||
assert result.errors
|
||||
|
||||
|
||||
def test_validate_entities_empty_is_vacuously_valid() -> None:
|
||||
result = SchemaValidator(_schema()).validate_entities([])
|
||||
assert result.valid
|
||||
assert result.score == 1.0
|
||||
|
||||
|
||||
def test_validate_entities_batch_returns_list_with_index() -> None:
|
||||
results = SchemaValidator(_schema()).validate_entities([[_person()], [_product()]])
|
||||
assert isinstance(results, list)
|
||||
assert len(results) == 2
|
||||
assert results[0].valid
|
||||
assert not results[1].valid
|
||||
assert results[0].metadata["batch_index"] == 0
|
||||
assert results[1].metadata["batch_index"] == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# SchemaValidator — relations
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_validate_relations_conforming() -> None:
|
||||
rels = [
|
||||
Relation(subject=_person(), predicate="worksAt", object=_org()),
|
||||
Relation(subject=_person(), predicate="knows", object=_city()),
|
||||
]
|
||||
result = SchemaValidator(_schema()).validate_relations(rels)
|
||||
assert result.valid
|
||||
assert result.score == 1.0
|
||||
|
||||
|
||||
def test_validate_relations_flags_unknown_predicate_and_domain_range() -> None:
|
||||
rels = [
|
||||
Relation(subject=_person(), predicate="worksAt", object=_org()), # ok
|
||||
Relation(subject=_person(), predicate="founded", object=_org()), # unknown pred
|
||||
Relation(subject=_person(), predicate="worksAt", object=_city()), # range viol
|
||||
]
|
||||
result = SchemaValidator(_schema()).validate_relations(rels)
|
||||
assert not result.valid
|
||||
assert result.metrics["unknown_predicate"] == 1
|
||||
assert result.metrics["domain_range_violation"] == 1
|
||||
assert result.metrics["conforming"] == 1
|
||||
assert result.score == 1 / 3
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Filtering
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_filter_by_schema_drops_off_vocabulary() -> None:
|
||||
kept = SchemaValidator(_schema()).filter_by_schema([_person(), _org(), _product()])
|
||||
assert [e.label for e in kept] == ["Person", "Organization"]
|
||||
|
||||
|
||||
def test_filter_relations_by_schema_keeps_only_conforming() -> None:
|
||||
rels = [
|
||||
Relation(subject=_person(), predicate="worksAt", object=_org()), # keep
|
||||
Relation(subject=_person(), predicate="founded", object=_org()), # drop
|
||||
Relation(subject=_person(), predicate="worksAt", object=_city()), # drop
|
||||
Relation(subject=_person(), predicate="knows", object=_city()), # keep
|
||||
]
|
||||
kept = SchemaValidator(_schema()).filter_relations_by_schema(rels)
|
||||
assert [r.predicate for r in kept] == ["worksAt", "knows"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Composition with the confidence-based ExtractionValidator (orthogonal axis)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_composes_with_extraction_validator_same_shape() -> None:
|
||||
entities = [_person(), _org()]
|
||||
confidence = ExtractionValidator().validate_entities(entities)
|
||||
conformance = SchemaValidator(_schema()).validate_entities(entities)
|
||||
assert isinstance(confidence, ValidationResult)
|
||||
assert isinstance(conformance, ValidationResult)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Robustness fixes surfaced in review
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_owl_thing_domain_range_is_unconstrained() -> None:
|
||||
# OntologyGenerator emits owl:Thing when it cannot resolve endpoint types;
|
||||
# it must behave as "any concept", not a literal {"Thing"} constraint.
|
||||
ont = {
|
||||
"classes": [{"name": "Person"}, {"name": "Organization"}],
|
||||
"properties": [
|
||||
{"name": "relatedTo", "domain": ["owl:Thing"], "range": ["owl:Thing"]}
|
||||
],
|
||||
}
|
||||
schema = ExtractionSchema.from_ontology(ont)
|
||||
assert schema.predicates["relatedTo"].domain == frozenset()
|
||||
assert schema.predicates["relatedTo"].range == frozenset()
|
||||
assert schema.allows_relation("Person", "relatedTo", "Organization")
|
||||
|
||||
|
||||
def test_from_owl_prefers_rdfs_label_and_supports_rdfs_class() -> None:
|
||||
ttl = """
|
||||
@prefix : <https://example.org/> .
|
||||
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
|
||||
:Cls1 a owl:Class ; rdfs:label "Person" .
|
||||
:Org a rdfs:Class .
|
||||
:worksAt a owl:ObjectProperty ;
|
||||
rdfs:domain :Cls1 ;
|
||||
rdfs:range :Org .
|
||||
"""
|
||||
schema = ExtractionSchema.from_owl(ttl, format="turtle")
|
||||
# rdfs:label wins over the URI suffix "Cls1"
|
||||
assert "Person" in schema.concepts
|
||||
assert "Cls1" not in schema.concepts
|
||||
# rdfs:Class is picked up too
|
||||
assert "Org" in schema.concepts
|
||||
assert schema.predicates["worksAt"].domain == frozenset({"Person"})
|
||||
|
||||
|
||||
def test_validate_relations_handles_malformed_without_crashing() -> None:
|
||||
# A relation missing an endpoint must be reported, not raise AttributeError.
|
||||
good = Relation(subject=_person(), predicate="worksAt", object=_org())
|
||||
bad = Relation(subject=_person(), predicate="worksAt", object=None) # type: ignore[arg-type]
|
||||
result = SchemaValidator(_schema()).validate_relations([good, bad])
|
||||
assert isinstance(result, ValidationResult)
|
||||
assert not result.valid
|
||||
assert result.metrics["malformed"] == 1
|
||||
assert result.metrics["conforming"] == 1
|
||||
# Filtering also drops the malformed one instead of crashing.
|
||||
kept = SchemaValidator(_schema()).filter_relations_by_schema([good, bad])
|
||||
assert kept == [good]
|
||||
|
||||
|
||||
def test_from_ontology_folds_endpoint_types_like_from_owl() -> None:
|
||||
# pkupt's case: an endpoint type (Org) that didn't clear the class-frequency
|
||||
# gate is absent from "classes" but referenced in a property's range. Both
|
||||
# constructors must agree that (Person, worksFor, Org) is allowed.
|
||||
ont = {
|
||||
"classes": [{"name": "Person"}],
|
||||
"properties": [{"name": "worksFor", "domain": ["Person"], "range": ["Org"]}],
|
||||
}
|
||||
dict_schema = ExtractionSchema.from_ontology(ont)
|
||||
assert {"Person", "Org"} <= dict_schema.concepts
|
||||
assert dict_schema.allows_relation("Person", "worksFor", "Org")
|
||||
|
||||
ttl = """
|
||||
@prefix : <https://example.org/> .
|
||||
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
|
||||
:Person a owl:Class .
|
||||
:worksFor a owl:ObjectProperty ;
|
||||
rdfs:domain :Person ;
|
||||
rdfs:range :Org .
|
||||
"""
|
||||
owl_schema = ExtractionSchema.from_owl(ttl, format="turtle")
|
||||
assert owl_schema.allows_relation(
|
||||
"Person", "worksFor", "Org"
|
||||
) == dict_schema.allows_relation("Person", "worksFor", "Org")
|
||||
|
||||
|
||||
def test_from_ontology_accepts_ontologydata_like_object() -> None:
|
||||
# semantica.ingest.OntologyIngestor.ingest_ontology returns an OntologyData
|
||||
# whose ontology dict is held in `.data`; from_ontology should unwrap it
|
||||
# instead of raising AttributeError on `.get()`.
|
||||
from types import SimpleNamespace
|
||||
|
||||
wrapped = SimpleNamespace(data=ONTOLOGY)
|
||||
schema = ExtractionSchema.from_ontology(wrapped)
|
||||
assert schema.concepts == frozenset({"Person", "Organization", "City"})
|
||||
assert "worksAt" in schema.predicates
|
||||
@@ -18,40 +18,29 @@ from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# ── Mock optional heavyweight dependencies before any semantica import ──────
|
||||
_MOCKED_MODULES = [
|
||||
"spacy",
|
||||
"instructor",
|
||||
"openai",
|
||||
"groq",
|
||||
"sentence_transformers",
|
||||
"transformers",
|
||||
]
|
||||
_original_modules = {k: sys.modules.get(k) for k in _MOCKED_MODULES}
|
||||
|
||||
for k in _MOCKED_MODULES:
|
||||
sys.modules.setdefault(k, MagicMock())
|
||||
sys.modules.setdefault("spacy", MagicMock())
|
||||
sys.modules.setdefault("instructor", MagicMock())
|
||||
_openai_mock = MagicMock()
|
||||
sys.modules.setdefault("openai", _openai_mock)
|
||||
sys.modules.setdefault("groq", MagicMock())
|
||||
sys.modules.setdefault("sentence_transformers", MagicMock())
|
||||
sys.modules.setdefault("transformers", MagicMock())
|
||||
sys.modules.setdefault("torch", MagicMock())
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
|
||||
|
||||
from semantica.semantic_extract.methods import extract_relations_llm # noqa: E402
|
||||
|
||||
for _key, _original in _original_modules.items():
|
||||
if _original is None:
|
||||
sys.modules.pop(_key, None)
|
||||
else:
|
||||
sys.modules[_key] = _original
|
||||
|
||||
from semantica.semantic_extract.ner_extractor import Entity # noqa: E402
|
||||
from semantica.semantic_extract.schemas import ( # noqa: E402
|
||||
from semantica.semantic_extract.methods import extract_relations_llm
|
||||
from semantica.semantic_extract.ner_extractor import Entity
|
||||
from semantica.semantic_extract.schemas import (
|
||||
RelationsResponse,
|
||||
RelationsWithTemporalResponse,
|
||||
)
|
||||
from semantica.kg.temporal_normalizer import TemporalNormalizer # noqa: E402
|
||||
from semantica.utils.exceptions import TemporalAmbiguityWarning # noqa: E402
|
||||
from semantica.kg.temporal_normalizer import TemporalNormalizer
|
||||
from semantica.utils.exceptions import TemporalAmbiguityWarning
|
||||
|
||||
|
||||
# ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_entities():
|
||||
return [
|
||||
Entity(text="Apple", label="ORG", start_char=0, end_char=5),
|
||||
@@ -67,17 +56,15 @@ def _ref_date():
|
||||
# Part 1 – extract_relations_llm() temporal flag
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTemporalExtractionFlag(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
from semantica.semantic_extract.methods import _result_cache
|
||||
|
||||
_result_cache.clear()
|
||||
|
||||
@patch("semantica.semantic_extract.methods.create_provider")
|
||||
def test_extract_temporal_bounds_true_adds_four_fields(self, mock_create):
|
||||
"""With extract_temporal_bounds=True all four temporal keys appear."""
|
||||
"""With extract_temporal_bounds=True all four temporal keys appear in metadata."""
|
||||
mock_prov = MagicMock()
|
||||
mock_prov.is_available.return_value = True
|
||||
mock_prov.generate_typed.return_value = RelationsWithTemporalResponse(
|
||||
@@ -145,9 +132,7 @@ class TestTemporalExtractionFlag(unittest.TestCase):
|
||||
self.assertNotIn("temporal_source_text", meta)
|
||||
|
||||
@patch("semantica.semantic_extract.methods.create_provider")
|
||||
def test_no_temporal_signal_returns_zero_confidence_and_null_dates(
|
||||
self, mock_create
|
||||
):
|
||||
def test_no_temporal_signal_returns_zero_confidence_and_null_dates(self, mock_create):
|
||||
"""When LLM returns no temporal signal, confidence=0.0 and dates are null."""
|
||||
mock_prov = MagicMock()
|
||||
mock_prov.is_available.return_value = True
|
||||
@@ -212,12 +197,10 @@ class TestTemporalExtractionFlag(unittest.TestCase):
|
||||
|
||||
@patch("semantica.semantic_extract.methods.create_provider")
|
||||
def test_correct_schema_used_when_temporal_true(self, mock_create):
|
||||
"""generate_typed is called with RelationsWithTemporalResponse."""
|
||||
"""generate_typed is called with RelationsWithTemporalResponse when flag=True."""
|
||||
mock_prov = MagicMock()
|
||||
mock_prov.is_available.return_value = True
|
||||
mock_prov.generate_typed.return_value = RelationsWithTemporalResponse(
|
||||
relations=[]
|
||||
)
|
||||
mock_prov.generate_typed.return_value = RelationsWithTemporalResponse(relations=[])
|
||||
mock_create.return_value = mock_prov
|
||||
|
||||
extract_relations_llm(
|
||||
@@ -252,7 +235,6 @@ class TestTemporalExtractionFlag(unittest.TestCase):
|
||||
# Part 2 – TemporalNormalizer: relative dates
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTemporalNormalizerRelativeDates(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
@@ -331,13 +313,10 @@ class TestTemporalNormalizerRelativeDates(unittest.TestCase):
|
||||
# Part 3 – TemporalNormalizer: partial / structured dates
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTemporalNormalizerPartialDates(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tn = TemporalNormalizer(
|
||||
reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)
|
||||
)
|
||||
self.tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
|
||||
|
||||
def test_year_only(self):
|
||||
result = self.tn.normalize("2021")
|
||||
@@ -417,13 +396,10 @@ class TestTemporalNormalizerPartialDates(unittest.TestCase):
|
||||
# Part 4 – TemporalNormalizer: ambiguous formats
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTemporalNormalizerAmbiguity(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tn = TemporalNormalizer(
|
||||
reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)
|
||||
)
|
||||
self.tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
|
||||
|
||||
def test_ambiguous_slash_date_raises_warning_and_returns_none(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
@@ -456,19 +432,14 @@ class TestTemporalNormalizerAmbiguity(unittest.TestCase):
|
||||
# Part 5 – TemporalNormalizer: domain phrase map
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTemporalNormalizerDomainPhrases(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tn = TemporalNormalizer(
|
||||
reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)
|
||||
)
|
||||
self.tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
|
||||
|
||||
def _assert_recognized(self, phrase):
|
||||
result = self.tn.normalize_phrase(phrase)
|
||||
self.assertIsNotNone(
|
||||
result, f"Expected phrase {phrase!r} to be recognized but got None"
|
||||
)
|
||||
self.assertIsNotNone(result, f"Expected phrase {phrase!r} to be recognized but got None")
|
||||
return result
|
||||
|
||||
# General / Policy
|
||||
@@ -551,7 +522,6 @@ class TestTemporalNormalizerDomainPhrases(unittest.TestCase):
|
||||
# Part 6 – TemporalNormalizer: custom phrase map
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTemporalNormalizerCustomPhraseMap(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
@@ -595,12 +565,10 @@ class TestTemporalNormalizerCustomPhraseMap(unittest.TestCase):
|
||||
# Part 7 – Full pipeline: extract → normalize → BiTemporalFact
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestFullPipelineTemporalToBiTemporal(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
from semantica.semantic_extract.methods import _result_cache
|
||||
|
||||
_result_cache.clear()
|
||||
|
||||
@patch("semantica.semantic_extract.methods.create_provider")
|
||||
@@ -652,12 +620,10 @@ class TestFullPipelineTemporalToBiTemporal(unittest.TestCase):
|
||||
self.assertEqual(vf[0].day, 1)
|
||||
|
||||
# Feed into BiTemporalFact
|
||||
fact = BiTemporalFact.from_relationship(
|
||||
{
|
||||
"valid_from": "2014-05-01T00:00:00Z",
|
||||
"valid_until": None,
|
||||
}
|
||||
)
|
||||
fact = BiTemporalFact.from_relationship({
|
||||
"valid_from": "2014-05-01T00:00:00Z",
|
||||
"valid_until": None,
|
||||
})
|
||||
self.assertIsNotNone(fact.valid_from)
|
||||
self.assertEqual(fact.valid_from.year, 2014)
|
||||
self.assertEqual(fact.valid_from.month, 5)
|
||||
@@ -694,9 +660,7 @@ class TestFullPipelineTemporalToBiTemporal(unittest.TestCase):
|
||||
extract_temporal_bounds=True,
|
||||
)
|
||||
meta = rels[0].metadata
|
||||
tn = TemporalNormalizer(
|
||||
reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)
|
||||
)
|
||||
tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
|
||||
|
||||
vf = tn.normalize(meta["valid_from"])
|
||||
vu = tn.normalize(meta["valid_until"])
|
||||
|
||||
+11
-40
@@ -2090,16 +2090,12 @@ class TestMCP:
|
||||
# Table renders correctly — at minimum the column header is present
|
||||
assert "Tool" in result.output or "tool" in result.output.lower()
|
||||
|
||||
def test_list_tools_reads_server_catalog(self, runner, monkeypatch):
|
||||
"""list-tools must read TOOL_DEFINITIONS (what the server serves via
|
||||
tools/list), not the module's ``__all__`` (issue #1355)."""
|
||||
import semantica_mcp.mcp.tools as tools_mod
|
||||
fake = [{"name": "fake_tool_from_catalog", "description": "", "inputSchema": {},
|
||||
"_handler": lambda a: {}}]
|
||||
monkeypatch.setattr(tools_mod, "TOOL_DEFINITIONS", fake)
|
||||
def test_list_tools_with_mock_shows_known_tools(self, runner, monkeypatch):
|
||||
fake_tools = _fake_module(__all__=["extract_entities", "query_graph"])
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica_mcp.mcp.tools", fake_tools)
|
||||
result = runner.invoke(cli_module.main, ["mcp", "list-tools"])
|
||||
_ok(result)
|
||||
assert "fake_tool_from_catalog" in result.output
|
||||
assert "extract_entities" in result.output
|
||||
|
||||
def test_list_tools_json(self, runner):
|
||||
result = runner.invoke(cli_module.main, ["mcp", "list-tools", "--json"])
|
||||
@@ -2142,39 +2138,14 @@ class TestMCP:
|
||||
err = json.loads(result.stderr)
|
||||
assert err["error"].startswith("Invalid JSON in --args")
|
||||
|
||||
def test_call_dispatches_through_packaged_server(self, runner):
|
||||
"""Regression for issue #1355: ``mcp call`` dispatches in-process through
|
||||
``semantica_mcp.mcp.server`` (the server ``mcp start`` spawns) instead
|
||||
of importing the nonexistent ``MCPSession``."""
|
||||
result = runner.invoke(
|
||||
cli_module.main, ["--json", "mcp", "call", "extract_entities"]
|
||||
)
|
||||
_ok(result)
|
||||
# Empty args short-circuit before heavy imports; reaching the
|
||||
# handler's own validation proves the dispatch path works.
|
||||
assert "text is required" in result.output
|
||||
|
||||
def test_call_unknown_tool_fails_cleanly(self, runner):
|
||||
result = runner.invoke(cli_module.main, ["mcp", "call", "no_such_tool"])
|
||||
def test_call_import_error_is_clean(self, runner):
|
||||
with patch("builtins.__import__", side_effect=lambda n, *a, **k: (
|
||||
(_ for _ in ()).throw(ImportError(n))
|
||||
if n.startswith("mcp") else __import__(n, *a, **k)
|
||||
)):
|
||||
result = runner.invoke(cli_module.main, ["mcp", "call", "extract_entities"])
|
||||
assert result.exit_code != 0
|
||||
assert "Traceback" not in result.output
|
||||
assert "Unknown tool" in result.output
|
||||
|
||||
def test_call_non_object_args_rejected(self, runner):
|
||||
result = runner.invoke(
|
||||
cli_module.main, ["mcp", "call", "extract_entities", "--args", "[1, 2]"]
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "Traceback" not in result.output
|
||||
assert "--args must be a JSON object" in result.output
|
||||
|
||||
def test_list_tools_json_matches_server_catalog(self, runner):
|
||||
"""The CLI catalog and the MCP server catalog must be the same list."""
|
||||
from semantica_mcp.mcp.tools import TOOL_DEFINITIONS
|
||||
result = runner.invoke(cli_module.main, ["mcp", "list-tools", "--json"])
|
||||
_ok(result)
|
||||
data = _json_output(result)
|
||||
assert data["tools"] == [t["name"] for t in TOOL_DEFINITIONS]
|
||||
|
||||
|
||||
# ─── services group (backward-compat wrapper) ─────────────────────────────────
|
||||
@@ -2331,7 +2302,7 @@ class TestDoctorEmbeddings:
|
||||
checks = self._doctor_checks(runner)
|
||||
st = checks["Embeddings (sentence-transformers)"]
|
||||
assert st["status"] == "fail"
|
||||
assert st["hint"] == "pip install 'semantica[embeddings-local]'"
|
||||
assert st["hint"] == "pip install sentence-transformers"
|
||||
|
||||
def test_deep_probe_detects_fallback_active(self, runner, monkeypatch):
|
||||
self._with_fake_st(monkeypatch)
|
||||
|
||||
@@ -1,470 +0,0 @@
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
import pytest
|
||||
|
||||
from semantica.parse.docx_parser import DOCXParser
|
||||
from semantica.parse.excel_parser import ExcelParser
|
||||
from semantica.parse.html_parser import HTMLParser
|
||||
from semantica.parse.xml_parser import XMLParser
|
||||
from semantica.utils.exceptions import ProcessingError
|
||||
|
||||
|
||||
def _load_toml(file_path: Path) -> dict:
|
||||
"""Load and parse a TOML file across Python 3.8-3.14+ without mode mismatches."""
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
try:
|
||||
import tomllib # Python 3.11+ standard library
|
||||
|
||||
return tomllib.loads(content)
|
||||
except ImportError:
|
||||
try:
|
||||
import tomli # Fast PEP 680 compatible parser for Python < 3.11
|
||||
|
||||
return tomli.loads(content)
|
||||
except ImportError:
|
||||
import toml # Fallback toml parser
|
||||
|
||||
return toml.loads(content)
|
||||
|
||||
|
||||
def test_core_dependencies_count():
|
||||
"""pyproject.toml must contain exactly 22 unique core dependencies."""
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
data = _load_toml(repo_root / "pyproject.toml")
|
||||
deps = data["project"]["dependencies"]
|
||||
normalized_names = {
|
||||
d.split(";")[0].split(">=")[0].split("<")[0].split("==")[0].strip()
|
||||
for d in deps
|
||||
}
|
||||
expected_22 = {
|
||||
"numpy",
|
||||
"pandas",
|
||||
"scipy",
|
||||
"scikit-learn",
|
||||
"rdflib",
|
||||
"networkx",
|
||||
"requests",
|
||||
"chardet",
|
||||
"protobuf",
|
||||
"grpcio",
|
||||
"pillow",
|
||||
"pydantic",
|
||||
"click",
|
||||
"rich",
|
||||
"tqdm",
|
||||
"pyyaml",
|
||||
"toml",
|
||||
"python-dotenv",
|
||||
"loguru",
|
||||
"structlog",
|
||||
"httpx",
|
||||
"pyarrow",
|
||||
}
|
||||
assert normalized_names == expected_22
|
||||
assert len(normalized_names) == 22
|
||||
|
||||
|
||||
def test_optional_extras_defined():
|
||||
"""All required optional extras must be declared in pyproject.toml."""
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
data = _load_toml(repo_root / "pyproject.toml")
|
||||
extras = data["project"]["optional-dependencies"]
|
||||
for extra in [
|
||||
"documents",
|
||||
"ingest-git",
|
||||
"embeddings-local",
|
||||
"nlp-spacy",
|
||||
"viz",
|
||||
"media",
|
||||
"vectorstore-faiss",
|
||||
"graph-embeddings",
|
||||
"all",
|
||||
]:
|
||||
assert extra in extras, f"Missing extra {extra}"
|
||||
all_extra_str = str(extras["all"])
|
||||
for expected_ref in [
|
||||
"documents",
|
||||
"ingest-git",
|
||||
"embeddings-local",
|
||||
"nlp-spacy",
|
||||
"viz",
|
||||
"media",
|
||||
"graph-embeddings",
|
||||
"vectorstore-all",
|
||||
]:
|
||||
assert expected_ref in all_extra_str, f"Missing {expected_ref} in all"
|
||||
# And vectorstore-faiss is in vectorstore-all
|
||||
assert "vectorstore-faiss" in str(extras["vectorstore-all"])
|
||||
|
||||
# Verify nlp-spacy does not declare thinc directly (Qodo bot issue 1)
|
||||
nlp_spacy_deps = str(extras.get("nlp-spacy", []))
|
||||
assert "thinc" not in nlp_spacy_deps, "nlp-spacy should not directly declare thinc"
|
||||
assert "spacy" in nlp_spacy_deps, "nlp-spacy must declare spacy"
|
||||
|
||||
|
||||
def test_core_modules_importable():
|
||||
"""Core modules must be importable without requiring optional extras."""
|
||||
import semantica
|
||||
import semantica.cli
|
||||
import semantica.parse
|
||||
import semantica.ingest
|
||||
import semantica.embeddings
|
||||
import semantica.export
|
||||
import semantica.kg
|
||||
import semantica.vector_store
|
||||
import semantica.visualization
|
||||
import semantica.semantic_extract
|
||||
import semantica.pipeline
|
||||
|
||||
assert semantica.__version__ is not None
|
||||
|
||||
|
||||
def test_docx_parser_lazy_construction_and_parse_hint():
|
||||
with patch("semantica.parse.docx_parser.Document", None):
|
||||
parser = DOCXParser()
|
||||
assert parser is not None
|
||||
with pytest.raises(ProcessingError, match=r"semantica\[documents\]"):
|
||||
parser.parse("nonexistent.docx")
|
||||
|
||||
|
||||
def test_excel_parser_lazy_construction_and_parse_hint():
|
||||
with patch("semantica.parse.excel_parser.load_workbook", None):
|
||||
parser = ExcelParser()
|
||||
assert parser is not None
|
||||
with pytest.raises(ProcessingError, match=r"semantica\[documents\]"):
|
||||
parser.parse("nonexistent.xlsx")
|
||||
|
||||
|
||||
def test_html_parser_lazy_construction_and_parse_hint():
|
||||
with patch("semantica.parse.html_parser.BeautifulSoup", None):
|
||||
parser = HTMLParser()
|
||||
assert parser is not None
|
||||
with pytest.raises(ProcessingError, match=r"semantica\[documents\]"):
|
||||
parser.parse("nonexistent.html")
|
||||
|
||||
|
||||
def test_xml_parser_etree_fallback():
|
||||
with patch("semantica.parse.xml_parser.etree", None):
|
||||
parser = XMLParser()
|
||||
assert parser is not None
|
||||
result = parser.parse("<root><item id='1'>Test</item></root>")
|
||||
assert result is not None
|
||||
assert result.root is not None
|
||||
assert result.root.tag == "root"
|
||||
|
||||
|
||||
def test_xml_parser_lxml_explicit_requires_documents_extra():
|
||||
with patch("semantica.parse.xml_parser.etree", None):
|
||||
parser = XMLParser(engine="lxml")
|
||||
assert parser is not None
|
||||
with pytest.raises(ProcessingError, match=r"semantica\[documents\]"):
|
||||
parser.parse("<root/>")
|
||||
|
||||
|
||||
def test_xml_ingestor_missing_hint():
|
||||
with patch("semantica.ingest.xml_ingestor.etree", None):
|
||||
from semantica.ingest.xml_ingestor import XMLIngestor
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[documents\]"):
|
||||
XMLIngestor()
|
||||
|
||||
|
||||
def test_xml_ingestor_package_import_missing_hint():
|
||||
import semantica.ingest as ingest_mod
|
||||
|
||||
ingest_mod.__dict__.pop("XMLIngestor", None)
|
||||
with patch("semantica.ingest.xml_ingestor.etree", None):
|
||||
with pytest.raises(ImportError, match=r"semantica\[documents\]"):
|
||||
_ = ingest_mod.XMLIngestor
|
||||
|
||||
|
||||
def test_repo_ingestor_missing_hint():
|
||||
with patch("semantica.ingest.repo_ingestor.git", None):
|
||||
from semantica.ingest.repo_ingestor import RepoIngestor
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[ingest-git\]"):
|
||||
RepoIngestor()
|
||||
|
||||
|
||||
def test_repo_ingestor_package_import_missing_hint():
|
||||
import semantica.ingest as ingest_mod
|
||||
|
||||
ingest_mod.__dict__.pop("RepoIngestor", None)
|
||||
with patch("semantica.ingest.repo_ingestor.git", None):
|
||||
with pytest.raises(ImportError, match=r"semantica\[ingest-git\]"):
|
||||
_ = ingest_mod.RepoIngestor
|
||||
|
||||
|
||||
def test_git_analyzer_package_import_succeeds_without_git():
|
||||
import semantica.ingest as ingest_mod
|
||||
|
||||
ingest_mod.__dict__.pop("GitAnalyzer", None)
|
||||
with patch("semantica.ingest.repo_ingestor.git", None):
|
||||
analyzer_cls = ingest_mod.GitAnalyzer
|
||||
assert analyzer_cls is not None
|
||||
analyzer = analyzer_cls()
|
||||
assert analyzer is not None
|
||||
|
||||
|
||||
def test_salesforce_ingestor_package_import_missing_hint():
|
||||
import semantica.ingest as ingest_mod
|
||||
|
||||
ingest_mod.__dict__.pop("SalesforceIngestor", None)
|
||||
with patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", False):
|
||||
with pytest.raises(ImportError, match=r"semantica\[db-salesforce\]"):
|
||||
_ = ingest_mod.SalesforceIngestor
|
||||
|
||||
|
||||
def test_parse_methods_dynamic_default_resolution():
|
||||
""""default" must NOT be registered in the method registry: each built-in
|
||||
dispatcher (parse_document, ...) starts with
|
||||
method_registry.get(<task>, method), so a self-registered "default" would
|
||||
resolve to the dispatcher itself and recurse infinitely. "default" stays
|
||||
the built-in code path, reached only by falling through an unregistered
|
||||
lookup, and callers can still register their own "default" to override
|
||||
it."""
|
||||
from semantica.parse.methods import get_parse_method, list_available_methods
|
||||
|
||||
assert get_parse_method("document", "default") is None
|
||||
methods = list_available_methods()
|
||||
assert "default" not in methods.get("document", [])
|
||||
assert "default" not in methods.get("structured", [])
|
||||
|
||||
|
||||
def test_node_embedder_gensim_missing_hint():
|
||||
with patch("semantica.kg.node_embeddings.GENSIM_AVAILABLE", False):
|
||||
from semantica.kg.node_embeddings import NodeEmbedder
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[graph-embeddings\]"):
|
||||
NodeEmbedder()
|
||||
|
||||
|
||||
def test_faiss_store_missing_hint():
|
||||
with patch("semantica.vector_store.faiss_store.FAISS_AVAILABLE", False):
|
||||
from semantica.vector_store.faiss_store import FAISSIndexBuilder, FAISSStore
|
||||
|
||||
builder = FAISSIndexBuilder(128)
|
||||
with pytest.raises(ProcessingError, match=r"semantica\[vectorstore-faiss\]"):
|
||||
builder.build_index("flat")
|
||||
store = FAISSStore(128)
|
||||
with pytest.raises(ProcessingError, match=r"semantica\[vectorstore-faiss\]"):
|
||||
store.load_index("nonexistent.faiss")
|
||||
|
||||
|
||||
def test_visualization_missing_hint():
|
||||
import numpy as np
|
||||
from semantica.visualization.embedding_visualizer import EmbeddingVisualizer
|
||||
|
||||
# Plotly is checked via px and go in _check_dependencies
|
||||
with patch("semantica.visualization.embedding_visualizer.px", None):
|
||||
visualizer = EmbeddingVisualizer()
|
||||
with pytest.raises(
|
||||
ProcessingError, match=r"Plotly is required.*semantica\[viz\]"
|
||||
):
|
||||
visualizer.visualize_2d_projection(np.array([[0.1, 0.2], [0.3, 0.4]]))
|
||||
|
||||
with patch("semantica.visualization.embedding_visualizer.go", None):
|
||||
visualizer = EmbeddingVisualizer()
|
||||
with pytest.raises(
|
||||
ProcessingError, match=r"Plotly is required.*semantica\[viz\]"
|
||||
):
|
||||
visualizer.visualize_2d_projection(np.array([[0.1, 0.2], [0.3, 0.4]]))
|
||||
|
||||
|
||||
def test_visualization_umap_missing_hint():
|
||||
import numpy as np
|
||||
from unittest.mock import MagicMock
|
||||
from semantica.visualization.embedding_visualizer import EmbeddingVisualizer
|
||||
|
||||
# Stand in for Plotly so we reach dimensionality reduction
|
||||
with patch("semantica.visualization.embedding_visualizer.px", MagicMock()), patch(
|
||||
"semantica.visualization.embedding_visualizer.go", MagicMock()
|
||||
), patch("semantica.visualization.embedding_visualizer.umap", None):
|
||||
visualizer = EmbeddingVisualizer()
|
||||
# High-dimensional embeddings (>2D) trigger dimensionality reduction
|
||||
# with method="umap"
|
||||
embeddings = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])
|
||||
with pytest.raises(
|
||||
ProcessingError, match=r"UMAP is required.*semantica\[viz\]"
|
||||
):
|
||||
visualizer.visualize_2d_projection(embeddings, method="umap")
|
||||
|
||||
# Also verify 3D projection triggers the same actionable error on >3D embeddings
|
||||
embeddings_4d = np.array(
|
||||
[[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1.0, 1.1, 1.2]]
|
||||
)
|
||||
with pytest.raises(
|
||||
ProcessingError, match=r"UMAP is required.*semantica\[viz\]"
|
||||
):
|
||||
visualizer.visualize_3d_projection(embeddings_4d, method="umap")
|
||||
|
||||
|
||||
def test_visualization_sklearn_missing_hint():
|
||||
import numpy as np
|
||||
from unittest.mock import MagicMock
|
||||
from semantica.visualization.embedding_visualizer import EmbeddingVisualizer
|
||||
|
||||
# Stand in for Plotly so we reach dimensionality reduction
|
||||
with patch("semantica.visualization.embedding_visualizer.px", MagicMock()), patch(
|
||||
"semantica.visualization.embedding_visualizer.go", MagicMock()
|
||||
):
|
||||
visualizer = EmbeddingVisualizer()
|
||||
embeddings = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])
|
||||
|
||||
# Test direct dependency check
|
||||
with patch("semantica.visualization.embedding_visualizer.PCA", None):
|
||||
with pytest.raises(ProcessingError, match=r"scikit-learn is required"):
|
||||
visualizer._check_dependencies(require_sklearn=True)
|
||||
|
||||
with patch("semantica.visualization.embedding_visualizer.PCA", None):
|
||||
with pytest.raises(ProcessingError, match=r"scikit-learn is required"):
|
||||
visualizer.visualize_2d_projection(embeddings, method="pca")
|
||||
|
||||
with patch("semantica.visualization.embedding_visualizer.TSNE", None):
|
||||
with pytest.raises(ProcessingError, match=r"scikit-learn is required"):
|
||||
visualizer.visualize_2d_projection(embeddings, method="tsne")
|
||||
|
||||
|
||||
def test_visualization_options_collision_free():
|
||||
"""Options like n_components, perplexity must not cause keyword collisions."""
|
||||
import numpy as np
|
||||
from unittest.mock import MagicMock
|
||||
from semantica.visualization.embedding_visualizer import EmbeddingVisualizer
|
||||
|
||||
mock_pca = MagicMock()
|
||||
mock_tsne = MagicMock()
|
||||
mock_umap_cls = MagicMock()
|
||||
mock_umap_module = MagicMock()
|
||||
mock_umap_module.UMAP = mock_umap_cls
|
||||
|
||||
with patch("semantica.visualization.embedding_visualizer.PCA", mock_pca), patch(
|
||||
"semantica.visualization.embedding_visualizer.TSNE", mock_tsne
|
||||
), patch(
|
||||
"semantica.visualization.embedding_visualizer.umap", mock_umap_module
|
||||
), patch(
|
||||
"semantica.visualization.embedding_visualizer.px", MagicMock()
|
||||
), patch(
|
||||
"semantica.visualization.embedding_visualizer.go", MagicMock()
|
||||
):
|
||||
visualizer = EmbeddingVisualizer()
|
||||
embeddings = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])
|
||||
|
||||
# PCA with n_components
|
||||
visualizer.visualize_2d_projection(embeddings, method="pca", n_components=2)
|
||||
# TSNE with perplexity and random_state
|
||||
visualizer.visualize_2d_projection(
|
||||
embeddings, method="tsne", perplexity=1, random_state=42
|
||||
)
|
||||
# UMAP with n_neighbors and min_dist
|
||||
visualizer.visualize_2d_projection(
|
||||
embeddings, method="umap", n_neighbors=2, min_dist=0.1
|
||||
)
|
||||
# 3D with n_components
|
||||
visualizer.visualize_3d_projection(embeddings, method="pca", n_components=3)
|
||||
|
||||
|
||||
def test_spacy_load_missing_hint():
|
||||
from semantica.semantic_extract.methods import load_spacy_model
|
||||
|
||||
with patch("semantica.semantic_extract.methods.spacy", None):
|
||||
with pytest.raises(ImportError, match=r"semantica\[nlp-spacy\]"):
|
||||
load_spacy_model("en_core_web_sm")
|
||||
|
||||
|
||||
def test_xml_parser_handles_comments():
|
||||
from semantica.parse.xml_parser import etree as real_lxml_etree
|
||||
|
||||
xml_content = (
|
||||
"<root><!-- top comment --><item id='1'>Value</item>"
|
||||
"<!-- bottom comment --></root>"
|
||||
)
|
||||
# etree engine (always available in a core-only install)
|
||||
p_etree = XMLParser(engine="etree")
|
||||
res_etree = p_etree.parse(xml_content)
|
||||
assert res_etree.root.tag == "root"
|
||||
assert len(res_etree.root.children) == 1
|
||||
assert res_etree.root.children[0].tag == "item"
|
||||
assert res_etree.root.children[0].text == "Value"
|
||||
|
||||
# lxml engine (only meaningful when the 'documents' extra is installed)
|
||||
if real_lxml_etree is None:
|
||||
pytest.skip("lxml not installed (requires semantica[documents])")
|
||||
p_lxml = XMLParser(engine="lxml")
|
||||
res_lxml = p_lxml.parse(xml_content)
|
||||
assert res_lxml.root.tag == "root"
|
||||
assert len(res_lxml.root.children) == 1
|
||||
assert res_lxml.root.children[0].tag == "item"
|
||||
assert res_lxml.root.children[0].text == "Value"
|
||||
|
||||
|
||||
def test_public_api_ingestor_handles_xml_comments():
|
||||
from semantica.ingest.public_api_ingestor import (
|
||||
PublicAPIIngestor,
|
||||
lxml_etree as real_lxml_etree,
|
||||
safe_xml_etree as real_safe_xml_etree,
|
||||
)
|
||||
|
||||
if real_lxml_etree is None and real_safe_xml_etree is None:
|
||||
pytest.skip("neither defusedxml nor lxml installed (requires semantica[documents]/[explorer])")
|
||||
|
||||
xml_content = "<root><!-- comment --><item id='1'>Value</item></root>"
|
||||
ingestor = PublicAPIIngestor(rate_limit_delay=0)
|
||||
|
||||
# 1. Default (defusedxml if available)
|
||||
parsed = ingestor._parse_xml(xml_content)
|
||||
assert parsed["tag"] == "root"
|
||||
assert len(parsed["children"]) == 1
|
||||
assert parsed["children"][0]["tag"] == "item"
|
||||
assert parsed["children"][0]["text"] == "Value"
|
||||
|
||||
# 2. lxml fallback (only meaningful when lxml is actually installed)
|
||||
if real_lxml_etree is None:
|
||||
pytest.skip("lxml not installed (requires semantica[documents])")
|
||||
with patch("semantica.ingest.public_api_ingestor.safe_xml_etree", None):
|
||||
parsed_lxml = ingestor._parse_xml(xml_content)
|
||||
assert parsed_lxml["tag"] == "root"
|
||||
assert len(parsed_lxml["children"]) == 1
|
||||
assert parsed_lxml["children"][0]["tag"] == "item"
|
||||
assert parsed_lxml["children"][0]["text"] == "Value"
|
||||
|
||||
|
||||
def test_huggingface_model_loader_catches_oserror():
|
||||
import builtins
|
||||
from unittest.mock import MagicMock
|
||||
from semantica.semantic_extract.providers import HuggingFaceModelLoader
|
||||
|
||||
mock_torch = MagicMock()
|
||||
mock_torch.Tensor = type("Tensor", (), {})
|
||||
with patch.dict("sys.modules", {"torch": mock_torch}):
|
||||
loader = HuggingFaceModelLoader()
|
||||
# 1. Test ModuleNotFoundError / ImportError
|
||||
with patch.dict("sys.modules", {"transformers": None}):
|
||||
with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
|
||||
loader.load_ner_model("bert-base-cased")
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
|
||||
loader.load_relation_model("bert-base-cased")
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
|
||||
loader.load_triplet_model("t5-base")
|
||||
|
||||
# 2. Test OSError (e.g. corrupt DLL / missing shared library)
|
||||
real_import = builtins.__import__
|
||||
|
||||
def fake_import(name, *args, **kwargs):
|
||||
if name == "transformers":
|
||||
raise OSError("DLL load failed")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
try:
|
||||
builtins.__import__ = fake_import
|
||||
with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
|
||||
loader.load_ner_model("bert-base-cased-oserror")
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
|
||||
loader.load_relation_model("bert-base-cased-oserror")
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
|
||||
loader.load_triplet_model("t5-base-oserror")
|
||||
finally:
|
||||
builtins.__import__ = real_import
|
||||
@@ -1,64 +0,0 @@
|
||||
"""Tests for the shared in-process tool entry point (issue #1355).
|
||||
|
||||
``semantica_mcp.mcp.server.call_tool`` is the dispatch used by both the
|
||||
JSON-RPC ``tools/call`` handler and the ``semantica mcp call`` CLI command,
|
||||
so the two surfaces cannot expose different tool sets.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
from semantica_mcp.mcp import server
|
||||
from semantica_mcp.mcp.server import UnknownToolError, _handle_tools_call, call_tool
|
||||
|
||||
|
||||
class TestCallTool(unittest.TestCase):
|
||||
|
||||
def test_known_tool_dispatches_to_handler(self):
|
||||
# Empty args hit extract_entities' own validation before any heavy
|
||||
# imports, which is enough to prove dispatch reached the handler.
|
||||
result = call_tool("extract_entities", {})
|
||||
self.assertEqual(result["error"], "text is required")
|
||||
|
||||
def test_unknown_tool_raises_unknown_tool_error(self):
|
||||
with self.assertRaises(UnknownToolError):
|
||||
call_tool("no_such_tool", {})
|
||||
|
||||
def test_unknown_tool_error_is_not_a_key_error(self):
|
||||
"""A handler's own KeyError (missing required arg) must remain
|
||||
distinguishable from an unknown tool name."""
|
||||
self.assertFalse(issubclass(UnknownToolError, KeyError))
|
||||
|
||||
|
||||
class TestToolsCallDispatch(unittest.TestCase):
|
||||
|
||||
@staticmethod
|
||||
def _tools_call(name, arguments):
|
||||
return _handle_tools_call(1, {"name": name, "arguments": arguments})
|
||||
|
||||
def test_unknown_tool_returns_method_not_found(self):
|
||||
response = self._tools_call("no_such_tool", {})
|
||||
self.assertEqual(response["error"]["code"], -32601)
|
||||
self.assertEqual(response["error"]["message"], "Unknown tool: no_such_tool")
|
||||
|
||||
def test_handler_key_error_is_internal_error_not_unknown_tool(self):
|
||||
def _boom(args):
|
||||
raise KeyError("category")
|
||||
|
||||
fake = {"name": "boom", "description": "", "inputSchema": {}, "_handler": _boom}
|
||||
with patch.dict(server._TOOL_INDEX, {"boom": fake}):
|
||||
response = self._tools_call("boom", {})
|
||||
self.assertEqual(response["error"]["code"], -32603)
|
||||
|
||||
def test_known_tool_returns_result_content(self):
|
||||
response = self._tools_call("extract_entities", {})
|
||||
self.assertIn("content", response["result"])
|
||||
self.assertTrue(response["result"]["isError"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,630 +0,0 @@
|
||||
"""Regression tests for MCP export_graph — GraphML and Parquet branches.
|
||||
|
||||
Pre-fix bugs
|
||||
------------
|
||||
GraphML:
|
||||
- handle_export_graph({"format": "graphml"}) imported ``GraphMLExporter``
|
||||
which does not exist in ``semantica.export``, raising ImportError on every
|
||||
call and returning ``{"error": "...GraphMLExporter..."}``.
|
||||
- Even if the import were corrected, ``exporter.export(graph)`` passed the
|
||||
raw ContextGraph object and ignored the required ``file_path`` argument.
|
||||
``GraphExporter.export()`` writes to a file and returns ``None``; the old
|
||||
code used its return value as the response data.
|
||||
|
||||
Parquet:
|
||||
- ``exporter.export(graph, include_metadata)`` passed the ContextGraph object
|
||||
as ``data`` (not the kg dict) and ``include_metadata`` (a bool) as
|
||||
``file_path``. Both arguments are wrong: the exporter expects a list/dict
|
||||
of plain dicts as ``data`` and a filesystem path as ``file_path``.
|
||||
- ``ParquetExporter.export()`` returns ``None``; wrapping it in ``str()``
|
||||
always produced the string ``"None"`` as the response data.
|
||||
|
||||
Post-fix expectations
|
||||
---------------------
|
||||
GraphML:
|
||||
- Returns ``{"format": "graphml", "data": "<graphml ...>..."}``
|
||||
- ``data`` is a non-empty string containing valid GraphML XML.
|
||||
- The TemporaryDirectory is cleaned up after the call (no leaked temp files).
|
||||
|
||||
Parquet:
|
||||
- Returns ``{"format": "parquet", "data": {...}, "encoding": "base64"}``
|
||||
- ``data`` is a dict mapping filename → base64-encoded bytes.
|
||||
- At least one ``*.parquet`` key is present.
|
||||
- Each value decodes to non-empty bytes (valid Parquet file magic: b"PAR1").
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import os
|
||||
import unittest
|
||||
|
||||
# Disable progress tracking before any Semantica import so no singleton
|
||||
# writes to stdout during testing.
|
||||
os.environ["SEMANTICA_DISABLE_PROGRESS"] = "1"
|
||||
|
||||
|
||||
def _make_graph():
|
||||
"""Return a ContextGraph with two entities and one relationship."""
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
g = ContextGraph()
|
||||
g.add_node("n1", node_type="entity")
|
||||
g.add_node("n2", node_type="entity")
|
||||
g.add_edge("n1", "n2", "related_to")
|
||||
return g
|
||||
|
||||
|
||||
class TestMCPExportGraphML(unittest.TestCase):
|
||||
"""GraphML export must use GraphExporter (not GraphMLExporter) and return
|
||||
valid GraphML XML."""
|
||||
|
||||
def setUp(self):
|
||||
import semantica_mcp.mcp.session as _session
|
||||
self._orig = _session._graph
|
||||
_session._graph = _make_graph()
|
||||
|
||||
def tearDown(self):
|
||||
import semantica_mcp.mcp.session as _session
|
||||
_session._graph = self._orig
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Regression: old code raised ImportError for GraphMLExporter
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_graphml_does_not_return_import_error(self):
|
||||
"""The old code: ``from semantica.export import GraphMLExporter`` —
|
||||
that class does not exist. Result must not contain 'GraphMLExporter'
|
||||
in the error message."""
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
if "error" in result:
|
||||
self.assertNotIn(
|
||||
"GraphMLExporter", result["error"],
|
||||
f"Import of non-existent GraphMLExporter still present: {result}",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Correctness
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_graphml_returns_success_not_error(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, f"GraphML export returned error: {result}")
|
||||
|
||||
def test_graphml_format_key_is_correct(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertEqual(result.get("format"), "graphml")
|
||||
|
||||
def test_graphml_data_is_non_empty_string(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertIsInstance(result.get("data"), str)
|
||||
self.assertGreater(len(result["data"]), 0)
|
||||
|
||||
def test_graphml_data_contains_xml_declaration(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertIn('<?xml version="1.0"', result["data"])
|
||||
|
||||
def test_graphml_data_contains_graphml_element(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertIn("<graphml", result["data"])
|
||||
self.assertIn("</graphml>", result["data"])
|
||||
|
||||
def test_graphml_data_is_not_the_string_None(self):
|
||||
"""The old code called ``str(exporter.export(graph))`` which returns
|
||||
``'None'`` because ``export()`` returns ``None``."""
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertNotEqual(result.get("data"), "None")
|
||||
|
||||
def test_graphml_is_parseable_xml(self):
|
||||
"""The response must be well-formed XML, not an error string."""
|
||||
import xml.etree.ElementTree as ET
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
try:
|
||||
ET.fromstring(result["data"])
|
||||
except ET.ParseError as exc:
|
||||
self.fail(f"GraphML output is not valid XML: {exc}\n{result['data'][:500]}")
|
||||
|
||||
def test_graphml_no_contextgraph_attribute_error(self):
|
||||
"""The old code passed the ContextGraph object directly. Verify the
|
||||
error 'ContextGraph' object has no attribute 'get' (or similar) is gone."""
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
if "error" in result:
|
||||
self.assertNotIn("ContextGraph", result["error"])
|
||||
self.assertNotIn("has no attribute", result["error"])
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Regression: undeclared / wrongly-scoped GraphML keys.
|
||||
#
|
||||
# Pre-fix, _export_graphml declared:
|
||||
# <key id="type" for="node" …/>
|
||||
# <key id="confidence" for="node" …/>
|
||||
#
|
||||
# and then referenced:
|
||||
# <data key="label"> on BOTH nodes and edges (no declaration at all)
|
||||
# <data key="confidence"> on edges (declared for="node" only)
|
||||
#
|
||||
# Schema-validating consumers (Cytoscape, yEd, any XSD-aware reader)
|
||||
# reject documents with undeclared or out-of-scope key references.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _graphml_key_declarations(self, xml_text: str) -> dict:
|
||||
"""Parse the XML and return {key_id: for_value} for every <key>."""
|
||||
import xml.etree.ElementTree as ET
|
||||
root = ET.fromstring(xml_text)
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
return {k.get("id"): k.get("for") for k in root.findall("g:key", ns)}
|
||||
|
||||
def test_graphml_label_key_is_declared(self):
|
||||
"""label key must be declared; pre-fix it was missing entirely."""
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
keys = self._graphml_key_declarations(result["data"])
|
||||
self.assertIn(
|
||||
"label", keys,
|
||||
f"<key id='label'> is missing from GraphML header; declared keys: {list(keys)}",
|
||||
)
|
||||
|
||||
def test_graphml_label_key_scope_covers_edges(self):
|
||||
"""label is written on both nodes (node label) and edges (edge type).
|
||||
Its for= scope must be 'all' or 'edge'. Pre-fix it was not declared."""
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
keys = self._graphml_key_declarations(result["data"])
|
||||
scope = keys.get("label", "")
|
||||
self.assertIn(
|
||||
scope, ("all", "edge"),
|
||||
f"<key id='label'> has for={scope!r}; must be 'all' or 'edge' "
|
||||
f"because edges write <data key='label'>",
|
||||
)
|
||||
|
||||
def test_graphml_confidence_key_scope_covers_edges(self):
|
||||
"""confidence is written on both nodes and edges when include_attributes
|
||||
is True. Pre-fix the key was declared for='node' only, making every
|
||||
edge confidence reference schema-invalid."""
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
keys = self._graphml_key_declarations(result["data"])
|
||||
scope = keys.get("confidence", "")
|
||||
self.assertIn(
|
||||
scope, ("all", "edge"),
|
||||
f"<key id='confidence'> has for={scope!r}; must be 'all' or 'edge' "
|
||||
f"because edges also write <data key='confidence'>",
|
||||
)
|
||||
|
||||
def test_graphml_all_data_key_refs_are_declared(self):
|
||||
"""Every <data key=X> reference in the document must have a matching
|
||||
<key id=X> declaration — with a graph that has both nodes AND edges,
|
||||
so edge-only violations are not hidden by an edge-free export."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from semantica.export import GraphExporter
|
||||
|
||||
kg = {
|
||||
"entities": [
|
||||
{"id": "n1", "text": "Alice", "type": "Person"},
|
||||
{"id": "n2", "text": "Bob", "type": "Person"},
|
||||
],
|
||||
"relationships": [
|
||||
{"source_id": "n1", "target_id": "n2", "type": "knows"},
|
||||
],
|
||||
}
|
||||
exporter = GraphExporter(format="graphml")
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_path = Path(tmpdir) / "out.graphml"
|
||||
exporter.export_knowledge_graph(kg, file_path=tmp_path)
|
||||
xml_text = tmp_path.read_text(encoding="utf-8")
|
||||
|
||||
root = ET.fromstring(xml_text)
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
|
||||
declared_ids = {k.get("id") for k in root.findall("g:key", ns)}
|
||||
referenced_ids = {d.get("key") for d in root.findall(".//g:data", ns)}
|
||||
|
||||
undeclared = referenced_ids - declared_ids
|
||||
self.assertEqual(
|
||||
undeclared, set(),
|
||||
f"GraphML references key id(s) with no <key> declaration: {undeclared}. "
|
||||
f"Declared: {declared_ids}",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Regression: export() consumed nodes/edges while to_kg_dict() returns
|
||||
# entities/relationships — the GraphML was structurally valid XML but
|
||||
# silently contained zero nodes and zero edges.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_graphml_contains_graph_nodes(self):
|
||||
"""Entities in the source graph must appear as <node> elements.
|
||||
Pre-fix: exporter.export(kg_dict) read 'nodes' key (absent in
|
||||
to_kg_dict()); export_knowledge_graph() converts entities -> nodes."""
|
||||
import xml.etree.ElementTree as ET
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
root = ET.fromstring(result["data"])
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
nodes = root.findall(".//g:node", ns)
|
||||
self.assertGreater(
|
||||
len(nodes), 0,
|
||||
"GraphML contains zero <node> elements; to_kg_dict() entities were "
|
||||
"not converted — export_knowledge_graph() must be used, not export().",
|
||||
)
|
||||
|
||||
def test_graphml_contains_graph_edges(self):
|
||||
"""Relationships in the source graph must appear as <edge> elements."""
|
||||
import xml.etree.ElementTree as ET
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
root = ET.fromstring(result["data"])
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
edges = root.findall(".//g:edge", ns)
|
||||
self.assertGreater(
|
||||
len(edges), 0,
|
||||
"GraphML contains zero <edge> elements; to_kg_dict() relationships were "
|
||||
"not converted — export_knowledge_graph() must be used, not export().",
|
||||
)
|
||||
|
||||
def test_graphml_node_ids_match_source_graph(self):
|
||||
"""The <node id=...> values must match the entity IDs from the source graph."""
|
||||
import xml.etree.ElementTree as ET
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
root = ET.fromstring(result["data"])
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
node_ids = {n.get("id") for n in root.findall(".//g:node", ns)}
|
||||
# _make_graph() adds nodes with id "n1" and "n2"
|
||||
self.assertIn("n1", node_ids, f"Expected 'n1' in GraphML node ids; got {node_ids}")
|
||||
self.assertIn("n2", node_ids, f"Expected 'n2' in GraphML node ids; got {node_ids}")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Regression: XML-special characters in graph values must be escaped.
|
||||
# Pre-fix _export_graphml used bare f-string interpolation so a node
|
||||
# id of 'a&b' produced <node id="a&b"> which is malformed XML.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_graphml_xml_special_chars_in_node_id_produce_well_formed_xml(self):
|
||||
"""A node id containing & < > must be escaped in the id= attribute so
|
||||
the output remains well-formed XML. Pre-fix this produced
|
||||
<node id="a&b"> which is a parse error."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import semantica_mcp.mcp.session as _session
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
|
||||
g = ContextGraph()
|
||||
g.add_node("a&b<c>d", node_type="entity")
|
||||
_orig = _session._graph
|
||||
_session._graph = g
|
||||
try:
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
finally:
|
||||
_session._graph = _orig
|
||||
|
||||
self.assertNotIn("error", result, result)
|
||||
# Must parse without raising; pre-fix this raised ET.ParseError
|
||||
try:
|
||||
root = ET.fromstring(result["data"])
|
||||
except ET.ParseError as exc:
|
||||
self.fail(
|
||||
f"GraphML with special-char node id is malformed XML: {exc}\n"
|
||||
f"{result['data'][:600]}"
|
||||
)
|
||||
# The id attribute value must round-trip to the original string
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
node_ids = {n.get("id") for n in root.findall(".//g:node", ns)}
|
||||
self.assertIn(
|
||||
"a&b<c>d", node_ids,
|
||||
f"Node id did not round-trip correctly; got {node_ids}",
|
||||
)
|
||||
|
||||
def test_graphml_xml_special_chars_in_label_produce_well_formed_xml(self):
|
||||
"""A label containing & < > must be escaped in the <data> text
|
||||
node. Pre-fix <data key="label">A & B</data> is malformed XML.
|
||||
|
||||
Drives GraphExporter directly with a hand-crafted KG dict to avoid
|
||||
relying on ContextGraph internal field mapping."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from semantica.export import GraphExporter
|
||||
|
||||
special_label = "price < 100 & qty > 0"
|
||||
kg = {
|
||||
"entities": [
|
||||
{"id": "e1", "text": special_label, "type": "metric"},
|
||||
],
|
||||
"relationships": [],
|
||||
}
|
||||
|
||||
exporter = GraphExporter(format="graphml")
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_path = Path(tmpdir) / "out.graphml"
|
||||
exporter.export_knowledge_graph(kg, file_path=tmp_path)
|
||||
xml_text = tmp_path.read_text(encoding="utf-8")
|
||||
|
||||
# Must parse without error
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError as exc:
|
||||
self.fail(
|
||||
f"GraphML with special-char label is malformed XML: {exc}\n"
|
||||
f"{xml_text[:600]}"
|
||||
)
|
||||
# Label text must round-trip to the original string
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
data_texts = [
|
||||
d.text
|
||||
for d in root.findall(".//g:data", ns)
|
||||
if d.get("key") == "label"
|
||||
]
|
||||
self.assertIn(
|
||||
special_label, data_texts,
|
||||
f"Label text did not round-trip; found label data: {data_texts}",
|
||||
)
|
||||
|
||||
def test_graphml_double_quotes_in_node_id_produce_well_formed_xml(self):
|
||||
"""A node id containing double-quotes must not break the id=\" attribute
|
||||
boundary. Pre-fix: <node id="say "hi""> is malformed."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import semantica_mcp.mcp.session as _session
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
|
||||
g = ContextGraph()
|
||||
g.add_node('say "hi"', node_type="entity")
|
||||
_orig = _session._graph
|
||||
_session._graph = g
|
||||
try:
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
finally:
|
||||
_session._graph = _orig
|
||||
|
||||
self.assertNotIn("error", result, result)
|
||||
try:
|
||||
root = ET.fromstring(result["data"])
|
||||
except ET.ParseError as exc:
|
||||
self.fail(
|
||||
f"GraphML with double-quote node id is malformed XML: {exc}\n"
|
||||
f"{result['data'][:600]}"
|
||||
)
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
node_ids = {n.get("id") for n in root.findall(".//g:node", ns)}
|
||||
self.assertIn(
|
||||
'say "hi"', node_ids,
|
||||
f"Node id with double-quotes did not round-trip; got {node_ids}",
|
||||
)
|
||||
|
||||
def test_graphml_xml_special_chars_in_edge_source_target_produce_well_formed_xml(self):
|
||||
"""Edge source= and target= attributes must also be escaped."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import semantica_mcp.mcp.session as _session
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
|
||||
g = ContextGraph()
|
||||
g.add_node("src&node", node_type="entity")
|
||||
g.add_node("tgt<node>", node_type="entity")
|
||||
g.add_edge("src&node", "tgt<node>", "link")
|
||||
_orig = _session._graph
|
||||
_session._graph = g
|
||||
try:
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
finally:
|
||||
_session._graph = _orig
|
||||
|
||||
self.assertNotIn("error", result, result)
|
||||
try:
|
||||
root = ET.fromstring(result["data"])
|
||||
except ET.ParseError as exc:
|
||||
self.fail(
|
||||
f"GraphML with special-char edge endpoints is malformed XML: {exc}\n"
|
||||
f"{result['data'][:600]}"
|
||||
)
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
edge_els = root.findall(".//g:edge", ns)
|
||||
self.assertEqual(len(edge_els), 1, "Expected exactly one edge element")
|
||||
self.assertEqual(edge_els[0].get("source"), "src&node")
|
||||
self.assertEqual(edge_els[0].get("target"), "tgt<node>")
|
||||
|
||||
|
||||
class TestMCPExportParquet(unittest.TestCase):
|
||||
"""Parquet export must use export_knowledge_graph(), return base64-encoded
|
||||
Parquet bytes, and clean up temporary files."""
|
||||
|
||||
def setUp(self):
|
||||
import semantica_mcp.mcp.session as _session
|
||||
self._orig = _session._graph
|
||||
_session._graph = _make_graph()
|
||||
|
||||
def tearDown(self):
|
||||
import semantica_mcp.mcp.session as _session
|
||||
_session._graph = self._orig
|
||||
|
||||
def _skip_if_no_pyarrow(self):
|
||||
try:
|
||||
import pyarrow # noqa: F401
|
||||
except ImportError:
|
||||
self.skipTest("pyarrow not installed")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Regression: old code produced {"format": "parquet", "data": "None"}
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_parquet_data_is_not_the_string_None(self):
|
||||
"""The old code: ``str(exporter.export(graph, include_metadata))``
|
||||
always produced ``"None"``."""
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertNotEqual(result.get("data"), "None")
|
||||
self.assertNotEqual(result.get("data"), None)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Regression: old code passed ContextGraph and bool to export()
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_parquet_does_not_return_contextgraph_type_error(self):
|
||||
"""The old code passed a ContextGraph as ``data`` and bool as
|
||||
``file_path``. Ensure neither TypeError appears in the result."""
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
if "error" in result:
|
||||
self.assertNotIn("ContextGraph", result["error"])
|
||||
self.assertNotIn("file_path", result["error"])
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Correctness
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_parquet_returns_success_not_error(self):
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, f"Parquet export returned error: {result}")
|
||||
|
||||
def test_parquet_format_key_is_correct(self):
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertEqual(result.get("format"), "parquet")
|
||||
|
||||
def test_parquet_encoding_is_base64(self):
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertEqual(result.get("encoding"), "base64")
|
||||
|
||||
def test_parquet_data_is_dict_of_filenames_to_strings(self):
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, result)
|
||||
data = result.get("data")
|
||||
self.assertIsInstance(data, dict, f"Expected dict, got {type(data)}: {data!r}")
|
||||
for k, v in data.items():
|
||||
self.assertIsInstance(k, str, f"key {k!r} is not str")
|
||||
self.assertIsInstance(v, str, f"value for {k!r} is not str")
|
||||
|
||||
def test_parquet_data_contains_at_least_one_parquet_file(self):
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, result)
|
||||
data = result.get("data", {})
|
||||
parquet_keys = [k for k in data if k.endswith(".parquet")]
|
||||
self.assertGreater(len(parquet_keys), 0,
|
||||
f"No .parquet keys in data: {list(data.keys())}")
|
||||
|
||||
def test_parquet_values_decode_to_valid_parquet_magic(self):
|
||||
"""Each base64 value must decode to bytes starting with the Parquet
|
||||
magic number b'PAR1' (first 4 bytes)."""
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, result)
|
||||
data = result.get("data", {})
|
||||
for filename, b64_str in data.items():
|
||||
raw = base64.b64decode(b64_str)
|
||||
self.assertGreater(len(raw), 4, f"{filename}: decoded to {len(raw)} bytes")
|
||||
self.assertEqual(
|
||||
raw[:4], b"PAR1",
|
||||
f"{filename}: expected Parquet magic b'PAR1', got {raw[:4]!r}",
|
||||
)
|
||||
|
||||
def test_parquet_no_pyarrow_returns_graceful_error(self):
|
||||
"""When pyarrow is absent the handler must return a dict with 'error'
|
||||
key, not raise an exception. Simulate by patching the import."""
|
||||
import sys
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
# Temporarily hide pyarrow
|
||||
real_pyarrow = sys.modules.pop("pyarrow", None)
|
||||
# Also hide the real ParquetExporter so the import inside the handler fails
|
||||
import semantica.export as _export_mod
|
||||
real_exporter_class = _export_mod.ParquetExporter
|
||||
# Replace with the dummy that raises ImportError on init
|
||||
class _MissingParquet:
|
||||
def __init__(self, *a, **kw):
|
||||
raise ImportError("pyarrow is not installed")
|
||||
_export_mod.ParquetExporter = _MissingParquet
|
||||
try:
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertIn("error", result)
|
||||
self.assertIn("pyarrow", result["error"].lower())
|
||||
finally:
|
||||
_export_mod.ParquetExporter = real_exporter_class
|
||||
if real_pyarrow is not None:
|
||||
sys.modules["pyarrow"] = real_pyarrow
|
||||
|
||||
|
||||
class TestMCPExportPreservesExistingFormats(unittest.TestCase):
|
||||
"""Adding GraphML/Parquet fixes must not regress JSON, CSV, or RDF."""
|
||||
|
||||
def setUp(self):
|
||||
import semantica_mcp.mcp.session as _session
|
||||
self._orig = _session._graph
|
||||
_session._graph = _make_graph()
|
||||
|
||||
def tearDown(self):
|
||||
import semantica_mcp.mcp.session as _session
|
||||
_session._graph = self._orig
|
||||
|
||||
def test_json_still_works(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "json"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertEqual(result.get("format"), "json")
|
||||
self.assertIsInstance(result.get("data"), dict)
|
||||
|
||||
def test_csv_still_works(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "csv"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertEqual(result.get("format"), "csv")
|
||||
self.assertIn("id,label,type", result.get("data", ""))
|
||||
|
||||
def test_turtle_still_works(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "turtle"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertIsInstance(result.get("data"), str)
|
||||
self.assertIn("@prefix", result["data"])
|
||||
|
||||
def test_nt_still_works(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "nt"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertIsInstance(result.get("data"), str)
|
||||
self.assertGreater(len(result["data"]), 0)
|
||||
|
||||
def test_unsupported_format_still_returns_error(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "yaml"})
|
||||
self.assertIn("error", result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -91,7 +91,7 @@ _INIT_REQUEST = _jsonrpc("initialize", 1, {
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMCPStdioFramingContract(unittest.TestCase):
|
||||
"""Run 'python -m semantica_mcp.mcp' exactly as an MCP client would, over a real pipe.
|
||||
"""Run 'python -m mcp' exactly as an MCP client would, over a real pipe.
|
||||
|
||||
Each test sends a complete JSON-RPC session through stdin and asserts that
|
||||
every byte on stdout is valid JSON — catching the exact failure mode from
|
||||
@@ -102,7 +102,7 @@ class TestMCPStdioFramingContract(unittest.TestCase):
|
||||
|
||||
def _run(self, *requests: bytes) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
[sys.executable, "-m", "semantica_mcp.mcp"],
|
||||
[sys.executable, "-m", "mcp"],
|
||||
input=b"".join(requests),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
"""Guard for the aggregate ``all`` extras in pyproject.toml.
|
||||
|
||||
Review of #1508 caught ``parse-pdf`` missing from every ``all`` bundle:
|
||||
``semantica[all]`` installed the package without pdfplumber, so the default
|
||||
PDFParser raised ProcessingError on first use. tomllib is stdlib only from
|
||||
Python 3.11, so the bundle block is parsed textually — the repo still
|
||||
supports 3.9/3.10.
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
PYPROJECT = Path(__file__).resolve().parents[1] / "pyproject.toml"
|
||||
|
||||
|
||||
def _extras_in_all_bundles():
|
||||
"""Collect every extra name referenced inside the ``all = [...]`` block."""
|
||||
text = PYPROJECT.read_text(encoding="utf-8")
|
||||
match = re.search(r"^all = \[\n(.*?)^\]", text, re.MULTILINE | re.DOTALL)
|
||||
assert match, "pyproject.toml must define an aggregate 'all' extra"
|
||||
|
||||
names = set()
|
||||
for bundle in re.findall(r"semantica\[([^\]]+)\]", match.group(1)):
|
||||
names.update(part.strip() for part in bundle.split(","))
|
||||
assert names, "'all' bundles must reference at least one extra"
|
||||
return names
|
||||
|
||||
|
||||
def test_all_bundles_include_parse_pdf():
|
||||
"""An all-features install must include built-in PDF parsing support."""
|
||||
assert "parse-pdf" in _extras_in_all_bundles()
|
||||
|
||||
|
||||
def test_all_bundle_extraction_finds_known_extra():
|
||||
"""Control: the extraction reads the right block (parse-docling is in)."""
|
||||
assert "parse-docling" in _extras_in_all_bundles()
|
||||
@@ -50,11 +50,10 @@ CLOUD_BACKENDS = sorted(_AVAILABILITY_FLAG)
|
||||
# Backends that store locally and need no connection step.
|
||||
_LOCAL_BACKENDS = {"inmemory", "faiss", "sqlite", "pgvector"}
|
||||
|
||||
# The facade dispatches store_vectors() to `add`, `add_vectors`, or
|
||||
# `insert_vectors`. Milvus exposes add_vectors and qdrant insert_vectors, so
|
||||
# both resolve; the remaining two name their write method differently and fall
|
||||
# through to NotImplementedError.
|
||||
_NO_WRITE_DISPATCH = {"pinecone", "weaviate"}
|
||||
# The facade dispatches store_vectors() to `add` or `add_vectors`. Milvus
|
||||
# exposes add_vectors so it already resolves; the other three name their write
|
||||
# method differently and fall through to NotImplementedError.
|
||||
_NO_WRITE_DISPATCH = {"qdrant", "pinecone", "weaviate"}
|
||||
|
||||
|
||||
def _construct(backend):
|
||||
|
||||
@@ -1,614 +0,0 @@
|
||||
"""Regression tests for issue #1029: VectorStore inmemory backend must not
|
||||
reissue live vector IDs after a deletion.
|
||||
|
||||
Before the fix, `store_vectors()` derived new IDs as
|
||||
``f"vec_{len(self.vectors) + i}"``. After any deletion `len` decreases,
|
||||
so the next insertion generates an ID that already belongs to a surviving
|
||||
vector, silently overwriting its embedding and metadata.
|
||||
|
||||
Three test groups:
|
||||
|
||||
1. ``TestInmemoryIdNoReuseAfterDelete`` — direct VectorStore path
|
||||
2. ``TestAgentMemoryIdNoReuseAfterDelete`` — AgentMemory path
|
||||
3. ``TestInmemoryIdPersistence`` — save / load / delete / store cycle
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper: build a lightweight VectorStore(backend="inmemory") without
|
||||
# triggering the real EmbeddingGenerator or VectorIndexer.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_store(dim: int = 4) -> "VectorStore": # noqa: F821
|
||||
"""Return an inmemory VectorStore with heavy components mocked out."""
|
||||
from semantica.vector_store.vector_store import VectorStore
|
||||
|
||||
with patch("semantica.vector_store.vector_store.get_logger",
|
||||
return_value=MagicMock()), \
|
||||
patch("semantica.vector_store.vector_store.get_progress_tracker",
|
||||
return_value=MagicMock()), \
|
||||
patch("semantica.vector_store.vector_store.VectorIndexer"), \
|
||||
patch("semantica.vector_store.vector_store.VectorRetriever"), \
|
||||
patch("semantica.vector_store.vector_store.EmbeddingGenerator"):
|
||||
store = VectorStore(backend="inmemory", config={"dimension": dim})
|
||||
store.embedder = None
|
||||
return store
|
||||
|
||||
|
||||
_RNG = np.random.default_rng(seed=1029)
|
||||
|
||||
|
||||
def _vec(dim: int = 4) -> np.ndarray:
|
||||
return _RNG.random(dim).astype(np.float32)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Direct VectorStore path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestInmemoryIdNoReuseAfterDelete(unittest.TestCase):
|
||||
"""Store A+B, delete A, store C — B and C must have distinct IDs and both
|
||||
survive with correct metadata."""
|
||||
|
||||
def _store(self) -> "VectorStore": # noqa: F821
|
||||
return _make_store()
|
||||
|
||||
def test_b_and_c_have_different_ids(self):
|
||||
"""Core regression: id_c must not equal id_b."""
|
||||
store = self._store()
|
||||
id_a, id_b = store.store_vectors(
|
||||
[_vec(), _vec()], [{"label": "A"}, {"label": "B"}]
|
||||
)
|
||||
store.delete_vectors([id_a])
|
||||
(id_c,) = store.store_vectors([_vec()], [{"label": "C"}])
|
||||
|
||||
self.assertNotEqual(
|
||||
id_b, id_c,
|
||||
f"ID collision: both B and C got id={id_b!r}",
|
||||
)
|
||||
|
||||
def test_both_vectors_remain_after_delete_reinsert(self):
|
||||
"""B's entry must survive the deletion of A and the insertion of C."""
|
||||
store = self._store()
|
||||
id_a, id_b = store.store_vectors(
|
||||
[_vec(), _vec()], [{"label": "A"}, {"label": "B"}]
|
||||
)
|
||||
store.delete_vectors([id_a])
|
||||
(id_c,) = store.store_vectors([_vec()], [{"label": "C"}])
|
||||
|
||||
self.assertIn(id_b, store.vectors, "B's vector was lost")
|
||||
self.assertIn(id_c, store.vectors, "C's vector was not stored")
|
||||
self.assertIn(id_b, store.metadata, "B's metadata was lost")
|
||||
self.assertIn(id_c, store.metadata, "C's metadata was not stored")
|
||||
|
||||
def test_count_is_two_after_store_delete_store(self):
|
||||
"""After storing 2, deleting 1, storing 1 the count must be 2."""
|
||||
store = self._store()
|
||||
id_a, _ = store.store_vectors(
|
||||
[_vec(), _vec()], [{}, {}]
|
||||
)
|
||||
store.delete_vectors([id_a])
|
||||
store.store_vectors([_vec()], [{}])
|
||||
|
||||
self.assertEqual(len(store.vectors), 2)
|
||||
|
||||
def test_b_metadata_not_overwritten_by_c(self):
|
||||
"""B's metadata must be unchanged after C is stored."""
|
||||
store = self._store()
|
||||
id_a, id_b = store.store_vectors(
|
||||
[_vec(), _vec()],
|
||||
[{"label": "A"}, {"label": "B", "sentinel": True}],
|
||||
)
|
||||
store.delete_vectors([id_a])
|
||||
store.store_vectors([_vec()], [{"label": "C"}])
|
||||
|
||||
self.assertEqual(
|
||||
store.metadata[id_b],
|
||||
{"label": "B", "sentinel": True},
|
||||
"B's metadata was silently overwritten",
|
||||
)
|
||||
|
||||
def test_id_uniqueness_across_multiple_delete_reinsert_cycles(self):
|
||||
"""Each cycle of delete-then-store must produce a fresh, unique ID."""
|
||||
store = self._store()
|
||||
seen_ids: set = set()
|
||||
|
||||
# Initial batch
|
||||
batch = store.store_vectors([_vec() for _ in range(3)], [{} for _ in range(3)])
|
||||
seen_ids.update(batch)
|
||||
|
||||
# Three delete-then-store cycles
|
||||
for current_id in list(batch):
|
||||
store.delete_vectors([current_id])
|
||||
(new_id,) = store.store_vectors([_vec()], [{}])
|
||||
self.assertNotIn(
|
||||
new_id, seen_ids,
|
||||
f"Generated ID {new_id!r} collides with a previously used ID",
|
||||
)
|
||||
seen_ids.add(new_id)
|
||||
|
||||
def test_counter_skips_explicit_vec_n_ids(self):
|
||||
"""The monotonic counter must skip over any explicit ``vec_N`` already
|
||||
present so it never collides with a manually supplied ID."""
|
||||
store = self._store()
|
||||
|
||||
# Manually insert vec_1 so the counter must skip it
|
||||
store.vectors["vec_1"] = _vec()
|
||||
store.metadata["vec_1"] = {"explicit": True}
|
||||
|
||||
# Ask for two auto-generated IDs; one would be vec_1 if the counter
|
||||
# did not skip it.
|
||||
ids = store.store_vectors([_vec(), _vec()], [{}, {}])
|
||||
|
||||
self.assertNotIn("vec_1", ids, "Monotonic counter re-generated an explicit ID")
|
||||
# vec_1's explicit entry must be intact
|
||||
self.assertEqual(store.metadata["vec_1"], {"explicit": True})
|
||||
# Both new vectors must actually be in the store
|
||||
for new_id in ids:
|
||||
self.assertIn(new_id, store.vectors)
|
||||
|
||||
def test_auto_generated_id_never_silently_overwrites_live_vector(self):
|
||||
"""An automatically generated ID must never land on top of a live
|
||||
auto-generated vector, regardless of deletion history.
|
||||
|
||||
This is the core of 'Never overwrite an existing live vector id
|
||||
silently' from issue #1029: after any sequence of stores and deletes
|
||||
every auto-generated ID must map to exactly one vector.
|
||||
"""
|
||||
store = self._store()
|
||||
# Store 5 vectors — auto-ids vec_0..vec_4
|
||||
first_batch = store.store_vectors([_vec() for _ in range(5)], [{} for _ in range(5)])
|
||||
# Delete vec_0, vec_1, vec_2 — _next_id stays at 5, so the next
|
||||
# auto-id should be vec_5, vec_6 … NOT vec_2/vec_3/vec_4.
|
||||
store.delete_vectors(first_batch[:3])
|
||||
surviving = set(store.vectors.keys()) # {vec_3, vec_4}
|
||||
|
||||
second_batch = store.store_vectors([_vec(), _vec()], [{}, {}])
|
||||
|
||||
# None of the new IDs must collide with surviving ones
|
||||
for new_id in second_batch:
|
||||
self.assertNotIn(
|
||||
new_id, surviving,
|
||||
f"Auto-generated ID {new_id!r} silently landed on a live vector",
|
||||
)
|
||||
# Both new vectors must be independently present
|
||||
for new_id in second_batch:
|
||||
self.assertIn(new_id, store.vectors)
|
||||
self.assertIn(new_id, store.metadata)
|
||||
# Total count: 2 surviving + 2 new
|
||||
self.assertEqual(len(store.vectors), 4)
|
||||
|
||||
def test_collision_detection_no_silent_overwrite_even_with_corrupted_counter(self):
|
||||
"""Even if _next_id is externally wound back (simulating a corrupt
|
||||
load), the generator must never silently overwrite a live vector.
|
||||
|
||||
The while-loop guarantees this by skipping every occupied candidate
|
||||
until it finds a free slot. The existing vector and its metadata
|
||||
must be completely unchanged after the call.
|
||||
"""
|
||||
store = self._store()
|
||||
# Store vec_0 and vec_1 (_next_id advances to 2)
|
||||
ids = store.store_vectors([_vec(), _vec()], [{"orig": 0}, {"orig": 1}])
|
||||
id_b = ids[1] # vec_1
|
||||
vec_b_before = store.vectors[id_b].copy()
|
||||
meta_b_before = dict(store.metadata[id_b])
|
||||
|
||||
# Corrupt the counter: reset to 0 so candidates start at vec_0/vec_1
|
||||
store._next_id = 0
|
||||
|
||||
# store_vectors must succeed without raising and without overwriting
|
||||
new_ids = store.store_vectors([_vec()], [{"new": True}])
|
||||
|
||||
# The new ID must be some other slot — not vec_0 or vec_1
|
||||
self.assertNotIn(
|
||||
new_ids[0], {ids[0], ids[1]},
|
||||
f"New vector landed on a live ID {new_ids[0]!r} "
|
||||
"(no-silent-overwrite invariant violated)",
|
||||
)
|
||||
# vec_1 must be completely unchanged
|
||||
np.testing.assert_array_equal(
|
||||
store.vectors[id_b], vec_b_before,
|
||||
err_msg="Live vector vec_1 was overwritten by the post-corruption store call",
|
||||
)
|
||||
self.assertEqual(
|
||||
store.metadata[id_b], meta_b_before,
|
||||
"Live metadata for vec_1 was overwritten by the post-corruption store call",
|
||||
)
|
||||
# New vector must actually be in the store
|
||||
self.assertIn(new_ids[0], store.vectors)
|
||||
self.assertEqual(store.metadata[new_ids[0]], {"new": True})
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestAgentMemoryIdNoReuseAfterDelete(unittest.TestCase):
|
||||
"""Exercise the ID-collision fix through AgentMemory.store() /
|
||||
delete_memory() rather than VectorStore directly."""
|
||||
|
||||
def setUp(self):
|
||||
"""Build a real VectorStore(inmemory) and bind it to AgentMemory.
|
||||
|
||||
EmbeddingGenerator is mocked so the test doesn't need a model.
|
||||
"""
|
||||
from semantica.context.agent_memory import AgentMemory
|
||||
from semantica.vector_store.vector_store import VectorStore
|
||||
|
||||
self._embedding_patch = patch(
|
||||
"semantica.context.agent_memory.AgentMemory._generate_embedding",
|
||||
side_effect=lambda text: np.ones(8, dtype=np.float32),
|
||||
)
|
||||
self._embedding_patch.start()
|
||||
|
||||
self.store = VectorStore(backend="inmemory", config={"dimension": 8})
|
||||
# Suppress real EmbeddingGenerator on the store itself
|
||||
self.store.embedder = None
|
||||
|
||||
self.memory = AgentMemory(vector_store=self.store)
|
||||
|
||||
def tearDown(self):
|
||||
self._embedding_patch.stop()
|
||||
|
||||
def test_b_and_c_have_different_vector_ids(self):
|
||||
"""After storing A+B, deleting A, storing C, B and C must have
|
||||
distinct vector IDs."""
|
||||
self.memory.store("content A", memory_id="mem_a", skip_graph=True)
|
||||
self.memory.store("content B", memory_id="mem_b", skip_graph=True)
|
||||
|
||||
vid_b_before = self.memory.vector_ids_for("mem_b")
|
||||
|
||||
self.memory.delete_memory("mem_a")
|
||||
self.memory.store("content C", memory_id="mem_c", skip_graph=True)
|
||||
|
||||
vid_b = self.memory.vector_ids_for("mem_b")
|
||||
vid_c = self.memory.vector_ids_for("mem_c")
|
||||
|
||||
# B's vector IDs must be unchanged — it was never touched.
|
||||
self.assertEqual(vid_b, vid_b_before, "mem_b's vector IDs changed unexpectedly")
|
||||
self.assertTrue(vid_b, "mem_b has no tracked vector IDs")
|
||||
self.assertTrue(vid_c, "mem_c has no tracked vector IDs")
|
||||
self.assertTrue(
|
||||
set(vid_b).isdisjoint(set(vid_c)),
|
||||
f"B and C share vector IDs: {set(vid_b) & set(vid_c)}",
|
||||
)
|
||||
|
||||
def test_both_memories_remain_independently_retrievable(self):
|
||||
"""mem_b and mem_c must both survive and report correct vector-store
|
||||
embeddings after the delete-reinsert cycle."""
|
||||
self.memory.store("content B", memory_id="mem_b", skip_graph=True)
|
||||
self.memory.store("content A", memory_id="mem_a", skip_graph=True)
|
||||
self.memory.delete_memory("mem_a")
|
||||
self.memory.store("content C", memory_id="mem_c", skip_graph=True)
|
||||
|
||||
self.assertIn("mem_b", self.memory.memory_items)
|
||||
self.assertIn("mem_c", self.memory.memory_items)
|
||||
self.assertNotIn("mem_a", self.memory.memory_items)
|
||||
|
||||
# Each surviving memory must have its own live vector in the store
|
||||
for mid in ("mem_b", "mem_c"):
|
||||
vids = self.memory.vector_ids_for(mid)
|
||||
for vid in vids:
|
||||
self.assertIn(
|
||||
vid, self.store.vectors,
|
||||
f"{mid!r} vector id {vid!r} is missing from the store",
|
||||
)
|
||||
|
||||
def test_vector_store_count_is_correct(self):
|
||||
"""After storing 2, deleting 1, storing 1 the vector count must be 2."""
|
||||
self.memory.store("content A", memory_id="mem_a", skip_graph=True)
|
||||
self.memory.store("content B", memory_id="mem_b", skip_graph=True)
|
||||
self.memory.delete_memory("mem_a")
|
||||
self.memory.store("content C", memory_id="mem_c", skip_graph=True)
|
||||
|
||||
self.assertEqual(self.store.count(), 2)
|
||||
|
||||
def test_b_embedding_not_overwritten(self):
|
||||
"""mem_b's vector must be the original embedding, not C's."""
|
||||
# Give B a distinct embedding so we can detect overwriting
|
||||
call_order: list = []
|
||||
|
||||
def _side_effect(text: str) -> np.ndarray:
|
||||
call_order.append(text)
|
||||
# Unique per-call vector based on call order length
|
||||
v = np.zeros(8, dtype=np.float32)
|
||||
v[len(call_order) % 8] = float(len(call_order))
|
||||
return v
|
||||
|
||||
with patch(
|
||||
"semantica.context.agent_memory.AgentMemory._generate_embedding",
|
||||
side_effect=_side_effect,
|
||||
):
|
||||
mem2 = __import__(
|
||||
"semantica.context.agent_memory", fromlist=["AgentMemory"]
|
||||
).AgentMemory(vector_store=self.store)
|
||||
mem2.store("content A", memory_id="mem_a2", skip_graph=True)
|
||||
mem2.store("content B", memory_id="mem_b2", skip_graph=True)
|
||||
b_embedding = mem2.memory_items["mem_b2"].embedding
|
||||
|
||||
mem2.delete_memory("mem_a2")
|
||||
mem2.store("content C", memory_id="mem_c2", skip_graph=True)
|
||||
|
||||
vid_b = mem2.vector_ids_for("mem_b2")
|
||||
self.assertTrue(vid_b, "mem_b2 has no tracked vector ID")
|
||||
stored_b_vec = self.store.vectors.get(vid_b[0])
|
||||
self.assertIsNotNone(stored_b_vec)
|
||||
np.testing.assert_array_equal(
|
||||
stored_b_vec,
|
||||
b_embedding if hasattr(b_embedding, "__len__") else np.array(b_embedding),
|
||||
err_msg="B's stored vector was silently overwritten by C's embedding",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Persistence: save → load → delete → store must not collide
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestInmemoryIdPersistence(unittest.TestCase):
|
||||
"""The _next_id counter must survive save/load so that inserting after a
|
||||
delete-and-reload cycle cannot generate an ID already held by a surviving
|
||||
vector."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmpdir, ignore_errors=True)
|
||||
|
||||
def _fresh_store(self, dim: int = 4):
|
||||
return _make_store(dim=dim)
|
||||
|
||||
def test_next_id_is_persisted_in_json(self):
|
||||
"""save() must write ``next_id`` into store_data.json."""
|
||||
store = self._fresh_store()
|
||||
store.store_vectors([_vec(), _vec(), _vec()], [{}, {}, {}])
|
||||
store.save(self.tmpdir)
|
||||
|
||||
with open(f"{self.tmpdir}/store_data.json", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
|
||||
self.assertIn("next_id", data, "save() did not write 'next_id' to JSON")
|
||||
self.assertGreaterEqual(data["next_id"], 3)
|
||||
|
||||
def test_load_restores_next_id(self):
|
||||
"""load() must restore _next_id so the counter does not restart at 0."""
|
||||
store = self._fresh_store()
|
||||
store.store_vectors([_vec(), _vec(), _vec()], [{}, {}, {}])
|
||||
store.save(self.tmpdir)
|
||||
|
||||
loaded = self._fresh_store()
|
||||
loaded.load(self.tmpdir)
|
||||
|
||||
self.assertGreaterEqual(
|
||||
loaded._next_id, 3,
|
||||
f"load() set _next_id={loaded._next_id}, expected >= 3",
|
||||
)
|
||||
|
||||
def test_no_collision_after_save_load_delete_store(self):
|
||||
"""Full lifecycle: save → load → delete one → store one new vector.
|
||||
The new ID must not collide with any surviving vector."""
|
||||
store = self._fresh_store()
|
||||
# Store vec_0, vec_1, vec_2
|
||||
ids = store.store_vectors([_vec() for _ in range(3)], [{} for _ in range(3)])
|
||||
store.save(self.tmpdir)
|
||||
|
||||
# Load fresh instance
|
||||
loaded = self._fresh_store()
|
||||
loaded.load(self.tmpdir)
|
||||
|
||||
# Delete vec_0 (ntotal drops to 2; without the fix, next id = vec_2)
|
||||
loaded.delete_vectors([ids[0]])
|
||||
surviving = set(loaded.vectors.keys())
|
||||
|
||||
# Store a new vector — must not reuse any surviving ID
|
||||
(new_id,) = loaded.store_vectors([_vec()], [{"new": True}])
|
||||
|
||||
self.assertNotIn(
|
||||
new_id, surviving,
|
||||
f"Generated ID {new_id!r} collides with a surviving ID "
|
||||
f"(surviving={sorted(surviving)})",
|
||||
)
|
||||
self.assertEqual(len(loaded.vectors), 3)
|
||||
|
||||
def test_stale_next_id_in_json_is_clamped(self):
|
||||
"""load() must clamp a stale persisted next_id to at least
|
||||
max(vec_N suffix)+1, guarding against corrupted saves."""
|
||||
store = self._fresh_store()
|
||||
# Stores vec_0, vec_1, vec_2
|
||||
store.store_vectors([_vec() for _ in range(3)], [{}, {}, {}])
|
||||
store.save(self.tmpdir)
|
||||
|
||||
# Corrupt the JSON: set next_id to 1 (below vec_2's suffix+1 = 3)
|
||||
json_path = f"{self.tmpdir}/store_data.json"
|
||||
with open(json_path, encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
data["next_id"] = 1
|
||||
with open(json_path, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh)
|
||||
|
||||
loaded = self._fresh_store()
|
||||
loaded.load(self.tmpdir)
|
||||
|
||||
self.assertGreaterEqual(
|
||||
loaded._next_id, 3,
|
||||
f"Stale next_id=1 was not clamped; got {loaded._next_id} (expected >= 3)",
|
||||
)
|
||||
|
||||
def test_missing_next_id_in_old_json_inferred_from_vec_suffixes(self):
|
||||
"""Older store files without 'next_id' must have the counter inferred
|
||||
from the highest ``vec_N`` suffix so that loading them is safe."""
|
||||
store = self._fresh_store()
|
||||
store.store_vectors([_vec() for _ in range(4)], [{} for _ in range(4)])
|
||||
store.save(self.tmpdir)
|
||||
|
||||
# Remove next_id to simulate an older save file
|
||||
json_path = f"{self.tmpdir}/store_data.json"
|
||||
with open(json_path, encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
data.pop("next_id", None)
|
||||
with open(json_path, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh)
|
||||
|
||||
loaded = self._fresh_store()
|
||||
loaded.load(self.tmpdir)
|
||||
|
||||
# _next_id must be at least 4 (vec_0..vec_3 → max suffix+1 = 4)
|
||||
self.assertGreaterEqual(
|
||||
loaded._next_id, 4,
|
||||
f"Missing next_id not inferred correctly; got {loaded._next_id}",
|
||||
)
|
||||
|
||||
# And a subsequent insert must not collide
|
||||
surviving = set(loaded.vectors.keys())
|
||||
(new_id,) = loaded.store_vectors([_vec()], [{}])
|
||||
self.assertNotIn(
|
||||
new_id, surviving,
|
||||
f"Post-load insert collided: {new_id!r} already in {sorted(surviving)}",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Concurrency: concurrent store_vectors calls must not produce duplicate IDs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestInmemoryIdConcurrency(unittest.TestCase):
|
||||
"""Concurrent store_vectors calls on the same VectorStore must each get
|
||||
unique IDs and all vectors must survive (no silent overwrites)."""
|
||||
|
||||
def test_concurrent_store_vectors_produce_unique_ids(self):
|
||||
"""Two threads storing vectors simultaneously must not collide."""
|
||||
import threading as _threading
|
||||
|
||||
store = _make_store(dim=4)
|
||||
results: list = []
|
||||
errors: list = []
|
||||
|
||||
def _store_batch(n: int) -> None:
|
||||
try:
|
||||
ids = store.store_vectors(
|
||||
[_vec() for _ in range(n)],
|
||||
[{"batch": n, "idx": i} for i in range(n)],
|
||||
)
|
||||
results.extend(ids)
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
threads = [_threading.Thread(target=_store_batch, args=(5,)) for _ in range(6)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
self.assertFalse(errors, f"Threads raised: {errors}")
|
||||
total = 6 * 5
|
||||
self.assertEqual(len(results), total, "Some store calls lost vectors")
|
||||
# All returned IDs must be unique — no two threads got the same ID
|
||||
self.assertEqual(
|
||||
len(set(results)), total,
|
||||
f"Duplicate IDs produced under concurrency: "
|
||||
f"{[x for x in results if results.count(x) > 1]}",
|
||||
)
|
||||
# Every returned ID must be present in the store
|
||||
for vid in results:
|
||||
self.assertIn(vid, store.vectors, f"ID {vid!r} not in store after concurrent insert")
|
||||
|
||||
def test_concurrent_delete_and_store_no_phantom_ids(self):
|
||||
"""A thread deleting while another is storing must not leave the store
|
||||
with stale index entries or inconsistent counts."""
|
||||
import threading as _threading
|
||||
|
||||
store = _make_store(dim=4)
|
||||
initial = store.store_vectors([_vec() for _ in range(4)], [{} for _ in range(4)])
|
||||
errors: list = []
|
||||
|
||||
def _deleter():
|
||||
try:
|
||||
store.delete_vectors(initial[:2])
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
def _storer():
|
||||
try:
|
||||
store.store_vectors([_vec(), _vec()], [{}, {}])
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
t1 = _threading.Thread(target=_deleter)
|
||||
t2 = _threading.Thread(target=_storer)
|
||||
t1.start()
|
||||
t2.start()
|
||||
t1.join()
|
||||
t2.join()
|
||||
|
||||
self.assertFalse(errors, f"Threads raised: {errors}")
|
||||
# After the dust settles the vectors dict and metadata must agree
|
||||
self.assertEqual(
|
||||
set(store.vectors.keys()), set(store.metadata.keys()),
|
||||
"vectors and metadata dicts are out of sync after concurrent delete+store",
|
||||
)
|
||||
|
||||
def test_concurrent_search_and_delete_no_runtime_error(self):
|
||||
"""search_vectors() must not raise RuntimeError when a concurrent
|
||||
delete_vectors() modifies the store during iteration.
|
||||
|
||||
Uses threading.Barrier to make the race deterministic: the search
|
||||
thread announces it is ready just before calling search_similar, and
|
||||
the delete thread fires only after that signal has been received.
|
||||
Without the lock-protected snapshot in search_vectors(), the delete
|
||||
would mutate self.vectors while list() is iterating it, reliably
|
||||
causing 'RuntimeError: dictionary changed size during iteration'.
|
||||
"""
|
||||
import threading as _threading
|
||||
|
||||
store = _make_store(dim=4)
|
||||
vecs = store.store_vectors([_vec() for _ in range(8)], [{} for _ in range(8)])
|
||||
query = _vec()
|
||||
errors: list = []
|
||||
|
||||
# Barrier with 2 parties: searcher + deleter.
|
||||
barrier = _threading.Barrier(2)
|
||||
|
||||
original_search_similar = store.retriever.search_similar
|
||||
|
||||
def _patched_search_similar(q, vectors, keys, k, **kw):
|
||||
# Signal the deleter that iteration is about to begin, then wait
|
||||
# for it to be ready too. Both threads proceed together.
|
||||
barrier.wait(timeout=5)
|
||||
return original_search_similar(q, vectors, keys, k, **kw)
|
||||
|
||||
store.retriever.search_similar = _patched_search_similar
|
||||
|
||||
def _searcher():
|
||||
try:
|
||||
store.search_vectors(query, k=4)
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
def _deleter():
|
||||
barrier.wait(timeout=5) # wait until searcher is mid-search
|
||||
try:
|
||||
store.delete_vectors(vecs[:4])
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
t1 = _threading.Thread(target=_searcher)
|
||||
t2 = _threading.Thread(target=_deleter)
|
||||
t1.start()
|
||||
t2.start()
|
||||
t1.join(timeout=10)
|
||||
t2.join(timeout=10)
|
||||
|
||||
self.assertFalse(
|
||||
errors,
|
||||
f"Concurrent search+delete raised: {errors}",
|
||||
)
|
||||
@@ -1,377 +0,0 @@
|
||||
"""Tests for QdrantCollection.search_points and QdrantStore.get_stats.
|
||||
|
||||
These cover the qdrant-client >=1.16.0 compatibility fixes:
|
||||
|
||||
1. search_points() must call client.query_points() (not the removed .search()),
|
||||
read ScoredPoints from response.points, and map them to the documented
|
||||
Semantica result shape.
|
||||
|
||||
2. get_stats() must not access vectors_count unconditionally; when the field
|
||||
is absent (qdrant-client >=1.16), it falls back to points_count for
|
||||
single-vector collections, and to None for named/multi-vector collections
|
||||
where the per-point vector count is unknown.
|
||||
|
||||
All tests drive the real implementation against a MagicMock client, following
|
||||
the established pattern in test_qdrant_store.py.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from semantica.utils.exceptions import ProcessingError
|
||||
from semantica.vector_store.qdrant_store import QdrantCollection, QdrantStore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _scored_point(point_id, score, payload=None):
|
||||
"""Build a stand-in for a qdrant_client ScoredPoint."""
|
||||
sp = MagicMock()
|
||||
sp.id = point_id
|
||||
sp.score = score
|
||||
sp.payload = payload
|
||||
return sp
|
||||
|
||||
|
||||
def _query_response(*scored_points):
|
||||
"""Build a stand-in for a qdrant_client QueryResponse."""
|
||||
qr = MagicMock()
|
||||
qr.points = list(scored_points)
|
||||
return qr
|
||||
|
||||
|
||||
def _collection_with_query_response(*scored_points):
|
||||
"""QdrantCollection whose client.query_points() returns the given points."""
|
||||
client = MagicMock()
|
||||
client.query_points.return_value = _query_response(*scored_points)
|
||||
return QdrantCollection(client, "test_collection")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# QdrantCollection.search_points — API call
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_calls_query_points_not_search():
|
||||
"""search_points() must call .query_points(), NOT the removed .search()."""
|
||||
collection = _collection_with_query_response()
|
||||
query = np.array([0.1, 0.2, 0.3, 0.4])
|
||||
|
||||
collection.search_points(query, limit=5)
|
||||
|
||||
collection.client.query_points.assert_called_once()
|
||||
collection.client.search.assert_not_called()
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_passes_correct_arguments():
|
||||
"""query_points() must receive collection_name, query list, limit, and payload flag."""
|
||||
collection = _collection_with_query_response()
|
||||
query = np.array([0.1, 0.2, 0.3, 0.4])
|
||||
|
||||
collection.search_points(query, limit=7)
|
||||
|
||||
_, kwargs = collection.client.query_points.call_args
|
||||
assert kwargs["collection_name"] == "test_collection"
|
||||
assert kwargs["query"] == [0.1, 0.2, 0.3, 0.4]
|
||||
assert kwargs["limit"] == 7
|
||||
assert kwargs["with_payload"] is True
|
||||
assert kwargs["with_vectors"] is False
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_passes_query_filter_through():
|
||||
"""The query_filter argument must be forwarded verbatim to query_points()."""
|
||||
collection = _collection_with_query_response()
|
||||
mock_filter = MagicMock()
|
||||
query = np.array([0.5, 0.6])
|
||||
|
||||
collection.search_points(query, limit=3, query_filter=mock_filter)
|
||||
|
||||
_, kwargs = collection.client.query_points.call_args
|
||||
assert kwargs["query_filter"] is mock_filter
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_passes_none_filter_when_unfiltered():
|
||||
"""query_filter=None must be passed through (not omitted) so the server
|
||||
returns all matching vectors rather than raising a missing-argument error."""
|
||||
collection = _collection_with_query_response()
|
||||
query = np.array([0.1, 0.2])
|
||||
|
||||
collection.search_points(query, limit=5, query_filter=None)
|
||||
|
||||
_, kwargs = collection.client.query_points.call_args
|
||||
assert kwargs["query_filter"] is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# QdrantCollection.search_points — result shape
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_result_shape():
|
||||
"""Each result dict must contain id, score, metadata, vector, distance."""
|
||||
sp = _scored_point(42, 0.8, payload={"tag": "ml"})
|
||||
collection = _collection_with_query_response(sp)
|
||||
query = np.array([0.1, 0.2, 0.3])
|
||||
|
||||
results = collection.search_points(query, limit=1)
|
||||
|
||||
assert len(results) == 1
|
||||
r = results[0]
|
||||
assert set(r.keys()) == {"id", "score", "metadata", "vector", "distance"}
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_maps_id_and_payload():
|
||||
"""id and metadata must come from ScoredPoint.id and ScoredPoint.payload."""
|
||||
sp = _scored_point(99, 0.5, payload={"source": "wiki", "year": 2024})
|
||||
collection = _collection_with_query_response(sp)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
|
||||
assert results[0]["id"] == 99
|
||||
assert results[0]["metadata"] == {"source": "wiki", "year": 2024}
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_null_payload_becomes_empty_dict():
|
||||
"""A ScoredPoint with payload=None must produce metadata={}."""
|
||||
sp = _scored_point(7, 0.9, payload=None)
|
||||
collection = _collection_with_query_response(sp)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
|
||||
assert results[0]["metadata"] == {}
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_vector_and_distance_are_none():
|
||||
"""vector and distance fields must always be None (vectors are not fetched)."""
|
||||
sp = _scored_point(1, 0.7, payload={})
|
||||
collection = _collection_with_query_response(sp)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
|
||||
assert results[0]["vector"] is None
|
||||
assert results[0]["distance"] is None
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_score_normalization_midrange():
|
||||
"""Score=0 must map to exactly 0.5 under the normalization formula."""
|
||||
sp = _scored_point(1, 0.0)
|
||||
collection = _collection_with_query_response(sp)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
|
||||
assert results[0]["score"] == pytest.approx(0.5)
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_score_normalization_positive():
|
||||
"""Positive raw scores must map to (0.5, 1.0) under the normalization formula."""
|
||||
sp = _scored_point(1, 1.0)
|
||||
collection = _collection_with_query_response(sp)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
|
||||
# (1.0/(1+1.0) + 1.0) / 2.0 = (0.5 + 1.0) / 2.0 = 0.75
|
||||
assert results[0]["score"] == pytest.approx(0.75)
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_score_normalization_negative():
|
||||
"""Negative raw scores must map to (0.0, 0.5) under the normalization formula."""
|
||||
sp = _scored_point(1, -1.0)
|
||||
collection = _collection_with_query_response(sp)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
|
||||
# (-1.0/(1+1.0) + 1.0) / 2.0 = (−0.5 + 1.0) / 2.0 = 0.25
|
||||
assert results[0]["score"] == pytest.approx(0.25)
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_multiple_results_preserve_order():
|
||||
"""All ScoredPoints in response.points must appear in the output, in order."""
|
||||
points = [_scored_point(i, 1.0 - i * 0.1) for i in range(5)]
|
||||
collection = _collection_with_query_response(*points)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=5)
|
||||
|
||||
assert len(results) == 5
|
||||
assert [r["id"] for r in results] == [0, 1, 2, 3, 4]
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_empty_response():
|
||||
"""An empty response.points list must produce an empty result list."""
|
||||
collection = _collection_with_query_response() # zero points
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=10)
|
||||
|
||||
assert results == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# QdrantCollection.search_points — error handling
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", False)
|
||||
def test_search_points_raises_when_qdrant_unavailable():
|
||||
client = MagicMock()
|
||||
collection = QdrantCollection(client, "test_collection")
|
||||
|
||||
with pytest.raises(ProcessingError):
|
||||
collection.search_points(np.array([0.1, 0.2]), limit=5)
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_wraps_client_errors_as_processing_error():
|
||||
client = MagicMock()
|
||||
client.query_points.side_effect = RuntimeError("network failure")
|
||||
collection = QdrantCollection(client, "test_collection")
|
||||
|
||||
with pytest.raises(ProcessingError, match="network failure"):
|
||||
collection.search_points(np.array([0.1, 0.2]), limit=5)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# QdrantStore.get_stats — vectors_count compatibility
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _store_with_collection_info(**info_attrs):
|
||||
"""QdrantStore with a mocked client.get_collection() response."""
|
||||
store = QdrantStore()
|
||||
store.client = MagicMock()
|
||||
store.collection = MagicMock()
|
||||
store.collection.collection_name = "test_coll"
|
||||
|
||||
info = MagicMock(spec=list(info_attrs.keys()))
|
||||
for attr, val in info_attrs.items():
|
||||
setattr(info, attr, val)
|
||||
store.client.get_collection.return_value = info
|
||||
return store
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_get_stats_uses_vectors_count_when_present():
|
||||
"""On qdrant-client <1.16, vectors_count exists and must be returned."""
|
||||
store = _store_with_collection_info(
|
||||
points_count=10, vectors_count=10, status="green"
|
||||
)
|
||||
|
||||
stats = store.get_stats()
|
||||
|
||||
assert stats["points_count"] == 10
|
||||
assert stats["vectors_count"] == 10
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_get_stats_uses_points_count_when_vectors_count_absent():
|
||||
"""On qdrant-client >=1.16, vectors_count is absent.
|
||||
For a single unnamed-vector collection (config.params.vectors is a
|
||||
VectorParams instance), points_count is the correct substitute.
|
||||
indexed_vectors_count must NOT be used: it counts only vectors
|
||||
in optimised segments and is 0 for freshly-inserted data."""
|
||||
from qdrant_client.models import VectorParams, Distance
|
||||
store = QdrantStore()
|
||||
store.client = MagicMock()
|
||||
store.collection = MagicMock()
|
||||
store.collection.collection_name = "test_coll"
|
||||
|
||||
info = MagicMock(spec=["points_count", "indexed_vectors_count", "config", "status"])
|
||||
info.points_count = 5
|
||||
info.indexed_vectors_count = 0 # typical for freshly-inserted, unoptimised data
|
||||
info.config.params.vectors = VectorParams(size=4, distance=Distance.COSINE)
|
||||
info.status = "green"
|
||||
store.client.get_collection.return_value = info
|
||||
|
||||
stats = store.get_stats()
|
||||
|
||||
assert stats["points_count"] == 5
|
||||
# Must equal points_count (5), NOT indexed_vectors_count (0)
|
||||
assert stats["vectors_count"] == 5
|
||||
assert stats["vectors_count"] != info.indexed_vectors_count
|
||||
assert stats["status"] == "green"
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_get_stats_vectors_count_equals_points_count_when_vectors_count_absent():
|
||||
"""On qdrant-client >=1.16, vectors_count is absent. For a single unnamed-
|
||||
vector collection the fallback is points_count, so both keys are equal.
|
||||
indexed_vectors_count is intentionally absent from this mock to confirm
|
||||
it is not required by the fallback path."""
|
||||
from qdrant_client.models import VectorParams, Distance
|
||||
store = QdrantStore()
|
||||
store.client = MagicMock()
|
||||
store.collection = MagicMock()
|
||||
store.collection.collection_name = "test_coll"
|
||||
|
||||
info = MagicMock(spec=["points_count", "config", "status"])
|
||||
info.points_count = 7
|
||||
info.config.params.vectors = VectorParams(size=8, distance=Distance.COSINE)
|
||||
info.status = "green"
|
||||
store.client.get_collection.return_value = info
|
||||
|
||||
stats = store.get_stats()
|
||||
|
||||
assert stats["points_count"] == 7
|
||||
assert stats["vectors_count"] == 7
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_get_stats_vectors_count_is_none_for_named_multi_vector_collection():
|
||||
"""When vectors_count is absent and the collection uses named/multi vectors
|
||||
(config.params.vectors is a dict), the total cannot be inferred and
|
||||
vectors_count must be None rather than a misleading points_count value."""
|
||||
from qdrant_client.models import VectorParams, Distance
|
||||
store = QdrantStore()
|
||||
store.client = MagicMock()
|
||||
store.collection = MagicMock()
|
||||
store.collection.collection_name = "test_coll"
|
||||
|
||||
info = MagicMock(spec=["points_count", "config", "status"])
|
||||
info.points_count = 4
|
||||
# Named multi-vector: qdrant-client returns a dict of VectorParams
|
||||
info.config.params.vectors = {
|
||||
"text": VectorParams(size=4, distance=Distance.COSINE),
|
||||
"image": VectorParams(size=8, distance=Distance.DOT),
|
||||
}
|
||||
info.status = "green"
|
||||
store.client.get_collection.return_value = info
|
||||
|
||||
stats = store.get_stats()
|
||||
|
||||
assert stats["points_count"] == 4
|
||||
# vectors_count must be None: total vectors = points * num_named_vectors,
|
||||
# and that multiplier is unknown to the caller.
|
||||
assert stats["vectors_count"] is None
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_get_stats_vectors_count_is_none_when_config_inaccessible():
|
||||
"""If the collection config cannot be read (e.g. an older schema or
|
||||
unexpected server response), vectors_count must fall back to None safely
|
||||
without raising."""
|
||||
store = QdrantStore()
|
||||
store.client = MagicMock()
|
||||
store.collection = MagicMock()
|
||||
store.collection.collection_name = "test_coll"
|
||||
|
||||
# Simulate a CollectionInfo that has no config attribute at all
|
||||
info = MagicMock(spec=["points_count", "status"])
|
||||
info.points_count = 3
|
||||
info.status = "green"
|
||||
store.client.get_collection.return_value = info
|
||||
|
||||
stats = store.get_stats()
|
||||
|
||||
assert stats["points_count"] == 3
|
||||
assert stats["vectors_count"] is None
|
||||
@@ -1,126 +0,0 @@
|
||||
"""Regression tests for the Qdrant write path behind the VectorStore facade.
|
||||
|
||||
Qodo review of #1508 found three bugs in the newly wired qdrant dispatch:
|
||||
stored IDs were swallowed (the upsert status dict was returned instead),
|
||||
mismatched ids/vectors silently truncated the write via zip(), and lazy
|
||||
collection init ignored the documented ``collection_name`` option. These
|
||||
tests pin all three.
|
||||
|
||||
qdrant-client is not installed in this environment, so QDRANT_AVAILABLE is
|
||||
patched and PointStruct is replaced with a plain stand-in, following the
|
||||
pattern in test_vector_store_deepdive.py and test_qdrant_store.py.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from semantica.utils.exceptions import ValidationError
|
||||
from semantica.vector_store import VectorStore
|
||||
from semantica.vector_store.qdrant_store import QdrantStore
|
||||
|
||||
VECTORS = [np.array([1.0, 0.0, 0.0]), np.array([0.0, 1.0, 0.0])]
|
||||
METADATA = [{"type": "a"}, {"type": "b"}]
|
||||
|
||||
|
||||
class _Point:
|
||||
"""Stand-in for qdrant_client PointStruct that keeps the point id."""
|
||||
|
||||
def __init__(self, id, vector=None, payload=None):
|
||||
self.id = id
|
||||
self.vector = vector
|
||||
self.payload = payload
|
||||
|
||||
|
||||
def _qdrant_facade(**config):
|
||||
"""VectorStore built through the real qdrant init path, with the network
|
||||
client and an attached collection replaced by mocks."""
|
||||
store = VectorStore(backend="qdrant", config={"dimension": 3, **config})
|
||||
backend = store._backend_store
|
||||
backend.client = MagicMock()
|
||||
backend.collection = MagicMock()
|
||||
return store
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.PointStruct", _Point)
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_facade_returns_generated_ids_not_upsert_status():
|
||||
"""store_vectors() promises callers the stored vector IDs; the qdrant
|
||||
branch used to leak insert_vectors()' upsert status dict instead."""
|
||||
store = _qdrant_facade()
|
||||
|
||||
ids = store.store_vectors(VECTORS, METADATA)
|
||||
|
||||
assert isinstance(ids, list)
|
||||
assert len(ids) == len(VECTORS)
|
||||
assert all(isinstance(i, str) and i for i in ids)
|
||||
upserted = store._backend_store.collection.upsert_points.call_args[0][0]
|
||||
assert [p.id for p in upserted] == ids
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.PointStruct", _Point)
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_facade_returns_caller_supplied_ids_verbatim():
|
||||
store = _qdrant_facade()
|
||||
|
||||
ids = store.store_vectors(VECTORS, METADATA, ids=["doc-a", "doc-b"])
|
||||
|
||||
assert ids == ["doc-a", "doc-b"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ids", [["doc-a"], ["doc-a", "doc-b", "doc-c"]])
|
||||
@patch("semantica.vector_store.qdrant_store.PointStruct", _Point)
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_facade_rejects_ids_count_mismatch(ids):
|
||||
"""insert_vectors() pairs points with zip(vectors, ids); a mismatched
|
||||
batch must fail loudly instead of silently dropping vectors."""
|
||||
store = _qdrant_facade()
|
||||
|
||||
with pytest.raises(ValidationError, match="must match number of vectors"):
|
||||
store.store_vectors(VECTORS, METADATA, ids=ids)
|
||||
store._backend_store.collection.upsert_points.assert_not_called()
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.PointStruct", _Point)
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_backend_insert_vectors_rejects_count_mismatch():
|
||||
"""Direct QdrantStore callers get the same guard as facade callers."""
|
||||
store = QdrantStore()
|
||||
store.client = MagicMock()
|
||||
store.collection = MagicMock()
|
||||
|
||||
with pytest.raises(ValidationError, match="must match number of vectors"):
|
||||
store.insert_vectors(VECTORS, ["only-one"])
|
||||
store.collection.upsert_points.assert_not_called()
|
||||
|
||||
|
||||
def _lazy_collection_name(store):
|
||||
"""Drive _ensure_default_collection and report the name it selected."""
|
||||
store.client = MagicMock()
|
||||
with (
|
||||
patch.object(store, "create_collection") as create,
|
||||
patch.object(store, "get_collection"),
|
||||
):
|
||||
store._ensure_default_collection(3)
|
||||
return create.call_args[0][0]
|
||||
|
||||
|
||||
def test_lazy_collection_uses_documented_collection_name():
|
||||
"""The docs and facade configure qdrant with collection_name=...; lazy
|
||||
init used to look up 'collection' and always fall back to the default."""
|
||||
store = QdrantStore(collection_name="semantica")
|
||||
|
||||
assert _lazy_collection_name(store) == "semantica"
|
||||
|
||||
|
||||
def test_lazy_collection_accepts_legacy_collection_alias():
|
||||
store = QdrantStore(collection="legacy_name")
|
||||
|
||||
assert _lazy_collection_name(store) == "legacy_name"
|
||||
|
||||
|
||||
def test_lazy_collection_defaults_without_config():
|
||||
store = QdrantStore()
|
||||
|
||||
assert _lazy_collection_name(store) == "semantica_default"
|
||||
@@ -112,9 +112,7 @@ class TestQdrantSearchSchema(unittest.TestCase):
|
||||
mock_hit.id = "q_1"
|
||||
mock_hit.score = 0.88
|
||||
mock_hit.payload = {"category": "x"}
|
||||
# search_points prefers the modern query_points API; a MagicMock
|
||||
# exposes it, so the response must carry the hits in .points.
|
||||
mock_client.query_points.return_value = MagicMock(points=[mock_hit])
|
||||
mock_client.search.return_value = [mock_hit]
|
||||
|
||||
coll = QdrantCollection(mock_client, "test_col")
|
||||
results = coll.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
@@ -134,9 +132,7 @@ class TestQdrantSearchSchema(unittest.TestCase):
|
||||
mock_hit_high = MagicMock(id="q_hi", score=50.0, payload={})
|
||||
mock_hit_mid = MagicMock(id="q_mid", score=2.0, payload={})
|
||||
mock_hit_low = MagicMock(id="q_lo", score=1.0, payload={})
|
||||
mock_client.query_points.return_value = MagicMock(
|
||||
points=[mock_hit_high, mock_hit_mid, mock_hit_low]
|
||||
)
|
||||
mock_client.search.return_value = [mock_hit_high, mock_hit_mid, mock_hit_low]
|
||||
|
||||
coll = QdrantCollection(mock_client, "test_col")
|
||||
results = coll.search_points(np.array([0.1, 0.2]), limit=3)
|
||||
|
||||
@@ -209,12 +209,12 @@ class TestVectorStoreDeepDive(unittest.TestCase):
|
||||
mock_client = MagicMock()
|
||||
mock_qdrant_cls.return_value = mock_client
|
||||
|
||||
# Mock search response (modern query_points API: hits in .points)
|
||||
# Mock search response
|
||||
mock_hit = MagicMock()
|
||||
mock_hit.id = "vec_1"
|
||||
mock_hit.score = 0.9
|
||||
mock_hit.payload = {"type": "a"}
|
||||
mock_client.query_points.return_value = MagicMock(points=[mock_hit])
|
||||
mock_client.search.return_value = [mock_hit]
|
||||
|
||||
store = QdrantStore(url="http://localhost:6333")
|
||||
|
||||
|
||||
@@ -38,13 +38,14 @@ class TestOptionalDependencies(unittest.TestCase):
|
||||
with import_without(
|
||||
"semantica.visualization.embedding_visualizer", "umap"
|
||||
) as module:
|
||||
with plotly_doubles(module):
|
||||
with plotly_doubles(module), patch.object(module, "PCA") as mock_pca_class:
|
||||
mock_pca_class.return_value.fit_transform.return_value = np.zeros((4, 2))
|
||||
|
||||
viz = module.EmbeddingVisualizer()
|
||||
embeddings = np.array([[0, 1, 2], [1, 0, 3], [0, 0, 0], [1, 1, 1]])
|
||||
with self.assertRaises(module.ProcessingError) as cm:
|
||||
viz.visualize_2d_projection(embeddings, method="umap")
|
||||
self.assertIn("UMAP is required", str(cm.exception))
|
||||
self.assertIn("semantica[viz]", str(cm.exception))
|
||||
viz.visualize_2d_projection(embeddings, method="umap")
|
||||
|
||||
mock_pca_class.assert_called()
|
||||
|
||||
def test_ontology_visualizer_without_graphviz(self):
|
||||
"""Test OntologyVisualizer behavior when graphviz is missing."""
|
||||
|
||||
Reference in New Issue
Block a user