mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-10 04:00:35 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Sameer Kadam
|
d79695cd42 |
+2694
-256
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+18
-26
@@ -97,10 +97,24 @@ jobs:
|
||||
- name: Build Explorer frontend
|
||||
working-directory: explorer
|
||||
run: npm run build
|
||||
- name: Install core package and base dependencies
|
||||
- name: Install Explorer backend test dependencies
|
||||
run: |
|
||||
# Verify that core semantica installs cleanly with only its base dependencies
|
||||
# (no optional extras) and that core imports and lazy missing-dependency hints work.
|
||||
# Run the deterministic backend path before the all-extras CI
|
||||
# environment is installed. The Explorer extra supplies the
|
||||
# production API dependencies without importing optional vector
|
||||
# providers such as Pinecone during test collection.
|
||||
#
|
||||
# --no-deps + a separate hash-pinned install (rather than the old
|
||||
# `pip install -e ".[explorer]" pytest==9.1.1`) so every fetched
|
||||
# package is hash-verified (Scorecard Pinned-Dependencies); the
|
||||
# local editable install itself has nothing to hash.
|
||||
# .github/requirements/explorer-extra-py311.txt is
|
||||
# `uv pip compile pyproject.toml --extra explorer --python-version 3.11 --constraint requirements-ci.txt --generate-hashes`
|
||||
# - regenerate it the same way if pyproject.toml's base/explorer
|
||||
# deps change. Resolved specifically for this job's python 3.11
|
||||
# (see the Dockerfile's explorer-extra-py313.txt for why this
|
||||
# can't be shared with python 3.13: audioread needs extra
|
||||
# standard-aifc/standard-sunau hashes only on 3.13+).
|
||||
#
|
||||
# --no-deps only skips *runtime* dependency resolution - `-e .`
|
||||
# still does a PEP 517 build, which by default creates an isolated
|
||||
@@ -111,30 +125,8 @@ jobs:
|
||||
# copies instead of fetching its own.
|
||||
pip install -r .github/requirements/pep517-build.txt --require-hashes
|
||||
pip install --no-deps --no-build-isolation -e .
|
||||
pip install -r .github/requirements/base-deps.txt --require-hashes
|
||||
pip install -r .github/requirements/pytest-tool.txt --require-hashes
|
||||
- name: Verify core-only package importability and slim behavior
|
||||
run: |
|
||||
python -c "
|
||||
import semantica
|
||||
print('semantica', semantica.__version__, 'core installed and importable')
|
||||
"
|
||||
pytest -q tests/test_issue_1513_slim_core.py
|
||||
- name: Install Explorer backend test dependencies
|
||||
run: |
|
||||
# Run the deterministic backend path before the all-extras CI
|
||||
# environment is installed. The Explorer extra supplies the
|
||||
# production API dependencies without importing optional vector
|
||||
# providers such as Pinecone during test collection.
|
||||
#
|
||||
# .github/requirements/explorer-extra-py311.txt is
|
||||
# `uv pip compile pyproject.toml --extra explorer --python-version 3.11 --constraint requirements-ci.txt --generate-hashes`
|
||||
# - regenerate it the same way if pyproject.toml's base/explorer
|
||||
# deps change. Resolved specifically for this job's python 3.11
|
||||
# (see the Dockerfile's explorer-extra-py313.txt for why this
|
||||
# can't be shared with python 3.13: audioread needs extra
|
||||
# standard-aifc/standard-sunau hashes only on 3.13+).
|
||||
pip install -r .github/requirements/explorer-extra-py311.txt --require-hashes
|
||||
pip install -r .github/requirements/pytest-tool.txt --require-hashes
|
||||
- name: Test deterministic Explorer backend path
|
||||
run: |
|
||||
pytest -q tests/explorer/test_explorer_deterministic_rendering_e2e.py
|
||||
|
||||
@@ -154,13 +154,13 @@ jobs:
|
||||
fi
|
||||
|
||||
# Vulnerability IDs reviewed and accepted as non-actionable for this
|
||||
# project:
|
||||
# - GHSA-4j2p-28q2-5m79: accelerate<=1.14.0 (transitive via docling-slim).
|
||||
# Path traversal in sharded checkpoint index loading (load_checkpoint_in_model).
|
||||
# 1.14.0 is the latest available PyPI release; no upstream patch exists yet.
|
||||
# Semantica does not load arbitrary user checkpoints. Re-evaluate once
|
||||
# accelerate releases a fixed version.
|
||||
IGNORED_VULN_IDS="GHSA-4j2p-28q2-5m79"
|
||||
# project. Empty for now: pip-audit's OSV-backed database doesn't
|
||||
# currently carry either of the findings Safety used to flag here
|
||||
# (cuda-toolkit CVE-2025-33228, torchvision CVE-2026-65918), so
|
||||
# there's nothing to exclude. Left in place so a future finding can
|
||||
# be added the same way without restructuring this step - see git
|
||||
# history on this file for the reasoning behind past entries.
|
||||
IGNORED_VULN_IDS=""
|
||||
|
||||
# Exported so the "Comment PR with Security Results" step below can
|
||||
# apply the same exclusion list to the raw report - it reads
|
||||
|
||||
BIN
Binary file not shown.
@@ -9,41 +9,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- **Schema-guided extraction validation** (#1510) by @Besokus
|
||||
- New `SchemaValidator` (`semantica.semantic_extract`, lazy export): a deterministic sibling of `ExtractionValidator` that checks extraction output for *conformance to a domain ontology* — an axis orthogonal to `ExtractionValidator`'s confidence checks. It mirrors the same interface (`validate_entities()` / `validate_relations()` returning `ValidationResult`, batch-aware), so the two compose back-to-back
|
||||
- Entity labels must be concepts in the schema; relation predicates must be in the schema and satisfy their `domain` / `range`. Violations are reported in `ValidationResult.errors` with counts in `metrics` and `score` = conformance ratio; `filter_by_schema()` / `filter_relations_by_schema()` return the conforming subset (mirroring `filter_by_confidence`). No LLM required
|
||||
- New `ExtractionSchema` (`semantica.semantic_extract`, lazy export): a lightweight, read-only view over a domain ontology (allowed concepts + predicates with optional `domain` / `range`). Reuses the project's existing OWL ontology representation rather than a parallel type — build one from a `generate_ontology`-style dict (`ExtractionSchema.from_ontology`) or an OWL/Turtle file/string (`ExtractionSchema.from_owl`, via the existing `rdflib` dependency). An empty `domain`/`range` means unconstrained, matching OWL
|
||||
- Implements the deterministic core of ontology-based information extraction (OBIE; Wimalasuriya & Dou, 2010). No new runtime dependencies
|
||||
- New `tests/semantic_extract/test_schema_validator.py`
|
||||
|
||||
## [0.7.0] - 2026-09-07
|
||||
|
||||
### Changed
|
||||
|
||||
- **Slim core dependencies: moved ~22 heavy packages to optional extras** (#1513)
|
||||
- Core dependencies in `pyproject.toml` are now reduced to exactly 22 direct packages: `numpy`, `pandas`, `scipy`, `scikit-learn`, `rdflib`, `networkx`, `requests`, `chardet`, `protobuf`, `grpcio`, `pillow`, `pydantic`, `click`, `rich`, `tqdm`, `pyyaml`, `toml`, `python-dotenv`, `loguru`, `structlog`, `httpx`, and `pyarrow`.
|
||||
- Heavy ML/NLP, visualization, document parsing, and ingestion packages moved into granular optional extras:
|
||||
- `models-huggingface`: `torch`, `transformers`
|
||||
- `embeddings-local`: `sentence-transformers`, `fastembed`, `onnxruntime`, `tokenizers`
|
||||
- `nlp-spacy`: `spacy`
|
||||
- `viz`: expanded to include `matplotlib`, `seaborn`, `plotly`, `ipywidgets`, `umap-learn`, alongside `pyvis`, `graphviz`, and `d3blocks`
|
||||
- `media`: `librosa`, `opencv-python`
|
||||
- `vectorstore-faiss`: `faiss-cpu` (also included in `vectorstore-all`)
|
||||
- `documents`: `python-docx`, `openpyxl`, `lxml`, `beautifulsoup4`
|
||||
- `ingest-git`: `GitPython`
|
||||
- `graph-embeddings`: `gensim` (also included in `graph-all`)
|
||||
- Full bundled behavior preserved via `pip install "semantica[all]"`, which includes all optional extras. Pinning `semantica<0.7.0` remains a permanent escape hatch for legacy workflows.
|
||||
- Safe lazy construction across parsers and visualizers:
|
||||
- `DOCXParser`, `ExcelParser`, `HTMLParser`, and `XMLParser` remain constructible without error on `__init__()`. They fail only upon calling `.parse()` with actionable error messages directing users to install `semantica[documents]`.
|
||||
- `XMLParser` automatically falls back to standard library `xml.etree` (`_parse_with_etree`) when `lxml` is not installed, preserving XML parsing capabilities without extra dependencies.
|
||||
- `EmbeddingVisualizer` and `OntologyVisualizer` safely guard `matplotlib` and optional reduction packages, advising `pip install 'semantica[viz]'`.
|
||||
- `RepoIngestor` guards `GitPython` with a clear error pointing to `semantica[ingest-git]`.
|
||||
- `PublicAPIIngestor` guards `lxml` and `_SAFE_XML_PARSER`.
|
||||
- Updated user-facing installation hints across CLI doctor commands, node embeddings (`NodeEmbedder`), vector stores (`FAISSStore`), and model loaders.
|
||||
- Recompiled CI lockfiles (`requirements-ci.txt`, `.github/requirements/explorer-extra-py311.txt`, `.github/requirements/explorer-extra-py313.txt`, and `.github/requirements/base-deps.txt`).
|
||||
|
||||
## [0.6.8] - 2026-09-05
|
||||
|
||||
### Added
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ authors:
|
||||
repository-code: "https://github.com/semantica-agi/semantica"
|
||||
url: "https://getsemantica.ai"
|
||||
license: MIT
|
||||
version: 0.7.0
|
||||
version: 0.6.8
|
||||
date-released: 2026-09-05
|
||||
keywords:
|
||||
- knowledge-graph
|
||||
|
||||
@@ -1480,14 +1480,6 @@ app = create_app(session=GraphSession(graph), agent_memory=memory)
|
||||
The Memories workspace is shown only when `agent_memory` is provided. Apply
|
||||
updates the supplied runtime object; it does not add disk persistence.
|
||||
|
||||
## What's New in v0.7.0
|
||||
|
||||
**Slim core dependencies: lightweight base install with granular optional extras** — `pip install semantica` now installs only 22 essential core dependencies, moving heavy packages into dedicated optional extras:
|
||||
- **Dramatically lighter and faster installation**: Core installation no longer pulls heavy machine learning or visualization packages by default.
|
||||
- **Granular extras**: Install only what your workload requires (`documents`, `embeddings-local`, `models-huggingface`, `nlp-spacy`, `viz`, `media`, `vectorstore-faiss`, `graph-embeddings`, `ingest-git`).
|
||||
- **Full backward compatibility**: `pip install "semantica[all]"` preserves the full bundled suite, while `semantica<0.7.0` remains a permanent escape hatch.
|
||||
- **Lazy parser construction & graceful fallbacks**: Document parsers can be constructed without extras and only raise actionable error hints upon calling `.parse()`; `XMLParser` automatically falls back to Python's standard library `xml.etree`.
|
||||
|
||||
---
|
||||
|
||||
## What's New in v0.6.8
|
||||
@@ -1526,42 +1518,32 @@ Semantica is designed for environments where AI outputs must be explainable, aud
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install semantica # lightweight core (22 essential dependencies)
|
||||
pip install "semantica[all]" # full bundled behavior with all extras
|
||||
pip install semantica # core
|
||||
pip install semantica[all] # everything
|
||||
```
|
||||
|
||||
> **Note for upgrades from <0.7.0**: In Semantica 0.7.0+, heavy machine learning, NLP, visualization, and document dependencies were moved into optional extras to make core installation significantly lighter and faster. If you want the previous bundled installation, install with `pip install "semantica[all]"` or pin `semantica<0.7.0`.
|
||||
|
||||
```bash
|
||||
# Granular Extras
|
||||
pip install "semantica[documents]" # Document parsing (docx, openpyxl, lxml, beautifulsoup4)
|
||||
pip install "semantica[embeddings-local]" # Local embeddings (sentence-transformers, fastembed, onnxruntime)
|
||||
pip install "semantica[models-huggingface]" # HuggingFace models (transformers, torch)
|
||||
pip install "semantica[nlp-spacy]" # spaCy NLP pipelines (spacy)
|
||||
pip install "semantica[viz]" # Visualization (matplotlib, seaborn, plotly, pyvis, graphviz)
|
||||
pip install "semantica[media]" # Audio & computer vision (librosa, opencv-python)
|
||||
pip install "semantica[graph-embeddings]" # Knowledge graph embeddings (gensim / Node2Vec)
|
||||
pip install "semantica[ingest-git]" # Git repository ingestor (GitPython)
|
||||
pip install "semantica[vectorstore-faiss]" # FAISS vector store
|
||||
pip install "semantica[vectorstore-all]" # All vector stores (Qdrant, Pinecone, Weaviate, FAISS, PgVector, SQLite)
|
||||
pip install "semantica[agno]" # Agno multi-agent integration
|
||||
pip install "semantica[crewai]" # CrewAI integration
|
||||
pip install "semantica[langchain]" # LangChain / LangGraph integration
|
||||
pip install "semantica[llm-all]" # All LLM provider clients
|
||||
pip install "semantica[graph-neo4j]" # Neo4j graph store (LPG)
|
||||
pip install "semantica[graph-falkordb]" # FalkorDB graph store (LPG)
|
||||
pip install "semantica[graph-apache-age]" # Apache AGE graph store (LPG)
|
||||
pip install "semantica[graph-amazon-neptune]" # AWS Neptune graph store (LPG)
|
||||
pip install "semantica[tripletstore-oxigraph]" # Embedded in-memory/on-disk RDF store
|
||||
pip install semantica[agno] # Agno multi-agent integration
|
||||
pip install semantica[crewai] # CrewAI integration
|
||||
pip install semantica[langchain] # LangChain / LangGraph integration
|
||||
pip install semantica[llm-litellm] # OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Bedrock, Ollama, DeepSeek, and more
|
||||
pip install semantica[graph-neo4j] # Neo4j graph store (LPG)
|
||||
pip install semantica[graph-falkordb] # FalkorDB graph store (LPG)
|
||||
pip install semantica[graph-apache-age] # Apache AGE graph store (LPG)
|
||||
pip install semantica[graph-amazon-neptune] # AWS Neptune graph store (LPG)
|
||||
pip install semantica[tripletstore-oxigraph] # Embedded in-memory/on-disk RDF store
|
||||
# RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J) need no extra:
|
||||
# semantica.triplet_store talks SPARQL over HTTP using the core `requests` dependency
|
||||
pip install "semantica[db-snowflake]" # Snowflake
|
||||
pip install "semantica[db-databricks]" # Databricks (SDK + SQL connector)
|
||||
pip install "semantica[ingest-sap]" # SAP OData
|
||||
pip install "semantica[ingest-parquet]" # Parquet / PyArrow
|
||||
pip install "semantica[ingest-arrow]" # Apache Arrow, Feather, IPC
|
||||
pip install "semantica[watch]" # Directory file watcher
|
||||
pip install "semantica[explorer]" # Knowledge Explorer dashboard
|
||||
pip install semantica[vectorstore-qdrant] # Qdrant vector store
|
||||
pip install semantica[vectorstore-pinecone] # Pinecone vector store
|
||||
pip install semantica[db-snowflake] # Snowflake
|
||||
pip install semantica[db-databricks] # Databricks (SDK + SQL connector)
|
||||
pip install semantica[ingest-sap] # SAP OData
|
||||
pip install semantica[ingest-parquet] # Parquet / PyArrow
|
||||
pip install semantica[ingest-arrow] # Apache Arrow, Feather, IPC
|
||||
pip install semantica[viz] # HTML interactive visualization
|
||||
pip install semantica[watch] # Directory file watcher
|
||||
pip install semantica[explorer] # Knowledge Explorer dashboard
|
||||
```
|
||||
|
||||
For production deployments, use Docker or Kubernetes rather than a local `pip install`. Set `SEMANTICA_API_KEY`, configure a persistent LPG graph store (Neo4j / FalkorDB / Apache AGE / AWS Neptune) and/or RDF triple store (Blazegraph / Apache Jena / Eclipse RDF4J), and point the vector store at a hosted backend (Qdrant / Pinecone). See [ARCHITECTURE.md](ARCHITECTURE.md) for the full deployment topology.
|
||||
|
||||
@@ -21,7 +21,6 @@ from typing import Any, List, Optional
|
||||
try:
|
||||
from google.adk.events import Event
|
||||
from google.adk.sessions import BaseSessionService, Session
|
||||
|
||||
try:
|
||||
from google.adk.sessions import ListSessionsResponse
|
||||
except ImportError:
|
||||
@@ -34,7 +33,7 @@ try:
|
||||
except ImportError:
|
||||
from google.adk.sessions.base_session_service import GetSessionConfig
|
||||
ADK_AVAILABLE = True
|
||||
except (ImportError, OSError):
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
ADK_AVAILABLE = False
|
||||
BaseSessionService = object
|
||||
Session = Any
|
||||
@@ -163,10 +162,10 @@ class SemanticaSessionService(BaseSessionService):
|
||||
return {}
|
||||
|
||||
def _find_session_node(
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
) -> Optional[Any]:
|
||||
"""Find a session node by its logical ADK session ID."""
|
||||
expected_node_id = self._node_id(app_name, user_id, session_id)
|
||||
@@ -183,18 +182,18 @@ class SemanticaSessionService(BaseSessionService):
|
||||
metadata = node.get("metadata")
|
||||
|
||||
if (
|
||||
isinstance(metadata, dict)
|
||||
and str(metadata.get("session_id")) == str(session_id)
|
||||
and str(metadata.get("app_name")) == str(app_name)
|
||||
and str(metadata.get("user_id")) == str(user_id)
|
||||
isinstance(metadata, dict)
|
||||
and str(metadata.get("session_id")) == str(session_id)
|
||||
and str(metadata.get("app_name")) == str(app_name)
|
||||
and str(metadata.get("user_id")) == str(user_id)
|
||||
):
|
||||
return node
|
||||
|
||||
return None
|
||||
|
||||
def _find_node_by_id(
|
||||
self,
|
||||
node_id: str,
|
||||
self,
|
||||
node_id: str,
|
||||
) -> Optional[Any]:
|
||||
"""Find a ContextGraph node by graph node ID."""
|
||||
for node in self.graph.find_nodes() or []:
|
||||
@@ -213,13 +212,13 @@ class SemanticaSessionService(BaseSessionService):
|
||||
data = SemanticaSessionService._safe_dict(event)
|
||||
|
||||
for field in (
|
||||
"id",
|
||||
"invocation_id",
|
||||
"author",
|
||||
"timestamp",
|
||||
"partial",
|
||||
"turn_complete",
|
||||
"branch",
|
||||
"id",
|
||||
"invocation_id",
|
||||
"author",
|
||||
"timestamp",
|
||||
"partial",
|
||||
"turn_complete",
|
||||
"branch",
|
||||
):
|
||||
if field not in data and hasattr(event, field):
|
||||
value = getattr(event, field)
|
||||
@@ -232,10 +231,10 @@ class SemanticaSessionService(BaseSessionService):
|
||||
return data
|
||||
|
||||
def _event_nodes(
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
) -> List[Any]:
|
||||
"""Return all event nodes connected to a session."""
|
||||
session_node_id = self._node_id(app_name, user_id, session_id)
|
||||
@@ -276,8 +275,8 @@ class SemanticaSessionService(BaseSessionService):
|
||||
return str(timestamp)
|
||||
|
||||
def _event_from_node(
|
||||
self,
|
||||
node: Any,
|
||||
self,
|
||||
node: Any,
|
||||
) -> Any:
|
||||
"""
|
||||
Reconstruct an ADK Event from its stored metadata.
|
||||
@@ -290,7 +289,7 @@ class SemanticaSessionService(BaseSessionService):
|
||||
if not event_id and graph_node_id:
|
||||
graph_node_id = str(graph_node_id)
|
||||
if graph_node_id.startswith("adk-event:"):
|
||||
event_id = graph_node_id[len("adk-event:") :]
|
||||
event_id = graph_node_id[len("adk-event:"):]
|
||||
|
||||
if event_id:
|
||||
properties["id"] = event_id
|
||||
@@ -311,11 +310,11 @@ class SemanticaSessionService(BaseSessionService):
|
||||
|
||||
@staticmethod
|
||||
def _session_kwargs(
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
state: Optional[dict],
|
||||
events: Optional[List[Any]],
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
state: Optional[dict],
|
||||
events: Optional[List[Any]],
|
||||
) -> dict:
|
||||
"""Build kwargs for the ADK Session model."""
|
||||
return {
|
||||
@@ -327,8 +326,8 @@ class SemanticaSessionService(BaseSessionService):
|
||||
}
|
||||
|
||||
def _session_from_node(
|
||||
self,
|
||||
node: Any,
|
||||
self,
|
||||
node: Any,
|
||||
) -> Session:
|
||||
"""Reconstruct an ADK Session from a ContextGraph node."""
|
||||
properties = self._node_properties(node)
|
||||
@@ -348,14 +347,14 @@ class SemanticaSessionService(BaseSessionService):
|
||||
# splitting on ':' after the prefix always yields
|
||||
# exactly 3 parts regardless of what characters the
|
||||
# original app_name/user_id/session_id contained.
|
||||
parts = graph_node_id[len("adk-session:") :].split(":")
|
||||
parts = graph_node_id[len("adk-session:"):].split(":")
|
||||
if len(parts) == 3:
|
||||
decoded = [urllib.parse.unquote(part) for part in parts]
|
||||
app_name = app_name or decoded[0]
|
||||
user_id = user_id or decoded[1]
|
||||
session_id = decoded[2]
|
||||
else:
|
||||
session_id = graph_node_id[len("adk-session:") :]
|
||||
session_id = graph_node_id[len("adk-session:"):]
|
||||
else:
|
||||
session_id = graph_node_id
|
||||
|
||||
@@ -384,12 +383,12 @@ class SemanticaSessionService(BaseSessionService):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def create_session(
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
state: Optional[dict[str, Any]] = None,
|
||||
session_id: Optional[str] = None,
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
state: Optional[dict[str, Any]] = None,
|
||||
session_id: Optional[str] = None,
|
||||
) -> Session:
|
||||
"""Create and persist an ADK session."""
|
||||
return await asyncio.to_thread(
|
||||
@@ -397,11 +396,11 @@ class SemanticaSessionService(BaseSessionService):
|
||||
)
|
||||
|
||||
def _create_session_sync(
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
state: Optional[dict[str, Any]],
|
||||
session_id: Optional[str],
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
state: Optional[dict[str, Any]],
|
||||
session_id: Optional[str],
|
||||
) -> Session:
|
||||
with self._lock:
|
||||
session_id = session_id or str(uuid.uuid4())
|
||||
@@ -431,12 +430,12 @@ class SemanticaSessionService(BaseSessionService):
|
||||
)
|
||||
|
||||
async def get_session(
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
config: Optional[GetSessionConfig] = None,
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
config: Optional[GetSessionConfig] = None,
|
||||
) -> Optional[Session]:
|
||||
"""Retrieve an ADK session from ContextGraph."""
|
||||
return await asyncio.to_thread(
|
||||
@@ -444,11 +443,11 @@ class SemanticaSessionService(BaseSessionService):
|
||||
)
|
||||
|
||||
def _get_session_sync(
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
config: Optional[GetSessionConfig],
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
config: Optional[GetSessionConfig],
|
||||
) -> Optional[Session]:
|
||||
with self._lock:
|
||||
node = self._find_session_node(app_name, user_id, session_id)
|
||||
@@ -469,7 +468,7 @@ class SemanticaSessionService(BaseSessionService):
|
||||
# trims the already-built Session object.
|
||||
if config:
|
||||
if config.num_recent_events:
|
||||
session.events = session.events[-config.num_recent_events :]
|
||||
session.events = session.events[-config.num_recent_events:]
|
||||
if config.after_timestamp:
|
||||
i = len(session.events) - 1
|
||||
while i >= 0:
|
||||
@@ -477,14 +476,14 @@ class SemanticaSessionService(BaseSessionService):
|
||||
break
|
||||
i -= 1
|
||||
if i >= 0:
|
||||
session.events = session.events[i + 1 :]
|
||||
session.events = session.events[i + 1:]
|
||||
|
||||
return session
|
||||
|
||||
async def append_event(
|
||||
self,
|
||||
session: Session,
|
||||
event: Event,
|
||||
self,
|
||||
session: Session,
|
||||
event: Event,
|
||||
) -> Event:
|
||||
"""Persist an ADK event and associate it with a session."""
|
||||
# ADK's own base implementation is a no-op for partial/streaming
|
||||
@@ -511,13 +510,8 @@ class SemanticaSessionService(BaseSessionService):
|
||||
|
||||
# Verify cross-tenant security
|
||||
properties = self._node_properties(session_node)
|
||||
if (
|
||||
properties.get("app_name") != app_name
|
||||
or properties.get("user_id") != user_id
|
||||
):
|
||||
raise ValueError(
|
||||
"Cross-tenant session write denied: app_name or user_id mismatch."
|
||||
)
|
||||
if properties.get("app_name") != app_name or properties.get("user_id") != user_id:
|
||||
raise ValueError("Cross-tenant session write denied: app_name or user_id mismatch.")
|
||||
|
||||
# Apply ADK in-memory event and state delta semantics. This
|
||||
# runs inside asyncio.to_thread's worker thread, which has no
|
||||
@@ -559,11 +553,11 @@ class SemanticaSessionService(BaseSessionService):
|
||||
)
|
||||
|
||||
async def delete_session(
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
) -> None:
|
||||
"""Delete a session and all of its graph-backed events."""
|
||||
await asyncio.to_thread(
|
||||
@@ -571,10 +565,10 @@ class SemanticaSessionService(BaseSessionService):
|
||||
)
|
||||
|
||||
def _delete_session_sync(
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: str,
|
||||
session_id: str,
|
||||
) -> None:
|
||||
with self._lock:
|
||||
session_node = self._find_session_node(app_name, user_id, session_id)
|
||||
@@ -596,9 +590,9 @@ class SemanticaSessionService(BaseSessionService):
|
||||
continue
|
||||
|
||||
if (
|
||||
edge.get("source") == session_node_id
|
||||
and edge.get("type") == "HAS_EVENT"
|
||||
and edge.get("target")
|
||||
edge.get("source") == session_node_id
|
||||
and edge.get("type") == "HAS_EVENT"
|
||||
and edge.get("target")
|
||||
):
|
||||
event_node_ids.append(str(edge["target"]))
|
||||
|
||||
@@ -608,18 +602,18 @@ class SemanticaSessionService(BaseSessionService):
|
||||
self.graph.purge_node(session_node_id)
|
||||
|
||||
async def list_sessions(
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: Optional[str] = None,
|
||||
self,
|
||||
*,
|
||||
app_name: str,
|
||||
user_id: Optional[str] = None,
|
||||
) -> ListSessionsResponse:
|
||||
"""List sessions for an app, optionally scoped to one user."""
|
||||
return await asyncio.to_thread(self._list_sessions_sync, app_name, user_id)
|
||||
|
||||
def _list_sessions_sync(
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: Optional[str],
|
||||
self,
|
||||
app_name: str,
|
||||
user_id: Optional[str],
|
||||
) -> ListSessionsResponse:
|
||||
with self._lock:
|
||||
sessions: List[Session] = []
|
||||
@@ -649,4 +643,4 @@ class SemanticaSessionService(BaseSessionService):
|
||||
__all__ = [
|
||||
"ADK_AVAILABLE",
|
||||
"SemanticaSessionService",
|
||||
]
|
||||
]
|
||||
+52
-88
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "semantica"
|
||||
version = "0.7.0"
|
||||
version = "0.6.8"
|
||||
description = "Graph-Native Infrastructure for Context and Accountable AI Systems: context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable."
|
||||
readme = "README.md"
|
||||
license = { text = "MIT" }
|
||||
@@ -12,11 +12,7 @@ license = { text = "MIT" }
|
||||
authors = [{ name = "Semantica", email = "kaif@getsemantica.ai" }]
|
||||
maintainers = [{ name = "Semantica", email = "kaif@getsemantica.ai" }]
|
||||
|
||||
# 3.8 is already unsatisfiable in practice (numpy>=2.0.2 requires >=3.9) and is
|
||||
# not exercised by the Install Matrix (3.9-3.12). The floor is 3.9.2 rather
|
||||
# than 3.9.0 because cryptography (db-snowflake) excludes 3.9.0/3.9.1 from
|
||||
# every release's requires-python, so those patch levels can never resolve.
|
||||
requires-python = ">=3.9.2"
|
||||
requires-python = ">=3.8"
|
||||
|
||||
classifiers = [
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
@@ -26,6 +22,7 @@ classifiers = [
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
@@ -55,13 +52,30 @@ dependencies = [
|
||||
# last 3.9-compatible release line; 3.10+ is left unconstrained.
|
||||
"scikit-learn>=1.6.1,<1.7.0; python_version < '3.10'",
|
||||
"scikit-learn>=1.7.2; python_version >= '3.10'",
|
||||
"umap-learn>=0.5.12",
|
||||
# thinc (spacy's core dep) dropped Python 3.9 wheels at 8.3.10, and later
|
||||
# spacy patch releases (3.8.8+) require thinc>=8.3.9-only-on-3.10+ ranges,
|
||||
# which forces a source build that fails outright on 3.9 (see Install
|
||||
# Matrix run history). Capping both keeps 3.9 on the last wheel-compatible
|
||||
# pair; 3.10+ is left unconstrained to always get the latest spacy/thinc.
|
||||
"spacy>=3.4.0,<3.8.8; python_version < '3.10'",
|
||||
"spacy>=3.4.0; python_version >= '3.10'",
|
||||
"thinc<8.3.5; python_version < '3.10'",
|
||||
"transformers>=4.20.0",
|
||||
"torch>=1.13.1",
|
||||
"sentence-transformers>=2.2.0",
|
||||
"rdflib>=6.2.0",
|
||||
"networkx>=2.8.0",
|
||||
"matplotlib>=3.9.4",
|
||||
"seaborn>=0.13.2",
|
||||
"plotly>=6.8.0",
|
||||
"ipywidgets>=8.0.0",
|
||||
# requests dropped Python 3.9 support at 2.33.0 (requires_python >=3.10),
|
||||
# so an unqualified >=2.34.2 floor is unsatisfiable on 3.9. Cap 3.9 to the
|
||||
# last 3.9-compatible release; 3.10+ is left unconstrained.
|
||||
"requests>=2.32.5,<2.33.0; python_version < '3.10'",
|
||||
"requests>=2.34.2; python_version >= '3.10'",
|
||||
"GitPython>=3.1.58",
|
||||
# chardet dropped Python 3.9 support at 6.0.0 (requires_python >=3.10), so
|
||||
# an unqualified >=7.4.3 floor is unsatisfiable on 3.9. Cap 3.9 to the last
|
||||
# 3.9-compatible release; 3.10+ is left unconstrained.
|
||||
@@ -73,11 +87,26 @@ dependencies = [
|
||||
# 3.9-compatible release; 3.10+ is left unconstrained.
|
||||
"grpcio>=1.80.0,<1.81.0; python_version < '3.10'",
|
||||
"grpcio>=1.81.1; python_version >= '3.10'",
|
||||
"beautifulsoup4>=4.15.0",
|
||||
"lxml>=6.1.1",
|
||||
"python-docx>=1.2.0",
|
||||
"openpyxl>=3.1.5",
|
||||
# pillow dropped Python 3.9 support at 12.0.0 (requires_python >=3.10), so
|
||||
# an unqualified >=12.2.0 floor is unsatisfiable on 3.9. Cap 3.9 to the last
|
||||
# 3.9-compatible release; 3.10+ is left unconstrained.
|
||||
"pillow>=11.3.0,<12.0.0; python_version < '3.10'",
|
||||
"pillow>=12.2.0; python_version >= '3.10'",
|
||||
"librosa>=0.9.0",
|
||||
"opencv-python>=4.13.0.92",
|
||||
"faiss-cpu>=1.7.0",
|
||||
"fastembed>=0.2.0",
|
||||
# onnxruntime stopped shipping cp39 wheels at 1.20.0 (its PyPI metadata
|
||||
# still claims requires_python >=3.9, but no matching wheel exists), so an
|
||||
# unqualified >=1.20.1 floor is unsatisfiable on 3.9. Cap 3.9 to the last
|
||||
# release with a cp39 wheel; 3.10+ is left unconstrained.
|
||||
"onnxruntime>=1.19.2,<1.20.0; python_version < '3.10'",
|
||||
"onnxruntime>=1.20.1; python_version >= '3.10'",
|
||||
"tokenizers>=0.15.0",
|
||||
"pydantic>=2.13.4",
|
||||
# click dropped Python 3.9 support at 8.2.0 (requires_python >=3.10), so an
|
||||
# unqualified >=8.4.2 floor is unsatisfiable on 3.9. Cap 3.9 to the last
|
||||
@@ -91,6 +120,7 @@ dependencies = [
|
||||
"python-dotenv>=1.2.1",
|
||||
"loguru>=0.7.3",
|
||||
"structlog>=22.1.0",
|
||||
"gensim>=4.4.0",
|
||||
"httpx<0.29.0",
|
||||
"pyarrow>=14.0.0"
|
||||
]
|
||||
@@ -114,10 +144,7 @@ llm-anthropic = ["anthropic>=0.122.0"]
|
||||
llm-ollama = ["ollama>=0.1.0"]
|
||||
llm-deepseek = ["openai>=1.0.0"]
|
||||
llm-novita = ["openai>=1.0.0"]
|
||||
# litellm>=1.83.10 requires Python>=3.10, and the last 3.9-compatible release
|
||||
# (1.83.9) pins python-dotenv==1.0.1, which conflicts with our >=1.2.1 core
|
||||
# floor — so no litellm satisfies 3.9 at all. Gate it to 3.10+.
|
||||
llm-litellm = ["litellm>=1.83.9; python_version >= '3.10'"]
|
||||
llm-litellm = ["litellm>=1.83.9"]
|
||||
llm-instructor = ["instructor>=1.15.3"]
|
||||
|
||||
llm-all = [
|
||||
@@ -125,49 +152,19 @@ llm-all = [
|
||||
]
|
||||
|
||||
# ---- Document Parsing ----
|
||||
documents = [
|
||||
"python-docx>=1.2.0",
|
||||
"openpyxl>=3.1.5",
|
||||
"lxml>=6.1.1",
|
||||
"beautifulsoup4>=4.15.0"
|
||||
]
|
||||
# every docling release requires Python>=3.10 (no 3.9-compatible version
|
||||
# exists to cap to), so gate it like google-adk below rather than split it.
|
||||
parse-docling = ["docling>=2.107.0; python_version >= '3.10'"]
|
||||
# pdfplumber powers the default PDFParser; not pulled in by any other extra.
|
||||
parse-pdf = ["pdfplumber>=0.10.0"]
|
||||
parse-docling = ["docling>=2.107.0"]
|
||||
|
||||
# ---- SHACL Validation ----
|
||||
shacl = ["pyshacl>=0.25.0"]
|
||||
|
||||
# ---- Database Connectors ----
|
||||
# snowflake-connector-python dropped Python 3.9 support at 4.6.0
|
||||
# (requires_python >=3.10), so an unqualified >=4.6.0 floor is unsatisfiable
|
||||
# on 3.9. Cap 3.9 below it; 3.10+ keeps the newer floor.
|
||||
db-snowflake = [
|
||||
"snowflake-connector-python>=4.6.0; python_version >= '3.10'",
|
||||
"snowflake-connector-python>=3.13.0,<4.6.0; python_version < '3.10'",
|
||||
"cryptography>=49.0.0"
|
||||
]
|
||||
db-snowflake = ["snowflake-connector-python>=4.6.0", "cryptography>=49.0.0"]
|
||||
db-databricks = ["databricks-sdk>=0.60.0", "databricks-sql-connector>=4.0.0"]
|
||||
# pyarrow dropped Python 3.9 support at 24.0.0 (requires_python >=3.10), so an
|
||||
# unqualified >=24.0.0 floor is unsatisfiable on 3.9. Cap 3.9 below the last
|
||||
# 3.9-compatible release line; 3.10+ is left unconstrained.
|
||||
db-arrow = [
|
||||
"pyarrow>=24.0.0; python_version >= '3.10'",
|
||||
"pyarrow>=14.0.0,<24.0.0; python_version < '3.10'"
|
||||
]
|
||||
db-arrow = ["pyarrow>=24.0.0"]
|
||||
db-salesforce = ["simple-salesforce>=1.12.0"]
|
||||
ingest-parquet = [
|
||||
"pyarrow>=24.0.0; python_version >= '3.10'",
|
||||
"pyarrow>=14.0.0,<24.0.0; python_version < '3.10'"
|
||||
]
|
||||
ingest-arrow = [
|
||||
"pyarrow>=24.0.0; python_version >= '3.10'",
|
||||
"pyarrow>=14.0.0,<24.0.0; python_version < '3.10'"
|
||||
]
|
||||
ingest-parquet = ["pyarrow>=24.0.0"]
|
||||
ingest-arrow = ["pyarrow>=24.0.0"]
|
||||
ingest-sap = ["requests>=2.28.0"]
|
||||
ingest-git = ["GitPython>=3.1.58"]
|
||||
|
||||
db-all = [
|
||||
"semantica[db-snowflake,db-databricks,db-salesforce,db-arrow]"
|
||||
@@ -178,35 +175,22 @@ models-huggingface = [
|
||||
"transformers>=4.20.0",
|
||||
"torch>=1.13.1"
|
||||
]
|
||||
embeddings-local = [
|
||||
"sentence-transformers>=2.2.0",
|
||||
"fastembed>=0.2.0",
|
||||
"onnxruntime>=1.19.2,<1.20.0; python_version < '3.10'",
|
||||
"onnxruntime>=1.20.1; python_version >= '3.10'",
|
||||
"tokenizers>=0.15.0"
|
||||
]
|
||||
nlp-spacy = [
|
||||
"spacy>=3.4.0,<3.8.8; python_version < '3.10'",
|
||||
"spacy>=3.4.0; python_version >= '3.10'"
|
||||
]
|
||||
|
||||
# ---- Graph Backends ----
|
||||
graph-neo4j = ["neo4j>=5.0.0"]
|
||||
graph-falkordb = ["falkordb>=1.0.0", "redis>=4.3.0"]
|
||||
graph-amazon-neptune = ["boto3>=1.24.0", "neo4j>=5.0.0"]
|
||||
graph-apache-age = ["psycopg2-binary>=2.9.0"]
|
||||
graph-embeddings = ["gensim>=4.4.0"]
|
||||
|
||||
graph-all = [
|
||||
"semantica[graph-neo4j,graph-falkordb,graph-amazon-neptune,graph-apache-age,graph-embeddings]"
|
||||
"semantica[graph-neo4j,graph-falkordb,graph-amazon-neptune,graph-apache-age]"
|
||||
]
|
||||
|
||||
# ---- Triplet Store Backends ----
|
||||
tripletstore-oxigraph = ["pyoxigraph>=0.5.0"]
|
||||
|
||||
# ---- Vector Store Backends ----
|
||||
vectorstore-faiss = ["faiss-cpu>=1.7.0"]
|
||||
vectorstore-qdrant = ["qdrant-client>=1.10.0"]
|
||||
vectorstore-qdrant = ["qdrant-client>=1.0.0"]
|
||||
vectorstore-weaviate = ["weaviate-client>=4.0.0"]
|
||||
vectorstore-pinecone = ["pinecone>=3.0.0"]
|
||||
vectorstore-milvus = ["pymilvus>=2.0.0"]
|
||||
@@ -214,7 +198,7 @@ vectorstore-pgvector = ["psycopg[binary,pool]>=3.0.0", "pgvector>=0.2.0"]
|
||||
vectorstore-sqlite = ["sqlite-vec>=0.1.1"]
|
||||
|
||||
vectorstore-all = [
|
||||
"semantica[vectorstore-qdrant,vectorstore-weaviate,vectorstore-pinecone,vectorstore-milvus,vectorstore-pgvector,vectorstore-sqlite,vectorstore-faiss]"
|
||||
"semantica[vectorstore-qdrant,vectorstore-weaviate,vectorstore-pinecone,vectorstore-milvus,vectorstore-pgvector,vectorstore-sqlite]"
|
||||
]
|
||||
|
||||
# ---- Infra / Queues / Workers ----
|
||||
@@ -246,18 +230,7 @@ monitoring = [
|
||||
viz = [
|
||||
"pyvis>=0.3.0",
|
||||
"graphviz>=0.21",
|
||||
"d3blocks>=1.0.0",
|
||||
"matplotlib>=3.9.4",
|
||||
"seaborn>=0.13.2",
|
||||
"plotly>=6.8.0",
|
||||
"ipywidgets>=8.0.0",
|
||||
"umap-learn>=0.5.12"
|
||||
]
|
||||
|
||||
# ---- Media ----
|
||||
media = [
|
||||
"librosa>=0.9.0",
|
||||
"opencv-python>=4.13.0.92"
|
||||
"d3blocks>=1.0.0"
|
||||
]
|
||||
|
||||
# ---- GPU ----
|
||||
@@ -271,9 +244,7 @@ agno = ["agno>=1.0.0"]
|
||||
# crewai core provides BaseTool and BaseKnowledgeSource; crewai-tools is not
|
||||
# needed (it pulls vulnerable transitive deps like chromadb) and would only
|
||||
# duplicate the prebuilt tooling users can install separately.
|
||||
# No un-yanked crewai release supports Python 3.9 (all require >=3.10), so
|
||||
# gate the extra to 3.10+.
|
||||
crewai = ["crewai>=0.80.0; python_version >= '3.10'"]
|
||||
crewai = ["crewai>=0.80.0"]
|
||||
langchain = ["langchain-core>=0.3.0"]
|
||||
google-adk = ["google-adk>=1.27.0; python_version >= '3.10'"]
|
||||
|
||||
@@ -298,23 +269,15 @@ dev = [
|
||||
"isort>=6.1.0",
|
||||
"flake8>=4.0.0",
|
||||
"mypy>=0.971",
|
||||
# pre-commit dropped Python 3.9 support at 4.6.0 (requires_python >=3.10).
|
||||
# Cap 3.9 below it; 3.10+ keeps the >=4.6.0 floor.
|
||||
"pre-commit>=4.0.0,<4.6.0; python_version < '3.10'",
|
||||
"pre-commit>=4.6.0; python_version >= '3.10'",
|
||||
"pre-commit>=4.6.0",
|
||||
"jupyter>=1.0.0",
|
||||
"ipykernel>=6.15.0"
|
||||
]
|
||||
|
||||
# Explorer Dashboard
|
||||
# fastapi dropped Python 3.9 support at 0.129.0 (requires_python >=3.10), and
|
||||
# every fastapi below that caps starlette<0.53.0 — so the 3.10+ starlette
|
||||
# floor is unsatisfiable on 3.9. Cap both on 3.9; 3.10+ keeps the newer floors.
|
||||
explorer = [
|
||||
"fastapi>=0.109.2,<0.129.0; python_version < '3.10'",
|
||||
"fastapi>=0.109.2; python_version >= '3.10'",
|
||||
"starlette>=0.36.3,<0.53.0; python_version < '3.10'",
|
||||
"starlette>=0.53.0; python_version >= '3.10'",
|
||||
"fastapi>=0.109.2",
|
||||
"starlette>=0.53.0",
|
||||
"uvicorn[standard]>=0.22.0",
|
||||
"websockets>=15.0.1",
|
||||
"python-multipart>=0.0.7",
|
||||
@@ -331,7 +294,8 @@ explorer-lite = [
|
||||
# (CVE-2026-45829) with no fixed release — including it here would fail the CI
|
||||
# dependency-audit/security gates. Install it explicitly via ``semantica[crewai]``.
|
||||
all = [
|
||||
"semantica[dev,viz,media,infra,cloud,monitoring,watch,llm-all,models-huggingface,embeddings-local,nlp-spacy,documents,ingest-git,graph-embeddings,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,parse-pdf,ingest-parquet,ingest-arrow,shacl,explorer,agno,langchain,google-adk]"
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,explorer]",
|
||||
"semantica[dev,viz,infra,cloud,monitoring,watch,llm-all,models-huggingface,split-all,graph-all,tripletstore-oxigraph,vectorstore-all,parse-docling,ingest-parquet,ingest-arrow,shacl,agno,langchain,google-adk]"
|
||||
]
|
||||
|
||||
# ---------------- ENTRYPOINTS ----------------
|
||||
|
||||
+2
-17
@@ -181,7 +181,6 @@ anyio==4.14.2 \
|
||||
# jupyter-server
|
||||
# langsmith
|
||||
# openai
|
||||
# pinecone
|
||||
# starlette
|
||||
# watchfiles
|
||||
argon2-cffi==25.1.0 \
|
||||
@@ -787,9 +786,7 @@ charset-normalizer==3.5.1 \
|
||||
--hash=sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f \
|
||||
--hash=sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0 \
|
||||
--hash=sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f
|
||||
# via
|
||||
# pdfminer-six
|
||||
# requests
|
||||
# via requests
|
||||
click==8.5.0 \
|
||||
--hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \
|
||||
--hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34
|
||||
@@ -1100,7 +1097,6 @@ cryptography==50.0.1 \
|
||||
# azure-storage-blob
|
||||
# google-auth
|
||||
# joserfc
|
||||
# pdfminer-six
|
||||
cuda-bindings==13.3.1 \
|
||||
--hash=sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708 \
|
||||
--hash=sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86 \
|
||||
@@ -4302,14 +4298,6 @@ patsy==1.0.3 \
|
||||
--hash=sha256:79ebf4c93ff4d296e58a9d5be2b2ee31bd49d737cf11d70ffbd8a44b2de42e65 \
|
||||
--hash=sha256:d3dbebe8fd5f46e29912d030b63c6268647b59bf788a99e2af28a30234cf357c
|
||||
# via statsmodels
|
||||
pdfminer-six==20260107 \
|
||||
--hash=sha256:366585ba97e80dffa8f00cebe303d2f381884d8637af4ce422f1df3ef38111a9 \
|
||||
--hash=sha256:96bfd431e3577a55a0efd25676968ca4ce8fd5b53f14565f85716ff363889602
|
||||
# via pdfplumber
|
||||
pdfplumber==0.11.10 \
|
||||
--hash=sha256:7741ea81bf165b474b153e6789d10d18e06b6ddcf3ec84289c3ef2fed6802580 \
|
||||
--hash=sha256:b95b2d28c66efb0a794a83b88c6c6aea5987532a445d20a1cbcfa657022e6e57
|
||||
# via semantica (pyproject.toml)
|
||||
pexpect==4.9.0 \
|
||||
--hash=sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523 \
|
||||
--hash=sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f
|
||||
@@ -4418,7 +4406,6 @@ pillow==12.3.0 \
|
||||
# docling-slim
|
||||
# fastembed
|
||||
# matplotlib
|
||||
# pdfplumber
|
||||
# python-pptx
|
||||
# rapidocr
|
||||
# torchvision
|
||||
@@ -5306,9 +5293,7 @@ pypdfium2==5.13.0 \
|
||||
--hash=sha256:d96929bde3bd64c771ab3558ca1ffd7704cc4d872ab92cd9f8f8b8a20f7f36b8 \
|
||||
--hash=sha256:d96beb7f379e6c76d874ca93fcd182ac3168dd499056407070f9927fb1061b8e \
|
||||
--hash=sha256:da5c7b74eebf40b5c1fbe1de01aa1edc8827a79fb1efd999616bc20dcaf77ba4
|
||||
# via
|
||||
# docling-slim
|
||||
# pdfplumber
|
||||
# via docling-slim
|
||||
pypickle==2.0.2 \
|
||||
--hash=sha256:d3307127314465fe3dc8f0162e11777d5e8284f3a29dc48b0f770d364a85d998 \
|
||||
--hash=sha256:d577e39cf501c7c80b1387f6d7dc885cf4efeba65f213df41226d1f24881b1e8
|
||||
|
||||
@@ -10,7 +10,7 @@ Main exports:
|
||||
- Config: Configuration management
|
||||
"""
|
||||
|
||||
__version__ = "0.7.0"
|
||||
__version__ = "0.6.8"
|
||||
__author__ = "Semantica Contributors"
|
||||
__license__ = "MIT"
|
||||
|
||||
|
||||
+22
-26
@@ -465,7 +465,7 @@ def _show_startup(cli_ctx: CLIContext) -> None:
|
||||
return
|
||||
cfg = cli_ctx.config.to_dict()
|
||||
graph_store = (
|
||||
cli_ctx.store_backend or cfg.get("graph_db", {}).get("backend", "neo4j")
|
||||
cli_ctx.store_backend or cfg.get("graph_db", {}).get("backend", "memory")
|
||||
)
|
||||
vector_store = (
|
||||
cli_ctx.vector_store_backend
|
||||
@@ -851,7 +851,9 @@ def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None
|
||||
# Graph store reachability
|
||||
def _graph() -> str:
|
||||
cfg = cli_ctx.config.to_dict()
|
||||
backend = cli_ctx.store_backend or cfg.get("graph_db", {}).get("backend", "neo4j")
|
||||
backend = cli_ctx.store_backend or cfg.get("graph_db", {}).get("backend", "memory")
|
||||
if backend == "memory":
|
||||
return "memory (always available)"
|
||||
gs = _get_graph_store(cli_ctx)
|
||||
gs.ping() if hasattr(gs, "ping") else gs.connect()
|
||||
return f"{backend} reachable"
|
||||
@@ -878,18 +880,10 @@ def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None
|
||||
def _embedding_backend(method: str) -> str:
|
||||
if method == "sentence_transformers":
|
||||
import sentence_transformers # noqa: F401
|
||||
try:
|
||||
ver = importlib.metadata.version("sentence-transformers")
|
||||
except Exception:
|
||||
ver = getattr(sentence_transformers, "__version__", "installed")
|
||||
note = f"importable ({ver})"
|
||||
note = f"importable ({importlib.metadata.version('sentence-transformers')})"
|
||||
else:
|
||||
import fastembed # noqa: F401
|
||||
try:
|
||||
ver = importlib.metadata.version("fastembed")
|
||||
except Exception:
|
||||
ver = getattr(fastembed, "__version__", "installed")
|
||||
note = f"importable ({ver})"
|
||||
note = f"importable ({importlib.metadata.version('fastembed')})"
|
||||
if not deep:
|
||||
return note
|
||||
try:
|
||||
@@ -910,12 +904,12 @@ def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None
|
||||
checks.append(_check(
|
||||
"Embeddings (sentence-transformers)",
|
||||
lambda: _embedding_backend("sentence_transformers"),
|
||||
hint="pip install 'semantica[embeddings-local]'",
|
||||
hint="pip install sentence-transformers",
|
||||
))
|
||||
checks.append(_check(
|
||||
"Embeddings (fastembed)",
|
||||
lambda: _embedding_backend("fastembed"),
|
||||
hint="pip install 'semantica[embeddings-local]'",
|
||||
hint="pip install fastembed",
|
||||
))
|
||||
|
||||
# LLM provider keys
|
||||
@@ -4770,10 +4764,15 @@ def mcp_list_tools(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
cli_ctx = _require_ctx(cli_ctx)
|
||||
|
||||
def _action() -> None:
|
||||
# Same catalog the server exposes via tools/list, so `list-tools`
|
||||
# and `mcp start` can't drift (issue #1355).
|
||||
from semantica_mcp.mcp.tools import TOOL_DEFINITIONS
|
||||
tools = [t["name"] for t in TOOL_DEFINITIONS]
|
||||
try:
|
||||
from semantica_mcp.mcp.tools import __all__ as tools
|
||||
except ImportError:
|
||||
tools = [
|
||||
"extract_entities", "extract_relations", "build_graph",
|
||||
"query_graph", "get_graph_analytics", "run_reasoning",
|
||||
"record_decision", "get_decisions", "export_graph",
|
||||
"validate_shacl", "get_provenance", "embed_and_search",
|
||||
]
|
||||
if _is_json(cli_ctx, local_json):
|
||||
_jecho({"tools": list(tools)})
|
||||
else:
|
||||
@@ -4806,15 +4805,12 @@ def mcp_call(cli_ctx: CLIContext, tool_name: str, args: str, local_json: bool) -
|
||||
tool_args = json.loads(args)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise click.ClickException(f"Invalid JSON in --args: {exc}") from exc
|
||||
if not isinstance(tool_args, dict):
|
||||
raise click.ClickException("--args must be a JSON object")
|
||||
# Dispatch through the same server `mcp start` spawns; its session
|
||||
# module never defined MCPSession (issue #1355).
|
||||
from semantica_mcp.mcp.server import UnknownToolError, call_tool
|
||||
try:
|
||||
result = call_tool(tool_name, tool_args)
|
||||
except UnknownToolError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
from semantica_mcp.mcp.session import MCPSession
|
||||
session = MCPSession(config=cli_ctx.config.to_dict())
|
||||
result = session.call_tool(tool_name, **tool_args)
|
||||
except ImportError as exc:
|
||||
raise click.ClickException(f"MCP module not available: {exc}") from exc
|
||||
if _is_json(cli_ctx, local_json):
|
||||
_jecho(result if isinstance(result, (dict, list)) else {"result": str(result)})
|
||||
else:
|
||||
|
||||
@@ -212,7 +212,7 @@ class FastEmbedStore(ProviderStore):
|
||||
self.logger.info(f"Loaded FastEmbed model: {self.model_name}")
|
||||
except (ImportError, OSError):
|
||||
self.logger.warning(
|
||||
"fastembed not available. Install with: pip install 'semantica[embeddings-local]'"
|
||||
"fastembed not available. Install with: pip install fastembed"
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to load FastEmbed model: {e}")
|
||||
|
||||
@@ -35,7 +35,6 @@ try:
|
||||
|
||||
SENTENCE_TRANSFORMERS_AVAILABLE = True
|
||||
except (ImportError, OSError):
|
||||
SentenceTransformer = None
|
||||
SENTENCE_TRANSFORMERS_AVAILABLE = False
|
||||
|
||||
try:
|
||||
@@ -43,7 +42,6 @@ try:
|
||||
|
||||
FASTEMBED_AVAILABLE = True
|
||||
except (ImportError, OSError):
|
||||
TextEmbedding = None
|
||||
FASTEMBED_AVAILABLE = False
|
||||
|
||||
|
||||
@@ -158,7 +156,7 @@ class TextEmbedder:
|
||||
else:
|
||||
self.logger.warning(
|
||||
"fastembed not available. "
|
||||
"Install with: pip install 'semantica[embeddings-local]'. "
|
||||
"Install with: pip install fastembed. "
|
||||
"Using fallback embedding method."
|
||||
)
|
||||
else:
|
||||
@@ -180,7 +178,7 @@ class TextEmbedder:
|
||||
else:
|
||||
self.logger.warning(
|
||||
"sentence-transformers not available. "
|
||||
"Install with: pip install 'semantica[embeddings-local]'. "
|
||||
"Install with: pip install sentence-transformers. "
|
||||
"Using fallback embedding method."
|
||||
)
|
||||
|
||||
|
||||
@@ -52,23 +52,6 @@ async def list_decisions(
|
||||
return [_node_to_decision(node) for node in nodes[skip : skip + limit]]
|
||||
|
||||
|
||||
@router.get("/causal-distance", response_model=CausalDistanceReport)
|
||||
async def causal_distance(
|
||||
source: str = Query(..., description="Source node/decision ID"),
|
||||
target: str = Query(..., description="Target node/decision ID"),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""FR-8 — Compute causal distance between two decisions via causal-edge-only traversal."""
|
||||
from ...context.causal_analyzer import CausalChainAnalyzer
|
||||
|
||||
analyzer = CausalChainAnalyzer(session.graph)
|
||||
report = await asyncio.to_thread(analyzer.interpret_causal_distance, source, target)
|
||||
return CausalDistanceReport(**report)
|
||||
|
||||
|
||||
# NOTE: static routes (e.g. /causal-distance above) must stay above this
|
||||
# dynamic route — Starlette matches in definition order, otherwise the static
|
||||
# path is captured as decision_id (see issue #1531).
|
||||
@router.get("/{decision_id}", response_model=DecisionResponse)
|
||||
async def get_decision(
|
||||
decision_id: str,
|
||||
@@ -142,6 +125,20 @@ async def get_precedents(
|
||||
return [_node_to_decision(decision) for _, decision in scored[:limit]]
|
||||
|
||||
|
||||
@router.get("/causal-distance", response_model=CausalDistanceReport)
|
||||
async def causal_distance(
|
||||
source: str = Query(..., description="Source node/decision ID"),
|
||||
target: str = Query(..., description="Target node/decision ID"),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""FR-8 — Compute causal distance between two decisions via causal-edge-only traversal."""
|
||||
from ...context.causal_analyzer import CausalChainAnalyzer
|
||||
|
||||
analyzer = CausalChainAnalyzer(session.graph)
|
||||
report = await asyncio.to_thread(analyzer.interpret_causal_distance, source, target)
|
||||
return CausalDistanceReport(**report)
|
||||
|
||||
|
||||
@router.get("/{decision_id}/compliance", response_model=ComplianceResponse)
|
||||
async def check_compliance(
|
||||
decision_id: str,
|
||||
|
||||
@@ -329,27 +329,12 @@ class GraphExporter:
|
||||
lines.append(' http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">')
|
||||
lines.append("")
|
||||
|
||||
# Define attribute keys.
|
||||
# GraphML requires every key id referenced by a <data> element to be
|
||||
# declared here with a matching <key> element.
|
||||
#
|
||||
# for="all" — key is valid on both nodes and edges
|
||||
# for="node" — key is valid on nodes only
|
||||
# for="edge" — key is valid on edges only
|
||||
#
|
||||
# label : used on <node> (human-readable label) and <edge> (type).
|
||||
# Declared for="all" so both uses are schema-valid.
|
||||
# type : node entity type; only written on nodes.
|
||||
# confidence: written on both nodes and edges when include_attributes
|
||||
# is True; declared for="all" so edge confidence is valid.
|
||||
lines.append(
|
||||
' <key id="label" for="all" attr.name="label" attr.type="string"/>'
|
||||
)
|
||||
# Define attribute keys
|
||||
lines.append(
|
||||
' <key id="type" for="node" attr.name="type" attr.type="string"/>'
|
||||
)
|
||||
lines.append(
|
||||
' <key id="confidence" for="all" attr.name="confidence" attr.type="double"/>'
|
||||
' <key id="confidence" for="node" attr.name="confidence" attr.type="double"/>'
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
@@ -357,31 +342,23 @@ class GraphExporter:
|
||||
lines.append(' <graph id="G" edgedefault="directed">')
|
||||
lines.append("")
|
||||
|
||||
# xml.sax.saxutils helpers:
|
||||
# escape(v) – escapes & < > in text node content
|
||||
# quoteattr(v) – escapes & < > " ' and wraps in the
|
||||
# appropriate quote character for use as an
|
||||
# XML attribute value (including the quotes)
|
||||
from xml.sax.saxutils import escape, quoteattr
|
||||
|
||||
# Export nodes
|
||||
nodes = graph_data.get("nodes", [])
|
||||
for node in nodes:
|
||||
node_id = str(node.get("id") or "")
|
||||
label = str(node.get("label") or "")
|
||||
node_type = str(node.get("type") or "")
|
||||
node_id = node.get("id", "")
|
||||
label = node.get("label", "")
|
||||
node_type = node.get("type", "")
|
||||
|
||||
# quoteattr produces the surrounding quotes; do NOT add extra "…"
|
||||
lines.append(f" <node id={quoteattr(node_id)}>")
|
||||
lines.append(f" <data key=\"label\">{escape(label)}</data>")
|
||||
lines.append(f" <data key=\"type\">{escape(node_type)}</data>")
|
||||
lines.append(f' <node id="{node_id}">')
|
||||
lines.append(f' <data key="label">{label}</data>')
|
||||
lines.append(f' <data key="type">{node_type}</data>')
|
||||
|
||||
# Add attributes if requested
|
||||
if self.include_attributes and "attributes" in node:
|
||||
attrs = node["attributes"]
|
||||
if "confidence" in attrs:
|
||||
lines.append(
|
||||
f" <data key=\"confidence\">{escape(str(attrs['confidence']))}</data>"
|
||||
f' <data key="confidence">{attrs["confidence"]}</data>'
|
||||
)
|
||||
|
||||
lines.append(" </node>")
|
||||
@@ -391,19 +368,19 @@ class GraphExporter:
|
||||
# Export edges
|
||||
edges = graph_data.get("edges", [])
|
||||
for edge in edges:
|
||||
source = str(edge.get("source") or "")
|
||||
target = str(edge.get("target") or "")
|
||||
edge_type = str(edge.get("type") or "")
|
||||
source = edge.get("source", "")
|
||||
target = edge.get("target", "")
|
||||
edge_type = edge.get("type", "")
|
||||
|
||||
lines.append(f" <edge source={quoteattr(source)} target={quoteattr(target)}>")
|
||||
lines.append(f" <data key=\"label\">{escape(edge_type)}</data>")
|
||||
lines.append(f' <edge source="{source}" target="{target}">')
|
||||
lines.append(f' <data key="label">{edge_type}</data>')
|
||||
|
||||
# Add attributes if requested
|
||||
if self.include_attributes and "attributes" in edge:
|
||||
attrs = edge["attributes"]
|
||||
if "confidence" in attrs:
|
||||
lines.append(
|
||||
f" <data key=\"confidence\">{escape(str(attrs['confidence']))}</data>"
|
||||
f' <data key="confidence">{attrs["confidence"]}</data>'
|
||||
)
|
||||
|
||||
lines.append(" </edge>")
|
||||
|
||||
@@ -421,7 +421,7 @@ class VectorExporter:
|
||||
import numpy as np
|
||||
except (ImportError, OSError):
|
||||
raise ImportError(
|
||||
"FAISS not installed. Install with: pip install 'semantica[vectorstore-faiss]' (or 'semantica[gpu]' for CUDA)"
|
||||
"FAISS not installed. Install with: pip install faiss-cpu or faiss-gpu"
|
||||
)
|
||||
|
||||
# Extract vectors and IDs
|
||||
|
||||
@@ -133,11 +133,7 @@ import importlib
|
||||
from typing import TYPE_CHECKING, Any, Dict, Tuple
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .salesforce_ingestor import (
|
||||
SalesforceConnector,
|
||||
SalesforceData,
|
||||
SalesforceIngestor,
|
||||
)
|
||||
from .salesforce_ingestor import SalesforceConnector, SalesforceData, SalesforceIngestor
|
||||
|
||||
from .config import IngestConfig, ingest_config
|
||||
from .file_ingestor import (
|
||||
@@ -252,42 +248,32 @@ _LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
|
||||
_OPTIONAL_DEPENDENCY_MESSAGES = {
|
||||
".repo_ingestor": (
|
||||
"Repository ingestion requires optional dependency 'GitPython'. "
|
||||
"Install it before importing RepoIngestor or using ingest_repository(). "
|
||||
"Install it with: pip install 'semantica[ingest-git]'"
|
||||
"Install it before importing RepoIngestor or using ingest_repository()."
|
||||
),
|
||||
".web_ingestor": (
|
||||
"Web ingestion requires optional dependency 'beautifulsoup4'. "
|
||||
"Install it before importing WebIngestor or using ingest_web(). "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
"Install it before importing WebIngestor or using ingest_web()."
|
||||
),
|
||||
".feed_ingestor": (
|
||||
"Feed ingestion requires optional dependency 'beautifulsoup4'. "
|
||||
"Install it before importing FeedIngestor or using ingest_feed(). "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
"Install it before importing FeedIngestor or using ingest_feed()."
|
||||
),
|
||||
".email_ingestor": (
|
||||
"Email ingestion requires optional dependency 'beautifulsoup4'. "
|
||||
"Install it before importing EmailIngestor or using ingest_email(). "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
),
|
||||
".xml_ingestor": (
|
||||
"XML ingestion requires optional dependency 'lxml'. "
|
||||
"Install it before importing XMLIngestor or using ingest_xml(). "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
"Install it before importing EmailIngestor or using ingest_email()."
|
||||
),
|
||||
".parquet_ingestor": (
|
||||
"Parquet ingestion requires optional dependency 'pyarrow'. "
|
||||
"Install it before importing ParquetIngestor or using ingest_parquet(). "
|
||||
"Install it with: pip install 'semantica[ingest-parquet]'"
|
||||
"Install it before importing ParquetIngestor or using ingest_parquet()."
|
||||
),
|
||||
".arrow_ingestor": (
|
||||
"Arrow ingestion requires optional dependency 'pyarrow'. "
|
||||
"Install it before importing ArrowIngestor or using ingest_arrow(). "
|
||||
"Install it with: pip install 'semantica[ingest-arrow]'"
|
||||
"Install it before importing ArrowIngestor or using ingest_arrow()."
|
||||
),
|
||||
".salesforce_ingestor": (
|
||||
"Salesforce ingestion requires optional dependency 'simple-salesforce'. "
|
||||
"Install it with: pip install 'semantica[db-salesforce]'"
|
||||
"Install it with: pip install \"semantica[db-salesforce]\" "
|
||||
"or: pip install simple-salesforce>=1.12.0"
|
||||
),
|
||||
}
|
||||
|
||||
@@ -300,55 +286,13 @@ def __getattr__(name: str) -> Any:
|
||||
module_name, attr_name = _LAZY_EXPORTS[name]
|
||||
try:
|
||||
module = importlib.import_module(module_name, __name__)
|
||||
except (ImportError, OSError) as exc:
|
||||
except ModuleNotFoundError as exc:
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
missing_name = getattr(exc, "name", None)
|
||||
if message and (
|
||||
missing_name is None
|
||||
or any(
|
||||
pkg in missing_name
|
||||
for pkg in ("git", "bs4", "pyarrow", "simple_salesforce", "lxml")
|
||||
)
|
||||
):
|
||||
if message and missing_name in {"git", "bs4", "pyarrow", "simple_salesforce"}:
|
||||
raise ImportError(message) from exc
|
||||
raise
|
||||
|
||||
# Guard against backends whose modules imported cleanly with dependencies
|
||||
# set to None; ensure probe imports (e.g. try: from semantica.ingest import ...)
|
||||
# fail at import time rather than postponing failure to construction time.
|
||||
if module_name == ".repo_ingestor" and name in {"RepoIngestor"}:
|
||||
if getattr(module, "git", None) is None:
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
if message:
|
||||
raise ImportError(message)
|
||||
|
||||
if module_name == ".xml_ingestor" and name in {"XMLIngestor"}:
|
||||
if getattr(module, "etree", None) is None:
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
if message:
|
||||
raise ImportError(message)
|
||||
|
||||
if module_name == ".parquet_ingestor" and name in {"ParquetIngestor"}:
|
||||
if not getattr(module, "PARQUET_AVAILABLE", True):
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
if message:
|
||||
raise ImportError(message)
|
||||
|
||||
if module_name == ".arrow_ingestor" and name in {"ArrowIngestor"}:
|
||||
if not getattr(module, "ARROW_AVAILABLE", True):
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
if message:
|
||||
raise ImportError(message)
|
||||
|
||||
if module_name == ".salesforce_ingestor" and name in {
|
||||
"SalesforceIngestor",
|
||||
"SalesforceConnector",
|
||||
}:
|
||||
if not getattr(module, "SALESFORCE_AVAILABLE", True):
|
||||
message = _OPTIONAL_DEPENDENCY_MESSAGES.get(module_name)
|
||||
if message:
|
||||
raise ImportError(message)
|
||||
|
||||
value = getattr(module, attr_name)
|
||||
globals()[name] = value
|
||||
return value
|
||||
|
||||
+129
-200
@@ -193,7 +193,6 @@ def _is_scp_like_repo_source(source: str) -> bool:
|
||||
"""Return True for scp-like SSH remotes (``user@host:path``)."""
|
||||
return bool(_SCP_LIKE_REPO_URL_RE.match(source.strip()))
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .api_ingestor import APIData
|
||||
from .arrow_ingestor import ArrowData
|
||||
@@ -253,12 +252,7 @@ def ingest_file(
|
||||
if custom_method and custom_method != ingest_file:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -324,18 +318,21 @@ def ingest_parquet(
|
||||
if custom_method and custom_method != ingest_parquet:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .parquet_ingestor import ParquetIngestor
|
||||
try:
|
||||
from .parquet_ingestor import ParquetIngestor
|
||||
except ModuleNotFoundError as exc:
|
||||
if _is_missing_dependency(exc, "pyarrow"):
|
||||
raise _missing_optional_dependency(
|
||||
"Parquet ingestion",
|
||||
"pyarrow",
|
||||
) from exc
|
||||
raise
|
||||
|
||||
config = ingest_config.get_method_config("parquet")
|
||||
config.update(kwargs)
|
||||
@@ -400,18 +397,21 @@ def ingest_arrow(
|
||||
if custom_method and custom_method != ingest_arrow:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .arrow_ingestor import ArrowIngestor
|
||||
try:
|
||||
from .arrow_ingestor import ArrowIngestor
|
||||
except ModuleNotFoundError as exc:
|
||||
if _is_missing_dependency(exc, "pyarrow"):
|
||||
raise _missing_optional_dependency(
|
||||
"Arrow ingestion",
|
||||
"pyarrow",
|
||||
) from exc
|
||||
raise
|
||||
|
||||
config = ingest_config.get_method_config("arrow")
|
||||
config.update(kwargs)
|
||||
@@ -481,12 +481,7 @@ def ingest_xml(
|
||||
if custom_method and custom_method != ingest_xml:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -496,10 +491,7 @@ def ingest_xml(
|
||||
|
||||
config = ingest_config.get_method_config("xml")
|
||||
config.update(kwargs)
|
||||
try:
|
||||
ingestor = XMLIngestor(**config)
|
||||
except ImportError as exc:
|
||||
raise _missing_optional_dependency("XML ingestion", "lxml") from exc
|
||||
ingestor = XMLIngestor(**config)
|
||||
|
||||
def _run_single(
|
||||
path: Union[str, Path],
|
||||
@@ -519,8 +511,6 @@ def ingest_xml(
|
||||
|
||||
return _run_single(source_path)
|
||||
|
||||
except ConfigurationError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to ingest XML: {e}")
|
||||
raise
|
||||
@@ -555,12 +545,7 @@ def ingest_web(
|
||||
if custom_method and custom_method != ingest_web:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -650,12 +635,7 @@ def ingest_public_api(
|
||||
if custom_method and custom_method != ingest_public_api:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -742,12 +722,7 @@ def ingest_feed(
|
||||
if custom_method and custom_method != ingest_feed:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -816,12 +791,7 @@ def ingest_stream(
|
||||
if custom_method and custom_method != ingest_stream:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -898,29 +868,26 @@ def ingest_repository(
|
||||
if custom_method and custom_method != ingest_repository:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .repo_ingestor import RepoIngestor
|
||||
try:
|
||||
from .repo_ingestor import RepoIngestor
|
||||
except ModuleNotFoundError as exc:
|
||||
if _is_missing_dependency(exc, "git"):
|
||||
raise _missing_optional_dependency(
|
||||
"Repository ingestion", "GitPython"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
# Get config
|
||||
config = ingest_config.get_method_config("repo")
|
||||
config.update(kwargs)
|
||||
|
||||
try:
|
||||
ingestor = RepoIngestor(**config)
|
||||
except ImportError as exc:
|
||||
raise _missing_optional_dependency(
|
||||
"Repository ingestion", "GitPython"
|
||||
) from exc
|
||||
ingestor = RepoIngestor(**config)
|
||||
|
||||
if method == "clone" or (
|
||||
isinstance(source, str)
|
||||
@@ -973,12 +940,7 @@ def ingest_email(
|
||||
if custom_method and custom_method != ingest_email:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -1057,12 +1019,7 @@ def ingest_ontology(
|
||||
if custom_method and custom_method != ingest_ontology:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -1128,12 +1085,7 @@ def ingest_database(
|
||||
if custom_method and custom_method != ingest_database:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -1281,122 +1233,105 @@ def ingest_salesforce(
|
||||
if custom_method and custom_method != ingest_salesforce:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source,
|
||||
fallback_on_custom_error=fallback, **kwargs,
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .salesforce_ingestor import SalesforceIngestor
|
||||
|
||||
# Unpack credential dict (if given); everything else stays in kwargs.
|
||||
creds: Dict[str, Any] = {}
|
||||
if source is not None:
|
||||
if not isinstance(source, dict):
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() source must be a credential dict or None. "
|
||||
"Pass sobject_name / soql as keyword arguments."
|
||||
)
|
||||
creds = dict(source)
|
||||
|
||||
# Merge any ingest_config method config under "salesforce".
|
||||
# get_method_config() now returns a copy, so this dict is safe to mutate.
|
||||
# We build the final connector config in order of increasing priority:
|
||||
# 1. base method config (lowest — global defaults set by operator)
|
||||
# 2. per-call credential dict supplied via `source`
|
||||
# 3. per-call connector params supplied as kwargs
|
||||
# Credentials are extracted from kwargs and removed so they don't also
|
||||
# flow into the ingest method call (which doesn't understand them).
|
||||
_CONNECTOR_PARAMS = frozenset(
|
||||
{
|
||||
"username",
|
||||
"password",
|
||||
"security_token",
|
||||
"domain",
|
||||
"instance_url",
|
||||
"session_id",
|
||||
"api_version",
|
||||
}
|
||||
)
|
||||
connector_kwargs = {k: v for k, v in kwargs.items() if k in _CONNECTOR_PARAMS}
|
||||
for k in _CONNECTOR_PARAMS:
|
||||
kwargs.pop(k, None)
|
||||
|
||||
# Build a fresh per-call config dict — never mutate the global store.
|
||||
config: Dict[str, Any] = {
|
||||
**ingest_config.get_method_config("salesforce"), # base (already a copy)
|
||||
**creds, # source dict credentials
|
||||
**connector_kwargs, # kwarg credentials
|
||||
}
|
||||
|
||||
try:
|
||||
ingestor = SalesforceIngestor(**config)
|
||||
except ImportError as exc:
|
||||
except ModuleNotFoundError as exc:
|
||||
if _is_missing_dependency(exc, "simple_salesforce"):
|
||||
raise _missing_optional_dependency(
|
||||
"Salesforce ingestion", "simple-salesforce"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
if method == "sobject":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='sobject' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
|
||||
elif method == "query":
|
||||
soql = kwargs.pop("soql", None)
|
||||
if not soql:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='query' requires "
|
||||
"soql keyword argument."
|
||||
)
|
||||
return ingestor.ingest_query(soql, **kwargs)
|
||||
|
||||
elif method == "list_sobjects":
|
||||
return ingestor.list_sobjects()
|
||||
|
||||
elif method == "schema":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='schema' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.get_sobject_schema(sobject_name)
|
||||
|
||||
elif method == "documents":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='documents' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
id_field = kwargs.pop("id_field", "Id")
|
||||
text_fields = kwargs.pop("text_fields", None)
|
||||
data = ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
return ingestor.export_as_documents(
|
||||
data, id_field=id_field, text_fields=text_fields
|
||||
)
|
||||
|
||||
else:
|
||||
# Unpack credential dict (if given); everything else stays in kwargs.
|
||||
creds: Dict[str, Any] = {}
|
||||
if source is not None:
|
||||
if not isinstance(source, dict):
|
||||
raise ProcessingError(
|
||||
f"Unknown ingest_salesforce method: {method!r}. "
|
||||
"Valid methods: 'sobject', 'query', 'list_sobjects', 'schema', "
|
||||
"'documents'."
|
||||
"ingest_salesforce() source must be a credential dict or None. "
|
||||
"Pass sobject_name / soql as keyword arguments."
|
||||
)
|
||||
creds = dict(source)
|
||||
|
||||
except ConfigurationError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to ingest salesforce: {e}")
|
||||
raise
|
||||
# Merge any ingest_config method config under "salesforce".
|
||||
# get_method_config() now returns a copy, so this dict is safe to mutate.
|
||||
# We build the final connector config in order of increasing priority:
|
||||
# 1. base method config (lowest — global defaults set by operator)
|
||||
# 2. per-call credential dict supplied via `source`
|
||||
# 3. per-call connector params supplied as kwargs
|
||||
# Credentials are extracted from kwargs and removed so they don't also
|
||||
# flow into the ingest method call (which doesn't understand them).
|
||||
_CONNECTOR_PARAMS = frozenset({
|
||||
"username", "password", "security_token", "domain",
|
||||
"instance_url", "session_id", "api_version",
|
||||
})
|
||||
connector_kwargs = {k: v for k, v in kwargs.items() if k in _CONNECTOR_PARAMS}
|
||||
for k in _CONNECTOR_PARAMS:
|
||||
kwargs.pop(k, None)
|
||||
|
||||
# Build a fresh per-call config dict — never mutate the global store.
|
||||
config: Dict[str, Any] = {
|
||||
**ingest_config.get_method_config("salesforce"), # base (already a copy)
|
||||
**creds, # source dict credentials
|
||||
**connector_kwargs, # kwarg credentials
|
||||
}
|
||||
|
||||
ingestor = SalesforceIngestor(**config)
|
||||
|
||||
if method == "sobject":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='sobject' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
|
||||
elif method == "query":
|
||||
soql = kwargs.pop("soql", None)
|
||||
if not soql:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='query' requires "
|
||||
"soql keyword argument."
|
||||
)
|
||||
return ingestor.ingest_query(soql, **kwargs)
|
||||
|
||||
elif method == "list_sobjects":
|
||||
return ingestor.list_sobjects()
|
||||
|
||||
elif method == "schema":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='schema' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
return ingestor.get_sobject_schema(sobject_name)
|
||||
|
||||
elif method == "documents":
|
||||
sobject_name = kwargs.pop("sobject_name", None)
|
||||
if not sobject_name:
|
||||
raise ProcessingError(
|
||||
"ingest_salesforce() with method='documents' requires "
|
||||
"sobject_name keyword argument."
|
||||
)
|
||||
id_field = kwargs.pop("id_field", "Id")
|
||||
text_fields = kwargs.pop("text_fields", None)
|
||||
data = ingestor.ingest_sobject(sobject_name, **kwargs)
|
||||
return ingestor.export_as_documents(data, id_field=id_field,
|
||||
text_fields=text_fields)
|
||||
|
||||
else:
|
||||
raise ProcessingError(
|
||||
f"Unknown ingest_salesforce method: {method!r}. "
|
||||
"Valid methods: 'sobject', 'query', 'list_sobjects', 'schema', "
|
||||
"'documents'."
|
||||
)
|
||||
|
||||
|
||||
def ingest_mcp(
|
||||
@@ -1464,12 +1399,7 @@ def ingest_mcp(
|
||||
if custom_method and custom_method != ingest_mcp:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
source,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -1708,9 +1638,8 @@ def ingest(
|
||||
elif source_type == "mcp":
|
||||
return {"data": ingest_mcp(sources, method=method or "resources", **kwargs)}
|
||||
elif source_type == "salesforce":
|
||||
return {
|
||||
"data": ingest_salesforce(sources, method=method or "sobject", **kwargs)
|
||||
}
|
||||
return {"data": ingest_salesforce(sources,
|
||||
method=method or "sobject", **kwargs)}
|
||||
else:
|
||||
raise ProcessingError(f"Unknown source type: {source_type}")
|
||||
|
||||
|
||||
@@ -28,29 +28,12 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import requests
|
||||
|
||||
try:
|
||||
from lxml import etree as lxml_etree
|
||||
|
||||
_SAFE_XML_PARSER = lxml_etree.XMLParser(
|
||||
resolve_entities=False,
|
||||
no_network=True,
|
||||
recover=False,
|
||||
huge_tree=False,
|
||||
load_dtd=False,
|
||||
remove_comments=True,
|
||||
remove_pis=True,
|
||||
)
|
||||
_LXML_SYNTAX_ERRORS: Tuple[type, ...] = (lxml_etree.XMLSyntaxError,)
|
||||
except (ImportError, OSError):
|
||||
lxml_etree = None
|
||||
_SAFE_XML_PARSER = None
|
||||
_LXML_SYNTAX_ERRORS = ()
|
||||
from lxml import etree as lxml_etree
|
||||
|
||||
try:
|
||||
from defusedxml import ElementTree as safe_xml_etree
|
||||
from defusedxml.common import DefusedXmlException
|
||||
except (ImportError, OSError): # pragma: no cover - fallback for minimal installs
|
||||
except ModuleNotFoundError: # pragma: no cover - fallback for minimal installs
|
||||
safe_xml_etree = None
|
||||
|
||||
class DefusedXmlException(Exception):
|
||||
@@ -88,6 +71,14 @@ AUTH_PARAM_NAMES = {
|
||||
"subscription-key",
|
||||
}
|
||||
|
||||
_SAFE_XML_PARSER = lxml_etree.XMLParser(
|
||||
resolve_entities=False,
|
||||
no_network=True,
|
||||
recover=False,
|
||||
huge_tree=False,
|
||||
load_dtd=False,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PublicAPIExample:
|
||||
@@ -452,9 +443,7 @@ class PublicAPIIngestor(RESTIngestor):
|
||||
APIData: Normalized public API response and metadata
|
||||
"""
|
||||
self._validate_endpoint(endpoint)
|
||||
self._validate_no_auth_request(
|
||||
headers=headers, params=params, options=options, endpoint=endpoint
|
||||
)
|
||||
self._validate_no_auth_request(headers=headers, params=params, options=options, endpoint=endpoint)
|
||||
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=endpoint,
|
||||
@@ -615,9 +604,7 @@ class PublicAPIIngestor(RESTIngestor):
|
||||
for endpoint in endpoints:
|
||||
try:
|
||||
results.append(
|
||||
self.ingest_public_api(
|
||||
endpoint, method=method, **copy.deepcopy(options)
|
||||
)
|
||||
self.ingest_public_api(endpoint, method=method, **copy.deepcopy(options))
|
||||
)
|
||||
except Exception as exc:
|
||||
self.logger.warning(f"Failed to fetch public API {endpoint}: {exc}")
|
||||
@@ -743,7 +730,7 @@ class PublicAPIIngestor(RESTIngestor):
|
||||
raise ProcessingError(
|
||||
f"Failed to parse {detected_format.upper()} public API response"
|
||||
) from exc
|
||||
except (DefusedXmlException, *_LXML_SYNTAX_ERRORS) as exc:
|
||||
except (DefusedXmlException, lxml_etree.XMLSyntaxError) as exc:
|
||||
raise ProcessingError("Failed to parse XML public API response") from exc
|
||||
|
||||
def _detect_response_format(
|
||||
@@ -783,40 +770,15 @@ class PublicAPIIngestor(RESTIngestor):
|
||||
def _parse_xml(self, xml_text: str) -> Dict[str, Any]:
|
||||
if safe_xml_etree is not None:
|
||||
root = safe_xml_etree.fromstring(xml_text)
|
||||
elif lxml_etree is not None and _SAFE_XML_PARSER is not None:
|
||||
else:
|
||||
root = lxml_etree.fromstring(
|
||||
xml_text.encode("utf-8"),
|
||||
parser=_SAFE_XML_PARSER,
|
||||
)
|
||||
for elem in root.iter():
|
||||
if (
|
||||
elem.tag is lxml_etree.Comment
|
||||
or elem.tag is lxml_etree.PI
|
||||
or getattr(elem.tag, "__name__", "")
|
||||
in ("Comment", "ProcessingInstruction", "PI")
|
||||
):
|
||||
continue
|
||||
if callable(elem.tag) or not isinstance(elem.tag, str):
|
||||
raise ProcessingError("Failed to parse XML public API response")
|
||||
else:
|
||||
raise ProcessingError(
|
||||
"XML parsing requires 'defusedxml' or 'lxml'. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
return self._element_to_dict(root)
|
||||
|
||||
def _element_to_dict(self, element: Any) -> Dict[str, Any]:
|
||||
children = [
|
||||
self._element_to_dict(child)
|
||||
for child in list(element)
|
||||
if not (
|
||||
callable(child.tag)
|
||||
or (
|
||||
lxml_etree is not None
|
||||
and (child.tag is lxml_etree.Comment or child.tag is lxml_etree.PI)
|
||||
)
|
||||
)
|
||||
]
|
||||
children = [self._element_to_dict(child) for child in list(element)]
|
||||
return {
|
||||
"tag": self._strip_namespace(element.tag),
|
||||
"attributes": {
|
||||
@@ -827,9 +789,7 @@ class PublicAPIIngestor(RESTIngestor):
|
||||
"children": children,
|
||||
}
|
||||
|
||||
def _strip_namespace(self, value: Any) -> str:
|
||||
if not isinstance(value, str):
|
||||
return str(value)
|
||||
def _strip_namespace(self, value: str) -> str:
|
||||
if value.startswith("{") and "}" in value:
|
||||
return value.split("}", 1)[1]
|
||||
return value
|
||||
|
||||
@@ -29,8 +29,6 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import os
|
||||
import re
|
||||
@@ -46,10 +44,7 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple, Union
|
||||
from urllib.parse import urlparse
|
||||
|
||||
try:
|
||||
import git
|
||||
except (ImportError, OSError):
|
||||
git = None
|
||||
import git
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -62,7 +57,9 @@ ALLOWED_CLONE_OPTIONS: Set[str] = {"depth", "branch", "single_branch", "no_tags"
|
||||
ALLOWED_REPO_URL_SCHEMES = frozenset({"https", "http", "git", "ssh"})
|
||||
# SCP-like SSH remotes: user@host:path/to/repo.git (no scheme)
|
||||
_SCP_LIKE_REPO_URL_RE = re.compile(r"^[^@\s]+@[^:\s]+:.+$")
|
||||
_ENV_VAR_TOKEN_RE = re.compile(r"\$(\{[A-Za-z_][A-Za-z0-9_]*\}|[A-Za-z_][A-Za-z0-9_]*)")
|
||||
_ENV_VAR_TOKEN_RE = re.compile(
|
||||
r"\$(\{[A-Za-z_][A-Za-z0-9_]*\}|[A-Za-z_][A-Za-z0-9_]*)"
|
||||
)
|
||||
# Short-lived DNS cache for host validation. This reduces repeated lookups but
|
||||
# does not eliminate DNS-rebinding / TOCTOU races between validate and clone —
|
||||
# network egress controls remain recommended.
|
||||
@@ -528,11 +525,6 @@ class RepoIngestor:
|
||||
**kwargs: Additional configuration parameters (merged into config)
|
||||
"""
|
||||
self.logger = get_logger("repo_ingestor")
|
||||
if git is None:
|
||||
raise ImportError(
|
||||
"GitPython is required for repository ingestion. "
|
||||
"Install it with: pip install 'semantica[ingest-git]'"
|
||||
)
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
|
||||
@@ -598,7 +590,10 @@ class RepoIngestor:
|
||||
networks). Those addresses are not SSRF-sensitive.
|
||||
"""
|
||||
return bool(
|
||||
ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_unspecified
|
||||
ip.is_private
|
||||
or ip.is_loopback
|
||||
or ip.is_link_local
|
||||
or ip.is_unspecified
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -625,7 +620,9 @@ class RepoIngestor:
|
||||
# or hanging lookup for one host cannot stall cache access for
|
||||
# concurrent lookups of other hosts.
|
||||
try:
|
||||
addrinfos = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)
|
||||
addrinfos = socket.getaddrinfo(
|
||||
host, None, type=socket.SOCK_STREAM
|
||||
)
|
||||
except socket.gaierror as exc:
|
||||
raise ValidationError(
|
||||
f"Cannot resolve repository host {host!r}: {exc}"
|
||||
@@ -798,7 +795,9 @@ class RepoIngestor:
|
||||
f"Allowed schemes: {sorted(ALLOWED_REPO_URL_SCHEMES)}"
|
||||
)
|
||||
if not parsed.netloc or not host:
|
||||
raise ValidationError(f"Repository URL must include a host: {repo_url}")
|
||||
raise ValidationError(
|
||||
f"Repository URL must include a host: {repo_url}"
|
||||
)
|
||||
|
||||
RepoIngestor._validate_repo_host(host)
|
||||
|
||||
@@ -813,7 +812,9 @@ class RepoIngestor:
|
||||
"include_extensions",
|
||||
"max_depth",
|
||||
}
|
||||
candidate = {k: v for k, v in options.items() if k not in non_git_options}
|
||||
candidate = {
|
||||
k: v for k, v in options.items() if k not in non_git_options
|
||||
}
|
||||
unsafe = set(candidate) - ALLOWED_CLONE_OPTIONS
|
||||
if unsafe:
|
||||
raise ValidationError(
|
||||
@@ -892,7 +893,9 @@ class RepoIngestor:
|
||||
if "include_extensions" in options:
|
||||
# Normalize extensions to include dot prefix
|
||||
exts = options["include_extensions"]
|
||||
normalized_exts = [e if e.startswith(".") else f".{e}" for e in exts]
|
||||
normalized_exts = [
|
||||
e if e.startswith(".") else f".{e}" for e in exts
|
||||
]
|
||||
file_filters["extensions"] = normalized_exts
|
||||
|
||||
# Process code files
|
||||
@@ -1059,14 +1062,14 @@ class RepoIngestor:
|
||||
return code_files
|
||||
|
||||
def get_repository_info(
|
||||
self, repo_url: str, repo: Optional[Any] = None
|
||||
self, repo_url: str, repo: Optional[git.Repo] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Get repository metadata and information.
|
||||
|
||||
Args:
|
||||
repo_url: Repository URL
|
||||
repo: Git repository object (git.Repo, optional)
|
||||
repo: Git repository object (optional)
|
||||
|
||||
Returns:
|
||||
dict: Repository information
|
||||
@@ -1108,7 +1111,6 @@ class RepoIngestor:
|
||||
def cleanup(self):
|
||||
"""Cleanup temporary repository files."""
|
||||
if self.temp_dir and os.path.exists(self.temp_dir):
|
||||
|
||||
def onexc(func, path, exc_info):
|
||||
"""
|
||||
Error handler for shutil.rmtree.
|
||||
@@ -1121,7 +1123,6 @@ class RepoIngestor:
|
||||
Usage : shutil.rmtree(path, onerror=onexc)
|
||||
"""
|
||||
import stat
|
||||
|
||||
if not os.access(path, os.W_OK):
|
||||
# Is the error an access error ?
|
||||
os.chmod(path, stat.S_IWUSR)
|
||||
|
||||
@@ -24,10 +24,7 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
try:
|
||||
from lxml import etree
|
||||
except (ImportError, OSError):
|
||||
etree = None
|
||||
from lxml import etree
|
||||
|
||||
from ..utils.constants import FILE_SIZE_LIMITS
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
@@ -75,11 +72,6 @@ class XMLIngestor:
|
||||
**kwargs: Additional configuration values
|
||||
"""
|
||||
self.logger = get_logger("xml_ingestor")
|
||||
if etree is None:
|
||||
raise ImportError(
|
||||
"lxml is required for XMLIngestor. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
@@ -792,8 +784,8 @@ class XMLIngestor:
|
||||
first_error = errors[0] if errors else "No detailed validation error available."
|
||||
return f"{prefix} for {source}: {first_error}"
|
||||
|
||||
def _format_xml_error(self, exc: Any) -> str:
|
||||
if hasattr(exc, "error_log") and exc.error_log:
|
||||
def _format_xml_error(self, exc: etree.XMLSyntaxError) -> str:
|
||||
if exc.error_log:
|
||||
return str(exc.error_log.last_error)
|
||||
return str(exc)
|
||||
|
||||
|
||||
@@ -141,7 +141,7 @@ class NodeEmbedder:
|
||||
|
||||
if method == "node2vec" and not GENSIM_AVAILABLE:
|
||||
raise ImportError(
|
||||
"gensim is required for Node2Vec. Install with: pip install 'semantica[graph-embeddings]'"
|
||||
"gensim is required for Node2Vec. Install with: pip install gensim"
|
||||
)
|
||||
|
||||
def compute_embeddings(
|
||||
|
||||
@@ -32,20 +32,12 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
try:
|
||||
from docx import Document
|
||||
from docx.document import Document as DocxDocument
|
||||
from docx.oxml.table import CT_Tbl
|
||||
from docx.oxml.text.paragraph import CT_P
|
||||
from docx.table import Table
|
||||
from docx.text.paragraph import Paragraph
|
||||
except (ImportError, OSError):
|
||||
Document = None
|
||||
DocxDocument = None
|
||||
CT_Tbl = None
|
||||
CT_P = None
|
||||
Table = None
|
||||
Paragraph = None
|
||||
from docx import Document
|
||||
from docx.document import Document as DocxDocument
|
||||
from docx.oxml.table import CT_Tbl
|
||||
from docx.oxml.text.paragraph import CT_P
|
||||
from docx.table import Table
|
||||
from docx.text.paragraph import Paragraph
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -92,9 +84,7 @@ class DOCXParser:
|
||||
self.config = config
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
def parse(
|
||||
self, file_path: Union[str, Path], pipeline_id: Optional[str] = None, **options
|
||||
) -> Dict[str, Any]:
|
||||
def parse(self, file_path: Union[str, Path], pipeline_id: Optional[str] = None, **options) -> Dict[str, Any]:
|
||||
"""
|
||||
Parse DOCX document.
|
||||
|
||||
@@ -109,12 +99,6 @@ class DOCXParser:
|
||||
Returns:
|
||||
dict: Parsed document data
|
||||
"""
|
||||
if Document is None:
|
||||
raise ProcessingError(
|
||||
"python-docx is required to parse DOCX files. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
|
||||
file_path = Path(file_path)
|
||||
|
||||
# Track DOCX parsing
|
||||
|
||||
@@ -33,11 +33,7 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import pandas as pd
|
||||
|
||||
try:
|
||||
from openpyxl import load_workbook
|
||||
except (ImportError, OSError):
|
||||
load_workbook = None
|
||||
from openpyxl import load_workbook
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -96,12 +92,6 @@ class ExcelParser:
|
||||
Returns:
|
||||
ExcelData or ExcelSheet: Parsed Excel data
|
||||
"""
|
||||
if load_workbook is None:
|
||||
raise ProcessingError(
|
||||
"openpyxl is required to parse Excel files. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
|
||||
file_path = Path(file_path)
|
||||
|
||||
# Track Excel parsing
|
||||
|
||||
@@ -33,10 +33,7 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from urllib.parse import urljoin
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except (ImportError, OSError):
|
||||
BeautifulSoup = None
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -116,12 +113,6 @@ class HTMLParser:
|
||||
Returns:
|
||||
HTMLData: Parsed HTML data
|
||||
"""
|
||||
if BeautifulSoup is None:
|
||||
raise ProcessingError(
|
||||
"beautifulsoup4 is required to parse HTML files. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
|
||||
# Track HTML parsing
|
||||
file_path = None
|
||||
if isinstance(html_content, Path) or (
|
||||
|
||||
+33
-99
@@ -123,6 +123,7 @@ Example Usage:
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ConfigurationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
|
||||
from .code_parser import CodeParser
|
||||
@@ -177,16 +178,10 @@ def parse_document(
|
||||
>>> text = parse_document("document.pdf", method="default", extract_text=True)
|
||||
"""
|
||||
custom_method = method_registry.get("document", method)
|
||||
if custom_method and custom_method != parse_document:
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
file_type,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, file_type, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -253,8 +248,7 @@ def parse_document_docling(
|
||||
|
||||
# Register Docling method
|
||||
try:
|
||||
from . import docling_parser # noqa: F401
|
||||
|
||||
from .docling_parser import DoclingParser
|
||||
method_registry.register("document", "docling", parse_document_docling)
|
||||
except (ImportError, OSError):
|
||||
# Docling not available, skip registration
|
||||
@@ -295,17 +289,10 @@ def parse_web_content(
|
||||
>>> html = parse_web_content("page.html", content_type="html", method="default")
|
||||
"""
|
||||
custom_method = method_registry.get("web", method)
|
||||
if custom_method and custom_method != parse_web_content:
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
content,
|
||||
content_type,
|
||||
base_url,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, content, content_type, base_url, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -358,16 +345,10 @@ def parse_structured_data(
|
||||
>>> csv_data = parse_structured_data("data.csv", data_format="csv", method="default")
|
||||
"""
|
||||
custom_method = method_registry.get("structured", method)
|
||||
if custom_method and custom_method != parse_structured_data:
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
data,
|
||||
data_format,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, data, data_format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -411,15 +392,10 @@ def parse_email(
|
||||
>>> headers = parse_email("email.eml", method="headers")
|
||||
"""
|
||||
custom_method = method_registry.get("email", method)
|
||||
if custom_method and custom_method != parse_email:
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
email_content,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, email_content, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -466,16 +442,10 @@ def parse_code(
|
||||
>>> structure = parse_code("script.py", method="ast")
|
||||
"""
|
||||
custom_method = method_registry.get("code", method)
|
||||
if custom_method and custom_method != parse_code:
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
language,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, language, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -525,16 +495,10 @@ def parse_media(
|
||||
>>> video = parse_media("video.mp4", method="default")
|
||||
"""
|
||||
custom_method = method_registry.get("media", method)
|
||||
if custom_method and custom_method != parse_media:
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
media_type,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, media_type, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -577,15 +541,10 @@ def parse_pdf(
|
||||
>>> pages = parse_pdf("document.pdf", method="default", pages=[1, 2, 3])
|
||||
"""
|
||||
custom_method = method_registry.get("document", method)
|
||||
if custom_method and custom_method not in (parse_pdf, parse_document):
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -626,15 +585,10 @@ def parse_docx(
|
||||
>>> docx = parse_docx("document.docx", method="default")
|
||||
"""
|
||||
custom_method = method_registry.get("document", method)
|
||||
if custom_method and custom_method not in (parse_docx, parse_document):
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -674,15 +628,10 @@ def parse_json(file_path: Union[str, Path], method: str = "default", **kwargs) -
|
||||
>>> flattened = parse_json("data.json", method="default", flatten=True)
|
||||
"""
|
||||
custom_method = method_registry.get("structured", method)
|
||||
if custom_method and custom_method not in (parse_json, parse_structured_data):
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -726,16 +675,10 @@ def parse_csv(
|
||||
>>> tab_separated = parse_csv("data.tsv", delimiter="\t", method="default")
|
||||
"""
|
||||
custom_method = method_registry.get("structured", method)
|
||||
if custom_method and custom_method not in (parse_csv, parse_structured_data):
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
delimiter,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, delimiter, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -771,15 +714,10 @@ def parse_xml(file_path: Union[str, Path], method: str = "default", **kwargs) ->
|
||||
>>> xml_data = parse_xml("data.xml", method="default")
|
||||
"""
|
||||
custom_method = method_registry.get("structured", method)
|
||||
if custom_method and custom_method not in (parse_xml, parse_structured_data):
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -824,15 +762,10 @@ def parse_image(
|
||||
>>> ocr_text = image.get("ocr_result", {}).get("text", "")
|
||||
"""
|
||||
custom_method = method_registry.get("media", method)
|
||||
if custom_method and custom_method not in (parse_image, parse_media):
|
||||
if custom_method:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger,
|
||||
method,
|
||||
custom_method,
|
||||
file_path,
|
||||
fallback_on_custom_error=fallback,
|
||||
**kwargs,
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
@@ -859,9 +792,10 @@ def list_available_methods(task: Optional[str] = None) -> Dict[str, List[str]]:
|
||||
return method_registry.list_all(task)
|
||||
|
||||
|
||||
# NOTE: the built-in dispatchers (parse_document, parse_web_content, ...) must
|
||||
# NOT be registered under their own task's "default" method name. Each
|
||||
# dispatcher starts with method_registry.get(<task>, method) and would find
|
||||
# itself, re-entering infinitely until RecursionError. "default" is the
|
||||
# built-in code path and stays unregistered; users can still register their
|
||||
# own "default" (or any other name) to override it.
|
||||
# Register default methods
|
||||
method_registry.register("document", "default", parse_document)
|
||||
method_registry.register("web", "default", parse_web_content)
|
||||
method_registry.register("structured", "default", parse_structured_data)
|
||||
method_registry.register("email", "default", parse_email)
|
||||
method_registry.register("code", "default", parse_code)
|
||||
method_registry.register("media", "default", parse_media)
|
||||
|
||||
@@ -33,10 +33,7 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except (ImportError, OSError):
|
||||
BeautifulSoup = None
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -190,12 +187,6 @@ class HTMLContentParser(HTMLParser):
|
||||
}
|
||||
|
||||
# Load HTML for structure extraction
|
||||
if BeautifulSoup is None:
|
||||
raise ProcessingError(
|
||||
"beautifulsoup4 is required for HTML structure extraction. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
|
||||
if isinstance(html_content, Path) or (
|
||||
isinstance(html_content, str) and Path(html_content).exists()
|
||||
):
|
||||
@@ -252,12 +243,6 @@ class HTMLContentParser(HTMLParser):
|
||||
else:
|
||||
html_string = html_content
|
||||
|
||||
if BeautifulSoup is None:
|
||||
raise ProcessingError(
|
||||
"beautifulsoup4 is required for HTML cleaning. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
|
||||
soup = BeautifulSoup(html_string, "html.parser")
|
||||
|
||||
# Remove scripts and styles
|
||||
|
||||
@@ -34,10 +34,7 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
try:
|
||||
from lxml import etree
|
||||
except (ImportError, OSError):
|
||||
etree = None
|
||||
from lxml import etree
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -108,13 +105,7 @@ class XMLParser:
|
||||
)
|
||||
|
||||
try:
|
||||
explicit_engine = options.get("engine") or self.config.get("engine")
|
||||
engine = explicit_engine or ("lxml" if etree is not None else "etree")
|
||||
if engine == "lxml" and etree is None:
|
||||
raise ProcessingError(
|
||||
"lxml is required to parse XML with engine='lxml'. "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
engine = options.get("engine", "lxml")
|
||||
|
||||
# Load XML content
|
||||
if file_path_obj:
|
||||
@@ -156,7 +147,7 @@ class XMLParser:
|
||||
self, xml_string: str, source: str, options: Dict[str, Any]
|
||||
) -> XMLData:
|
||||
"""Parse XML using lxml."""
|
||||
parser = etree.XMLParser(remove_blank_text=True, remove_comments=True)
|
||||
parser = etree.XMLParser(remove_blank_text=True)
|
||||
root = etree.fromstring(xml_string.encode("utf-8"), parser)
|
||||
|
||||
# Extract namespaces
|
||||
@@ -197,10 +188,8 @@ class XMLParser:
|
||||
metadata={"source": source, "engine": "etree"},
|
||||
)
|
||||
|
||||
def _element_to_xml_element(self, element) -> Optional[XMLElement]:
|
||||
def _element_to_xml_element(self, element) -> XMLElement:
|
||||
"""Convert lxml element to XMLElement."""
|
||||
if not hasattr(element, "tag") or not isinstance(element.tag, str):
|
||||
return None
|
||||
tag = element.tag
|
||||
if "}" in tag:
|
||||
namespace, tag = tag.split("}", 1)
|
||||
@@ -217,18 +206,12 @@ class XMLParser:
|
||||
|
||||
# Process children
|
||||
for child in element:
|
||||
child_elem = self._element_to_xml_element(child)
|
||||
if child_elem is not None:
|
||||
xml_elem.children.append(child_elem)
|
||||
xml_elem.children.append(self._element_to_xml_element(child))
|
||||
|
||||
return xml_elem
|
||||
|
||||
def _etree_element_to_xml_element(
|
||||
self, element: ET.Element
|
||||
) -> Optional[XMLElement]:
|
||||
def _etree_element_to_xml_element(self, element: ET.Element) -> XMLElement:
|
||||
"""Convert ElementTree element to XMLElement."""
|
||||
if not hasattr(element, "tag") or not isinstance(element.tag, str):
|
||||
return None
|
||||
tag = element.tag
|
||||
if "}" in tag:
|
||||
namespace, tag = tag.split("}", 1)
|
||||
@@ -245,9 +228,7 @@ class XMLParser:
|
||||
|
||||
# Process children
|
||||
for child in element:
|
||||
child_elem = self._etree_element_to_xml_element(child)
|
||||
if child_elem is not None:
|
||||
xml_elem.children.append(child_elem)
|
||||
xml_elem.children.append(self._etree_element_to_xml_element(child))
|
||||
|
||||
return xml_elem
|
||||
|
||||
@@ -268,30 +249,18 @@ class XMLParser:
|
||||
xml_data = self.parse(file_path, **options)
|
||||
|
||||
# Use lxml for XPath queries
|
||||
if etree is None:
|
||||
raise ProcessingError(
|
||||
"lxml is required for find_elements (XPath queries). "
|
||||
"Install it with: pip install 'semantica[documents]'"
|
||||
)
|
||||
|
||||
xml_string = (
|
||||
file_path
|
||||
if isinstance(file_path, str) and not Path(file_path).exists()
|
||||
else Path(file_path).read_text(encoding="utf-8")
|
||||
else Path(file_path).read_text()
|
||||
)
|
||||
parser = etree.XMLParser(remove_blank_text=True, remove_comments=True)
|
||||
root = etree.fromstring(xml_string.encode("utf-8"), parser=parser)
|
||||
root = etree.fromstring(xml_string.encode("utf-8"))
|
||||
|
||||
# Register namespaces for XPath
|
||||
namespaces = xml_data.namespaces
|
||||
elements = root.xpath(xpath, namespaces=namespaces)
|
||||
|
||||
results = []
|
||||
for elem in elements:
|
||||
xml_elem = self._element_to_xml_element(elem)
|
||||
if xml_elem is not None:
|
||||
results.append(xml_elem)
|
||||
return results
|
||||
return [self._element_to_xml_element(elem) for elem in elements]
|
||||
|
||||
def extract_by_tag(
|
||||
self, file_path: Union[str, Path], tag_name: str, **options
|
||||
|
||||
@@ -34,11 +34,11 @@ Example Usage:
|
||||
>>> from semantica.semantic_extract import NamedEntityRecognizer
|
||||
>>> ner = NamedEntityRecognizer(confidence_threshold=0.7)
|
||||
>>> entities = ner.extract_entities("Steve Jobs founded Apple.")
|
||||
|
||||
|
||||
>>> from semantica.semantic_extract import RelationExtractor
|
||||
>>> rel_extractor = RelationExtractor(confidence_threshold=0.6)
|
||||
>>> relations = rel_extractor.extract_relations(text, entities=entities)
|
||||
|
||||
|
||||
>>> from semantica.semantic_extract import TripletExtractor
|
||||
>>> triplet_extractor = TripletExtractor(include_temporal=True)
|
||||
>>> triplets = triplet_extractor.extract_triplets(text)
|
||||
@@ -52,6 +52,7 @@ from __future__ import annotations
|
||||
import importlib
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
|
||||
_LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
|
||||
# Named Entity Recognition
|
||||
"NamedEntityRecognizer": (".named_entity_recognizer", "NamedEntityRecognizer"),
|
||||
@@ -91,10 +92,7 @@ _LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
|
||||
"RoleLabeler": (".semantic_analyzer", "RoleLabeler"),
|
||||
"SemanticClusterer": (".semantic_analyzer", "SemanticClusterer"),
|
||||
# Semantic Network
|
||||
"SemanticNetworkExtractor": (
|
||||
".semantic_network_extractor",
|
||||
"SemanticNetworkExtractor",
|
||||
),
|
||||
"SemanticNetworkExtractor": (".semantic_network_extractor", "SemanticNetworkExtractor"),
|
||||
"SemanticNode": (".semantic_network_extractor", "SemanticNode"),
|
||||
"SemanticEdge": (".semantic_network_extractor", "SemanticEdge"),
|
||||
"SemanticNetwork": (".semantic_network_extractor", "SemanticNetwork"),
|
||||
@@ -105,10 +103,6 @@ _LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
|
||||
# Validation
|
||||
"ExtractionValidator": (".extraction_validator", "ExtractionValidator"),
|
||||
"ValidationResult": (".extraction_validator", "ValidationResult"),
|
||||
# Schema-guided validation
|
||||
"ExtractionSchema": (".schema", "ExtractionSchema"),
|
||||
"Predicate": (".schema", "Predicate"),
|
||||
"SchemaValidator": (".schema_validator", "SchemaValidator"),
|
||||
# Providers
|
||||
"BaseProvider": (".providers", "BaseProvider"),
|
||||
"OpenAIProvider": (".providers", "OpenAIProvider"),
|
||||
@@ -145,7 +139,6 @@ def __getattr__(name: str) -> Any:
|
||||
globals()[name] = value
|
||||
return value
|
||||
|
||||
|
||||
__all__ = [
|
||||
# Named Entity Recognition
|
||||
"NamedEntityRecognizer",
|
||||
@@ -195,10 +188,6 @@ __all__ = [
|
||||
# Validation
|
||||
"ExtractionValidator",
|
||||
"ValidationResult",
|
||||
# Schema-guided validation
|
||||
"ExtractionSchema",
|
||||
"Predicate",
|
||||
"SchemaValidator",
|
||||
# Providers
|
||||
"BaseProvider",
|
||||
"OpenAIProvider",
|
||||
|
||||
@@ -209,11 +209,6 @@ def load_spacy_model(name: str):
|
||||
Raises whatever ``spacy.load`` raises (``OSError`` for a missing model), so
|
||||
callers keep their existing fallback behavior.
|
||||
"""
|
||||
if spacy is None:
|
||||
raise ImportError(
|
||||
"spaCy is not installed. Install with: pip install 'semantica[nlp-spacy]'"
|
||||
)
|
||||
|
||||
cached = _spacy_model_cache.get(name)
|
||||
if cached is not None and cached[0] is spacy:
|
||||
return cached[1]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,234 +0,0 @@
|
||||
"""Domain schema view over an ontology, for schema-guided extraction.
|
||||
|
||||
An :class:`ExtractionSchema` is a lightweight, read-only view over a domain
|
||||
ontology: the set of allowed *concept* names (entity labels) and the allowed
|
||||
*predicates* with optional ``domain`` / ``range`` constraints. It is what
|
||||
:class:`~semantica.semantic_extract.schema_validator.SchemaValidator` checks
|
||||
extraction output against.
|
||||
|
||||
The schema deliberately **reuses the project's existing OWL ontology
|
||||
representation** instead of introducing a parallel "template" type. Build one
|
||||
from the dict produced by :func:`semantica.ontology.generate_ontology`
|
||||
(:meth:`ExtractionSchema.from_ontology`), or from an OWL / Turtle file or string
|
||||
(:meth:`ExtractionSchema.from_owl`).
|
||||
|
||||
An empty ``domain`` / ``range`` means "unconstrained", matching OWL's convention
|
||||
that an object property with no ``rdfs:domain`` / ``rdfs:range`` places no
|
||||
restriction on its subjects / objects.
|
||||
|
||||
Reference
|
||||
---------
|
||||
Using an ontology to constrain what may be extracted is the defining idea of
|
||||
ontology-based information extraction (OBIE): Wimalasuriya & Dou, "Ontology-Based
|
||||
Information Extraction: An Introduction and a Survey" (2010).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, FrozenSet, Iterable, Mapping, Optional, Set
|
||||
|
||||
|
||||
def _as_name_set(value: Any) -> Set[str]:
|
||||
"""Coerce a ``domain`` / ``range`` value to a set of concept names.
|
||||
|
||||
Accepts a string, an iterable of strings / mappings, a mapping (reads its
|
||||
``name`` / ``label``), or ``None``. ``None`` / empty yields an empty set,
|
||||
interpreted downstream as "unconstrained".
|
||||
"""
|
||||
if value is None:
|
||||
return set()
|
||||
if isinstance(value, str):
|
||||
return {value}
|
||||
if isinstance(value, Mapping):
|
||||
name = value.get("name") or value.get("label")
|
||||
return {str(name)} if name else {str(k) for k in value}
|
||||
if isinstance(value, Iterable):
|
||||
out: Set[str] = set()
|
||||
for item in value:
|
||||
out |= _as_name_set(item)
|
||||
return out
|
||||
return {str(value)}
|
||||
|
||||
|
||||
_OWL_THING = {
|
||||
"owl:Thing",
|
||||
"Thing",
|
||||
"http://www.w3.org/2002/07/owl#Thing",
|
||||
}
|
||||
|
||||
|
||||
def _drop_thing(names: Set[str]) -> Set[str]:
|
||||
"""Collapse an ``owl:Thing`` domain / range to *unconstrained* (empty set).
|
||||
|
||||
``owl:Thing`` is the universal class, so a property whose ``domain`` / ``range``
|
||||
is ``owl:Thing`` places no restriction. ``OntologyGenerator`` emits it as the
|
||||
fallback when it cannot resolve endpoint types; keeping it as a literal
|
||||
``{"Thing"}`` constraint would reject every real endpoint, so we treat its
|
||||
presence as "any concept".
|
||||
"""
|
||||
return set() if names & _OWL_THING else names
|
||||
|
||||
|
||||
def _constraint_set(value: Any) -> Set[str]:
|
||||
"""A ``domain`` / ``range`` constraint set, with ``owl:Thing`` meaning unconstrained."""
|
||||
return _drop_thing(_as_name_set(value))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Predicate:
|
||||
"""An allowed predicate with optional ``domain`` / ``range`` constraints.
|
||||
|
||||
Empty ``domain`` / ``range`` means any concept is allowed in that position.
|
||||
"""
|
||||
|
||||
name: str
|
||||
domain: FrozenSet[str] = frozenset()
|
||||
range: FrozenSet[str] = frozenset()
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractionSchema:
|
||||
"""Read-only view over a domain ontology used to gate extraction.
|
||||
|
||||
Names are matched **exactly**: the schema vocabulary and the extraction labels
|
||||
must share a normalization convention. ``OntologyGenerator`` normalizes concept
|
||||
names to PascalCase and predicate names to camelCase, so entity labels /
|
||||
relation predicates validated against a generated schema should follow the same
|
||||
convention (e.g. label entities ``Person`` rather than ``person``).
|
||||
"""
|
||||
|
||||
concepts: FrozenSet[str] = field(default_factory=frozenset)
|
||||
predicates: Dict[str, Predicate] = field(default_factory=dict)
|
||||
|
||||
# ---- constructors -------------------------------------------------
|
||||
|
||||
@classmethod
|
||||
def from_ontology(cls, ontology: Any) -> "ExtractionSchema":
|
||||
"""Build a schema from a ``generate_ontology``-style ontology.
|
||||
|
||||
Accepts the mapping returned by :func:`semantica.ontology.generate_ontology`,
|
||||
or an object exposing such a mapping via a ``.data`` attribute — e.g. the
|
||||
``OntologyData`` returned by ``semantica.ingest.OntologyIngestor``.
|
||||
|
||||
Reads ``ontology["classes"]`` (each carrying a ``name`` / ``label``) as
|
||||
concepts and ``ontology["properties"]`` (each carrying a ``name`` and
|
||||
optional ``domain`` / ``range``) as predicates. Missing ``domain`` /
|
||||
``range`` means unconstrained; unrecognised keys are ignored.
|
||||
|
||||
Endpoint types named in a property's ``domain`` / ``range`` are also folded
|
||||
into the concept set (consistent with :meth:`from_owl`), so a type referenced
|
||||
only as an endpoint — e.g. one that didn't clear the class-frequency gate
|
||||
during induction — is still a known concept.
|
||||
"""
|
||||
if not isinstance(ontology, Mapping) and hasattr(ontology, "data"):
|
||||
ontology = ontology.data # unwrap OntologyData-like objects
|
||||
|
||||
concepts: Set[str] = set()
|
||||
for c in ontology.get("classes", []) or []:
|
||||
name = (c.get("name") or c.get("label")) if isinstance(c, Mapping) else c
|
||||
if name:
|
||||
concepts.add(str(name))
|
||||
|
||||
predicates: Dict[str, Predicate] = {}
|
||||
for p in ontology.get("properties", []) or []:
|
||||
if not isinstance(p, Mapping):
|
||||
continue
|
||||
name = p.get("name") or p.get("label")
|
||||
if not name:
|
||||
continue
|
||||
domain = _constraint_set(p.get("domain"))
|
||||
rng = _constraint_set(p.get("range"))
|
||||
predicates[str(name)] = Predicate(
|
||||
name=str(name),
|
||||
domain=frozenset(domain),
|
||||
range=frozenset(rng),
|
||||
)
|
||||
concepts |= domain | rng
|
||||
return cls(concepts=frozenset(concepts), predicates=predicates)
|
||||
|
||||
@classmethod
|
||||
def from_owl(
|
||||
cls, source: str, *, format: Optional[str] = None
|
||||
) -> "ExtractionSchema":
|
||||
"""Build a schema from an OWL / RDF file path or serialized string.
|
||||
|
||||
``owl:Class`` / ``rdfs:Class`` become concepts; ``owl:ObjectProperty`` with
|
||||
``rdfs:domain`` / ``rdfs:range`` becomes a predicate (its domain / range
|
||||
names are folded into the concept set, with ``owl:Thing`` treated as
|
||||
unconstrained). Names prefer an explicit ``rdfs:label``, falling back to the
|
||||
URI's local name, so the vocabulary matches :meth:`from_ontology`. Requires
|
||||
``rdflib`` (an existing project dependency).
|
||||
"""
|
||||
from rdflib import OWL, RDF, RDFS, Graph, URIRef
|
||||
|
||||
graph = Graph()
|
||||
if os.path.exists(source):
|
||||
graph.parse(source, format=format)
|
||||
else:
|
||||
graph.parse(data=source, format=format or "turtle")
|
||||
|
||||
def _local(term: Any) -> str:
|
||||
text = str(term)
|
||||
for sep in ("#", "/"):
|
||||
if sep in text:
|
||||
text = text.rsplit(sep, 1)[-1]
|
||||
return text
|
||||
|
||||
def _name_of(term: Any) -> str:
|
||||
label = graph.value(term, RDFS.label)
|
||||
return str(label) if label is not None else _local(term)
|
||||
|
||||
concepts: Set[str] = {
|
||||
_name_of(c)
|
||||
for class_type in (OWL.Class, RDFS.Class)
|
||||
for c in graph.subjects(RDF.type, class_type)
|
||||
if isinstance(c, URIRef)
|
||||
}
|
||||
predicates: Dict[str, Predicate] = {}
|
||||
for prop in graph.subjects(RDF.type, OWL.ObjectProperty):
|
||||
name = _name_of(prop)
|
||||
domain = _drop_thing(
|
||||
{_name_of(d) for d in graph.objects(prop, RDFS.domain)}
|
||||
)
|
||||
rng = _drop_thing({_name_of(r) for r in graph.objects(prop, RDFS.range)})
|
||||
predicates[name] = Predicate(
|
||||
name=name, domain=frozenset(domain), range=frozenset(rng)
|
||||
)
|
||||
concepts |= domain | rng
|
||||
return cls(concepts=frozenset(concepts), predicates=predicates)
|
||||
|
||||
# ---- queries ------------------------------------------------------
|
||||
|
||||
def has_concept(self, name: str) -> bool:
|
||||
"""Whether ``name`` is an allowed concept (entity label)."""
|
||||
return name in self.concepts
|
||||
|
||||
def has_predicate(self, name: str) -> bool:
|
||||
"""Whether ``name`` is an allowed predicate."""
|
||||
return name in self.predicates
|
||||
|
||||
def allows_relation(
|
||||
self, subject_label: str, predicate: str, object_label: str
|
||||
) -> bool:
|
||||
"""Whether a relation conforms to the schema.
|
||||
|
||||
True iff subject and object are known concepts, the predicate is known,
|
||||
and subject / object satisfy the predicate's ``domain`` / ``range``
|
||||
(an empty ``domain`` / ``range`` allows any concept).
|
||||
|
||||
Membership is exact; ``subClassOf`` hierarchies are not traversed, so a
|
||||
subclass endpoint is not accepted for a superclass ``domain`` / ``range``
|
||||
(subclass-aware validation is a possible follow-up).
|
||||
"""
|
||||
if subject_label not in self.concepts or object_label not in self.concepts:
|
||||
return False
|
||||
pred = self.predicates.get(predicate)
|
||||
if pred is None:
|
||||
return False
|
||||
if pred.domain and subject_label not in pred.domain:
|
||||
return False
|
||||
if pred.range and object_label not in pred.range:
|
||||
return False
|
||||
return True
|
||||
@@ -1,196 +0,0 @@
|
||||
"""Schema-guided validation for semantic extractions.
|
||||
|
||||
:class:`SchemaValidator` is a sibling of
|
||||
:class:`~semantica.semantic_extract.extraction_validator.ExtractionValidator`:
|
||||
same two entry points (:meth:`validate_entities` / :meth:`validate_relations`)
|
||||
returning the same :class:`ValidationResult`, so the two compose back-to-back.
|
||||
|
||||
The two validators check **orthogonal** axes. ``ExtractionValidator`` checks
|
||||
*confidence* and structural sanity; ``SchemaValidator`` checks *conformance to a
|
||||
domain ontology*:
|
||||
|
||||
* every entity label must be a concept in the schema;
|
||||
* every relation predicate must be in the schema and satisfy its ``domain`` /
|
||||
``range``.
|
||||
|
||||
This is the deterministic core of ontology-based information extraction (OBIE,
|
||||
Wimalasuriya & Dou 2010): it needs no LLM and is fully unit-testable.
|
||||
:meth:`validate_entities` / :meth:`validate_relations` *report* conformance;
|
||||
:meth:`filter_by_schema` / :meth:`filter_relations_by_schema` return the
|
||||
conforming subset (mirroring ``ExtractionValidator.filter_by_confidence``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from .extraction_validator import ValidationResult
|
||||
from .ner_extractor import Entity
|
||||
from .relation_extractor import Relation
|
||||
from .schema import ExtractionSchema
|
||||
|
||||
|
||||
class SchemaValidator:
|
||||
"""Validate extractions against a domain ontology (:class:`ExtractionSchema`)."""
|
||||
|
||||
def __init__(
|
||||
self, schema: ExtractionSchema, method: Optional[str] = None, **config: Any
|
||||
) -> None:
|
||||
"""Initialize the validator.
|
||||
|
||||
Args:
|
||||
schema: The domain ontology view to validate against.
|
||||
method: Reserved for future method-specific validation (unused),
|
||||
mirroring ``ExtractionValidator``.
|
||||
**config: Reserved configuration options.
|
||||
"""
|
||||
self.schema = schema
|
||||
self.method = method
|
||||
self.config = config
|
||||
|
||||
def validate_entities(
|
||||
self, entities: Union[List[Entity], List[List[Entity]]], **options: Any
|
||||
) -> Union[ValidationResult, List[ValidationResult]]:
|
||||
"""Validate that entity labels are concepts in the schema.
|
||||
|
||||
Handles both a single list and a batch (list of lists), like
|
||||
``ExtractionValidator.validate_entities``.
|
||||
"""
|
||||
if entities and isinstance(entities, list) and isinstance(entities[0], list):
|
||||
results = []
|
||||
for idx, batch in enumerate(entities):
|
||||
res = self.validate_entities(batch, **options)
|
||||
if "batch_index" not in res.metadata:
|
||||
res.metadata["batch_index"] = idx
|
||||
results.append(res)
|
||||
return results
|
||||
|
||||
errors: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
out_of_vocab = [e for e in entities if not self.schema.has_concept(e.label)]
|
||||
unknown_labels = sorted({e.label for e in out_of_vocab})
|
||||
if out_of_vocab:
|
||||
errors.append(
|
||||
f"{len(out_of_vocab)} entities with labels outside the schema: "
|
||||
f"{', '.join(unknown_labels)}"
|
||||
)
|
||||
|
||||
total = len(entities)
|
||||
conforming = total - len(out_of_vocab)
|
||||
metrics = {
|
||||
"total_entities": total,
|
||||
"in_vocabulary": conforming,
|
||||
"out_of_vocabulary": len(out_of_vocab),
|
||||
"unknown_labels": unknown_labels,
|
||||
"schema_concepts": len(self.schema.concepts),
|
||||
}
|
||||
score = conforming / total if total else 1.0
|
||||
|
||||
return ValidationResult(
|
||||
valid=not errors,
|
||||
score=score,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
metrics=metrics,
|
||||
metadata=self._metadata(entities),
|
||||
)
|
||||
|
||||
def validate_relations(
|
||||
self, relations: Union[List[Relation], List[List[Relation]]], **options: Any
|
||||
) -> Union[ValidationResult, List[ValidationResult]]:
|
||||
"""Validate relation predicates and ``domain`` / ``range`` against the schema.
|
||||
|
||||
Handles both a single list and a batch (list of lists), like
|
||||
``ExtractionValidator.validate_relations``.
|
||||
"""
|
||||
if relations and isinstance(relations, list) and isinstance(relations[0], list):
|
||||
results = []
|
||||
for idx, batch in enumerate(relations):
|
||||
res = self.validate_relations(batch, **options)
|
||||
if "batch_index" not in res.metadata:
|
||||
res.metadata["batch_index"] = idx
|
||||
results.append(res)
|
||||
return results
|
||||
|
||||
errors: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
# Guard malformed relations (missing subject/object) before dereferencing
|
||||
# their endpoints, matching ExtractionValidator's own leniency.
|
||||
malformed = [r for r in relations if not r.subject or not r.object]
|
||||
well_formed = [r for r in relations if r.subject and r.object]
|
||||
|
||||
unknown_predicate = [
|
||||
r for r in well_formed if not self.schema.has_predicate(r.predicate)
|
||||
]
|
||||
dr_violation = [
|
||||
r
|
||||
for r in well_formed
|
||||
if self.schema.has_predicate(r.predicate)
|
||||
and not self.schema.allows_relation(
|
||||
r.subject.label, r.predicate, r.object.label
|
||||
)
|
||||
]
|
||||
if malformed:
|
||||
errors.append(f"{len(malformed)} relations missing a subject or object")
|
||||
if unknown_predicate:
|
||||
preds = sorted({r.predicate for r in unknown_predicate})
|
||||
errors.append(
|
||||
f"{len(unknown_predicate)} relations with predicates outside the "
|
||||
f"schema: {', '.join(preds)}"
|
||||
)
|
||||
if dr_violation:
|
||||
errors.append(
|
||||
f"{len(dr_violation)} relations violating domain/range constraints"
|
||||
)
|
||||
|
||||
total = len(relations)
|
||||
conforming = total - len(malformed) - len(unknown_predicate) - len(dr_violation)
|
||||
metrics = {
|
||||
"total_relations": total,
|
||||
"conforming": conforming,
|
||||
"malformed": len(malformed),
|
||||
"unknown_predicate": len(unknown_predicate),
|
||||
"domain_range_violation": len(dr_violation),
|
||||
"schema_predicates": len(self.schema.predicates),
|
||||
}
|
||||
score = conforming / total if total else 1.0
|
||||
|
||||
return ValidationResult(
|
||||
valid=not errors,
|
||||
score=score,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
metrics=metrics,
|
||||
metadata=self._metadata(relations),
|
||||
)
|
||||
|
||||
def filter_by_schema(self, entities: List[Entity]) -> List[Entity]:
|
||||
"""Return only entities whose label is a concept in the schema."""
|
||||
return [e for e in entities if self.schema.has_concept(e.label)]
|
||||
|
||||
def filter_relations_by_schema(self, relations: List[Relation]) -> List[Relation]:
|
||||
"""Return only relations that fully conform to the schema."""
|
||||
return [
|
||||
r
|
||||
for r in relations
|
||||
if r.subject
|
||||
and r.object
|
||||
and self.schema.has_predicate(r.predicate)
|
||||
and self.schema.allows_relation(
|
||||
r.subject.label, r.predicate, r.object.label
|
||||
)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _metadata(items: List[Any]) -> Dict[str, Any]:
|
||||
"""Carry ``batch_index`` / ``document_id`` through, like ``ExtractionValidator``."""
|
||||
metadata: Dict[str, Any] = {}
|
||||
if items:
|
||||
first = items[0]
|
||||
if getattr(first, "metadata", None):
|
||||
for key in ("batch_index", "document_id"):
|
||||
if key in first.metadata:
|
||||
metadata[key] = first.metadata[key]
|
||||
return metadata
|
||||
+13
-21
@@ -38,15 +38,15 @@ Example Usage:
|
||||
>>> from semantica.utils import clean_text, normalize_entities
|
||||
>>> cleaned = clean_text(" Hello World ")
|
||||
>>> entities = normalize_entities([{"id": "e1", "text": "John", "type": "PERSON"}])
|
||||
>>>
|
||||
>>>
|
||||
>>> from semantica.utils import hash_data, safe_filename
|
||||
>>> data_hash = hash_data({"key": "value"})
|
||||
>>> safe_name = safe_filename("my file.txt")
|
||||
>>>
|
||||
>>>
|
||||
>>> from semantica.utils import merge_dicts, get_nested_value
|
||||
>>> merged = merge_dicts({"a": 1}, {"b": 2}, deep=True)
|
||||
>>> value = get_nested_value(config, "database.host", default="localhost")
|
||||
>>>
|
||||
>>>
|
||||
>>> from semantica.utils import retry_on_error
|
||||
>>> @retry_on_error(max_retries=3, delay=1.0)
|
||||
... def fetch_data():
|
||||
@@ -457,9 +457,7 @@ def chunk_list(items: List[Any], chunk_size: int) -> List[List[Any]]:
|
||||
Returns:
|
||||
List of chunks
|
||||
"""
|
||||
return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)]
|
||||
|
||||
|
||||
return [items[i : i + chunk_size] for i in range(0, len(items), chunk_size)]
|
||||
def flatten_dict(
|
||||
d: Dict[str, Any], parent_key: str = "", sep: str = "."
|
||||
) -> Dict[str, Any]:
|
||||
@@ -558,27 +556,27 @@ def safe_import(
|
||||
) -> Tuple[Any, bool]:
|
||||
"""
|
||||
Safely import an optional module, handling both ImportError and OSError.
|
||||
|
||||
|
||||
This is useful for optional dependencies that may fail to import due to:
|
||||
- Missing package (ImportError)
|
||||
- DLL loading failures on Windows, e.g., PyTorch (OSError)
|
||||
|
||||
|
||||
Args:
|
||||
module_name: Name of the module to import (e.g., "spacy", "docling.document_converter")
|
||||
package: Optional package name for relative imports
|
||||
default: Default value to return if import fails
|
||||
error_message: Optional custom error message for logging
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (module_or_default, success_flag):
|
||||
- If import succeeds: (imported_module, True)
|
||||
- If import fails: (default, False)
|
||||
|
||||
|
||||
Example:
|
||||
>>> spacy, available = safe_import("spacy")
|
||||
>>> if available:
|
||||
... doc = spacy.load("en_core_web_sm")
|
||||
>>>
|
||||
>>>
|
||||
>>> converter, available = safe_import("docling.document_converter", default=None)
|
||||
>>> if available:
|
||||
... converter = converter()
|
||||
@@ -589,13 +587,11 @@ def safe_import(
|
||||
else:
|
||||
module = importlib.import_module(module_name)
|
||||
return module, True
|
||||
except (ImportError, OSError) as e:
|
||||
except (ImportError, ModuleNotFoundError, OSError) as e:
|
||||
if error_message:
|
||||
import sys
|
||||
|
||||
if "logging" in sys.modules:
|
||||
from .logging import get_logger
|
||||
|
||||
logger = get_logger("utils.helpers")
|
||||
logger.debug(f"{error_message}: {e}")
|
||||
return default, False
|
||||
@@ -811,13 +807,9 @@ def _is_record(value: Any) -> bool:
|
||||
exporters rather than a ``ValidationError`` at the boundary where the
|
||||
problem is visible.
|
||||
"""
|
||||
return (
|
||||
isinstance(value, Mapping)
|
||||
or is_dataclass(value)
|
||||
or (
|
||||
hasattr(value, "__dict__")
|
||||
and not isinstance(value, (types.ModuleType, type))
|
||||
)
|
||||
return isinstance(value, Mapping) or is_dataclass(value) or (
|
||||
hasattr(value, "__dict__")
|
||||
and not isinstance(value, (types.ModuleType, type))
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -343,9 +343,7 @@ class FAISSIndex:
|
||||
was originally saved.
|
||||
"""
|
||||
if not FAISS_AVAILABLE:
|
||||
raise ProcessingError(
|
||||
"FAISS not available. Install with: pip install 'semantica[vectorstore-faiss]' (or 'semantica[gpu]' for CUDA)"
|
||||
)
|
||||
raise ProcessingError("FAISS not available")
|
||||
|
||||
path = Path(path)
|
||||
index = faiss.read_index(str(path))
|
||||
@@ -484,7 +482,7 @@ class FAISSIndexBuilder:
|
||||
"""
|
||||
if not FAISS_AVAILABLE:
|
||||
raise ProcessingError(
|
||||
"FAISS is not available. Install it with: pip install 'semantica[vectorstore-faiss]' (or 'semantica[gpu]' for CUDA)"
|
||||
"FAISS is not available. Install it with: pip install faiss-cpu or faiss-gpu"
|
||||
)
|
||||
|
||||
# Create index based on type
|
||||
@@ -556,7 +554,7 @@ class FAISSStore:
|
||||
# Check FAISS availability
|
||||
if not FAISS_AVAILABLE:
|
||||
self.logger.warning(
|
||||
"FAISS not available. Install with: pip install 'semantica[vectorstore-faiss]' (or 'semantica[gpu]' for CUDA)"
|
||||
"FAISS not available. Install with: pip install faiss-cpu or faiss-gpu"
|
||||
)
|
||||
|
||||
def create_index(
|
||||
@@ -755,9 +753,7 @@ class FAISSStore:
|
||||
FAISSIndex instance
|
||||
"""
|
||||
if not FAISS_AVAILABLE:
|
||||
raise ProcessingError(
|
||||
"FAISS not available. Install with: pip install 'semantica[vectorstore-faiss]' (or 'semantica[gpu]' for CUDA)"
|
||||
)
|
||||
raise ProcessingError("FAISS not available")
|
||||
|
||||
path = Path(path)
|
||||
if path.exists() and not _metadata_path(path).exists():
|
||||
|
||||
@@ -153,12 +153,9 @@ class QdrantCollection:
|
||||
raise ProcessingError("Qdrant not available")
|
||||
|
||||
try:
|
||||
# qdrant-client >=1.10.0: query_points() supersedes the removed search().
|
||||
# It returns a QueryResponse whose .points attribute is a list of
|
||||
# ScoredPoint objects (id, score, payload, …).
|
||||
response = self.client.query_points(
|
||||
search_results = self.client.search(
|
||||
collection_name=self.collection_name,
|
||||
query=query_vector.tolist(),
|
||||
query_vector=query_vector.tolist(),
|
||||
limit=limit,
|
||||
query_filter=query_filter,
|
||||
with_payload=True,
|
||||
@@ -167,19 +164,19 @@ class QdrantCollection:
|
||||
)
|
||||
|
||||
results = []
|
||||
for point in response.points:
|
||||
for result in search_results:
|
||||
results.append(
|
||||
{
|
||||
"id": point.id,
|
||||
"id": result.id,
|
||||
# See pinecone_store.py PineconeIndex.search_vectors for why
|
||||
# this uses x/(1+|x|) rather than clamping distance-to-zero:
|
||||
# Qdrant's Dot distance metric is unbounded, and the old
|
||||
# clamped formula collapsed every score >= 1.0 to 1.0.
|
||||
"score": (
|
||||
float(point.score) / (1.0 + abs(float(point.score))) + 1.0
|
||||
float(result.score) / (1.0 + abs(float(result.score))) + 1.0
|
||||
)
|
||||
/ 2.0,
|
||||
"metadata": point.payload or {},
|
||||
"metadata": result.payload or {},
|
||||
"vector": None,
|
||||
"distance": None,
|
||||
}
|
||||
@@ -384,27 +381,6 @@ class QdrantStore:
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to get collection: {str(e)}")
|
||||
|
||||
def _ensure_default_collection(self, dim: int = 384) -> QdrantCollection:
|
||||
"""Lazily attach the configured collection, creating it on first use.
|
||||
|
||||
Mirrors FAISSStore's automatic index creation so the VectorStore
|
||||
facade can read/write without an explicit create_collection() call.
|
||||
Reuses the existing collection if a previous process created it.
|
||||
"""
|
||||
# ``collection_name`` is the option the VectorStore facade and the
|
||||
# docs pass through; accept the legacy ``collection`` spelling too.
|
||||
name = (
|
||||
self.config.get("collection_name")
|
||||
or self.config.get("collection")
|
||||
or "semantica_default"
|
||||
)
|
||||
try:
|
||||
self.create_collection(name, vector_size=dim)
|
||||
except ProcessingError:
|
||||
self.get_collection(name)
|
||||
self.logger.info(f"Auto-initialized Qdrant collection '{name}' (dim={dim})")
|
||||
return self.collection
|
||||
|
||||
def insert_vectors(
|
||||
self,
|
||||
vectors: List[Union[np.ndarray, List[float]]],
|
||||
@@ -424,14 +400,6 @@ class QdrantStore:
|
||||
Returns:
|
||||
Insert response
|
||||
"""
|
||||
if len(ids) != len(vectors):
|
||||
# Points are paired with zip(vectors, ids), so a mismatched ID
|
||||
# list would silently drop the unpaired vectors while the
|
||||
# completion message still reports the full batch as inserted.
|
||||
raise ValidationError(
|
||||
f"Number of ids ({len(ids)}) must match number of vectors ({len(vectors)})"
|
||||
)
|
||||
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="vector_store",
|
||||
submodule="QdrantStore",
|
||||
@@ -440,10 +408,12 @@ class QdrantStore:
|
||||
|
||||
try:
|
||||
if self.collection is None:
|
||||
# len() not truthiness: vectors may be a 2-D ndarray, whose
|
||||
# truth value is ambiguous.
|
||||
dim = int(len(vectors[0])) if len(vectors) else 384
|
||||
self._ensure_default_collection(dim)
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message="Collection not initialized"
|
||||
)
|
||||
raise ProcessingError(
|
||||
"Collection not initialized. Call create_collection() or get_collection() first."
|
||||
)
|
||||
|
||||
if not QDRANT_AVAILABLE:
|
||||
self.progress_tracker.stop_tracking(
|
||||
@@ -508,7 +478,12 @@ class QdrantStore:
|
||||
|
||||
try:
|
||||
if self.search_engine is None:
|
||||
self._ensure_default_collection(int(len(query_vector)))
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message="Collection not initialized"
|
||||
)
|
||||
raise ProcessingError(
|
||||
"Collection not initialized. Call create_collection() or get_collection() first."
|
||||
)
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Performing similarity search..."
|
||||
@@ -720,31 +695,9 @@ class QdrantStore:
|
||||
collection_info = self.client.get_collection(
|
||||
self.collection.collection_name
|
||||
)
|
||||
# vectors_count was removed in qdrant-client 1.16.0.
|
||||
# When it is absent, only infer the total from points_count if we
|
||||
# can confirm the collection uses a single unnamed vector per point
|
||||
# (VectorParams). Named/multi-vector collections (dict of VectorParams)
|
||||
# have an unknown multiplier, so return None rather than a wrong value.
|
||||
# get_collection() accepts externally-created collections without schema
|
||||
# validation, so the schema must be inspected at stats time.
|
||||
vectors_count_fallback: Optional[int]
|
||||
try:
|
||||
vectors_cfg = collection_info.config.params.vectors
|
||||
vectors_count_fallback = (
|
||||
collection_info.points_count
|
||||
if QDRANT_AVAILABLE and isinstance(vectors_cfg, VectorParams)
|
||||
else None
|
||||
)
|
||||
except Exception:
|
||||
vectors_count_fallback = None
|
||||
|
||||
return {
|
||||
"points_count": collection_info.points_count,
|
||||
"vectors_count": getattr(
|
||||
collection_info,
|
||||
"vectors_count",
|
||||
vectors_count_fallback,
|
||||
),
|
||||
"vectors_count": collection_info.vectors_count,
|
||||
"status": str(collection_info.status)
|
||||
if hasattr(collection_info, "status")
|
||||
else "unknown",
|
||||
|
||||
@@ -68,7 +68,6 @@ License: MIT
|
||||
from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union, cast
|
||||
import concurrent.futures
|
||||
import inspect
|
||||
import uuid
|
||||
|
||||
import numpy as np
|
||||
|
||||
@@ -483,11 +482,7 @@ class VectorStore:
|
||||
doc_meta = doc.metadata
|
||||
elif isinstance(doc, dict):
|
||||
doc_meta = doc.get("metadata", {})
|
||||
elif isinstance(doc, str):
|
||||
# Plain-text documents: keep the text itself in the
|
||||
# payload, otherwise it is silently dropped.
|
||||
doc_meta = {"document": doc}
|
||||
|
||||
|
||||
final_metadata[i].update(doc_meta)
|
||||
|
||||
return self.store_vectors(vectors, metadata=final_metadata, **options)
|
||||
@@ -530,16 +525,6 @@ class VectorStore:
|
||||
if supports_metadata:
|
||||
return self._backend_store.add_vectors(vectors, metadata=metadata, **options)
|
||||
return self._backend_store.add_vectors(vectors, **options)
|
||||
elif hasattr(self._backend_store, 'insert_vectors'):
|
||||
# QdrantStore: insert_vectors(vectors, ids, payloads=None)
|
||||
# upserts the points but returns the client's status dict,
|
||||
# while this facade promises callers the stored vector IDs
|
||||
# (decision storage indexes the result at position 0).
|
||||
# metadata entries already carry the source document (folded
|
||||
# in by store()), so they map directly to Qdrant payloads.
|
||||
ids = options.pop('ids', None) or [str(uuid.uuid4()) for _ in range(len(vectors))]
|
||||
self._backend_store.insert_vectors(vectors, ids, payloads=metadata, **options)
|
||||
return ids
|
||||
else:
|
||||
raise NotImplementedError(f"Backend store {type(self._backend_store).__name__} does not have add or add_vectors method")
|
||||
|
||||
|
||||
@@ -85,7 +85,7 @@ class AnalyticsVisualizer:
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for analytics visualization. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
if np is None:
|
||||
raise ProcessingError(
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
"""
|
||||
Embedding Visualizer Module
|
||||
|
||||
This module provides comprehensive visualization capabilities for vector
|
||||
embeddings in the Semantica framework, including 2D/3D dimensionality
|
||||
reduction projections, similarity heatmaps, clustering visualizations,
|
||||
multi-modal comparisons, and quality metrics analysis.
|
||||
This module provides comprehensive visualization capabilities for vector embeddings in the
|
||||
Semantica framework, including 2D/3D dimensionality reduction projections, similarity heatmaps,
|
||||
clustering visualizations, multi-modal comparisons, and quality metrics analysis.
|
||||
|
||||
Key Features:
|
||||
- 2D and 3D dimensionality reduction (UMAP, t-SNE, PCA)
|
||||
@@ -32,8 +31,9 @@ License: MIT
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
@@ -45,12 +45,8 @@ except (ImportError, OSError):
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
try:
|
||||
from sklearn.decomposition import PCA
|
||||
from sklearn.manifold import TSNE
|
||||
except (ImportError, OSError):
|
||||
PCA = None
|
||||
TSNE = None
|
||||
from sklearn.decomposition import PCA
|
||||
from sklearn.manifold import TSNE
|
||||
|
||||
try:
|
||||
import umap
|
||||
@@ -61,7 +57,7 @@ from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .utils.color_schemes import ColorPalette, ColorScheme
|
||||
from .utils.export_formats import export_plotly_figure
|
||||
from .utils.export_formats import export_matplotlib_figure, export_plotly_figure
|
||||
|
||||
|
||||
class EmbeddingVisualizer:
|
||||
@@ -98,17 +94,12 @@ class EmbeddingVisualizer:
|
||||
self.color_scheme = ColorScheme.DEFAULT
|
||||
self.point_size = config.get("point_size", 5)
|
||||
|
||||
def _check_dependencies(self, require_sklearn: bool = False):
|
||||
def _check_dependencies(self):
|
||||
"""Check if dependencies are available."""
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for embedding visualization. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
)
|
||||
if require_sklearn and (PCA is None or TSNE is None):
|
||||
raise ProcessingError(
|
||||
"scikit-learn is required for dimensionality reduction. "
|
||||
"Reinstall scikit-learn or install dependencies."
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_2d_projection(
|
||||
@@ -159,12 +150,10 @@ class EmbeddingVisualizer:
|
||||
|
||||
try:
|
||||
self.logger.info(f"Visualizing 2D projection using {method}")
|
||||
|
||||
|
||||
# Step 2: Data Analysis
|
||||
n_samples, n_features = embeddings.shape
|
||||
self.logger.info(
|
||||
f"Embedding Analysis: {n_samples} samples, {n_features} dimensions"
|
||||
)
|
||||
self.logger.info(f"Embedding Analysis: {n_samples} samples, {n_features} dimensions")
|
||||
|
||||
if embeddings.shape[1] <= 2:
|
||||
# Already 2D or less, use directly
|
||||
@@ -176,32 +165,28 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message=f"Reducing dimensions using {method}..."
|
||||
)
|
||||
dim_options = dict(options)
|
||||
n_comp = dim_options.pop("n_components", 2)
|
||||
projected = self._reduce_dimensions(
|
||||
embeddings, method=method, n_components=n_comp, **dim_options
|
||||
embeddings, method=method, n_components=2, **options
|
||||
)
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Generating visualization..."
|
||||
)
|
||||
result = self._visualize_2d_plotly(
|
||||
projected,
|
||||
labels,
|
||||
output,
|
||||
file_path,
|
||||
projected,
|
||||
labels,
|
||||
output,
|
||||
file_path,
|
||||
color_by=color_by,
|
||||
size_by=size_by,
|
||||
hover_data=hover_data,
|
||||
**options,
|
||||
**options
|
||||
)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=(
|
||||
f"2D projection visualization generated: {len(projected)} points"
|
||||
),
|
||||
message=f"2D projection visualization generated: {len(projected)} points",
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
@@ -252,10 +237,8 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message=f"Reducing dimensions using {method}..."
|
||||
)
|
||||
dim_options = dict(options)
|
||||
n_comp = dim_options.pop("n_components", 3)
|
||||
projected = self._reduce_dimensions(
|
||||
embeddings, method=method, n_components=n_comp, **dim_options
|
||||
embeddings, method=method, n_components=3, **options
|
||||
)
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
@@ -268,9 +251,7 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=(
|
||||
f"3D projection visualization generated: {len(projected)} points"
|
||||
),
|
||||
message=f"3D projection visualization generated: {len(projected)} points",
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
@@ -361,10 +342,7 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=(
|
||||
f"Similarity heatmap generated: "
|
||||
f"{len(embeddings)}x{len(embeddings)} matrix"
|
||||
),
|
||||
message=f"Similarity heatmap generated: {len(embeddings)}x{len(embeddings)} matrix",
|
||||
)
|
||||
return fig
|
||||
elif file_path:
|
||||
@@ -426,10 +404,8 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message=f"Reducing dimensions using {method}..."
|
||||
)
|
||||
dim_options = dict(options)
|
||||
n_comp = dim_options.pop("n_components", 2)
|
||||
projected = self._reduce_dimensions(
|
||||
embeddings, method=method, n_components=n_comp, **dim_options
|
||||
embeddings, method=method, n_components=2, **options
|
||||
)
|
||||
|
||||
num_clusters = len(set(cluster_labels))
|
||||
@@ -469,10 +445,7 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=(
|
||||
f"Clustering visualization generated: "
|
||||
f"{num_clusters} clusters, {len(embeddings)} points"
|
||||
),
|
||||
message=f"Clustering visualization generated: {num_clusters} clusters, {len(embeddings)} points",
|
||||
)
|
||||
return fig
|
||||
elif file_path:
|
||||
@@ -568,13 +541,8 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message=f"Reducing dimensions using {method}..."
|
||||
)
|
||||
dim_options = dict(options)
|
||||
n_comp = dim_options.pop("n_components", 2)
|
||||
projected = self._reduce_dimensions(
|
||||
combined_embeddings,
|
||||
method=method,
|
||||
n_components=n_comp,
|
||||
**dim_options,
|
||||
combined_embeddings, method=method, n_components=2, **options
|
||||
)
|
||||
|
||||
# Color by type
|
||||
@@ -611,10 +579,7 @@ class EmbeddingVisualizer:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=(
|
||||
f"Multi-modal comparison generated: "
|
||||
f"{len(combined_embeddings)} embeddings"
|
||||
),
|
||||
message=f"Multi-modal comparison generated: {len(combined_embeddings)} embeddings",
|
||||
)
|
||||
return fig
|
||||
elif file_path:
|
||||
@@ -633,6 +598,8 @@ class EmbeddingVisualizer:
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
|
||||
def _reduce_dimensions(
|
||||
self,
|
||||
embeddings: np.ndarray,
|
||||
@@ -641,56 +608,39 @@ class EmbeddingVisualizer:
|
||||
**options,
|
||||
) -> np.ndarray:
|
||||
"""Reduce embedding dimensions using specified method."""
|
||||
opts = dict(options)
|
||||
opts.pop("n_components", None)
|
||||
|
||||
if method == "pca":
|
||||
if PCA is None:
|
||||
raise ProcessingError(
|
||||
"scikit-learn is required for dimensionality reduction. "
|
||||
"Reinstall scikit-learn or install dependencies."
|
||||
)
|
||||
pca = PCA(n_components=n_components, **opts)
|
||||
pca = PCA(n_components=n_components, **options)
|
||||
return pca.fit_transform(embeddings)
|
||||
|
||||
elif method == "tsne":
|
||||
if TSNE is None:
|
||||
raise ProcessingError(
|
||||
"scikit-learn is required for dimensionality reduction. "
|
||||
"Reinstall scikit-learn or install dependencies."
|
||||
)
|
||||
perplexity = opts.pop("perplexity", min(30, len(embeddings) - 1))
|
||||
random_state = opts.pop("random_state", 42)
|
||||
perplexity = options.get("perplexity", min(30, len(embeddings) - 1))
|
||||
tsne = TSNE(
|
||||
n_components=n_components,
|
||||
perplexity=perplexity,
|
||||
random_state=random_state,
|
||||
**opts,
|
||||
random_state=42,
|
||||
**options,
|
||||
)
|
||||
return tsne.fit_transform(embeddings)
|
||||
|
||||
elif method == "umap":
|
||||
if umap is not None:
|
||||
n_neighbors = opts.pop("n_neighbors", min(15, len(embeddings) - 1))
|
||||
n_neighbors = options.get("n_neighbors", min(15, len(embeddings) - 1))
|
||||
reducer = umap.UMAP(
|
||||
n_components=n_components, n_neighbors=n_neighbors, **opts
|
||||
n_components=n_components, n_neighbors=n_neighbors, **options
|
||||
)
|
||||
return reducer.fit_transform(embeddings)
|
||||
else:
|
||||
raise ProcessingError(
|
||||
"UMAP is required for UMAP dimensionality reduction. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
# Fallback to PCA if UMAP not available
|
||||
self.logger.warning(
|
||||
"UMAP not available, using PCA. Install with: pip install umap-learn"
|
||||
)
|
||||
pca = PCA(n_components=n_components)
|
||||
return pca.fit_transform(embeddings)
|
||||
|
||||
else:
|
||||
if PCA is None:
|
||||
raise ProcessingError(
|
||||
"scikit-learn is required for dimensionality reduction. "
|
||||
"Reinstall scikit-learn or install dependencies."
|
||||
)
|
||||
# Fallback to PCA
|
||||
self.logger.warning(f"Method {method} not available, using PCA")
|
||||
pca = PCA(n_components=n_components, **opts)
|
||||
pca = PCA(n_components=n_components)
|
||||
return pca.fit_transform(embeddings)
|
||||
|
||||
def _visualize_2d_plotly(
|
||||
|
||||
@@ -124,7 +124,7 @@ class KGVisualizer:
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for KG visualization. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def _convert_knowledge_graph(self, kg: Any) -> Dict[str, Any]:
|
||||
|
||||
@@ -35,14 +35,8 @@ License: MIT
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
try:
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import FancyBboxPatch
|
||||
except (ImportError, OSError):
|
||||
mpatches = None
|
||||
plt = None
|
||||
FancyBboxPatch = None
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
try:
|
||||
import plotly.express as px
|
||||
@@ -53,6 +47,8 @@ except (ImportError, OSError):
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
from matplotlib.patches import FancyBboxPatch
|
||||
|
||||
try:
|
||||
import graphviz
|
||||
except (ImportError, OSError):
|
||||
@@ -110,13 +106,13 @@ class OntologyVisualizer:
|
||||
if graphviz is None:
|
||||
raise ProcessingError(
|
||||
"Graphviz is required for DOT export. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
"Install with: pip install graphviz"
|
||||
)
|
||||
else:
|
||||
if go is None:
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for ontology visualization. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_hierarchy(
|
||||
@@ -896,7 +892,7 @@ class OntologyVisualizer:
|
||||
"""Create Graphviz hierarchy visualization."""
|
||||
if graphviz is None:
|
||||
raise ProcessingError(
|
||||
"Graphviz not available. Install with: pip install 'semantica[viz]'"
|
||||
"Graphviz not available. Install with: pip install graphviz"
|
||||
)
|
||||
|
||||
dot = graphviz.Digraph(comment="Ontology Hierarchy")
|
||||
|
||||
@@ -74,7 +74,7 @@ class SemanticNetworkVisualizer:
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for semantic network visualization. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_network(
|
||||
|
||||
@@ -83,7 +83,7 @@ class TemporalVisualizer:
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for temporal visualization. "
|
||||
"Install with: pip install 'semantica[viz]'"
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_temporal_dashboard(
|
||||
|
||||
@@ -51,27 +51,6 @@ _INTERNAL_ERROR = -32603
|
||||
_TOOL_INDEX: dict[str, dict] = {t["name"]: t for t in TOOL_DEFINITIONS}
|
||||
|
||||
|
||||
class UnknownToolError(Exception):
|
||||
"""Raised by :func:`call_tool` when the tool name is not in the catalog.
|
||||
|
||||
A dedicated type (rather than ``KeyError``) so callers can distinguish
|
||||
a bad tool name from a ``KeyError`` raised inside a handler indexing a
|
||||
required argument (e.g. ``args["category"]``).
|
||||
"""
|
||||
|
||||
|
||||
def call_tool(name: str, arguments: dict) -> dict:
|
||||
"""Invoke a tool in-process by name and return its raw result dict.
|
||||
|
||||
Shared by the JSON-RPC ``tools/call`` handler and ``semantica mcp call``
|
||||
(issue #1355), so both expose exactly the same tool set.
|
||||
"""
|
||||
tool = _TOOL_INDEX.get(name)
|
||||
if tool is None:
|
||||
raise UnknownToolError(f"Unknown tool: {name}")
|
||||
return tool["_handler"](arguments)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request handlers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -106,10 +85,12 @@ def _handle_tools_call(req_id: Any, params: dict) -> dict:
|
||||
name = params.get("name", "")
|
||||
args = params.get("arguments", {}) or {}
|
||||
|
||||
tool = _TOOL_INDEX.get(name)
|
||||
if tool is None:
|
||||
return _err(req_id, _METHOD_NOT_FOUND, f"Unknown tool: {name}")
|
||||
|
||||
try:
|
||||
result = call_tool(name, args)
|
||||
except UnknownToolError as exc:
|
||||
return _err(req_id, _METHOD_NOT_FOUND, str(exc))
|
||||
result = tool["_handler"](args)
|
||||
except Exception as exc:
|
||||
log.exception("Tool %s raised an exception", name)
|
||||
# The exception's class name (e.g. "ValidationError", "TimeoutError")
|
||||
|
||||
@@ -58,69 +58,21 @@ def handle_export_graph(args: dict) -> dict:
|
||||
return {"format": "csv", "data": header + "\n" + "\n".join(rows)}
|
||||
|
||||
if fmt in ("graphml",):
|
||||
# GraphMLExporter does not exist. GraphExporter is the correct
|
||||
# class; it writes to a file and returns None, so we need a
|
||||
# temporary directory for safe cleanup regardless of success or
|
||||
# failure.
|
||||
#
|
||||
# export_knowledge_graph() is the required entry point because
|
||||
# to_kg_dict() returns {"entities": [...], "relationships": [...]}
|
||||
# while export() / _export_graphml() only consumes {"nodes": [...],
|
||||
# "edges": [...]}. Calling export() directly therefore produces a
|
||||
# structurally-valid but empty GraphML document (zero nodes, zero
|
||||
# edges). export_knowledge_graph() runs _convert_kg_to_graph()
|
||||
# first, which maps entities→nodes and relationships→edges.
|
||||
try:
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from semantica.export import GraphExporter
|
||||
kg_dict = graph.to_kg_dict()
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_path = Path(tmpdir) / "export.graphml"
|
||||
exporter = GraphExporter(format="graphml")
|
||||
exporter.export_knowledge_graph(kg_dict, file_path=tmp_path)
|
||||
data = tmp_path.read_text(encoding="utf-8")
|
||||
from semantica.export import GraphMLExporter
|
||||
exporter = GraphMLExporter()
|
||||
data = exporter.export(graph)
|
||||
return {"format": "graphml", "data": data}
|
||||
except ImportError as exc:
|
||||
log.warning("GraphML export unavailable: %s", exc)
|
||||
return {"error": f"GraphML export unavailable: {exc}"}
|
||||
except Exception as exc:
|
||||
log.exception("GraphML export failed")
|
||||
return {"error": f"GraphML export failed: {exc}"}
|
||||
|
||||
if fmt in ("parquet",):
|
||||
# ParquetExporter.export() writes to file(s) and returns None;
|
||||
# passing the ContextGraph object or using the return value as
|
||||
# data were both wrong. export_knowledge_graph() is the correct
|
||||
# entry point for a full KG: it splits into _entities.parquet and
|
||||
# _relationships.parquet under a provided base path. We read both
|
||||
# files and return them base64-encoded so the MCP client can
|
||||
# reconstruct them without a shared filesystem.
|
||||
try:
|
||||
import base64
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
try:
|
||||
from semantica.export import ParquetExporter
|
||||
exporter = ParquetExporter()
|
||||
except ImportError as exc:
|
||||
log.warning("Parquet export unavailable: %s", exc)
|
||||
return {"error": f"Parquet export unavailable (pyarrow not installed): {exc}"}
|
||||
kg_dict = graph.to_kg_dict()
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
base_path = Path(tmpdir) / "kg"
|
||||
exporter.export_knowledge_graph(kg_dict, base_path)
|
||||
# Collect the produced files and return them as base64
|
||||
files = {}
|
||||
for p in sorted(Path(tmpdir).glob("*.parquet")):
|
||||
files[p.name] = base64.b64encode(p.read_bytes()).decode("ascii")
|
||||
if not files:
|
||||
return {"error": "Parquet export produced no files"}
|
||||
return {"format": "parquet", "data": files,
|
||||
"encoding": "base64",
|
||||
"note": "Each value is a base64-encoded Parquet file."}
|
||||
from semantica.export import ParquetExporter
|
||||
exporter = ParquetExporter()
|
||||
data = exporter.export(graph, include_metadata)
|
||||
return {"format": "parquet", "data": str(data)}
|
||||
except Exception as exc:
|
||||
log.exception("Parquet export failed")
|
||||
return {"error": f"Parquet export failed: {exc}"}
|
||||
|
||||
# RDF formats
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
"""Regression tests for issue #1531.
|
||||
|
||||
``GET /api/decisions/causal-distance`` was unreachable because the dynamic
|
||||
route ``GET /api/decisions/{decision_id}`` is registered before the static
|
||||
``/causal-distance`` route in ``semantica/explorer/routes/decisions.py``.
|
||||
Starlette matches routes in definition order, so a request to
|
||||
``causal-distance`` was bound as ``decision_id="causal-distance"`` and
|
||||
returned ``404 Decision 'causal-distance' not found``.
|
||||
|
||||
Each test below fails while the static route sits after ``/{decision_id}``
|
||||
and passes once it is moved above it.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
# fastapi ships in the optional `explorer` extra, not in `dev`, so this module
|
||||
# must skip rather than fail collection when it is absent.
|
||||
pytest.importorskip("fastapi")
|
||||
|
||||
from fastapi.testclient import TestClient # noqa: E402
|
||||
|
||||
from semantica.context.context_graph import ContextGraph # noqa: E402
|
||||
from semantica.explorer.app import create_app # noqa: E402
|
||||
from semantica.explorer.session import GraphSession # noqa: E402
|
||||
|
||||
|
||||
def _client() -> TestClient:
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
return TestClient(create_app(session=GraphSession(graph)))
|
||||
|
||||
|
||||
def test_causal_distance_route_is_not_shadowed_by_decision_id():
|
||||
"""GET /api/decisions/causal-distance must reach the distance handler.
|
||||
|
||||
Fails before the fix with: 404 {"detail": "Decision 'causal-distance' not found"}.
|
||||
After the fix the analyzer returns an unreachable-distance report (200).
|
||||
"""
|
||||
with _client() as client:
|
||||
response = client.get(
|
||||
"/api/decisions/causal-distance", params={"source": "x", "target": "y"}
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
payload = response.json()
|
||||
assert payload["source_id"] == "x"
|
||||
assert payload["target_id"] == "y"
|
||||
|
||||
|
||||
def test_causal_distance_between_linked_decisions():
|
||||
"""Two causally linked decisions must report a 1-hop path."""
|
||||
graph = ContextGraph(advanced_analytics=False)
|
||||
first = graph.record_decision(
|
||||
category="credit_application",
|
||||
scenario="Personal loan review",
|
||||
reasoning="Income meets threshold",
|
||||
outcome="proceed_to_underwriting",
|
||||
confidence=0.88,
|
||||
)
|
||||
second = graph.record_decision(
|
||||
category="loan_underwriting",
|
||||
scenario="Underwriting review",
|
||||
reasoning="DTI within policy",
|
||||
outcome="approved",
|
||||
confidence=0.94,
|
||||
)
|
||||
graph.add_causal_relationship(first, second, relationship_type="CAUSED")
|
||||
|
||||
with TestClient(create_app(session=GraphSession(graph))) as client:
|
||||
response = client.get(
|
||||
"/api/decisions/causal-distance",
|
||||
params={"source": first, "target": second},
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["causal_hop_count"] == 1
|
||||
|
||||
|
||||
def test_unknown_decision_id_still_404s():
|
||||
"""Moving the static route must not change param-route 404 behavior."""
|
||||
with _client() as client:
|
||||
response = client.get("/api/decisions/does-not-exist")
|
||||
|
||||
assert response.status_code == 404
|
||||
assert response.json() == {"detail": "Decision 'does-not-exist' not found"}
|
||||
@@ -141,134 +141,3 @@ else:
|
||||
assert "ConfigurationError" in result.stdout
|
||||
assert "Parquet ingestion" in result.stdout
|
||||
assert "pyarrow" in result.stdout
|
||||
|
||||
|
||||
def test_repo_ingestor_probe_fails_without_gitpython() -> None:
|
||||
result = _run_python_with_blocked_modules(
|
||||
"""
|
||||
try:
|
||||
from semantica.ingest import RepoIngestor
|
||||
has_git = True
|
||||
except ImportError:
|
||||
has_git = False
|
||||
|
||||
assert not has_git, "Expected RepoIngestor import to fail without GitPython"
|
||||
print("RepoIngestor probe passed")
|
||||
""",
|
||||
("git",),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "RepoIngestor probe passed" in result.stdout
|
||||
|
||||
|
||||
def test_xml_ingestor_probe_fails_without_lxml() -> None:
|
||||
result = _run_python_with_blocked_modules(
|
||||
"""
|
||||
try:
|
||||
from semantica.ingest import XMLIngestor
|
||||
has_lxml = True
|
||||
except ImportError:
|
||||
has_lxml = False
|
||||
|
||||
assert not has_lxml, "Expected XMLIngestor import to fail without lxml"
|
||||
print("XMLIngestor probe passed")
|
||||
""",
|
||||
("lxml",),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "XMLIngestor probe passed" in result.stdout
|
||||
|
||||
|
||||
def test_xml_ingestion_reports_missing_lxml_when_used() -> None:
|
||||
result = _run_python_with_blocked_modules(
|
||||
"""
|
||||
from semantica.ingest import ingest_xml
|
||||
|
||||
try:
|
||||
ingest_xml("catalog.xml")
|
||||
except Exception as exc:
|
||||
print(type(exc).__name__, exc)
|
||||
else:
|
||||
raise SystemExit("expected XML ingestion to fail without lxml")
|
||||
""",
|
||||
("lxml",),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "ConfigurationError" in result.stdout
|
||||
assert "XML ingestion" in result.stdout
|
||||
assert "lxml" in result.stdout
|
||||
|
||||
|
||||
def test_sibling_imports_succeed_without_optional_backends() -> None:
|
||||
result = _run_python_with_blocked_modules(
|
||||
"""
|
||||
from semantica.ingest import (
|
||||
CodeExtractor,
|
||||
CodeFile,
|
||||
CommitInfo,
|
||||
GitAnalyzer,
|
||||
XMLIngestionData,
|
||||
SalesforceData,
|
||||
)
|
||||
print(
|
||||
CodeExtractor.__name__,
|
||||
CodeFile.__name__,
|
||||
CommitInfo.__name__,
|
||||
GitAnalyzer.__name__,
|
||||
XMLIngestionData.__name__,
|
||||
SalesforceData.__name__,
|
||||
)
|
||||
""",
|
||||
("git", "lxml", "simple_salesforce"),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert (
|
||||
"CodeExtractor CodeFile CommitInfo GitAnalyzer XMLIngestionData SalesforceData"
|
||||
in result.stdout
|
||||
)
|
||||
|
||||
|
||||
def test_salesforce_ingestor_probe_fails_without_simple_salesforce() -> None:
|
||||
result = _run_python_with_blocked_modules(
|
||||
"""
|
||||
try:
|
||||
from semantica.ingest import SalesforceIngestor
|
||||
has_salesforce = True
|
||||
except ImportError:
|
||||
has_salesforce = False
|
||||
|
||||
assert not has_salesforce, (
|
||||
"Expected SalesforceIngestor import to fail without simple-salesforce"
|
||||
)
|
||||
print("SalesforceIngestor probe passed")
|
||||
""",
|
||||
("simple_salesforce",),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "SalesforceIngestor probe passed" in result.stdout
|
||||
|
||||
|
||||
def test_salesforce_ingestion_reports_missing_dep_when_used() -> None:
|
||||
result = _run_python_with_blocked_modules(
|
||||
"""
|
||||
from semantica.ingest import ingest_salesforce
|
||||
|
||||
try:
|
||||
ingest_salesforce()
|
||||
except Exception as exc:
|
||||
print(type(exc).__name__, exc)
|
||||
else:
|
||||
raise SystemExit("expected Salesforce ingestion to fail without simple-salesforce")
|
||||
""",
|
||||
("simple_salesforce",),
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "ConfigurationError" in result.stdout
|
||||
assert "Salesforce ingestion" in result.stdout
|
||||
assert "simple-salesforce" in result.stdout
|
||||
|
||||
@@ -1,282 +0,0 @@
|
||||
"""Unit tests for ExtractionSchema and SchemaValidator (schema-guided validation)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from semantica.semantic_extract import (
|
||||
Entity,
|
||||
ExtractionSchema,
|
||||
ExtractionValidator,
|
||||
Relation,
|
||||
SchemaValidator,
|
||||
ValidationResult,
|
||||
)
|
||||
|
||||
ONTOLOGY = {
|
||||
"classes": [{"name": "Person"}, {"name": "Organization"}, {"label": "City"}],
|
||||
"properties": [
|
||||
{"name": "worksAt", "domain": ["Person"], "range": ["Organization"]},
|
||||
{"name": "locatedIn", "domain": "Organization", "range": "City"},
|
||||
{"name": "knows"}, # unconstrained domain / range
|
||||
],
|
||||
}
|
||||
|
||||
TTL = """
|
||||
@prefix : <https://example.org/> .
|
||||
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
|
||||
:Person a owl:Class .
|
||||
:Organization a owl:Class .
|
||||
:worksAt a owl:ObjectProperty ;
|
||||
rdfs:domain :Person ;
|
||||
rdfs:range :Organization .
|
||||
"""
|
||||
|
||||
|
||||
def _schema() -> ExtractionSchema:
|
||||
return ExtractionSchema.from_ontology(ONTOLOGY)
|
||||
|
||||
|
||||
def _person() -> Entity:
|
||||
return Entity(text="Alice", label="Person", start_char=0, end_char=5)
|
||||
|
||||
|
||||
def _org() -> Entity:
|
||||
return Entity(text="Acme", label="Organization", start_char=0, end_char=4)
|
||||
|
||||
|
||||
def _city() -> Entity:
|
||||
return Entity(text="Paris", label="City", start_char=0, end_char=5)
|
||||
|
||||
|
||||
def _product() -> Entity:
|
||||
return Entity(text="Widget", label="Product", start_char=0, end_char=6)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# ExtractionSchema
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_from_ontology_parses_concepts_and_predicates() -> None:
|
||||
schema = _schema()
|
||||
assert schema.concepts == frozenset({"Person", "Organization", "City"})
|
||||
assert set(schema.predicates) == {"worksAt", "locatedIn", "knows"}
|
||||
assert schema.predicates["worksAt"].domain == frozenset({"Person"})
|
||||
assert schema.predicates["worksAt"].range == frozenset({"Organization"})
|
||||
# Missing domain / range means unconstrained.
|
||||
assert schema.predicates["knows"].domain == frozenset()
|
||||
assert schema.predicates["knows"].range == frozenset()
|
||||
|
||||
|
||||
def test_allows_relation_respects_domain_range() -> None:
|
||||
schema = _schema()
|
||||
assert schema.allows_relation("Person", "worksAt", "Organization")
|
||||
assert not schema.allows_relation("Person", "worksAt", "City") # range violation
|
||||
assert not schema.allows_relation("Person", "unknownPred", "Organization")
|
||||
assert not schema.allows_relation("Product", "worksAt", "Organization") # off-vocab
|
||||
# Unconstrained predicate accepts any known concepts.
|
||||
assert schema.allows_relation("Person", "knows", "City")
|
||||
|
||||
|
||||
def test_from_owl_parses_turtle() -> None:
|
||||
schema = ExtractionSchema.from_owl(TTL, format="turtle")
|
||||
assert {"Person", "Organization"} <= schema.concepts
|
||||
assert schema.predicates["worksAt"].domain == frozenset({"Person"})
|
||||
assert schema.predicates["worksAt"].range == frozenset({"Organization"})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# SchemaValidator — entities
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_validate_entities_all_conforming() -> None:
|
||||
result = SchemaValidator(_schema()).validate_entities([_person(), _org(), _city()])
|
||||
assert isinstance(result, ValidationResult)
|
||||
assert result.valid
|
||||
assert result.score == 1.0
|
||||
assert result.metrics["out_of_vocabulary"] == 0
|
||||
|
||||
|
||||
def test_validate_entities_flags_out_of_vocabulary() -> None:
|
||||
result = SchemaValidator(_schema()).validate_entities([_person(), _product()])
|
||||
assert not result.valid
|
||||
assert result.metrics["out_of_vocabulary"] == 1
|
||||
assert result.metrics["unknown_labels"] == ["Product"]
|
||||
assert result.score == 0.5
|
||||
assert result.errors
|
||||
|
||||
|
||||
def test_validate_entities_empty_is_vacuously_valid() -> None:
|
||||
result = SchemaValidator(_schema()).validate_entities([])
|
||||
assert result.valid
|
||||
assert result.score == 1.0
|
||||
|
||||
|
||||
def test_validate_entities_batch_returns_list_with_index() -> None:
|
||||
results = SchemaValidator(_schema()).validate_entities([[_person()], [_product()]])
|
||||
assert isinstance(results, list)
|
||||
assert len(results) == 2
|
||||
assert results[0].valid
|
||||
assert not results[1].valid
|
||||
assert results[0].metadata["batch_index"] == 0
|
||||
assert results[1].metadata["batch_index"] == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# SchemaValidator — relations
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_validate_relations_conforming() -> None:
|
||||
rels = [
|
||||
Relation(subject=_person(), predicate="worksAt", object=_org()),
|
||||
Relation(subject=_person(), predicate="knows", object=_city()),
|
||||
]
|
||||
result = SchemaValidator(_schema()).validate_relations(rels)
|
||||
assert result.valid
|
||||
assert result.score == 1.0
|
||||
|
||||
|
||||
def test_validate_relations_flags_unknown_predicate_and_domain_range() -> None:
|
||||
rels = [
|
||||
Relation(subject=_person(), predicate="worksAt", object=_org()), # ok
|
||||
Relation(subject=_person(), predicate="founded", object=_org()), # unknown pred
|
||||
Relation(subject=_person(), predicate="worksAt", object=_city()), # range viol
|
||||
]
|
||||
result = SchemaValidator(_schema()).validate_relations(rels)
|
||||
assert not result.valid
|
||||
assert result.metrics["unknown_predicate"] == 1
|
||||
assert result.metrics["domain_range_violation"] == 1
|
||||
assert result.metrics["conforming"] == 1
|
||||
assert result.score == 1 / 3
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Filtering
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_filter_by_schema_drops_off_vocabulary() -> None:
|
||||
kept = SchemaValidator(_schema()).filter_by_schema([_person(), _org(), _product()])
|
||||
assert [e.label for e in kept] == ["Person", "Organization"]
|
||||
|
||||
|
||||
def test_filter_relations_by_schema_keeps_only_conforming() -> None:
|
||||
rels = [
|
||||
Relation(subject=_person(), predicate="worksAt", object=_org()), # keep
|
||||
Relation(subject=_person(), predicate="founded", object=_org()), # drop
|
||||
Relation(subject=_person(), predicate="worksAt", object=_city()), # drop
|
||||
Relation(subject=_person(), predicate="knows", object=_city()), # keep
|
||||
]
|
||||
kept = SchemaValidator(_schema()).filter_relations_by_schema(rels)
|
||||
assert [r.predicate for r in kept] == ["worksAt", "knows"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Composition with the confidence-based ExtractionValidator (orthogonal axis)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_composes_with_extraction_validator_same_shape() -> None:
|
||||
entities = [_person(), _org()]
|
||||
confidence = ExtractionValidator().validate_entities(entities)
|
||||
conformance = SchemaValidator(_schema()).validate_entities(entities)
|
||||
assert isinstance(confidence, ValidationResult)
|
||||
assert isinstance(conformance, ValidationResult)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Robustness fixes surfaced in review
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_owl_thing_domain_range_is_unconstrained() -> None:
|
||||
# OntologyGenerator emits owl:Thing when it cannot resolve endpoint types;
|
||||
# it must behave as "any concept", not a literal {"Thing"} constraint.
|
||||
ont = {
|
||||
"classes": [{"name": "Person"}, {"name": "Organization"}],
|
||||
"properties": [
|
||||
{"name": "relatedTo", "domain": ["owl:Thing"], "range": ["owl:Thing"]}
|
||||
],
|
||||
}
|
||||
schema = ExtractionSchema.from_ontology(ont)
|
||||
assert schema.predicates["relatedTo"].domain == frozenset()
|
||||
assert schema.predicates["relatedTo"].range == frozenset()
|
||||
assert schema.allows_relation("Person", "relatedTo", "Organization")
|
||||
|
||||
|
||||
def test_from_owl_prefers_rdfs_label_and_supports_rdfs_class() -> None:
|
||||
ttl = """
|
||||
@prefix : <https://example.org/> .
|
||||
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
|
||||
:Cls1 a owl:Class ; rdfs:label "Person" .
|
||||
:Org a rdfs:Class .
|
||||
:worksAt a owl:ObjectProperty ;
|
||||
rdfs:domain :Cls1 ;
|
||||
rdfs:range :Org .
|
||||
"""
|
||||
schema = ExtractionSchema.from_owl(ttl, format="turtle")
|
||||
# rdfs:label wins over the URI suffix "Cls1"
|
||||
assert "Person" in schema.concepts
|
||||
assert "Cls1" not in schema.concepts
|
||||
# rdfs:Class is picked up too
|
||||
assert "Org" in schema.concepts
|
||||
assert schema.predicates["worksAt"].domain == frozenset({"Person"})
|
||||
|
||||
|
||||
def test_validate_relations_handles_malformed_without_crashing() -> None:
|
||||
# A relation missing an endpoint must be reported, not raise AttributeError.
|
||||
good = Relation(subject=_person(), predicate="worksAt", object=_org())
|
||||
bad = Relation(subject=_person(), predicate="worksAt", object=None) # type: ignore[arg-type]
|
||||
result = SchemaValidator(_schema()).validate_relations([good, bad])
|
||||
assert isinstance(result, ValidationResult)
|
||||
assert not result.valid
|
||||
assert result.metrics["malformed"] == 1
|
||||
assert result.metrics["conforming"] == 1
|
||||
# Filtering also drops the malformed one instead of crashing.
|
||||
kept = SchemaValidator(_schema()).filter_relations_by_schema([good, bad])
|
||||
assert kept == [good]
|
||||
|
||||
|
||||
def test_from_ontology_folds_endpoint_types_like_from_owl() -> None:
|
||||
# pkupt's case: an endpoint type (Org) that didn't clear the class-frequency
|
||||
# gate is absent from "classes" but referenced in a property's range. Both
|
||||
# constructors must agree that (Person, worksFor, Org) is allowed.
|
||||
ont = {
|
||||
"classes": [{"name": "Person"}],
|
||||
"properties": [{"name": "worksFor", "domain": ["Person"], "range": ["Org"]}],
|
||||
}
|
||||
dict_schema = ExtractionSchema.from_ontology(ont)
|
||||
assert {"Person", "Org"} <= dict_schema.concepts
|
||||
assert dict_schema.allows_relation("Person", "worksFor", "Org")
|
||||
|
||||
ttl = """
|
||||
@prefix : <https://example.org/> .
|
||||
@prefix owl: <http://www.w3.org/2002/07/owl#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
|
||||
:Person a owl:Class .
|
||||
:worksFor a owl:ObjectProperty ;
|
||||
rdfs:domain :Person ;
|
||||
rdfs:range :Org .
|
||||
"""
|
||||
owl_schema = ExtractionSchema.from_owl(ttl, format="turtle")
|
||||
assert owl_schema.allows_relation(
|
||||
"Person", "worksFor", "Org"
|
||||
) == dict_schema.allows_relation("Person", "worksFor", "Org")
|
||||
|
||||
|
||||
def test_from_ontology_accepts_ontologydata_like_object() -> None:
|
||||
# semantica.ingest.OntologyIngestor.ingest_ontology returns an OntologyData
|
||||
# whose ontology dict is held in `.data`; from_ontology should unwrap it
|
||||
# instead of raising AttributeError on `.get()`.
|
||||
from types import SimpleNamespace
|
||||
|
||||
wrapped = SimpleNamespace(data=ONTOLOGY)
|
||||
schema = ExtractionSchema.from_ontology(wrapped)
|
||||
assert schema.concepts == frozenset({"Person", "Organization", "City"})
|
||||
assert "worksAt" in schema.predicates
|
||||
@@ -18,40 +18,29 @@ from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# ── Mock optional heavyweight dependencies before any semantica import ──────
|
||||
_MOCKED_MODULES = [
|
||||
"spacy",
|
||||
"instructor",
|
||||
"openai",
|
||||
"groq",
|
||||
"sentence_transformers",
|
||||
"transformers",
|
||||
]
|
||||
_original_modules = {k: sys.modules.get(k) for k in _MOCKED_MODULES}
|
||||
|
||||
for k in _MOCKED_MODULES:
|
||||
sys.modules.setdefault(k, MagicMock())
|
||||
sys.modules.setdefault("spacy", MagicMock())
|
||||
sys.modules.setdefault("instructor", MagicMock())
|
||||
_openai_mock = MagicMock()
|
||||
sys.modules.setdefault("openai", _openai_mock)
|
||||
sys.modules.setdefault("groq", MagicMock())
|
||||
sys.modules.setdefault("sentence_transformers", MagicMock())
|
||||
sys.modules.setdefault("transformers", MagicMock())
|
||||
sys.modules.setdefault("torch", MagicMock())
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
|
||||
|
||||
from semantica.semantic_extract.methods import extract_relations_llm # noqa: E402
|
||||
|
||||
for _key, _original in _original_modules.items():
|
||||
if _original is None:
|
||||
sys.modules.pop(_key, None)
|
||||
else:
|
||||
sys.modules[_key] = _original
|
||||
|
||||
from semantica.semantic_extract.ner_extractor import Entity # noqa: E402
|
||||
from semantica.semantic_extract.schemas import ( # noqa: E402
|
||||
from semantica.semantic_extract.methods import extract_relations_llm
|
||||
from semantica.semantic_extract.ner_extractor import Entity
|
||||
from semantica.semantic_extract.schemas import (
|
||||
RelationsResponse,
|
||||
RelationsWithTemporalResponse,
|
||||
)
|
||||
from semantica.kg.temporal_normalizer import TemporalNormalizer # noqa: E402
|
||||
from semantica.utils.exceptions import TemporalAmbiguityWarning # noqa: E402
|
||||
from semantica.kg.temporal_normalizer import TemporalNormalizer
|
||||
from semantica.utils.exceptions import TemporalAmbiguityWarning
|
||||
|
||||
|
||||
# ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_entities():
|
||||
return [
|
||||
Entity(text="Apple", label="ORG", start_char=0, end_char=5),
|
||||
@@ -67,17 +56,15 @@ def _ref_date():
|
||||
# Part 1 – extract_relations_llm() temporal flag
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTemporalExtractionFlag(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
from semantica.semantic_extract.methods import _result_cache
|
||||
|
||||
_result_cache.clear()
|
||||
|
||||
@patch("semantica.semantic_extract.methods.create_provider")
|
||||
def test_extract_temporal_bounds_true_adds_four_fields(self, mock_create):
|
||||
"""With extract_temporal_bounds=True all four temporal keys appear."""
|
||||
"""With extract_temporal_bounds=True all four temporal keys appear in metadata."""
|
||||
mock_prov = MagicMock()
|
||||
mock_prov.is_available.return_value = True
|
||||
mock_prov.generate_typed.return_value = RelationsWithTemporalResponse(
|
||||
@@ -145,9 +132,7 @@ class TestTemporalExtractionFlag(unittest.TestCase):
|
||||
self.assertNotIn("temporal_source_text", meta)
|
||||
|
||||
@patch("semantica.semantic_extract.methods.create_provider")
|
||||
def test_no_temporal_signal_returns_zero_confidence_and_null_dates(
|
||||
self, mock_create
|
||||
):
|
||||
def test_no_temporal_signal_returns_zero_confidence_and_null_dates(self, mock_create):
|
||||
"""When LLM returns no temporal signal, confidence=0.0 and dates are null."""
|
||||
mock_prov = MagicMock()
|
||||
mock_prov.is_available.return_value = True
|
||||
@@ -212,12 +197,10 @@ class TestTemporalExtractionFlag(unittest.TestCase):
|
||||
|
||||
@patch("semantica.semantic_extract.methods.create_provider")
|
||||
def test_correct_schema_used_when_temporal_true(self, mock_create):
|
||||
"""generate_typed is called with RelationsWithTemporalResponse."""
|
||||
"""generate_typed is called with RelationsWithTemporalResponse when flag=True."""
|
||||
mock_prov = MagicMock()
|
||||
mock_prov.is_available.return_value = True
|
||||
mock_prov.generate_typed.return_value = RelationsWithTemporalResponse(
|
||||
relations=[]
|
||||
)
|
||||
mock_prov.generate_typed.return_value = RelationsWithTemporalResponse(relations=[])
|
||||
mock_create.return_value = mock_prov
|
||||
|
||||
extract_relations_llm(
|
||||
@@ -252,7 +235,6 @@ class TestTemporalExtractionFlag(unittest.TestCase):
|
||||
# Part 2 – TemporalNormalizer: relative dates
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTemporalNormalizerRelativeDates(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
@@ -331,13 +313,10 @@ class TestTemporalNormalizerRelativeDates(unittest.TestCase):
|
||||
# Part 3 – TemporalNormalizer: partial / structured dates
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTemporalNormalizerPartialDates(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tn = TemporalNormalizer(
|
||||
reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)
|
||||
)
|
||||
self.tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
|
||||
|
||||
def test_year_only(self):
|
||||
result = self.tn.normalize("2021")
|
||||
@@ -417,13 +396,10 @@ class TestTemporalNormalizerPartialDates(unittest.TestCase):
|
||||
# Part 4 – TemporalNormalizer: ambiguous formats
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTemporalNormalizerAmbiguity(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tn = TemporalNormalizer(
|
||||
reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)
|
||||
)
|
||||
self.tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
|
||||
|
||||
def test_ambiguous_slash_date_raises_warning_and_returns_none(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
@@ -456,19 +432,14 @@ class TestTemporalNormalizerAmbiguity(unittest.TestCase):
|
||||
# Part 5 – TemporalNormalizer: domain phrase map
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTemporalNormalizerDomainPhrases(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tn = TemporalNormalizer(
|
||||
reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)
|
||||
)
|
||||
self.tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
|
||||
|
||||
def _assert_recognized(self, phrase):
|
||||
result = self.tn.normalize_phrase(phrase)
|
||||
self.assertIsNotNone(
|
||||
result, f"Expected phrase {phrase!r} to be recognized but got None"
|
||||
)
|
||||
self.assertIsNotNone(result, f"Expected phrase {phrase!r} to be recognized but got None")
|
||||
return result
|
||||
|
||||
# General / Policy
|
||||
@@ -551,7 +522,6 @@ class TestTemporalNormalizerDomainPhrases(unittest.TestCase):
|
||||
# Part 6 – TemporalNormalizer: custom phrase map
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestTemporalNormalizerCustomPhraseMap(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
@@ -595,12 +565,10 @@ class TestTemporalNormalizerCustomPhraseMap(unittest.TestCase):
|
||||
# Part 7 – Full pipeline: extract → normalize → BiTemporalFact
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestFullPipelineTemporalToBiTemporal(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
from semantica.semantic_extract.methods import _result_cache
|
||||
|
||||
_result_cache.clear()
|
||||
|
||||
@patch("semantica.semantic_extract.methods.create_provider")
|
||||
@@ -652,12 +620,10 @@ class TestFullPipelineTemporalToBiTemporal(unittest.TestCase):
|
||||
self.assertEqual(vf[0].day, 1)
|
||||
|
||||
# Feed into BiTemporalFact
|
||||
fact = BiTemporalFact.from_relationship(
|
||||
{
|
||||
"valid_from": "2014-05-01T00:00:00Z",
|
||||
"valid_until": None,
|
||||
}
|
||||
)
|
||||
fact = BiTemporalFact.from_relationship({
|
||||
"valid_from": "2014-05-01T00:00:00Z",
|
||||
"valid_until": None,
|
||||
})
|
||||
self.assertIsNotNone(fact.valid_from)
|
||||
self.assertEqual(fact.valid_from.year, 2014)
|
||||
self.assertEqual(fact.valid_from.month, 5)
|
||||
@@ -694,9 +660,7 @@ class TestFullPipelineTemporalToBiTemporal(unittest.TestCase):
|
||||
extract_temporal_bounds=True,
|
||||
)
|
||||
meta = rels[0].metadata
|
||||
tn = TemporalNormalizer(
|
||||
reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc)
|
||||
)
|
||||
tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
|
||||
|
||||
vf = tn.normalize(meta["valid_from"])
|
||||
vu = tn.normalize(meta["valid_until"])
|
||||
|
||||
+11
-40
@@ -2090,16 +2090,12 @@ class TestMCP:
|
||||
# Table renders correctly — at minimum the column header is present
|
||||
assert "Tool" in result.output or "tool" in result.output.lower()
|
||||
|
||||
def test_list_tools_reads_server_catalog(self, runner, monkeypatch):
|
||||
"""list-tools must read TOOL_DEFINITIONS (what the server serves via
|
||||
tools/list), not the module's ``__all__`` (issue #1355)."""
|
||||
import semantica_mcp.mcp.tools as tools_mod
|
||||
fake = [{"name": "fake_tool_from_catalog", "description": "", "inputSchema": {},
|
||||
"_handler": lambda a: {}}]
|
||||
monkeypatch.setattr(tools_mod, "TOOL_DEFINITIONS", fake)
|
||||
def test_list_tools_with_mock_shows_known_tools(self, runner, monkeypatch):
|
||||
fake_tools = _fake_module(__all__=["extract_entities", "query_graph"])
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica_mcp.mcp.tools", fake_tools)
|
||||
result = runner.invoke(cli_module.main, ["mcp", "list-tools"])
|
||||
_ok(result)
|
||||
assert "fake_tool_from_catalog" in result.output
|
||||
assert "extract_entities" in result.output
|
||||
|
||||
def test_list_tools_json(self, runner):
|
||||
result = runner.invoke(cli_module.main, ["mcp", "list-tools", "--json"])
|
||||
@@ -2142,39 +2138,14 @@ class TestMCP:
|
||||
err = json.loads(result.stderr)
|
||||
assert err["error"].startswith("Invalid JSON in --args")
|
||||
|
||||
def test_call_dispatches_through_packaged_server(self, runner):
|
||||
"""Regression for issue #1355: ``mcp call`` dispatches in-process through
|
||||
``semantica_mcp.mcp.server`` (the server ``mcp start`` spawns) instead
|
||||
of importing the nonexistent ``MCPSession``."""
|
||||
result = runner.invoke(
|
||||
cli_module.main, ["--json", "mcp", "call", "extract_entities"]
|
||||
)
|
||||
_ok(result)
|
||||
# Empty args short-circuit before heavy imports; reaching the
|
||||
# handler's own validation proves the dispatch path works.
|
||||
assert "text is required" in result.output
|
||||
|
||||
def test_call_unknown_tool_fails_cleanly(self, runner):
|
||||
result = runner.invoke(cli_module.main, ["mcp", "call", "no_such_tool"])
|
||||
def test_call_import_error_is_clean(self, runner):
|
||||
with patch("builtins.__import__", side_effect=lambda n, *a, **k: (
|
||||
(_ for _ in ()).throw(ImportError(n))
|
||||
if n.startswith("mcp") else __import__(n, *a, **k)
|
||||
)):
|
||||
result = runner.invoke(cli_module.main, ["mcp", "call", "extract_entities"])
|
||||
assert result.exit_code != 0
|
||||
assert "Traceback" not in result.output
|
||||
assert "Unknown tool" in result.output
|
||||
|
||||
def test_call_non_object_args_rejected(self, runner):
|
||||
result = runner.invoke(
|
||||
cli_module.main, ["mcp", "call", "extract_entities", "--args", "[1, 2]"]
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "Traceback" not in result.output
|
||||
assert "--args must be a JSON object" in result.output
|
||||
|
||||
def test_list_tools_json_matches_server_catalog(self, runner):
|
||||
"""The CLI catalog and the MCP server catalog must be the same list."""
|
||||
from semantica_mcp.mcp.tools import TOOL_DEFINITIONS
|
||||
result = runner.invoke(cli_module.main, ["mcp", "list-tools", "--json"])
|
||||
_ok(result)
|
||||
data = _json_output(result)
|
||||
assert data["tools"] == [t["name"] for t in TOOL_DEFINITIONS]
|
||||
|
||||
|
||||
# ─── services group (backward-compat wrapper) ─────────────────────────────────
|
||||
@@ -2331,7 +2302,7 @@ class TestDoctorEmbeddings:
|
||||
checks = self._doctor_checks(runner)
|
||||
st = checks["Embeddings (sentence-transformers)"]
|
||||
assert st["status"] == "fail"
|
||||
assert st["hint"] == "pip install 'semantica[embeddings-local]'"
|
||||
assert st["hint"] == "pip install sentence-transformers"
|
||||
|
||||
def test_deep_probe_detects_fallback_active(self, runner, monkeypatch):
|
||||
self._with_fake_st(monkeypatch)
|
||||
|
||||
@@ -1,470 +0,0 @@
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
import pytest
|
||||
|
||||
from semantica.parse.docx_parser import DOCXParser
|
||||
from semantica.parse.excel_parser import ExcelParser
|
||||
from semantica.parse.html_parser import HTMLParser
|
||||
from semantica.parse.xml_parser import XMLParser
|
||||
from semantica.utils.exceptions import ProcessingError
|
||||
|
||||
|
||||
def _load_toml(file_path: Path) -> dict:
|
||||
"""Load and parse a TOML file across Python 3.8-3.14+ without mode mismatches."""
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
try:
|
||||
import tomllib # Python 3.11+ standard library
|
||||
|
||||
return tomllib.loads(content)
|
||||
except ImportError:
|
||||
try:
|
||||
import tomli # Fast PEP 680 compatible parser for Python < 3.11
|
||||
|
||||
return tomli.loads(content)
|
||||
except ImportError:
|
||||
import toml # Fallback toml parser
|
||||
|
||||
return toml.loads(content)
|
||||
|
||||
|
||||
def test_core_dependencies_count():
|
||||
"""pyproject.toml must contain exactly 22 unique core dependencies."""
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
data = _load_toml(repo_root / "pyproject.toml")
|
||||
deps = data["project"]["dependencies"]
|
||||
normalized_names = {
|
||||
d.split(";")[0].split(">=")[0].split("<")[0].split("==")[0].strip()
|
||||
for d in deps
|
||||
}
|
||||
expected_22 = {
|
||||
"numpy",
|
||||
"pandas",
|
||||
"scipy",
|
||||
"scikit-learn",
|
||||
"rdflib",
|
||||
"networkx",
|
||||
"requests",
|
||||
"chardet",
|
||||
"protobuf",
|
||||
"grpcio",
|
||||
"pillow",
|
||||
"pydantic",
|
||||
"click",
|
||||
"rich",
|
||||
"tqdm",
|
||||
"pyyaml",
|
||||
"toml",
|
||||
"python-dotenv",
|
||||
"loguru",
|
||||
"structlog",
|
||||
"httpx",
|
||||
"pyarrow",
|
||||
}
|
||||
assert normalized_names == expected_22
|
||||
assert len(normalized_names) == 22
|
||||
|
||||
|
||||
def test_optional_extras_defined():
|
||||
"""All required optional extras must be declared in pyproject.toml."""
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
data = _load_toml(repo_root / "pyproject.toml")
|
||||
extras = data["project"]["optional-dependencies"]
|
||||
for extra in [
|
||||
"documents",
|
||||
"ingest-git",
|
||||
"embeddings-local",
|
||||
"nlp-spacy",
|
||||
"viz",
|
||||
"media",
|
||||
"vectorstore-faiss",
|
||||
"graph-embeddings",
|
||||
"all",
|
||||
]:
|
||||
assert extra in extras, f"Missing extra {extra}"
|
||||
all_extra_str = str(extras["all"])
|
||||
for expected_ref in [
|
||||
"documents",
|
||||
"ingest-git",
|
||||
"embeddings-local",
|
||||
"nlp-spacy",
|
||||
"viz",
|
||||
"media",
|
||||
"graph-embeddings",
|
||||
"vectorstore-all",
|
||||
]:
|
||||
assert expected_ref in all_extra_str, f"Missing {expected_ref} in all"
|
||||
# And vectorstore-faiss is in vectorstore-all
|
||||
assert "vectorstore-faiss" in str(extras["vectorstore-all"])
|
||||
|
||||
# Verify nlp-spacy does not declare thinc directly (Qodo bot issue 1)
|
||||
nlp_spacy_deps = str(extras.get("nlp-spacy", []))
|
||||
assert "thinc" not in nlp_spacy_deps, "nlp-spacy should not directly declare thinc"
|
||||
assert "spacy" in nlp_spacy_deps, "nlp-spacy must declare spacy"
|
||||
|
||||
|
||||
def test_core_modules_importable():
|
||||
"""Core modules must be importable without requiring optional extras."""
|
||||
import semantica
|
||||
import semantica.cli
|
||||
import semantica.parse
|
||||
import semantica.ingest
|
||||
import semantica.embeddings
|
||||
import semantica.export
|
||||
import semantica.kg
|
||||
import semantica.vector_store
|
||||
import semantica.visualization
|
||||
import semantica.semantic_extract
|
||||
import semantica.pipeline
|
||||
|
||||
assert semantica.__version__ is not None
|
||||
|
||||
|
||||
def test_docx_parser_lazy_construction_and_parse_hint():
|
||||
with patch("semantica.parse.docx_parser.Document", None):
|
||||
parser = DOCXParser()
|
||||
assert parser is not None
|
||||
with pytest.raises(ProcessingError, match=r"semantica\[documents\]"):
|
||||
parser.parse("nonexistent.docx")
|
||||
|
||||
|
||||
def test_excel_parser_lazy_construction_and_parse_hint():
|
||||
with patch("semantica.parse.excel_parser.load_workbook", None):
|
||||
parser = ExcelParser()
|
||||
assert parser is not None
|
||||
with pytest.raises(ProcessingError, match=r"semantica\[documents\]"):
|
||||
parser.parse("nonexistent.xlsx")
|
||||
|
||||
|
||||
def test_html_parser_lazy_construction_and_parse_hint():
|
||||
with patch("semantica.parse.html_parser.BeautifulSoup", None):
|
||||
parser = HTMLParser()
|
||||
assert parser is not None
|
||||
with pytest.raises(ProcessingError, match=r"semantica\[documents\]"):
|
||||
parser.parse("nonexistent.html")
|
||||
|
||||
|
||||
def test_xml_parser_etree_fallback():
|
||||
with patch("semantica.parse.xml_parser.etree", None):
|
||||
parser = XMLParser()
|
||||
assert parser is not None
|
||||
result = parser.parse("<root><item id='1'>Test</item></root>")
|
||||
assert result is not None
|
||||
assert result.root is not None
|
||||
assert result.root.tag == "root"
|
||||
|
||||
|
||||
def test_xml_parser_lxml_explicit_requires_documents_extra():
|
||||
with patch("semantica.parse.xml_parser.etree", None):
|
||||
parser = XMLParser(engine="lxml")
|
||||
assert parser is not None
|
||||
with pytest.raises(ProcessingError, match=r"semantica\[documents\]"):
|
||||
parser.parse("<root/>")
|
||||
|
||||
|
||||
def test_xml_ingestor_missing_hint():
|
||||
with patch("semantica.ingest.xml_ingestor.etree", None):
|
||||
from semantica.ingest.xml_ingestor import XMLIngestor
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[documents\]"):
|
||||
XMLIngestor()
|
||||
|
||||
|
||||
def test_xml_ingestor_package_import_missing_hint():
|
||||
import semantica.ingest as ingest_mod
|
||||
|
||||
ingest_mod.__dict__.pop("XMLIngestor", None)
|
||||
with patch("semantica.ingest.xml_ingestor.etree", None):
|
||||
with pytest.raises(ImportError, match=r"semantica\[documents\]"):
|
||||
_ = ingest_mod.XMLIngestor
|
||||
|
||||
|
||||
def test_repo_ingestor_missing_hint():
|
||||
with patch("semantica.ingest.repo_ingestor.git", None):
|
||||
from semantica.ingest.repo_ingestor import RepoIngestor
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[ingest-git\]"):
|
||||
RepoIngestor()
|
||||
|
||||
|
||||
def test_repo_ingestor_package_import_missing_hint():
|
||||
import semantica.ingest as ingest_mod
|
||||
|
||||
ingest_mod.__dict__.pop("RepoIngestor", None)
|
||||
with patch("semantica.ingest.repo_ingestor.git", None):
|
||||
with pytest.raises(ImportError, match=r"semantica\[ingest-git\]"):
|
||||
_ = ingest_mod.RepoIngestor
|
||||
|
||||
|
||||
def test_git_analyzer_package_import_succeeds_without_git():
|
||||
import semantica.ingest as ingest_mod
|
||||
|
||||
ingest_mod.__dict__.pop("GitAnalyzer", None)
|
||||
with patch("semantica.ingest.repo_ingestor.git", None):
|
||||
analyzer_cls = ingest_mod.GitAnalyzer
|
||||
assert analyzer_cls is not None
|
||||
analyzer = analyzer_cls()
|
||||
assert analyzer is not None
|
||||
|
||||
|
||||
def test_salesforce_ingestor_package_import_missing_hint():
|
||||
import semantica.ingest as ingest_mod
|
||||
|
||||
ingest_mod.__dict__.pop("SalesforceIngestor", None)
|
||||
with patch("semantica.ingest.salesforce_ingestor.SALESFORCE_AVAILABLE", False):
|
||||
with pytest.raises(ImportError, match=r"semantica\[db-salesforce\]"):
|
||||
_ = ingest_mod.SalesforceIngestor
|
||||
|
||||
|
||||
def test_parse_methods_dynamic_default_resolution():
|
||||
""""default" must NOT be registered in the method registry: each built-in
|
||||
dispatcher (parse_document, ...) starts with
|
||||
method_registry.get(<task>, method), so a self-registered "default" would
|
||||
resolve to the dispatcher itself and recurse infinitely. "default" stays
|
||||
the built-in code path, reached only by falling through an unregistered
|
||||
lookup, and callers can still register their own "default" to override
|
||||
it."""
|
||||
from semantica.parse.methods import get_parse_method, list_available_methods
|
||||
|
||||
assert get_parse_method("document", "default") is None
|
||||
methods = list_available_methods()
|
||||
assert "default" not in methods.get("document", [])
|
||||
assert "default" not in methods.get("structured", [])
|
||||
|
||||
|
||||
def test_node_embedder_gensim_missing_hint():
|
||||
with patch("semantica.kg.node_embeddings.GENSIM_AVAILABLE", False):
|
||||
from semantica.kg.node_embeddings import NodeEmbedder
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[graph-embeddings\]"):
|
||||
NodeEmbedder()
|
||||
|
||||
|
||||
def test_faiss_store_missing_hint():
|
||||
with patch("semantica.vector_store.faiss_store.FAISS_AVAILABLE", False):
|
||||
from semantica.vector_store.faiss_store import FAISSIndexBuilder, FAISSStore
|
||||
|
||||
builder = FAISSIndexBuilder(128)
|
||||
with pytest.raises(ProcessingError, match=r"semantica\[vectorstore-faiss\]"):
|
||||
builder.build_index("flat")
|
||||
store = FAISSStore(128)
|
||||
with pytest.raises(ProcessingError, match=r"semantica\[vectorstore-faiss\]"):
|
||||
store.load_index("nonexistent.faiss")
|
||||
|
||||
|
||||
def test_visualization_missing_hint():
|
||||
import numpy as np
|
||||
from semantica.visualization.embedding_visualizer import EmbeddingVisualizer
|
||||
|
||||
# Plotly is checked via px and go in _check_dependencies
|
||||
with patch("semantica.visualization.embedding_visualizer.px", None):
|
||||
visualizer = EmbeddingVisualizer()
|
||||
with pytest.raises(
|
||||
ProcessingError, match=r"Plotly is required.*semantica\[viz\]"
|
||||
):
|
||||
visualizer.visualize_2d_projection(np.array([[0.1, 0.2], [0.3, 0.4]]))
|
||||
|
||||
with patch("semantica.visualization.embedding_visualizer.go", None):
|
||||
visualizer = EmbeddingVisualizer()
|
||||
with pytest.raises(
|
||||
ProcessingError, match=r"Plotly is required.*semantica\[viz\]"
|
||||
):
|
||||
visualizer.visualize_2d_projection(np.array([[0.1, 0.2], [0.3, 0.4]]))
|
||||
|
||||
|
||||
def test_visualization_umap_missing_hint():
|
||||
import numpy as np
|
||||
from unittest.mock import MagicMock
|
||||
from semantica.visualization.embedding_visualizer import EmbeddingVisualizer
|
||||
|
||||
# Stand in for Plotly so we reach dimensionality reduction
|
||||
with patch("semantica.visualization.embedding_visualizer.px", MagicMock()), patch(
|
||||
"semantica.visualization.embedding_visualizer.go", MagicMock()
|
||||
), patch("semantica.visualization.embedding_visualizer.umap", None):
|
||||
visualizer = EmbeddingVisualizer()
|
||||
# High-dimensional embeddings (>2D) trigger dimensionality reduction
|
||||
# with method="umap"
|
||||
embeddings = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])
|
||||
with pytest.raises(
|
||||
ProcessingError, match=r"UMAP is required.*semantica\[viz\]"
|
||||
):
|
||||
visualizer.visualize_2d_projection(embeddings, method="umap")
|
||||
|
||||
# Also verify 3D projection triggers the same actionable error on >3D embeddings
|
||||
embeddings_4d = np.array(
|
||||
[[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8], [0.9, 1.0, 1.1, 1.2]]
|
||||
)
|
||||
with pytest.raises(
|
||||
ProcessingError, match=r"UMAP is required.*semantica\[viz\]"
|
||||
):
|
||||
visualizer.visualize_3d_projection(embeddings_4d, method="umap")
|
||||
|
||||
|
||||
def test_visualization_sklearn_missing_hint():
|
||||
import numpy as np
|
||||
from unittest.mock import MagicMock
|
||||
from semantica.visualization.embedding_visualizer import EmbeddingVisualizer
|
||||
|
||||
# Stand in for Plotly so we reach dimensionality reduction
|
||||
with patch("semantica.visualization.embedding_visualizer.px", MagicMock()), patch(
|
||||
"semantica.visualization.embedding_visualizer.go", MagicMock()
|
||||
):
|
||||
visualizer = EmbeddingVisualizer()
|
||||
embeddings = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])
|
||||
|
||||
# Test direct dependency check
|
||||
with patch("semantica.visualization.embedding_visualizer.PCA", None):
|
||||
with pytest.raises(ProcessingError, match=r"scikit-learn is required"):
|
||||
visualizer._check_dependencies(require_sklearn=True)
|
||||
|
||||
with patch("semantica.visualization.embedding_visualizer.PCA", None):
|
||||
with pytest.raises(ProcessingError, match=r"scikit-learn is required"):
|
||||
visualizer.visualize_2d_projection(embeddings, method="pca")
|
||||
|
||||
with patch("semantica.visualization.embedding_visualizer.TSNE", None):
|
||||
with pytest.raises(ProcessingError, match=r"scikit-learn is required"):
|
||||
visualizer.visualize_2d_projection(embeddings, method="tsne")
|
||||
|
||||
|
||||
def test_visualization_options_collision_free():
|
||||
"""Options like n_components, perplexity must not cause keyword collisions."""
|
||||
import numpy as np
|
||||
from unittest.mock import MagicMock
|
||||
from semantica.visualization.embedding_visualizer import EmbeddingVisualizer
|
||||
|
||||
mock_pca = MagicMock()
|
||||
mock_tsne = MagicMock()
|
||||
mock_umap_cls = MagicMock()
|
||||
mock_umap_module = MagicMock()
|
||||
mock_umap_module.UMAP = mock_umap_cls
|
||||
|
||||
with patch("semantica.visualization.embedding_visualizer.PCA", mock_pca), patch(
|
||||
"semantica.visualization.embedding_visualizer.TSNE", mock_tsne
|
||||
), patch(
|
||||
"semantica.visualization.embedding_visualizer.umap", mock_umap_module
|
||||
), patch(
|
||||
"semantica.visualization.embedding_visualizer.px", MagicMock()
|
||||
), patch(
|
||||
"semantica.visualization.embedding_visualizer.go", MagicMock()
|
||||
):
|
||||
visualizer = EmbeddingVisualizer()
|
||||
embeddings = np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]])
|
||||
|
||||
# PCA with n_components
|
||||
visualizer.visualize_2d_projection(embeddings, method="pca", n_components=2)
|
||||
# TSNE with perplexity and random_state
|
||||
visualizer.visualize_2d_projection(
|
||||
embeddings, method="tsne", perplexity=1, random_state=42
|
||||
)
|
||||
# UMAP with n_neighbors and min_dist
|
||||
visualizer.visualize_2d_projection(
|
||||
embeddings, method="umap", n_neighbors=2, min_dist=0.1
|
||||
)
|
||||
# 3D with n_components
|
||||
visualizer.visualize_3d_projection(embeddings, method="pca", n_components=3)
|
||||
|
||||
|
||||
def test_spacy_load_missing_hint():
|
||||
from semantica.semantic_extract.methods import load_spacy_model
|
||||
|
||||
with patch("semantica.semantic_extract.methods.spacy", None):
|
||||
with pytest.raises(ImportError, match=r"semantica\[nlp-spacy\]"):
|
||||
load_spacy_model("en_core_web_sm")
|
||||
|
||||
|
||||
def test_xml_parser_handles_comments():
|
||||
from semantica.parse.xml_parser import etree as real_lxml_etree
|
||||
|
||||
xml_content = (
|
||||
"<root><!-- top comment --><item id='1'>Value</item>"
|
||||
"<!-- bottom comment --></root>"
|
||||
)
|
||||
# etree engine (always available in a core-only install)
|
||||
p_etree = XMLParser(engine="etree")
|
||||
res_etree = p_etree.parse(xml_content)
|
||||
assert res_etree.root.tag == "root"
|
||||
assert len(res_etree.root.children) == 1
|
||||
assert res_etree.root.children[0].tag == "item"
|
||||
assert res_etree.root.children[0].text == "Value"
|
||||
|
||||
# lxml engine (only meaningful when the 'documents' extra is installed)
|
||||
if real_lxml_etree is None:
|
||||
pytest.skip("lxml not installed (requires semantica[documents])")
|
||||
p_lxml = XMLParser(engine="lxml")
|
||||
res_lxml = p_lxml.parse(xml_content)
|
||||
assert res_lxml.root.tag == "root"
|
||||
assert len(res_lxml.root.children) == 1
|
||||
assert res_lxml.root.children[0].tag == "item"
|
||||
assert res_lxml.root.children[0].text == "Value"
|
||||
|
||||
|
||||
def test_public_api_ingestor_handles_xml_comments():
|
||||
from semantica.ingest.public_api_ingestor import (
|
||||
PublicAPIIngestor,
|
||||
lxml_etree as real_lxml_etree,
|
||||
safe_xml_etree as real_safe_xml_etree,
|
||||
)
|
||||
|
||||
if real_lxml_etree is None and real_safe_xml_etree is None:
|
||||
pytest.skip("neither defusedxml nor lxml installed (requires semantica[documents]/[explorer])")
|
||||
|
||||
xml_content = "<root><!-- comment --><item id='1'>Value</item></root>"
|
||||
ingestor = PublicAPIIngestor(rate_limit_delay=0)
|
||||
|
||||
# 1. Default (defusedxml if available)
|
||||
parsed = ingestor._parse_xml(xml_content)
|
||||
assert parsed["tag"] == "root"
|
||||
assert len(parsed["children"]) == 1
|
||||
assert parsed["children"][0]["tag"] == "item"
|
||||
assert parsed["children"][0]["text"] == "Value"
|
||||
|
||||
# 2. lxml fallback (only meaningful when lxml is actually installed)
|
||||
if real_lxml_etree is None:
|
||||
pytest.skip("lxml not installed (requires semantica[documents])")
|
||||
with patch("semantica.ingest.public_api_ingestor.safe_xml_etree", None):
|
||||
parsed_lxml = ingestor._parse_xml(xml_content)
|
||||
assert parsed_lxml["tag"] == "root"
|
||||
assert len(parsed_lxml["children"]) == 1
|
||||
assert parsed_lxml["children"][0]["tag"] == "item"
|
||||
assert parsed_lxml["children"][0]["text"] == "Value"
|
||||
|
||||
|
||||
def test_huggingface_model_loader_catches_oserror():
|
||||
import builtins
|
||||
from unittest.mock import MagicMock
|
||||
from semantica.semantic_extract.providers import HuggingFaceModelLoader
|
||||
|
||||
mock_torch = MagicMock()
|
||||
mock_torch.Tensor = type("Tensor", (), {})
|
||||
with patch.dict("sys.modules", {"torch": mock_torch}):
|
||||
loader = HuggingFaceModelLoader()
|
||||
# 1. Test ModuleNotFoundError / ImportError
|
||||
with patch.dict("sys.modules", {"transformers": None}):
|
||||
with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
|
||||
loader.load_ner_model("bert-base-cased")
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
|
||||
loader.load_relation_model("bert-base-cased")
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
|
||||
loader.load_triplet_model("t5-base")
|
||||
|
||||
# 2. Test OSError (e.g. corrupt DLL / missing shared library)
|
||||
real_import = builtins.__import__
|
||||
|
||||
def fake_import(name, *args, **kwargs):
|
||||
if name == "transformers":
|
||||
raise OSError("DLL load failed")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
try:
|
||||
builtins.__import__ = fake_import
|
||||
with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
|
||||
loader.load_ner_model("bert-base-cased-oserror")
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
|
||||
loader.load_relation_model("bert-base-cased-oserror")
|
||||
|
||||
with pytest.raises(ImportError, match=r"semantica\[models-huggingface\]"):
|
||||
loader.load_triplet_model("t5-base-oserror")
|
||||
finally:
|
||||
builtins.__import__ = real_import
|
||||
@@ -1,64 +0,0 @@
|
||||
"""Tests for the shared in-process tool entry point (issue #1355).
|
||||
|
||||
``semantica_mcp.mcp.server.call_tool`` is the dispatch used by both the
|
||||
JSON-RPC ``tools/call`` handler and the ``semantica mcp call`` CLI command,
|
||||
so the two surfaces cannot expose different tool sets.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
from semantica_mcp.mcp import server
|
||||
from semantica_mcp.mcp.server import UnknownToolError, _handle_tools_call, call_tool
|
||||
|
||||
|
||||
class TestCallTool(unittest.TestCase):
|
||||
|
||||
def test_known_tool_dispatches_to_handler(self):
|
||||
# Empty args hit extract_entities' own validation before any heavy
|
||||
# imports, which is enough to prove dispatch reached the handler.
|
||||
result = call_tool("extract_entities", {})
|
||||
self.assertEqual(result["error"], "text is required")
|
||||
|
||||
def test_unknown_tool_raises_unknown_tool_error(self):
|
||||
with self.assertRaises(UnknownToolError):
|
||||
call_tool("no_such_tool", {})
|
||||
|
||||
def test_unknown_tool_error_is_not_a_key_error(self):
|
||||
"""A handler's own KeyError (missing required arg) must remain
|
||||
distinguishable from an unknown tool name."""
|
||||
self.assertFalse(issubclass(UnknownToolError, KeyError))
|
||||
|
||||
|
||||
class TestToolsCallDispatch(unittest.TestCase):
|
||||
|
||||
@staticmethod
|
||||
def _tools_call(name, arguments):
|
||||
return _handle_tools_call(1, {"name": name, "arguments": arguments})
|
||||
|
||||
def test_unknown_tool_returns_method_not_found(self):
|
||||
response = self._tools_call("no_such_tool", {})
|
||||
self.assertEqual(response["error"]["code"], -32601)
|
||||
self.assertEqual(response["error"]["message"], "Unknown tool: no_such_tool")
|
||||
|
||||
def test_handler_key_error_is_internal_error_not_unknown_tool(self):
|
||||
def _boom(args):
|
||||
raise KeyError("category")
|
||||
|
||||
fake = {"name": "boom", "description": "", "inputSchema": {}, "_handler": _boom}
|
||||
with patch.dict(server._TOOL_INDEX, {"boom": fake}):
|
||||
response = self._tools_call("boom", {})
|
||||
self.assertEqual(response["error"]["code"], -32603)
|
||||
|
||||
def test_known_tool_returns_result_content(self):
|
||||
response = self._tools_call("extract_entities", {})
|
||||
self.assertIn("content", response["result"])
|
||||
self.assertTrue(response["result"]["isError"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,630 +0,0 @@
|
||||
"""Regression tests for MCP export_graph — GraphML and Parquet branches.
|
||||
|
||||
Pre-fix bugs
|
||||
------------
|
||||
GraphML:
|
||||
- handle_export_graph({"format": "graphml"}) imported ``GraphMLExporter``
|
||||
which does not exist in ``semantica.export``, raising ImportError on every
|
||||
call and returning ``{"error": "...GraphMLExporter..."}``.
|
||||
- Even if the import were corrected, ``exporter.export(graph)`` passed the
|
||||
raw ContextGraph object and ignored the required ``file_path`` argument.
|
||||
``GraphExporter.export()`` writes to a file and returns ``None``; the old
|
||||
code used its return value as the response data.
|
||||
|
||||
Parquet:
|
||||
- ``exporter.export(graph, include_metadata)`` passed the ContextGraph object
|
||||
as ``data`` (not the kg dict) and ``include_metadata`` (a bool) as
|
||||
``file_path``. Both arguments are wrong: the exporter expects a list/dict
|
||||
of plain dicts as ``data`` and a filesystem path as ``file_path``.
|
||||
- ``ParquetExporter.export()`` returns ``None``; wrapping it in ``str()``
|
||||
always produced the string ``"None"`` as the response data.
|
||||
|
||||
Post-fix expectations
|
||||
---------------------
|
||||
GraphML:
|
||||
- Returns ``{"format": "graphml", "data": "<graphml ...>..."}``
|
||||
- ``data`` is a non-empty string containing valid GraphML XML.
|
||||
- The TemporaryDirectory is cleaned up after the call (no leaked temp files).
|
||||
|
||||
Parquet:
|
||||
- Returns ``{"format": "parquet", "data": {...}, "encoding": "base64"}``
|
||||
- ``data`` is a dict mapping filename → base64-encoded bytes.
|
||||
- At least one ``*.parquet`` key is present.
|
||||
- Each value decodes to non-empty bytes (valid Parquet file magic: b"PAR1").
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import os
|
||||
import unittest
|
||||
|
||||
# Disable progress tracking before any Semantica import so no singleton
|
||||
# writes to stdout during testing.
|
||||
os.environ["SEMANTICA_DISABLE_PROGRESS"] = "1"
|
||||
|
||||
|
||||
def _make_graph():
|
||||
"""Return a ContextGraph with two entities and one relationship."""
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
g = ContextGraph()
|
||||
g.add_node("n1", node_type="entity")
|
||||
g.add_node("n2", node_type="entity")
|
||||
g.add_edge("n1", "n2", "related_to")
|
||||
return g
|
||||
|
||||
|
||||
class TestMCPExportGraphML(unittest.TestCase):
|
||||
"""GraphML export must use GraphExporter (not GraphMLExporter) and return
|
||||
valid GraphML XML."""
|
||||
|
||||
def setUp(self):
|
||||
import semantica_mcp.mcp.session as _session
|
||||
self._orig = _session._graph
|
||||
_session._graph = _make_graph()
|
||||
|
||||
def tearDown(self):
|
||||
import semantica_mcp.mcp.session as _session
|
||||
_session._graph = self._orig
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Regression: old code raised ImportError for GraphMLExporter
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_graphml_does_not_return_import_error(self):
|
||||
"""The old code: ``from semantica.export import GraphMLExporter`` —
|
||||
that class does not exist. Result must not contain 'GraphMLExporter'
|
||||
in the error message."""
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
if "error" in result:
|
||||
self.assertNotIn(
|
||||
"GraphMLExporter", result["error"],
|
||||
f"Import of non-existent GraphMLExporter still present: {result}",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Correctness
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_graphml_returns_success_not_error(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, f"GraphML export returned error: {result}")
|
||||
|
||||
def test_graphml_format_key_is_correct(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertEqual(result.get("format"), "graphml")
|
||||
|
||||
def test_graphml_data_is_non_empty_string(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertIsInstance(result.get("data"), str)
|
||||
self.assertGreater(len(result["data"]), 0)
|
||||
|
||||
def test_graphml_data_contains_xml_declaration(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertIn('<?xml version="1.0"', result["data"])
|
||||
|
||||
def test_graphml_data_contains_graphml_element(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertIn("<graphml", result["data"])
|
||||
self.assertIn("</graphml>", result["data"])
|
||||
|
||||
def test_graphml_data_is_not_the_string_None(self):
|
||||
"""The old code called ``str(exporter.export(graph))`` which returns
|
||||
``'None'`` because ``export()`` returns ``None``."""
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertNotEqual(result.get("data"), "None")
|
||||
|
||||
def test_graphml_is_parseable_xml(self):
|
||||
"""The response must be well-formed XML, not an error string."""
|
||||
import xml.etree.ElementTree as ET
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
try:
|
||||
ET.fromstring(result["data"])
|
||||
except ET.ParseError as exc:
|
||||
self.fail(f"GraphML output is not valid XML: {exc}\n{result['data'][:500]}")
|
||||
|
||||
def test_graphml_no_contextgraph_attribute_error(self):
|
||||
"""The old code passed the ContextGraph object directly. Verify the
|
||||
error 'ContextGraph' object has no attribute 'get' (or similar) is gone."""
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
if "error" in result:
|
||||
self.assertNotIn("ContextGraph", result["error"])
|
||||
self.assertNotIn("has no attribute", result["error"])
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Regression: undeclared / wrongly-scoped GraphML keys.
|
||||
#
|
||||
# Pre-fix, _export_graphml declared:
|
||||
# <key id="type" for="node" …/>
|
||||
# <key id="confidence" for="node" …/>
|
||||
#
|
||||
# and then referenced:
|
||||
# <data key="label"> on BOTH nodes and edges (no declaration at all)
|
||||
# <data key="confidence"> on edges (declared for="node" only)
|
||||
#
|
||||
# Schema-validating consumers (Cytoscape, yEd, any XSD-aware reader)
|
||||
# reject documents with undeclared or out-of-scope key references.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _graphml_key_declarations(self, xml_text: str) -> dict:
|
||||
"""Parse the XML and return {key_id: for_value} for every <key>."""
|
||||
import xml.etree.ElementTree as ET
|
||||
root = ET.fromstring(xml_text)
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
return {k.get("id"): k.get("for") for k in root.findall("g:key", ns)}
|
||||
|
||||
def test_graphml_label_key_is_declared(self):
|
||||
"""label key must be declared; pre-fix it was missing entirely."""
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
keys = self._graphml_key_declarations(result["data"])
|
||||
self.assertIn(
|
||||
"label", keys,
|
||||
f"<key id='label'> is missing from GraphML header; declared keys: {list(keys)}",
|
||||
)
|
||||
|
||||
def test_graphml_label_key_scope_covers_edges(self):
|
||||
"""label is written on both nodes (node label) and edges (edge type).
|
||||
Its for= scope must be 'all' or 'edge'. Pre-fix it was not declared."""
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
keys = self._graphml_key_declarations(result["data"])
|
||||
scope = keys.get("label", "")
|
||||
self.assertIn(
|
||||
scope, ("all", "edge"),
|
||||
f"<key id='label'> has for={scope!r}; must be 'all' or 'edge' "
|
||||
f"because edges write <data key='label'>",
|
||||
)
|
||||
|
||||
def test_graphml_confidence_key_scope_covers_edges(self):
|
||||
"""confidence is written on both nodes and edges when include_attributes
|
||||
is True. Pre-fix the key was declared for='node' only, making every
|
||||
edge confidence reference schema-invalid."""
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
keys = self._graphml_key_declarations(result["data"])
|
||||
scope = keys.get("confidence", "")
|
||||
self.assertIn(
|
||||
scope, ("all", "edge"),
|
||||
f"<key id='confidence'> has for={scope!r}; must be 'all' or 'edge' "
|
||||
f"because edges also write <data key='confidence'>",
|
||||
)
|
||||
|
||||
def test_graphml_all_data_key_refs_are_declared(self):
|
||||
"""Every <data key=X> reference in the document must have a matching
|
||||
<key id=X> declaration — with a graph that has both nodes AND edges,
|
||||
so edge-only violations are not hidden by an edge-free export."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from semantica.export import GraphExporter
|
||||
|
||||
kg = {
|
||||
"entities": [
|
||||
{"id": "n1", "text": "Alice", "type": "Person"},
|
||||
{"id": "n2", "text": "Bob", "type": "Person"},
|
||||
],
|
||||
"relationships": [
|
||||
{"source_id": "n1", "target_id": "n2", "type": "knows"},
|
||||
],
|
||||
}
|
||||
exporter = GraphExporter(format="graphml")
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_path = Path(tmpdir) / "out.graphml"
|
||||
exporter.export_knowledge_graph(kg, file_path=tmp_path)
|
||||
xml_text = tmp_path.read_text(encoding="utf-8")
|
||||
|
||||
root = ET.fromstring(xml_text)
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
|
||||
declared_ids = {k.get("id") for k in root.findall("g:key", ns)}
|
||||
referenced_ids = {d.get("key") for d in root.findall(".//g:data", ns)}
|
||||
|
||||
undeclared = referenced_ids - declared_ids
|
||||
self.assertEqual(
|
||||
undeclared, set(),
|
||||
f"GraphML references key id(s) with no <key> declaration: {undeclared}. "
|
||||
f"Declared: {declared_ids}",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Regression: export() consumed nodes/edges while to_kg_dict() returns
|
||||
# entities/relationships — the GraphML was structurally valid XML but
|
||||
# silently contained zero nodes and zero edges.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_graphml_contains_graph_nodes(self):
|
||||
"""Entities in the source graph must appear as <node> elements.
|
||||
Pre-fix: exporter.export(kg_dict) read 'nodes' key (absent in
|
||||
to_kg_dict()); export_knowledge_graph() converts entities -> nodes."""
|
||||
import xml.etree.ElementTree as ET
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
root = ET.fromstring(result["data"])
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
nodes = root.findall(".//g:node", ns)
|
||||
self.assertGreater(
|
||||
len(nodes), 0,
|
||||
"GraphML contains zero <node> elements; to_kg_dict() entities were "
|
||||
"not converted — export_knowledge_graph() must be used, not export().",
|
||||
)
|
||||
|
||||
def test_graphml_contains_graph_edges(self):
|
||||
"""Relationships in the source graph must appear as <edge> elements."""
|
||||
import xml.etree.ElementTree as ET
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
root = ET.fromstring(result["data"])
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
edges = root.findall(".//g:edge", ns)
|
||||
self.assertGreater(
|
||||
len(edges), 0,
|
||||
"GraphML contains zero <edge> elements; to_kg_dict() relationships were "
|
||||
"not converted — export_knowledge_graph() must be used, not export().",
|
||||
)
|
||||
|
||||
def test_graphml_node_ids_match_source_graph(self):
|
||||
"""The <node id=...> values must match the entity IDs from the source graph."""
|
||||
import xml.etree.ElementTree as ET
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
self.assertNotIn("error", result, result)
|
||||
root = ET.fromstring(result["data"])
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
node_ids = {n.get("id") for n in root.findall(".//g:node", ns)}
|
||||
# _make_graph() adds nodes with id "n1" and "n2"
|
||||
self.assertIn("n1", node_ids, f"Expected 'n1' in GraphML node ids; got {node_ids}")
|
||||
self.assertIn("n2", node_ids, f"Expected 'n2' in GraphML node ids; got {node_ids}")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Regression: XML-special characters in graph values must be escaped.
|
||||
# Pre-fix _export_graphml used bare f-string interpolation so a node
|
||||
# id of 'a&b' produced <node id="a&b"> which is malformed XML.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_graphml_xml_special_chars_in_node_id_produce_well_formed_xml(self):
|
||||
"""A node id containing & < > must be escaped in the id= attribute so
|
||||
the output remains well-formed XML. Pre-fix this produced
|
||||
<node id="a&b"> which is a parse error."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import semantica_mcp.mcp.session as _session
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
|
||||
g = ContextGraph()
|
||||
g.add_node("a&b<c>d", node_type="entity")
|
||||
_orig = _session._graph
|
||||
_session._graph = g
|
||||
try:
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
finally:
|
||||
_session._graph = _orig
|
||||
|
||||
self.assertNotIn("error", result, result)
|
||||
# Must parse without raising; pre-fix this raised ET.ParseError
|
||||
try:
|
||||
root = ET.fromstring(result["data"])
|
||||
except ET.ParseError as exc:
|
||||
self.fail(
|
||||
f"GraphML with special-char node id is malformed XML: {exc}\n"
|
||||
f"{result['data'][:600]}"
|
||||
)
|
||||
# The id attribute value must round-trip to the original string
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
node_ids = {n.get("id") for n in root.findall(".//g:node", ns)}
|
||||
self.assertIn(
|
||||
"a&b<c>d", node_ids,
|
||||
f"Node id did not round-trip correctly; got {node_ids}",
|
||||
)
|
||||
|
||||
def test_graphml_xml_special_chars_in_label_produce_well_formed_xml(self):
|
||||
"""A label containing & < > must be escaped in the <data> text
|
||||
node. Pre-fix <data key="label">A & B</data> is malformed XML.
|
||||
|
||||
Drives GraphExporter directly with a hand-crafted KG dict to avoid
|
||||
relying on ContextGraph internal field mapping."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from semantica.export import GraphExporter
|
||||
|
||||
special_label = "price < 100 & qty > 0"
|
||||
kg = {
|
||||
"entities": [
|
||||
{"id": "e1", "text": special_label, "type": "metric"},
|
||||
],
|
||||
"relationships": [],
|
||||
}
|
||||
|
||||
exporter = GraphExporter(format="graphml")
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_path = Path(tmpdir) / "out.graphml"
|
||||
exporter.export_knowledge_graph(kg, file_path=tmp_path)
|
||||
xml_text = tmp_path.read_text(encoding="utf-8")
|
||||
|
||||
# Must parse without error
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
except ET.ParseError as exc:
|
||||
self.fail(
|
||||
f"GraphML with special-char label is malformed XML: {exc}\n"
|
||||
f"{xml_text[:600]}"
|
||||
)
|
||||
# Label text must round-trip to the original string
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
data_texts = [
|
||||
d.text
|
||||
for d in root.findall(".//g:data", ns)
|
||||
if d.get("key") == "label"
|
||||
]
|
||||
self.assertIn(
|
||||
special_label, data_texts,
|
||||
f"Label text did not round-trip; found label data: {data_texts}",
|
||||
)
|
||||
|
||||
def test_graphml_double_quotes_in_node_id_produce_well_formed_xml(self):
|
||||
"""A node id containing double-quotes must not break the id=\" attribute
|
||||
boundary. Pre-fix: <node id="say "hi""> is malformed."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import semantica_mcp.mcp.session as _session
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
|
||||
g = ContextGraph()
|
||||
g.add_node('say "hi"', node_type="entity")
|
||||
_orig = _session._graph
|
||||
_session._graph = g
|
||||
try:
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
finally:
|
||||
_session._graph = _orig
|
||||
|
||||
self.assertNotIn("error", result, result)
|
||||
try:
|
||||
root = ET.fromstring(result["data"])
|
||||
except ET.ParseError as exc:
|
||||
self.fail(
|
||||
f"GraphML with double-quote node id is malformed XML: {exc}\n"
|
||||
f"{result['data'][:600]}"
|
||||
)
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
node_ids = {n.get("id") for n in root.findall(".//g:node", ns)}
|
||||
self.assertIn(
|
||||
'say "hi"', node_ids,
|
||||
f"Node id with double-quotes did not round-trip; got {node_ids}",
|
||||
)
|
||||
|
||||
def test_graphml_xml_special_chars_in_edge_source_target_produce_well_formed_xml(self):
|
||||
"""Edge source= and target= attributes must also be escaped."""
|
||||
import xml.etree.ElementTree as ET
|
||||
import semantica_mcp.mcp.session as _session
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
|
||||
g = ContextGraph()
|
||||
g.add_node("src&node", node_type="entity")
|
||||
g.add_node("tgt<node>", node_type="entity")
|
||||
g.add_edge("src&node", "tgt<node>", "link")
|
||||
_orig = _session._graph
|
||||
_session._graph = g
|
||||
try:
|
||||
result = handle_export_graph({"format": "graphml"})
|
||||
finally:
|
||||
_session._graph = _orig
|
||||
|
||||
self.assertNotIn("error", result, result)
|
||||
try:
|
||||
root = ET.fromstring(result["data"])
|
||||
except ET.ParseError as exc:
|
||||
self.fail(
|
||||
f"GraphML with special-char edge endpoints is malformed XML: {exc}\n"
|
||||
f"{result['data'][:600]}"
|
||||
)
|
||||
ns = {"g": "http://graphml.graphdrawing.org/xmlns"}
|
||||
edge_els = root.findall(".//g:edge", ns)
|
||||
self.assertEqual(len(edge_els), 1, "Expected exactly one edge element")
|
||||
self.assertEqual(edge_els[0].get("source"), "src&node")
|
||||
self.assertEqual(edge_els[0].get("target"), "tgt<node>")
|
||||
|
||||
|
||||
class TestMCPExportParquet(unittest.TestCase):
|
||||
"""Parquet export must use export_knowledge_graph(), return base64-encoded
|
||||
Parquet bytes, and clean up temporary files."""
|
||||
|
||||
def setUp(self):
|
||||
import semantica_mcp.mcp.session as _session
|
||||
self._orig = _session._graph
|
||||
_session._graph = _make_graph()
|
||||
|
||||
def tearDown(self):
|
||||
import semantica_mcp.mcp.session as _session
|
||||
_session._graph = self._orig
|
||||
|
||||
def _skip_if_no_pyarrow(self):
|
||||
try:
|
||||
import pyarrow # noqa: F401
|
||||
except ImportError:
|
||||
self.skipTest("pyarrow not installed")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Regression: old code produced {"format": "parquet", "data": "None"}
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_parquet_data_is_not_the_string_None(self):
|
||||
"""The old code: ``str(exporter.export(graph, include_metadata))``
|
||||
always produced ``"None"``."""
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertNotEqual(result.get("data"), "None")
|
||||
self.assertNotEqual(result.get("data"), None)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Regression: old code passed ContextGraph and bool to export()
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_parquet_does_not_return_contextgraph_type_error(self):
|
||||
"""The old code passed a ContextGraph as ``data`` and bool as
|
||||
``file_path``. Ensure neither TypeError appears in the result."""
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
if "error" in result:
|
||||
self.assertNotIn("ContextGraph", result["error"])
|
||||
self.assertNotIn("file_path", result["error"])
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Correctness
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_parquet_returns_success_not_error(self):
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, f"Parquet export returned error: {result}")
|
||||
|
||||
def test_parquet_format_key_is_correct(self):
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertEqual(result.get("format"), "parquet")
|
||||
|
||||
def test_parquet_encoding_is_base64(self):
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertEqual(result.get("encoding"), "base64")
|
||||
|
||||
def test_parquet_data_is_dict_of_filenames_to_strings(self):
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, result)
|
||||
data = result.get("data")
|
||||
self.assertIsInstance(data, dict, f"Expected dict, got {type(data)}: {data!r}")
|
||||
for k, v in data.items():
|
||||
self.assertIsInstance(k, str, f"key {k!r} is not str")
|
||||
self.assertIsInstance(v, str, f"value for {k!r} is not str")
|
||||
|
||||
def test_parquet_data_contains_at_least_one_parquet_file(self):
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, result)
|
||||
data = result.get("data", {})
|
||||
parquet_keys = [k for k in data if k.endswith(".parquet")]
|
||||
self.assertGreater(len(parquet_keys), 0,
|
||||
f"No .parquet keys in data: {list(data.keys())}")
|
||||
|
||||
def test_parquet_values_decode_to_valid_parquet_magic(self):
|
||||
"""Each base64 value must decode to bytes starting with the Parquet
|
||||
magic number b'PAR1' (first 4 bytes)."""
|
||||
self._skip_if_no_pyarrow()
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertNotIn("error", result, result)
|
||||
data = result.get("data", {})
|
||||
for filename, b64_str in data.items():
|
||||
raw = base64.b64decode(b64_str)
|
||||
self.assertGreater(len(raw), 4, f"{filename}: decoded to {len(raw)} bytes")
|
||||
self.assertEqual(
|
||||
raw[:4], b"PAR1",
|
||||
f"{filename}: expected Parquet magic b'PAR1', got {raw[:4]!r}",
|
||||
)
|
||||
|
||||
def test_parquet_no_pyarrow_returns_graceful_error(self):
|
||||
"""When pyarrow is absent the handler must return a dict with 'error'
|
||||
key, not raise an exception. Simulate by patching the import."""
|
||||
import sys
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
# Temporarily hide pyarrow
|
||||
real_pyarrow = sys.modules.pop("pyarrow", None)
|
||||
# Also hide the real ParquetExporter so the import inside the handler fails
|
||||
import semantica.export as _export_mod
|
||||
real_exporter_class = _export_mod.ParquetExporter
|
||||
# Replace with the dummy that raises ImportError on init
|
||||
class _MissingParquet:
|
||||
def __init__(self, *a, **kw):
|
||||
raise ImportError("pyarrow is not installed")
|
||||
_export_mod.ParquetExporter = _MissingParquet
|
||||
try:
|
||||
result = handle_export_graph({"format": "parquet"})
|
||||
self.assertIn("error", result)
|
||||
self.assertIn("pyarrow", result["error"].lower())
|
||||
finally:
|
||||
_export_mod.ParquetExporter = real_exporter_class
|
||||
if real_pyarrow is not None:
|
||||
sys.modules["pyarrow"] = real_pyarrow
|
||||
|
||||
|
||||
class TestMCPExportPreservesExistingFormats(unittest.TestCase):
|
||||
"""Adding GraphML/Parquet fixes must not regress JSON, CSV, or RDF."""
|
||||
|
||||
def setUp(self):
|
||||
import semantica_mcp.mcp.session as _session
|
||||
self._orig = _session._graph
|
||||
_session._graph = _make_graph()
|
||||
|
||||
def tearDown(self):
|
||||
import semantica_mcp.mcp.session as _session
|
||||
_session._graph = self._orig
|
||||
|
||||
def test_json_still_works(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "json"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertEqual(result.get("format"), "json")
|
||||
self.assertIsInstance(result.get("data"), dict)
|
||||
|
||||
def test_csv_still_works(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "csv"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertEqual(result.get("format"), "csv")
|
||||
self.assertIn("id,label,type", result.get("data", ""))
|
||||
|
||||
def test_turtle_still_works(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "turtle"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertIsInstance(result.get("data"), str)
|
||||
self.assertIn("@prefix", result["data"])
|
||||
|
||||
def test_nt_still_works(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "nt"})
|
||||
self.assertNotIn("error", result, result)
|
||||
self.assertIsInstance(result.get("data"), str)
|
||||
self.assertGreater(len(result["data"]), 0)
|
||||
|
||||
def test_unsupported_format_still_returns_error(self):
|
||||
from semantica_mcp.mcp.tools.export import handle_export_graph
|
||||
result = handle_export_graph({"format": "yaml"})
|
||||
self.assertIn("error", result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -91,7 +91,7 @@ _INIT_REQUEST = _jsonrpc("initialize", 1, {
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMCPStdioFramingContract(unittest.TestCase):
|
||||
"""Run 'python -m semantica_mcp.mcp' exactly as an MCP client would, over a real pipe.
|
||||
"""Run 'python -m mcp' exactly as an MCP client would, over a real pipe.
|
||||
|
||||
Each test sends a complete JSON-RPC session through stdin and asserts that
|
||||
every byte on stdout is valid JSON — catching the exact failure mode from
|
||||
@@ -102,7 +102,7 @@ class TestMCPStdioFramingContract(unittest.TestCase):
|
||||
|
||||
def _run(self, *requests: bytes) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
[sys.executable, "-m", "semantica_mcp.mcp"],
|
||||
[sys.executable, "-m", "mcp"],
|
||||
input=b"".join(requests),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
"""Guard for the aggregate ``all`` extras in pyproject.toml.
|
||||
|
||||
Review of #1508 caught ``parse-pdf`` missing from every ``all`` bundle:
|
||||
``semantica[all]`` installed the package without pdfplumber, so the default
|
||||
PDFParser raised ProcessingError on first use. tomllib is stdlib only from
|
||||
Python 3.11, so the bundle block is parsed textually — the repo still
|
||||
supports 3.9/3.10.
|
||||
"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
PYPROJECT = Path(__file__).resolve().parents[1] / "pyproject.toml"
|
||||
|
||||
|
||||
def _extras_in_all_bundles():
|
||||
"""Collect every extra name referenced inside the ``all = [...]`` block."""
|
||||
text = PYPROJECT.read_text(encoding="utf-8")
|
||||
match = re.search(r"^all = \[\n(.*?)^\]", text, re.MULTILINE | re.DOTALL)
|
||||
assert match, "pyproject.toml must define an aggregate 'all' extra"
|
||||
|
||||
names = set()
|
||||
for bundle in re.findall(r"semantica\[([^\]]+)\]", match.group(1)):
|
||||
names.update(part.strip() for part in bundle.split(","))
|
||||
assert names, "'all' bundles must reference at least one extra"
|
||||
return names
|
||||
|
||||
|
||||
def test_all_bundles_include_parse_pdf():
|
||||
"""An all-features install must include built-in PDF parsing support."""
|
||||
assert "parse-pdf" in _extras_in_all_bundles()
|
||||
|
||||
|
||||
def test_all_bundle_extraction_finds_known_extra():
|
||||
"""Control: the extraction reads the right block (parse-docling is in)."""
|
||||
assert "parse-docling" in _extras_in_all_bundles()
|
||||
@@ -50,11 +50,10 @@ CLOUD_BACKENDS = sorted(_AVAILABILITY_FLAG)
|
||||
# Backends that store locally and need no connection step.
|
||||
_LOCAL_BACKENDS = {"inmemory", "faiss", "sqlite", "pgvector"}
|
||||
|
||||
# The facade dispatches store_vectors() to `add`, `add_vectors`, or
|
||||
# `insert_vectors`. Milvus exposes add_vectors and qdrant insert_vectors, so
|
||||
# both resolve; the remaining two name their write method differently and fall
|
||||
# through to NotImplementedError.
|
||||
_NO_WRITE_DISPATCH = {"pinecone", "weaviate"}
|
||||
# The facade dispatches store_vectors() to `add` or `add_vectors`. Milvus
|
||||
# exposes add_vectors so it already resolves; the other three name their write
|
||||
# method differently and fall through to NotImplementedError.
|
||||
_NO_WRITE_DISPATCH = {"qdrant", "pinecone", "weaviate"}
|
||||
|
||||
|
||||
def _construct(backend):
|
||||
|
||||
@@ -1,377 +0,0 @@
|
||||
"""Tests for QdrantCollection.search_points and QdrantStore.get_stats.
|
||||
|
||||
These cover the qdrant-client >=1.16.0 compatibility fixes:
|
||||
|
||||
1. search_points() must call client.query_points() (not the removed .search()),
|
||||
read ScoredPoints from response.points, and map them to the documented
|
||||
Semantica result shape.
|
||||
|
||||
2. get_stats() must not access vectors_count unconditionally; when the field
|
||||
is absent (qdrant-client >=1.16), it falls back to points_count for
|
||||
single-vector collections, and to None for named/multi-vector collections
|
||||
where the per-point vector count is unknown.
|
||||
|
||||
All tests drive the real implementation against a MagicMock client, following
|
||||
the established pattern in test_qdrant_store.py.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from semantica.utils.exceptions import ProcessingError
|
||||
from semantica.vector_store.qdrant_store import QdrantCollection, QdrantStore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _scored_point(point_id, score, payload=None):
|
||||
"""Build a stand-in for a qdrant_client ScoredPoint."""
|
||||
sp = MagicMock()
|
||||
sp.id = point_id
|
||||
sp.score = score
|
||||
sp.payload = payload
|
||||
return sp
|
||||
|
||||
|
||||
def _query_response(*scored_points):
|
||||
"""Build a stand-in for a qdrant_client QueryResponse."""
|
||||
qr = MagicMock()
|
||||
qr.points = list(scored_points)
|
||||
return qr
|
||||
|
||||
|
||||
def _collection_with_query_response(*scored_points):
|
||||
"""QdrantCollection whose client.query_points() returns the given points."""
|
||||
client = MagicMock()
|
||||
client.query_points.return_value = _query_response(*scored_points)
|
||||
return QdrantCollection(client, "test_collection")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# QdrantCollection.search_points — API call
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_calls_query_points_not_search():
|
||||
"""search_points() must call .query_points(), NOT the removed .search()."""
|
||||
collection = _collection_with_query_response()
|
||||
query = np.array([0.1, 0.2, 0.3, 0.4])
|
||||
|
||||
collection.search_points(query, limit=5)
|
||||
|
||||
collection.client.query_points.assert_called_once()
|
||||
collection.client.search.assert_not_called()
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_passes_correct_arguments():
|
||||
"""query_points() must receive collection_name, query list, limit, and payload flag."""
|
||||
collection = _collection_with_query_response()
|
||||
query = np.array([0.1, 0.2, 0.3, 0.4])
|
||||
|
||||
collection.search_points(query, limit=7)
|
||||
|
||||
_, kwargs = collection.client.query_points.call_args
|
||||
assert kwargs["collection_name"] == "test_collection"
|
||||
assert kwargs["query"] == [0.1, 0.2, 0.3, 0.4]
|
||||
assert kwargs["limit"] == 7
|
||||
assert kwargs["with_payload"] is True
|
||||
assert kwargs["with_vectors"] is False
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_passes_query_filter_through():
|
||||
"""The query_filter argument must be forwarded verbatim to query_points()."""
|
||||
collection = _collection_with_query_response()
|
||||
mock_filter = MagicMock()
|
||||
query = np.array([0.5, 0.6])
|
||||
|
||||
collection.search_points(query, limit=3, query_filter=mock_filter)
|
||||
|
||||
_, kwargs = collection.client.query_points.call_args
|
||||
assert kwargs["query_filter"] is mock_filter
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_passes_none_filter_when_unfiltered():
|
||||
"""query_filter=None must be passed through (not omitted) so the server
|
||||
returns all matching vectors rather than raising a missing-argument error."""
|
||||
collection = _collection_with_query_response()
|
||||
query = np.array([0.1, 0.2])
|
||||
|
||||
collection.search_points(query, limit=5, query_filter=None)
|
||||
|
||||
_, kwargs = collection.client.query_points.call_args
|
||||
assert kwargs["query_filter"] is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# QdrantCollection.search_points — result shape
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_result_shape():
|
||||
"""Each result dict must contain id, score, metadata, vector, distance."""
|
||||
sp = _scored_point(42, 0.8, payload={"tag": "ml"})
|
||||
collection = _collection_with_query_response(sp)
|
||||
query = np.array([0.1, 0.2, 0.3])
|
||||
|
||||
results = collection.search_points(query, limit=1)
|
||||
|
||||
assert len(results) == 1
|
||||
r = results[0]
|
||||
assert set(r.keys()) == {"id", "score", "metadata", "vector", "distance"}
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_maps_id_and_payload():
|
||||
"""id and metadata must come from ScoredPoint.id and ScoredPoint.payload."""
|
||||
sp = _scored_point(99, 0.5, payload={"source": "wiki", "year": 2024})
|
||||
collection = _collection_with_query_response(sp)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
|
||||
assert results[0]["id"] == 99
|
||||
assert results[0]["metadata"] == {"source": "wiki", "year": 2024}
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_null_payload_becomes_empty_dict():
|
||||
"""A ScoredPoint with payload=None must produce metadata={}."""
|
||||
sp = _scored_point(7, 0.9, payload=None)
|
||||
collection = _collection_with_query_response(sp)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
|
||||
assert results[0]["metadata"] == {}
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_vector_and_distance_are_none():
|
||||
"""vector and distance fields must always be None (vectors are not fetched)."""
|
||||
sp = _scored_point(1, 0.7, payload={})
|
||||
collection = _collection_with_query_response(sp)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
|
||||
assert results[0]["vector"] is None
|
||||
assert results[0]["distance"] is None
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_score_normalization_midrange():
|
||||
"""Score=0 must map to exactly 0.5 under the normalization formula."""
|
||||
sp = _scored_point(1, 0.0)
|
||||
collection = _collection_with_query_response(sp)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
|
||||
assert results[0]["score"] == pytest.approx(0.5)
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_score_normalization_positive():
|
||||
"""Positive raw scores must map to (0.5, 1.0) under the normalization formula."""
|
||||
sp = _scored_point(1, 1.0)
|
||||
collection = _collection_with_query_response(sp)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
|
||||
# (1.0/(1+1.0) + 1.0) / 2.0 = (0.5 + 1.0) / 2.0 = 0.75
|
||||
assert results[0]["score"] == pytest.approx(0.75)
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_score_normalization_negative():
|
||||
"""Negative raw scores must map to (0.0, 0.5) under the normalization formula."""
|
||||
sp = _scored_point(1, -1.0)
|
||||
collection = _collection_with_query_response(sp)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
|
||||
# (-1.0/(1+1.0) + 1.0) / 2.0 = (−0.5 + 1.0) / 2.0 = 0.25
|
||||
assert results[0]["score"] == pytest.approx(0.25)
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_multiple_results_preserve_order():
|
||||
"""All ScoredPoints in response.points must appear in the output, in order."""
|
||||
points = [_scored_point(i, 1.0 - i * 0.1) for i in range(5)]
|
||||
collection = _collection_with_query_response(*points)
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=5)
|
||||
|
||||
assert len(results) == 5
|
||||
assert [r["id"] for r in results] == [0, 1, 2, 3, 4]
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_empty_response():
|
||||
"""An empty response.points list must produce an empty result list."""
|
||||
collection = _collection_with_query_response() # zero points
|
||||
|
||||
results = collection.search_points(np.array([0.1, 0.2]), limit=10)
|
||||
|
||||
assert results == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# QdrantCollection.search_points — error handling
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", False)
|
||||
def test_search_points_raises_when_qdrant_unavailable():
|
||||
client = MagicMock()
|
||||
collection = QdrantCollection(client, "test_collection")
|
||||
|
||||
with pytest.raises(ProcessingError):
|
||||
collection.search_points(np.array([0.1, 0.2]), limit=5)
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_search_points_wraps_client_errors_as_processing_error():
|
||||
client = MagicMock()
|
||||
client.query_points.side_effect = RuntimeError("network failure")
|
||||
collection = QdrantCollection(client, "test_collection")
|
||||
|
||||
with pytest.raises(ProcessingError, match="network failure"):
|
||||
collection.search_points(np.array([0.1, 0.2]), limit=5)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# QdrantStore.get_stats — vectors_count compatibility
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _store_with_collection_info(**info_attrs):
|
||||
"""QdrantStore with a mocked client.get_collection() response."""
|
||||
store = QdrantStore()
|
||||
store.client = MagicMock()
|
||||
store.collection = MagicMock()
|
||||
store.collection.collection_name = "test_coll"
|
||||
|
||||
info = MagicMock(spec=list(info_attrs.keys()))
|
||||
for attr, val in info_attrs.items():
|
||||
setattr(info, attr, val)
|
||||
store.client.get_collection.return_value = info
|
||||
return store
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_get_stats_uses_vectors_count_when_present():
|
||||
"""On qdrant-client <1.16, vectors_count exists and must be returned."""
|
||||
store = _store_with_collection_info(
|
||||
points_count=10, vectors_count=10, status="green"
|
||||
)
|
||||
|
||||
stats = store.get_stats()
|
||||
|
||||
assert stats["points_count"] == 10
|
||||
assert stats["vectors_count"] == 10
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_get_stats_uses_points_count_when_vectors_count_absent():
|
||||
"""On qdrant-client >=1.16, vectors_count is absent.
|
||||
For a single unnamed-vector collection (config.params.vectors is a
|
||||
VectorParams instance), points_count is the correct substitute.
|
||||
indexed_vectors_count must NOT be used: it counts only vectors
|
||||
in optimised segments and is 0 for freshly-inserted data."""
|
||||
from qdrant_client.models import VectorParams, Distance
|
||||
store = QdrantStore()
|
||||
store.client = MagicMock()
|
||||
store.collection = MagicMock()
|
||||
store.collection.collection_name = "test_coll"
|
||||
|
||||
info = MagicMock(spec=["points_count", "indexed_vectors_count", "config", "status"])
|
||||
info.points_count = 5
|
||||
info.indexed_vectors_count = 0 # typical for freshly-inserted, unoptimised data
|
||||
info.config.params.vectors = VectorParams(size=4, distance=Distance.COSINE)
|
||||
info.status = "green"
|
||||
store.client.get_collection.return_value = info
|
||||
|
||||
stats = store.get_stats()
|
||||
|
||||
assert stats["points_count"] == 5
|
||||
# Must equal points_count (5), NOT indexed_vectors_count (0)
|
||||
assert stats["vectors_count"] == 5
|
||||
assert stats["vectors_count"] != info.indexed_vectors_count
|
||||
assert stats["status"] == "green"
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_get_stats_vectors_count_equals_points_count_when_vectors_count_absent():
|
||||
"""On qdrant-client >=1.16, vectors_count is absent. For a single unnamed-
|
||||
vector collection the fallback is points_count, so both keys are equal.
|
||||
indexed_vectors_count is intentionally absent from this mock to confirm
|
||||
it is not required by the fallback path."""
|
||||
from qdrant_client.models import VectorParams, Distance
|
||||
store = QdrantStore()
|
||||
store.client = MagicMock()
|
||||
store.collection = MagicMock()
|
||||
store.collection.collection_name = "test_coll"
|
||||
|
||||
info = MagicMock(spec=["points_count", "config", "status"])
|
||||
info.points_count = 7
|
||||
info.config.params.vectors = VectorParams(size=8, distance=Distance.COSINE)
|
||||
info.status = "green"
|
||||
store.client.get_collection.return_value = info
|
||||
|
||||
stats = store.get_stats()
|
||||
|
||||
assert stats["points_count"] == 7
|
||||
assert stats["vectors_count"] == 7
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_get_stats_vectors_count_is_none_for_named_multi_vector_collection():
|
||||
"""When vectors_count is absent and the collection uses named/multi vectors
|
||||
(config.params.vectors is a dict), the total cannot be inferred and
|
||||
vectors_count must be None rather than a misleading points_count value."""
|
||||
from qdrant_client.models import VectorParams, Distance
|
||||
store = QdrantStore()
|
||||
store.client = MagicMock()
|
||||
store.collection = MagicMock()
|
||||
store.collection.collection_name = "test_coll"
|
||||
|
||||
info = MagicMock(spec=["points_count", "config", "status"])
|
||||
info.points_count = 4
|
||||
# Named multi-vector: qdrant-client returns a dict of VectorParams
|
||||
info.config.params.vectors = {
|
||||
"text": VectorParams(size=4, distance=Distance.COSINE),
|
||||
"image": VectorParams(size=8, distance=Distance.DOT),
|
||||
}
|
||||
info.status = "green"
|
||||
store.client.get_collection.return_value = info
|
||||
|
||||
stats = store.get_stats()
|
||||
|
||||
assert stats["points_count"] == 4
|
||||
# vectors_count must be None: total vectors = points * num_named_vectors,
|
||||
# and that multiplier is unknown to the caller.
|
||||
assert stats["vectors_count"] is None
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_get_stats_vectors_count_is_none_when_config_inaccessible():
|
||||
"""If the collection config cannot be read (e.g. an older schema or
|
||||
unexpected server response), vectors_count must fall back to None safely
|
||||
without raising."""
|
||||
store = QdrantStore()
|
||||
store.client = MagicMock()
|
||||
store.collection = MagicMock()
|
||||
store.collection.collection_name = "test_coll"
|
||||
|
||||
# Simulate a CollectionInfo that has no config attribute at all
|
||||
info = MagicMock(spec=["points_count", "status"])
|
||||
info.points_count = 3
|
||||
info.status = "green"
|
||||
store.client.get_collection.return_value = info
|
||||
|
||||
stats = store.get_stats()
|
||||
|
||||
assert stats["points_count"] == 3
|
||||
assert stats["vectors_count"] is None
|
||||
@@ -1,126 +0,0 @@
|
||||
"""Regression tests for the Qdrant write path behind the VectorStore facade.
|
||||
|
||||
Qodo review of #1508 found three bugs in the newly wired qdrant dispatch:
|
||||
stored IDs were swallowed (the upsert status dict was returned instead),
|
||||
mismatched ids/vectors silently truncated the write via zip(), and lazy
|
||||
collection init ignored the documented ``collection_name`` option. These
|
||||
tests pin all three.
|
||||
|
||||
qdrant-client is not installed in this environment, so QDRANT_AVAILABLE is
|
||||
patched and PointStruct is replaced with a plain stand-in, following the
|
||||
pattern in test_vector_store_deepdive.py and test_qdrant_store.py.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from semantica.utils.exceptions import ValidationError
|
||||
from semantica.vector_store import VectorStore
|
||||
from semantica.vector_store.qdrant_store import QdrantStore
|
||||
|
||||
VECTORS = [np.array([1.0, 0.0, 0.0]), np.array([0.0, 1.0, 0.0])]
|
||||
METADATA = [{"type": "a"}, {"type": "b"}]
|
||||
|
||||
|
||||
class _Point:
|
||||
"""Stand-in for qdrant_client PointStruct that keeps the point id."""
|
||||
|
||||
def __init__(self, id, vector=None, payload=None):
|
||||
self.id = id
|
||||
self.vector = vector
|
||||
self.payload = payload
|
||||
|
||||
|
||||
def _qdrant_facade(**config):
|
||||
"""VectorStore built through the real qdrant init path, with the network
|
||||
client and an attached collection replaced by mocks."""
|
||||
store = VectorStore(backend="qdrant", config={"dimension": 3, **config})
|
||||
backend = store._backend_store
|
||||
backend.client = MagicMock()
|
||||
backend.collection = MagicMock()
|
||||
return store
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.PointStruct", _Point)
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_facade_returns_generated_ids_not_upsert_status():
|
||||
"""store_vectors() promises callers the stored vector IDs; the qdrant
|
||||
branch used to leak insert_vectors()' upsert status dict instead."""
|
||||
store = _qdrant_facade()
|
||||
|
||||
ids = store.store_vectors(VECTORS, METADATA)
|
||||
|
||||
assert isinstance(ids, list)
|
||||
assert len(ids) == len(VECTORS)
|
||||
assert all(isinstance(i, str) and i for i in ids)
|
||||
upserted = store._backend_store.collection.upsert_points.call_args[0][0]
|
||||
assert [p.id for p in upserted] == ids
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.PointStruct", _Point)
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_facade_returns_caller_supplied_ids_verbatim():
|
||||
store = _qdrant_facade()
|
||||
|
||||
ids = store.store_vectors(VECTORS, METADATA, ids=["doc-a", "doc-b"])
|
||||
|
||||
assert ids == ["doc-a", "doc-b"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ids", [["doc-a"], ["doc-a", "doc-b", "doc-c"]])
|
||||
@patch("semantica.vector_store.qdrant_store.PointStruct", _Point)
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_facade_rejects_ids_count_mismatch(ids):
|
||||
"""insert_vectors() pairs points with zip(vectors, ids); a mismatched
|
||||
batch must fail loudly instead of silently dropping vectors."""
|
||||
store = _qdrant_facade()
|
||||
|
||||
with pytest.raises(ValidationError, match="must match number of vectors"):
|
||||
store.store_vectors(VECTORS, METADATA, ids=ids)
|
||||
store._backend_store.collection.upsert_points.assert_not_called()
|
||||
|
||||
|
||||
@patch("semantica.vector_store.qdrant_store.PointStruct", _Point)
|
||||
@patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True)
|
||||
def test_backend_insert_vectors_rejects_count_mismatch():
|
||||
"""Direct QdrantStore callers get the same guard as facade callers."""
|
||||
store = QdrantStore()
|
||||
store.client = MagicMock()
|
||||
store.collection = MagicMock()
|
||||
|
||||
with pytest.raises(ValidationError, match="must match number of vectors"):
|
||||
store.insert_vectors(VECTORS, ["only-one"])
|
||||
store.collection.upsert_points.assert_not_called()
|
||||
|
||||
|
||||
def _lazy_collection_name(store):
|
||||
"""Drive _ensure_default_collection and report the name it selected."""
|
||||
store.client = MagicMock()
|
||||
with (
|
||||
patch.object(store, "create_collection") as create,
|
||||
patch.object(store, "get_collection"),
|
||||
):
|
||||
store._ensure_default_collection(3)
|
||||
return create.call_args[0][0]
|
||||
|
||||
|
||||
def test_lazy_collection_uses_documented_collection_name():
|
||||
"""The docs and facade configure qdrant with collection_name=...; lazy
|
||||
init used to look up 'collection' and always fall back to the default."""
|
||||
store = QdrantStore(collection_name="semantica")
|
||||
|
||||
assert _lazy_collection_name(store) == "semantica"
|
||||
|
||||
|
||||
def test_lazy_collection_accepts_legacy_collection_alias():
|
||||
store = QdrantStore(collection="legacy_name")
|
||||
|
||||
assert _lazy_collection_name(store) == "legacy_name"
|
||||
|
||||
|
||||
def test_lazy_collection_defaults_without_config():
|
||||
store = QdrantStore()
|
||||
|
||||
assert _lazy_collection_name(store) == "semantica_default"
|
||||
@@ -112,9 +112,7 @@ class TestQdrantSearchSchema(unittest.TestCase):
|
||||
mock_hit.id = "q_1"
|
||||
mock_hit.score = 0.88
|
||||
mock_hit.payload = {"category": "x"}
|
||||
# search_points prefers the modern query_points API; a MagicMock
|
||||
# exposes it, so the response must carry the hits in .points.
|
||||
mock_client.query_points.return_value = MagicMock(points=[mock_hit])
|
||||
mock_client.search.return_value = [mock_hit]
|
||||
|
||||
coll = QdrantCollection(mock_client, "test_col")
|
||||
results = coll.search_points(np.array([0.1, 0.2]), limit=1)
|
||||
@@ -134,9 +132,7 @@ class TestQdrantSearchSchema(unittest.TestCase):
|
||||
mock_hit_high = MagicMock(id="q_hi", score=50.0, payload={})
|
||||
mock_hit_mid = MagicMock(id="q_mid", score=2.0, payload={})
|
||||
mock_hit_low = MagicMock(id="q_lo", score=1.0, payload={})
|
||||
mock_client.query_points.return_value = MagicMock(
|
||||
points=[mock_hit_high, mock_hit_mid, mock_hit_low]
|
||||
)
|
||||
mock_client.search.return_value = [mock_hit_high, mock_hit_mid, mock_hit_low]
|
||||
|
||||
coll = QdrantCollection(mock_client, "test_col")
|
||||
results = coll.search_points(np.array([0.1, 0.2]), limit=3)
|
||||
|
||||
@@ -209,12 +209,12 @@ class TestVectorStoreDeepDive(unittest.TestCase):
|
||||
mock_client = MagicMock()
|
||||
mock_qdrant_cls.return_value = mock_client
|
||||
|
||||
# Mock search response (modern query_points API: hits in .points)
|
||||
# Mock search response
|
||||
mock_hit = MagicMock()
|
||||
mock_hit.id = "vec_1"
|
||||
mock_hit.score = 0.9
|
||||
mock_hit.payload = {"type": "a"}
|
||||
mock_client.query_points.return_value = MagicMock(points=[mock_hit])
|
||||
mock_client.search.return_value = [mock_hit]
|
||||
|
||||
store = QdrantStore(url="http://localhost:6333")
|
||||
|
||||
|
||||
@@ -38,13 +38,14 @@ class TestOptionalDependencies(unittest.TestCase):
|
||||
with import_without(
|
||||
"semantica.visualization.embedding_visualizer", "umap"
|
||||
) as module:
|
||||
with plotly_doubles(module):
|
||||
with plotly_doubles(module), patch.object(module, "PCA") as mock_pca_class:
|
||||
mock_pca_class.return_value.fit_transform.return_value = np.zeros((4, 2))
|
||||
|
||||
viz = module.EmbeddingVisualizer()
|
||||
embeddings = np.array([[0, 1, 2], [1, 0, 3], [0, 0, 0], [1, 1, 1]])
|
||||
with self.assertRaises(module.ProcessingError) as cm:
|
||||
viz.visualize_2d_projection(embeddings, method="umap")
|
||||
self.assertIn("UMAP is required", str(cm.exception))
|
||||
self.assertIn("semantica[viz]", str(cm.exception))
|
||||
viz.visualize_2d_projection(embeddings, method="umap")
|
||||
|
||||
mock_pca_class.assert_called()
|
||||
|
||||
def test_ontology_visualizer_without_graphviz(self):
|
||||
"""Test OntologyVisualizer behavior when graphviz is missing."""
|
||||
|
||||
Reference in New Issue
Block a user