Merge branch 'main' into codex/harden-markdown-import-symlinks

This commit is contained in:
Mohd Kaif
2026-08-23 16:55:43 +05:30
committed by GitHub
19 changed files with 841 additions and 73 deletions
+59 -21
View File
@@ -8,7 +8,7 @@ icon: "shield-check"
SHACL (Shapes Constraint Language) is a standard for validating graph-based data. While an ontology defines the conceptual *schema* (the "what" exists in your domain), SHACL defines the structural *rules and constraints* (the "how" it should be structured).
In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and `_run_pyshacl` evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated.
In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and the public `run_shacl_validation` function evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated. The historical `_run_pyshacl` name remains available as a compatibility alias.
## Why Use SHACL Validation?
@@ -55,7 +55,7 @@ Let's look at a simple, universally understood example: ensuring every `Employee
```python
from semantica.context import ContextGraph
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
from semantica.ontology.ontology_validator import _run_pyshacl
from semantica.ontology import run_shacl_validation
# 1. Prepare your data graph
graph = ContextGraph()
@@ -95,7 +95,7 @@ data_ttl = """
"""
# 5. Run Validation
report = _run_pyshacl(data_ttl, shacl_ttl)
report = run_shacl_validation(data_ttl, shacl_ttl)
# 6. Analyze the Report
print(f"Graph conforms: {report.conforms}")
@@ -265,10 +265,10 @@ cve_id_shape = NodeShape(
## Step 4 — Run validation and read the report
Serialize the graph to RDF, then run `_run_pyshacl` against the shapes.
Serialize the graph to RDF, then run `run_shacl_validation` against the shapes.
```python
from semantica.ontology.ontology_validator import _run_pyshacl
from semantica.ontology import run_shacl_validation
# Prepare your RDF data string (since export_rdf primarily exports structural metadata,
# you typically serialize your custom data graph to Turtle using rdflib or similar).
@@ -281,7 +281,7 @@ data_ttl = """
"""
# Run SHACL validation
report = _run_pyshacl(
report = run_shacl_validation(
data_ttl,
shacl_ttl,
data_graph_format="turtle",
@@ -366,8 +366,8 @@ print(f"Malware nodes missing 'family': {len(missing_family)}")
# e.g. graph.update_node(node_id, {"family": "UNKNOWN — requires triage"})
# After remediation, re-run validation to confirm the fix
# (re-export the patched graph to Turtle first, then call _run_pyshacl again)
report2 = _run_pyshacl(patched_data_ttl, shacl_ttl)
# (re-export the patched graph to Turtle first, then call run_shacl_validation again)
report2 = run_shacl_validation(patched_data_ttl, shacl_ttl)
print(f"Violations after remediation: {report2.violation_count}")
# Violations after remediation: 0
```
@@ -377,10 +377,49 @@ print(f"Violations after remediation: {report2.violation_count}")
## Common Pitfalls
- **Assuming the ontology automatically enforces data quality**: `SHACLGenerator` generates shapes based on what it observes in the data. If your data is missing a field, the generator won't know it was mandatory unless you explicitly inject the constraint (as shown in Step 3).
- **Passing `ContextGraph` directly to SHACL validators**: The `_run_pyshacl` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object.
- **Passing `ContextGraph` directly to SHACL validators**: The `run_shacl_validation` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object.
- **Forgetting RDF serialization**: You must serialize your graph (often via a temporary file using `export_rdf`) before validating it.
- **Treating validation as a one-time step**: Validation should be integrated as an automated step in your CI/CD pipeline or data ingestion flow, acting as a recurring gatekeeper rather than a one-off script.
- **Ignoring validation reports**: A graph that does not conform must be remediated. Failing to review the `violation_count` and address the issues negates the purpose of SHACL validation.
- **Validating `sh:class`/`sh:node` range checks on a property that declares `rdfs:range` with RDFS entailment on**: RDFS is an entailment rule, not a constraint. When pyshacl runs with `inference="rdfs"`, it infers the range class onto every object of the property, so class-based constraints on that property can never fail — the report says `conforms: True` on data that does not conform:
```python
from pyshacl import validate
from rdflib import Graph
data = Graph()
data.parse(
data="""
@prefix ex: <https://example.org/ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
ex:contains rdfs:domain ex:Container ; rdfs:range ex:Item .
ex:box a ex:Container ; ex:contains ex:notAnItem .
ex:notAnItem a ex:Fish .
""",
format="turtle",
)
shapes = Graph()
shapes.parse(
data="""
@prefix ex: <https://example.org/ns#> .
@prefix sh: <http://www.w3.org/ns/shacl#> .
ex:ContainerShape a sh:NodeShape ;
sh:targetClass ex:Container ;
sh:property [ sh:path ex:contains ; sh:class ex:Item ] .
""",
format="turtle",
)
for inference in ("none", "rdfs"):
conforms, _, _ = validate(data, shacl_graph=shapes, inference=inference)
print(inference, conforms)
# none False <- correct: notAnItem is a Fish, not an Item
# rdfs True <- the entailment manufactured the type
```
Mitigations: prefer not to declare `rdfs:range` on properties you intend to constrain with `sh:class`; when class membership is the thing under test, run validation without RDFS entailment (`inference="none"`); or express the check as a constraint the entailment cannot satisfy (for example a literal property constraint). Note the trade-off: with entailment off, `sh:targetClass` no longer reaches subclasses, so subclass hierarchies need explicit typing or inference-aware target selection. Semantica's own `run_shacl_validation` wrapper already calls pyshacl with `inference="none"`, so this pitfall only bites when calling `pyshacl.validate` directly with entailment enabled.
- **Trusting `conforms: True` without checking the inference mode**: an inference-enabled run can hide the exact violations the shapes were written to catch (see above). Record which inference mode validation ran under alongside the result, and re-run shape sets that contain `sh:class`/`sh:node` with entailment off before treating a pass as authoritative.
---
@@ -396,7 +435,7 @@ A DoD CTI team enforces STIX-compatible constraints on a threat graph before sha
from semantica.context import AgentContext, ContextGraph
from semantica.vector_store import VectorStore
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
from semantica.ontology.ontology_validator import _run_pyshacl
from semantica.ontology import run_shacl_validation
graph = ContextGraph()
ctx = AgentContext(
@@ -448,7 +487,7 @@ data_ttl = """
<http://example.org/hammertoss> a ex:Malware .
"""
report = _run_pyshacl(data_ttl, shacl_ttl)
report = run_shacl_validation(data_ttl, shacl_ttl)
print(f"CTI graph conforms : {report.conforms}")
print(f"Violations : {report.violation_count}")
print(f"Warnings : {report.warning_count}")
@@ -469,7 +508,7 @@ A SOC team validates zero-trust policy nodes before publishing them to the polic
```python
from semantica.context import ContextGraph
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
from semantica.ontology.ontology_validator import _run_pyshacl
from semantica.ontology import run_shacl_validation
graph = ContextGraph()
graph.add_node("policy-001", "Policy", "MFA Required for Tier-1 Resources",
@@ -516,7 +555,7 @@ data_ttl = """
<http://example.org/policy-002> a ex:Policy .
"""
report = _run_pyshacl(data_ttl, shacl_ttl)
report = run_shacl_validation(data_ttl, shacl_ttl)
print(f"Policy graph conforms: {report.conforms}")
# Policy graph conforms: False
@@ -534,7 +573,7 @@ A clinical informatics team validates trial ontology nodes before loading them i
```python
from semantica.ontology import LLMOntologyGenerator, SHACLGenerator, PropertyShape
from semantica.ontology.ontology_validator import _run_pyshacl
from semantica.ontology import run_shacl_validation
from semantica.export import export_rdf
import tempfile, os
@@ -586,7 +625,7 @@ with open(tmp.name) as f:
data_ttl = f.read()
os.unlink(tmp.name)
report = _run_pyshacl(data_ttl, shacl_ttl)
report = run_shacl_validation(data_ttl, shacl_ttl)
print(f"Trial data conforms: {report.conforms}")
print(f"Warnings : {report.warning_count}")
```
@@ -600,7 +639,7 @@ A credit risk team validates every `LoanApplication` node against Basel III CRE2
```python
from semantica.context import ContextGraph
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
from semantica.ontology.ontology_validator import _run_pyshacl
from semantica.ontology import run_shacl_validation
graph = ContextGraph()
graph.add_node("loan-001", "LoanApplication", "Prime mortgage APP-2025-88421",
@@ -645,7 +684,7 @@ data_ttl = """
ex:ltv "0.65" .
"""
report = _run_pyshacl(data_ttl, shacl_ttl)
report = run_shacl_validation(data_ttl, shacl_ttl)
print(f"Loan portfolio conforms: {report.conforms}")
# Loan portfolio conforms: False
@@ -675,14 +714,14 @@ Call this function as a pre-publish gate; exit code 1 blocks the pipeline.
```python
import sys
from semantica.ontology import OntologyGenerator, SHACLGenerator
from semantica.ontology.ontology_validator import _run_pyshacl
from semantica.ontology import run_shacl_validation
def validate_before_publish(data_graph_str: str, ontology: dict) -> None:
shacl_gen = SHACLGenerator(base_uri="https://example.org/shapes/")
shacl_graph = shacl_gen.generate(ontology)
shacl_ttl = shacl_gen.serialize(shacl_graph, format="turtle")
report = _run_pyshacl(data_graph_str, shacl_ttl)
report = run_shacl_validation(data_graph_str, shacl_ttl)
if not report.conforms:
print(f"Graph validation FAILED — {report.violation_count} violation(s)")
@@ -700,7 +739,6 @@ def validate_before_publish(data_graph_str: str, ontology: dict) -> None:
- [Ontology Management](ontology) — generate the OWL ontology that SHACL shapes are derived from
- [Reasoning & Rules](reasoning) — complement SHACL structural constraints with logical inference rules
- [Export & Serialization](export) — serialize graph data to Turtle/RDF/XML for `_run_pyshacl` input
- [Export & Serialization](export) — serialize graph data to Turtle/RDF/XML for `run_shacl_validation` input
- [Conflict Resolution](conflict-resolution) — detect and resolve data conflicts before SHACL validation
- [Change Management](change-management) — version-gate SHACL shapes alongside ontology versions
+2 -1
View File
@@ -85,7 +85,8 @@ dependencies = [
"loguru>=0.7.3",
"structlog>=22.1.0",
"gensim>=4.4.0",
"httpx<0.29.0"
"httpx<0.29.0",
"pyarrow>=14.0.0"
]
[project.urls]
+9 -9
View File
@@ -6,9 +6,9 @@ accelerate==1.14.0 \
# via
# docling-ibm-models
# docling-slim
agno==2.8.7 \
--hash=sha256:6a2763eb469163f7b79ab1da6ca2f22d8619f6b9d614574f975d9c12bb4323ea \
--hash=sha256:d49396a2062ee6994ca82695b9bd1e1b95667fec432c544afa38133e564bf090
agno==2.9.0 \
--hash=sha256:7777674b3931b341fad4fcf02a61b185a08588c509101348facf87feb2144c0c \
--hash=sha256:7d9c134703e3c2798023cd57dcb9caa8e1174f6914813f9b130becfc3521a46f
# via semantica (pyproject.toml)
agnoctl==0.1.3 \
--hash=sha256:6fce1d2482b1f2e0a3d14b0a7c12fbd49d8df4f0bf0a4fd9fd91753cbff5efdc \
@@ -403,9 +403,9 @@ boto3==1.43.69 \
--hash=sha256:4eb494d05b2bd08a7eee61b8ac4c34745c99e9bbce435c91f8d15d372dd8c2db \
--hash=sha256:76297a0b415849c63575ae08a4f1661b2dc8ee0100f104b86f98aa69b47fa2c7
# via semantica (pyproject.toml)
botocore==1.43.69 \
--hash=sha256:5caa46b740d9a886137146ffbb69edb691f702bfe74c64e85621947ae00181fd \
--hash=sha256:b1f0e01c53d6b84ee9c184ebf3636c3b3aef85e0ae8498c74afb8734ff224f87
botocore==1.43.73 \
--hash=sha256:068433028e011ccbeab1dd7c46b1090c24e378397693c66e67ca571176498daa \
--hash=sha256:0fa1e63c24b3531be3e1bc1687a88b3be9e63a430153f24edd93efc162bb1c51
# via
# boto3
# s3transfer
@@ -1731,9 +1731,9 @@ google-crc32c==1.8.0 \
# via
# google-cloud-storage
# google-resumable-media
google-genai==2.17.0 \
--hash=sha256:6b640a2390c82b4a240873eddb9f518c6d2c33244b2de16ed3d14526a6da7f57 \
--hash=sha256:a4835563c60aee646c9c4b261c507aa4a624710d25017012d20dc65abf3d9a54
google-genai==2.18.1 \
--hash=sha256:36a5949233e64a60f6cc4521bff7a76b7c569d0aa227bbe9fa642213b8a3a3b2 \
--hash=sha256:a1e2be75c16234adc6641afd1ad4dd44218c9eec005d938bdc428585a048918a
# via semantica (pyproject.toml)
google-resumable-media==2.10.1 \
--hash=sha256:224975032ddb73f7ed9e2f0f4cc08ed1b06874c52d48cc8533e3eb72980b21a0 \
+104 -2
View File
@@ -773,10 +773,22 @@ def changelog(cli_ctx: CLIContext, local_json: bool) -> None:
_run_with_error_handling(_action)
class _DeepEmbeddingFailure(Exception):
"""A deep-probe failure from doctor's embedding checks.
Marks failures that happened AFTER the backend imported cleanly — model
load, probe, or runtime problems — so the check's hint can point at the
real remediation instead of `pip install`.
"""
@main.command()
@click.option("--json", "local_json", is_flag=True, default=False)
@click.option("--deep-embeddings", "deep_embeddings", is_flag=True, default=False,
help="Also instantiate the local embedding backends and embed a probe "
"text (catches backends that import cleanly but cannot load).")
@click.pass_obj
def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None:
"""Run a health check on all Semantica components and backends."""
import importlib.metadata
cli_ctx = _require_ctx(cli_ctx)
@@ -787,6 +799,16 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
try:
note = fn()
return label, "ok", note, None
except _DeepEmbeddingFailure as exc:
# A deep-probe failure means the package IMPORTED fine: the pip
# hint would be the wrong remediation for what is actually a
# runtime/model-load problem (broken torch, failed model
# download, missing shared libs).
return label, "fail", str(exc), (
"runtime/model-load failure — reinstalling the package usually "
"does not help; check the warnings above (torch install, model "
"download, disk space)"
)
except Exception as exc:
return label, "fail", str(exc), hint
@@ -827,6 +849,50 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
return f"{backend} importable"
checks.append(_check("Vector store", _vector, hint="pip install semantica[vectorstore-…]"))
# Embedding backends (#994): `doctor` used to report all green while
# every local embedding backend was non-functional — import success
# says nothing about model loading. Default checks stay cheap
# (import + version); --deep-embeddings (or
# SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates the backend through
# TextEmbedder and embeds a probe, which is the only level that
# catches a backend that imports cleanly but cannot actually load.
deep = deep_embeddings or os.environ.get("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", "").strip().lower() in ("1", "true", "yes", "on")
def _embedding_backend(method: str) -> str:
if method == "sentence_transformers":
import sentence_transformers # noqa: F401
note = f"importable ({importlib.metadata.version('sentence-transformers')})"
else:
import fastembed # noqa: F401
note = f"importable ({importlib.metadata.version('fastembed')})"
if not deep:
return note
try:
from .embeddings import TextEmbedder
embedder = TextEmbedder(method=method)
if embedder.model is None and embedder.fastembed_model is None:
raise RuntimeError(
"model failed to load — the hash fallback is active "
"(see warnings above); embedding quality is degraded"
)
probe = embedder.embed_text("semantica doctor embedding probe")
except _DeepEmbeddingFailure:
raise
except Exception as exc:
raise _DeepEmbeddingFailure(str(exc)) from exc
return f"{note}; deep probe ok ({len(probe)}-dim)"
checks.append(_check(
"Embeddings (sentence-transformers)",
lambda: _embedding_backend("sentence_transformers"),
hint="pip install sentence-transformers",
))
checks.append(_check(
"Embeddings (fastembed)",
lambda: _embedding_backend("fastembed"),
hint="pip install fastembed",
))
# LLM provider keys
for provider, var in [("OpenAI", "OPENAI_API_KEY"), ("Anthropic", "ANTHROPIC_API_KEY"),
("Groq", "GROQ_API_KEY")]:
@@ -1708,7 +1774,43 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str,
except ImportError as exc:
raise click.ClickException(f"Embeddings module not available: {exc}") from exc
if output:
Path(output).write_text(json.dumps(result, default=str), encoding="utf-8")
output_path = Path(output)
suffix = output_path.suffix.lower()
try:
import numpy as np
import pandas as pd
arr = np.asarray(result)
if arr.ndim == 1:
arr = arr[np.newaxis, :]
if arr.ndim != 2:
raise click.ClickException(
f"embed generate --output expects a 1-D or 2-D array, "
f"got {arr.ndim}-D (shape {arr.shape})"
)
rows = [list(row) for row in arr]
if suffix == ".parquet":
# Schema: single 'embedding' column (list[float] per row).
# embed index detects vector columns via
# isinstance(df[c].iloc[0], (list, np.ndarray)).
df = pd.DataFrame({"embedding": rows})
df.to_parquet(output_path, index=False)
elif suffix in (".json", ".jsonl"):
df = pd.DataFrame({"embedding": rows})
df.to_json(
output_path,
orient="records",
lines=(suffix == ".jsonl"),
)
else:
raise click.ClickException(
f"Unsupported output format '{suffix}'. "
"Use .parquet, .json, or .jsonl"
)
except ImportError as exc:
raise click.ClickException(
f"Missing dependency for --output: {exc}. "
"Install pyarrow with: pip install pyarrow"
) from exc
_ok(cli_ctx, f"Wrote {output}")
elif _is_json(cli_ctx, local_json):
_jecho(result if isinstance(result, dict) else {"status": "ok"})
+46 -11
View File
@@ -438,6 +438,23 @@ _ATTRS_MISSING = object()
#: entities and timestamps.
_CAUSAL_EDGE_TYPES = ("CAUSED", "INFLUENCED", "PRECEDENT_FOR")
# Causal edges circulate under two vocabularies: this module's canonical
# spellings above, and the present-tense spellings CausalChainAnalyzer also
# accepts ("causes", "influences", "leads_to", "supports"). The present-tense
# forms normalize onto the canonical types for storage; traversal accepts
# both vocabularies so an edge recorded either way is never invisible.
_CAUSAL_EDGE_ALIASES = {
"CAUSES": "CAUSED",
"CAUSED": "CAUSED",
"INFLUENCES": "INFLUENCED",
"INFLUENCED": "INFLUENCED",
"PRECEDES": "PRECEDENT_FOR",
"PRECEDENT_FOR": "PRECEDENT_FOR",
}
_CAUSAL_TRAVERSAL_TYPES = frozenset(_CAUSAL_EDGE_ALIASES) | {
"LEADS_TO", "LEAD_TO", "SUPPORTS", "SUPPORT",
}
class ContextGraph:
"""
@@ -2745,9 +2762,15 @@ class ContextGraph:
target_decision_id: Target decision ID
relationship_type: Type of relationship (CAUSED, INFLUENCED, PRECEDENT_FOR)
"""
valid_types = ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]
if relationship_type not in valid_types:
raise ValueError(f"Relationship type must be one of: {valid_types}")
# Normalize so callers may use either vocabulary's spelling
# ("causes" from CausalChainAnalyzer, or "CAUSED" from this module's
# canonical constant); the stored form is always canonical. Invalid
# inputs keep raising ValueError rather than AttributeError.
if not isinstance(relationship_type, str):
raise ValueError(f"Relationship type must be one of: {_CAUSAL_EDGE_TYPES}")
relationship_type = _CAUSAL_EDGE_ALIASES.get(relationship_type.strip().upper())
if relationship_type is None:
raise ValueError(f"Relationship type must be one of: {_CAUSAL_EDGE_TYPES}")
# Check if decisions exist - if not, skip adding relationship
if source_decision_id not in self.nodes or target_decision_id not in self.nodes:
@@ -2839,11 +2862,11 @@ class ContextGraph:
# Find connected decisions
for edge in self.edges:
if direction == "upstream":
if edge.target_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]:
if edge.target_id == current_id and edge.edge_type.upper() in _CAUSAL_TRAVERSAL_TYPES:
if edge.source_id not in visited and depth < max_depth:
queue.append((edge.source_id, depth + 1))
else: # downstream
if edge.source_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]:
if edge.source_id == current_id and edge.edge_type.upper() in _CAUSAL_TRAVERSAL_TYPES:
if edge.target_id not in visited and depth < max_depth:
queue.append((edge.target_id, depth + 1))
@@ -2866,10 +2889,13 @@ class ContextGraph:
Returns:
List of precedent decisions
"""
# Find decisions connected via PRECEDENT_FOR relationships
# Find decisions connected via PRECEDENT_FOR relationships, accepting
# the analyzer vocabulary's "precedes" spelling as well (issue #1184).
precedent_ids = []
for edge in self.edges:
if edge.target_id == decision_id and edge.edge_type == "PRECEDENT_FOR":
if edge.target_id == decision_id and edge.edge_type.upper() in {
"PRECEDENT_FOR", "PRECEDES",
}:
precedent_ids.append(edge.source_id)
# Convert to Decision objects
@@ -3430,8 +3456,13 @@ class ContextGraph:
# Explicit causal relationships recorded via add_causal_relationship() are
# ground truth and always count as direct influence, in either direction.
for edge_type in _CAUSAL_EDGE_TYPES:
for edge in self.edge_type_index.get(edge_type, []):
# The index is keyed by the raw edge_type string ("causes" and "CAUSED"
# are separate keys), so filter by normalized type instead of iterating
# a fixed spelling list.
for edge_type, edges in self.edge_type_index.items():
if edge_type.upper() not in _CAUSAL_TRAVERSAL_TYPES:
continue
for edge in edges:
if edge.source_id == decision_id and edge.target_id in self._decisions:
direct_influence.add(edge.target_id)
elif edge.target_id == decision_id and edge.source_id in self._decisions:
@@ -3577,8 +3608,12 @@ class ContextGraph:
# record_decision() (e.g. a graph restored via from_dict), so only
# causes with a known decision record are kept.
incoming_causal_edges = defaultdict(list)
for edge_type in _CAUSAL_EDGE_TYPES:
for edge in self.edge_type_index.get(edge_type, []):
# The index is keyed by the raw edge_type string ("causes" and
# "CAUSED" are separate keys), so filter by normalized type.
for edge_type, edges in self.edge_type_index.items():
if edge_type.upper() not in _CAUSAL_TRAVERSAL_TYPES:
continue
for edge in edges:
if edge.source_id in self._decisions:
incoming_causal_edges[edge.target_id].append(edge)
@@ -69,6 +69,15 @@ class EmbeddingGeneratorWithProvenance:
return embeddings
def __getattr__(self, name):
# __getattr__ only runs when normal lookup fails. Accessing
# self._generator by attribute syntax HERE would re-enter
# __getattr__ for ever when _generator itself is missing — the shape
# pickle/copy protocol probes hit when __init__ never completed
# (#994's RecursionError family). Fail fast on private probes.
if name.startswith("_"):
raise AttributeError(
f"{type(self).__name__!r} object has no attribute {name!r}"
)
return getattr(self._generator, name)
+8 -8
View File
@@ -117,9 +117,9 @@ def generate_embeddings(
>>> emb = generate_embeddings("Hello world", method="default")
>>> embs = generate_embeddings(["text1", "text2"], method="text")
"""
# Check for custom method in registry
# Check for custom method in registry, skip self-reference
custom_method = method_registry.get("generation", method)
if custom_method:
if custom_method and custom_method is not generate_embeddings:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, data, data_type=data_type, fallback_on_custom_error=fallback, **kwargs
@@ -165,9 +165,9 @@ def embed_text(
>>> emb = embed_text("Hello world", method="sentence_transformers")
>>> embs = embed_text(["text1", "text2"], method="sentence_transformers")
"""
# Check for custom method in registry
# Check for custom method in registry, skip self-reference
custom_method = method_registry.get("text", method)
if custom_method:
if custom_method and custom_method is not embed_text:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
@@ -225,9 +225,9 @@ def calculate_similarity(
>>> similarity = calculate_similarity(emb1, emb2, method="cosine")
>>> print(f"Similarity: {similarity:.3f}")
"""
# Check for custom method in registry
# Check for custom method in registry, skip self-reference
custom_method = method_registry.get("similarity", method)
if custom_method:
if custom_method and custom_method is not calculate_similarity:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, embedding1, embedding2, fallback_on_custom_error=fallback, **kwargs
@@ -272,9 +272,9 @@ def pool_embeddings(
>>> pooled = pool_embeddings(embeddings, method="mean")
>>> attention_pooled = pool_embeddings(embeddings, method="attention")
"""
# Check for custom method in registry
# Check for custom method in registry, skip self-reference
custom_method = method_registry.get("pooling", method)
if custom_method:
if custom_method and custom_method is not pool_embeddings:
fallback = kwargs.pop("fallback_on_custom_error", False)
result = call_custom_method(
logger, method, custom_method, embeddings, fallback_on_custom_error=fallback, **kwargs
+52 -8
View File
@@ -28,12 +28,35 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.helpers import ensure_directory, utc_now_iso, write_json_file
from ..utils.helpers import ensure_directory, hash_data, utc_now_iso, write_json_file
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri
def _content_iri(prefix: str, payload: Any) -> str:
"""Mint a document IRI from what was exported, not when.
Minting from ``utc_now_iso()`` gave every export of the same graph a new
identity a few microseconds apart, so re-exporting an unchanged graph was
never idempotent and merging exports duplicated every node (#1147). This
mirrors ``mint_entity_iri`` (#1109): identical content hashes to the same
IRI, and any change to the content changes it too. ``default=str`` keeps
the hash defined for values ``json.dumps`` would otherwise reject, such as
``datetime`` objects a caller may have left in the graph.
Args:
prefix: IRI prefix the digest is appended to
payload: JSON-serializable value whose content determines the digest
Returns:
A stable IRI of the form ``{prefix}{16-hex-char digest}``
"""
canonical = json.dumps(payload, sort_keys=True, default=str)
digest = hash_data(canonical)[:16]
return f"{prefix}{digest}"
def _is_jsonld_document(data: Dict[str, Any]) -> bool:
"""
Report whether a dictionary is already a JSON-LD document.
@@ -230,7 +253,10 @@ class JSONExporter:
- statistics: Statistics dictionary (optional)
file_path: Output JSON file path
format: Export format - 'json' or 'json-ld' (default: self.format)
**options: Additional options passed to conversion methods
**options: Additional options passed to conversion methods:
- graph_uri: Caller-supplied IRI for the graph node when
format='json-ld', overriding the default content-derived
IRI (see #1147)
Example:
>>> kg = {
@@ -401,7 +427,9 @@ class JSONExporter:
data: Data to convert (dict, list, or any value)
include_metadata: Whether to include metadata (default: True)
include_provenance: Whether to include provenance (default: True)
**options: Additional options passed to knowledge graph conversion
**options: Additional options passed to knowledge graph conversion:
- document_uri: Caller-supplied IRI for the document node,
overriding the default content-derived IRI (see #1147)
Returns:
Dictionary in JSON-LD format with @context, @graph/@value, and metadata
@@ -451,13 +479,17 @@ class JSONExporter:
# Add metadata and provenance if requested
if include_metadata:
self._attach_document_metadata(jsonld, include_provenance)
self._attach_document_metadata(
jsonld, include_provenance, options.get("document_uri")
)
return jsonld
@staticmethod
def _attach_document_metadata(
jsonld: Dict[str, Any], include_provenance: bool
jsonld: Dict[str, Any],
include_provenance: bool,
document_uri: Optional[str] = None,
) -> None:
"""
Attach the export's own metadata without naming the graph.
@@ -473,6 +505,9 @@ class JSONExporter:
Args:
jsonld: Document being built, modified in place
include_provenance: Whether to record how and when it was exported
document_uri: Caller-supplied IRI for the document node. Falls back
to a content-derived IRI (#1147) so re-exporting unchanged data
is idempotent instead of minting a new identity every time.
"""
# A caller may hand us a document that is deliberately a named graph.
# That name is theirs to keep, but our own statements must not end up
@@ -483,7 +518,10 @@ class JSONExporter:
# Do not overwrite an identifier the payload already carries: the
# knowledge-graph conversion names its own document node.
if "@id" not in jsonld or payload_is_named_graph:
document["@id"] = f"https://semantica.dev/data/{utc_now_iso()}"
content = {key: value for key, value in jsonld.items() if key != "@context"}
document["@id"] = document_uri or _content_iri(
"https://semantica.dev/data/", content
)
if include_provenance:
document["semantica:exportedAt"] = utc_now_iso()
document["semantica:format"] = "json-ld"
@@ -553,7 +591,9 @@ class JSONExporter:
- entities: List of entity dictionaries
- relationships: List of relationship dictionaries
- metadata: Metadata dictionary (optional)
**options: Additional options (unused)
**options: Additional options:
- graph_uri: Caller-supplied IRI for the graph node,
overriding the default content-derived IRI (see #1147)
Returns:
Dictionary in JSON-LD format with @context, @id, @type, and graph data
@@ -566,7 +606,11 @@ class JSONExporter:
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
},
"@id": f"https://semantica.dev/graph/{utc_now_iso()}",
# Minted from the graph's own content rather than the wall clock
# (#1147): re-exporting an unchanged graph must produce the same
# subject, or merging repeated exports duplicates every node.
"@id": options.get("graph_uri")
or _content_iri("https://semantica.dev/graph/", kg),
"@type": "semantica:KnowledgeGraph",
}
+2
View File
@@ -159,6 +159,7 @@ from .ontology_validator import (
SHACLValidationReport,
SHACLViolation,
ValidationResult,
run_shacl_validation,
validate_ontology,
)
from .owl_generator import OWLGenerator
@@ -192,6 +193,7 @@ __all__ = [
"PropertyShape",
"SHACLValidationReport",
"SHACLViolation",
"run_shacl_validation",
# OWL/RDF generation
"OWLGenerator",
# Requirements and competency questions
+17 -2
View File
@@ -145,14 +145,14 @@ class SHACLValidationReport:
}
def _run_pyshacl(
def run_shacl_validation(
data_graph_str: str,
shacl_str: str,
data_graph_format: str = "turtle",
shacl_format: str = "turtle",
) -> SHACLValidationReport:
"""
Run pyshacl validation and return a structured SHACLValidationReport.
Run pySHACL validation and return a structured SHACLValidationReport.
Args:
data_graph_str: Serialized data graph string.
@@ -272,6 +272,21 @@ def _run_pyshacl(
raw_report=results_text,
)
def _run_pyshacl(
data_graph_str: str,
shacl_str: str,
data_graph_format: str = "turtle",
shacl_format: str = "turtle",
) -> SHACLValidationReport:
"""Backward-compatible alias for :func:`run_shacl_validation`."""
return run_shacl_validation(
data_graph_str,
shacl_str,
data_graph_format=data_graph_format,
shacl_format=shacl_format,
)
@dataclass
class ValidationResult:
"""Result of an ontology validation operation."""
+4 -2
View File
@@ -779,9 +779,11 @@ def extract_entities_huggingface(
"""
loader = HuggingFaceModelLoader(device=device)
# Pass kwargs (like aggregation_strategy) to load_ner_model
model_obj = loader.load_ner_model(model, **kwargs)
loader_kwargs = {
key: value for key, value in kwargs.items() if key != "huggingface_model"
}
model_obj = loader.load_ner_model(model, **loader_kwargs)
results = loader.extract_entities(model_obj, text)
entities = []
# Check if manual aggregation is needed (raw IOB tags detected)
+5 -1
View File
@@ -297,7 +297,11 @@ def create_index(
config = vector_store_config.get_all()
backend = config.get("default_backend", "faiss")
dimension = config.get("dimension", 768)
indexer = VectorIndexer(backend=backend, dimension=dimension, **config)
# backend/dimension are already passed explicitly; drop them from the
# forwarded config so VectorIndexer(..., **remaining_config) doesn't
# receive duplicate keyword arguments.
remaining_config = {k: v for k, v in config.items() if k not in ("default_backend", "dimension")}
indexer = VectorIndexer(backend=backend, dimension=dimension, **remaining_config)
return indexer.create_index(vectors, ids, **options)
@@ -7,6 +7,8 @@ extraction found nothing, the chain came back empty even though an explicit
``CAUSED`` edge was stored in the graph.
"""
import pytest
from semantica.context import ContextGraph
from semantica.context.context_graph import ContextEdge
@@ -323,3 +325,130 @@ def test_entity_based_inference_still_applies_without_explicit_edges():
hop["from"] == earlier and hop["to"] == later and hop["type"] == "influences"
for hop in hops
)
def test_get_causal_chain_accepts_lowercase_causal_edge_types():
"""Issue #1184: edges recorded with the analyzer's lowercase vocabulary
must be traversed by get_causal_chain().
CausalChainAnalyzer documents causal types as lowercase ("causes",
"influences", ...) while get_causal_chain() matched only the uppercase
spellings, so an edge recorded as "causes" produced an empty audit
chain silent and in the dangerous direction.
"""
graph = ContextGraph(advanced_analytics=True)
cause = graph.record_decision(
category="a", scenario="upstream", reasoning="r",
outcome="x", confidence=0.9,
)
effect = graph.record_decision(
category="b", scenario="downstream", reasoning="r",
outcome="y", confidence=0.9,
)
graph.add_edge(cause, effect, "causes")
chain = graph.get_causal_chain(effect, direction="upstream")
assert [decision.decision_id for decision in chain] == [cause]
def test_add_causal_relationship_accepts_any_case_and_stores_canonical():
"""Issue #1184: add_causal_relationship() should accept either spelling
and store the canonical uppercase vocabulary."""
graph = ContextGraph(advanced_analytics=True)
cause = graph.record_decision(
category="a", scenario="upstream", reasoning="r",
outcome="x", confidence=0.9,
)
effect = graph.record_decision(
category="b", scenario="downstream", reasoning="r",
outcome="y", confidence=0.9,
)
graph.add_causal_relationship(cause, effect, relationship_type="causes")
edges = [
edge for edge in graph.edges
if edge.source_id == cause and edge.target_id == effect
]
assert edges, "add_causal_relationship must store the edge"
assert edges[0].edge_type == "CAUSED"
def test_add_causal_relationship_rejects_non_string_with_value_error():
"""Invalid relationship types must keep raising ValueError (issue #1184
follow-up): normalization must not turn them into AttributeError."""
graph = ContextGraph(advanced_analytics=True)
cause = graph.record_decision(
category="a", scenario="upstream", reasoning="r",
outcome="x", confidence=0.9,
)
effect = graph.record_decision(
category="b", scenario="downstream", reasoning="r",
outcome="y", confidence=0.9,
)
for bad_type in (None, 42, ["CAUSED"]):
with pytest.raises(ValueError):
graph.add_causal_relationship(cause, effect, relationship_type=bad_type)
def test_analyze_decision_influence_sees_lowercase_causal_edge():
"""Issue #1184 follow-up: influence analysis reads the same edge index as
the causal traversal, so lowercase edges must count as direct influence."""
graph = ContextGraph(advanced_analytics=True)
cause = graph.record_decision(
category="a", scenario="upstream", reasoning="r",
outcome="x", confidence=0.9,
)
effect = graph.record_decision(
category="b", scenario="downstream", reasoning="r",
outcome="y", confidence=0.9,
)
graph.add_edge(cause, effect, "causes")
impact = graph.analyze_decision_influence(cause)
direct_ids = {entry["decision_id"] for entry in impact["direct_influence"]}
assert effect in direct_ids
def test_trace_decision_causality_sees_lowercase_causal_edge():
"""Issue #1184 follow-up: the trace must not return an empty audit chain
for a decision with an explicit lowercase upstream causal edge."""
graph = ContextGraph(advanced_analytics=True)
cause = graph.record_decision(
category="a", scenario="upstream", reasoning="r",
outcome="x", confidence=0.9,
)
effect = graph.record_decision(
category="b", scenario="downstream", reasoning="r",
outcome="y", confidence=0.9,
)
graph.add_edge(cause, effect, "causes")
trace = graph.trace_decision_causality(effect)
assert any(
hop["from"] == cause and hop["to"] == effect
for chain in trace for hop in chain["hops"]
), "lowercase causal edge must appear in the traced chain"
def test_find_precedents_sees_lowercase_precedent_edge():
"""Issue #1184 follow-up: precedent lookup must accept the analyzer's
spelling alongside the canonical PRECEDENT_FOR."""
graph = ContextGraph(advanced_analytics=True)
precedent = graph.record_decision(
category="a", scenario="earlier", reasoning="r",
outcome="x", confidence=0.9,
)
later = graph.record_decision(
category="b", scenario="later", reasoning="r",
outcome="y", confidence=0.9,
)
graph.add_edge(precedent, later, "precedes")
precedents = graph.find_precedents(later)
assert [d.decision_id for d in precedents] == [precedent]
+134
View File
@@ -0,0 +1,134 @@
"""The document IRI of a JSON-LD export must depend on content, not the clock
(issue #1147).
``_convert_kg_to_jsonld`` minted the graph's ``@id`` from ``utc_now_iso()``,
and the generic ``_attach_document_metadata`` path did the same for a plain
document ``@id``. Exporting an unchanged graph therefore produced a new
subject every time: three exports of one one-entity graph merged into 3
``semantica:KnowledgeGraph`` nodes and 15 triples for what should have been a
single graph. Neither identifier resolves and the timestamp is already
recorded correctly in ``semantica:exportedAt``, so the fix mints the IRI from
the exported content instead (mirroring ``mint_entity_iri``, #1109), with an
optional caller-supplied override for callers who already name their graphs.
"""
import json
from rdflib import RDF, Graph, URIRef
from semantica.export.json_exporter import JSONExporter
KG = {
"entities": [{"id": "https://example.org/e1", "text": "Acme Corp", "type": "ORG"}],
"relationships": [],
}
OTHER_KG = {
"entities": [
{"id": "https://example.org/e1", "text": "Acme Corp Renamed", "type": "ORG"}
],
"relationships": [],
}
def _export(kg, tmp_path, name="out.jsonld", **options):
path = tmp_path / name
JSONExporter().export_knowledge_graph(kg, path, format="json-ld", **options)
return path
def test_reexporting_an_unchanged_graph_is_idempotent(tmp_path):
"""The whole point of an identifier: same content, same @id."""
first = json.loads(_export(KG, tmp_path, "a.jsonld").read_text())
second = json.loads(_export(KG, tmp_path, "b.jsonld").read_text())
assert first["@id"] == second["@id"]
def test_a_changed_graph_gets_a_different_id(tmp_path):
unchanged = json.loads(_export(KG, tmp_path, "a.jsonld").read_text())
changed = json.loads(_export(OTHER_KG, tmp_path, "b.jsonld").read_text())
assert unchanged["@id"] != changed["@id"]
def test_merging_repeated_exports_yields_one_graph_node(tmp_path):
"""Regression for the exact repro in #1147: churn no longer multiplies nodes."""
merged = Graph()
for i in range(3):
path = _export(KG, tmp_path, f"churn{i}.jsonld")
merged.parse(str(path), format="json-ld")
# Exactly one subject typed as a KnowledgeGraph, regardless of how many
# times the unchanged graph was exported and merged.
kg_nodes = set(
merged.subjects(RDF.type, URIRef("https://semantica.dev/ns#KnowledgeGraph"))
)
assert len(kg_nodes) == 1
entity_nodes = set(
merged.subjects(RDF.type, URIRef("https://semantica.dev/vocab/ORG"))
)
assert len(entity_nodes) == 1
def test_exported_at_still_varies_between_exports(tmp_path):
"""Identity is now content-derived, but provenance still records each run."""
first = json.loads(_export(KG, tmp_path, "a.jsonld").read_text())
second = json.loads(_export(KG, tmp_path, "b.jsonld").read_text())
assert first["@id"] == second["@id"]
assert first["semantica:exportedAt"] != second["semantica:exportedAt"]
def test_caller_supplied_graph_uri_is_honored(tmp_path):
path = _export(KG, tmp_path, graph_uri="https://example.org/my-graph")
document = json.loads(path.read_text())
assert document["@id"] == "https://example.org/my-graph"
def _document_node_id(document):
"""The generic (non-knowledge-graph) path hangs its own @id off a member
of @graph rather than the top level, to avoid re-creating the named-graph
bug fixed by #1145. Find that member and return its @id."""
for node in document["@graph"]:
if "semantica:exportedAt" in node:
return node["@id"]
raise AssertionError(f"no document metadata node in @graph: {document}")
def test_caller_supplied_document_uri_is_honored_for_a_generic_export(tmp_path):
payload = {"note": "no entities or relationships here"}
path = tmp_path / "generic.jsonld"
JSONExporter().export(
payload, path, format="json-ld", document_uri="https://example.org/my-doc"
)
document = json.loads(path.read_text())
assert _document_node_id(document) == "https://example.org/my-doc"
def test_generic_document_id_is_also_content_derived(tmp_path):
"""The non-knowledge-graph path (_attach_document_metadata) gets the same fix."""
payload = {"note": "plain data, no @id of its own"}
first = tmp_path / "a.jsonld"
second = tmp_path / "b.jsonld"
JSONExporter().export(payload, first, format="json-ld")
JSONExporter().export(dict(payload), second, format="json-ld")
first_id = _document_node_id(json.loads(first.read_text()))
second_id = _document_node_id(json.loads(second.read_text()))
assert first_id == second_id
def test_document_id_still_differs_for_different_generic_payloads(tmp_path):
a = tmp_path / "a.jsonld"
b = tmp_path / "b.jsonld"
JSONExporter().export({"note": "one"}, a, format="json-ld")
JSONExporter().export({"note": "two"}, b, format="json-ld")
a_id = _document_node_id(json.loads(a.read_text()))
b_id = _document_node_id(json.loads(b.read_text()))
assert a_id != b_id
+19 -8
View File
@@ -107,20 +107,31 @@ def test_exported_timestamp_survives_a_timezone_qualified_sparql_filter():
assert len(rows) == 1, "the export was dropped by a timezone-qualified filter"
def test_document_iri_carrying_an_offset_is_a_valid_iri():
"""The offset puts '+' and ':' in the @id; both are legal in a path."""
def test_document_iri_is_a_valid_iri():
"""The graph @id must be a valid IRI regardless of how it is minted.
Before #1147, this @id was minted from the offset-carrying timestamp
itself (``+00:00`` interpolated straight into the path), so this test
asserted the offset survived without breaking IRI validity. #1147 mints
the @id from the graph's content instead, so the timestamp no longer
appears here at all it stays in ``semantica:exportedAt`` (still
offset-aware, per ``test_jsonld_export_timestamp_is_offset_aware`` above).
What's left worth guarding is the general case: whatever the @id is
minted from, it has to be a valid IRI that round-trips through RDF.
"""
rdflib = pytest.importorskip("rdflib")
document_iri = JSONExporter()._convert_kg_to_jsonld(KG)["@id"]
assert "+00:00" in document_iri
assert rdflib.term._is_valid_uri(document_iri)
graph = rdflib.Graph()
graph.add((
rdflib.URIRef(document_iri),
rdflib.RDF.type,
rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"),
))
graph.add(
(
rdflib.URIRef(document_iri),
rdflib.RDF.type,
rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"),
)
)
reparsed = rdflib.Graph().parse(data=graph.serialize(format="nt"), format="nt")
assert document_iri in {str(s) for s in reparsed.subjects()}
+59
View File
@@ -525,6 +525,65 @@ class TestSHACLHierarchicalAndValidation(unittest.TestCase):
self.assertEqual(mc[0].max_count, 2)
# 33
def test_public_run_shacl_validation_api(self):
"""The public API validates data and retains the legacy alias."""
try:
import pyshacl # noqa: F401
import rdflib # noqa: F401
except ImportError:
self.skipTest("pyshacl/rdflib not installed")
from semantica.ontology import run_shacl_validation
from semantica.ontology.ontology_validator import _run_pyshacl
data = "@prefix ex: <http://example.org/> . ex:alice a ex:Person ."
shacl = """
@prefix ex: <http://example.org/> .
@prefix sh: <http://www.w3.org/ns/shacl#> .
ex:PersonShape a sh:NodeShape ; sh:targetClass ex:Person ;
sh:property [ sh:path ex:name ; sh:minCount 1 ] .
"""
public_report = run_shacl_validation(data, shacl)
legacy_report = _run_pyshacl(data, shacl)
self.assertFalse(public_report.conforms)
self.assertEqual(public_report.violation_count, 1)
self.assertEqual(legacy_report.conforms, public_report.conforms)
self.assertEqual(legacy_report.violation_count, public_report.violation_count)
self.assertEqual(
[
(v.focus_node, v.result_path, v.constraint, v.severity, v.message)
for v in legacy_report.violations
],
[
(v.focus_node, v.result_path, v.constraint, v.severity, v.message)
for v in public_report.violations
],
)
# 34
def test_public_run_shacl_validation_conforming_graph(self):
"""The public API reports a valid graph without violations."""
try:
import pyshacl # noqa: F401
import rdflib # noqa: F401
except ImportError:
self.skipTest("pyshacl/rdflib not installed")
from semantica.ontology import run_shacl_validation
data = """
@prefix ex: <http://example.org/> .
ex:alice a ex:Person ; ex:name "Alice" .
"""
shacl = """
@prefix ex: <http://example.org/> .
@prefix sh: <http://www.w3.org/ns/shacl#> .
ex:PersonShape a sh:NodeShape ; sh:targetClass ex:Person ;
sh:property [ sh:path ex:name ; sh:minCount 1 ] .
"""
report = run_shacl_validation(data, shacl)
self.assertTrue(report.conforms)
self.assertEqual(report.violation_count, 0)
def test_shacl_violation_to_dict(self):
from semantica.ontology.ontology_validator import SHACLViolation
v = SHACLViolation(
+123
View File
@@ -1851,3 +1851,126 @@ class TestExitCodes:
assert "Traceback" not in result.output, (
f"Traceback found for {argv}: {result.output}"
)
class TestDoctorEmbeddings:
"""#994: doctor must surface non-functional embedding backends instead of
reporting all green. Default = import-level check; --deep-embeddings (or
SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates via TextEmbedder."""
def _doctor_checks(self, runner, *extra):
result = runner.invoke(cli_module.main, ["doctor", "--json", *extra])
_ok(result)
import json as _json
return {c["check"]: c for c in _json.loads(result.output)}
def _with_fake_st(self, monkeypatch, **embedder_attrs):
fake_st = _fake_module(
__version__="9.9.9",
SentenceTransformer=object,
)
monkeypatch.setitem(__import__("sys").modules, "sentence_transformers", fake_st)
def test_doctor_reports_embedding_checks(self, runner):
checks = self._doctor_checks(runner)
assert "Embeddings (sentence-transformers)" in checks
assert "Embeddings (fastembed)" in checks
def test_import_failure_is_fail_status_with_hint(self, runner, monkeypatch):
# Force the 'import sentence_transformers' inside _embedding_backend to
# raise ImportError regardless of whether the package is installed on
# this machine. Setting a module entry to None is the standard Python
# mechanism: any subsequent 'import <name>' raises
# "import of <name> halted; None in sys.modules".
monkeypatch.setitem(
__import__("sys").modules, "sentence_transformers", None
)
checks = self._doctor_checks(runner)
st = checks["Embeddings (sentence-transformers)"]
assert st["status"] == "fail"
assert st["hint"] == "pip install sentence-transformers"
def test_deep_probe_detects_fallback_active(self, runner, monkeypatch):
self._with_fake_st(monkeypatch)
fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None)
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
checks = self._doctor_checks(runner, "--deep-embeddings")
st = checks["Embeddings (sentence-transformers)"]
assert st["status"] == "fail"
assert "hash fallback" in st["note"]
def test_deep_probe_ok_when_model_loads(self, runner, monkeypatch):
self._with_fake_st(monkeypatch)
import numpy as np
fake_embedder = types.SimpleNamespace(
model=object(),
fastembed_model=None,
embed_text=lambda text: np.zeros(384, dtype=np.float32),
)
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
checks = self._doctor_checks(runner, "--deep-embeddings")
st = checks["Embeddings (sentence-transformers)"]
assert st["status"] == "ok"
assert "384-dim" in st["note"]
def test_env_var_enables_deep_mode(self, runner, monkeypatch):
monkeypatch.setenv("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", "1")
self._with_fake_st(monkeypatch)
fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None)
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
checks = self._doctor_checks(runner)
st = checks["Embeddings (sentence-transformers)"]
assert st["status"] == "fail"
assert "hash fallback" in st["note"]
class TestDoctorEmbeddingHintsAndEnv:
"""Review follow-ups: deep failures must not carry the pip-install hint,
and the env toggle tolerates case/whitespace variants."""
def _doctor_checks(self, runner, *extra):
result = runner.invoke(cli_module.main, ["doctor", "--json", *extra])
_ok(result)
import json as _json
return {c["check"]: c for c in _json.loads(result.output)}
def _with_fake_st(self, monkeypatch):
fake_st = _fake_module(
__version__="9.9.9",
SentenceTransformer=object,
)
monkeypatch.setitem(__import__("sys").modules, "sentence_transformers", fake_st)
def test_deep_failure_hint_is_not_pip_install(self, runner, monkeypatch):
self._with_fake_st(monkeypatch)
fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None)
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
checks = self._doctor_checks(runner, "--deep-embeddings")
st = checks["Embeddings (sentence-transformers)"]
assert st["status"] == "fail"
assert "pip install" not in (st["hint"] or ""), (
"a deep probe failure means the package imported fine — pointing "
"users at pip sends them to reinstall for a runtime/model problem"
)
assert "runtime/model-load" in st["hint"]
def test_env_var_tolerates_case_and_whitespace(self, runner, monkeypatch):
monkeypatch.setenv("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", " TRUE ")
self._with_fake_st(monkeypatch)
fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None)
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
checks = self._doctor_checks(runner)
st = checks["Embeddings (sentence-transformers)"]
assert st["status"] == "fail"
assert "hash fallback" in st["note"], "padded/caps env value must enable deep mode"
+46
View File
@@ -85,3 +85,49 @@ if __name__ == '__main__':
runner = unittest.TextTestRunner(stream=f, verbosity=2)
unittest.main(testRunner=runner, exit=False)
class TestMethodDispatchRecursion(unittest.TestCase):
"""#994: built-in aliases are registered in the method registry onto the
wrapper functions themselves, so dispatching through the registry called a
wrapper back into itself with the same default method a recursion storm
that surfaced as `maximum recursion depth exceeded` during model loading."""
def test_generate_embeddings_default_does_not_self_recurse(self):
from semantica.embeddings.methods import generate_embeddings
emb = generate_embeddings("recursion probe")
self.assertIsNotNone(emb)
def test_embed_text_default_does_not_self_recurse(self):
from semantica.embeddings.methods import embed_text
emb = embed_text("recursion probe", method="sentence_transformers")
self.assertIsNotNone(emb)
def test_custom_registered_method_still_wins(self):
from semantica.embeddings.methods import method_registry
calls = []
def spy(data, *a, **k):
calls.append(data)
return {"custom": True}
method_registry.register("generation", "my_custom_gen", spy)
try:
from semantica.embeddings.methods import generate_embeddings
out = generate_embeddings("payload", method="my_custom_gen")
self.assertEqual(out, {"custom": True})
self.assertEqual(calls, ["payload"])
finally:
method_registry.unregister("generation", "my_custom_gen")
def test_provenance_wrapper_missing_generator_raises_attribute_error(self):
# Partially-initialised wrappers (failed __init__, pickle/copy probes)
# must raise AttributeError, not RecursionError via __getattr__.
from semantica.embeddings.embeddings_provenance import (
EmbeddingGeneratorWithProvenance,
)
bare = EmbeddingGeneratorWithProvenance.__new__(
EmbeddingGeneratorWithProvenance
)
with self.assertRaises(AttributeError):
getattr(bare, "model")
+14
View File
@@ -243,5 +243,19 @@ class TestVectorStore(unittest.TestCase):
shutil.rmtree(tmpdir, ignore_errors=True)
class TestCreateIndexFunction(unittest.TestCase):
"""create_index() forwards vector_store_config's defaults into VectorIndexer,
which already receives backend/dimension as explicit args. Regression for the
'got multiple values for keyword argument dimension' crash on the default
(unmocked) config, hit by e.g. `semantica embed index`."""
def test_create_index_with_default_config(self):
from semantica.vector_store.methods import create_index
vectors = [np.array([0.1, 0.2, 0.3]), np.array([0.4, 0.5, 0.6])]
index = create_index(vectors, ids=["a", "b"])
self.assertIsNotNone(index)
if __name__ == '__main__':
unittest.main()