mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Merge branch 'main' into codex/context-graph-markdown-round-trip
This commit is contained in:
@@ -408,6 +408,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Markdown import followed filesystem links even though Markdown export already refused to overwrite them** (#851, follow-up to #765, #786) by @SaurabhScripts
|
||||
- `AgentMemory._read_markdown_path()` now rejects symlink files, broken symlinks, symlinked directories, Windows directory junctions, and other Windows reparse points supplied directly; linked entries discovered inside an otherwise valid directory are safely skipped, preserving the current directory-import contract
|
||||
- `_read_markdown_file_content()` re-checks the file and parent directory immediately before and after opening, uses `O_NOFOLLOW` where available, and verifies the resulting descriptor is a regular file via `fstat`/`S_ISREG`, so link swaps are rejected rather than silently followed
|
||||
- Junction detection uses `os.path.isjunction()` where available and falls back to the Windows reparse-point file attribute on older Python versions; export applies the same link check before replacing a Markdown file
|
||||
- Documented the import restriction in `docs/reference/context.md`; added 11 tests to `tests/context/test_agent_memory_markdown.py` covering file/directory/broken-symlink rejection, simulated open races, mocked and real Windows junctions, and the reparse-point fallback
|
||||
- Any additional review follow-up commits land in this same PR/entry rather than as a separate changelog item
|
||||
|
||||
- **`PipelineWithProvenance` raised `ModuleNotFoundError` on import and `AttributeError` on `.run()`** (#858, closes #858) by @Karunasagar12
|
||||
- `from .pipeline import Pipeline` failed because `semantica/pipeline/pipeline.py` does not exist; corrected to `from .pipeline_builder import Pipeline`
|
||||
- `.run()` called `self._pipeline.run()` on the `Pipeline` dataclass, which has no such method; replaced with `self._engine.execute_pipeline(self._pipeline, ...)` delegating to `ExecutionEngine`
|
||||
|
||||
@@ -8,7 +8,7 @@ icon: "shield-check"
|
||||
|
||||
SHACL (Shapes Constraint Language) is a standard for validating graph-based data. While an ontology defines the conceptual *schema* (the "what" exists in your domain), SHACL defines the structural *rules and constraints* (the "how" it should be structured).
|
||||
|
||||
In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and `_run_pyshacl` evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated.
|
||||
In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and the public `run_shacl_validation` function evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated. The historical `_run_pyshacl` name remains available as a compatibility alias.
|
||||
|
||||
## Why Use SHACL Validation?
|
||||
|
||||
@@ -55,7 +55,7 @@ Let's look at a simple, universally understood example: ensuring every `Employee
|
||||
```python
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
# 1. Prepare your data graph
|
||||
graph = ContextGraph()
|
||||
@@ -95,7 +95,7 @@ data_ttl = """
|
||||
"""
|
||||
|
||||
# 5. Run Validation
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
|
||||
# 6. Analyze the Report
|
||||
print(f"Graph conforms: {report.conforms}")
|
||||
@@ -265,10 +265,10 @@ cve_id_shape = NodeShape(
|
||||
|
||||
## Step 4 — Run validation and read the report
|
||||
|
||||
Serialize the graph to RDF, then run `_run_pyshacl` against the shapes.
|
||||
Serialize the graph to RDF, then run `run_shacl_validation` against the shapes.
|
||||
|
||||
```python
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
# Prepare your RDF data string (since export_rdf primarily exports structural metadata,
|
||||
# you typically serialize your custom data graph to Turtle using rdflib or similar).
|
||||
@@ -281,7 +281,7 @@ data_ttl = """
|
||||
"""
|
||||
|
||||
# Run SHACL validation
|
||||
report = _run_pyshacl(
|
||||
report = run_shacl_validation(
|
||||
data_ttl,
|
||||
shacl_ttl,
|
||||
data_graph_format="turtle",
|
||||
@@ -366,8 +366,8 @@ print(f"Malware nodes missing 'family': {len(missing_family)}")
|
||||
# e.g. graph.update_node(node_id, {"family": "UNKNOWN — requires triage"})
|
||||
|
||||
# After remediation, re-run validation to confirm the fix
|
||||
# (re-export the patched graph to Turtle first, then call _run_pyshacl again)
|
||||
report2 = _run_pyshacl(patched_data_ttl, shacl_ttl)
|
||||
# (re-export the patched graph to Turtle first, then call run_shacl_validation again)
|
||||
report2 = run_shacl_validation(patched_data_ttl, shacl_ttl)
|
||||
print(f"Violations after remediation: {report2.violation_count}")
|
||||
# Violations after remediation: 0
|
||||
```
|
||||
@@ -377,10 +377,49 @@ print(f"Violations after remediation: {report2.violation_count}")
|
||||
## Common Pitfalls
|
||||
|
||||
- **Assuming the ontology automatically enforces data quality**: `SHACLGenerator` generates shapes based on what it observes in the data. If your data is missing a field, the generator won't know it was mandatory unless you explicitly inject the constraint (as shown in Step 3).
|
||||
- **Passing `ContextGraph` directly to SHACL validators**: The `_run_pyshacl` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object.
|
||||
- **Passing `ContextGraph` directly to SHACL validators**: The `run_shacl_validation` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object.
|
||||
- **Forgetting RDF serialization**: You must serialize your graph (often via a temporary file using `export_rdf`) before validating it.
|
||||
- **Treating validation as a one-time step**: Validation should be integrated as an automated step in your CI/CD pipeline or data ingestion flow, acting as a recurring gatekeeper rather than a one-off script.
|
||||
- **Ignoring validation reports**: A graph that does not conform must be remediated. Failing to review the `violation_count` and address the issues negates the purpose of SHACL validation.
|
||||
- **Validating `sh:class`/`sh:node` range checks on a property that declares `rdfs:range` with RDFS entailment on**: RDFS is an entailment rule, not a constraint. When pyshacl runs with `inference="rdfs"`, it infers the range class onto every object of the property, so class-based constraints on that property can never fail — the report says `conforms: True` on data that does not conform:
|
||||
|
||||
```python
|
||||
from pyshacl import validate
|
||||
from rdflib import Graph
|
||||
|
||||
data = Graph()
|
||||
data.parse(
|
||||
data="""
|
||||
@prefix ex: <https://example.org/ns#> .
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||
ex:contains rdfs:domain ex:Container ; rdfs:range ex:Item .
|
||||
ex:box a ex:Container ; ex:contains ex:notAnItem .
|
||||
ex:notAnItem a ex:Fish .
|
||||
""",
|
||||
format="turtle",
|
||||
)
|
||||
|
||||
shapes = Graph()
|
||||
shapes.parse(
|
||||
data="""
|
||||
@prefix ex: <https://example.org/ns#> .
|
||||
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
||||
ex:ContainerShape a sh:NodeShape ;
|
||||
sh:targetClass ex:Container ;
|
||||
sh:property [ sh:path ex:contains ; sh:class ex:Item ] .
|
||||
""",
|
||||
format="turtle",
|
||||
)
|
||||
|
||||
for inference in ("none", "rdfs"):
|
||||
conforms, _, _ = validate(data, shacl_graph=shapes, inference=inference)
|
||||
print(inference, conforms)
|
||||
# none False <- correct: notAnItem is a Fish, not an Item
|
||||
# rdfs True <- the entailment manufactured the type
|
||||
```
|
||||
|
||||
Mitigations: prefer not to declare `rdfs:range` on properties you intend to constrain with `sh:class`; when class membership is the thing under test, run validation without RDFS entailment (`inference="none"`); or express the check as a constraint the entailment cannot satisfy (for example a literal property constraint). Note the trade-off: with entailment off, `sh:targetClass` no longer reaches subclasses, so subclass hierarchies need explicit typing or inference-aware target selection. Semantica's own `run_shacl_validation` wrapper already calls pyshacl with `inference="none"`, so this pitfall only bites when calling `pyshacl.validate` directly with entailment enabled.
|
||||
- **Trusting `conforms: True` without checking the inference mode**: an inference-enabled run can hide the exact violations the shapes were written to catch (see above). Record which inference mode validation ran under alongside the result, and re-run shape sets that contain `sh:class`/`sh:node` with entailment off before treating a pass as authoritative.
|
||||
|
||||
---
|
||||
|
||||
@@ -396,7 +435,7 @@ A DoD CTI team enforces STIX-compatible constraints on a threat graph before sha
|
||||
from semantica.context import AgentContext, ContextGraph
|
||||
from semantica.vector_store import VectorStore
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
graph = ContextGraph()
|
||||
ctx = AgentContext(
|
||||
@@ -448,7 +487,7 @@ data_ttl = """
|
||||
<http://example.org/hammertoss> a ex:Malware .
|
||||
"""
|
||||
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
print(f"CTI graph conforms : {report.conforms}")
|
||||
print(f"Violations : {report.violation_count}")
|
||||
print(f"Warnings : {report.warning_count}")
|
||||
@@ -469,7 +508,7 @@ A SOC team validates zero-trust policy nodes before publishing them to the polic
|
||||
```python
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node("policy-001", "Policy", "MFA Required for Tier-1 Resources",
|
||||
@@ -516,7 +555,7 @@ data_ttl = """
|
||||
<http://example.org/policy-002> a ex:Policy .
|
||||
"""
|
||||
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
print(f"Policy graph conforms: {report.conforms}")
|
||||
# Policy graph conforms: False
|
||||
|
||||
@@ -534,7 +573,7 @@ A clinical informatics team validates trial ontology nodes before loading them i
|
||||
|
||||
```python
|
||||
from semantica.ontology import LLMOntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
from semantica.export import export_rdf
|
||||
import tempfile, os
|
||||
|
||||
@@ -586,7 +625,7 @@ with open(tmp.name) as f:
|
||||
data_ttl = f.read()
|
||||
os.unlink(tmp.name)
|
||||
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
print(f"Trial data conforms: {report.conforms}")
|
||||
print(f"Warnings : {report.warning_count}")
|
||||
```
|
||||
@@ -600,7 +639,7 @@ A credit risk team validates every `LoanApplication` node against Basel III CRE2
|
||||
```python
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
graph = ContextGraph()
|
||||
graph.add_node("loan-001", "LoanApplication", "Prime mortgage APP-2025-88421",
|
||||
@@ -645,7 +684,7 @@ data_ttl = """
|
||||
ex:ltv "0.65" .
|
||||
"""
|
||||
|
||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
||||
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||
print(f"Loan portfolio conforms: {report.conforms}")
|
||||
# Loan portfolio conforms: False
|
||||
|
||||
@@ -675,14 +714,14 @@ Call this function as a pre-publish gate; exit code 1 blocks the pipeline.
|
||||
```python
|
||||
import sys
|
||||
from semantica.ontology import OntologyGenerator, SHACLGenerator
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
def validate_before_publish(data_graph_str: str, ontology: dict) -> None:
|
||||
shacl_gen = SHACLGenerator(base_uri="https://example.org/shapes/")
|
||||
shacl_graph = shacl_gen.generate(ontology)
|
||||
shacl_ttl = shacl_gen.serialize(shacl_graph, format="turtle")
|
||||
|
||||
report = _run_pyshacl(data_graph_str, shacl_ttl)
|
||||
report = run_shacl_validation(data_graph_str, shacl_ttl)
|
||||
|
||||
if not report.conforms:
|
||||
print(f"Graph validation FAILED — {report.violation_count} violation(s)")
|
||||
@@ -700,7 +739,6 @@ def validate_before_publish(data_graph_str: str, ontology: dict) -> None:
|
||||
|
||||
- [Ontology Management](ontology) — generate the OWL ontology that SHACL shapes are derived from
|
||||
- [Reasoning & Rules](reasoning) — complement SHACL structural constraints with logical inference rules
|
||||
- [Export & Serialization](export) — serialize graph data to Turtle/RDF/XML for `_run_pyshacl` input
|
||||
- [Export & Serialization](export) — serialize graph data to Turtle/RDF/XML for `run_shacl_validation` input
|
||||
- [Conflict Resolution](conflict-resolution) — detect and resolve data conflicts before SHACL validation
|
||||
- [Change Management](change-management) — version-gate SHACL shapes alongside ontology versions
|
||||
|
||||
|
||||
@@ -625,8 +625,10 @@ malformed or duplicate fields before changing memory, and re-importing unchanged
|
||||
files is idempotent. Memory-local `entities` and `relationships` are preserved as
|
||||
provenance but are not applied to `ContextGraph` by Markdown import. Use a dedicated
|
||||
export directory: matching files are overwritten, but unrelated or stale Markdown
|
||||
files are not deleted automatically. Export refuses to overwrite symbolic links and
|
||||
uses atomic file replacement. Timestamp offsets are preserved in Markdown and
|
||||
files are not deleted automatically. Export refuses to overwrite filesystem links and
|
||||
uses atomic file replacement; import also refuses symlinks, Windows directory
|
||||
junctions, and other Windows reparse points.
|
||||
Timestamp offsets are preserved in Markdown and
|
||||
normalized to UTC only for comparisons, so aware and local-naive records can be
|
||||
queried together safely. Vector-store writes are deferred until the in-memory import
|
||||
commits; adapter synchronization remains best-effort and logs failures.
|
||||
|
||||
+2
-1
@@ -85,7 +85,8 @@ dependencies = [
|
||||
"loguru>=0.7.3",
|
||||
"structlog>=22.1.0",
|
||||
"gensim>=4.4.0",
|
||||
"httpx<0.29.0"
|
||||
"httpx<0.29.0",
|
||||
"pyarrow>=14.0.0"
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
|
||||
+9
-9
@@ -6,9 +6,9 @@ accelerate==1.14.0 \
|
||||
# via
|
||||
# docling-ibm-models
|
||||
# docling-slim
|
||||
agno==2.8.7 \
|
||||
--hash=sha256:6a2763eb469163f7b79ab1da6ca2f22d8619f6b9d614574f975d9c12bb4323ea \
|
||||
--hash=sha256:d49396a2062ee6994ca82695b9bd1e1b95667fec432c544afa38133e564bf090
|
||||
agno==2.9.0 \
|
||||
--hash=sha256:7777674b3931b341fad4fcf02a61b185a08588c509101348facf87feb2144c0c \
|
||||
--hash=sha256:7d9c134703e3c2798023cd57dcb9caa8e1174f6914813f9b130becfc3521a46f
|
||||
# via semantica (pyproject.toml)
|
||||
agnoctl==0.1.3 \
|
||||
--hash=sha256:6fce1d2482b1f2e0a3d14b0a7c12fbd49d8df4f0bf0a4fd9fd91753cbff5efdc \
|
||||
@@ -403,9 +403,9 @@ boto3==1.43.69 \
|
||||
--hash=sha256:4eb494d05b2bd08a7eee61b8ac4c34745c99e9bbce435c91f8d15d372dd8c2db \
|
||||
--hash=sha256:76297a0b415849c63575ae08a4f1661b2dc8ee0100f104b86f98aa69b47fa2c7
|
||||
# via semantica (pyproject.toml)
|
||||
botocore==1.43.69 \
|
||||
--hash=sha256:5caa46b740d9a886137146ffbb69edb691f702bfe74c64e85621947ae00181fd \
|
||||
--hash=sha256:b1f0e01c53d6b84ee9c184ebf3636c3b3aef85e0ae8498c74afb8734ff224f87
|
||||
botocore==1.43.73 \
|
||||
--hash=sha256:068433028e011ccbeab1dd7c46b1090c24e378397693c66e67ca571176498daa \
|
||||
--hash=sha256:0fa1e63c24b3531be3e1bc1687a88b3be9e63a430153f24edd93efc162bb1c51
|
||||
# via
|
||||
# boto3
|
||||
# s3transfer
|
||||
@@ -1731,9 +1731,9 @@ google-crc32c==1.8.0 \
|
||||
# via
|
||||
# google-cloud-storage
|
||||
# google-resumable-media
|
||||
google-genai==2.17.0 \
|
||||
--hash=sha256:6b640a2390c82b4a240873eddb9f518c6d2c33244b2de16ed3d14526a6da7f57 \
|
||||
--hash=sha256:a4835563c60aee646c9c4b261c507aa4a624710d25017012d20dc65abf3d9a54
|
||||
google-genai==2.18.1 \
|
||||
--hash=sha256:36a5949233e64a60f6cc4521bff7a76b7c569d0aa227bbe9fa642213b8a3a3b2 \
|
||||
--hash=sha256:a1e2be75c16234adc6641afd1ad4dd44218c9eec005d938bdc428585a048918a
|
||||
# via semantica (pyproject.toml)
|
||||
google-resumable-media==2.10.1 \
|
||||
--hash=sha256:224975032ddb73f7ed9e2f0f4cc08ed1b06874c52d48cc8533e3eb72980b21a0 \
|
||||
|
||||
+104
-2
@@ -773,10 +773,22 @@ def changelog(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
_run_with_error_handling(_action)
|
||||
|
||||
|
||||
class _DeepEmbeddingFailure(Exception):
|
||||
"""A deep-probe failure from doctor's embedding checks.
|
||||
|
||||
Marks failures that happened AFTER the backend imported cleanly — model
|
||||
load, probe, or runtime problems — so the check's hint can point at the
|
||||
real remediation instead of `pip install`.
|
||||
"""
|
||||
|
||||
|
||||
@main.command()
|
||||
@click.option("--json", "local_json", is_flag=True, default=False)
|
||||
@click.option("--deep-embeddings", "deep_embeddings", is_flag=True, default=False,
|
||||
help="Also instantiate the local embedding backends and embed a probe "
|
||||
"text (catches backends that import cleanly but cannot load).")
|
||||
@click.pass_obj
|
||||
def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None:
|
||||
"""Run a health check on all Semantica components and backends."""
|
||||
import importlib.metadata
|
||||
cli_ctx = _require_ctx(cli_ctx)
|
||||
@@ -787,6 +799,16 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
try:
|
||||
note = fn()
|
||||
return label, "ok", note, None
|
||||
except _DeepEmbeddingFailure as exc:
|
||||
# A deep-probe failure means the package IMPORTED fine: the pip
|
||||
# hint would be the wrong remediation for what is actually a
|
||||
# runtime/model-load problem (broken torch, failed model
|
||||
# download, missing shared libs).
|
||||
return label, "fail", str(exc), (
|
||||
"runtime/model-load failure — reinstalling the package usually "
|
||||
"does not help; check the warnings above (torch install, model "
|
||||
"download, disk space)"
|
||||
)
|
||||
except Exception as exc:
|
||||
return label, "fail", str(exc), hint
|
||||
|
||||
@@ -827,6 +849,50 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
|
||||
return f"{backend} importable"
|
||||
checks.append(_check("Vector store", _vector, hint="pip install semantica[vectorstore-…]"))
|
||||
|
||||
# Embedding backends (#994): `doctor` used to report all green while
|
||||
# every local embedding backend was non-functional — import success
|
||||
# says nothing about model loading. Default checks stay cheap
|
||||
# (import + version); --deep-embeddings (or
|
||||
# SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates the backend through
|
||||
# TextEmbedder and embeds a probe, which is the only level that
|
||||
# catches a backend that imports cleanly but cannot actually load.
|
||||
deep = deep_embeddings or os.environ.get("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", "").strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
def _embedding_backend(method: str) -> str:
|
||||
if method == "sentence_transformers":
|
||||
import sentence_transformers # noqa: F401
|
||||
note = f"importable ({importlib.metadata.version('sentence-transformers')})"
|
||||
else:
|
||||
import fastembed # noqa: F401
|
||||
note = f"importable ({importlib.metadata.version('fastembed')})"
|
||||
if not deep:
|
||||
return note
|
||||
try:
|
||||
from .embeddings import TextEmbedder
|
||||
embedder = TextEmbedder(method=method)
|
||||
if embedder.model is None and embedder.fastembed_model is None:
|
||||
raise RuntimeError(
|
||||
"model failed to load — the hash fallback is active "
|
||||
"(see warnings above); embedding quality is degraded"
|
||||
)
|
||||
probe = embedder.embed_text("semantica doctor embedding probe")
|
||||
except _DeepEmbeddingFailure:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise _DeepEmbeddingFailure(str(exc)) from exc
|
||||
return f"{note}; deep probe ok ({len(probe)}-dim)"
|
||||
|
||||
checks.append(_check(
|
||||
"Embeddings (sentence-transformers)",
|
||||
lambda: _embedding_backend("sentence_transformers"),
|
||||
hint="pip install sentence-transformers",
|
||||
))
|
||||
checks.append(_check(
|
||||
"Embeddings (fastembed)",
|
||||
lambda: _embedding_backend("fastembed"),
|
||||
hint="pip install fastembed",
|
||||
))
|
||||
|
||||
# LLM provider keys
|
||||
for provider, var in [("OpenAI", "OPENAI_API_KEY"), ("Anthropic", "ANTHROPIC_API_KEY"),
|
||||
("Groq", "GROQ_API_KEY")]:
|
||||
@@ -1708,7 +1774,43 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str,
|
||||
except ImportError as exc:
|
||||
raise click.ClickException(f"Embeddings module not available: {exc}") from exc
|
||||
if output:
|
||||
Path(output).write_text(json.dumps(result, default=str), encoding="utf-8")
|
||||
output_path = Path(output)
|
||||
suffix = output_path.suffix.lower()
|
||||
try:
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
arr = np.asarray(result)
|
||||
if arr.ndim == 1:
|
||||
arr = arr[np.newaxis, :]
|
||||
if arr.ndim != 2:
|
||||
raise click.ClickException(
|
||||
f"embed generate --output expects a 1-D or 2-D array, "
|
||||
f"got {arr.ndim}-D (shape {arr.shape})"
|
||||
)
|
||||
rows = [list(row) for row in arr]
|
||||
if suffix == ".parquet":
|
||||
# Schema: single 'embedding' column (list[float] per row).
|
||||
# embed index detects vector columns via
|
||||
# isinstance(df[c].iloc[0], (list, np.ndarray)).
|
||||
df = pd.DataFrame({"embedding": rows})
|
||||
df.to_parquet(output_path, index=False)
|
||||
elif suffix in (".json", ".jsonl"):
|
||||
df = pd.DataFrame({"embedding": rows})
|
||||
df.to_json(
|
||||
output_path,
|
||||
orient="records",
|
||||
lines=(suffix == ".jsonl"),
|
||||
)
|
||||
else:
|
||||
raise click.ClickException(
|
||||
f"Unsupported output format '{suffix}'. "
|
||||
"Use .parquet, .json, or .jsonl"
|
||||
)
|
||||
except ImportError as exc:
|
||||
raise click.ClickException(
|
||||
f"Missing dependency for --output: {exc}. "
|
||||
"Install pyarrow with: pip install pyarrow"
|
||||
) from exc
|
||||
_ok(cli_ctx, f"Wrote {output}")
|
||||
elif _is_json(cli_ctx, local_json):
|
||||
_jecho(result if isinstance(result, dict) else {"status": "ok"})
|
||||
|
||||
@@ -77,6 +77,7 @@ import yaml
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from ..utils.types import EntityDict, RelationshipDict
|
||||
from ._markdown_filesystem import find_filesystem_link
|
||||
|
||||
|
||||
class _UniqueKeySafeLoader(yaml.SafeLoader):
|
||||
@@ -1742,9 +1743,10 @@ class AgentMemory:
|
||||
@staticmethod
|
||||
def _write_markdown_file(file_path: Path, document: str) -> None:
|
||||
"""Atomically replace a Markdown file without following output symlinks."""
|
||||
if file_path.is_symlink():
|
||||
if find_filesystem_link(file_path) is not None:
|
||||
raise ValueError(
|
||||
f"Refusing to overwrite Markdown symbolic link: {file_path}"
|
||||
"Refusing to overwrite Markdown symbolic link or junction: "
|
||||
f"{file_path}"
|
||||
)
|
||||
|
||||
temporary_path = None
|
||||
@@ -1867,7 +1869,8 @@ class AgentMemory:
|
||||
if "\n" not in data and "\r" not in data:
|
||||
candidate = Path(data)
|
||||
try:
|
||||
candidate_exists = candidate.exists()
|
||||
candidate_is_link = find_filesystem_link(candidate) is not None
|
||||
candidate_exists = candidate_is_link or candidate.exists()
|
||||
except OSError as exc:
|
||||
error_message = (
|
||||
"Failed to inspect possible Markdown import "
|
||||
@@ -1909,62 +1912,71 @@ class AgentMemory:
|
||||
return memories
|
||||
|
||||
def _read_markdown_file_content(self, file_path: Path) -> str:
|
||||
if file_path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {file_path}")
|
||||
if find_filesystem_link(file_path) is not None:
|
||||
raise ValueError(
|
||||
"Symlink Markdown import paths are rejected; symbolic links and "
|
||||
f"junctions are unsafe: {file_path}"
|
||||
)
|
||||
|
||||
flags = os.O_RDONLY
|
||||
if hasattr(os, "O_NOFOLLOW"):
|
||||
# On POSIX, O_NOFOLLOW makes os.open() fail with ELOOP if the
|
||||
# final path component is a symlink, atomically closing the TOCTOU
|
||||
# window between the is_symlink() check above and the open call.
|
||||
# On Windows, O_NOFOLLOW is not available; the is_symlink() pre-check
|
||||
# above is the only symlink defense and remains vulnerable to a narrow
|
||||
# race. The fstat()/S_ISREG guard below still rejects special files
|
||||
# (FIFOs, devices) on both platforms.
|
||||
flags |= os.O_NOFOLLOW
|
||||
nofollow_flag = getattr(os, "O_NOFOLLOW", 0)
|
||||
flags |= nofollow_flag
|
||||
|
||||
try:
|
||||
fd = os.open(str(file_path), flags)
|
||||
except OSError as exc:
|
||||
if exc.errno == getattr(errno, "ELOOP", None):
|
||||
if (
|
||||
(nofollow_flag and exc.errno == errno.ELOOP)
|
||||
or find_filesystem_link(file_path) is not None
|
||||
):
|
||||
raise ValueError(
|
||||
f"Symlink Markdown import paths are rejected: {file_path}"
|
||||
"Symlink Markdown import paths are rejected; symbolic links "
|
||||
f"and junctions are unsafe: {file_path}"
|
||||
) from exc
|
||||
raise
|
||||
|
||||
try:
|
||||
stat_res = os.fstat(fd)
|
||||
if not stat.S_ISREG(stat_res.st_mode):
|
||||
if find_filesystem_link(file_path) is not None:
|
||||
raise ValueError(
|
||||
"Symlink Markdown import paths are rejected; symbolic links "
|
||||
f"and junctions are unsafe: {file_path}"
|
||||
)
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||
raise ValueError(
|
||||
f"Markdown import path is not a regular file: {file_path}"
|
||||
)
|
||||
with open(fd, "r", encoding="utf-8", closefd=True) as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
try:
|
||||
with os.fdopen(fd, mode="r", encoding="utf-8") as source:
|
||||
fd = -1
|
||||
return source.read()
|
||||
finally:
|
||||
if fd >= 0:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def _read_markdown_path(self, path: Path) -> List[Tuple[str, str]]:
|
||||
if path.is_symlink():
|
||||
raise ValueError(f"Symlink Markdown import paths are rejected: {path}")
|
||||
if find_filesystem_link(path) is not None:
|
||||
raise ValueError(
|
||||
"Symlink Markdown import paths are rejected; symbolic links and "
|
||||
f"junctions are unsafe: {path}"
|
||||
)
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Markdown import path does not exist: {path}")
|
||||
|
||||
if path.is_dir():
|
||||
file_paths = sorted(
|
||||
(
|
||||
file_path
|
||||
for file_path in path.iterdir()
|
||||
if file_path.is_file()
|
||||
and not file_path.is_symlink()
|
||||
and file_path.suffix.lower() in self._MARKDOWN_EXTENSIONS
|
||||
),
|
||||
key=lambda file_path: (file_path.name.casefold(), file_path.name),
|
||||
)
|
||||
file_paths = []
|
||||
for file_path in path.iterdir():
|
||||
if file_path.suffix.lower() not in self._MARKDOWN_EXTENSIONS:
|
||||
continue
|
||||
if find_filesystem_link(file_path) is not None:
|
||||
continue
|
||||
if file_path.is_file():
|
||||
file_paths.append(file_path)
|
||||
if find_filesystem_link(path) is not None:
|
||||
raise ValueError(
|
||||
"Symlink Markdown import paths are rejected; symbolic links "
|
||||
f"and junctions are unsafe: {path}"
|
||||
)
|
||||
file_paths.sort(key=lambda item: (item.name.casefold(), item.name))
|
||||
elif path.is_file():
|
||||
file_paths = [path]
|
||||
else:
|
||||
|
||||
@@ -493,6 +493,23 @@ _ATTRS_MISSING = object()
|
||||
#: entities and timestamps.
|
||||
_CAUSAL_EDGE_TYPES = ("CAUSED", "INFLUENCED", "PRECEDENT_FOR")
|
||||
|
||||
# Causal edges circulate under two vocabularies: this module's canonical
|
||||
# spellings above, and the present-tense spellings CausalChainAnalyzer also
|
||||
# accepts ("causes", "influences", "leads_to", "supports"). The present-tense
|
||||
# forms normalize onto the canonical types for storage; traversal accepts
|
||||
# both vocabularies so an edge recorded either way is never invisible.
|
||||
_CAUSAL_EDGE_ALIASES = {
|
||||
"CAUSES": "CAUSED",
|
||||
"CAUSED": "CAUSED",
|
||||
"INFLUENCES": "INFLUENCED",
|
||||
"INFLUENCED": "INFLUENCED",
|
||||
"PRECEDES": "PRECEDENT_FOR",
|
||||
"PRECEDENT_FOR": "PRECEDENT_FOR",
|
||||
}
|
||||
_CAUSAL_TRAVERSAL_TYPES = frozenset(_CAUSAL_EDGE_ALIASES) | {
|
||||
"LEADS_TO", "LEAD_TO", "SUPPORTS", "SUPPORT",
|
||||
}
|
||||
|
||||
|
||||
class ContextGraph:
|
||||
"""
|
||||
@@ -3623,9 +3640,15 @@ class ContextGraph:
|
||||
target_decision_id: Target decision ID
|
||||
relationship_type: Type of relationship (CAUSED, INFLUENCED, PRECEDENT_FOR)
|
||||
"""
|
||||
valid_types = ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]
|
||||
if relationship_type not in valid_types:
|
||||
raise ValueError(f"Relationship type must be one of: {valid_types}")
|
||||
# Normalize so callers may use either vocabulary's spelling
|
||||
# ("causes" from CausalChainAnalyzer, or "CAUSED" from this module's
|
||||
# canonical constant); the stored form is always canonical. Invalid
|
||||
# inputs keep raising ValueError rather than AttributeError.
|
||||
if not isinstance(relationship_type, str):
|
||||
raise ValueError(f"Relationship type must be one of: {_CAUSAL_EDGE_TYPES}")
|
||||
relationship_type = _CAUSAL_EDGE_ALIASES.get(relationship_type.strip().upper())
|
||||
if relationship_type is None:
|
||||
raise ValueError(f"Relationship type must be one of: {_CAUSAL_EDGE_TYPES}")
|
||||
|
||||
# Check if decisions exist - if not, skip adding relationship
|
||||
if source_decision_id not in self.nodes or target_decision_id not in self.nodes:
|
||||
@@ -3717,11 +3740,11 @@ class ContextGraph:
|
||||
# Find connected decisions
|
||||
for edge in self.edges:
|
||||
if direction == "upstream":
|
||||
if edge.target_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]:
|
||||
if edge.target_id == current_id and edge.edge_type.upper() in _CAUSAL_TRAVERSAL_TYPES:
|
||||
if edge.source_id not in visited and depth < max_depth:
|
||||
queue.append((edge.source_id, depth + 1))
|
||||
else: # downstream
|
||||
if edge.source_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]:
|
||||
if edge.source_id == current_id and edge.edge_type.upper() in _CAUSAL_TRAVERSAL_TYPES:
|
||||
if edge.target_id not in visited and depth < max_depth:
|
||||
queue.append((edge.target_id, depth + 1))
|
||||
|
||||
@@ -3744,10 +3767,13 @@ class ContextGraph:
|
||||
Returns:
|
||||
List of precedent decisions
|
||||
"""
|
||||
# Find decisions connected via PRECEDENT_FOR relationships
|
||||
# Find decisions connected via PRECEDENT_FOR relationships, accepting
|
||||
# the analyzer vocabulary's "precedes" spelling as well (issue #1184).
|
||||
precedent_ids = []
|
||||
for edge in self.edges:
|
||||
if edge.target_id == decision_id and edge.edge_type == "PRECEDENT_FOR":
|
||||
if edge.target_id == decision_id and edge.edge_type.upper() in {
|
||||
"PRECEDENT_FOR", "PRECEDES",
|
||||
}:
|
||||
precedent_ids.append(edge.source_id)
|
||||
|
||||
# Convert to Decision objects
|
||||
@@ -4308,8 +4334,13 @@ class ContextGraph:
|
||||
|
||||
# Explicit causal relationships recorded via add_causal_relationship() are
|
||||
# ground truth and always count as direct influence, in either direction.
|
||||
for edge_type in _CAUSAL_EDGE_TYPES:
|
||||
for edge in self.edge_type_index.get(edge_type, []):
|
||||
# The index is keyed by the raw edge_type string ("causes" and "CAUSED"
|
||||
# are separate keys), so filter by normalized type instead of iterating
|
||||
# a fixed spelling list.
|
||||
for edge_type, edges in self.edge_type_index.items():
|
||||
if edge_type.upper() not in _CAUSAL_TRAVERSAL_TYPES:
|
||||
continue
|
||||
for edge in edges:
|
||||
if edge.source_id == decision_id and edge.target_id in self._decisions:
|
||||
direct_influence.add(edge.target_id)
|
||||
elif edge.target_id == decision_id and edge.source_id in self._decisions:
|
||||
@@ -4455,8 +4486,12 @@ class ContextGraph:
|
||||
# record_decision() (e.g. a graph restored via from_dict), so only
|
||||
# causes with a known decision record are kept.
|
||||
incoming_causal_edges = defaultdict(list)
|
||||
for edge_type in _CAUSAL_EDGE_TYPES:
|
||||
for edge in self.edge_type_index.get(edge_type, []):
|
||||
# The index is keyed by the raw edge_type string ("causes" and
|
||||
# "CAUSED" are separate keys), so filter by normalized type.
|
||||
for edge_type, edges in self.edge_type_index.items():
|
||||
if edge_type.upper() not in _CAUSAL_TRAVERSAL_TYPES:
|
||||
continue
|
||||
for edge in edges:
|
||||
if edge.source_id in self._decisions:
|
||||
incoming_causal_edges[edge.target_id].append(edge)
|
||||
|
||||
|
||||
@@ -69,6 +69,15 @@ class EmbeddingGeneratorWithProvenance:
|
||||
return embeddings
|
||||
|
||||
def __getattr__(self, name):
|
||||
# __getattr__ only runs when normal lookup fails. Accessing
|
||||
# self._generator by attribute syntax HERE would re-enter
|
||||
# __getattr__ for ever when _generator itself is missing — the shape
|
||||
# pickle/copy protocol probes hit when __init__ never completed
|
||||
# (#994's RecursionError family). Fail fast on private probes.
|
||||
if name.startswith("_"):
|
||||
raise AttributeError(
|
||||
f"{type(self).__name__!r} object has no attribute {name!r}"
|
||||
)
|
||||
return getattr(self._generator, name)
|
||||
|
||||
|
||||
|
||||
@@ -117,9 +117,9 @@ def generate_embeddings(
|
||||
>>> emb = generate_embeddings("Hello world", method="default")
|
||||
>>> embs = generate_embeddings(["text1", "text2"], method="text")
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
# Check for custom method in registry, skip self-reference
|
||||
custom_method = method_registry.get("generation", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not generate_embeddings:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, data_type=data_type, fallback_on_custom_error=fallback, **kwargs
|
||||
@@ -165,9 +165,9 @@ def embed_text(
|
||||
>>> emb = embed_text("Hello world", method="sentence_transformers")
|
||||
>>> embs = embed_text(["text1", "text2"], method="sentence_transformers")
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
# Check for custom method in registry, skip self-reference
|
||||
custom_method = method_registry.get("text", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not embed_text:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
|
||||
@@ -225,9 +225,9 @@ def calculate_similarity(
|
||||
>>> similarity = calculate_similarity(emb1, emb2, method="cosine")
|
||||
>>> print(f"Similarity: {similarity:.3f}")
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
# Check for custom method in registry, skip self-reference
|
||||
custom_method = method_registry.get("similarity", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not calculate_similarity:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, embedding1, embedding2, fallback_on_custom_error=fallback, **kwargs
|
||||
@@ -272,9 +272,9 @@ def pool_embeddings(
|
||||
>>> pooled = pool_embeddings(embeddings, method="mean")
|
||||
>>> attention_pooled = pool_embeddings(embeddings, method="attention")
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
# Check for custom method in registry, skip self-reference
|
||||
custom_method = method_registry.get("pooling", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not pool_embeddings:
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, embeddings, fallback_on_custom_error=fallback, **kwargs
|
||||
|
||||
@@ -28,12 +28,35 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.helpers import ensure_directory, utc_now_iso, write_json_file
|
||||
from ..utils.helpers import ensure_directory, hash_data, utc_now_iso, write_json_file
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri
|
||||
|
||||
|
||||
def _content_iri(prefix: str, payload: Any) -> str:
|
||||
"""Mint a document IRI from what was exported, not when.
|
||||
|
||||
Minting from ``utc_now_iso()`` gave every export of the same graph a new
|
||||
identity a few microseconds apart, so re-exporting an unchanged graph was
|
||||
never idempotent and merging exports duplicated every node (#1147). This
|
||||
mirrors ``mint_entity_iri`` (#1109): identical content hashes to the same
|
||||
IRI, and any change to the content changes it too. ``default=str`` keeps
|
||||
the hash defined for values ``json.dumps`` would otherwise reject, such as
|
||||
``datetime`` objects a caller may have left in the graph.
|
||||
|
||||
Args:
|
||||
prefix: IRI prefix the digest is appended to
|
||||
payload: JSON-serializable value whose content determines the digest
|
||||
|
||||
Returns:
|
||||
A stable IRI of the form ``{prefix}{16-hex-char digest}``
|
||||
"""
|
||||
canonical = json.dumps(payload, sort_keys=True, default=str)
|
||||
digest = hash_data(canonical)[:16]
|
||||
return f"{prefix}{digest}"
|
||||
|
||||
|
||||
def _is_jsonld_document(data: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
Report whether a dictionary is already a JSON-LD document.
|
||||
@@ -230,7 +253,10 @@ class JSONExporter:
|
||||
- statistics: Statistics dictionary (optional)
|
||||
file_path: Output JSON file path
|
||||
format: Export format - 'json' or 'json-ld' (default: self.format)
|
||||
**options: Additional options passed to conversion methods
|
||||
**options: Additional options passed to conversion methods:
|
||||
- graph_uri: Caller-supplied IRI for the graph node when
|
||||
format='json-ld', overriding the default content-derived
|
||||
IRI (see #1147)
|
||||
|
||||
Example:
|
||||
>>> kg = {
|
||||
@@ -401,7 +427,9 @@ class JSONExporter:
|
||||
data: Data to convert (dict, list, or any value)
|
||||
include_metadata: Whether to include metadata (default: True)
|
||||
include_provenance: Whether to include provenance (default: True)
|
||||
**options: Additional options passed to knowledge graph conversion
|
||||
**options: Additional options passed to knowledge graph conversion:
|
||||
- document_uri: Caller-supplied IRI for the document node,
|
||||
overriding the default content-derived IRI (see #1147)
|
||||
|
||||
Returns:
|
||||
Dictionary in JSON-LD format with @context, @graph/@value, and metadata
|
||||
@@ -451,13 +479,17 @@ class JSONExporter:
|
||||
|
||||
# Add metadata and provenance if requested
|
||||
if include_metadata:
|
||||
self._attach_document_metadata(jsonld, include_provenance)
|
||||
self._attach_document_metadata(
|
||||
jsonld, include_provenance, options.get("document_uri")
|
||||
)
|
||||
|
||||
return jsonld
|
||||
|
||||
@staticmethod
|
||||
def _attach_document_metadata(
|
||||
jsonld: Dict[str, Any], include_provenance: bool
|
||||
jsonld: Dict[str, Any],
|
||||
include_provenance: bool,
|
||||
document_uri: Optional[str] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Attach the export's own metadata without naming the graph.
|
||||
@@ -473,6 +505,9 @@ class JSONExporter:
|
||||
Args:
|
||||
jsonld: Document being built, modified in place
|
||||
include_provenance: Whether to record how and when it was exported
|
||||
document_uri: Caller-supplied IRI for the document node. Falls back
|
||||
to a content-derived IRI (#1147) so re-exporting unchanged data
|
||||
is idempotent instead of minting a new identity every time.
|
||||
"""
|
||||
# A caller may hand us a document that is deliberately a named graph.
|
||||
# That name is theirs to keep, but our own statements must not end up
|
||||
@@ -483,7 +518,10 @@ class JSONExporter:
|
||||
# Do not overwrite an identifier the payload already carries: the
|
||||
# knowledge-graph conversion names its own document node.
|
||||
if "@id" not in jsonld or payload_is_named_graph:
|
||||
document["@id"] = f"https://semantica.dev/data/{utc_now_iso()}"
|
||||
content = {key: value for key, value in jsonld.items() if key != "@context"}
|
||||
document["@id"] = document_uri or _content_iri(
|
||||
"https://semantica.dev/data/", content
|
||||
)
|
||||
if include_provenance:
|
||||
document["semantica:exportedAt"] = utc_now_iso()
|
||||
document["semantica:format"] = "json-ld"
|
||||
@@ -553,7 +591,9 @@ class JSONExporter:
|
||||
- entities: List of entity dictionaries
|
||||
- relationships: List of relationship dictionaries
|
||||
- metadata: Metadata dictionary (optional)
|
||||
**options: Additional options (unused)
|
||||
**options: Additional options:
|
||||
- graph_uri: Caller-supplied IRI for the graph node,
|
||||
overriding the default content-derived IRI (see #1147)
|
||||
|
||||
Returns:
|
||||
Dictionary in JSON-LD format with @context, @id, @type, and graph data
|
||||
@@ -566,7 +606,11 @@ class JSONExporter:
|
||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||
},
|
||||
"@id": f"https://semantica.dev/graph/{utc_now_iso()}",
|
||||
# Minted from the graph's own content rather than the wall clock
|
||||
# (#1147): re-exporting an unchanged graph must produce the same
|
||||
# subject, or merging repeated exports duplicates every node.
|
||||
"@id": options.get("graph_uri")
|
||||
or _content_iri("https://semantica.dev/graph/", kg),
|
||||
"@type": "semantica:KnowledgeGraph",
|
||||
}
|
||||
|
||||
|
||||
@@ -159,6 +159,7 @@ from .ontology_validator import (
|
||||
SHACLValidationReport,
|
||||
SHACLViolation,
|
||||
ValidationResult,
|
||||
run_shacl_validation,
|
||||
validate_ontology,
|
||||
)
|
||||
from .owl_generator import OWLGenerator
|
||||
@@ -192,6 +193,7 @@ __all__ = [
|
||||
"PropertyShape",
|
||||
"SHACLValidationReport",
|
||||
"SHACLViolation",
|
||||
"run_shacl_validation",
|
||||
# OWL/RDF generation
|
||||
"OWLGenerator",
|
||||
# Requirements and competency questions
|
||||
|
||||
@@ -145,14 +145,14 @@ class SHACLValidationReport:
|
||||
}
|
||||
|
||||
|
||||
def _run_pyshacl(
|
||||
def run_shacl_validation(
|
||||
data_graph_str: str,
|
||||
shacl_str: str,
|
||||
data_graph_format: str = "turtle",
|
||||
shacl_format: str = "turtle",
|
||||
) -> SHACLValidationReport:
|
||||
"""
|
||||
Run pyshacl validation and return a structured SHACLValidationReport.
|
||||
Run pySHACL validation and return a structured SHACLValidationReport.
|
||||
|
||||
Args:
|
||||
data_graph_str: Serialized data graph string.
|
||||
@@ -272,6 +272,21 @@ def _run_pyshacl(
|
||||
raw_report=results_text,
|
||||
)
|
||||
|
||||
|
||||
def _run_pyshacl(
|
||||
data_graph_str: str,
|
||||
shacl_str: str,
|
||||
data_graph_format: str = "turtle",
|
||||
shacl_format: str = "turtle",
|
||||
) -> SHACLValidationReport:
|
||||
"""Backward-compatible alias for :func:`run_shacl_validation`."""
|
||||
return run_shacl_validation(
|
||||
data_graph_str,
|
||||
shacl_str,
|
||||
data_graph_format=data_graph_format,
|
||||
shacl_format=shacl_format,
|
||||
)
|
||||
|
||||
@dataclass
|
||||
class ValidationResult:
|
||||
"""Result of an ontology validation operation."""
|
||||
|
||||
@@ -779,9 +779,11 @@ def extract_entities_huggingface(
|
||||
"""
|
||||
loader = HuggingFaceModelLoader(device=device)
|
||||
# Pass kwargs (like aggregation_strategy) to load_ner_model
|
||||
model_obj = loader.load_ner_model(model, **kwargs)
|
||||
loader_kwargs = {
|
||||
key: value for key, value in kwargs.items() if key != "huggingface_model"
|
||||
}
|
||||
model_obj = loader.load_ner_model(model, **loader_kwargs)
|
||||
results = loader.extract_entities(model_obj, text)
|
||||
|
||||
entities = []
|
||||
|
||||
# Check if manual aggregation is needed (raw IOB tags detected)
|
||||
|
||||
@@ -297,7 +297,11 @@ def create_index(
|
||||
config = vector_store_config.get_all()
|
||||
backend = config.get("default_backend", "faiss")
|
||||
dimension = config.get("dimension", 768)
|
||||
indexer = VectorIndexer(backend=backend, dimension=dimension, **config)
|
||||
# backend/dimension are already passed explicitly; drop them from the
|
||||
# forwarded config so VectorIndexer(..., **remaining_config) doesn't
|
||||
# receive duplicate keyword arguments.
|
||||
remaining_config = {k: v for k, v in config.items() if k not in ("default_backend", "dimension")}
|
||||
indexer = VectorIndexer(backend=backend, dimension=dimension, **remaining_config)
|
||||
return indexer.create_index(vectors, ids, **options)
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import errno
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -56,6 +60,25 @@ def markdown_document(frontmatter, body=""):
|
||||
return f"---\n{yaml_text}---\n\n{body}"
|
||||
|
||||
|
||||
def _require_symlink_support(tmp_path):
|
||||
"""Skip the test if this environment cannot create symbolic links.
|
||||
|
||||
Symlink creation can be unavailable even on POSIX (e.g. restricted
|
||||
containers) and commonly requires elevated privilege or Developer Mode
|
||||
on Windows. Probe actual capability instead of assuming based on
|
||||
platform, so these tests still run wherever symlinks genuinely work.
|
||||
"""
|
||||
probe_target = tmp_path / ".symlink_probe_target"
|
||||
probe_link = tmp_path / ".symlink_probe_link"
|
||||
probe_target.write_text("", encoding="utf-8")
|
||||
try:
|
||||
probe_link.symlink_to(probe_target)
|
||||
except OSError as exc:
|
||||
pytest.skip(f"environment cannot create symbolic links: {exc}")
|
||||
probe_link.unlink()
|
||||
probe_target.unlink()
|
||||
|
||||
|
||||
def required_frontmatter(memory_id="mem_test", **overrides):
|
||||
frontmatter = {
|
||||
"id": memory_id,
|
||||
@@ -707,6 +730,151 @@ def test_markdown_string_path_inspection_errors_are_actionable():
|
||||
assert exc_info.value.__cause__ is original_error
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_string_path", [False, True])
|
||||
def test_markdown_import_rejects_symlinked_file(tmp_path, use_string_path):
|
||||
_require_symlink_support(tmp_path)
|
||||
outside = tmp_path / "outside.md"
|
||||
outside.write_text(
|
||||
markdown_document(required_frontmatter(), "Do not import"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
source = tmp_path / "memory.md"
|
||||
source.symlink_to(outside)
|
||||
payload = str(source) if use_string_path else source
|
||||
memory = AgentMemory()
|
||||
|
||||
with pytest.raises(ValueError, match="symbolic link"):
|
||||
memory.import_data(payload, format="markdown")
|
||||
|
||||
assert memory.count() == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_string_path", [False, True])
|
||||
def test_markdown_import_rejects_broken_symlink(tmp_path, use_string_path):
|
||||
_require_symlink_support(tmp_path)
|
||||
source = tmp_path / "missing-memory.md"
|
||||
source.symlink_to(tmp_path / "missing-target.md")
|
||||
payload = str(source) if use_string_path else source
|
||||
memory = AgentMemory()
|
||||
|
||||
with pytest.raises(ValueError, match="symbolic link"):
|
||||
memory.import_data(payload, format="markdown")
|
||||
|
||||
assert memory.count() == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_string_path", [False, True])
|
||||
def test_markdown_import_rejects_symlinked_directory(tmp_path, use_string_path):
|
||||
_require_symlink_support(tmp_path)
|
||||
outside = tmp_path / "outside"
|
||||
outside.mkdir()
|
||||
(outside / "memory.md").write_text(
|
||||
markdown_document(required_frontmatter(), "Do not import"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
source = tmp_path / "memory-export"
|
||||
source.symlink_to(outside, target_is_directory=True)
|
||||
payload = str(source) if use_string_path else source
|
||||
memory = AgentMemory()
|
||||
|
||||
with pytest.raises(ValueError, match="symbolic link"):
|
||||
memory.import_data(payload, format="markdown")
|
||||
|
||||
assert memory.count() == 0
|
||||
|
||||
|
||||
def test_markdown_import_skips_symlinked_file_in_directory(tmp_path):
|
||||
_require_symlink_support(tmp_path)
|
||||
outside = tmp_path / "outside.md"
|
||||
outside.write_text(
|
||||
markdown_document(required_frontmatter(), "Do not import"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
source = tmp_path / "memory-export"
|
||||
source.mkdir()
|
||||
(source / "memory.md").symlink_to(outside)
|
||||
memory = AgentMemory()
|
||||
|
||||
assert memory.import_data(source, format="markdown") == 0
|
||||
assert memory.count() == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not hasattr(os, "O_NOFOLLOW"), reason="requires O_NOFOLLOW")
|
||||
def test_markdown_import_does_not_follow_symlink_raced_before_open(tmp_path):
|
||||
_require_symlink_support(tmp_path)
|
||||
outside = tmp_path / "outside.md"
|
||||
outside.write_text(
|
||||
markdown_document(required_frontmatter(), "Do not import"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
source = tmp_path / "memory.md"
|
||||
source.symlink_to(outside)
|
||||
|
||||
with patch.object(Path, "is_symlink", return_value=False):
|
||||
with pytest.raises(ValueError, match="symbolic link"):
|
||||
AgentMemory()._read_markdown_file_content(source)
|
||||
|
||||
|
||||
def test_markdown_import_rejects_windows_junction(tmp_path, monkeypatch):
|
||||
source = tmp_path / "junction"
|
||||
source.mkdir()
|
||||
(source / "memory.md").write_text("not read", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(
|
||||
os.path,
|
||||
"isjunction",
|
||||
lambda candidate: Path(candidate) == source,
|
||||
raising=False,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="junction"):
|
||||
AgentMemory().import_data(source, format="markdown")
|
||||
|
||||
|
||||
def test_markdown_import_rejects_windows_reparse_point_fallback(tmp_path, monkeypatch):
|
||||
source = tmp_path / "reparse-point"
|
||||
source.mkdir()
|
||||
real_lstat = os.lstat
|
||||
|
||||
class ReparseStat:
|
||||
st_file_attributes = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
|
||||
|
||||
monkeypatch.delattr(os.path, "isjunction", raising=False)
|
||||
monkeypatch.setattr(Path, "is_symlink", lambda self: False)
|
||||
monkeypatch.setattr(
|
||||
os,
|
||||
"lstat",
|
||||
lambda candidate: (
|
||||
ReparseStat() if Path(candidate) == source else real_lstat(candidate)
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="junction"):
|
||||
AgentMemory().import_data(source, format="markdown")
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.name != "nt", reason="requires Windows junctions")
|
||||
def test_markdown_import_rejects_real_windows_junction(tmp_path):
|
||||
outside = tmp_path / "outside"
|
||||
outside.mkdir()
|
||||
(outside / "memory.md").write_text("not read", encoding="utf-8")
|
||||
source = tmp_path / "junction"
|
||||
result = subprocess.run(
|
||||
["cmd.exe", "/c", "mklink", "/J", str(source), str(outside)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
pytest.skip(f"could not create Windows junction: {result.stderr}")
|
||||
|
||||
try:
|
||||
with pytest.raises(ValueError, match="junction"):
|
||||
AgentMemory().import_data(source, format="markdown")
|
||||
finally:
|
||||
os.rmdir(source)
|
||||
|
||||
|
||||
def test_legacy_dict_import_behavior_is_unchanged():
|
||||
memory = AgentMemory()
|
||||
data = {
|
||||
@@ -751,6 +919,7 @@ def test_markdown_import_file_open_security_rejects_symlink(tmp_path):
|
||||
with pytest.raises(ValueError, match="Symlink Markdown import paths are rejected"):
|
||||
memory._read_markdown_file_content(symlink_file)
|
||||
|
||||
|
||||
def test_markdown_import_public_api_rejects_symlink(tmp_path):
|
||||
"""
|
||||
import_data(..., format="markdown") must propagate the symlink rejection
|
||||
|
||||
@@ -7,6 +7,8 @@ extraction found nothing, the chain came back empty even though an explicit
|
||||
``CAUSED`` edge was stored in the graph.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.context.context_graph import ContextEdge
|
||||
|
||||
@@ -323,3 +325,130 @@ def test_entity_based_inference_still_applies_without_explicit_edges():
|
||||
hop["from"] == earlier and hop["to"] == later and hop["type"] == "influences"
|
||||
for hop in hops
|
||||
)
|
||||
|
||||
|
||||
def test_get_causal_chain_accepts_lowercase_causal_edge_types():
|
||||
"""Issue #1184: edges recorded with the analyzer's lowercase vocabulary
|
||||
must be traversed by get_causal_chain().
|
||||
|
||||
CausalChainAnalyzer documents causal types as lowercase ("causes",
|
||||
"influences", ...) while get_causal_chain() matched only the uppercase
|
||||
spellings, so an edge recorded as "causes" produced an empty audit
|
||||
chain — silent and in the dangerous direction.
|
||||
"""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
cause = graph.record_decision(
|
||||
category="a", scenario="upstream", reasoning="r",
|
||||
outcome="x", confidence=0.9,
|
||||
)
|
||||
effect = graph.record_decision(
|
||||
category="b", scenario="downstream", reasoning="r",
|
||||
outcome="y", confidence=0.9,
|
||||
)
|
||||
graph.add_edge(cause, effect, "causes")
|
||||
|
||||
chain = graph.get_causal_chain(effect, direction="upstream")
|
||||
|
||||
assert [decision.decision_id for decision in chain] == [cause]
|
||||
|
||||
|
||||
def test_add_causal_relationship_accepts_any_case_and_stores_canonical():
|
||||
"""Issue #1184: add_causal_relationship() should accept either spelling
|
||||
and store the canonical uppercase vocabulary."""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
cause = graph.record_decision(
|
||||
category="a", scenario="upstream", reasoning="r",
|
||||
outcome="x", confidence=0.9,
|
||||
)
|
||||
effect = graph.record_decision(
|
||||
category="b", scenario="downstream", reasoning="r",
|
||||
outcome="y", confidence=0.9,
|
||||
)
|
||||
|
||||
graph.add_causal_relationship(cause, effect, relationship_type="causes")
|
||||
|
||||
edges = [
|
||||
edge for edge in graph.edges
|
||||
if edge.source_id == cause and edge.target_id == effect
|
||||
]
|
||||
assert edges, "add_causal_relationship must store the edge"
|
||||
assert edges[0].edge_type == "CAUSED"
|
||||
|
||||
|
||||
def test_add_causal_relationship_rejects_non_string_with_value_error():
|
||||
"""Invalid relationship types must keep raising ValueError (issue #1184
|
||||
follow-up): normalization must not turn them into AttributeError."""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
cause = graph.record_decision(
|
||||
category="a", scenario="upstream", reasoning="r",
|
||||
outcome="x", confidence=0.9,
|
||||
)
|
||||
effect = graph.record_decision(
|
||||
category="b", scenario="downstream", reasoning="r",
|
||||
outcome="y", confidence=0.9,
|
||||
)
|
||||
|
||||
for bad_type in (None, 42, ["CAUSED"]):
|
||||
with pytest.raises(ValueError):
|
||||
graph.add_causal_relationship(cause, effect, relationship_type=bad_type)
|
||||
|
||||
|
||||
def test_analyze_decision_influence_sees_lowercase_causal_edge():
|
||||
"""Issue #1184 follow-up: influence analysis reads the same edge index as
|
||||
the causal traversal, so lowercase edges must count as direct influence."""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
cause = graph.record_decision(
|
||||
category="a", scenario="upstream", reasoning="r",
|
||||
outcome="x", confidence=0.9,
|
||||
)
|
||||
effect = graph.record_decision(
|
||||
category="b", scenario="downstream", reasoning="r",
|
||||
outcome="y", confidence=0.9,
|
||||
)
|
||||
graph.add_edge(cause, effect, "causes")
|
||||
|
||||
impact = graph.analyze_decision_influence(cause)
|
||||
|
||||
direct_ids = {entry["decision_id"] for entry in impact["direct_influence"]}
|
||||
assert effect in direct_ids
|
||||
|
||||
|
||||
def test_trace_decision_causality_sees_lowercase_causal_edge():
|
||||
"""Issue #1184 follow-up: the trace must not return an empty audit chain
|
||||
for a decision with an explicit lowercase upstream causal edge."""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
cause = graph.record_decision(
|
||||
category="a", scenario="upstream", reasoning="r",
|
||||
outcome="x", confidence=0.9,
|
||||
)
|
||||
effect = graph.record_decision(
|
||||
category="b", scenario="downstream", reasoning="r",
|
||||
outcome="y", confidence=0.9,
|
||||
)
|
||||
graph.add_edge(cause, effect, "causes")
|
||||
|
||||
trace = graph.trace_decision_causality(effect)
|
||||
|
||||
assert any(
|
||||
hop["from"] == cause and hop["to"] == effect
|
||||
for chain in trace for hop in chain["hops"]
|
||||
), "lowercase causal edge must appear in the traced chain"
|
||||
|
||||
|
||||
def test_find_precedents_sees_lowercase_precedent_edge():
|
||||
"""Issue #1184 follow-up: precedent lookup must accept the analyzer's
|
||||
spelling alongside the canonical PRECEDENT_FOR."""
|
||||
graph = ContextGraph(advanced_analytics=True)
|
||||
precedent = graph.record_decision(
|
||||
category="a", scenario="earlier", reasoning="r",
|
||||
outcome="x", confidence=0.9,
|
||||
)
|
||||
later = graph.record_decision(
|
||||
category="b", scenario="later", reasoning="r",
|
||||
outcome="y", confidence=0.9,
|
||||
)
|
||||
graph.add_edge(precedent, later, "precedes")
|
||||
|
||||
precedents = graph.find_precedents(later)
|
||||
|
||||
assert [d.decision_id for d in precedents] == [precedent]
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
"""The document IRI of a JSON-LD export must depend on content, not the clock
|
||||
(issue #1147).
|
||||
|
||||
``_convert_kg_to_jsonld`` minted the graph's ``@id`` from ``utc_now_iso()``,
|
||||
and the generic ``_attach_document_metadata`` path did the same for a plain
|
||||
document ``@id``. Exporting an unchanged graph therefore produced a new
|
||||
subject every time: three exports of one one-entity graph merged into 3
|
||||
``semantica:KnowledgeGraph`` nodes and 15 triples for what should have been a
|
||||
single graph. Neither identifier resolves and the timestamp is already
|
||||
recorded correctly in ``semantica:exportedAt``, so the fix mints the IRI from
|
||||
the exported content instead (mirroring ``mint_entity_iri``, #1109), with an
|
||||
optional caller-supplied override for callers who already name their graphs.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from rdflib import RDF, Graph, URIRef
|
||||
|
||||
from semantica.export.json_exporter import JSONExporter
|
||||
|
||||
KG = {
|
||||
"entities": [{"id": "https://example.org/e1", "text": "Acme Corp", "type": "ORG"}],
|
||||
"relationships": [],
|
||||
}
|
||||
|
||||
OTHER_KG = {
|
||||
"entities": [
|
||||
{"id": "https://example.org/e1", "text": "Acme Corp Renamed", "type": "ORG"}
|
||||
],
|
||||
"relationships": [],
|
||||
}
|
||||
|
||||
|
||||
def _export(kg, tmp_path, name="out.jsonld", **options):
|
||||
path = tmp_path / name
|
||||
JSONExporter().export_knowledge_graph(kg, path, format="json-ld", **options)
|
||||
return path
|
||||
|
||||
|
||||
def test_reexporting_an_unchanged_graph_is_idempotent(tmp_path):
|
||||
"""The whole point of an identifier: same content, same @id."""
|
||||
first = json.loads(_export(KG, tmp_path, "a.jsonld").read_text())
|
||||
second = json.loads(_export(KG, tmp_path, "b.jsonld").read_text())
|
||||
|
||||
assert first["@id"] == second["@id"]
|
||||
|
||||
|
||||
def test_a_changed_graph_gets_a_different_id(tmp_path):
|
||||
unchanged = json.loads(_export(KG, tmp_path, "a.jsonld").read_text())
|
||||
changed = json.loads(_export(OTHER_KG, tmp_path, "b.jsonld").read_text())
|
||||
|
||||
assert unchanged["@id"] != changed["@id"]
|
||||
|
||||
|
||||
def test_merging_repeated_exports_yields_one_graph_node(tmp_path):
|
||||
"""Regression for the exact repro in #1147: churn no longer multiplies nodes."""
|
||||
merged = Graph()
|
||||
for i in range(3):
|
||||
path = _export(KG, tmp_path, f"churn{i}.jsonld")
|
||||
merged.parse(str(path), format="json-ld")
|
||||
|
||||
# Exactly one subject typed as a KnowledgeGraph, regardless of how many
|
||||
# times the unchanged graph was exported and merged.
|
||||
kg_nodes = set(
|
||||
merged.subjects(RDF.type, URIRef("https://semantica.dev/ns#KnowledgeGraph"))
|
||||
)
|
||||
assert len(kg_nodes) == 1
|
||||
|
||||
entity_nodes = set(
|
||||
merged.subjects(RDF.type, URIRef("https://semantica.dev/vocab/ORG"))
|
||||
)
|
||||
assert len(entity_nodes) == 1
|
||||
|
||||
|
||||
def test_exported_at_still_varies_between_exports(tmp_path):
|
||||
"""Identity is now content-derived, but provenance still records each run."""
|
||||
first = json.loads(_export(KG, tmp_path, "a.jsonld").read_text())
|
||||
second = json.loads(_export(KG, tmp_path, "b.jsonld").read_text())
|
||||
|
||||
assert first["@id"] == second["@id"]
|
||||
assert first["semantica:exportedAt"] != second["semantica:exportedAt"]
|
||||
|
||||
|
||||
def test_caller_supplied_graph_uri_is_honored(tmp_path):
|
||||
path = _export(KG, tmp_path, graph_uri="https://example.org/my-graph")
|
||||
document = json.loads(path.read_text())
|
||||
|
||||
assert document["@id"] == "https://example.org/my-graph"
|
||||
|
||||
|
||||
def _document_node_id(document):
|
||||
"""The generic (non-knowledge-graph) path hangs its own @id off a member
|
||||
of @graph rather than the top level, to avoid re-creating the named-graph
|
||||
bug fixed by #1145. Find that member and return its @id."""
|
||||
for node in document["@graph"]:
|
||||
if "semantica:exportedAt" in node:
|
||||
return node["@id"]
|
||||
raise AssertionError(f"no document metadata node in @graph: {document}")
|
||||
|
||||
|
||||
def test_caller_supplied_document_uri_is_honored_for_a_generic_export(tmp_path):
|
||||
payload = {"note": "no entities or relationships here"}
|
||||
path = tmp_path / "generic.jsonld"
|
||||
JSONExporter().export(
|
||||
payload, path, format="json-ld", document_uri="https://example.org/my-doc"
|
||||
)
|
||||
document = json.loads(path.read_text())
|
||||
|
||||
assert _document_node_id(document) == "https://example.org/my-doc"
|
||||
|
||||
|
||||
def test_generic_document_id_is_also_content_derived(tmp_path):
|
||||
"""The non-knowledge-graph path (_attach_document_metadata) gets the same fix."""
|
||||
payload = {"note": "plain data, no @id of its own"}
|
||||
|
||||
first = tmp_path / "a.jsonld"
|
||||
second = tmp_path / "b.jsonld"
|
||||
JSONExporter().export(payload, first, format="json-ld")
|
||||
JSONExporter().export(dict(payload), second, format="json-ld")
|
||||
|
||||
first_id = _document_node_id(json.loads(first.read_text()))
|
||||
second_id = _document_node_id(json.loads(second.read_text()))
|
||||
assert first_id == second_id
|
||||
|
||||
|
||||
def test_document_id_still_differs_for_different_generic_payloads(tmp_path):
|
||||
a = tmp_path / "a.jsonld"
|
||||
b = tmp_path / "b.jsonld"
|
||||
JSONExporter().export({"note": "one"}, a, format="json-ld")
|
||||
JSONExporter().export({"note": "two"}, b, format="json-ld")
|
||||
|
||||
a_id = _document_node_id(json.loads(a.read_text()))
|
||||
b_id = _document_node_id(json.loads(b.read_text()))
|
||||
assert a_id != b_id
|
||||
@@ -107,20 +107,31 @@ def test_exported_timestamp_survives_a_timezone_qualified_sparql_filter():
|
||||
assert len(rows) == 1, "the export was dropped by a timezone-qualified filter"
|
||||
|
||||
|
||||
def test_document_iri_carrying_an_offset_is_a_valid_iri():
|
||||
"""The offset puts '+' and ':' in the @id; both are legal in a path."""
|
||||
def test_document_iri_is_a_valid_iri():
|
||||
"""The graph @id must be a valid IRI regardless of how it is minted.
|
||||
|
||||
Before #1147, this @id was minted from the offset-carrying timestamp
|
||||
itself (``+00:00`` interpolated straight into the path), so this test
|
||||
asserted the offset survived without breaking IRI validity. #1147 mints
|
||||
the @id from the graph's content instead, so the timestamp no longer
|
||||
appears here at all — it stays in ``semantica:exportedAt`` (still
|
||||
offset-aware, per ``test_jsonld_export_timestamp_is_offset_aware`` above).
|
||||
What's left worth guarding is the general case: whatever the @id is
|
||||
minted from, it has to be a valid IRI that round-trips through RDF.
|
||||
"""
|
||||
rdflib = pytest.importorskip("rdflib")
|
||||
|
||||
document_iri = JSONExporter()._convert_kg_to_jsonld(KG)["@id"]
|
||||
assert "+00:00" in document_iri
|
||||
assert rdflib.term._is_valid_uri(document_iri)
|
||||
|
||||
graph = rdflib.Graph()
|
||||
graph.add((
|
||||
rdflib.URIRef(document_iri),
|
||||
rdflib.RDF.type,
|
||||
rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"),
|
||||
))
|
||||
graph.add(
|
||||
(
|
||||
rdflib.URIRef(document_iri),
|
||||
rdflib.RDF.type,
|
||||
rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"),
|
||||
)
|
||||
)
|
||||
reparsed = rdflib.Graph().parse(data=graph.serialize(format="nt"), format="nt")
|
||||
assert document_iri in {str(s) for s in reparsed.subjects()}
|
||||
|
||||
|
||||
@@ -525,6 +525,65 @@ class TestSHACLHierarchicalAndValidation(unittest.TestCase):
|
||||
self.assertEqual(mc[0].max_count, 2)
|
||||
|
||||
# 33
|
||||
def test_public_run_shacl_validation_api(self):
|
||||
"""The public API validates data and retains the legacy alias."""
|
||||
try:
|
||||
import pyshacl # noqa: F401
|
||||
import rdflib # noqa: F401
|
||||
except ImportError:
|
||||
self.skipTest("pyshacl/rdflib not installed")
|
||||
from semantica.ontology import run_shacl_validation
|
||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||
|
||||
data = "@prefix ex: <http://example.org/> . ex:alice a ex:Person ."
|
||||
shacl = """
|
||||
@prefix ex: <http://example.org/> .
|
||||
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
||||
ex:PersonShape a sh:NodeShape ; sh:targetClass ex:Person ;
|
||||
sh:property [ sh:path ex:name ; sh:minCount 1 ] .
|
||||
"""
|
||||
public_report = run_shacl_validation(data, shacl)
|
||||
legacy_report = _run_pyshacl(data, shacl)
|
||||
self.assertFalse(public_report.conforms)
|
||||
self.assertEqual(public_report.violation_count, 1)
|
||||
self.assertEqual(legacy_report.conforms, public_report.conforms)
|
||||
self.assertEqual(legacy_report.violation_count, public_report.violation_count)
|
||||
self.assertEqual(
|
||||
[
|
||||
(v.focus_node, v.result_path, v.constraint, v.severity, v.message)
|
||||
for v in legacy_report.violations
|
||||
],
|
||||
[
|
||||
(v.focus_node, v.result_path, v.constraint, v.severity, v.message)
|
||||
for v in public_report.violations
|
||||
],
|
||||
)
|
||||
|
||||
# 34
|
||||
def test_public_run_shacl_validation_conforming_graph(self):
|
||||
"""The public API reports a valid graph without violations."""
|
||||
try:
|
||||
import pyshacl # noqa: F401
|
||||
import rdflib # noqa: F401
|
||||
except ImportError:
|
||||
self.skipTest("pyshacl/rdflib not installed")
|
||||
from semantica.ontology import run_shacl_validation
|
||||
|
||||
data = """
|
||||
@prefix ex: <http://example.org/> .
|
||||
ex:alice a ex:Person ; ex:name "Alice" .
|
||||
"""
|
||||
shacl = """
|
||||
@prefix ex: <http://example.org/> .
|
||||
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
||||
ex:PersonShape a sh:NodeShape ; sh:targetClass ex:Person ;
|
||||
sh:property [ sh:path ex:name ; sh:minCount 1 ] .
|
||||
"""
|
||||
|
||||
report = run_shacl_validation(data, shacl)
|
||||
|
||||
self.assertTrue(report.conforms)
|
||||
self.assertEqual(report.violation_count, 0)
|
||||
def test_shacl_violation_to_dict(self):
|
||||
from semantica.ontology.ontology_validator import SHACLViolation
|
||||
v = SHACLViolation(
|
||||
|
||||
@@ -1851,3 +1851,126 @@ class TestExitCodes:
|
||||
assert "Traceback" not in result.output, (
|
||||
f"Traceback found for {argv}: {result.output}"
|
||||
)
|
||||
|
||||
|
||||
class TestDoctorEmbeddings:
|
||||
"""#994: doctor must surface non-functional embedding backends instead of
|
||||
reporting all green. Default = import-level check; --deep-embeddings (or
|
||||
SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates via TextEmbedder."""
|
||||
|
||||
def _doctor_checks(self, runner, *extra):
|
||||
result = runner.invoke(cli_module.main, ["doctor", "--json", *extra])
|
||||
_ok(result)
|
||||
import json as _json
|
||||
return {c["check"]: c for c in _json.loads(result.output)}
|
||||
|
||||
def _with_fake_st(self, monkeypatch, **embedder_attrs):
|
||||
fake_st = _fake_module(
|
||||
__version__="9.9.9",
|
||||
SentenceTransformer=object,
|
||||
)
|
||||
monkeypatch.setitem(__import__("sys").modules, "sentence_transformers", fake_st)
|
||||
|
||||
def test_doctor_reports_embedding_checks(self, runner):
|
||||
checks = self._doctor_checks(runner)
|
||||
assert "Embeddings (sentence-transformers)" in checks
|
||||
assert "Embeddings (fastembed)" in checks
|
||||
|
||||
def test_import_failure_is_fail_status_with_hint(self, runner, monkeypatch):
|
||||
# Force the 'import sentence_transformers' inside _embedding_backend to
|
||||
# raise ImportError regardless of whether the package is installed on
|
||||
# this machine. Setting a module entry to None is the standard Python
|
||||
# mechanism: any subsequent 'import <name>' raises
|
||||
# "import of <name> halted; None in sys.modules".
|
||||
monkeypatch.setitem(
|
||||
__import__("sys").modules, "sentence_transformers", None
|
||||
)
|
||||
checks = self._doctor_checks(runner)
|
||||
st = checks["Embeddings (sentence-transformers)"]
|
||||
assert st["status"] == "fail"
|
||||
assert st["hint"] == "pip install sentence-transformers"
|
||||
|
||||
def test_deep_probe_detects_fallback_active(self, runner, monkeypatch):
|
||||
self._with_fake_st(monkeypatch)
|
||||
fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None)
|
||||
|
||||
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
|
||||
|
||||
checks = self._doctor_checks(runner, "--deep-embeddings")
|
||||
st = checks["Embeddings (sentence-transformers)"]
|
||||
assert st["status"] == "fail"
|
||||
assert "hash fallback" in st["note"]
|
||||
|
||||
def test_deep_probe_ok_when_model_loads(self, runner, monkeypatch):
|
||||
self._with_fake_st(monkeypatch)
|
||||
import numpy as np
|
||||
fake_embedder = types.SimpleNamespace(
|
||||
model=object(),
|
||||
fastembed_model=None,
|
||||
embed_text=lambda text: np.zeros(384, dtype=np.float32),
|
||||
)
|
||||
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
|
||||
|
||||
checks = self._doctor_checks(runner, "--deep-embeddings")
|
||||
st = checks["Embeddings (sentence-transformers)"]
|
||||
assert st["status"] == "ok"
|
||||
assert "384-dim" in st["note"]
|
||||
|
||||
def test_env_var_enables_deep_mode(self, runner, monkeypatch):
|
||||
monkeypatch.setenv("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", "1")
|
||||
self._with_fake_st(monkeypatch)
|
||||
fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None)
|
||||
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
|
||||
|
||||
checks = self._doctor_checks(runner)
|
||||
st = checks["Embeddings (sentence-transformers)"]
|
||||
assert st["status"] == "fail"
|
||||
assert "hash fallback" in st["note"]
|
||||
|
||||
|
||||
class TestDoctorEmbeddingHintsAndEnv:
|
||||
"""Review follow-ups: deep failures must not carry the pip-install hint,
|
||||
and the env toggle tolerates case/whitespace variants."""
|
||||
|
||||
def _doctor_checks(self, runner, *extra):
|
||||
result = runner.invoke(cli_module.main, ["doctor", "--json", *extra])
|
||||
_ok(result)
|
||||
import json as _json
|
||||
return {c["check"]: c for c in _json.loads(result.output)}
|
||||
|
||||
def _with_fake_st(self, monkeypatch):
|
||||
fake_st = _fake_module(
|
||||
__version__="9.9.9",
|
||||
SentenceTransformer=object,
|
||||
)
|
||||
monkeypatch.setitem(__import__("sys").modules, "sentence_transformers", fake_st)
|
||||
|
||||
def test_deep_failure_hint_is_not_pip_install(self, runner, monkeypatch):
|
||||
self._with_fake_st(monkeypatch)
|
||||
fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None)
|
||||
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
|
||||
|
||||
checks = self._doctor_checks(runner, "--deep-embeddings")
|
||||
st = checks["Embeddings (sentence-transformers)"]
|
||||
assert st["status"] == "fail"
|
||||
assert "pip install" not in (st["hint"] or ""), (
|
||||
"a deep probe failure means the package imported fine — pointing "
|
||||
"users at pip sends them to reinstall for a runtime/model problem"
|
||||
)
|
||||
assert "runtime/model-load" in st["hint"]
|
||||
|
||||
def test_env_var_tolerates_case_and_whitespace(self, runner, monkeypatch):
|
||||
monkeypatch.setenv("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", " TRUE ")
|
||||
self._with_fake_st(monkeypatch)
|
||||
fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None)
|
||||
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
|
||||
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
|
||||
|
||||
checks = self._doctor_checks(runner)
|
||||
st = checks["Embeddings (sentence-transformers)"]
|
||||
assert st["status"] == "fail"
|
||||
assert "hash fallback" in st["note"], "padded/caps env value must enable deep mode"
|
||||
|
||||
@@ -85,3 +85,49 @@ if __name__ == '__main__':
|
||||
runner = unittest.TextTestRunner(stream=f, verbosity=2)
|
||||
unittest.main(testRunner=runner, exit=False)
|
||||
|
||||
|
||||
class TestMethodDispatchRecursion(unittest.TestCase):
|
||||
"""#994: built-in aliases are registered in the method registry onto the
|
||||
wrapper functions themselves, so dispatching through the registry called a
|
||||
wrapper back into itself with the same default method — a recursion storm
|
||||
that surfaced as `maximum recursion depth exceeded` during model loading."""
|
||||
|
||||
def test_generate_embeddings_default_does_not_self_recurse(self):
|
||||
from semantica.embeddings.methods import generate_embeddings
|
||||
emb = generate_embeddings("recursion probe")
|
||||
self.assertIsNotNone(emb)
|
||||
|
||||
def test_embed_text_default_does_not_self_recurse(self):
|
||||
from semantica.embeddings.methods import embed_text
|
||||
emb = embed_text("recursion probe", method="sentence_transformers")
|
||||
self.assertIsNotNone(emb)
|
||||
|
||||
def test_custom_registered_method_still_wins(self):
|
||||
from semantica.embeddings.methods import method_registry
|
||||
calls = []
|
||||
|
||||
def spy(data, *a, **k):
|
||||
calls.append(data)
|
||||
return {"custom": True}
|
||||
|
||||
method_registry.register("generation", "my_custom_gen", spy)
|
||||
try:
|
||||
from semantica.embeddings.methods import generate_embeddings
|
||||
out = generate_embeddings("payload", method="my_custom_gen")
|
||||
self.assertEqual(out, {"custom": True})
|
||||
self.assertEqual(calls, ["payload"])
|
||||
finally:
|
||||
method_registry.unregister("generation", "my_custom_gen")
|
||||
|
||||
def test_provenance_wrapper_missing_generator_raises_attribute_error(self):
|
||||
# Partially-initialised wrappers (failed __init__, pickle/copy probes)
|
||||
# must raise AttributeError, not RecursionError via __getattr__.
|
||||
from semantica.embeddings.embeddings_provenance import (
|
||||
EmbeddingGeneratorWithProvenance,
|
||||
)
|
||||
bare = EmbeddingGeneratorWithProvenance.__new__(
|
||||
EmbeddingGeneratorWithProvenance
|
||||
)
|
||||
with self.assertRaises(AttributeError):
|
||||
getattr(bare, "model")
|
||||
|
||||
|
||||
@@ -243,5 +243,19 @@ class TestVectorStore(unittest.TestCase):
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
|
||||
class TestCreateIndexFunction(unittest.TestCase):
|
||||
"""create_index() forwards vector_store_config's defaults into VectorIndexer,
|
||||
which already receives backend/dimension as explicit args. Regression for the
|
||||
'got multiple values for keyword argument dimension' crash on the default
|
||||
(unmocked) config, hit by e.g. `semantica embed index`."""
|
||||
|
||||
def test_create_index_with_default_config(self):
|
||||
from semantica.vector_store.methods import create_index
|
||||
|
||||
vectors = [np.array([0.1, 0.2, 0.3]), np.array([0.4, 0.5, 0.6])]
|
||||
index = create_index(vectors, ids=["a", "b"])
|
||||
self.assertIsNotNone(index)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user