mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Merge branch 'main' into codex/context-graph-markdown-round-trip
This commit is contained in:
@@ -408,6 +408,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
|
- **Markdown import followed filesystem links even though Markdown export already refused to overwrite them** (#851, follow-up to #765, #786) by @SaurabhScripts
|
||||||
|
- `AgentMemory._read_markdown_path()` now rejects symlink files, broken symlinks, symlinked directories, Windows directory junctions, and other Windows reparse points supplied directly; linked entries discovered inside an otherwise valid directory are safely skipped, preserving the current directory-import contract
|
||||||
|
- `_read_markdown_file_content()` re-checks the file and parent directory immediately before and after opening, uses `O_NOFOLLOW` where available, and verifies the resulting descriptor is a regular file via `fstat`/`S_ISREG`, so link swaps are rejected rather than silently followed
|
||||||
|
- Junction detection uses `os.path.isjunction()` where available and falls back to the Windows reparse-point file attribute on older Python versions; export applies the same link check before replacing a Markdown file
|
||||||
|
- Documented the import restriction in `docs/reference/context.md`; added 11 tests to `tests/context/test_agent_memory_markdown.py` covering file/directory/broken-symlink rejection, simulated open races, mocked and real Windows junctions, and the reparse-point fallback
|
||||||
|
- Any additional review follow-up commits land in this same PR/entry rather than as a separate changelog item
|
||||||
|
|
||||||
- **`PipelineWithProvenance` raised `ModuleNotFoundError` on import and `AttributeError` on `.run()`** (#858, closes #858) by @Karunasagar12
|
- **`PipelineWithProvenance` raised `ModuleNotFoundError` on import and `AttributeError` on `.run()`** (#858, closes #858) by @Karunasagar12
|
||||||
- `from .pipeline import Pipeline` failed because `semantica/pipeline/pipeline.py` does not exist; corrected to `from .pipeline_builder import Pipeline`
|
- `from .pipeline import Pipeline` failed because `semantica/pipeline/pipeline.py` does not exist; corrected to `from .pipeline_builder import Pipeline`
|
||||||
- `.run()` called `self._pipeline.run()` on the `Pipeline` dataclass, which has no such method; replaced with `self._engine.execute_pipeline(self._pipeline, ...)` delegating to `ExecutionEngine`
|
- `.run()` called `self._pipeline.run()` on the `Pipeline` dataclass, which has no such method; replaced with `self._engine.execute_pipeline(self._pipeline, ...)` delegating to `ExecutionEngine`
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ icon: "shield-check"
|
|||||||
|
|
||||||
SHACL (Shapes Constraint Language) is a standard for validating graph-based data. While an ontology defines the conceptual *schema* (the "what" exists in your domain), SHACL defines the structural *rules and constraints* (the "how" it should be structured).
|
SHACL (Shapes Constraint Language) is a standard for validating graph-based data. While an ontology defines the conceptual *schema* (the "what" exists in your domain), SHACL defines the structural *rules and constraints* (the "how" it should be structured).
|
||||||
|
|
||||||
In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and `_run_pyshacl` evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated.
|
In Semantica, `SHACLGenerator` produces constraint rules (shapes) based on your ontology, and the public `run_shacl_validation` function evaluates your actual data against these rules. If a node violates a rule (e.g., missing a required property or using the wrong datatype), a detailed violation report is generated. The historical `_run_pyshacl` name remains available as a compatibility alias.
|
||||||
|
|
||||||
## Why Use SHACL Validation?
|
## Why Use SHACL Validation?
|
||||||
|
|
||||||
@@ -55,7 +55,7 @@ Let's look at a simple, universally understood example: ensuring every `Employee
|
|||||||
```python
|
```python
|
||||||
from semantica.context import ContextGraph
|
from semantica.context import ContextGraph
|
||||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
from semantica.ontology import run_shacl_validation
|
||||||
|
|
||||||
# 1. Prepare your data graph
|
# 1. Prepare your data graph
|
||||||
graph = ContextGraph()
|
graph = ContextGraph()
|
||||||
@@ -95,7 +95,7 @@ data_ttl = """
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# 5. Run Validation
|
# 5. Run Validation
|
||||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||||
|
|
||||||
# 6. Analyze the Report
|
# 6. Analyze the Report
|
||||||
print(f"Graph conforms: {report.conforms}")
|
print(f"Graph conforms: {report.conforms}")
|
||||||
@@ -265,10 +265,10 @@ cve_id_shape = NodeShape(
|
|||||||
|
|
||||||
## Step 4 — Run validation and read the report
|
## Step 4 — Run validation and read the report
|
||||||
|
|
||||||
Serialize the graph to RDF, then run `_run_pyshacl` against the shapes.
|
Serialize the graph to RDF, then run `run_shacl_validation` against the shapes.
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
from semantica.ontology import run_shacl_validation
|
||||||
|
|
||||||
# Prepare your RDF data string (since export_rdf primarily exports structural metadata,
|
# Prepare your RDF data string (since export_rdf primarily exports structural metadata,
|
||||||
# you typically serialize your custom data graph to Turtle using rdflib or similar).
|
# you typically serialize your custom data graph to Turtle using rdflib or similar).
|
||||||
@@ -281,7 +281,7 @@ data_ttl = """
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# Run SHACL validation
|
# Run SHACL validation
|
||||||
report = _run_pyshacl(
|
report = run_shacl_validation(
|
||||||
data_ttl,
|
data_ttl,
|
||||||
shacl_ttl,
|
shacl_ttl,
|
||||||
data_graph_format="turtle",
|
data_graph_format="turtle",
|
||||||
@@ -366,8 +366,8 @@ print(f"Malware nodes missing 'family': {len(missing_family)}")
|
|||||||
# e.g. graph.update_node(node_id, {"family": "UNKNOWN — requires triage"})
|
# e.g. graph.update_node(node_id, {"family": "UNKNOWN — requires triage"})
|
||||||
|
|
||||||
# After remediation, re-run validation to confirm the fix
|
# After remediation, re-run validation to confirm the fix
|
||||||
# (re-export the patched graph to Turtle first, then call _run_pyshacl again)
|
# (re-export the patched graph to Turtle first, then call run_shacl_validation again)
|
||||||
report2 = _run_pyshacl(patched_data_ttl, shacl_ttl)
|
report2 = run_shacl_validation(patched_data_ttl, shacl_ttl)
|
||||||
print(f"Violations after remediation: {report2.violation_count}")
|
print(f"Violations after remediation: {report2.violation_count}")
|
||||||
# Violations after remediation: 0
|
# Violations after remediation: 0
|
||||||
```
|
```
|
||||||
@@ -377,10 +377,49 @@ print(f"Violations after remediation: {report2.violation_count}")
|
|||||||
## Common Pitfalls
|
## Common Pitfalls
|
||||||
|
|
||||||
- **Assuming the ontology automatically enforces data quality**: `SHACLGenerator` generates shapes based on what it observes in the data. If your data is missing a field, the generator won't know it was mandatory unless you explicitly inject the constraint (as shown in Step 3).
|
- **Assuming the ontology automatically enforces data quality**: `SHACLGenerator` generates shapes based on what it observes in the data. If your data is missing a field, the generator won't know it was mandatory unless you explicitly inject the constraint (as shown in Step 3).
|
||||||
- **Passing `ContextGraph` directly to SHACL validators**: The `_run_pyshacl` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object.
|
- **Passing `ContextGraph` directly to SHACL validators**: The `run_shacl_validation` function expects an RDF string (like Turtle format), not a raw Python dictionary or `ContextGraph` object.
|
||||||
- **Forgetting RDF serialization**: You must serialize your graph (often via a temporary file using `export_rdf`) before validating it.
|
- **Forgetting RDF serialization**: You must serialize your graph (often via a temporary file using `export_rdf`) before validating it.
|
||||||
- **Treating validation as a one-time step**: Validation should be integrated as an automated step in your CI/CD pipeline or data ingestion flow, acting as a recurring gatekeeper rather than a one-off script.
|
- **Treating validation as a one-time step**: Validation should be integrated as an automated step in your CI/CD pipeline or data ingestion flow, acting as a recurring gatekeeper rather than a one-off script.
|
||||||
- **Ignoring validation reports**: A graph that does not conform must be remediated. Failing to review the `violation_count` and address the issues negates the purpose of SHACL validation.
|
- **Ignoring validation reports**: A graph that does not conform must be remediated. Failing to review the `violation_count` and address the issues negates the purpose of SHACL validation.
|
||||||
|
- **Validating `sh:class`/`sh:node` range checks on a property that declares `rdfs:range` with RDFS entailment on**: RDFS is an entailment rule, not a constraint. When pyshacl runs with `inference="rdfs"`, it infers the range class onto every object of the property, so class-based constraints on that property can never fail — the report says `conforms: True` on data that does not conform:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from pyshacl import validate
|
||||||
|
from rdflib import Graph
|
||||||
|
|
||||||
|
data = Graph()
|
||||||
|
data.parse(
|
||||||
|
data="""
|
||||||
|
@prefix ex: <https://example.org/ns#> .
|
||||||
|
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
|
||||||
|
ex:contains rdfs:domain ex:Container ; rdfs:range ex:Item .
|
||||||
|
ex:box a ex:Container ; ex:contains ex:notAnItem .
|
||||||
|
ex:notAnItem a ex:Fish .
|
||||||
|
""",
|
||||||
|
format="turtle",
|
||||||
|
)
|
||||||
|
|
||||||
|
shapes = Graph()
|
||||||
|
shapes.parse(
|
||||||
|
data="""
|
||||||
|
@prefix ex: <https://example.org/ns#> .
|
||||||
|
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
||||||
|
ex:ContainerShape a sh:NodeShape ;
|
||||||
|
sh:targetClass ex:Container ;
|
||||||
|
sh:property [ sh:path ex:contains ; sh:class ex:Item ] .
|
||||||
|
""",
|
||||||
|
format="turtle",
|
||||||
|
)
|
||||||
|
|
||||||
|
for inference in ("none", "rdfs"):
|
||||||
|
conforms, _, _ = validate(data, shacl_graph=shapes, inference=inference)
|
||||||
|
print(inference, conforms)
|
||||||
|
# none False <- correct: notAnItem is a Fish, not an Item
|
||||||
|
# rdfs True <- the entailment manufactured the type
|
||||||
|
```
|
||||||
|
|
||||||
|
Mitigations: prefer not to declare `rdfs:range` on properties you intend to constrain with `sh:class`; when class membership is the thing under test, run validation without RDFS entailment (`inference="none"`); or express the check as a constraint the entailment cannot satisfy (for example a literal property constraint). Note the trade-off: with entailment off, `sh:targetClass` no longer reaches subclasses, so subclass hierarchies need explicit typing or inference-aware target selection. Semantica's own `run_shacl_validation` wrapper already calls pyshacl with `inference="none"`, so this pitfall only bites when calling `pyshacl.validate` directly with entailment enabled.
|
||||||
|
- **Trusting `conforms: True` without checking the inference mode**: an inference-enabled run can hide the exact violations the shapes were written to catch (see above). Record which inference mode validation ran under alongside the result, and re-run shape sets that contain `sh:class`/`sh:node` with entailment off before treating a pass as authoritative.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -396,7 +435,7 @@ A DoD CTI team enforces STIX-compatible constraints on a threat graph before sha
|
|||||||
from semantica.context import AgentContext, ContextGraph
|
from semantica.context import AgentContext, ContextGraph
|
||||||
from semantica.vector_store import VectorStore
|
from semantica.vector_store import VectorStore
|
||||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
from semantica.ontology import run_shacl_validation
|
||||||
|
|
||||||
graph = ContextGraph()
|
graph = ContextGraph()
|
||||||
ctx = AgentContext(
|
ctx = AgentContext(
|
||||||
@@ -448,7 +487,7 @@ data_ttl = """
|
|||||||
<http://example.org/hammertoss> a ex:Malware .
|
<http://example.org/hammertoss> a ex:Malware .
|
||||||
"""
|
"""
|
||||||
|
|
||||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||||
print(f"CTI graph conforms : {report.conforms}")
|
print(f"CTI graph conforms : {report.conforms}")
|
||||||
print(f"Violations : {report.violation_count}")
|
print(f"Violations : {report.violation_count}")
|
||||||
print(f"Warnings : {report.warning_count}")
|
print(f"Warnings : {report.warning_count}")
|
||||||
@@ -469,7 +508,7 @@ A SOC team validates zero-trust policy nodes before publishing them to the polic
|
|||||||
```python
|
```python
|
||||||
from semantica.context import ContextGraph
|
from semantica.context import ContextGraph
|
||||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
from semantica.ontology import run_shacl_validation
|
||||||
|
|
||||||
graph = ContextGraph()
|
graph = ContextGraph()
|
||||||
graph.add_node("policy-001", "Policy", "MFA Required for Tier-1 Resources",
|
graph.add_node("policy-001", "Policy", "MFA Required for Tier-1 Resources",
|
||||||
@@ -516,7 +555,7 @@ data_ttl = """
|
|||||||
<http://example.org/policy-002> a ex:Policy .
|
<http://example.org/policy-002> a ex:Policy .
|
||||||
"""
|
"""
|
||||||
|
|
||||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||||
print(f"Policy graph conforms: {report.conforms}")
|
print(f"Policy graph conforms: {report.conforms}")
|
||||||
# Policy graph conforms: False
|
# Policy graph conforms: False
|
||||||
|
|
||||||
@@ -534,7 +573,7 @@ A clinical informatics team validates trial ontology nodes before loading them i
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
from semantica.ontology import LLMOntologyGenerator, SHACLGenerator, PropertyShape
|
from semantica.ontology import LLMOntologyGenerator, SHACLGenerator, PropertyShape
|
||||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
from semantica.ontology import run_shacl_validation
|
||||||
from semantica.export import export_rdf
|
from semantica.export import export_rdf
|
||||||
import tempfile, os
|
import tempfile, os
|
||||||
|
|
||||||
@@ -586,7 +625,7 @@ with open(tmp.name) as f:
|
|||||||
data_ttl = f.read()
|
data_ttl = f.read()
|
||||||
os.unlink(tmp.name)
|
os.unlink(tmp.name)
|
||||||
|
|
||||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||||
print(f"Trial data conforms: {report.conforms}")
|
print(f"Trial data conforms: {report.conforms}")
|
||||||
print(f"Warnings : {report.warning_count}")
|
print(f"Warnings : {report.warning_count}")
|
||||||
```
|
```
|
||||||
@@ -600,7 +639,7 @@ A credit risk team validates every `LoanApplication` node against Basel III CRE2
|
|||||||
```python
|
```python
|
||||||
from semantica.context import ContextGraph
|
from semantica.context import ContextGraph
|
||||||
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
from semantica.ontology import OntologyGenerator, SHACLGenerator, PropertyShape
|
||||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
from semantica.ontology import run_shacl_validation
|
||||||
|
|
||||||
graph = ContextGraph()
|
graph = ContextGraph()
|
||||||
graph.add_node("loan-001", "LoanApplication", "Prime mortgage APP-2025-88421",
|
graph.add_node("loan-001", "LoanApplication", "Prime mortgage APP-2025-88421",
|
||||||
@@ -645,7 +684,7 @@ data_ttl = """
|
|||||||
ex:ltv "0.65" .
|
ex:ltv "0.65" .
|
||||||
"""
|
"""
|
||||||
|
|
||||||
report = _run_pyshacl(data_ttl, shacl_ttl)
|
report = run_shacl_validation(data_ttl, shacl_ttl)
|
||||||
print(f"Loan portfolio conforms: {report.conforms}")
|
print(f"Loan portfolio conforms: {report.conforms}")
|
||||||
# Loan portfolio conforms: False
|
# Loan portfolio conforms: False
|
||||||
|
|
||||||
@@ -675,14 +714,14 @@ Call this function as a pre-publish gate; exit code 1 blocks the pipeline.
|
|||||||
```python
|
```python
|
||||||
import sys
|
import sys
|
||||||
from semantica.ontology import OntologyGenerator, SHACLGenerator
|
from semantica.ontology import OntologyGenerator, SHACLGenerator
|
||||||
from semantica.ontology.ontology_validator import _run_pyshacl
|
from semantica.ontology import run_shacl_validation
|
||||||
|
|
||||||
def validate_before_publish(data_graph_str: str, ontology: dict) -> None:
|
def validate_before_publish(data_graph_str: str, ontology: dict) -> None:
|
||||||
shacl_gen = SHACLGenerator(base_uri="https://example.org/shapes/")
|
shacl_gen = SHACLGenerator(base_uri="https://example.org/shapes/")
|
||||||
shacl_graph = shacl_gen.generate(ontology)
|
shacl_graph = shacl_gen.generate(ontology)
|
||||||
shacl_ttl = shacl_gen.serialize(shacl_graph, format="turtle")
|
shacl_ttl = shacl_gen.serialize(shacl_graph, format="turtle")
|
||||||
|
|
||||||
report = _run_pyshacl(data_graph_str, shacl_ttl)
|
report = run_shacl_validation(data_graph_str, shacl_ttl)
|
||||||
|
|
||||||
if not report.conforms:
|
if not report.conforms:
|
||||||
print(f"Graph validation FAILED — {report.violation_count} violation(s)")
|
print(f"Graph validation FAILED — {report.violation_count} violation(s)")
|
||||||
@@ -700,7 +739,6 @@ def validate_before_publish(data_graph_str: str, ontology: dict) -> None:
|
|||||||
|
|
||||||
- [Ontology Management](ontology) — generate the OWL ontology that SHACL shapes are derived from
|
- [Ontology Management](ontology) — generate the OWL ontology that SHACL shapes are derived from
|
||||||
- [Reasoning & Rules](reasoning) — complement SHACL structural constraints with logical inference rules
|
- [Reasoning & Rules](reasoning) — complement SHACL structural constraints with logical inference rules
|
||||||
- [Export & Serialization](export) — serialize graph data to Turtle/RDF/XML for `_run_pyshacl` input
|
- [Export & Serialization](export) — serialize graph data to Turtle/RDF/XML for `run_shacl_validation` input
|
||||||
- [Conflict Resolution](conflict-resolution) — detect and resolve data conflicts before SHACL validation
|
- [Conflict Resolution](conflict-resolution) — detect and resolve data conflicts before SHACL validation
|
||||||
- [Change Management](change-management) — version-gate SHACL shapes alongside ontology versions
|
- [Change Management](change-management) — version-gate SHACL shapes alongside ontology versions
|
||||||
|
|
||||||
|
|||||||
@@ -625,8 +625,10 @@ malformed or duplicate fields before changing memory, and re-importing unchanged
|
|||||||
files is idempotent. Memory-local `entities` and `relationships` are preserved as
|
files is idempotent. Memory-local `entities` and `relationships` are preserved as
|
||||||
provenance but are not applied to `ContextGraph` by Markdown import. Use a dedicated
|
provenance but are not applied to `ContextGraph` by Markdown import. Use a dedicated
|
||||||
export directory: matching files are overwritten, but unrelated or stale Markdown
|
export directory: matching files are overwritten, but unrelated or stale Markdown
|
||||||
files are not deleted automatically. Export refuses to overwrite symbolic links and
|
files are not deleted automatically. Export refuses to overwrite filesystem links and
|
||||||
uses atomic file replacement. Timestamp offsets are preserved in Markdown and
|
uses atomic file replacement; import also refuses symlinks, Windows directory
|
||||||
|
junctions, and other Windows reparse points.
|
||||||
|
Timestamp offsets are preserved in Markdown and
|
||||||
normalized to UTC only for comparisons, so aware and local-naive records can be
|
normalized to UTC only for comparisons, so aware and local-naive records can be
|
||||||
queried together safely. Vector-store writes are deferred until the in-memory import
|
queried together safely. Vector-store writes are deferred until the in-memory import
|
||||||
commits; adapter synchronization remains best-effort and logs failures.
|
commits; adapter synchronization remains best-effort and logs failures.
|
||||||
|
|||||||
+2
-1
@@ -85,7 +85,8 @@ dependencies = [
|
|||||||
"loguru>=0.7.3",
|
"loguru>=0.7.3",
|
||||||
"structlog>=22.1.0",
|
"structlog>=22.1.0",
|
||||||
"gensim>=4.4.0",
|
"gensim>=4.4.0",
|
||||||
"httpx<0.29.0"
|
"httpx<0.29.0",
|
||||||
|
"pyarrow>=14.0.0"
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.urls]
|
[project.urls]
|
||||||
|
|||||||
+9
-9
@@ -6,9 +6,9 @@ accelerate==1.14.0 \
|
|||||||
# via
|
# via
|
||||||
# docling-ibm-models
|
# docling-ibm-models
|
||||||
# docling-slim
|
# docling-slim
|
||||||
agno==2.8.7 \
|
agno==2.9.0 \
|
||||||
--hash=sha256:6a2763eb469163f7b79ab1da6ca2f22d8619f6b9d614574f975d9c12bb4323ea \
|
--hash=sha256:7777674b3931b341fad4fcf02a61b185a08588c509101348facf87feb2144c0c \
|
||||||
--hash=sha256:d49396a2062ee6994ca82695b9bd1e1b95667fec432c544afa38133e564bf090
|
--hash=sha256:7d9c134703e3c2798023cd57dcb9caa8e1174f6914813f9b130becfc3521a46f
|
||||||
# via semantica (pyproject.toml)
|
# via semantica (pyproject.toml)
|
||||||
agnoctl==0.1.3 \
|
agnoctl==0.1.3 \
|
||||||
--hash=sha256:6fce1d2482b1f2e0a3d14b0a7c12fbd49d8df4f0bf0a4fd9fd91753cbff5efdc \
|
--hash=sha256:6fce1d2482b1f2e0a3d14b0a7c12fbd49d8df4f0bf0a4fd9fd91753cbff5efdc \
|
||||||
@@ -403,9 +403,9 @@ boto3==1.43.69 \
|
|||||||
--hash=sha256:4eb494d05b2bd08a7eee61b8ac4c34745c99e9bbce435c91f8d15d372dd8c2db \
|
--hash=sha256:4eb494d05b2bd08a7eee61b8ac4c34745c99e9bbce435c91f8d15d372dd8c2db \
|
||||||
--hash=sha256:76297a0b415849c63575ae08a4f1661b2dc8ee0100f104b86f98aa69b47fa2c7
|
--hash=sha256:76297a0b415849c63575ae08a4f1661b2dc8ee0100f104b86f98aa69b47fa2c7
|
||||||
# via semantica (pyproject.toml)
|
# via semantica (pyproject.toml)
|
||||||
botocore==1.43.69 \
|
botocore==1.43.73 \
|
||||||
--hash=sha256:5caa46b740d9a886137146ffbb69edb691f702bfe74c64e85621947ae00181fd \
|
--hash=sha256:068433028e011ccbeab1dd7c46b1090c24e378397693c66e67ca571176498daa \
|
||||||
--hash=sha256:b1f0e01c53d6b84ee9c184ebf3636c3b3aef85e0ae8498c74afb8734ff224f87
|
--hash=sha256:0fa1e63c24b3531be3e1bc1687a88b3be9e63a430153f24edd93efc162bb1c51
|
||||||
# via
|
# via
|
||||||
# boto3
|
# boto3
|
||||||
# s3transfer
|
# s3transfer
|
||||||
@@ -1731,9 +1731,9 @@ google-crc32c==1.8.0 \
|
|||||||
# via
|
# via
|
||||||
# google-cloud-storage
|
# google-cloud-storage
|
||||||
# google-resumable-media
|
# google-resumable-media
|
||||||
google-genai==2.17.0 \
|
google-genai==2.18.1 \
|
||||||
--hash=sha256:6b640a2390c82b4a240873eddb9f518c6d2c33244b2de16ed3d14526a6da7f57 \
|
--hash=sha256:36a5949233e64a60f6cc4521bff7a76b7c569d0aa227bbe9fa642213b8a3a3b2 \
|
||||||
--hash=sha256:a4835563c60aee646c9c4b261c507aa4a624710d25017012d20dc65abf3d9a54
|
--hash=sha256:a1e2be75c16234adc6641afd1ad4dd44218c9eec005d938bdc428585a048918a
|
||||||
# via semantica (pyproject.toml)
|
# via semantica (pyproject.toml)
|
||||||
google-resumable-media==2.10.1 \
|
google-resumable-media==2.10.1 \
|
||||||
--hash=sha256:224975032ddb73f7ed9e2f0f4cc08ed1b06874c52d48cc8533e3eb72980b21a0 \
|
--hash=sha256:224975032ddb73f7ed9e2f0f4cc08ed1b06874c52d48cc8533e3eb72980b21a0 \
|
||||||
|
|||||||
+104
-2
@@ -773,10 +773,22 @@ def changelog(cli_ctx: CLIContext, local_json: bool) -> None:
|
|||||||
_run_with_error_handling(_action)
|
_run_with_error_handling(_action)
|
||||||
|
|
||||||
|
|
||||||
|
class _DeepEmbeddingFailure(Exception):
|
||||||
|
"""A deep-probe failure from doctor's embedding checks.
|
||||||
|
|
||||||
|
Marks failures that happened AFTER the backend imported cleanly — model
|
||||||
|
load, probe, or runtime problems — so the check's hint can point at the
|
||||||
|
real remediation instead of `pip install`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
@main.command()
|
@main.command()
|
||||||
@click.option("--json", "local_json", is_flag=True, default=False)
|
@click.option("--json", "local_json", is_flag=True, default=False)
|
||||||
|
@click.option("--deep-embeddings", "deep_embeddings", is_flag=True, default=False,
|
||||||
|
help="Also instantiate the local embedding backends and embed a probe "
|
||||||
|
"text (catches backends that import cleanly but cannot load).")
|
||||||
@click.pass_obj
|
@click.pass_obj
|
||||||
def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
|
def doctor(cli_ctx: CLIContext, local_json: bool, deep_embeddings: bool) -> None:
|
||||||
"""Run a health check on all Semantica components and backends."""
|
"""Run a health check on all Semantica components and backends."""
|
||||||
import importlib.metadata
|
import importlib.metadata
|
||||||
cli_ctx = _require_ctx(cli_ctx)
|
cli_ctx = _require_ctx(cli_ctx)
|
||||||
@@ -787,6 +799,16 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
|
|||||||
try:
|
try:
|
||||||
note = fn()
|
note = fn()
|
||||||
return label, "ok", note, None
|
return label, "ok", note, None
|
||||||
|
except _DeepEmbeddingFailure as exc:
|
||||||
|
# A deep-probe failure means the package IMPORTED fine: the pip
|
||||||
|
# hint would be the wrong remediation for what is actually a
|
||||||
|
# runtime/model-load problem (broken torch, failed model
|
||||||
|
# download, missing shared libs).
|
||||||
|
return label, "fail", str(exc), (
|
||||||
|
"runtime/model-load failure — reinstalling the package usually "
|
||||||
|
"does not help; check the warnings above (torch install, model "
|
||||||
|
"download, disk space)"
|
||||||
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return label, "fail", str(exc), hint
|
return label, "fail", str(exc), hint
|
||||||
|
|
||||||
@@ -827,6 +849,50 @@ def doctor(cli_ctx: CLIContext, local_json: bool) -> None:
|
|||||||
return f"{backend} importable"
|
return f"{backend} importable"
|
||||||
checks.append(_check("Vector store", _vector, hint="pip install semantica[vectorstore-…]"))
|
checks.append(_check("Vector store", _vector, hint="pip install semantica[vectorstore-…]"))
|
||||||
|
|
||||||
|
# Embedding backends (#994): `doctor` used to report all green while
|
||||||
|
# every local embedding backend was non-functional — import success
|
||||||
|
# says nothing about model loading. Default checks stay cheap
|
||||||
|
# (import + version); --deep-embeddings (or
|
||||||
|
# SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates the backend through
|
||||||
|
# TextEmbedder and embeds a probe, which is the only level that
|
||||||
|
# catches a backend that imports cleanly but cannot actually load.
|
||||||
|
deep = deep_embeddings or os.environ.get("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", "").strip().lower() in ("1", "true", "yes", "on")
|
||||||
|
|
||||||
|
def _embedding_backend(method: str) -> str:
|
||||||
|
if method == "sentence_transformers":
|
||||||
|
import sentence_transformers # noqa: F401
|
||||||
|
note = f"importable ({importlib.metadata.version('sentence-transformers')})"
|
||||||
|
else:
|
||||||
|
import fastembed # noqa: F401
|
||||||
|
note = f"importable ({importlib.metadata.version('fastembed')})"
|
||||||
|
if not deep:
|
||||||
|
return note
|
||||||
|
try:
|
||||||
|
from .embeddings import TextEmbedder
|
||||||
|
embedder = TextEmbedder(method=method)
|
||||||
|
if embedder.model is None and embedder.fastembed_model is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
"model failed to load — the hash fallback is active "
|
||||||
|
"(see warnings above); embedding quality is degraded"
|
||||||
|
)
|
||||||
|
probe = embedder.embed_text("semantica doctor embedding probe")
|
||||||
|
except _DeepEmbeddingFailure:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
raise _DeepEmbeddingFailure(str(exc)) from exc
|
||||||
|
return f"{note}; deep probe ok ({len(probe)}-dim)"
|
||||||
|
|
||||||
|
checks.append(_check(
|
||||||
|
"Embeddings (sentence-transformers)",
|
||||||
|
lambda: _embedding_backend("sentence_transformers"),
|
||||||
|
hint="pip install sentence-transformers",
|
||||||
|
))
|
||||||
|
checks.append(_check(
|
||||||
|
"Embeddings (fastembed)",
|
||||||
|
lambda: _embedding_backend("fastembed"),
|
||||||
|
hint="pip install fastembed",
|
||||||
|
))
|
||||||
|
|
||||||
# LLM provider keys
|
# LLM provider keys
|
||||||
for provider, var in [("OpenAI", "OPENAI_API_KEY"), ("Anthropic", "ANTHROPIC_API_KEY"),
|
for provider, var in [("OpenAI", "OPENAI_API_KEY"), ("Anthropic", "ANTHROPIC_API_KEY"),
|
||||||
("Groq", "GROQ_API_KEY")]:
|
("Groq", "GROQ_API_KEY")]:
|
||||||
@@ -1708,7 +1774,43 @@ def embed_generate(cli_ctx: CLIContext, input_path: str, model: str,
|
|||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise click.ClickException(f"Embeddings module not available: {exc}") from exc
|
raise click.ClickException(f"Embeddings module not available: {exc}") from exc
|
||||||
if output:
|
if output:
|
||||||
Path(output).write_text(json.dumps(result, default=str), encoding="utf-8")
|
output_path = Path(output)
|
||||||
|
suffix = output_path.suffix.lower()
|
||||||
|
try:
|
||||||
|
import numpy as np
|
||||||
|
import pandas as pd
|
||||||
|
arr = np.asarray(result)
|
||||||
|
if arr.ndim == 1:
|
||||||
|
arr = arr[np.newaxis, :]
|
||||||
|
if arr.ndim != 2:
|
||||||
|
raise click.ClickException(
|
||||||
|
f"embed generate --output expects a 1-D or 2-D array, "
|
||||||
|
f"got {arr.ndim}-D (shape {arr.shape})"
|
||||||
|
)
|
||||||
|
rows = [list(row) for row in arr]
|
||||||
|
if suffix == ".parquet":
|
||||||
|
# Schema: single 'embedding' column (list[float] per row).
|
||||||
|
# embed index detects vector columns via
|
||||||
|
# isinstance(df[c].iloc[0], (list, np.ndarray)).
|
||||||
|
df = pd.DataFrame({"embedding": rows})
|
||||||
|
df.to_parquet(output_path, index=False)
|
||||||
|
elif suffix in (".json", ".jsonl"):
|
||||||
|
df = pd.DataFrame({"embedding": rows})
|
||||||
|
df.to_json(
|
||||||
|
output_path,
|
||||||
|
orient="records",
|
||||||
|
lines=(suffix == ".jsonl"),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise click.ClickException(
|
||||||
|
f"Unsupported output format '{suffix}'. "
|
||||||
|
"Use .parquet, .json, or .jsonl"
|
||||||
|
)
|
||||||
|
except ImportError as exc:
|
||||||
|
raise click.ClickException(
|
||||||
|
f"Missing dependency for --output: {exc}. "
|
||||||
|
"Install pyarrow with: pip install pyarrow"
|
||||||
|
) from exc
|
||||||
_ok(cli_ctx, f"Wrote {output}")
|
_ok(cli_ctx, f"Wrote {output}")
|
||||||
elif _is_json(cli_ctx, local_json):
|
elif _is_json(cli_ctx, local_json):
|
||||||
_jecho(result if isinstance(result, dict) else {"status": "ok"})
|
_jecho(result if isinstance(result, dict) else {"status": "ok"})
|
||||||
|
|||||||
@@ -77,6 +77,7 @@ import yaml
|
|||||||
from ..utils.logging import get_logger
|
from ..utils.logging import get_logger
|
||||||
from ..utils.progress_tracker import get_progress_tracker
|
from ..utils.progress_tracker import get_progress_tracker
|
||||||
from ..utils.types import EntityDict, RelationshipDict
|
from ..utils.types import EntityDict, RelationshipDict
|
||||||
|
from ._markdown_filesystem import find_filesystem_link
|
||||||
|
|
||||||
|
|
||||||
class _UniqueKeySafeLoader(yaml.SafeLoader):
|
class _UniqueKeySafeLoader(yaml.SafeLoader):
|
||||||
@@ -1742,9 +1743,10 @@ class AgentMemory:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def _write_markdown_file(file_path: Path, document: str) -> None:
|
def _write_markdown_file(file_path: Path, document: str) -> None:
|
||||||
"""Atomically replace a Markdown file without following output symlinks."""
|
"""Atomically replace a Markdown file without following output symlinks."""
|
||||||
if file_path.is_symlink():
|
if find_filesystem_link(file_path) is not None:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Refusing to overwrite Markdown symbolic link: {file_path}"
|
"Refusing to overwrite Markdown symbolic link or junction: "
|
||||||
|
f"{file_path}"
|
||||||
)
|
)
|
||||||
|
|
||||||
temporary_path = None
|
temporary_path = None
|
||||||
@@ -1867,7 +1869,8 @@ class AgentMemory:
|
|||||||
if "\n" not in data and "\r" not in data:
|
if "\n" not in data and "\r" not in data:
|
||||||
candidate = Path(data)
|
candidate = Path(data)
|
||||||
try:
|
try:
|
||||||
candidate_exists = candidate.exists()
|
candidate_is_link = find_filesystem_link(candidate) is not None
|
||||||
|
candidate_exists = candidate_is_link or candidate.exists()
|
||||||
except OSError as exc:
|
except OSError as exc:
|
||||||
error_message = (
|
error_message = (
|
||||||
"Failed to inspect possible Markdown import "
|
"Failed to inspect possible Markdown import "
|
||||||
@@ -1909,62 +1912,71 @@ class AgentMemory:
|
|||||||
return memories
|
return memories
|
||||||
|
|
||||||
def _read_markdown_file_content(self, file_path: Path) -> str:
|
def _read_markdown_file_content(self, file_path: Path) -> str:
|
||||||
if file_path.is_symlink():
|
if find_filesystem_link(file_path) is not None:
|
||||||
raise ValueError(f"Symlink Markdown import paths are rejected: {file_path}")
|
raise ValueError(
|
||||||
|
"Symlink Markdown import paths are rejected; symbolic links and "
|
||||||
|
f"junctions are unsafe: {file_path}"
|
||||||
|
)
|
||||||
|
|
||||||
flags = os.O_RDONLY
|
flags = os.O_RDONLY
|
||||||
if hasattr(os, "O_NOFOLLOW"):
|
nofollow_flag = getattr(os, "O_NOFOLLOW", 0)
|
||||||
# On POSIX, O_NOFOLLOW makes os.open() fail with ELOOP if the
|
flags |= nofollow_flag
|
||||||
# final path component is a symlink, atomically closing the TOCTOU
|
|
||||||
# window between the is_symlink() check above and the open call.
|
|
||||||
# On Windows, O_NOFOLLOW is not available; the is_symlink() pre-check
|
|
||||||
# above is the only symlink defense and remains vulnerable to a narrow
|
|
||||||
# race. The fstat()/S_ISREG guard below still rejects special files
|
|
||||||
# (FIFOs, devices) on both platforms.
|
|
||||||
flags |= os.O_NOFOLLOW
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
fd = os.open(str(file_path), flags)
|
fd = os.open(str(file_path), flags)
|
||||||
except OSError as exc:
|
except OSError as exc:
|
||||||
if exc.errno == getattr(errno, "ELOOP", None):
|
if (
|
||||||
|
(nofollow_flag and exc.errno == errno.ELOOP)
|
||||||
|
or find_filesystem_link(file_path) is not None
|
||||||
|
):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Symlink Markdown import paths are rejected: {file_path}"
|
"Symlink Markdown import paths are rejected; symbolic links "
|
||||||
|
f"and junctions are unsafe: {file_path}"
|
||||||
) from exc
|
) from exc
|
||||||
raise
|
raise
|
||||||
|
|
||||||
try:
|
try:
|
||||||
stat_res = os.fstat(fd)
|
if find_filesystem_link(file_path) is not None:
|
||||||
if not stat.S_ISREG(stat_res.st_mode):
|
raise ValueError(
|
||||||
|
"Symlink Markdown import paths are rejected; symbolic links "
|
||||||
|
f"and junctions are unsafe: {file_path}"
|
||||||
|
)
|
||||||
|
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Markdown import path is not a regular file: {file_path}"
|
f"Markdown import path is not a regular file: {file_path}"
|
||||||
)
|
)
|
||||||
with open(fd, "r", encoding="utf-8", closefd=True) as f:
|
with os.fdopen(fd, mode="r", encoding="utf-8") as source:
|
||||||
return f.read()
|
fd = -1
|
||||||
except Exception:
|
return source.read()
|
||||||
try:
|
finally:
|
||||||
|
if fd >= 0:
|
||||||
os.close(fd)
|
os.close(fd)
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
raise
|
|
||||||
|
|
||||||
def _read_markdown_path(self, path: Path) -> List[Tuple[str, str]]:
|
def _read_markdown_path(self, path: Path) -> List[Tuple[str, str]]:
|
||||||
if path.is_symlink():
|
if find_filesystem_link(path) is not None:
|
||||||
raise ValueError(f"Symlink Markdown import paths are rejected: {path}")
|
raise ValueError(
|
||||||
|
"Symlink Markdown import paths are rejected; symbolic links and "
|
||||||
|
f"junctions are unsafe: {path}"
|
||||||
|
)
|
||||||
|
|
||||||
if not path.exists():
|
if not path.exists():
|
||||||
raise FileNotFoundError(f"Markdown import path does not exist: {path}")
|
raise FileNotFoundError(f"Markdown import path does not exist: {path}")
|
||||||
|
|
||||||
if path.is_dir():
|
if path.is_dir():
|
||||||
file_paths = sorted(
|
file_paths = []
|
||||||
(
|
for file_path in path.iterdir():
|
||||||
file_path
|
if file_path.suffix.lower() not in self._MARKDOWN_EXTENSIONS:
|
||||||
for file_path in path.iterdir()
|
continue
|
||||||
if file_path.is_file()
|
if find_filesystem_link(file_path) is not None:
|
||||||
and not file_path.is_symlink()
|
continue
|
||||||
and file_path.suffix.lower() in self._MARKDOWN_EXTENSIONS
|
if file_path.is_file():
|
||||||
),
|
file_paths.append(file_path)
|
||||||
key=lambda file_path: (file_path.name.casefold(), file_path.name),
|
if find_filesystem_link(path) is not None:
|
||||||
)
|
raise ValueError(
|
||||||
|
"Symlink Markdown import paths are rejected; symbolic links "
|
||||||
|
f"and junctions are unsafe: {path}"
|
||||||
|
)
|
||||||
|
file_paths.sort(key=lambda item: (item.name.casefold(), item.name))
|
||||||
elif path.is_file():
|
elif path.is_file():
|
||||||
file_paths = [path]
|
file_paths = [path]
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -493,6 +493,23 @@ _ATTRS_MISSING = object()
|
|||||||
#: entities and timestamps.
|
#: entities and timestamps.
|
||||||
_CAUSAL_EDGE_TYPES = ("CAUSED", "INFLUENCED", "PRECEDENT_FOR")
|
_CAUSAL_EDGE_TYPES = ("CAUSED", "INFLUENCED", "PRECEDENT_FOR")
|
||||||
|
|
||||||
|
# Causal edges circulate under two vocabularies: this module's canonical
|
||||||
|
# spellings above, and the present-tense spellings CausalChainAnalyzer also
|
||||||
|
# accepts ("causes", "influences", "leads_to", "supports"). The present-tense
|
||||||
|
# forms normalize onto the canonical types for storage; traversal accepts
|
||||||
|
# both vocabularies so an edge recorded either way is never invisible.
|
||||||
|
_CAUSAL_EDGE_ALIASES = {
|
||||||
|
"CAUSES": "CAUSED",
|
||||||
|
"CAUSED": "CAUSED",
|
||||||
|
"INFLUENCES": "INFLUENCED",
|
||||||
|
"INFLUENCED": "INFLUENCED",
|
||||||
|
"PRECEDES": "PRECEDENT_FOR",
|
||||||
|
"PRECEDENT_FOR": "PRECEDENT_FOR",
|
||||||
|
}
|
||||||
|
_CAUSAL_TRAVERSAL_TYPES = frozenset(_CAUSAL_EDGE_ALIASES) | {
|
||||||
|
"LEADS_TO", "LEAD_TO", "SUPPORTS", "SUPPORT",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class ContextGraph:
|
class ContextGraph:
|
||||||
"""
|
"""
|
||||||
@@ -3623,9 +3640,15 @@ class ContextGraph:
|
|||||||
target_decision_id: Target decision ID
|
target_decision_id: Target decision ID
|
||||||
relationship_type: Type of relationship (CAUSED, INFLUENCED, PRECEDENT_FOR)
|
relationship_type: Type of relationship (CAUSED, INFLUENCED, PRECEDENT_FOR)
|
||||||
"""
|
"""
|
||||||
valid_types = ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]
|
# Normalize so callers may use either vocabulary's spelling
|
||||||
if relationship_type not in valid_types:
|
# ("causes" from CausalChainAnalyzer, or "CAUSED" from this module's
|
||||||
raise ValueError(f"Relationship type must be one of: {valid_types}")
|
# canonical constant); the stored form is always canonical. Invalid
|
||||||
|
# inputs keep raising ValueError rather than AttributeError.
|
||||||
|
if not isinstance(relationship_type, str):
|
||||||
|
raise ValueError(f"Relationship type must be one of: {_CAUSAL_EDGE_TYPES}")
|
||||||
|
relationship_type = _CAUSAL_EDGE_ALIASES.get(relationship_type.strip().upper())
|
||||||
|
if relationship_type is None:
|
||||||
|
raise ValueError(f"Relationship type must be one of: {_CAUSAL_EDGE_TYPES}")
|
||||||
|
|
||||||
# Check if decisions exist - if not, skip adding relationship
|
# Check if decisions exist - if not, skip adding relationship
|
||||||
if source_decision_id not in self.nodes or target_decision_id not in self.nodes:
|
if source_decision_id not in self.nodes or target_decision_id not in self.nodes:
|
||||||
@@ -3717,11 +3740,11 @@ class ContextGraph:
|
|||||||
# Find connected decisions
|
# Find connected decisions
|
||||||
for edge in self.edges:
|
for edge in self.edges:
|
||||||
if direction == "upstream":
|
if direction == "upstream":
|
||||||
if edge.target_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]:
|
if edge.target_id == current_id and edge.edge_type.upper() in _CAUSAL_TRAVERSAL_TYPES:
|
||||||
if edge.source_id not in visited and depth < max_depth:
|
if edge.source_id not in visited and depth < max_depth:
|
||||||
queue.append((edge.source_id, depth + 1))
|
queue.append((edge.source_id, depth + 1))
|
||||||
else: # downstream
|
else: # downstream
|
||||||
if edge.source_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]:
|
if edge.source_id == current_id and edge.edge_type.upper() in _CAUSAL_TRAVERSAL_TYPES:
|
||||||
if edge.target_id not in visited and depth < max_depth:
|
if edge.target_id not in visited and depth < max_depth:
|
||||||
queue.append((edge.target_id, depth + 1))
|
queue.append((edge.target_id, depth + 1))
|
||||||
|
|
||||||
@@ -3744,10 +3767,13 @@ class ContextGraph:
|
|||||||
Returns:
|
Returns:
|
||||||
List of precedent decisions
|
List of precedent decisions
|
||||||
"""
|
"""
|
||||||
# Find decisions connected via PRECEDENT_FOR relationships
|
# Find decisions connected via PRECEDENT_FOR relationships, accepting
|
||||||
|
# the analyzer vocabulary's "precedes" spelling as well (issue #1184).
|
||||||
precedent_ids = []
|
precedent_ids = []
|
||||||
for edge in self.edges:
|
for edge in self.edges:
|
||||||
if edge.target_id == decision_id and edge.edge_type == "PRECEDENT_FOR":
|
if edge.target_id == decision_id and edge.edge_type.upper() in {
|
||||||
|
"PRECEDENT_FOR", "PRECEDES",
|
||||||
|
}:
|
||||||
precedent_ids.append(edge.source_id)
|
precedent_ids.append(edge.source_id)
|
||||||
|
|
||||||
# Convert to Decision objects
|
# Convert to Decision objects
|
||||||
@@ -4308,8 +4334,13 @@ class ContextGraph:
|
|||||||
|
|
||||||
# Explicit causal relationships recorded via add_causal_relationship() are
|
# Explicit causal relationships recorded via add_causal_relationship() are
|
||||||
# ground truth and always count as direct influence, in either direction.
|
# ground truth and always count as direct influence, in either direction.
|
||||||
for edge_type in _CAUSAL_EDGE_TYPES:
|
# The index is keyed by the raw edge_type string ("causes" and "CAUSED"
|
||||||
for edge in self.edge_type_index.get(edge_type, []):
|
# are separate keys), so filter by normalized type instead of iterating
|
||||||
|
# a fixed spelling list.
|
||||||
|
for edge_type, edges in self.edge_type_index.items():
|
||||||
|
if edge_type.upper() not in _CAUSAL_TRAVERSAL_TYPES:
|
||||||
|
continue
|
||||||
|
for edge in edges:
|
||||||
if edge.source_id == decision_id and edge.target_id in self._decisions:
|
if edge.source_id == decision_id and edge.target_id in self._decisions:
|
||||||
direct_influence.add(edge.target_id)
|
direct_influence.add(edge.target_id)
|
||||||
elif edge.target_id == decision_id and edge.source_id in self._decisions:
|
elif edge.target_id == decision_id and edge.source_id in self._decisions:
|
||||||
@@ -4455,8 +4486,12 @@ class ContextGraph:
|
|||||||
# record_decision() (e.g. a graph restored via from_dict), so only
|
# record_decision() (e.g. a graph restored via from_dict), so only
|
||||||
# causes with a known decision record are kept.
|
# causes with a known decision record are kept.
|
||||||
incoming_causal_edges = defaultdict(list)
|
incoming_causal_edges = defaultdict(list)
|
||||||
for edge_type in _CAUSAL_EDGE_TYPES:
|
# The index is keyed by the raw edge_type string ("causes" and
|
||||||
for edge in self.edge_type_index.get(edge_type, []):
|
# "CAUSED" are separate keys), so filter by normalized type.
|
||||||
|
for edge_type, edges in self.edge_type_index.items():
|
||||||
|
if edge_type.upper() not in _CAUSAL_TRAVERSAL_TYPES:
|
||||||
|
continue
|
||||||
|
for edge in edges:
|
||||||
if edge.source_id in self._decisions:
|
if edge.source_id in self._decisions:
|
||||||
incoming_causal_edges[edge.target_id].append(edge)
|
incoming_causal_edges[edge.target_id].append(edge)
|
||||||
|
|
||||||
|
|||||||
@@ -69,6 +69,15 @@ class EmbeddingGeneratorWithProvenance:
|
|||||||
return embeddings
|
return embeddings
|
||||||
|
|
||||||
def __getattr__(self, name):
|
def __getattr__(self, name):
|
||||||
|
# __getattr__ only runs when normal lookup fails. Accessing
|
||||||
|
# self._generator by attribute syntax HERE would re-enter
|
||||||
|
# __getattr__ for ever when _generator itself is missing — the shape
|
||||||
|
# pickle/copy protocol probes hit when __init__ never completed
|
||||||
|
# (#994's RecursionError family). Fail fast on private probes.
|
||||||
|
if name.startswith("_"):
|
||||||
|
raise AttributeError(
|
||||||
|
f"{type(self).__name__!r} object has no attribute {name!r}"
|
||||||
|
)
|
||||||
return getattr(self._generator, name)
|
return getattr(self._generator, name)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -117,9 +117,9 @@ def generate_embeddings(
|
|||||||
>>> emb = generate_embeddings("Hello world", method="default")
|
>>> emb = generate_embeddings("Hello world", method="default")
|
||||||
>>> embs = generate_embeddings(["text1", "text2"], method="text")
|
>>> embs = generate_embeddings(["text1", "text2"], method="text")
|
||||||
"""
|
"""
|
||||||
# Check for custom method in registry
|
# Check for custom method in registry, skip self-reference
|
||||||
custom_method = method_registry.get("generation", method)
|
custom_method = method_registry.get("generation", method)
|
||||||
if custom_method:
|
if custom_method and custom_method is not generate_embeddings:
|
||||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||||
result = call_custom_method(
|
result = call_custom_method(
|
||||||
logger, method, custom_method, data, data_type=data_type, fallback_on_custom_error=fallback, **kwargs
|
logger, method, custom_method, data, data_type=data_type, fallback_on_custom_error=fallback, **kwargs
|
||||||
@@ -165,9 +165,9 @@ def embed_text(
|
|||||||
>>> emb = embed_text("Hello world", method="sentence_transformers")
|
>>> emb = embed_text("Hello world", method="sentence_transformers")
|
||||||
>>> embs = embed_text(["text1", "text2"], method="sentence_transformers")
|
>>> embs = embed_text(["text1", "text2"], method="sentence_transformers")
|
||||||
"""
|
"""
|
||||||
# Check for custom method in registry
|
# Check for custom method in registry, skip self-reference
|
||||||
custom_method = method_registry.get("text", method)
|
custom_method = method_registry.get("text", method)
|
||||||
if custom_method:
|
if custom_method and custom_method is not embed_text:
|
||||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||||
result = call_custom_method(
|
result = call_custom_method(
|
||||||
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
|
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
|
||||||
@@ -225,9 +225,9 @@ def calculate_similarity(
|
|||||||
>>> similarity = calculate_similarity(emb1, emb2, method="cosine")
|
>>> similarity = calculate_similarity(emb1, emb2, method="cosine")
|
||||||
>>> print(f"Similarity: {similarity:.3f}")
|
>>> print(f"Similarity: {similarity:.3f}")
|
||||||
"""
|
"""
|
||||||
# Check for custom method in registry
|
# Check for custom method in registry, skip self-reference
|
||||||
custom_method = method_registry.get("similarity", method)
|
custom_method = method_registry.get("similarity", method)
|
||||||
if custom_method:
|
if custom_method and custom_method is not calculate_similarity:
|
||||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||||
result = call_custom_method(
|
result = call_custom_method(
|
||||||
logger, method, custom_method, embedding1, embedding2, fallback_on_custom_error=fallback, **kwargs
|
logger, method, custom_method, embedding1, embedding2, fallback_on_custom_error=fallback, **kwargs
|
||||||
@@ -272,9 +272,9 @@ def pool_embeddings(
|
|||||||
>>> pooled = pool_embeddings(embeddings, method="mean")
|
>>> pooled = pool_embeddings(embeddings, method="mean")
|
||||||
>>> attention_pooled = pool_embeddings(embeddings, method="attention")
|
>>> attention_pooled = pool_embeddings(embeddings, method="attention")
|
||||||
"""
|
"""
|
||||||
# Check for custom method in registry
|
# Check for custom method in registry, skip self-reference
|
||||||
custom_method = method_registry.get("pooling", method)
|
custom_method = method_registry.get("pooling", method)
|
||||||
if custom_method:
|
if custom_method and custom_method is not pool_embeddings:
|
||||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||||
result = call_custom_method(
|
result = call_custom_method(
|
||||||
logger, method, custom_method, embeddings, fallback_on_custom_error=fallback, **kwargs
|
logger, method, custom_method, embeddings, fallback_on_custom_error=fallback, **kwargs
|
||||||
|
|||||||
@@ -28,12 +28,35 @@ from pathlib import Path
|
|||||||
from typing import Any, Dict, List, Optional, Union
|
from typing import Any, Dict, List, Optional, Union
|
||||||
|
|
||||||
from ..utils.exceptions import ProcessingError, ValidationError
|
from ..utils.exceptions import ProcessingError, ValidationError
|
||||||
from ..utils.helpers import ensure_directory, utc_now_iso, write_json_file
|
from ..utils.helpers import ensure_directory, hash_data, utc_now_iso, write_json_file
|
||||||
from ..utils.logging import get_logger
|
from ..utils.logging import get_logger
|
||||||
from ..utils.progress_tracker import get_progress_tracker
|
from ..utils.progress_tracker import get_progress_tracker
|
||||||
from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri
|
from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri
|
||||||
|
|
||||||
|
|
||||||
|
def _content_iri(prefix: str, payload: Any) -> str:
|
||||||
|
"""Mint a document IRI from what was exported, not when.
|
||||||
|
|
||||||
|
Minting from ``utc_now_iso()`` gave every export of the same graph a new
|
||||||
|
identity a few microseconds apart, so re-exporting an unchanged graph was
|
||||||
|
never idempotent and merging exports duplicated every node (#1147). This
|
||||||
|
mirrors ``mint_entity_iri`` (#1109): identical content hashes to the same
|
||||||
|
IRI, and any change to the content changes it too. ``default=str`` keeps
|
||||||
|
the hash defined for values ``json.dumps`` would otherwise reject, such as
|
||||||
|
``datetime`` objects a caller may have left in the graph.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
prefix: IRI prefix the digest is appended to
|
||||||
|
payload: JSON-serializable value whose content determines the digest
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A stable IRI of the form ``{prefix}{16-hex-char digest}``
|
||||||
|
"""
|
||||||
|
canonical = json.dumps(payload, sort_keys=True, default=str)
|
||||||
|
digest = hash_data(canonical)[:16]
|
||||||
|
return f"{prefix}{digest}"
|
||||||
|
|
||||||
|
|
||||||
def _is_jsonld_document(data: Dict[str, Any]) -> bool:
|
def _is_jsonld_document(data: Dict[str, Any]) -> bool:
|
||||||
"""
|
"""
|
||||||
Report whether a dictionary is already a JSON-LD document.
|
Report whether a dictionary is already a JSON-LD document.
|
||||||
@@ -230,7 +253,10 @@ class JSONExporter:
|
|||||||
- statistics: Statistics dictionary (optional)
|
- statistics: Statistics dictionary (optional)
|
||||||
file_path: Output JSON file path
|
file_path: Output JSON file path
|
||||||
format: Export format - 'json' or 'json-ld' (default: self.format)
|
format: Export format - 'json' or 'json-ld' (default: self.format)
|
||||||
**options: Additional options passed to conversion methods
|
**options: Additional options passed to conversion methods:
|
||||||
|
- graph_uri: Caller-supplied IRI for the graph node when
|
||||||
|
format='json-ld', overriding the default content-derived
|
||||||
|
IRI (see #1147)
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
>>> kg = {
|
>>> kg = {
|
||||||
@@ -401,7 +427,9 @@ class JSONExporter:
|
|||||||
data: Data to convert (dict, list, or any value)
|
data: Data to convert (dict, list, or any value)
|
||||||
include_metadata: Whether to include metadata (default: True)
|
include_metadata: Whether to include metadata (default: True)
|
||||||
include_provenance: Whether to include provenance (default: True)
|
include_provenance: Whether to include provenance (default: True)
|
||||||
**options: Additional options passed to knowledge graph conversion
|
**options: Additional options passed to knowledge graph conversion:
|
||||||
|
- document_uri: Caller-supplied IRI for the document node,
|
||||||
|
overriding the default content-derived IRI (see #1147)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary in JSON-LD format with @context, @graph/@value, and metadata
|
Dictionary in JSON-LD format with @context, @graph/@value, and metadata
|
||||||
@@ -451,13 +479,17 @@ class JSONExporter:
|
|||||||
|
|
||||||
# Add metadata and provenance if requested
|
# Add metadata and provenance if requested
|
||||||
if include_metadata:
|
if include_metadata:
|
||||||
self._attach_document_metadata(jsonld, include_provenance)
|
self._attach_document_metadata(
|
||||||
|
jsonld, include_provenance, options.get("document_uri")
|
||||||
|
)
|
||||||
|
|
||||||
return jsonld
|
return jsonld
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _attach_document_metadata(
|
def _attach_document_metadata(
|
||||||
jsonld: Dict[str, Any], include_provenance: bool
|
jsonld: Dict[str, Any],
|
||||||
|
include_provenance: bool,
|
||||||
|
document_uri: Optional[str] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Attach the export's own metadata without naming the graph.
|
Attach the export's own metadata without naming the graph.
|
||||||
@@ -473,6 +505,9 @@ class JSONExporter:
|
|||||||
Args:
|
Args:
|
||||||
jsonld: Document being built, modified in place
|
jsonld: Document being built, modified in place
|
||||||
include_provenance: Whether to record how and when it was exported
|
include_provenance: Whether to record how and when it was exported
|
||||||
|
document_uri: Caller-supplied IRI for the document node. Falls back
|
||||||
|
to a content-derived IRI (#1147) so re-exporting unchanged data
|
||||||
|
is idempotent instead of minting a new identity every time.
|
||||||
"""
|
"""
|
||||||
# A caller may hand us a document that is deliberately a named graph.
|
# A caller may hand us a document that is deliberately a named graph.
|
||||||
# That name is theirs to keep, but our own statements must not end up
|
# That name is theirs to keep, but our own statements must not end up
|
||||||
@@ -483,7 +518,10 @@ class JSONExporter:
|
|||||||
# Do not overwrite an identifier the payload already carries: the
|
# Do not overwrite an identifier the payload already carries: the
|
||||||
# knowledge-graph conversion names its own document node.
|
# knowledge-graph conversion names its own document node.
|
||||||
if "@id" not in jsonld or payload_is_named_graph:
|
if "@id" not in jsonld or payload_is_named_graph:
|
||||||
document["@id"] = f"https://semantica.dev/data/{utc_now_iso()}"
|
content = {key: value for key, value in jsonld.items() if key != "@context"}
|
||||||
|
document["@id"] = document_uri or _content_iri(
|
||||||
|
"https://semantica.dev/data/", content
|
||||||
|
)
|
||||||
if include_provenance:
|
if include_provenance:
|
||||||
document["semantica:exportedAt"] = utc_now_iso()
|
document["semantica:exportedAt"] = utc_now_iso()
|
||||||
document["semantica:format"] = "json-ld"
|
document["semantica:format"] = "json-ld"
|
||||||
@@ -553,7 +591,9 @@ class JSONExporter:
|
|||||||
- entities: List of entity dictionaries
|
- entities: List of entity dictionaries
|
||||||
- relationships: List of relationship dictionaries
|
- relationships: List of relationship dictionaries
|
||||||
- metadata: Metadata dictionary (optional)
|
- metadata: Metadata dictionary (optional)
|
||||||
**options: Additional options (unused)
|
**options: Additional options:
|
||||||
|
- graph_uri: Caller-supplied IRI for the graph node,
|
||||||
|
overriding the default content-derived IRI (see #1147)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary in JSON-LD format with @context, @id, @type, and graph data
|
Dictionary in JSON-LD format with @context, @id, @type, and graph data
|
||||||
@@ -566,7 +606,11 @@ class JSONExporter:
|
|||||||
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
|
||||||
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
|
||||||
},
|
},
|
||||||
"@id": f"https://semantica.dev/graph/{utc_now_iso()}",
|
# Minted from the graph's own content rather than the wall clock
|
||||||
|
# (#1147): re-exporting an unchanged graph must produce the same
|
||||||
|
# subject, or merging repeated exports duplicates every node.
|
||||||
|
"@id": options.get("graph_uri")
|
||||||
|
or _content_iri("https://semantica.dev/graph/", kg),
|
||||||
"@type": "semantica:KnowledgeGraph",
|
"@type": "semantica:KnowledgeGraph",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -159,6 +159,7 @@ from .ontology_validator import (
|
|||||||
SHACLValidationReport,
|
SHACLValidationReport,
|
||||||
SHACLViolation,
|
SHACLViolation,
|
||||||
ValidationResult,
|
ValidationResult,
|
||||||
|
run_shacl_validation,
|
||||||
validate_ontology,
|
validate_ontology,
|
||||||
)
|
)
|
||||||
from .owl_generator import OWLGenerator
|
from .owl_generator import OWLGenerator
|
||||||
@@ -192,6 +193,7 @@ __all__ = [
|
|||||||
"PropertyShape",
|
"PropertyShape",
|
||||||
"SHACLValidationReport",
|
"SHACLValidationReport",
|
||||||
"SHACLViolation",
|
"SHACLViolation",
|
||||||
|
"run_shacl_validation",
|
||||||
# OWL/RDF generation
|
# OWL/RDF generation
|
||||||
"OWLGenerator",
|
"OWLGenerator",
|
||||||
# Requirements and competency questions
|
# Requirements and competency questions
|
||||||
|
|||||||
@@ -145,14 +145,14 @@ class SHACLValidationReport:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _run_pyshacl(
|
def run_shacl_validation(
|
||||||
data_graph_str: str,
|
data_graph_str: str,
|
||||||
shacl_str: str,
|
shacl_str: str,
|
||||||
data_graph_format: str = "turtle",
|
data_graph_format: str = "turtle",
|
||||||
shacl_format: str = "turtle",
|
shacl_format: str = "turtle",
|
||||||
) -> SHACLValidationReport:
|
) -> SHACLValidationReport:
|
||||||
"""
|
"""
|
||||||
Run pyshacl validation and return a structured SHACLValidationReport.
|
Run pySHACL validation and return a structured SHACLValidationReport.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
data_graph_str: Serialized data graph string.
|
data_graph_str: Serialized data graph string.
|
||||||
@@ -272,6 +272,21 @@ def _run_pyshacl(
|
|||||||
raw_report=results_text,
|
raw_report=results_text,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_pyshacl(
|
||||||
|
data_graph_str: str,
|
||||||
|
shacl_str: str,
|
||||||
|
data_graph_format: str = "turtle",
|
||||||
|
shacl_format: str = "turtle",
|
||||||
|
) -> SHACLValidationReport:
|
||||||
|
"""Backward-compatible alias for :func:`run_shacl_validation`."""
|
||||||
|
return run_shacl_validation(
|
||||||
|
data_graph_str,
|
||||||
|
shacl_str,
|
||||||
|
data_graph_format=data_graph_format,
|
||||||
|
shacl_format=shacl_format,
|
||||||
|
)
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ValidationResult:
|
class ValidationResult:
|
||||||
"""Result of an ontology validation operation."""
|
"""Result of an ontology validation operation."""
|
||||||
|
|||||||
@@ -779,9 +779,11 @@ def extract_entities_huggingface(
|
|||||||
"""
|
"""
|
||||||
loader = HuggingFaceModelLoader(device=device)
|
loader = HuggingFaceModelLoader(device=device)
|
||||||
# Pass kwargs (like aggregation_strategy) to load_ner_model
|
# Pass kwargs (like aggregation_strategy) to load_ner_model
|
||||||
model_obj = loader.load_ner_model(model, **kwargs)
|
loader_kwargs = {
|
||||||
|
key: value for key, value in kwargs.items() if key != "huggingface_model"
|
||||||
|
}
|
||||||
|
model_obj = loader.load_ner_model(model, **loader_kwargs)
|
||||||
results = loader.extract_entities(model_obj, text)
|
results = loader.extract_entities(model_obj, text)
|
||||||
|
|
||||||
entities = []
|
entities = []
|
||||||
|
|
||||||
# Check if manual aggregation is needed (raw IOB tags detected)
|
# Check if manual aggregation is needed (raw IOB tags detected)
|
||||||
|
|||||||
@@ -297,7 +297,11 @@ def create_index(
|
|||||||
config = vector_store_config.get_all()
|
config = vector_store_config.get_all()
|
||||||
backend = config.get("default_backend", "faiss")
|
backend = config.get("default_backend", "faiss")
|
||||||
dimension = config.get("dimension", 768)
|
dimension = config.get("dimension", 768)
|
||||||
indexer = VectorIndexer(backend=backend, dimension=dimension, **config)
|
# backend/dimension are already passed explicitly; drop them from the
|
||||||
|
# forwarded config so VectorIndexer(..., **remaining_config) doesn't
|
||||||
|
# receive duplicate keyword arguments.
|
||||||
|
remaining_config = {k: v for k, v in config.items() if k not in ("default_backend", "dimension")}
|
||||||
|
indexer = VectorIndexer(backend=backend, dimension=dimension, **remaining_config)
|
||||||
return indexer.create_index(vectors, ids, **options)
|
return indexer.create_index(vectors, ids, **options)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
import errno
|
import errno
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from pathlib import Path
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -56,6 +60,25 @@ def markdown_document(frontmatter, body=""):
|
|||||||
return f"---\n{yaml_text}---\n\n{body}"
|
return f"---\n{yaml_text}---\n\n{body}"
|
||||||
|
|
||||||
|
|
||||||
|
def _require_symlink_support(tmp_path):
|
||||||
|
"""Skip the test if this environment cannot create symbolic links.
|
||||||
|
|
||||||
|
Symlink creation can be unavailable even on POSIX (e.g. restricted
|
||||||
|
containers) and commonly requires elevated privilege or Developer Mode
|
||||||
|
on Windows. Probe actual capability instead of assuming based on
|
||||||
|
platform, so these tests still run wherever symlinks genuinely work.
|
||||||
|
"""
|
||||||
|
probe_target = tmp_path / ".symlink_probe_target"
|
||||||
|
probe_link = tmp_path / ".symlink_probe_link"
|
||||||
|
probe_target.write_text("", encoding="utf-8")
|
||||||
|
try:
|
||||||
|
probe_link.symlink_to(probe_target)
|
||||||
|
except OSError as exc:
|
||||||
|
pytest.skip(f"environment cannot create symbolic links: {exc}")
|
||||||
|
probe_link.unlink()
|
||||||
|
probe_target.unlink()
|
||||||
|
|
||||||
|
|
||||||
def required_frontmatter(memory_id="mem_test", **overrides):
|
def required_frontmatter(memory_id="mem_test", **overrides):
|
||||||
frontmatter = {
|
frontmatter = {
|
||||||
"id": memory_id,
|
"id": memory_id,
|
||||||
@@ -707,6 +730,151 @@ def test_markdown_string_path_inspection_errors_are_actionable():
|
|||||||
assert exc_info.value.__cause__ is original_error
|
assert exc_info.value.__cause__ is original_error
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("use_string_path", [False, True])
|
||||||
|
def test_markdown_import_rejects_symlinked_file(tmp_path, use_string_path):
|
||||||
|
_require_symlink_support(tmp_path)
|
||||||
|
outside = tmp_path / "outside.md"
|
||||||
|
outside.write_text(
|
||||||
|
markdown_document(required_frontmatter(), "Do not import"),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
source = tmp_path / "memory.md"
|
||||||
|
source.symlink_to(outside)
|
||||||
|
payload = str(source) if use_string_path else source
|
||||||
|
memory = AgentMemory()
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="symbolic link"):
|
||||||
|
memory.import_data(payload, format="markdown")
|
||||||
|
|
||||||
|
assert memory.count() == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("use_string_path", [False, True])
|
||||||
|
def test_markdown_import_rejects_broken_symlink(tmp_path, use_string_path):
|
||||||
|
_require_symlink_support(tmp_path)
|
||||||
|
source = tmp_path / "missing-memory.md"
|
||||||
|
source.symlink_to(tmp_path / "missing-target.md")
|
||||||
|
payload = str(source) if use_string_path else source
|
||||||
|
memory = AgentMemory()
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="symbolic link"):
|
||||||
|
memory.import_data(payload, format="markdown")
|
||||||
|
|
||||||
|
assert memory.count() == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("use_string_path", [False, True])
|
||||||
|
def test_markdown_import_rejects_symlinked_directory(tmp_path, use_string_path):
|
||||||
|
_require_symlink_support(tmp_path)
|
||||||
|
outside = tmp_path / "outside"
|
||||||
|
outside.mkdir()
|
||||||
|
(outside / "memory.md").write_text(
|
||||||
|
markdown_document(required_frontmatter(), "Do not import"),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
source = tmp_path / "memory-export"
|
||||||
|
source.symlink_to(outside, target_is_directory=True)
|
||||||
|
payload = str(source) if use_string_path else source
|
||||||
|
memory = AgentMemory()
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="symbolic link"):
|
||||||
|
memory.import_data(payload, format="markdown")
|
||||||
|
|
||||||
|
assert memory.count() == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_markdown_import_skips_symlinked_file_in_directory(tmp_path):
|
||||||
|
_require_symlink_support(tmp_path)
|
||||||
|
outside = tmp_path / "outside.md"
|
||||||
|
outside.write_text(
|
||||||
|
markdown_document(required_frontmatter(), "Do not import"),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
source = tmp_path / "memory-export"
|
||||||
|
source.mkdir()
|
||||||
|
(source / "memory.md").symlink_to(outside)
|
||||||
|
memory = AgentMemory()
|
||||||
|
|
||||||
|
assert memory.import_data(source, format="markdown") == 0
|
||||||
|
assert memory.count() == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not hasattr(os, "O_NOFOLLOW"), reason="requires O_NOFOLLOW")
|
||||||
|
def test_markdown_import_does_not_follow_symlink_raced_before_open(tmp_path):
|
||||||
|
_require_symlink_support(tmp_path)
|
||||||
|
outside = tmp_path / "outside.md"
|
||||||
|
outside.write_text(
|
||||||
|
markdown_document(required_frontmatter(), "Do not import"),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
source = tmp_path / "memory.md"
|
||||||
|
source.symlink_to(outside)
|
||||||
|
|
||||||
|
with patch.object(Path, "is_symlink", return_value=False):
|
||||||
|
with pytest.raises(ValueError, match="symbolic link"):
|
||||||
|
AgentMemory()._read_markdown_file_content(source)
|
||||||
|
|
||||||
|
|
||||||
|
def test_markdown_import_rejects_windows_junction(tmp_path, monkeypatch):
|
||||||
|
source = tmp_path / "junction"
|
||||||
|
source.mkdir()
|
||||||
|
(source / "memory.md").write_text("not read", encoding="utf-8")
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
os.path,
|
||||||
|
"isjunction",
|
||||||
|
lambda candidate: Path(candidate) == source,
|
||||||
|
raising=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="junction"):
|
||||||
|
AgentMemory().import_data(source, format="markdown")
|
||||||
|
|
||||||
|
|
||||||
|
def test_markdown_import_rejects_windows_reparse_point_fallback(tmp_path, monkeypatch):
|
||||||
|
source = tmp_path / "reparse-point"
|
||||||
|
source.mkdir()
|
||||||
|
real_lstat = os.lstat
|
||||||
|
|
||||||
|
class ReparseStat:
|
||||||
|
st_file_attributes = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
|
||||||
|
|
||||||
|
monkeypatch.delattr(os.path, "isjunction", raising=False)
|
||||||
|
monkeypatch.setattr(Path, "is_symlink", lambda self: False)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
os,
|
||||||
|
"lstat",
|
||||||
|
lambda candidate: (
|
||||||
|
ReparseStat() if Path(candidate) == source else real_lstat(candidate)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="junction"):
|
||||||
|
AgentMemory().import_data(source, format="markdown")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(os.name != "nt", reason="requires Windows junctions")
|
||||||
|
def test_markdown_import_rejects_real_windows_junction(tmp_path):
|
||||||
|
outside = tmp_path / "outside"
|
||||||
|
outside.mkdir()
|
||||||
|
(outside / "memory.md").write_text("not read", encoding="utf-8")
|
||||||
|
source = tmp_path / "junction"
|
||||||
|
result = subprocess.run(
|
||||||
|
["cmd.exe", "/c", "mklink", "/J", str(source), str(outside)],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
pytest.skip(f"could not create Windows junction: {result.stderr}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
with pytest.raises(ValueError, match="junction"):
|
||||||
|
AgentMemory().import_data(source, format="markdown")
|
||||||
|
finally:
|
||||||
|
os.rmdir(source)
|
||||||
|
|
||||||
|
|
||||||
def test_legacy_dict_import_behavior_is_unchanged():
|
def test_legacy_dict_import_behavior_is_unchanged():
|
||||||
memory = AgentMemory()
|
memory = AgentMemory()
|
||||||
data = {
|
data = {
|
||||||
@@ -751,6 +919,7 @@ def test_markdown_import_file_open_security_rejects_symlink(tmp_path):
|
|||||||
with pytest.raises(ValueError, match="Symlink Markdown import paths are rejected"):
|
with pytest.raises(ValueError, match="Symlink Markdown import paths are rejected"):
|
||||||
memory._read_markdown_file_content(symlink_file)
|
memory._read_markdown_file_content(symlink_file)
|
||||||
|
|
||||||
|
|
||||||
def test_markdown_import_public_api_rejects_symlink(tmp_path):
|
def test_markdown_import_public_api_rejects_symlink(tmp_path):
|
||||||
"""
|
"""
|
||||||
import_data(..., format="markdown") must propagate the symlink rejection
|
import_data(..., format="markdown") must propagate the symlink rejection
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ extraction found nothing, the chain came back empty even though an explicit
|
|||||||
``CAUSED`` edge was stored in the graph.
|
``CAUSED`` edge was stored in the graph.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
from semantica.context import ContextGraph
|
from semantica.context import ContextGraph
|
||||||
from semantica.context.context_graph import ContextEdge
|
from semantica.context.context_graph import ContextEdge
|
||||||
|
|
||||||
@@ -323,3 +325,130 @@ def test_entity_based_inference_still_applies_without_explicit_edges():
|
|||||||
hop["from"] == earlier and hop["to"] == later and hop["type"] == "influences"
|
hop["from"] == earlier and hop["to"] == later and hop["type"] == "influences"
|
||||||
for hop in hops
|
for hop in hops
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_causal_chain_accepts_lowercase_causal_edge_types():
|
||||||
|
"""Issue #1184: edges recorded with the analyzer's lowercase vocabulary
|
||||||
|
must be traversed by get_causal_chain().
|
||||||
|
|
||||||
|
CausalChainAnalyzer documents causal types as lowercase ("causes",
|
||||||
|
"influences", ...) while get_causal_chain() matched only the uppercase
|
||||||
|
spellings, so an edge recorded as "causes" produced an empty audit
|
||||||
|
chain — silent and in the dangerous direction.
|
||||||
|
"""
|
||||||
|
graph = ContextGraph(advanced_analytics=True)
|
||||||
|
cause = graph.record_decision(
|
||||||
|
category="a", scenario="upstream", reasoning="r",
|
||||||
|
outcome="x", confidence=0.9,
|
||||||
|
)
|
||||||
|
effect = graph.record_decision(
|
||||||
|
category="b", scenario="downstream", reasoning="r",
|
||||||
|
outcome="y", confidence=0.9,
|
||||||
|
)
|
||||||
|
graph.add_edge(cause, effect, "causes")
|
||||||
|
|
||||||
|
chain = graph.get_causal_chain(effect, direction="upstream")
|
||||||
|
|
||||||
|
assert [decision.decision_id for decision in chain] == [cause]
|
||||||
|
|
||||||
|
|
||||||
|
def test_add_causal_relationship_accepts_any_case_and_stores_canonical():
|
||||||
|
"""Issue #1184: add_causal_relationship() should accept either spelling
|
||||||
|
and store the canonical uppercase vocabulary."""
|
||||||
|
graph = ContextGraph(advanced_analytics=True)
|
||||||
|
cause = graph.record_decision(
|
||||||
|
category="a", scenario="upstream", reasoning="r",
|
||||||
|
outcome="x", confidence=0.9,
|
||||||
|
)
|
||||||
|
effect = graph.record_decision(
|
||||||
|
category="b", scenario="downstream", reasoning="r",
|
||||||
|
outcome="y", confidence=0.9,
|
||||||
|
)
|
||||||
|
|
||||||
|
graph.add_causal_relationship(cause, effect, relationship_type="causes")
|
||||||
|
|
||||||
|
edges = [
|
||||||
|
edge for edge in graph.edges
|
||||||
|
if edge.source_id == cause and edge.target_id == effect
|
||||||
|
]
|
||||||
|
assert edges, "add_causal_relationship must store the edge"
|
||||||
|
assert edges[0].edge_type == "CAUSED"
|
||||||
|
|
||||||
|
|
||||||
|
def test_add_causal_relationship_rejects_non_string_with_value_error():
|
||||||
|
"""Invalid relationship types must keep raising ValueError (issue #1184
|
||||||
|
follow-up): normalization must not turn them into AttributeError."""
|
||||||
|
graph = ContextGraph(advanced_analytics=True)
|
||||||
|
cause = graph.record_decision(
|
||||||
|
category="a", scenario="upstream", reasoning="r",
|
||||||
|
outcome="x", confidence=0.9,
|
||||||
|
)
|
||||||
|
effect = graph.record_decision(
|
||||||
|
category="b", scenario="downstream", reasoning="r",
|
||||||
|
outcome="y", confidence=0.9,
|
||||||
|
)
|
||||||
|
|
||||||
|
for bad_type in (None, 42, ["CAUSED"]):
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
graph.add_causal_relationship(cause, effect, relationship_type=bad_type)
|
||||||
|
|
||||||
|
|
||||||
|
def test_analyze_decision_influence_sees_lowercase_causal_edge():
|
||||||
|
"""Issue #1184 follow-up: influence analysis reads the same edge index as
|
||||||
|
the causal traversal, so lowercase edges must count as direct influence."""
|
||||||
|
graph = ContextGraph(advanced_analytics=True)
|
||||||
|
cause = graph.record_decision(
|
||||||
|
category="a", scenario="upstream", reasoning="r",
|
||||||
|
outcome="x", confidence=0.9,
|
||||||
|
)
|
||||||
|
effect = graph.record_decision(
|
||||||
|
category="b", scenario="downstream", reasoning="r",
|
||||||
|
outcome="y", confidence=0.9,
|
||||||
|
)
|
||||||
|
graph.add_edge(cause, effect, "causes")
|
||||||
|
|
||||||
|
impact = graph.analyze_decision_influence(cause)
|
||||||
|
|
||||||
|
direct_ids = {entry["decision_id"] for entry in impact["direct_influence"]}
|
||||||
|
assert effect in direct_ids
|
||||||
|
|
||||||
|
|
||||||
|
def test_trace_decision_causality_sees_lowercase_causal_edge():
|
||||||
|
"""Issue #1184 follow-up: the trace must not return an empty audit chain
|
||||||
|
for a decision with an explicit lowercase upstream causal edge."""
|
||||||
|
graph = ContextGraph(advanced_analytics=True)
|
||||||
|
cause = graph.record_decision(
|
||||||
|
category="a", scenario="upstream", reasoning="r",
|
||||||
|
outcome="x", confidence=0.9,
|
||||||
|
)
|
||||||
|
effect = graph.record_decision(
|
||||||
|
category="b", scenario="downstream", reasoning="r",
|
||||||
|
outcome="y", confidence=0.9,
|
||||||
|
)
|
||||||
|
graph.add_edge(cause, effect, "causes")
|
||||||
|
|
||||||
|
trace = graph.trace_decision_causality(effect)
|
||||||
|
|
||||||
|
assert any(
|
||||||
|
hop["from"] == cause and hop["to"] == effect
|
||||||
|
for chain in trace for hop in chain["hops"]
|
||||||
|
), "lowercase causal edge must appear in the traced chain"
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_precedents_sees_lowercase_precedent_edge():
|
||||||
|
"""Issue #1184 follow-up: precedent lookup must accept the analyzer's
|
||||||
|
spelling alongside the canonical PRECEDENT_FOR."""
|
||||||
|
graph = ContextGraph(advanced_analytics=True)
|
||||||
|
precedent = graph.record_decision(
|
||||||
|
category="a", scenario="earlier", reasoning="r",
|
||||||
|
outcome="x", confidence=0.9,
|
||||||
|
)
|
||||||
|
later = graph.record_decision(
|
||||||
|
category="b", scenario="later", reasoning="r",
|
||||||
|
outcome="y", confidence=0.9,
|
||||||
|
)
|
||||||
|
graph.add_edge(precedent, later, "precedes")
|
||||||
|
|
||||||
|
precedents = graph.find_precedents(later)
|
||||||
|
|
||||||
|
assert [d.decision_id for d in precedents] == [precedent]
|
||||||
|
|||||||
@@ -0,0 +1,134 @@
|
|||||||
|
"""The document IRI of a JSON-LD export must depend on content, not the clock
|
||||||
|
(issue #1147).
|
||||||
|
|
||||||
|
``_convert_kg_to_jsonld`` minted the graph's ``@id`` from ``utc_now_iso()``,
|
||||||
|
and the generic ``_attach_document_metadata`` path did the same for a plain
|
||||||
|
document ``@id``. Exporting an unchanged graph therefore produced a new
|
||||||
|
subject every time: three exports of one one-entity graph merged into 3
|
||||||
|
``semantica:KnowledgeGraph`` nodes and 15 triples for what should have been a
|
||||||
|
single graph. Neither identifier resolves and the timestamp is already
|
||||||
|
recorded correctly in ``semantica:exportedAt``, so the fix mints the IRI from
|
||||||
|
the exported content instead (mirroring ``mint_entity_iri``, #1109), with an
|
||||||
|
optional caller-supplied override for callers who already name their graphs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
from rdflib import RDF, Graph, URIRef
|
||||||
|
|
||||||
|
from semantica.export.json_exporter import JSONExporter
|
||||||
|
|
||||||
|
KG = {
|
||||||
|
"entities": [{"id": "https://example.org/e1", "text": "Acme Corp", "type": "ORG"}],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
OTHER_KG = {
|
||||||
|
"entities": [
|
||||||
|
{"id": "https://example.org/e1", "text": "Acme Corp Renamed", "type": "ORG"}
|
||||||
|
],
|
||||||
|
"relationships": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _export(kg, tmp_path, name="out.jsonld", **options):
|
||||||
|
path = tmp_path / name
|
||||||
|
JSONExporter().export_knowledge_graph(kg, path, format="json-ld", **options)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def test_reexporting_an_unchanged_graph_is_idempotent(tmp_path):
|
||||||
|
"""The whole point of an identifier: same content, same @id."""
|
||||||
|
first = json.loads(_export(KG, tmp_path, "a.jsonld").read_text())
|
||||||
|
second = json.loads(_export(KG, tmp_path, "b.jsonld").read_text())
|
||||||
|
|
||||||
|
assert first["@id"] == second["@id"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_changed_graph_gets_a_different_id(tmp_path):
|
||||||
|
unchanged = json.loads(_export(KG, tmp_path, "a.jsonld").read_text())
|
||||||
|
changed = json.loads(_export(OTHER_KG, tmp_path, "b.jsonld").read_text())
|
||||||
|
|
||||||
|
assert unchanged["@id"] != changed["@id"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_merging_repeated_exports_yields_one_graph_node(tmp_path):
|
||||||
|
"""Regression for the exact repro in #1147: churn no longer multiplies nodes."""
|
||||||
|
merged = Graph()
|
||||||
|
for i in range(3):
|
||||||
|
path = _export(KG, tmp_path, f"churn{i}.jsonld")
|
||||||
|
merged.parse(str(path), format="json-ld")
|
||||||
|
|
||||||
|
# Exactly one subject typed as a KnowledgeGraph, regardless of how many
|
||||||
|
# times the unchanged graph was exported and merged.
|
||||||
|
kg_nodes = set(
|
||||||
|
merged.subjects(RDF.type, URIRef("https://semantica.dev/ns#KnowledgeGraph"))
|
||||||
|
)
|
||||||
|
assert len(kg_nodes) == 1
|
||||||
|
|
||||||
|
entity_nodes = set(
|
||||||
|
merged.subjects(RDF.type, URIRef("https://semantica.dev/vocab/ORG"))
|
||||||
|
)
|
||||||
|
assert len(entity_nodes) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_exported_at_still_varies_between_exports(tmp_path):
|
||||||
|
"""Identity is now content-derived, but provenance still records each run."""
|
||||||
|
first = json.loads(_export(KG, tmp_path, "a.jsonld").read_text())
|
||||||
|
second = json.loads(_export(KG, tmp_path, "b.jsonld").read_text())
|
||||||
|
|
||||||
|
assert first["@id"] == second["@id"]
|
||||||
|
assert first["semantica:exportedAt"] != second["semantica:exportedAt"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_caller_supplied_graph_uri_is_honored(tmp_path):
|
||||||
|
path = _export(KG, tmp_path, graph_uri="https://example.org/my-graph")
|
||||||
|
document = json.loads(path.read_text())
|
||||||
|
|
||||||
|
assert document["@id"] == "https://example.org/my-graph"
|
||||||
|
|
||||||
|
|
||||||
|
def _document_node_id(document):
|
||||||
|
"""The generic (non-knowledge-graph) path hangs its own @id off a member
|
||||||
|
of @graph rather than the top level, to avoid re-creating the named-graph
|
||||||
|
bug fixed by #1145. Find that member and return its @id."""
|
||||||
|
for node in document["@graph"]:
|
||||||
|
if "semantica:exportedAt" in node:
|
||||||
|
return node["@id"]
|
||||||
|
raise AssertionError(f"no document metadata node in @graph: {document}")
|
||||||
|
|
||||||
|
|
||||||
|
def test_caller_supplied_document_uri_is_honored_for_a_generic_export(tmp_path):
|
||||||
|
payload = {"note": "no entities or relationships here"}
|
||||||
|
path = tmp_path / "generic.jsonld"
|
||||||
|
JSONExporter().export(
|
||||||
|
payload, path, format="json-ld", document_uri="https://example.org/my-doc"
|
||||||
|
)
|
||||||
|
document = json.loads(path.read_text())
|
||||||
|
|
||||||
|
assert _document_node_id(document) == "https://example.org/my-doc"
|
||||||
|
|
||||||
|
|
||||||
|
def test_generic_document_id_is_also_content_derived(tmp_path):
|
||||||
|
"""The non-knowledge-graph path (_attach_document_metadata) gets the same fix."""
|
||||||
|
payload = {"note": "plain data, no @id of its own"}
|
||||||
|
|
||||||
|
first = tmp_path / "a.jsonld"
|
||||||
|
second = tmp_path / "b.jsonld"
|
||||||
|
JSONExporter().export(payload, first, format="json-ld")
|
||||||
|
JSONExporter().export(dict(payload), second, format="json-ld")
|
||||||
|
|
||||||
|
first_id = _document_node_id(json.loads(first.read_text()))
|
||||||
|
second_id = _document_node_id(json.loads(second.read_text()))
|
||||||
|
assert first_id == second_id
|
||||||
|
|
||||||
|
|
||||||
|
def test_document_id_still_differs_for_different_generic_payloads(tmp_path):
|
||||||
|
a = tmp_path / "a.jsonld"
|
||||||
|
b = tmp_path / "b.jsonld"
|
||||||
|
JSONExporter().export({"note": "one"}, a, format="json-ld")
|
||||||
|
JSONExporter().export({"note": "two"}, b, format="json-ld")
|
||||||
|
|
||||||
|
a_id = _document_node_id(json.loads(a.read_text()))
|
||||||
|
b_id = _document_node_id(json.loads(b.read_text()))
|
||||||
|
assert a_id != b_id
|
||||||
@@ -107,20 +107,31 @@ def test_exported_timestamp_survives_a_timezone_qualified_sparql_filter():
|
|||||||
assert len(rows) == 1, "the export was dropped by a timezone-qualified filter"
|
assert len(rows) == 1, "the export was dropped by a timezone-qualified filter"
|
||||||
|
|
||||||
|
|
||||||
def test_document_iri_carrying_an_offset_is_a_valid_iri():
|
def test_document_iri_is_a_valid_iri():
|
||||||
"""The offset puts '+' and ':' in the @id; both are legal in a path."""
|
"""The graph @id must be a valid IRI regardless of how it is minted.
|
||||||
|
|
||||||
|
Before #1147, this @id was minted from the offset-carrying timestamp
|
||||||
|
itself (``+00:00`` interpolated straight into the path), so this test
|
||||||
|
asserted the offset survived without breaking IRI validity. #1147 mints
|
||||||
|
the @id from the graph's content instead, so the timestamp no longer
|
||||||
|
appears here at all — it stays in ``semantica:exportedAt`` (still
|
||||||
|
offset-aware, per ``test_jsonld_export_timestamp_is_offset_aware`` above).
|
||||||
|
What's left worth guarding is the general case: whatever the @id is
|
||||||
|
minted from, it has to be a valid IRI that round-trips through RDF.
|
||||||
|
"""
|
||||||
rdflib = pytest.importorskip("rdflib")
|
rdflib = pytest.importorskip("rdflib")
|
||||||
|
|
||||||
document_iri = JSONExporter()._convert_kg_to_jsonld(KG)["@id"]
|
document_iri = JSONExporter()._convert_kg_to_jsonld(KG)["@id"]
|
||||||
assert "+00:00" in document_iri
|
|
||||||
assert rdflib.term._is_valid_uri(document_iri)
|
assert rdflib.term._is_valid_uri(document_iri)
|
||||||
|
|
||||||
graph = rdflib.Graph()
|
graph = rdflib.Graph()
|
||||||
graph.add((
|
graph.add(
|
||||||
rdflib.URIRef(document_iri),
|
(
|
||||||
rdflib.RDF.type,
|
rdflib.URIRef(document_iri),
|
||||||
rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"),
|
rdflib.RDF.type,
|
||||||
))
|
rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"),
|
||||||
|
)
|
||||||
|
)
|
||||||
reparsed = rdflib.Graph().parse(data=graph.serialize(format="nt"), format="nt")
|
reparsed = rdflib.Graph().parse(data=graph.serialize(format="nt"), format="nt")
|
||||||
assert document_iri in {str(s) for s in reparsed.subjects()}
|
assert document_iri in {str(s) for s in reparsed.subjects()}
|
||||||
|
|
||||||
|
|||||||
@@ -525,6 +525,65 @@ class TestSHACLHierarchicalAndValidation(unittest.TestCase):
|
|||||||
self.assertEqual(mc[0].max_count, 2)
|
self.assertEqual(mc[0].max_count, 2)
|
||||||
|
|
||||||
# 33
|
# 33
|
||||||
|
def test_public_run_shacl_validation_api(self):
|
||||||
|
"""The public API validates data and retains the legacy alias."""
|
||||||
|
try:
|
||||||
|
import pyshacl # noqa: F401
|
||||||
|
import rdflib # noqa: F401
|
||||||
|
except ImportError:
|
||||||
|
self.skipTest("pyshacl/rdflib not installed")
|
||||||
|
from semantica.ontology import run_shacl_validation
|
||||||
|
from semantica.ontology.ontology_validator import _run_pyshacl
|
||||||
|
|
||||||
|
data = "@prefix ex: <http://example.org/> . ex:alice a ex:Person ."
|
||||||
|
shacl = """
|
||||||
|
@prefix ex: <http://example.org/> .
|
||||||
|
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
||||||
|
ex:PersonShape a sh:NodeShape ; sh:targetClass ex:Person ;
|
||||||
|
sh:property [ sh:path ex:name ; sh:minCount 1 ] .
|
||||||
|
"""
|
||||||
|
public_report = run_shacl_validation(data, shacl)
|
||||||
|
legacy_report = _run_pyshacl(data, shacl)
|
||||||
|
self.assertFalse(public_report.conforms)
|
||||||
|
self.assertEqual(public_report.violation_count, 1)
|
||||||
|
self.assertEqual(legacy_report.conforms, public_report.conforms)
|
||||||
|
self.assertEqual(legacy_report.violation_count, public_report.violation_count)
|
||||||
|
self.assertEqual(
|
||||||
|
[
|
||||||
|
(v.focus_node, v.result_path, v.constraint, v.severity, v.message)
|
||||||
|
for v in legacy_report.violations
|
||||||
|
],
|
||||||
|
[
|
||||||
|
(v.focus_node, v.result_path, v.constraint, v.severity, v.message)
|
||||||
|
for v in public_report.violations
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
# 34
|
||||||
|
def test_public_run_shacl_validation_conforming_graph(self):
|
||||||
|
"""The public API reports a valid graph without violations."""
|
||||||
|
try:
|
||||||
|
import pyshacl # noqa: F401
|
||||||
|
import rdflib # noqa: F401
|
||||||
|
except ImportError:
|
||||||
|
self.skipTest("pyshacl/rdflib not installed")
|
||||||
|
from semantica.ontology import run_shacl_validation
|
||||||
|
|
||||||
|
data = """
|
||||||
|
@prefix ex: <http://example.org/> .
|
||||||
|
ex:alice a ex:Person ; ex:name "Alice" .
|
||||||
|
"""
|
||||||
|
shacl = """
|
||||||
|
@prefix ex: <http://example.org/> .
|
||||||
|
@prefix sh: <http://www.w3.org/ns/shacl#> .
|
||||||
|
ex:PersonShape a sh:NodeShape ; sh:targetClass ex:Person ;
|
||||||
|
sh:property [ sh:path ex:name ; sh:minCount 1 ] .
|
||||||
|
"""
|
||||||
|
|
||||||
|
report = run_shacl_validation(data, shacl)
|
||||||
|
|
||||||
|
self.assertTrue(report.conforms)
|
||||||
|
self.assertEqual(report.violation_count, 0)
|
||||||
def test_shacl_violation_to_dict(self):
|
def test_shacl_violation_to_dict(self):
|
||||||
from semantica.ontology.ontology_validator import SHACLViolation
|
from semantica.ontology.ontology_validator import SHACLViolation
|
||||||
v = SHACLViolation(
|
v = SHACLViolation(
|
||||||
|
|||||||
@@ -1851,3 +1851,126 @@ class TestExitCodes:
|
|||||||
assert "Traceback" not in result.output, (
|
assert "Traceback" not in result.output, (
|
||||||
f"Traceback found for {argv}: {result.output}"
|
f"Traceback found for {argv}: {result.output}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDoctorEmbeddings:
|
||||||
|
"""#994: doctor must surface non-functional embedding backends instead of
|
||||||
|
reporting all green. Default = import-level check; --deep-embeddings (or
|
||||||
|
SEMANTICA_DOCTOR_DEEP_EMBEDDINGS=1) instantiates via TextEmbedder."""
|
||||||
|
|
||||||
|
def _doctor_checks(self, runner, *extra):
|
||||||
|
result = runner.invoke(cli_module.main, ["doctor", "--json", *extra])
|
||||||
|
_ok(result)
|
||||||
|
import json as _json
|
||||||
|
return {c["check"]: c for c in _json.loads(result.output)}
|
||||||
|
|
||||||
|
def _with_fake_st(self, monkeypatch, **embedder_attrs):
|
||||||
|
fake_st = _fake_module(
|
||||||
|
__version__="9.9.9",
|
||||||
|
SentenceTransformer=object,
|
||||||
|
)
|
||||||
|
monkeypatch.setitem(__import__("sys").modules, "sentence_transformers", fake_st)
|
||||||
|
|
||||||
|
def test_doctor_reports_embedding_checks(self, runner):
|
||||||
|
checks = self._doctor_checks(runner)
|
||||||
|
assert "Embeddings (sentence-transformers)" in checks
|
||||||
|
assert "Embeddings (fastembed)" in checks
|
||||||
|
|
||||||
|
def test_import_failure_is_fail_status_with_hint(self, runner, monkeypatch):
|
||||||
|
# Force the 'import sentence_transformers' inside _embedding_backend to
|
||||||
|
# raise ImportError regardless of whether the package is installed on
|
||||||
|
# this machine. Setting a module entry to None is the standard Python
|
||||||
|
# mechanism: any subsequent 'import <name>' raises
|
||||||
|
# "import of <name> halted; None in sys.modules".
|
||||||
|
monkeypatch.setitem(
|
||||||
|
__import__("sys").modules, "sentence_transformers", None
|
||||||
|
)
|
||||||
|
checks = self._doctor_checks(runner)
|
||||||
|
st = checks["Embeddings (sentence-transformers)"]
|
||||||
|
assert st["status"] == "fail"
|
||||||
|
assert st["hint"] == "pip install sentence-transformers"
|
||||||
|
|
||||||
|
def test_deep_probe_detects_fallback_active(self, runner, monkeypatch):
|
||||||
|
self._with_fake_st(monkeypatch)
|
||||||
|
fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None)
|
||||||
|
|
||||||
|
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
|
||||||
|
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
|
||||||
|
|
||||||
|
checks = self._doctor_checks(runner, "--deep-embeddings")
|
||||||
|
st = checks["Embeddings (sentence-transformers)"]
|
||||||
|
assert st["status"] == "fail"
|
||||||
|
assert "hash fallback" in st["note"]
|
||||||
|
|
||||||
|
def test_deep_probe_ok_when_model_loads(self, runner, monkeypatch):
|
||||||
|
self._with_fake_st(monkeypatch)
|
||||||
|
import numpy as np
|
||||||
|
fake_embedder = types.SimpleNamespace(
|
||||||
|
model=object(),
|
||||||
|
fastembed_model=None,
|
||||||
|
embed_text=lambda text: np.zeros(384, dtype=np.float32),
|
||||||
|
)
|
||||||
|
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
|
||||||
|
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
|
||||||
|
|
||||||
|
checks = self._doctor_checks(runner, "--deep-embeddings")
|
||||||
|
st = checks["Embeddings (sentence-transformers)"]
|
||||||
|
assert st["status"] == "ok"
|
||||||
|
assert "384-dim" in st["note"]
|
||||||
|
|
||||||
|
def test_env_var_enables_deep_mode(self, runner, monkeypatch):
|
||||||
|
monkeypatch.setenv("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", "1")
|
||||||
|
self._with_fake_st(monkeypatch)
|
||||||
|
fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None)
|
||||||
|
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
|
||||||
|
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
|
||||||
|
|
||||||
|
checks = self._doctor_checks(runner)
|
||||||
|
st = checks["Embeddings (sentence-transformers)"]
|
||||||
|
assert st["status"] == "fail"
|
||||||
|
assert "hash fallback" in st["note"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestDoctorEmbeddingHintsAndEnv:
|
||||||
|
"""Review follow-ups: deep failures must not carry the pip-install hint,
|
||||||
|
and the env toggle tolerates case/whitespace variants."""
|
||||||
|
|
||||||
|
def _doctor_checks(self, runner, *extra):
|
||||||
|
result = runner.invoke(cli_module.main, ["doctor", "--json", *extra])
|
||||||
|
_ok(result)
|
||||||
|
import json as _json
|
||||||
|
return {c["check"]: c for c in _json.loads(result.output)}
|
||||||
|
|
||||||
|
def _with_fake_st(self, monkeypatch):
|
||||||
|
fake_st = _fake_module(
|
||||||
|
__version__="9.9.9",
|
||||||
|
SentenceTransformer=object,
|
||||||
|
)
|
||||||
|
monkeypatch.setitem(__import__("sys").modules, "sentence_transformers", fake_st)
|
||||||
|
|
||||||
|
def test_deep_failure_hint_is_not_pip_install(self, runner, monkeypatch):
|
||||||
|
self._with_fake_st(monkeypatch)
|
||||||
|
fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None)
|
||||||
|
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
|
||||||
|
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
|
||||||
|
|
||||||
|
checks = self._doctor_checks(runner, "--deep-embeddings")
|
||||||
|
st = checks["Embeddings (sentence-transformers)"]
|
||||||
|
assert st["status"] == "fail"
|
||||||
|
assert "pip install" not in (st["hint"] or ""), (
|
||||||
|
"a deep probe failure means the package imported fine — pointing "
|
||||||
|
"users at pip sends them to reinstall for a runtime/model problem"
|
||||||
|
)
|
||||||
|
assert "runtime/model-load" in st["hint"]
|
||||||
|
|
||||||
|
def test_env_var_tolerates_case_and_whitespace(self, runner, monkeypatch):
|
||||||
|
monkeypatch.setenv("SEMANTICA_DOCTOR_DEEP_EMBEDDINGS", " TRUE ")
|
||||||
|
self._with_fake_st(monkeypatch)
|
||||||
|
fake_embedder = types.SimpleNamespace(model=None, fastembed_model=None)
|
||||||
|
fake_emb_mod = _fake_module(TextEmbedder=lambda **k: fake_embedder)
|
||||||
|
monkeypatch.setitem(__import__("sys").modules, "semantica.embeddings", fake_emb_mod)
|
||||||
|
|
||||||
|
checks = self._doctor_checks(runner)
|
||||||
|
st = checks["Embeddings (sentence-transformers)"]
|
||||||
|
assert st["status"] == "fail"
|
||||||
|
assert "hash fallback" in st["note"], "padded/caps env value must enable deep mode"
|
||||||
|
|||||||
@@ -85,3 +85,49 @@ if __name__ == '__main__':
|
|||||||
runner = unittest.TextTestRunner(stream=f, verbosity=2)
|
runner = unittest.TextTestRunner(stream=f, verbosity=2)
|
||||||
unittest.main(testRunner=runner, exit=False)
|
unittest.main(testRunner=runner, exit=False)
|
||||||
|
|
||||||
|
|
||||||
|
class TestMethodDispatchRecursion(unittest.TestCase):
|
||||||
|
"""#994: built-in aliases are registered in the method registry onto the
|
||||||
|
wrapper functions themselves, so dispatching through the registry called a
|
||||||
|
wrapper back into itself with the same default method — a recursion storm
|
||||||
|
that surfaced as `maximum recursion depth exceeded` during model loading."""
|
||||||
|
|
||||||
|
def test_generate_embeddings_default_does_not_self_recurse(self):
|
||||||
|
from semantica.embeddings.methods import generate_embeddings
|
||||||
|
emb = generate_embeddings("recursion probe")
|
||||||
|
self.assertIsNotNone(emb)
|
||||||
|
|
||||||
|
def test_embed_text_default_does_not_self_recurse(self):
|
||||||
|
from semantica.embeddings.methods import embed_text
|
||||||
|
emb = embed_text("recursion probe", method="sentence_transformers")
|
||||||
|
self.assertIsNotNone(emb)
|
||||||
|
|
||||||
|
def test_custom_registered_method_still_wins(self):
|
||||||
|
from semantica.embeddings.methods import method_registry
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def spy(data, *a, **k):
|
||||||
|
calls.append(data)
|
||||||
|
return {"custom": True}
|
||||||
|
|
||||||
|
method_registry.register("generation", "my_custom_gen", spy)
|
||||||
|
try:
|
||||||
|
from semantica.embeddings.methods import generate_embeddings
|
||||||
|
out = generate_embeddings("payload", method="my_custom_gen")
|
||||||
|
self.assertEqual(out, {"custom": True})
|
||||||
|
self.assertEqual(calls, ["payload"])
|
||||||
|
finally:
|
||||||
|
method_registry.unregister("generation", "my_custom_gen")
|
||||||
|
|
||||||
|
def test_provenance_wrapper_missing_generator_raises_attribute_error(self):
|
||||||
|
# Partially-initialised wrappers (failed __init__, pickle/copy probes)
|
||||||
|
# must raise AttributeError, not RecursionError via __getattr__.
|
||||||
|
from semantica.embeddings.embeddings_provenance import (
|
||||||
|
EmbeddingGeneratorWithProvenance,
|
||||||
|
)
|
||||||
|
bare = EmbeddingGeneratorWithProvenance.__new__(
|
||||||
|
EmbeddingGeneratorWithProvenance
|
||||||
|
)
|
||||||
|
with self.assertRaises(AttributeError):
|
||||||
|
getattr(bare, "model")
|
||||||
|
|
||||||
|
|||||||
@@ -243,5 +243,19 @@ class TestVectorStore(unittest.TestCase):
|
|||||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCreateIndexFunction(unittest.TestCase):
|
||||||
|
"""create_index() forwards vector_store_config's defaults into VectorIndexer,
|
||||||
|
which already receives backend/dimension as explicit args. Regression for the
|
||||||
|
'got multiple values for keyword argument dimension' crash on the default
|
||||||
|
(unmocked) config, hit by e.g. `semantica embed index`."""
|
||||||
|
|
||||||
|
def test_create_index_with_default_config(self):
|
||||||
|
from semantica.vector_store.methods import create_index
|
||||||
|
|
||||||
|
vectors = [np.array([0.1, 0.2, 0.3]), np.array([0.4, 0.5, 0.6])]
|
||||||
|
index = create_index(vectors, ids=["a", "b"])
|
||||||
|
self.assertIsNotNone(index)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user