From 08a6e7c0535afa5e747ce3c4e8f91620215a49f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=9F=E4=BF=8A=E6=9D=B0?= Date: Wed, 19 Aug 2026 16:30:36 +0800 Subject: [PATCH 01/13] fix(ontology): render real SHACL constraint values in explain_violations explain_violations previously rendered hardcoded placeholders (min_count=1, max_count=1) and misused the violation message as the datatype/class value, so plain-English explanations were inaccurate. The root cause is that _run_pyshacl never read the real constraint parameters from sh:sourceShape when building each SHACLViolation. Changes: - SHACLViolation: add min_count/max_count/datatype/class_ fields and include them in to_dict() - _run_pyshacl: back-reference sh:sourceShape to extract the real sh:minCount/sh:maxCount/sh:datatype/sh:class values - explain_violations: render the real values, falling back to "?" or descriptive text when unknown Note: sh:qualifiedMinCount/qualifiedMaxCount are not handled and fall back to the "?" placeholder. Adds regression tests covering both the formatting path and the sh:sourceShape back-reference (skips when pyshacl/rdflib are absent). --- semantica/ontology/ontology_validator.py | 48 +++++++++++- tests/ontology/test_ontology_advanced.py | 97 ++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 4 deletions(-) diff --git a/semantica/ontology/ontology_validator.py b/semantica/ontology/ontology_validator.py index 4a03f2d1..5d9d10df 100644 --- a/semantica/ontology/ontology_validator.py +++ b/semantica/ontology/ontology_validator.py @@ -33,6 +33,12 @@ class SHACLViolation: value: Optional[str] = None shape: Optional[str] = None explanation: Optional[str] = None + # Real constraint parameters extracted from the source shape (sh:sourceShape), + # used to render accurate plain-English explanations. + min_count: Optional[int] = None + max_count: Optional[int] = None + datatype: Optional[str] = None + class_: Optional[str] = None def to_dict(self) -> Dict[str, Any]: return { @@ -44,6 +50,10 @@ class SHACLViolation: "value": self.value, "shape": self.shape, "explanation": self.explanation, + "min_count": self.min_count, + "max_count": self.max_count, + "datatype": self.datatype, + "class_": self.class_, } @@ -118,10 +128,10 @@ class SHACLValidationReport: focus_node=v.focus_node, path=v.result_path or "", value=v.value or "", - min_count=1, - max_count=1, - datatype=v.message or "", - class_=v.message or "", + min_count=v.min_count if v.min_count is not None else "?", + max_count=v.max_count if v.max_count is not None else "?", + datatype=v.datatype or "the expected datatype", + class_=v.class_ or "the required class", ) def to_dict(self) -> Dict[str, Any]: @@ -208,6 +218,32 @@ def _run_pyshacl( shape_node = results_graph.value(result, SH.sourceShape) shape = str(shape_node) if shape_node is not None else None + # Look up the real constraint parameters from the source shape so that + # explain_violations can render accurate values instead of placeholders. + # Note: sh:qualifiedMinCount / sh:qualifiedMaxCount are not handled here; + # such violations fall back to the "?" placeholder in explain_violations. + min_count: Optional[int] = None + max_count: Optional[int] = None + datatype: Optional[str] = None + class_: Optional[str] = None + if shape_node is not None: + min_node = shacl_g.value(shape_node, SH.minCount) + if min_node is not None: + try: + min_count = int(str(min_node)) + except (TypeError, ValueError): + min_count = None + max_node = shacl_g.value(shape_node, SH.maxCount) + if max_node is not None: + try: + max_count = int(str(max_node)) + except (TypeError, ValueError): + max_count = None + dt_node = shacl_g.value(shape_node, SH.datatype) + datatype = str(dt_node) if dt_node is not None else None + cls_node = shacl_g.value(shape_node, SH["class"]) + class_ = str(cls_node) if cls_node is not None else None + v = SHACLViolation( focus_node=focus, result_path=path, @@ -216,6 +252,10 @@ def _run_pyshacl( message=msg, value=val, shape=shape, + min_count=min_count, + max_count=max_count, + datatype=datatype, + class_=class_, ) if sev_str == "Violation": violations.append(v) diff --git a/tests/ontology/test_ontology_advanced.py b/tests/ontology/test_ontology_advanced.py index 564ca758..14ac52d5 100644 --- a/tests/ontology/test_ontology_advanced.py +++ b/tests/ontology/test_ontology_advanced.py @@ -414,6 +414,103 @@ class TestSHACLHierarchicalAndValidation(unittest.TestCase): self.assertIsNotNone(v.explanation) self.assertIn("https://example.com/john", v.explanation) + # 32b + def test_explain_violations_uses_real_constraint_values(self): + """explain_violations must render the real min/max/datatype/class values, + not hardcoded placeholders (regression for PR #318).""" + from semantica.ontology.ontology_validator import ( + SHACLValidationReport, + SHACLViolation, + ) + + max_v = SHACLViolation( + focus_node="https://example.com/john", + result_path="ex:age", + constraint="MaxCountConstraintComponent", + max_count=3, + ) + dt_v = SHACLViolation( + focus_node="https://example.com/john", + result_path="ex:age", + constraint="DatatypeConstraintComponent", + value="abc", + datatype="http://www.w3.org/2001/XMLSchema#integer", + ) + cls_v = SHACLViolation( + focus_node="https://example.com/john", + result_path="ex:knows", + constraint="ClassConstraintComponent", + value="https://example.com/thing", + class_="https://example.com/Person", + ) + report = SHACLValidationReport( + conforms=False, violations=[max_v, dt_v, cls_v] + ) + report.explain_violations() + # MaxCount must show the real limit (3), not the hardcoded 1. + self.assertIn("3", max_v.explanation) + self.assertNotIn("At most 1 value", max_v.explanation) + # Datatype must show the real datatype IRI, not the message. + self.assertIn( + "http://www.w3.org/2001/XMLSchema#integer", dt_v.explanation + ) + # Class must show the real class IRI. + self.assertIn("https://example.com/Person", cls_v.explanation) + + # 32c + def test_run_pyshacl_extracts_constraint_values_from_shape(self): + """_run_pyshacl must back-reference sh:sourceShape to populate the real + constraint parameters on each violation.""" + try: + import pyshacl # noqa: F401 + import rdflib # noqa: F401 + except ImportError: + self.skipTest("pyshacl/rdflib not installed") + from semantica.ontology.ontology_validator import _run_pyshacl + + shacl = """ + @prefix sh: . + @prefix ex: . + @prefix xsd: . + + ex:PersonShape a sh:NodeShape ; + sh:targetClass ex:Person ; + sh:property [ + sh:path ex:age ; + sh:datatype xsd:integer ; + sh:maxCount 2 ; + ] . + """ + data = """ + @prefix ex: . + ex:john a ex:Person ; + ex:age "not-a-number" ; + ex:age 1 ; + ex:age 2 ; + ex:age 3 . + """ + report = _run_pyshacl(data, shacl) + self.assertFalse(report.conforms) + # Datatype violation should carry the real xsd:integer datatype. + dt = [ + v + for v in report.violations + if v.constraint == "DatatypeConstraintComponent" + ] + self.assertTrue(dt) + self.assertTrue( + dt[0].datatype.endswith("integer"), + f"expected integer datatype, got {dt[0].datatype}", + ) + # MaxCount violation should carry the real max_count == 2. + mc = [ + v + for v in report.violations + if v.constraint == "MaxCountConstraintComponent" + ] + if mc: + self.assertEqual(mc[0].max_count, 2) + # 33 def test_shacl_violation_to_dict(self): from semantica.ontology.ontology_validator import SHACLViolation From b77e3e8c3cc3b63755b62caf2e4ec22e70c167a5 Mon Sep 17 00:00:00 2001 From: "Guofang.Tang" <136770748@qq.com> Date: Wed, 19 Aug 2026 19:17:51 +0800 Subject: [PATCH 02/13] fix(kg): preserve entity_id aliases during entity merging (#1086) * fix(kg): preserve entity_id aliases during merge * fix(kg): unify entity ID extraction semantics --- semantica/deduplication/entity_merger.py | 3 +- semantica/deduplication/merge_strategy.py | 11 ++++- semantica/kg/entity_resolver.py | 13 +++--- semantica/utils/entity_ids.py | 16 +++++++ tests/kg/test_entity_pipeline.py | 57 ++++++++++++++++++++++- 5 files changed, 89 insertions(+), 11 deletions(-) create mode 100644 semantica/utils/entity_ids.py diff --git a/semantica/deduplication/entity_merger.py b/semantica/deduplication/entity_merger.py index b19447cd..6552f8cd 100644 --- a/semantica/deduplication/entity_merger.py +++ b/semantica/deduplication/entity_merger.py @@ -45,6 +45,7 @@ License: MIT from dataclasses import dataclass from typing import Any, Dict, List, Optional, Union +from ..utils.entity_ids import get_entity_id from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -504,7 +505,7 @@ class EntityMerger: # Record source entities provenance["merged_from"] = [ { - "id": self._get_entity_value(e, "id"), + "id": get_entity_id(e), "name": self._get_entity_value(e, "name"), "source": self._get_entity_value(e, "metadata", {}).get("source") if hasattr(e, "metadata") or isinstance(e, dict) else None, } diff --git a/semantica/deduplication/merge_strategy.py b/semantica/deduplication/merge_strategy.py index b5c2b29e..610aecb2 100644 --- a/semantica/deduplication/merge_strategy.py +++ b/semantica/deduplication/merge_strategy.py @@ -45,6 +45,7 @@ from dataclasses import dataclass, field from enum import Enum from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from ..utils.entity_ids import get_entity_id from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -317,14 +318,20 @@ class MergeStrategyManager: message=f"Building merged entity... ({current_step}/{total_steps}, remaining: {remaining_steps} steps)" ) # Build merged entity + merged_from = [] + for entity in entities: + entity_id = get_entity_id(entity) + if entity_id is not None: + merged_from.append(entity_id) + merged_entity = { - "id": base_entity.get("id"), + "id": get_entity_id(base_entity), "name": self._merge_top_level_field("name", entities, base_entity), "type": self._merge_top_level_field("type", entities, base_entity), "properties": merged_properties, "relationships": merged_relationships, "metadata": self._merge_metadata(entities, base_entity), - "merged_from": [e.get("id") for e in entities if e.get("id")], + "merged_from": merged_from, "merge_strategy": strategy.value, } diff --git a/semantica/kg/entity_resolver.py b/semantica/kg/entity_resolver.py index a56c8cdd..06f4d299 100644 --- a/semantica/kg/entity_resolver.py +++ b/semantica/kg/entity_resolver.py @@ -24,6 +24,7 @@ from typing import Any, Dict, List, Optional from ..deduplication.duplicate_detector import DuplicateDetector, DuplicateGroup from ..deduplication.entity_merger import EntityMerger +from ..utils.entity_ids import get_entity_id from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -240,9 +241,7 @@ class EntityResolver: @staticmethod def _get_entity_id(entity: Any) -> Any: """Return an entity ID while supporting dictionary and object inputs.""" - if isinstance(entity, dict): - return entity.get("id") or entity.get("entity_id") - return getattr(entity, "id", None) or getattr(entity, "entity_id", None) + return get_entity_id(entity) @staticmethod def _get_entity_name(entity: Any) -> Optional[str]: @@ -279,13 +278,13 @@ class EntityResolver: processed_ids = set() for op in merge_operations: for source_entity in op.source_entities: - entity_id = source_entity.get("id") or source_entity.get("entity_id") - if entity_id: + entity_id = get_entity_id(source_entity) + if entity_id is not None: processed_ids.add(entity_id) for entity in entities: - entity_id = entity.get("id") or entity.get("entity_id") - if entity_id and entity_id not in processed_ids: + entity_id = get_entity_id(entity) + if entity_id is not None and entity_id not in processed_ids: merged_entities.append(entity) self.logger.info(f"Merged to {len(merged_entities)} entities") diff --git a/semantica/utils/entity_ids.py b/semantica/utils/entity_ids.py new file mode 100644 index 00000000..e33b965e --- /dev/null +++ b/semantica/utils/entity_ids.py @@ -0,0 +1,16 @@ +"""Helpers for reading entity identifiers consistently across the KG pipeline.""" + +from typing import Any + + +def get_entity_id(entity: Any) -> Any: + """Return a truthy identifier from either supported entity ID field. + + The KG pipeline treats empty and otherwise falsy identifiers as missing. + Prefer the canonical ``id`` field when it is populated, then fall back to + the compatible ``entity_id`` alias. + """ + if isinstance(entity, dict): + return entity.get("id") or entity.get("entity_id") or None + + return getattr(entity, "id", None) or getattr(entity, "entity_id", None) or None diff --git a/tests/kg/test_entity_pipeline.py b/tests/kg/test_entity_pipeline.py index ec345286..7823ea1a 100644 --- a/tests/kg/test_entity_pipeline.py +++ b/tests/kg/test_entity_pipeline.py @@ -1,9 +1,10 @@ import pytest -from semantica.utils.types import Entity from semantica.kg.graph_builder import GraphBuilder from semantica.kg.entity_resolver import EntityResolver from semantica.kg.graph_analyzer import GraphAnalyzer +from semantica.utils.entity_ids import get_entity_id +from semantica.utils.types import Entity def test_full_entity_pipeline(): """ @@ -107,6 +108,60 @@ def test_direct_entity_objects_in_analyzer(): print("Direct Entity objects test passed successfully!") + +def test_entity_id_only_merge_remaps_relationship_endpoints(): + """Entity aliases must survive merging and relationship remapping.""" + builder = GraphBuilder( + merge_entities=True, + entity_resolution_strategy="exact", + resolve_conflicts=False, + ) + + graph = builder.build( + { + "entities": [ + {"entity_id": "alice:1", "name": "Alice", "type": "Person"}, + {"entity_id": "alice:2", "name": " Alice ", "type": "Person"}, + {"entity_id": "org:1", "name": "Acme", "type": "Organization"}, + ], + "relationships": [ + { + "source_id": "alice:2", + "target_id": "org:1", + "type": "WORKS_FOR", + } + ], + } + ) + + merged_alice = next( + entity for entity in graph["entities"] if entity["name"] == "Alice" + ) + relationship = graph["relationships"][0] + entity_ids = { + entity.get("id") or entity.get("entity_id") for entity in graph["entities"] + } + + assert merged_alice["id"] == "alice:1" + assert set(merged_alice["merged_from"]) == {"alice:1", "alice:2"} + assert { + item["id"] for item in merged_alice["metadata"]["provenance"]["merged_from"] + } == {"alice:1", "alice:2"} + assert relationship["source"] == "alice:1" + assert relationship["target"] == "org:1" + assert {relationship["source"], relationship["target"]} <= entity_ids + + +def test_entity_id_helper_ignores_falsy_identifiers(): + """ID extraction must match the KG pipeline's falsy-ID contract.""" + assert get_entity_id({"id": "", "entity_id": "alias:1"}) == "alias:1" + assert get_entity_id({"id": 0, "entity_id": "alias:2"}) == "alias:2" + assert ( + get_entity_id({"id": "primary:1", "entity_id": "alias:3"}) + == "primary:1" + ) + assert get_entity_id({"id": "", "entity_id": 0}) is None + if __name__ == "__main__": test_full_entity_pipeline() test_direct_entity_objects_in_analyzer() From e1092ac507b80c1fd310f10374933cf5587631d0 Mon Sep 17 00:00:00 2001 From: FABIOTESS Date: Wed, 19 Aug 2026 12:28:11 +0100 Subject: [PATCH 03/13] feat(ontology): declare the Semantica vocabulary, and mint entity IRIs deterministically Closes #1107, closes #1101. Every RDF export mints terms in https://semantica.dev/ns#, and nothing declared what those terms meant. The namespace returns 404 and no vocabulary shipped with the package, so a consumer receiving an export could not tell semantica:text from a typo of it: in the open world an undeclared IRI is unknown rather than wrong, and every RDF tool treats the two alike. Closed-world checking is what separates them, and it needs a document to check against. semantica/ontology/vocabulary/semantica-ns.ttl declares the fourteen terms the exporters actually emit, drawn from the emitting call sites rather than from what a vocabulary ought to contain. It ships inside the package so it loads without a network round trip, and is the same document intended to be served at the namespace IRI once hosting and content negotiation are sorted. tests/ontology/test_vocabulary.py ties the document to the code: every term the serializers can write must be declared, so adding a term to an exporter without declaring it fails the build rather than shipping an undeclared IRI. The vocabulary alone would not have made those IRIs resolve, because the fallback path minted them from Python's builtin hash(). That is randomised per process, so the same entity received a different IRI on every run and exports could not be diffed, deduplicated against an earlier load, or joined to a provenance record written by an earlier process. Minting now uses SHA-256 and writes a full IRI in the declared namespace rather than semantica:entity_N, which inside angle brackets is an IRI in the scheme "semantica" rather than the prefix expansion, and so never joined with anything written through the prefix. The same applies to the default entity and relationship types in the Turtle path. 134 export tests and 91 ontology tests pass. Co-Authored-By: Claude Opus 5 (1M context) --- pyproject.toml | 2 +- semantica/export/rdf_exporter.py | 44 +++++- semantica/ontology/vocabulary/__init__.py | 35 +++++ .../ontology/vocabulary/semantica-ns.ttl | 134 ++++++++++++++++++ tests/export/test_rdf_exporter_iri_minting.py | 88 ++++++++++++ tests/ontology/test_vocabulary.py | 85 +++++++++++ 6 files changed, 381 insertions(+), 7 deletions(-) create mode 100644 semantica/ontology/vocabulary/__init__.py create mode 100644 semantica/ontology/vocabulary/semantica-ns.ttl create mode 100644 tests/export/test_rdf_exporter_iri_minting.py create mode 100644 tests/ontology/test_vocabulary.py diff --git a/pyproject.toml b/pyproject.toml index 341e57ed..286a26b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -271,7 +271,7 @@ include = ["semantica*", "integrations*"] [tool.setuptools.package-data] # Explicit patterns are more reliable than **/* across setuptools versions. # static/* covers index.html / favicon; static/assets/* covers all JS/CSS chunks. -"semantica" = ["static/*", "static/assets/*"] +"semantica" = ["static/*", "static/assets/*", "ontology/vocabulary/*.ttl"] [tool.black] line-length = 88 diff --git a/semantica/export/rdf_exporter.py b/semantica/export/rdf_exporter.py index 7c50ffc5..40cc4341 100644 --- a/semantica/export/rdf_exporter.py +++ b/semantica/export/rdf_exporter.py @@ -30,6 +30,7 @@ License: MIT """ from pathlib import Path +import hashlib from typing import Any, Dict, List, Optional, Set, Union from ..utils.exceptions import ProcessingError, ValidationError @@ -38,6 +39,37 @@ from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker +SEMANTICA_NS = "https://semantica.dev/ns#" + +#: Written when an entity carries no type of its own. A full IRI rather than the +#: prefixed form, because the Turtle serializer writes it inside angle brackets, +#: where `semantica:Entity` would be read as an IRI in the scheme `semantica` +#: rather than as the prefix expansion (issue #1101). +DEFAULT_ENTITY_TYPE = f"{SEMANTICA_NS}Entity" + +#: Written when a relationship carries no type of its own. Same reasoning. +DEFAULT_RELATION_TYPE = f"{SEMANTICA_NS}related_to" + + +def mint_entity_iri(text: str) -> str: + """Mint a stable IRI for an entity that arrived without an id. + + Python's builtin ``hash()`` is randomised per process (PYTHONHASHSEED), so + minting from it gave the same entity a different IRI on every run: exports + could not be diffed, deduplicated against an earlier load, or joined to a + provenance record written by an earlier process. SHA-256 is stable across + runs and machines, which is what an identifier has to be. + """ + digest = hashlib.sha256(str(text).encode("utf-8")).hexdigest()[:16] + return f"{SEMANTICA_NS}entity_{digest}" + + +def mint_relationship_iri(index: int, source: Any, target: Any) -> str: + """Mint a stable IRI for a relationship that arrived without an id.""" + digest = hashlib.sha256(f"{source}\x00{target}".encode("utf-8")).hexdigest()[:16] + return f"{SEMANTICA_NS}rel_{index}_{digest}" + + class NamespaceManager: """ RDF namespace management engine. @@ -360,9 +392,9 @@ class RDFSerializer: entity_id = entity.get("id") if not entity_id: entity_text = entity.get("text", "") - entity_id = f"semantica:entity_{hash(entity_text)}" + entity_id = mint_entity_iri(entity_text) - entity_type = entity.get("type", "semantica:Entity") + entity_type = entity.get("type", DEFAULT_ENTITY_TYPE) text = entity.get("text") or entity.get("label", "") confidence = entity.get("confidence", 1.0) @@ -376,7 +408,7 @@ class RDFSerializer: for idx, rel in enumerate(relationships): source_id = rel.get("source_id") or rel.get("source") target_id = rel.get("target_id") or rel.get("target") - rel_type = rel.get("type", "semantica:related_to") + rel_type = rel.get("type", DEFAULT_RELATION_TYPE) lines.append(f"<{source_id}> <{rel_type}> <{target_id}> .") @@ -415,7 +447,7 @@ class RDFSerializer: rel_base_id = ( rel.get("id") - or f"semantica:rel_{idx}_{hash(str(rel.get('source_id', '')) + str(rel.get('target_id', '')))}" + or mint_relationship_iri(idx, rel.get('source_id', ''), rel.get('target_id', '')) ) lines = [""] # blank separator @@ -488,7 +520,7 @@ class RDFSerializer: entity_id = entity.get("id") if not entity_id: entity_text = entity.get("text", "") - entity_id = f"semantica:entity_{hash(entity_text)}" + entity_id = mint_entity_iri(entity_text) entity_type = entity.get("type", "semantica:Entity") text = entity.get("text") or entity.get("label", "") @@ -644,7 +676,7 @@ class RDFSerializer: entity_id = entity.get("id") if not entity_id: entity_text = entity.get("text", "") - entity_id = f"semantica:entity_{hash(entity_text)}" + entity_id = mint_entity_iri(entity_text) subject = expand_uri(entity_id) diff --git a/semantica/ontology/vocabulary/__init__.py b/semantica/ontology/vocabulary/__init__.py new file mode 100644 index 00000000..d0665830 --- /dev/null +++ b/semantica/ontology/vocabulary/__init__.py @@ -0,0 +1,35 @@ +"""The vocabulary Semantica's exporters emit terms from. + +Every RDF export mints terms in ``https://semantica.dev/ns#``: ``sem:text``, +``sem:confidence``, the default ``sem:Entity`` type, and the rest. Until this +file existed, nothing declared what those terms meant, so a consumer receiving +an export could not tell ``sem:text`` from a typo of it, and no closed-world +check could be run against them at all (issue #1107). + +The document ships inside the package so it can be loaded without a network +round trip, and is the same file intended to be served at the namespace IRI. + + >>> from semantica.ontology.vocabulary import vocabulary_turtle + >>> ttl = vocabulary_turtle() +""" + +from __future__ import annotations + +from pathlib import Path + +VOCABULARY_FILENAME = "semantica-ns.ttl" + +#: The namespace the vocabulary declares terms in. +NAMESPACE = "https://semantica.dev/ns#" + +__all__ = ["NAMESPACE", "VOCABULARY_FILENAME", "vocabulary_path", "vocabulary_turtle"] + + +def vocabulary_path() -> Path: + """Filesystem path to the vocabulary document.""" + return Path(__file__).parent / VOCABULARY_FILENAME + + +def vocabulary_turtle() -> str: + """The vocabulary document as Turtle.""" + return vocabulary_path().read_text(encoding="utf-8") diff --git a/semantica/ontology/vocabulary/semantica-ns.ttl b/semantica/ontology/vocabulary/semantica-ns.ttl new file mode 100644 index 00000000..b5695b08 --- /dev/null +++ b/semantica/ontology/vocabulary/semantica-ns.ttl @@ -0,0 +1,134 @@ +@prefix owl: . +@prefix rdf: . +@prefix rdfs: . +@prefix xsd: . +@prefix dct: . +@prefix prov: . +@prefix time: . +@prefix sem: . + + a owl:Ontology ; + rdfs:label "Semantica vocabulary" ; + rdfs:comment """Declares the terms the Semantica exporters emit in +https://semantica.dev/ns#. Drafted from the emitting call sites in +semantica 0.6.5: export/rdf_exporter.py, export/json_exporter.py and +provenance/manager.py. Every term below appears in output the package +produces today; no term has been invented for completeness.""" ; + owl:versionInfo "0.1.0-draft" ; + dct:created "2026-08-19"^^xsd:date . + +# ── Classes ────────────────────────────────────────────────────────────────── + +sem:Entity a owl:Class ; + rdfs:label "Entity" ; + rdfs:comment """The default type given to an extracted entity when the +source carries no type of its own. Emitted by serialize_to_turtle as the +fallback for entity.get("type").""" ; + rdfs:isDefinedBy . + +sem:Relationship a owl:Class ; + rdfs:label "Relationship" ; + rdfs:comment """A reified relationship, as emitted in the JSON-LD export +where a relationship carries sem:type, sem:source and sem:target rather than +being written as a single triple.""" ; + rdfs:isDefinedBy . + +# ── Properties on an entity ────────────────────────────────────────────────── + +sem:text a owl:DatatypeProperty ; + rdfs:label "text" ; + rdfs:comment """The surface text of an extracted entity. Carries the same +intent as rdfs:label; declared separately because the exporters emit it under +this IRI.""" ; + rdfs:domain sem:Entity ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + +sem:confidence a owl:DatatypeProperty ; + rdfs:label "confidence" ; + rdfs:comment """Extractor confidence in the assertion, on the unit interval. +Emitted for both entities and relationships, so the domain is left open rather +than tied to sem:Entity.""" ; + rdfs:range xsd:decimal ; + rdfs:isDefinedBy . + +sem:metadata a owl:AnnotationProperty ; + rdfs:label "metadata" ; + rdfs:comment """Free-form metadata carried through from extraction. An +annotation property because its value is an arbitrary structure rather than a +modelled one.""" ; + rdfs:isDefinedBy . + +# ── Relationship terms (JSON-LD export) ────────────────────────────────────── + +sem:related_to a owl:ObjectProperty ; + rdfs:label "related to" ; + rdfs:comment """The default predicate for a relationship whose type the +extractor did not determine. Deliberately unspecific: it asserts that two +entities are connected and nothing about how.""" ; + rdfs:isDefinedBy . + +sem:source a owl:ObjectProperty ; + rdfs:label "source" ; + rdfs:comment "The subject entity of a reified relationship." ; + rdfs:domain sem:Relationship ; + rdfs:isDefinedBy . + +sem:target a owl:ObjectProperty ; + rdfs:label "target" ; + rdfs:comment "The object entity of a reified relationship." ; + rdfs:domain sem:Relationship ; + rdfs:isDefinedBy . + +sem:type a owl:DatatypeProperty ; + rdfs:label "type" ; + rdfs:comment """The relationship type as a label, as emitted in the JSON-LD +export. Distinct from rdf:type, which relates a node to a class rather than to +a string.""" ; + rdfs:domain sem:Relationship ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + +# ── Document-level terms (JSON-LD export) ──────────────────────────────────── + +sem:entities a owl:ObjectProperty ; + rdfs:label "entities" ; + rdfs:comment "Ordered list of entities in an exported graph document." ; + rdfs:range sem:Entity ; + rdfs:isDefinedBy . + +sem:relationships a owl:ObjectProperty ; + rdfs:label "relationships" ; + rdfs:comment "Ordered list of relationships in an exported graph document." ; + rdfs:range sem:Relationship ; + rdfs:isDefinedBy . + +sem:exportedAt a owl:DatatypeProperty ; + rdfs:label "exported at" ; + rdfs:comment """When the export was written. Emitted as an ISO 8601 local +timestamp, so the range is xsd:dateTime rather than xsd:dateTimeStamp: the +values carry no timezone offset.""" ; + rdfs:range xsd:dateTime ; + rdfs:isDefinedBy . + +# ── Temporal term (OWL-Time export) ────────────────────────────────────────── + +sem:openEndedInterval a owl:DatatypeProperty ; + rdfs:label "open ended interval" ; + rdfs:comment """True when an interval has no known end. OWL-Time has no +standard predicate for this, which is the reason the exporter mints one: an +interval with no time:hasEnd is ambiguous between "ongoing" and "end not +recorded", and this term resolves that in favour of the first.""" ; + rdfs:domain time:Interval ; + rdfs:range xsd:boolean ; + rdfs:isDefinedBy . + +# ── Provenance roles ───────────────────────────────────────────────────────── + +sem:role_generator a prov:Role ; + rdfs:label "generator" ; + rdfs:comment """The default role in a prov:qualifiedAssociation, used when +an agent generated an entity rather than approving or reviewing it. Typed as +prov:Role so that prov:hadRole has a declared value rather than an undeclared +IRI.""" ; + rdfs:isDefinedBy . diff --git a/tests/export/test_rdf_exporter_iri_minting.py b/tests/export/test_rdf_exporter_iri_minting.py new file mode 100644 index 00000000..5734db73 --- /dev/null +++ b/tests/export/test_rdf_exporter_iri_minting.py @@ -0,0 +1,88 @@ +"""Minted IRIs must be stable and must sit in the declared namespace (issue #1101). + +An entity that arrives without an id gets one minted for it. That identifier was +built from Python's builtin ``hash()``, which is randomised per process, so the +same entity received a different IRI on every run and exports could not be +diffed, deduplicated against an earlier load, or joined to a provenance record +written by an earlier process. + +It was also written as ``semantica:entity_N`` inside angle brackets, which is an +IRI in the scheme ``semantica`` rather than the expansion of the declared +``semantica:`` prefix, so it never joined with anything written through it. +""" + +import subprocess +import sys + +from semantica.export.rdf_exporter import ( + DEFAULT_ENTITY_TYPE, + DEFAULT_RELATION_TYPE, + RDFExporter, + SEMANTICA_NS, + mint_entity_iri, + mint_relationship_iri, +) + +UNIDENTIFIED = { + "entities": [{"text": "Acme Corp", "type": "https://example.org/Org"}], + "relationships": [], +} + + +def test_minted_entity_iri_is_stable_within_a_process(): + assert mint_entity_iri("Acme Corp") == mint_entity_iri("Acme Corp") + + +def test_minted_entity_iri_is_stable_across_processes(): + """The regression that matters: identity must survive a restart.""" + script = ( + "from semantica.export.rdf_exporter import mint_entity_iri;" + "print(mint_entity_iri('Acme Corp'))" + ) + runs = { + subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=True, + env={"PYTHONHASHSEED": seed, "PATH": "/usr/bin:/bin"}, + ).stdout.strip() + for seed in ("0", "1", "random") + } + assert len(runs) == 1, f"minted IRI differs between processes: {runs}" + + +def test_minted_iris_are_in_the_declared_namespace(): + assert mint_entity_iri("Acme Corp").startswith(SEMANTICA_NS) + assert mint_relationship_iri(0, "a", "b").startswith(SEMANTICA_NS) + + +def test_distinct_entities_get_distinct_iris(): + assert mint_entity_iri("Acme Corp") != mint_entity_iri("Acme Corporation") + + +def test_turtle_export_writes_a_resolvable_minted_iri(): + turtle = RDFExporter().export_to_rdf(UNIDENTIFIED, format="turtle") + + assert f"<{SEMANTICA_NS}entity_" in turtle + assert "" in turtle + assert f"<{DEFAULT_RELATION_TYPE}>" in turtle + assert "" not in turtle + assert "" not in turtle diff --git a/tests/ontology/test_vocabulary.py b/tests/ontology/test_vocabulary.py new file mode 100644 index 00000000..49d2e794 --- /dev/null +++ b/tests/ontology/test_vocabulary.py @@ -0,0 +1,85 @@ +"""The vocabulary must stay true to what the exporters emit (issue #1107). + +A vocabulary document that drifts from the code is worse than none, because it +states that terms mean something while the exporters emit different ones. These +tests tie the two together: every term the serializers can write must be +declared here, so adding a term to an exporter without declaring it fails the +build rather than shipping an undeclared IRI. +""" + +import pytest + +rdflib = pytest.importorskip("rdflib") + +from semantica.export.rdf_exporter import ( # noqa: E402 + DEFAULT_ENTITY_TYPE, + DEFAULT_RELATION_TYPE, + SEMANTICA_NS, +) +from semantica.ontology.vocabulary import ( # noqa: E402 + NAMESPACE, + vocabulary_path, + vocabulary_turtle, +) + +#: Every term the exporters emit in the Semantica namespace, by local name. +#: RDF and OWL-Time paths in export/rdf_exporter.py, document and relationship +#: terms in export/json_exporter.py, roles in provenance/manager.py. +EMITTED_TERMS = { + "Entity", + "Relationship", + "text", + "confidence", + "metadata", + "related_to", + "source", + "target", + "type", + "entities", + "relationships", + "exportedAt", + "openEndedInterval", + "role_generator", +} + + +@pytest.fixture(scope="module") +def graph(): + g = rdflib.Graph() + g.parse(data=vocabulary_turtle(), format="turtle") + return g + + +def test_vocabulary_ships_with_the_package(): + assert vocabulary_path().is_file() + + +def test_vocabulary_parses(graph): + assert len(graph) > 0 + + +def test_namespace_matches_the_one_the_exporters_use(): + assert NAMESPACE == SEMANTICA_NS + + +def test_every_emitted_term_is_declared(graph): + declared = { + str(s)[len(NAMESPACE) :] + for s in set(graph.subjects()) + if isinstance(s, rdflib.URIRef) and str(s).startswith(NAMESPACE) + } + missing = EMITTED_TERMS - declared + assert not missing, f"emitted but not declared in the vocabulary: {sorted(missing)}" + + +def test_the_defaults_the_exporters_fall_back_to_are_declared(graph): + for iri in (DEFAULT_ENTITY_TYPE, DEFAULT_RELATION_TYPE): + assert (rdflib.URIRef(iri), None, None) in graph, f"{iri} is not declared" + + +def test_every_declared_term_carries_a_label_and_a_comment(graph): + for subject in set(graph.subjects()): + if not (isinstance(subject, rdflib.URIRef) and str(subject).startswith(NAMESPACE)): + continue + assert graph.value(subject, rdflib.RDFS.label), f"{subject} has no rdfs:label" + assert graph.value(subject, rdflib.RDFS.comment), f"{subject} has no rdfs:comment" From 4a886d970e01efeaad830e17e782e2e2691201f0 Mon Sep 17 00:00:00 2001 From: Zohaib Hassnain <109234410+ZohaibHassan16@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:06:01 +0500 Subject: [PATCH 04/13] fix seed SSRF (#942) Co-authored-by: Sameer Kadam Co-authored-by: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> --- semantica/seed/seed_manager.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/semantica/seed/seed_manager.py b/semantica/seed/seed_manager.py index 6e52c382..3e6ebbf8 100644 --- a/semantica/seed/seed_manager.py +++ b/semantica/seed/seed_manager.py @@ -38,6 +38,7 @@ from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Union +from ..ingest.ssrf import request_with_ssrf_guard from ..utils.exceptions import ProcessingError, ValidationError from ..utils.helpers import read_json_file, write_json_file from ..utils.logging import get_logger @@ -456,8 +457,8 @@ class SeedDataManager: """ Load seed data from API. - Makes an HTTP GET request to an API endpoint and parses the JSON - response. Handles various response structures (list, dict with + Makes an SSRF-protected HTTP GET request to an API endpoint and parses + the JSON response. Handles various response structures (list, dict with 'entities', 'data', 'results', 'items' keys). Automatically adds entity_type, relationship_type, and source metadata if provided. @@ -493,8 +494,6 @@ class SeedDataManager: ... ) """ try: - import requests - # Build full URL if endpoint: full_url = f"{api_url.rstrip('/')}/{endpoint.lstrip('/')}" From e55c03bd397817e634ac1330d345a5c28dc78763 Mon Sep 17 00:00:00 2001 From: FABIOTESS Date: Wed, 19 Aug 2026 13:35:39 +0100 Subject: [PATCH 05/13] fix: resolve temporal endpoints both ways, and stop declaring a range the exporters contradict Both from review on #1109. The temporal fallback minted from source_id only, while the main serializer accepts source_id or source. Relationships using the second form therefore hashed two empty strings, and once the IRI became deterministic that turned a latent problem into an active one: unrelated relationships at the same list index collided on the same IRI across exports, so their temporal data aliased when loaded together. Endpoints are now resolved the way serialize_to_turtle resolves them, before minting. The vocabulary declared sem:confidence with range xsd:decimal, which the N-Triples serializer contradicts by typing the same value xsd:float. Neither is safe to declare while the two serializers disagree, since the Turtle path writes the value bare and the Turtle grammar reads that as xsd:decimal. The range is dropped with the reasoning recorded on the term and a pointer to #1100, which tracks the disagreement itself. Extends the drift guard rather than only fixing the instance: a new test asserts that any range this vocabulary declares matches the datatype the serializers actually emit, so the class of contradiction that review caught fails the build next time. 228 export and ontology tests pass. Co-Authored-By: Claude Opus 5 (1M context) --- semantica/export/rdf_exporter.py | 12 ++++--- .../ontology/vocabulary/semantica-ns.ttl | 10 ++++-- tests/export/test_rdf_exporter_iri_minting.py | 36 +++++++++++++++++++ tests/ontology/test_vocabulary.py | 29 +++++++++++++++ 4 files changed, 81 insertions(+), 6 deletions(-) diff --git a/semantica/export/rdf_exporter.py b/semantica/export/rdf_exporter.py index 40cc4341..0d8dcdee 100644 --- a/semantica/export/rdf_exporter.py +++ b/semantica/export/rdf_exporter.py @@ -445,10 +445,14 @@ class RDFSerializer: if time_axis in ("transaction", "both"): axes.append(("tx", rel.get("recorded_at"), rel.get("superseded_at"))) - rel_base_id = ( - rel.get("id") - or mint_relationship_iri(idx, rel.get('source_id', ''), rel.get('target_id', '')) - ) + # Resolve endpoints the same way serialize_to_turtle does: both + # representations are accepted upstream, and minting from source_id + # alone hashes empty strings for every relationship that uses source, + # so unrelated relationships at the same index would collide on a + # deterministic IRI. + source_id = rel.get("source_id") or rel.get("source") or "" + target_id = rel.get("target_id") or rel.get("target") or "" + rel_base_id = rel.get("id") or mint_relationship_iri(idx, source_id, target_id) lines = [""] # blank separator for axis_name, from_val, until_val in axes: diff --git a/semantica/ontology/vocabulary/semantica-ns.ttl b/semantica/ontology/vocabulary/semantica-ns.ttl index b5695b08..19943eac 100644 --- a/semantica/ontology/vocabulary/semantica-ns.ttl +++ b/semantica/ontology/vocabulary/semantica-ns.ttl @@ -48,8 +48,14 @@ sem:confidence a owl:DatatypeProperty ; rdfs:label "confidence" ; rdfs:comment """Extractor confidence in the assertion, on the unit interval. Emitted for both entities and relationships, so the domain is left open rather -than tied to sem:Entity.""" ; - rdfs:range xsd:decimal ; +than tied to sem:Entity. + +No rdfs:range is declared, deliberately. The Turtle serializer writes the value +bare, which the Turtle grammar reads as xsd:decimal, while the N-Triples +serializer types it xsd:float explicitly, and those two datatypes are disjoint. +Declaring either one would make the vocabulary contradict one of the exporters. +Issue #1100 tracks the disagreement; a range belongs here once the serializers +agree on one.""" ; rdfs:isDefinedBy . sem:metadata a owl:AnnotationProperty ; diff --git a/tests/export/test_rdf_exporter_iri_minting.py b/tests/export/test_rdf_exporter_iri_minting.py index 5734db73..fc56b74e 100644 --- a/tests/export/test_rdf_exporter_iri_minting.py +++ b/tests/export/test_rdf_exporter_iri_minting.py @@ -86,3 +86,39 @@ def test_default_types_are_written_as_full_iris_in_turtle(): assert f"<{DEFAULT_RELATION_TYPE}>" in turtle assert "" not in turtle assert "" not in turtle + + +def test_temporal_minting_uses_either_endpoint_representation(): + """Relationships may carry source/target or source_id/target_id (#1109 review). + + Minting from source_id alone hashed empty strings for every relationship + that used the other representation, so once the IRI became deterministic, + unrelated relationships at the same index collided on it and their temporal + data aliased when the exports were loaded together. + """ + def temporal(rel): + return RDFExporter().export_to_rdf( + {"entities": [], "relationships": [rel]}, + format="turtle", + include_temporal=True, + ) + + a = temporal({"source": "https://example.org/a", "target": "https://example.org/b", + "type": "https://example.org/worksFor", "valid_from": "2020-01-01T00:00:00Z"}) + b = temporal({"source": "https://example.org/c", "target": "https://example.org/d", + "type": "https://example.org/worksFor", "valid_from": "2020-01-01T00:00:00Z"}) + + assert f"<{SEMANTICA_NS}rel_" in a + assert a != b, "different endpoints must not mint the same temporal IRI" + + +def test_temporal_minting_agrees_across_the_two_representations(): + """The same relationship written either way is the same relationship.""" + def mint(rel): + return mint_relationship_iri( + 0, + rel.get("source_id") or rel.get("source") or "", + rel.get("target_id") or rel.get("target") or "", + ) + + assert mint({"source": "a", "target": "b"}) == mint({"source_id": "a", "target_id": "b"}) diff --git a/tests/ontology/test_vocabulary.py b/tests/ontology/test_vocabulary.py index 49d2e794..e59d0d77 100644 --- a/tests/ontology/test_vocabulary.py +++ b/tests/ontology/test_vocabulary.py @@ -83,3 +83,32 @@ def test_every_declared_term_carries_a_label_and_a_comment(graph): continue assert graph.value(subject, rdflib.RDFS.label), f"{subject} has no rdfs:label" assert graph.value(subject, rdflib.RDFS.comment), f"{subject} has no rdfs:comment" + + +def test_declared_ranges_do_not_contradict_what_the_exporters_emit(graph): + """A declared range must match the datatype the serializers actually write. + + Caught by review on #1109: sem:confidence was declared xsd:decimal while the + N-Triples serializer types the same value xsd:float. A vocabulary that + contradicts the code is worse than no vocabulary, so any range declared here + has to be one the exporters really emit. + """ + import re + + from semantica.export.rdf_exporter import RDFExporter + + sample = { + "entities": [{"id": "https://example.org/e1", "text": "A", + "type": "https://example.org/T", "confidence": 0.5}], + "relationships": [], + } + emitted = RDFExporter().export_to_rdf(sample, format="ntriples") + + for subject, _, range_ in graph.triples((None, rdflib.RDFS.range, None)): + if not str(subject).startswith(NAMESPACE): + continue + local = str(subject)[len(NAMESPACE):] + for match in re.finditer(rf'<{NAMESPACE}{local}> "[^"]*"\^\^<([^>]+)>', emitted): + assert match.group(1) == str(range_), ( + f"{local}: vocabulary declares {range_}, N-Triples emits {match.group(1)}" + ) From 2d75952476f2e489838e3dc4a5fa88b690ee5c43 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Wed, 19 Aug 2026 19:09:02 +0530 Subject: [PATCH 06/13] fix: close remaining review gaps in vocabulary/deterministic-IRI PR serialize_to_rdfxml still defaulted entity_type to the bare string "semantica:Entity" written into an rdf:resource attribute, which isn't namespace-expanded the way a Turtle angle-bracket or XML element name is - the same #1101 failure mode, just on the path the original tests didn't cover. Now uses the full-IRI DEFAULT_ENTITY_TYPE like the Turtle path. json_exporter.py emits semantica:format and @type: "semantica:KnowledgeGraph", neither of which was declared in the vocabulary or included in EMITTED_TERMS, so the "undeclared terms fail the build" guarantee didn't actually cover them. Both are now declared with rdfs:label/comment and added to the guard set. MANIFEST.in didn't mirror the pyproject.toml package-data addition, so a source-distribution install could ship without the vocabulary file. The cross-process minting-stability test replaced the subprocess's entire environment with a POSIX-only PATH, breaking it on Windows and any host needing other inherited env vars; now overrides only PYTHONHASHSEED on top of the inherited environment. Also folds mint_entity_iri/mint_relationship_iri's hand-rolled hashlib.sha256(...).hexdigest() into the existing hash_data() helper this file already imports alongside. 229 export and ontology tests pass, including a new regression test for the RDF/XML default-type fix. Co-Authored-By: fabio-rovai --- CHANGELOG.md | 9 +++++++++ MANIFEST.in | 1 + semantica/export/rdf_exporter.py | 9 ++++----- .../ontology/vocabulary/semantica-ns.ttl | 15 +++++++++++++++ tests/export/test_rdf_exporter_iri_minting.py | 19 ++++++++++++++++++- tests/ontology/test_vocabulary.py | 2 ++ 6 files changed, 49 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbd7cfb2..2c81bc4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Semantica RDF vocabulary, and deterministic entity/relationship IRIs** (#1109, closes #1107, closes #1101) by @fabio-rovai, reviewed by @KaifAhmad1 + - Every RDF/JSON-LD export mints terms in `https://semantica.dev/ns#`, and until now nothing declared what those terms meant — the namespace 404s and no vocabulary shipped with the package, so a consumer receiving an export had no way to tell `sem:text` from a typo of it, and no closed-world checker could validate an export at all + - `semantica/ontology/vocabulary/semantica-ns.ttl` declares the terms the exporters actually emit — drawn from the emitting call sites in `export/rdf_exporter.py`, `export/json_exporter.py` and `provenance/manager.py`, not from what a vocabulary "ought" to contain. Ships inside the package (`from semantica.ontology.vocabulary import vocabulary_turtle`) so it loads without a network round trip, and is the same document intended to be served at the namespace IRI once hosting/content-negotiation is sorted + - `tests/ontology/test_vocabulary.py` ties the document to the code: every term a serializer can write must be declared, so adding a term to an exporter without declaring it fails the build + - The missing-id fallback minted entity/relationship IRIs from Python's builtin `hash()`, randomised per process (`PYTHONHASHSEED`), so the same entity got a different IRI on every run and exports couldn't be diffed, deduplicated, or joined to an earlier provenance record. It also wrote ``, an IRI in the scheme `semantica` rather than the expansion of the declared prefix, so those nodes never joined with anything written through it. Minting now uses SHA-256 and writes a full IRI in the declared namespace; the same fix applies to the default entity/relationship types in the Turtle path + - **Fixed during review** (Qodo): the temporal fallback minted from `source_id` only, while the main serializer accepts `source_id` or `source` — relationships using the second form hashed two empty strings, which the previous randomised `hash()` masked by making the IRI unstable anyway; once deterministic, unrelated relationships at the same list index collided on one IRI across exports. Endpoints are now resolved the same way `serialize_to_turtle` resolves them, before minting. `sem:confidence` also lost its declared `xsd:decimal` range: the N-Triples serializer types the same value `xsd:float`, and the two are disjoint, so declaring either contradicted one of the exporters (tracked in #1100) — a new `test_declared_ranges_do_not_contradict_what_the_exporters_emit` guards the whole class of that mistake + - **Fixed in follow-up**: `serialize_to_rdfxml`'s default entity type still wrote the bare string `"semantica:Entity"` into an `rdf:resource` attribute, which (unlike a Turtle angle-bracket or an XML element name) is not namespace-expanded — the exact #1101 failure mode, just on the untested RDF/XML path. `json_exporter.py`'s `semantica:format` and `@type: "semantica:KnowledgeGraph"` were emitted but absent from both the vocabulary and the test's `EMITTED_TERMS` guard set, so the "undeclared terms fail the build" claim didn't actually cover them — both are now declared and guarded. `MANIFEST.in` didn't mirror the `pyproject.toml` package-data addition, so a source-distribution install could omit the vocabulary file. The cross-process minting-stability test replaced the subprocess's entire environment with a POSIX-only `PATH`, breaking it on Windows; now overrides only `PYTHONHASHSEED` on top of the inherited environment + - 229 export and ontology tests pass + - **First-class CrewAI integration** (#962) - New `pip install semantica[crewai]` extra (`crewai>=0.80.0`) — crewai core provides `BaseTool`/`BaseKnowledgeSource`, so `crewai-tools` is intentionally not included, and the extra is intentionally **not** part of the `all` bundle: crewai hard-requires `chromadb~=1.1.0`, which is affected by the unpatched pre-auth code-injection CVE-2026-45829 (see `integrations/crewai/README.md`) - `integrations/crewai/SemanticaKGTool` — a CrewAI `BaseTool` exposing 5 KG actions (`extract_entities`, `extract_relations`, `add_to_graph`, `query_graph`, `find_related`) backed by `NERExtractor` / `RelationExtractor` / `ContextGraph`; supports both sync `run()` and async `arun()` diff --git a/MANIFEST.in b/MANIFEST.in index aa726d6b..7d60a31a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ recursive-include semantica/static * +recursive-include semantica/ontology/vocabulary *.ttl diff --git a/semantica/export/rdf_exporter.py b/semantica/export/rdf_exporter.py index 0d8dcdee..63eac4fe 100644 --- a/semantica/export/rdf_exporter.py +++ b/semantica/export/rdf_exporter.py @@ -30,11 +30,10 @@ License: MIT """ from pathlib import Path -import hashlib from typing import Any, Dict, List, Optional, Set, Union from ..utils.exceptions import ProcessingError, ValidationError -from ..utils.helpers import ensure_directory +from ..utils.helpers import ensure_directory, hash_data from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -60,13 +59,13 @@ def mint_entity_iri(text: str) -> str: provenance record written by an earlier process. SHA-256 is stable across runs and machines, which is what an identifier has to be. """ - digest = hashlib.sha256(str(text).encode("utf-8")).hexdigest()[:16] + digest = hash_data(str(text))[:16] return f"{SEMANTICA_NS}entity_{digest}" def mint_relationship_iri(index: int, source: Any, target: Any) -> str: """Mint a stable IRI for a relationship that arrived without an id.""" - digest = hashlib.sha256(f"{source}\x00{target}".encode("utf-8")).hexdigest()[:16] + digest = hash_data(f"{source}\x00{target}")[:16] return f"{SEMANTICA_NS}rel_{index}_{digest}" @@ -526,7 +525,7 @@ class RDFSerializer: entity_text = entity.get("text", "") entity_id = mint_entity_iri(entity_text) - entity_type = entity.get("type", "semantica:Entity") + entity_type = entity.get("type", DEFAULT_ENTITY_TYPE) text = entity.get("text") or entity.get("label", "") confidence = entity.get("confidence", 1.0) diff --git a/semantica/ontology/vocabulary/semantica-ns.ttl b/semantica/ontology/vocabulary/semantica-ns.ttl index 19943eac..c6843a32 100644 --- a/semantica/ontology/vocabulary/semantica-ns.ttl +++ b/semantica/ontology/vocabulary/semantica-ns.ttl @@ -33,6 +33,13 @@ where a relationship carries sem:type, sem:source and sem:target rather than being written as a single triple.""" ; rdfs:isDefinedBy . +sem:KnowledgeGraph a owl:Class ; + rdfs:label "Knowledge Graph" ; + rdfs:comment """The document-level type of a JSON-LD export: the @type of +the top-level node carrying sem:entities, sem:relationships and +sem:exportedAt. Emitted by _convert_kg_to_jsonld in export/json_exporter.py.""" ; + rdfs:isDefinedBy . + # ── Properties on an entity ────────────────────────────────────────────────── sem:text a owl:DatatypeProperty ; @@ -117,6 +124,14 @@ values carry no timezone offset.""" ; rdfs:range xsd:dateTime ; rdfs:isDefinedBy . +sem:format a owl:DatatypeProperty ; + rdfs:label "format" ; + rdfs:comment """The serialization format label written on a JSON-LD +document (currently always the literal "json-ld"). Emitted by +JSONExporter.export_to_jsonld in export/json_exporter.py.""" ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + # ── Temporal term (OWL-Time export) ────────────────────────────────────────── sem:openEndedInterval a owl:DatatypeProperty ; diff --git a/tests/export/test_rdf_exporter_iri_minting.py b/tests/export/test_rdf_exporter_iri_minting.py index fc56b74e..3650dec2 100644 --- a/tests/export/test_rdf_exporter_iri_minting.py +++ b/tests/export/test_rdf_exporter_iri_minting.py @@ -11,6 +11,7 @@ IRI in the scheme ``semantica`` rather than the expansion of the declared ``semantica:`` prefix, so it never joined with anything written through it. """ +import os import subprocess import sys @@ -45,7 +46,7 @@ def test_minted_entity_iri_is_stable_across_processes(): capture_output=True, text=True, check=True, - env={"PYTHONHASHSEED": seed, "PATH": "/usr/bin:/bin"}, + env={**os.environ, "PYTHONHASHSEED": seed}, ).stdout.strip() for seed in ("0", "1", "random") } @@ -88,6 +89,22 @@ def test_default_types_are_written_as_full_iris_in_turtle(): assert "" not in turtle +def test_default_entity_type_is_a_full_iri_in_rdfxml(): + """RDF/XML's rdf:resource is an attribute value, not a QName context, so a + + prefixed default there (``semantica:Entity``) resolves to the scheme + ``semantica`` rather than the declared namespace — the same failure mode + fixed for Turtle in #1101, missed here because the original tests only + checked Turtle output. + """ + untyped = {"entities": [{"id": "https://example.org/e1", "text": "A"}], + "relationships": []} + rdfxml = RDFExporter().export_to_rdf(untyped, format="rdfxml") + + assert f'rdf:resource="{DEFAULT_ENTITY_TYPE}"' in rdfxml + assert 'rdf:resource="semantica:Entity"' not in rdfxml + + def test_temporal_minting_uses_either_endpoint_representation(): """Relationships may carry source/target or source_id/target_id (#1109 review). diff --git a/tests/ontology/test_vocabulary.py b/tests/ontology/test_vocabulary.py index e59d0d77..34eb4e5d 100644 --- a/tests/ontology/test_vocabulary.py +++ b/tests/ontology/test_vocabulary.py @@ -28,6 +28,7 @@ from semantica.ontology.vocabulary import ( # noqa: E402 EMITTED_TERMS = { "Entity", "Relationship", + "KnowledgeGraph", "text", "confidence", "metadata", @@ -38,6 +39,7 @@ EMITTED_TERMS = { "entities", "relationships", "exportedAt", + "format", "openEndedInterval", "role_generator", } From 75b026c6ddb3dc8da98a17db0a86f1723625eafd Mon Sep 17 00:00:00 2001 From: FABIOTESS Date: Wed, 19 Aug 2026 15:09:41 +0100 Subject: [PATCH 07/13] fix(export): mint JSON-LD @ids the same way the RDF serializers do (#1101) The #1101 fix covered serialize_to_turtle, serialize_to_ntriples and serialize_to_rdfxml. Both JSON-LD writers were left interpolating the entity's own text into f"semantica:entity/{text}" and the endpoints into f"semantica:rel/{source}_{target}". Three consequences, all reproducible on 0.6.5 through the public API: - An entity whose text contains a space, which is most organisation and person names an extractor produces, mints an invalid IRI. A JSON-LD parser drops that node in full and says nothing, so the entity is simply missing from the export: rdflib reads 6 triples for {"text": "AcmeCorp"} and 2 for {"text": "Acme Corp"}. - serialize_to_jsonld resolved endpoints from source_id/target_id only, while the rest of the module accepts source/target too. Every relationship carrying the second form minted the identical "semantica:rel/_", so all of them collapsed onto one node and their types and endpoints merged into a graph nobody wrote. - The JSON-LD @id and the Turtle IRI for one entity disagreed (ns#entity/Acme Corp vs ns#entity_a73cb4563ee2e72c), so the two serializations of one knowledge graph were two different graphs. Both writers now use mint_entity_iri/mint_relationship_iri, resolving endpoints both ways and passing the list index the RDF paths pass, so one knowledge graph carries one node identity whichever serializer wrote it. JSONExporter.export_entities and export_relationships also declare the semantica prefix their @context was already writing "semantica:entities" against. Without the declaration a processor reads that as an IRI in the scheme semantica rather than the namespace expansion, which is the original #1101 defect on a third path: rdflib returns the predicate literally as semantica:entities. tests/export/test_jsonld_iri_minting.py parses each export with a real JSON-LD processor rather than asserting on the JSON text, and covers all seven claims above. Each test fails on the parent commit. 236 export and ontology tests pass. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 4 +- semantica/export/json_exporter.py | 36 +++--- semantica/export/rdf_exporter.py | 36 +++--- tests/export/test_jsonld_iri_minting.py | 139 ++++++++++++++++++++++++ 4 files changed, 183 insertions(+), 32 deletions(-) create mode 100644 tests/export/test_jsonld_iri_minting.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c81bc4a..5cb0fdd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The missing-id fallback minted entity/relationship IRIs from Python's builtin `hash()`, randomised per process (`PYTHONHASHSEED`), so the same entity got a different IRI on every run and exports couldn't be diffed, deduplicated, or joined to an earlier provenance record. It also wrote ``, an IRI in the scheme `semantica` rather than the expansion of the declared prefix, so those nodes never joined with anything written through it. Minting now uses SHA-256 and writes a full IRI in the declared namespace; the same fix applies to the default entity/relationship types in the Turtle path - **Fixed during review** (Qodo): the temporal fallback minted from `source_id` only, while the main serializer accepts `source_id` or `source` — relationships using the second form hashed two empty strings, which the previous randomised `hash()` masked by making the IRI unstable anyway; once deterministic, unrelated relationships at the same list index collided on one IRI across exports. Endpoints are now resolved the same way `serialize_to_turtle` resolves them, before minting. `sem:confidence` also lost its declared `xsd:decimal` range: the N-Triples serializer types the same value `xsd:float`, and the two are disjoint, so declaring either contradicted one of the exporters (tracked in #1100) — a new `test_declared_ranges_do_not_contradict_what_the_exporters_emit` guards the whole class of that mistake - **Fixed in follow-up**: `serialize_to_rdfxml`'s default entity type still wrote the bare string `"semantica:Entity"` into an `rdf:resource` attribute, which (unlike a Turtle angle-bracket or an XML element name) is not namespace-expanded — the exact #1101 failure mode, just on the untested RDF/XML path. `json_exporter.py`'s `semantica:format` and `@type: "semantica:KnowledgeGraph"` were emitted but absent from both the vocabulary and the test's `EMITTED_TERMS` guard set, so the "undeclared terms fail the build" claim didn't actually cover them — both are now declared and guarded. `MANIFEST.in` didn't mirror the `pyproject.toml` package-data addition, so a source-distribution install could omit the vocabulary file. The cross-process minting-stability test replaced the subprocess's entire environment with a POSIX-only `PATH`, breaking it on Windows; now overrides only `PYTHONHASHSEED` on top of the inherited environment - - 229 export and ontology tests pass + - **Also fixed, on the JSON-LD paths**: the first fix covered the Turtle, N-Triples and RDF/XML serializers, and left both JSON-LD writers interpolating the entity's own text into `f"semantica:entity/{text}"` and the endpoints into `f"semantica:rel/{source}_{target}"`. Three consequences, all live in 0.6.5: an entity whose text contained a space produced an invalid IRI, and a JSON-LD parser dropped that node in full rather than reporting it, so the entity disappeared from the export; every relationship carrying `source`/`target` rather than `source_id`/`target_id` minted the identical `semantica:rel/_`, collapsing all of them onto one node whose types and endpoints merged; and the JSON-LD `@id` disagreed with the Turtle IRI for the same entity, so the two serializations of one knowledge graph were two different graphs. Both JSON-LD writers now use `mint_entity_iri`/`mint_relationship_iri`, and `JSONExporter.export_entities`/`export_relationships` declare the `semantica` prefix their `@context` was already writing `semantica:entities` against — without it a processor reads that as an IRI in the scheme `semantica`, which is the original #1101 defect on a third path + - `tests/export/test_jsonld_iri_minting.py` parses each export with a real JSON-LD processor and asserts the entity survives, the relationships stay distinct, no term expands into the `semantica` scheme, and the JSON-LD `@id` equals the Turtle IRI + - 236 export and ontology tests pass - **First-class CrewAI integration** (#962) - New `pip install semantica[crewai]` extra (`crewai>=0.80.0`) — crewai core provides `BaseTool`/`BaseKnowledgeSource`, so `crewai-tools` is intentionally not included, and the extra is intentionally **not** part of the `all` bundle: crewai hard-requires `chromadb~=1.1.0`, which is affected by the unpatched pre-auth code-injection CVE-2026-45829 (see `integrations/crewai/README.md`) diff --git a/semantica/export/json_exporter.py b/semantica/export/json_exporter.py index 2bca8cbf..aa1641ae 100644 --- a/semantica/export/json_exporter.py +++ b/semantica/export/json_exporter.py @@ -32,6 +32,7 @@ from ..utils.exceptions import ProcessingError, ValidationError from ..utils.helpers import ensure_directory, write_json_file from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker +from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri class JSONExporter: @@ -265,6 +266,7 @@ class JSONExporter: json_data = { "@context": { "@vocab": "https://semantica.dev/vocab/", + "semantica": SEMANTICA_NS, "entities": {"@id": "semantica:entities", "@container": "@list"}, }, "entities": entities, @@ -294,6 +296,7 @@ class JSONExporter: json_data = { "@context": { "@vocab": "https://semantica.dev/vocab/", + "semantica": SEMANTICA_NS, "relationships": { "@id": "semantica:relationships", "@container": "@list", @@ -495,7 +498,8 @@ class JSONExporter: relationships = kg.get("relationships", []) if relationships: jsonld["semantica:relationships"] = [ - self._relationship_to_jsonld(r) for r in relationships + self._relationship_to_jsonld(r, index) + for index, r in enumerate(relationships) ] self.logger.debug( f"Converted {len(relationships)} relationship(s) to JSON-LD" @@ -526,11 +530,13 @@ class JSONExporter: Returns: Dictionary in JSON-LD format representing the entity """ - # Generate @id if not provided - entity_id = entity.get("id") - if not entity_id: - entity_text = entity.get("text") or entity.get("label", "unknown") - entity_id = f"semantica:entity/{entity_text}" + # Generate @id if not provided. Minted exactly as the RDF serializers + # mint it (#1101), so the JSON-LD and Turtle exports of one knowledge + # graph name the same entity with the same IRI. Interpolating the raw + # text into f"semantica:entity/{text}" produced an invalid IRI for any + # text containing a space, and a JSON-LD parser dropped the whole node. + entity_text = entity.get("text") or entity.get("label", "unknown") + entity_id = entity.get("id") or mint_entity_iri(entity_text) jsonld = { "@id": entity_id, @@ -545,7 +551,9 @@ class JSONExporter: return jsonld - def _relationship_to_jsonld(self, rel: Dict[str, Any]) -> Dict[str, Any]: + def _relationship_to_jsonld( + self, rel: Dict[str, Any], index: int = 0 + ) -> Dict[str, Any]: """ Convert relationship to JSON-LD format. @@ -560,16 +568,18 @@ class JSONExporter: - type: Relationship type (optional) - confidence: Confidence score (optional) - metadata: Metadata dictionary (optional) + index: Position of the relationship in the exported list, used when + minting an IRI for a relationship that arrived without an id Returns: Dictionary in JSON-LD format representing the relationship """ - # Generate @id if not provided - rel_id = rel.get("id") - if not rel_id: - source_id = rel.get("source_id") or rel.get("source", "") - target_id = rel.get("target_id") or rel.get("target", "") - rel_id = f"semantica:rel/{source_id}_{target_id}" + # Generate @id if not provided, from the same mint the RDF serializers + # use, including the list index that separates two relationships + # sharing a pair of endpoints (#1101). + source_id = rel.get("source_id") or rel.get("source", "") + target_id = rel.get("target_id") or rel.get("target", "") + rel_id = rel.get("id") or mint_relationship_iri(index, source_id, target_id) jsonld = { "@id": rel_id, diff --git a/semantica/export/rdf_exporter.py b/semantica/export/rdf_exporter.py index 63eac4fe..de3d0dd0 100644 --- a/semantica/export/rdf_exporter.py +++ b/semantica/export/rdf_exporter.py @@ -600,11 +600,13 @@ class RDFSerializer: # Convert entities to JSON-LD entities = rdf_data.get("entities", []) for entity in entities: - # Generate @id if not provided - entity_id = entity.get("id") - if not entity_id: - entity_text = entity.get("text", "") - entity_id = f"semantica:entity/{entity_text}" + # Generate @id if not provided. Minted the same way the Turtle and + # N-Triples paths mint it (#1101), so one knowledge graph carries + # the same node identity whichever serializer wrote it. The former + # f"semantica:entity/{text}" interpolated the raw text into an IRI: + # any entity whose text contained a space produced an invalid IRI + # and was dropped in full by a JSON-LD parser, silently. + entity_id = entity.get("id") or mint_entity_iri(entity.get("text", "")) jsonld["@graph"].append( { @@ -617,24 +619,22 @@ class RDFSerializer: # Convert relationships to JSON-LD relationships = rdf_data.get("relationships", []) - for rel in relationships: - # Generate @id if not provided - rel_id = rel.get("id") - if not rel_id: - source_id = rel.get("source_id", "") - target_id = rel.get("target_id", "") - rel_id = f"semantica:rel/{source_id}_{target_id}" + for index, rel in enumerate(relationships): + # Endpoints are resolved both ways, as serialize_to_turtle resolves + # them: a relationship carrying source/target rather than + # source_id/target_id used to hash into f"semantica:rel/_", so every + # such relationship in an export collapsed onto one node and their + # types and endpoints merged. + source = rel.get("source_id") or rel.get("source", "") + target = rel.get("target_id") or rel.get("target", "") + rel_id = rel.get("id") or mint_relationship_iri(index, source, target) jsonld["@graph"].append( { "@id": rel_id, "@type": "semantica:Relationship", - "semantica:source": { - "@id": rel.get("source_id") or rel.get("source") - }, - "semantica:target": { - "@id": rel.get("target_id") or rel.get("target") - }, + "semantica:source": {"@id": source}, + "semantica:target": {"@id": target}, "semantica:type": rel.get("type", "related_to"), } ) diff --git a/tests/export/test_jsonld_iri_minting.py b/tests/export/test_jsonld_iri_minting.py new file mode 100644 index 00000000..26b24bf7 --- /dev/null +++ b/tests/export/test_jsonld_iri_minting.py @@ -0,0 +1,139 @@ +"""The JSON-LD paths must mint the same IRIs as the RDF paths (issue #1101). + +#1101 was fixed for the Turtle, N-Triples and RDF/XML serializers: an entity +arriving without an id gets a deterministic IRI in the declared namespace. The +JSON-LD paths were left interpolating the entity's own text into +``f"semantica:entity/{text}"`` and the relationship's endpoints into +``f"semantica:rel/{source}_{target}"``, which fails three ways: + +* a text containing a space produces an invalid IRI, and a JSON-LD parser drops + the whole node rather than complaining, so the entity vanishes from the export; +* relationships carrying ``source``/``target`` rather than ``source_id``/ + ``target_id`` all minted ``semantica:rel/_``, so every one of them collapsed + onto a single node whose types and endpoints merged; +* the JSON-LD @id and the Turtle IRI for one entity disagreed, so the two + serializations of one knowledge graph were two different graphs. + +``JSONExporter.export_entities`` and ``export_relationships`` also wrote +``semantica:entities`` into a context that never declared the ``semantica`` +prefix, which a JSON-LD processor reads as an IRI in the scheme ``semantica`` — +the original #1101 failure mode, on a path the first fix did not cover. +""" + +import json + +import pytest + +from semantica.export.json_exporter import JSONExporter +from semantica.export.rdf_exporter import ( + RDFExporter, + SEMANTICA_NS, + mint_entity_iri, + mint_relationship_iri, +) + +KG = { + "entities": [ + {"text": "Acme Corp", "type": "https://example.org/Org"}, + {"id": "https://example.org/e2", "text": "Bob"}, + ], + "relationships": [ + {"source": "https://example.org/a", "target": "https://example.org/b", + "type": "https://example.org/employs"}, + {"source": "https://example.org/b", "target": "https://example.org/a", + "type": "https://example.org/works_for"}, + ], +} + + +def _graph(document: str): + """Parse a JSON-LD document the way a consumer would.""" + rdflib = pytest.importorskip("rdflib") + graph = rdflib.Graph() + graph.parse(data=document, format="json-ld") + return graph + + +def test_jsonld_entity_id_is_the_minted_iri_not_the_interpolated_text(): + graph = json.loads(RDFExporter().export_to_rdf(KG, format="jsonld"))["@graph"] + + assert graph[0]["@id"] == mint_entity_iri("Acme Corp") + assert graph[0]["@id"].startswith(SEMANTICA_NS) + assert "semantica:entity/" not in json.dumps(graph) + + +def test_json_exporter_mints_the_same_entity_iri_as_the_rdf_exporter(): + """One knowledge graph, two exporters, one node identity.""" + from_rdf = json.loads(RDFExporter().export_to_rdf(KG, format="jsonld"))["@graph"] + from_json = JSONExporter()._convert_kg_to_jsonld(KG) + + assert from_json["semantica:entities"][0]["@id"] == from_rdf[0]["@id"] + assert from_json["semantica:relationships"][0]["@id"] == from_rdf[2]["@id"] + + +def test_minted_jsonld_iri_agrees_with_the_turtle_serialization(): + """The two serializations of one graph must name the same entity alike.""" + turtle = RDFExporter().export_to_rdf(KG, format="turtle") + jsonld = json.loads(RDFExporter().export_to_rdf(KG, format="jsonld")) + + minted = mint_entity_iri("Acme Corp") + assert f"<{minted}>" in turtle + assert jsonld["@graph"][0]["@id"] == minted + + +def test_entity_whose_text_contains_a_space_survives_a_jsonld_parse(): + """The regression that lost data: an invalid IRI is dropped, not reported.""" + kg = {"entities": [{"text": "Acme Corp"}], "relationships": []} + graph = _graph(json.dumps(JSONExporter()._convert_kg_to_jsonld(kg))) + + subjects = {str(s) for s in graph.subjects()} + assert mint_entity_iri("Acme Corp") in subjects + + +def test_relationships_carrying_source_and_target_do_not_collide(): + """Two relationships, two nodes: ``semantica:rel/_`` merged them into one.""" + jsonld = json.loads(RDFExporter().export_to_rdf(KG, format="jsonld")) + relationships = [n for n in jsonld["@graph"] + if n["@type"] == "semantica:Relationship"] + + ids = {node["@id"] for node in relationships} + assert len(ids) == len(relationships) == 2 + assert ids == { + mint_relationship_iri(0, "https://example.org/a", "https://example.org/b"), + mint_relationship_iri(1, "https://example.org/b", "https://example.org/a"), + } + + graph = _graph(json.dumps(jsonld)) + assert len({str(s) for s in graph.subjects()} & ids) == 2 + + +def test_no_export_path_writes_an_iri_in_the_semantica_scheme(tmp_path): + """Nothing may expand to the scheme ``semantica`` rather than the namespace.""" + documents = [ + RDFExporter().export_to_rdf(KG, format="jsonld"), + json.dumps(JSONExporter()._convert_kg_to_jsonld(KG)), + ] + exporter = JSONExporter() + exporter.export_entities(KG["entities"], tmp_path / "entities.json") + exporter.export_relationships(KG["relationships"], tmp_path / "relationships.json") + documents.append((tmp_path / "entities.json").read_text()) + documents.append((tmp_path / "relationships.json").read_text()) + + for document in documents: + for term in _graph(document).all_nodes(): + assert not str(term).startswith("semantica:"), document + for predicate in _graph(document).predicates(): + assert not str(predicate).startswith("semantica:"), document + + +def test_entity_and_relationship_lists_expand_into_the_declared_namespace(tmp_path): + exporter = JSONExporter() + exporter.export_entities(KG["entities"], tmp_path / "entities.json") + exporter.export_relationships(KG["relationships"], tmp_path / "relationships.json") + + predicates = set() + for name in ("entities.json", "relationships.json"): + predicates |= {str(p) for p in _graph((tmp_path / name).read_text()).predicates()} + + assert f"{SEMANTICA_NS}entities" in predicates + assert f"{SEMANTICA_NS}relationships" in predicates From 83c04a57d6ad64081e953284cfb6d0240b8149c6 Mon Sep 17 00:00:00 2001 From: FABIOTESS Date: Wed, 19 Aug 2026 15:42:36 +0100 Subject: [PATCH 08/13] fix(export,provenance): write timestamps with an explicit UTC offset (#1114) semantica/export/ stamped every value with datetime.now().isoformat(), which reads the machine's local clock. semantica/provenance/ stamped its own with datetime.utcnow().isoformat(), which reads UTC. Both return a naive datetime and both serialize identically, so once the value is out of the process nothing distinguishes them: the same string means two different instants depending on which module wrote it. In RDF the consequence is silent rather than loud. Under XSD 1.1 a value with no timezone compared against one with a timezone is indeterminate whenever the two fall inside the 14-hour window, SPARQL turns an indeterminate comparison into an error, and FILTER discards errors as non-matches. Loading a Semantica-stamped export into Oxigraph next to two correctly stamped ones and asking which were written before a given instant returns the other two and drops ours, with no error anywhere. prov:generatedAtTime, prov:startedAtTime, prov:endedAtTime and prov:atTime all carry values written this way, so an audit trail cannot be ordered against timestamps from any other system. Adds utc_now()/utc_now_iso() to semantica/utils/helpers.py, exported from semantica.utils, and uses them at all 29 call sites in export/ (json_exporter, yaml_exporter, report_generator, export_provenance) and provenance/ (manager, schemas, bridge_axiom). Values now read 2026-08-19T14:19:04.229937+00:00: one unambiguous instant, comparable against any correctly stamped value, and valid xsd:dateTimeStamp. sem:exportedAt's range in the vocabulary that landed with #1109 is tightened from xsd:dateTime to xsd:dateTimeStamp accordingly. Its comment had to explain why the weaker range was necessary; that reason is gone. datetime.utcnow() is also deprecated as of Python 3.12 and scheduled for removal. Constructing a ProvenanceEntry under -W error::DeprecationWarning on 3.13 raised; it no longer does. Two new test modules cover offset presence on every export and provenance path, PROV-O literals valid as xsd:dateTimeStamp, comparison against a timezone-aware instant without TypeError, the Oxigraph filter that dropped the naive value, the declared range matching what the exporter writes, and the document @id remaining a valid IRI with +00:00 in it. The filter test picks a bound inside the indeterminate window on purpose: a bound years away is determinate even for a naive value, and the test would pass without the fix. 13 of the 14 fail with this commit's semantica/export, semantica/provenance and vocabulary reverted. The remaining 147 naive call sites, in context/, vector_store/, seed/ and elsewhere, are deliberately untouched: those timestamps are compared against values parsed back from previously stored naive strings, so converting the write side alone would raise TypeError on existing data. That sweep needs a read-side migration and belongs in its own change. No new failures across the suite: 329 pre-existing failures before and after, all from optional dependencies missing in the local environment. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 8 + semantica/export/export_provenance.py | 7 +- semantica/export/json_exporter.py | 23 ++- semantica/export/report_generator.py | 9 +- semantica/export/yaml_exporter.py | 12 +- .../ontology/vocabulary/semantica-ns.ttl | 10 +- semantica/provenance/bridge_axiom.py | 5 +- semantica/provenance/manager.py | 16 +- semantica/provenance/schemas.py | 4 +- semantica/utils/__init__.py | 4 + semantica/utils/helpers.py | 29 ++++ tests/export/test_timestamp_timezones.py | 141 ++++++++++++++++++ tests/provenance/test_timestamp_timezones.py | 87 +++++++++++ 13 files changed, 314 insertions(+), 41 deletions(-) create mode 100644 tests/export/test_timestamp_timezones.py create mode 100644 tests/provenance/test_timestamp_timezones.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c81bc4a..9b90b3a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Every timestamp an export or a provenance record wrote was timezone-naive** (closes #1114) by @fabio-rovai + - `semantica/export/` stamped with `datetime.now().isoformat()`, which reads the machine's **local** clock; `semantica/provenance/` stamped with `datetime.utcnow().isoformat()`, which reads **UTC**. Both produce a naive value and both serialize identically, so nothing downstream can tell which zone a given timestamp belongs to — the same string means two different instants depending on which module wrote it + - In RDF the consequence is silent rather than loud. Under XSD 1.1 a value with no timezone compared against one with a timezone is indeterminate whenever the two fall inside the ±14 hour window; SPARQL turns an indeterminate comparison into an error, and `FILTER` discards errors as non-matches. A timezone-qualified query over an Oxigraph store returns an answer with every Semantica-written record quietly absent from it, which is a poor property for `prov:generatedAtTime`, `prov:startedAtTime`, `prov:endedAtTime` and `prov:atTime` to have + - New `utc_now()`/`utc_now_iso()` in `semantica/utils/helpers.py`, exported from `semantica.utils`, and used at all 29 call sites in `export/` (`json_exporter`, `yaml_exporter`, `report_generator`, `export_provenance`) and `provenance/` (`manager`, `schemas`, `bridge_axiom`). Values now read `2026-08-19T14:19:04.229937+00:00`: one unambiguous instant, comparable against any correctly stamped value, and valid `xsd:dateTimeStamp`. `sem:exportedAt`'s range in `semantica/ontology/vocabulary/semantica-ns.ttl` is tightened from `xsd:dateTime` accordingly, and its comment no longer has to explain why the weaker range was necessary + - `datetime.utcnow()` is deprecated as of Python 3.12 and scheduled for removal; constructing a `ProvenanceEntry` under `-W error::DeprecationWarning` on 3.13 raised, and no longer does + - New `tests/export/test_timestamp_timezones.py` and `tests/provenance/test_timestamp_timezones.py`: offset presence on every export and provenance path, PROV-O literals valid as `xsd:dateTimeStamp`, comparison against a timezone-aware instant without `TypeError`, the Oxigraph filter that dropped the naive value (with a bound inside the indeterminate window, so the test cannot pass by accident), and the document `@id` remaining a valid IRI with `+00:00` in it. 11 of the 13 fail on the parent commit + - The remaining 147 naive call sites are in `context/`, `vector_store/`, `seed/` and elsewhere, where timestamps are compared against values parsed from previously stored naive strings. Converting those without a read-side migration would raise `TypeError: can't compare offset-naive and offset-aware datetimes` on existing data, so they are deliberately left for a separate change + - **`split`/chunking paths bypassed the centralized spaCy model cache, reloading the model on every call** (#1042, closes #998) by @Accute9, reviewed by @Sameer6305 - `semantica/split/methods.py`'s `split_by_sentences()` and `semantica/split/semantic_chunker.py`'s `SemanticChunker.__init__` each called `spacy.load()` directly instead of reusing the process-level cache added in #889/`semantic_extract/methods.py`'s `load_spacy_model()` — every call/construction re-paid the ~120ms model-load cost independently of `NERExtractor`, which already used the cache - Both now route through `load_spacy_model()`, sharing one cached `Language` instance per model name across `split_by_sentences()`, `SemanticChunker`, and `NERExtractor`; a missing model still falls back to regex/paragraph chunking without poisoning the cache for a later successful load diff --git a/semantica/export/export_provenance.py b/semantica/export/export_provenance.py index 551b0f2c..07925d75 100644 --- a/semantica/export/export_provenance.py +++ b/semantica/export/export_provenance.py @@ -14,9 +14,10 @@ License: MIT """ from typing import Any, Optional -from datetime import datetime import uuid +from ..utils.helpers import utc_now_iso + class ExporterWithProvenance: """Base exporter with provenance tracking.""" @@ -45,9 +46,9 @@ class ExporterWithProvenance: def export(self, data: Any, destination: str, **kwargs): """Export data with provenance tracking.""" - activity_started_at_time = datetime.utcnow().isoformat() + activity_started_at_time = utc_now_iso() result = self._exporter.export(data, destination, **kwargs) - activity_ended_at_time = datetime.utcnow().isoformat() + activity_ended_at_time = utc_now_iso() if self.provenance and self._prov_manager: self._prov_manager.track_entity( diff --git a/semantica/export/json_exporter.py b/semantica/export/json_exporter.py index 2bca8cbf..6932dfeb 100644 --- a/semantica/export/json_exporter.py +++ b/semantica/export/json_exporter.py @@ -24,12 +24,11 @@ License: MIT """ import json -from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError, ValidationError -from ..utils.helpers import ensure_directory, write_json_file +from ..utils.helpers import ensure_directory, utc_now_iso, write_json_file from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -269,7 +268,7 @@ class JSONExporter: }, "entities": entities, "metadata": { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "entity_count": len(entities), **options.get("metadata", {}), }, @@ -301,7 +300,7 @@ class JSONExporter: }, "relationships": relationships, "metadata": { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "relationship_count": len(relationships), **options.get("metadata", {}), }, @@ -339,7 +338,7 @@ class JSONExporter: if include_metadata: if "metadata" not in result: result["metadata"] = {} - result["metadata"]["exported_at"] = datetime.now().isoformat() + result["metadata"]["exported_at"] = utc_now_iso() if include_provenance: result["metadata"]["format"] = "json" @@ -349,7 +348,7 @@ class JSONExporter: "data": data, "count": len(data), "metadata": { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "format": "json" if include_provenance else None, **options.get("metadata", {}), }, @@ -358,7 +357,7 @@ class JSONExporter: # Single value return { "value": data, - "metadata": {"exported_at": datetime.now().isoformat()} + "metadata": {"exported_at": utc_now_iso()} if include_metadata else {}, } @@ -410,9 +409,9 @@ class JSONExporter: # Add metadata and provenance if requested if include_metadata: - jsonld["@id"] = f"https://semantica.dev/data/{datetime.now().isoformat()}" + jsonld["@id"] = f"https://semantica.dev/data/{utc_now_iso()}" if include_provenance: - jsonld["semantica:exportedAt"] = datetime.now().isoformat() + jsonld["semantica:exportedAt"] = utc_now_iso() jsonld["semantica:format"] = "json-ld" return jsonld @@ -444,7 +443,7 @@ class JSONExporter: "nodes": kg.get("nodes", []), "edges": kg.get("edges", []), "metadata": { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), **kg.get("metadata", {}), **options.get("metadata", {}), }, @@ -481,7 +480,7 @@ class JSONExporter: "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdfs": "http://www.w3.org/2000/01/rdf-schema#", }, - "@id": f"https://semantica.dev/graph/{datetime.now().isoformat()}", + "@id": f"https://semantica.dev/graph/{utc_now_iso()}", "@type": "semantica:KnowledgeGraph", } @@ -502,7 +501,7 @@ class JSONExporter: ) # Add metadata - jsonld["semantica:exportedAt"] = datetime.now().isoformat() + jsonld["semantica:exportedAt"] = utc_now_iso() if "metadata" in kg: jsonld["semantica:metadata"] = kg["metadata"] diff --git a/semantica/export/report_generator.py b/semantica/export/report_generator.py index fcee11a1..be9f92be 100644 --- a/semantica/export/report_generator.py +++ b/semantica/export/report_generator.py @@ -25,12 +25,11 @@ License: MIT import html import json -from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError, ValidationError -from ..utils.helpers import ensure_directory +from ..utils.helpers import ensure_directory, utc_now_iso from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -252,7 +251,7 @@ class ReportGenerator: # Build report data with summary report_data = { "title": "Quality Assurance Report", - "generated_at": datetime.now().isoformat(), + "generated_at": utc_now_iso(), "metrics": quality_metrics, "summary": self._generate_quality_summary(quality_metrics), } @@ -278,7 +277,7 @@ class ReportGenerator: """ report_data = { "title": "Analysis Report", - "generated_at": datetime.now().isoformat(), + "generated_at": utc_now_iso(), "analysis": analysis_results, "summary": self._generate_analysis_summary(analysis_results), } @@ -304,7 +303,7 @@ class ReportGenerator: """ report_data = { "title": "Framework Metrics Report", - "generated_at": datetime.now().isoformat(), + "generated_at": utc_now_iso(), "metrics": metrics, "summary": self._generate_metrics_summary(metrics), } diff --git a/semantica/export/yaml_exporter.py b/semantica/export/yaml_exporter.py index ecbd55fa..a9befedb 100644 --- a/semantica/export/yaml_exporter.py +++ b/semantica/export/yaml_exporter.py @@ -22,7 +22,6 @@ License: MIT """ from collections.abc import Mapping -from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Union @@ -33,6 +32,7 @@ from ..utils.helpers import ( _require_recognized_keys, ensure_directory, normalize_graph_payload, + utc_now_iso, ) from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -216,7 +216,7 @@ class SemanticNetworkYAMLExporter: records = normalize_graph_payload(semantic_network) yaml_data = { "metadata": { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "version": "1.0", **semantic_network.get("metadata", {}), }, @@ -309,7 +309,7 @@ class SemanticNetworkYAMLExporter: if include_metadata: yaml_data["metadata"] = { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "entity_count": len(entities), } @@ -333,7 +333,7 @@ class SemanticNetworkYAMLExporter: if include_properties: yaml_data["metadata"] = { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "relationship_count": len(relationships), } @@ -370,7 +370,7 @@ class SemanticNetworkYAMLExporter: } yaml_data["metadata"] = { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "triplet_count": len(triplets), } @@ -410,7 +410,7 @@ class SemanticNetworkYAMLExporter: yaml_data = { "pipeline_stage": pipeline_stage, "metadata": { - "extracted_at": datetime.now().isoformat(), + "extracted_at": utc_now_iso(), **extracted_data.get("metadata", {}), }, "semantic_network": semantic_network, diff --git a/semantica/ontology/vocabulary/semantica-ns.ttl b/semantica/ontology/vocabulary/semantica-ns.ttl index c6843a32..36e206bc 100644 --- a/semantica/ontology/vocabulary/semantica-ns.ttl +++ b/semantica/ontology/vocabulary/semantica-ns.ttl @@ -118,10 +118,12 @@ sem:relationships a owl:ObjectProperty ; sem:exportedAt a owl:DatatypeProperty ; rdfs:label "exported at" ; - rdfs:comment """When the export was written. Emitted as an ISO 8601 local -timestamp, so the range is xsd:dateTime rather than xsd:dateTimeStamp: the -values carry no timezone offset.""" ; - rdfs:range xsd:dateTime ; + rdfs:comment """When the export was written, as an ISO 8601 timestamp with +an explicit UTC offset. The range was xsd:dateTime while the exporters stamped +with a naive datetime.now(); with the offset present (#1114) the value is a +determinate instant, comparable against a timestamp written anywhere else, so +the range is the stricter xsd:dateTimeStamp, which requires the offset.""" ; + rdfs:range xsd:dateTimeStamp ; rdfs:isDefinedBy . sem:format a owl:DatatypeProperty ; diff --git a/semantica/provenance/bridge_axiom.py b/semantica/provenance/bridge_axiom.py index e04f054e..e5e96696 100644 --- a/semantica/provenance/bridge_axiom.py +++ b/semantica/provenance/bridge_axiom.py @@ -61,9 +61,10 @@ License: MIT from dataclasses import dataclass, field from typing import Optional, Dict, Any, List -from datetime import datetime import uuid +from ..utils.helpers import utc_now_iso + @dataclass class BridgeAxiom: @@ -280,7 +281,7 @@ class TranslationChain: "type": layer_type, "value": value, "source": source, - "timestamp": datetime.utcnow().isoformat(), + "timestamp": utc_now_iso(), **kwargs } self.layers.append(layer) diff --git a/semantica/provenance/manager.py b/semantica/provenance/manager.py index 8c2d1738..1a0efa3d 100644 --- a/semantica/provenance/manager.py +++ b/semantica/provenance/manager.py @@ -26,7 +26,6 @@ License: MIT from typing import Optional, List, Dict, Any, Union from collections.abc import Mapping -from datetime import datetime from contextlib import contextmanager import copy import inspect @@ -36,6 +35,7 @@ import threading from .schemas import ProvenanceEntry, SourceReference, AgentRecord, ActivityRecord from .storage import ProvenanceStorage, InMemoryStorage, SQLiteStorage from .integrity import compute_checksum, verify_checksum +from ..utils.helpers import utc_now_iso from ..utils.logging import get_logger # Issue #825, Part B Tier 3 — configurable base URI for export_prov(), shared @@ -363,8 +363,8 @@ class ProvenanceManager: source_quote=kwargs.get("source_quote"), confidence=kwargs.get("confidence", 1.0), metadata=metadata or {}, - first_seen=existing.first_seen if existing else datetime.utcnow().isoformat(), - last_updated=datetime.utcnow().isoformat(), + first_seen=existing.first_seen if existing else utc_now_iso(), + last_updated=utc_now_iso(), parent_entity_id=parent_id, used_entities=list(kwargs.get("used_entities", [])), activity_started_at_time=activity_info["activity_started_at_time"], @@ -455,8 +455,8 @@ class ProvenanceManager: source_location=kwargs.get("source_location"), confidence=kwargs.get("confidence", 1.0), metadata=metadata or {}, - first_seen=datetime.utcnow().isoformat(), - last_updated=datetime.utcnow().isoformat(), + first_seen=utc_now_iso(), + last_updated=utc_now_iso(), activity_started_at_time=activity_info["activity_started_at_time"], activity_ended_at_time=activity_info["activity_ended_at_time"], acted_on_behalf_of=kwargs.get("acted_on_behalf_of"), @@ -534,7 +534,7 @@ class ProvenanceManager: # split (issue #825, Part A item 4). derived_from_id=parent_chunk_id, metadata=metadata, - timestamp=datetime.utcnow().isoformat(), + timestamp=utc_now_iso(), activity_started_at_time=activity_info["activity_started_at_time"], activity_ended_at_time=activity_info["activity_ended_at_time"], ) @@ -604,7 +604,7 @@ class ProvenanceManager: **metadata, **source.metadata }, - timestamp=datetime.utcnow().isoformat(), + timestamp=utc_now_iso(), activity_started_at_time=activity_info["activity_started_at_time"], activity_ended_at_time=activity_info["activity_ended_at_time"], ) @@ -1071,7 +1071,7 @@ class ProvenanceManager: entry = copy.deepcopy(existing) entry.invalidated = True - entry.invalidated_at_time = datetime.utcnow().isoformat() + entry.invalidated_at_time = utc_now_iso() entry.invalidated_by = agent_id entry.invalidation_reason = reason entry.previous_version_id = history_id diff --git a/semantica/provenance/schemas.py b/semantica/provenance/schemas.py index df888db4..1847b341 100644 --- a/semantica/provenance/schemas.py +++ b/semantica/provenance/schemas.py @@ -30,6 +30,8 @@ from dataclasses import dataclass, field from typing import Optional, List, Dict, Any from datetime import datetime +from ..utils.helpers import utc_now_iso + @dataclass class ProvenanceEntry: @@ -91,7 +93,7 @@ class ProvenanceEntry: source_quote: Optional[str] = None # Temporal tracking (from kg.ProvenanceTracker) - timestamp: str = field(default_factory=lambda: datetime.utcnow().isoformat()) + timestamp: str = field(default_factory=lambda: utc_now_iso()) first_seen: Optional[str] = None last_updated: Optional[str] = None diff --git a/semantica/utils/__init__.py b/semantica/utils/__init__.py index 002a139d..599e0746 100644 --- a/semantica/utils/__init__.py +++ b/semantica/utils/__init__.py @@ -82,6 +82,8 @@ from .helpers import ( normalize_entities, normalize_graph_payload, parse_timestamp, + utc_now, + utc_now_iso, read_json_file, retry_on_error, safe_filename, @@ -193,6 +195,8 @@ __all__ = [ "get_file_size", "format_timestamp", "parse_timestamp", + "utc_now", + "utc_now_iso", "merge_dicts", "chunk_list", "flatten_dict", diff --git a/semantica/utils/helpers.py b/semantica/utils/helpers.py index 75031fe8..47f5865c 100644 --- a/semantica/utils/helpers.py +++ b/semantica/utils/helpers.py @@ -320,6 +320,35 @@ def format_timestamp( return dt.strftime(format_str) +def utc_now() -> datetime: + """ + Current instant as a timezone-aware UTC datetime. + + ``datetime.now()`` reads the local clock and ``datetime.utcnow()`` reads UTC, + but both return a naive datetime, and the two are indistinguishable once + serialized: a consumer cannot tell which zone the value belongs to, and an + RDF timestamp without an offset is not comparable against one that has an + offset (a SPARQL FILTER drops it rather than reporting an error). Use this + for any timestamp that leaves the process. + + Returns: + Current UTC time, timezone-aware + """ + return datetime.now(timezone.utc) + + +def utc_now_iso() -> str: + """ + Current instant as an ISO 8601 string carrying an explicit UTC offset. + + Returns: + Timestamp string such as ``2026-08-19T14:19:04.229937+00:00``, which is + a valid ``xsd:dateTimeStamp`` and orders correctly against timestamps + written in any other timezone + """ + return utc_now().isoformat() + + def parse_timestamp(timestamp_str: str, format_str: Optional[str] = None) -> datetime: """ Parse timestamp string to datetime. diff --git a/tests/export/test_timestamp_timezones.py b/tests/export/test_timestamp_timezones.py new file mode 100644 index 00000000..df34dd29 --- /dev/null +++ b/tests/export/test_timestamp_timezones.py @@ -0,0 +1,141 @@ +"""Timestamps that leave the process must carry a timezone (issue #1114). + +Every timestamp an exporter wrote was naive: ``datetime.now().isoformat()`` +reads the local clock, ``datetime.utcnow().isoformat()`` reads UTC, and the two +serialize identically, so nothing downstream can tell which zone a value belongs +to. In RDF the consequence is not a parse error but a silent one: under XSD 1.1 +a value with no timezone compared against one with a timezone is indeterminate +whenever they fall inside the +/-14 hour window, SPARQL turns that into an error, +and FILTER discards errors as non-matches. A timezone-qualified query therefore +returns an answer with every Semantica-written record quietly missing from it. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from semantica.export.json_exporter import JSONExporter +from semantica.export.report_generator import ReportGenerator +from semantica.export.yaml_exporter import SemanticNetworkYAMLExporter +from semantica.utils.helpers import utc_now, utc_now_iso + +KG = { + "entities": [{"id": "https://example.org/e1", "text": "Bob"}], + "relationships": [], +} + + +def assert_offset_aware(value): + """An ISO 8601 string is only an instant if it says which zone it is in.""" + assert isinstance(value, str), value + parsed = datetime.fromisoformat(value) + assert parsed.tzinfo is not None, f"timezone-naive timestamp: {value!r}" + assert parsed.utcoffset() is not None + + +def test_utc_now_iso_is_offset_aware(): + assert_offset_aware(utc_now_iso()) + assert utc_now().tzinfo is not None + + +def test_jsonld_export_timestamp_is_offset_aware(): + document = JSONExporter()._convert_kg_to_jsonld(KG) + assert_offset_aware(document["semantica:exportedAt"]) + + +def test_json_export_metadata_timestamp_is_offset_aware(tmp_path): + import json + + exporter = JSONExporter() + exporter.export_entities(KG["entities"], tmp_path / "entities.json") + exporter.export_relationships([], tmp_path / "relationships.json") + + for name in ("entities.json", "relationships.json"): + payload = json.loads((tmp_path / name).read_text()) + assert_offset_aware(payload["metadata"]["exported_at"]) + + +def test_yaml_export_timestamp_is_offset_aware(): + yaml = pytest.importorskip("yaml") + + document = SemanticNetworkYAMLExporter().export_entities(KG["entities"]) + payload = yaml.safe_load(document) + assert_offset_aware(payload["metadata"]["exported_at"]) + + +def test_report_timestamp_is_offset_aware(): + import json + + report = json.loads( + ReportGenerator().generate_quality_report({"score": 0.9}, format="json") + ) + assert_offset_aware(report["generated_at"]) + + +def test_exported_timestamp_compares_against_a_timezone_aware_instant(): + """The naive form raised TypeError here, or compared as if it were UTC.""" + exported = datetime.fromisoformat( + JSONExporter()._convert_kg_to_jsonld(KG)["semantica:exportedAt"] + ) + assert exported <= utc_now() + assert exported > datetime(2020, 1, 1, tzinfo=timezone.utc) + + +def test_exported_timestamp_survives_a_timezone_qualified_sparql_filter(): + """The regression in #1114: a strict engine dropped the naive value.""" + pyoxigraph = pytest.importorskip("pyoxigraph") + + exported = JSONExporter()._convert_kg_to_jsonld(KG)["semantica:exportedAt"] + store = pyoxigraph.Store() + store.load( + ( + ' ' + ' ' + f'"{exported}"^^ .' + ).encode(), + format=pyoxigraph.RdfFormat.N_TRIPLES, + ) + # The bound has to sit inside the +/-14 hour window that makes an + # untimezoned comparison indeterminate. A bound years away is determinate + # even for a naive value, and the test would pass without the fix. + bound = (utc_now() + timedelta(hours=1)).isoformat().replace("+00:00", "Z") + rows = list(store.query( + "PREFIX xsd: " + "SELECT ?e WHERE { ?e ?t . " + f'FILTER (?t < "{bound}"^^xsd:dateTime) }}' + )) + assert len(rows) == 1, "the export was dropped by a timezone-qualified filter" + + +def test_document_iri_carrying_an_offset_is_a_valid_iri(): + """The offset puts '+' and ':' in the @id; both are legal in a path.""" + rdflib = pytest.importorskip("rdflib") + + document_iri = JSONExporter()._convert_kg_to_jsonld(KG)["@id"] + assert "+00:00" in document_iri + assert rdflib.term._is_valid_uri(document_iri) + + graph = rdflib.Graph() + graph.add(( + rdflib.URIRef(document_iri), + rdflib.RDF.type, + rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"), + )) + reparsed = rdflib.Graph().parse(data=graph.serialize(format="nt"), format="nt") + assert document_iri in {str(s) for s in reparsed.subjects()} + + +def test_vocabulary_range_matches_what_the_exporter_writes(): + """The declared range says the offset is required; the export must carry it.""" + rdflib = pytest.importorskip("rdflib") + from rdflib.namespace import RDFS, XSD + + from semantica.ontology.vocabulary import NAMESPACE, vocabulary_turtle + + graph = rdflib.Graph() + graph.parse(data=vocabulary_turtle(), format="turtle") + declared = graph.value(rdflib.URIRef(f"{NAMESPACE}exportedAt"), RDFS.range) + assert declared == XSD.dateTimeStamp + + exported = JSONExporter()._convert_kg_to_jsonld(KG)["semantica:exportedAt"] + assert datetime.fromisoformat(exported).utcoffset() is not None diff --git a/tests/provenance/test_timestamp_timezones.py b/tests/provenance/test_timestamp_timezones.py new file mode 100644 index 00000000..f665b859 --- /dev/null +++ b/tests/provenance/test_timestamp_timezones.py @@ -0,0 +1,87 @@ +"""Provenance timestamps must carry a timezone (issue #1114). + +The provenance package stamped every record with ``datetime.utcnow()``, which +returns a naive datetime that happens to hold UTC. The exporters stamped theirs +with ``datetime.now()``, which returns a naive datetime holding local time. Both +serialize identically, so a graph mixing the two cannot be ordered, and the +values reach RDF as ``prov:generatedAtTime``/``startedAtTime``/``endedAtTime`` +typed ``xsd:dateTime``, where a timezone-qualified SPARQL comparison discards +them. ``datetime.utcnow()`` is also deprecated as of Python 3.12. +""" + +import warnings +from datetime import datetime + +import pytest + +from semantica.provenance.manager import ProvenanceManager +from semantica.provenance.schemas import ProvenanceEntry +from semantica.utils.helpers import utc_now + + +def assert_offset_aware(value): + parsed = datetime.fromisoformat(value) + assert parsed.tzinfo is not None, f"timezone-naive timestamp: {value!r}" + + +def test_provenance_entry_default_timestamp_is_offset_aware(): + entry = ProvenanceEntry(entity_id="e1", entity_type="Doc", activity_id="act1") + assert_offset_aware(entry.timestamp) + assert datetime.fromisoformat(entry.timestamp) <= utc_now() + + +def test_creating_an_entry_raises_no_deprecation_warning(): + """datetime.utcnow() is deprecated and scheduled for removal.""" + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + ProvenanceEntry(entity_id="e1", entity_type="Doc", activity_id="act1") + + +def test_tracked_entity_timestamps_are_offset_aware(): + manager = ProvenanceManager() + manager.track_entity("e1", source="doc.pdf") + + entry = manager.storage.retrieve_all()[0] + assert_offset_aware(entry.timestamp) + for field in ("first_seen", "last_updated"): + value = getattr(entry, field, None) + if value: + assert_offset_aware(value) + + +def test_prov_o_export_timestamps_are_offset_aware(): + """The values land in RDF typed xsd:dateTime, so the offset is the contract.""" + rdflib = pytest.importorskip("rdflib") + from rdflib.namespace import XSD + + manager = ProvenanceManager() + manager.track_entity("e_parent", source="doc.pdf") + manager.track_entity( + "e_child", source="doc.pdf", parent_entity_id="e_parent", + used_entities=["e_parent"], activity_id="act_transform", + ) + + graph = rdflib.Graph() + graph.parse(data=manager.export_prov(format="turtle"), format="turtle") + + stamps = [o for o in graph.objects() + if isinstance(o, rdflib.Literal) and o.datatype == XSD.dateTime] + assert stamps, "no xsd:dateTime literals in the PROV-O export" + for stamp in stamps: + assert_offset_aware(str(stamp)) + + +def test_prov_o_timestamps_are_valid_datetimestamp(): + """xsd:dateTimeStamp requires an explicit timezone; these now qualify.""" + rdflib = pytest.importorskip("rdflib") + from rdflib.namespace import XSD + + manager = ProvenanceManager() + manager.track_entity("e1", source="doc.pdf") + graph = rdflib.Graph() + graph.parse(data=manager.export_prov(format="turtle"), format="turtle") + + for stamp in [o for o in graph.objects() + if isinstance(o, rdflib.Literal) and o.datatype == XSD.dateTime]: + assert rdflib.Literal(str(stamp), datatype=XSD.dateTime).ill_typed is False + assert datetime.fromisoformat(str(stamp)).utcoffset() is not None From e03212cd6661e905dab851c23bcbd6435801828e Mon Sep 17 00:00:00 2001 From: FABIOTESS Date: Wed, 19 Aug 2026 15:58:24 +0100 Subject: [PATCH 09/13] fix(provenance): compare timestamp ranges by instant, not by spelling Review finding on #1121, and correct: with new entries carrying +00:00 and entries written earlier carrying nothing, query_recorded_between() and audit_log() compared ISO strings directly, which orders by how a timestamp is spelled rather than when it happened. Two consequences, both introduced by the offset this PR adds: - An inclusive naive bound naming a stored offset-bearing timestamp sorts below it, because the stored value is the longer string, so the record it names is excluded from its own range. - A bound in another offset lands wherever its digits fall. "2026-08-19T19:45:00+05:30" is 14:15Z, before an entry at 14:19Z, but string comparison puts it after. Both paths now compare instants, through a new to_utc_datetime() helper that reads a missing offset as UTC. That is what the naive values actually were: provenance stamped with datetime.utcnow(), so reading them as UTC keeps a stored naive value and the same instant written with an offset comparing equal instead of ordering by representation. It is also the read side the remaining 147 call sites will need whenever the rest of the package is converted. A bound that cannot be read as a timestamp keeps the historical string comparison rather than raising on a call that used to work. Five new tests cover the inclusive naive bound, the other-offset bound, legacy and offset-bearing entries ordered together, audit_log's since filter, and the unreadable-bound fallback. The first two fail with manager.py reverted; the rest are guards. 569 provenance, export and ontology tests pass, and the full-suite failure set is unchanged at 329, all from optional dependencies missing locally. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 1 + semantica/provenance/manager.py | 51 ++++++++++++--- semantica/utils/__init__.py | 2 + semantica/utils/helpers.py | 30 +++++++++ tests/provenance/test_timestamp_timezones.py | 66 ++++++++++++++++++++ 5 files changed, 142 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b90b3a3..9db73a09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New `utc_now()`/`utc_now_iso()` in `semantica/utils/helpers.py`, exported from `semantica.utils`, and used at all 29 call sites in `export/` (`json_exporter`, `yaml_exporter`, `report_generator`, `export_provenance`) and `provenance/` (`manager`, `schemas`, `bridge_axiom`). Values now read `2026-08-19T14:19:04.229937+00:00`: one unambiguous instant, comparable against any correctly stamped value, and valid `xsd:dateTimeStamp`. `sem:exportedAt`'s range in `semantica/ontology/vocabulary/semantica-ns.ttl` is tightened from `xsd:dateTime` accordingly, and its comment no longer has to explain why the weaker range was necessary - `datetime.utcnow()` is deprecated as of Python 3.12 and scheduled for removal; constructing a `ProvenanceEntry` under `-W error::DeprecationWarning` on 3.13 raised, and no longer does - New `tests/export/test_timestamp_timezones.py` and `tests/provenance/test_timestamp_timezones.py`: offset presence on every export and provenance path, PROV-O literals valid as `xsd:dateTimeStamp`, comparison against a timezone-aware instant without `TypeError`, the Oxigraph filter that dropped the naive value (with a bound inside the indeterminate window, so the test cannot pass by accident), and the document `@id` remaining a valid IRI with `+00:00` in it. 11 of the 13 fail on the parent commit + - **Fixed during review** (Qodo): once new entries carry `+00:00` and stored ones do not, `ProvenanceManager.query_recorded_between` and `audit_log` compared ISO timestamps as raw strings, so they ordered by spelling rather than by instant — an inclusive naive bound naming a stored offset-bearing timestamp sorted *below* it and dropped the record, and a bound written in another offset landed wherever its digits fell (`19:45+05:30` is 14:15Z, but sorted after 14:19Z). Both now compare instants through a new `to_utc_datetime()` helper that reads a missing offset as UTC, which is what the values written before this change actually were; a bound that cannot be read as a timestamp keeps the historical string comparison rather than raising on a call that used to work - The remaining 147 naive call sites are in `context/`, `vector_store/`, `seed/` and elsewhere, where timestamps are compared against values parsed from previously stored naive strings. Converting those without a read-side migration would raise `TypeError: can't compare offset-naive and offset-aware datetimes` on existing data, so they are deliberately left for a separate change - **`split`/chunking paths bypassed the centralized spaCy model cache, reloading the model on every call** (#1042, closes #998) by @Accute9, reviewed by @Sameer6305 diff --git a/semantica/provenance/manager.py b/semantica/provenance/manager.py index 1a0efa3d..1a2b26f1 100644 --- a/semantica/provenance/manager.py +++ b/semantica/provenance/manager.py @@ -26,6 +26,7 @@ License: MIT from typing import Optional, List, Dict, Any, Union from collections.abc import Mapping +from datetime import datetime, timezone from contextlib import contextmanager import copy import inspect @@ -35,9 +36,13 @@ import threading from .schemas import ProvenanceEntry, SourceReference, AgentRecord, ActivityRecord from .storage import ProvenanceStorage, InMemoryStorage, SQLiteStorage from .integrity import compute_checksum, verify_checksum -from ..utils.helpers import utc_now_iso +from ..utils.helpers import to_utc_datetime, utc_now_iso from ..utils.logging import get_logger +#: Sort key for an entry whose timestamp cannot be read as one, so an +#: unreadable value orders first instead of raising during a sort. +_EPOCH = datetime(1, 1, 1, tzinfo=timezone.utc) + # Issue #825, Part B Tier 3 — configurable base URI for export_prov(), shared # with RDFExporter's NamespaceManager "semantica" entry (semantica/export/ # rdf_exporter.py) so KG-exported and PROV-exported URIs for the same @@ -959,11 +964,27 @@ class ProvenanceManager: Returns: List of matching entries as dicts, sorted by timestamp ascending. """ - matches = [ - e for e in self.storage.retrieve_all() - if e.timestamp and start <= e.timestamp <= end - ] - matches.sort(key=lambda e: e.timestamp) + # Compare instants, not spellings. Since #1114 new entries carry a + # +00:00 offset while entries written earlier do not, and a raw string + # comparison orders those two by length: an inclusive naive bound equal + # to a stored offset-bearing timestamp would sort below it and drop the + # record. A bound in another offset was mis-ordered the same way. + start_at = to_utc_datetime(start) + end_at = to_utc_datetime(end) + entries = [e for e in self.storage.retrieve_all() if e.timestamp] + + if start_at is None or end_at is None: + # A bound this module cannot read as a timestamp keeps the historical + # string comparison rather than raising on a call that used to work. + matches = [e for e in entries if start <= e.timestamp <= end] + else: + matches = [ + e for e in entries + if (at := to_utc_datetime(e.timestamp)) is not None + and start_at <= at <= end_at + ] + + matches.sort(key=lambda e: (to_utc_datetime(e.timestamp) or _EPOCH, e.timestamp)) return [e.to_dict() for e in matches] def get_all_sources(self, entity_id: str) -> List[Dict[str, Any]]: @@ -1163,8 +1184,22 @@ class ProvenanceManager: """ entries = self.storage.retrieve_all() if since: - entries = [e for e in entries if getattr(e, "timestamp", "") >= since] - entries.sort(key=lambda e: getattr(e, "timestamp", "")) + since_at = to_utc_datetime(since) + if since_at is None: + entries = [e for e in entries + if getattr(e, "timestamp", "") >= since] + else: + entries = [ + e for e in entries + if (at := to_utc_datetime(getattr(e, "timestamp", None))) + is not None and at >= since_at + ] + entries.sort( + key=lambda e: ( + to_utc_datetime(getattr(e, "timestamp", None)) or _EPOCH, + getattr(e, "timestamp", ""), + ) + ) if format == "json": return [ diff --git a/semantica/utils/__init__.py b/semantica/utils/__init__.py index 599e0746..a6d7aa9e 100644 --- a/semantica/utils/__init__.py +++ b/semantica/utils/__init__.py @@ -82,6 +82,7 @@ from .helpers import ( normalize_entities, normalize_graph_payload, parse_timestamp, + to_utc_datetime, utc_now, utc_now_iso, read_json_file, @@ -195,6 +196,7 @@ __all__ = [ "get_file_size", "format_timestamp", "parse_timestamp", + "to_utc_datetime", "utc_now", "utc_now_iso", "merge_dicts", diff --git a/semantica/utils/helpers.py b/semantica/utils/helpers.py index 47f5865c..05ece01f 100644 --- a/semantica/utils/helpers.py +++ b/semantica/utils/helpers.py @@ -349,6 +349,36 @@ def utc_now_iso() -> str: return utc_now().isoformat() +def to_utc_datetime(value: Union[str, datetime, None]) -> Optional[datetime]: + """ + Read an ISO 8601 timestamp as a timezone-aware UTC instant. + + Timestamps written before #1114 carry no offset. They were produced by + ``datetime.utcnow()``, so a missing offset is read as UTC: that keeps a + stored naive value and the same instant written with an offset comparing + equal, instead of ordering by how the timestamp happens to be spelled. + + Args: + value: ISO 8601 string or datetime. ``Z`` is accepted as the offset. + + Returns: + Timezone-aware UTC datetime, or None if the value cannot be read as a + timestamp, so callers can fall back rather than raise on stored data + """ + if value is None: + return None + if isinstance(value, datetime): + parsed = value + else: + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + def parse_timestamp(timestamp_str: str, format_str: Optional[str] = None) -> datetime: """ Parse timestamp string to datetime. diff --git a/tests/provenance/test_timestamp_timezones.py b/tests/provenance/test_timestamp_timezones.py index f665b859..b2b91959 100644 --- a/tests/provenance/test_timestamp_timezones.py +++ b/tests/provenance/test_timestamp_timezones.py @@ -85,3 +85,69 @@ def test_prov_o_timestamps_are_valid_datetimestamp(): if isinstance(o, rdflib.Literal) and o.datatype == XSD.dateTime]: assert rdflib.Literal(str(stamp), datatype=XSD.dateTime).ill_typed is False assert datetime.fromisoformat(str(stamp)).utcoffset() is not None + + +class TestRangeQueriesCompareInstants: + """Range APIs compared ISO strings, so they ordered by spelling (#1121 review). + + Once new entries carry ``+00:00`` and stored ones do not, a raw string + comparison puts an inclusive naive bound *below* the offset-bearing + timestamp it names, dropping the record, and a bound written in another + offset lands wherever its digits fall rather than at its instant. + """ + + @staticmethod + def _manager_with(timestamps): + manager = ProvenanceManager() + for index, stamp in enumerate(timestamps): + manager.storage.store(ProvenanceEntry( + entity_id=f"e{index}", entity_type="Doc", + activity_id="act", timestamp=stamp, + )) + return manager + + def test_inclusive_bound_written_without_an_offset_still_matches(self): + manager = self._manager_with(["2026-08-19T14:19:04.229937+00:00"]) + + found = manager.query_recorded_between( + "2026-08-19T00:00:00", "2026-08-19T14:19:04.229937" + ) + assert [e["entity_id"] for e in found] == ["e0"] + + def test_bound_in_another_offset_selects_by_instant(self): + """19:45+05:30 is 14:15Z: before the entry, though its digits are after.""" + manager = self._manager_with(["2026-08-19T14:19:04+00:00"]) + + assert manager.query_recorded_between( + "2026-08-19T00:00:00Z", "2026-08-19T19:45:00+05:30" + ) == [] + assert len(manager.query_recorded_between( + "2026-08-19T00:00:00Z", "2026-08-19T19:50:00+05:30" + )) == 1 + + def test_legacy_and_offset_bearing_entries_are_both_found_and_ordered(self): + manager = self._manager_with([ + "2026-08-19T14:19:05+00:00", # written after #1114 + "2026-08-19T14:19:04", # written before it, meaning UTC + ]) + + found = manager.query_recorded_between( + "2026-08-19T14:00:00Z", "2026-08-19T15:00:00Z" + ) + assert [e["entity_id"] for e in found] == ["e1", "e0"] + + def test_audit_log_since_reads_a_naive_bound_as_utc(self): + manager = self._manager_with([ + "2026-08-19T14:19:05+00:00", + "2026-08-19T09:00:00", + ]) + + recent = manager.audit_log(since="2026-08-19T14:19:05", format="json") + assert [e["entity_id"] for e in recent] == ["e0"] + + def test_an_unreadable_bound_falls_back_to_the_previous_behaviour(self): + """A call that used to work with a non-timestamp bound must not raise.""" + manager = self._manager_with(["2026-08-19T14:19:04+00:00"]) + + assert manager.query_recorded_between("not-a-date", "also-not") == [] + assert manager.audit_log(since="not-a-date", format="json") == [] From 78fc9028a872072dcb2c65bf65ba1638a7a02341 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Thu, 20 Aug 2026 13:34:04 +0530 Subject: [PATCH 10/13] chore(release): prepare v0.6.6 Bump version, cut CHANGELOG's Unreleased section into 0.6.6, backfill changelog entries for merged PRs missing from it, and refresh version-dependent references in README/docs. --- CHANGELOG.md | 85 ++++++++++++++++++++++++++++++++++++++++- README.md | 20 +++++----- docs/citation.md | 10 ++--- docs/faq.md | 2 +- docs/getting-started.md | 2 +- pyproject.toml | 4 +- semantica/__init__.py | 2 +- 7 files changed, 103 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ef8e1bb..9ac9df42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.6.6] - 2026-08-20 + ### Added - **Semantica RDF vocabulary, and deterministic entity/relationship IRIs** (#1109, closes #1107, closes #1101) by @fabio-rovai, reviewed by @KaifAhmad1 @@ -22,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `tests/export/test_jsonld_iri_minting.py` parses each export with a real JSON-LD processor and asserts the entity survives, the relationships stay distinct, no term expands into the `semantica` scheme, and the JSON-LD `@id` equals the Turtle IRI - 236 export and ontology tests pass -- **First-class CrewAI integration** (#962) +- **First-class CrewAI integration** (#988, closes #962) by @Shindevrp - New `pip install semantica[crewai]` extra (`crewai>=0.80.0`) — crewai core provides `BaseTool`/`BaseKnowledgeSource`, so `crewai-tools` is intentionally not included, and the extra is intentionally **not** part of the `all` bundle: crewai hard-requires `chromadb~=1.1.0`, which is affected by the unpatched pre-auth code-injection CVE-2026-45829 (see `integrations/crewai/README.md`) - `integrations/crewai/SemanticaKGTool` — a CrewAI `BaseTool` exposing 5 KG actions (`extract_entities`, `extract_relations`, `add_to_graph`, `query_graph`, `find_related`) backed by `NERExtractor` / `RelationExtractor` / `ContextGraph`; supports both sync `run()` and async `arun()` - `integrations/crewai/SemanticaDecisionTool` — a CrewAI `BaseTool` wrapping `AgentContext` with 5 decision-intelligence actions (`record_decision`, `find_precedents`, `trace_causal_chain`, `analyze_impact`, `check_policy`) @@ -52,6 +54,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New `tests/export/test_distance_exporter_metric_errors.py`: 6 tests covering success, single/multiple failures, opt-out, the no-path-vs-error distinction, and default-schema stability; existing `tests/export/test_distance_exporter.py` updated for the new tuple return type - Full `tests/export/` suite: 77 passed +- **`ContextGraph.to_kg_dict()`: an adapter converting a `ContextGraph`'s internal `nodes`/`edges`/`source` shape into the canonical `entities`/`relationships`/`source_id` shape `RDFExporter` and `TemporalGraphQuery` consume** (#1081) by @cxzg007 + - Previously there was no supported way to feed a `ContextGraph` into those consumers without hand-rolling the field remapping; `to_kg_dict()` does it once, with an `entities_only` option that drops relationships left dangling by the filter + - **Fixed during review** (Qodo): null `properties`/`metadata` on a node loaded from JSON raised `TypeError` when copied — both are now guarded with `or {}`; entity ids are coerced to `str(node_id)` to match `ContextEdge`'s already-str-coerced endpoints, so valid relationships were no longer dropped by `entities_only` filtering + - `RDFExporter`'s validator and `TemporalGraphQuery` now also accept `source_id`/`target_id` endpoints, the shape `to_kg_dict()` emits + ### Changed - **`GraphBuilder`'s 6 public methods now have Google-style docstrings** (#878, closes #876) by @cakeni @@ -61,7 +68,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Corrected during review**: `add_temporal_edge`/`create_temporal_snapshot` docstrings overclaimed numeric-timestamp support; `_parse_time()` only special-cases `str` and `datetime`, falling back to a bare `str()` cast for anything else (not true numeric parsing). Narrowed to "datetime or ISO-formatted string" - **Fixed along the way**: `build()`'s `**options` documented a default only for `extract`; `extract_relations`, `extract_triplets`, `ner_method`, `relation_method`, and `triplet_method` all have concrete defaults in `_extract_from_text()` (`True`, `True`, `"llm"`, `"llm"`, `"llm"`) that were left unstated, inconsistent with CONTRIBUTING.md's own docstring example of noting defaults inline - `python -m pytest tests/kg/test_kg.py tests/kg/test_graph_builder_external.py -q`: 45 passed -- **`GraphBuilder` raw-text extraction now defaults to local extractors instead of LLM extraction** (closes #930) by @dex0shubham +- **`GraphBuilder` raw-text extraction now defaults to local extractors instead of LLM extraction** (#941, closes #930) by @dex0shubham - `GraphBuilder._extract_from_text()` defaulted `ner_method`, `relation_method`, and `triplet_method` to `"llm"`, and ran relation extraction unconditionally (`extract_relations` defaulted to `True`) — all four contradicting the defaults documented in the `build()` docstring at the time (`"ml"` / `"pattern"` / `False`), and diverging from the standalone extractors (`NERExtractor` defaults to `method="ml"`, `RelationExtractor` and `TripletExtractor` to `method="pattern"`). The practical effect was that any raw-text `build()` call silently required a configured provider, an API key, and network access - Defaults are now `ner_method="ml"`, `relation_method="pattern"`, `triplet_method="pattern"`, and `extract_relations=False`, matching the docstring. LLM extraction remains fully available and is now opt-in - **To restore the previous behaviour**, pass the methods explicitly: @@ -82,6 +89,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New regression coverage in `tests/kg/test_graph_builder_extraction_defaults.py` pinning all four defaults, verifying that no default resolves to `"llm"`, confirming explicit LLM opt-in still routes correctly, asserting extractors are constructed once across repeated texts, covering fallback method lists (e.g. `ner_method=["pattern", "ml"]`) for all three extractors, asserting relations are forwarded to triplet extraction (and that `None` is forwarded when relation extraction is disabled or fails), and running the real default path end to end with no provider mocked. Verified to fail against the pre-fix code - Full `kg` suite: 473 passed +- **Explorer graph canvas now renders edge labels** (#1013, closes #1009) by @yzxcj797 + - `GraphCanvas.tsx` had no edge-label rendering path at all; Sigma's edge-label renderer draws `data.label`, but the graph state stored the relationship type under `edgeType`, so simply enabling the renderer would have left every edge blank. `graphSceneState`'s edge reducer now maps `edgeType` onto `label` (suppressed for hidden edges) + - Rendering is gated behind a new `edgeLabelsEnabled` entry in the Effects panel (default on), wired through the existing `GraphEffectToggle`/`GraphEffectsState` plumbing, so dense graphs can still turn labels off + - **Fixed during review** (Qodo): two follow-up passes closed gaps the first cut left — label rendering wasn't wired through `explorationEffectsPluginPhaseC.tsx`'s Phase C variant, and toggling the effect off mid-session didn't clear already-rendered labels + - New coverage in `explorer/tests/graphSceneState.display.test.ts` + +- **Removed `GraphWorkspaceShell.tsx`, `GraphRuntimeStage.tsx`, and `useGraphData.ts` — a second, unused implementation of the graph-loading/error-handling logic already fixed in `GraphWorkspace.tsx`** (#984, resolves the cleanup tracked in #981 by #980's review note) by @lakshayxi + - 1,564 lines removed; the surviving `GraphWorkspace` path is now the only implementation, so the "two copies that drifted apart" root cause #980 fixed can't recur in the copy nobody was maintaining + +- **Explorer README and `docs/explorer-setup.md` corrected to describe the authentication 0.6.5 actually shipped**, plus a documented `/ws/graph-updates` auth note (#1040, fixes #1028) by @Kyou12138 + - Both docs still claimed the Explorer API had no built-in authentication after v0.6.5 added mandatory `SEMANTICA_API_KEY` enforcement with a `503` fail-closed default; corrected to describe the actual behavior, including that only protected routes require the key (`/api/health`/`/api/info` stay open), the non-loopback-bind CLI warning only fires in anonymous mode or when the key is unset, and `SEMANTICA_API_KEY`/`SEMANTICA_ALLOW_ANONYMOUS` are documented in the environment-variable table + +- **CI: pinned `github/codeql-action` to current v4** (#986) by @ZohaibHassan16, and **pinned Python dependencies in `requirements-ci.txt` for reproducible CI runs** (#945) by @yunaremaia, closing the gap where an unpinned CI dependency could silently change behavior between runs + +- **README now states up front that Semantica's explainability is system-level, not foundation-model-internal** (#1033, #1034) by @KaifAhmad1 + - Nothing in the README previously scoped what "explainable" meant, leaving readers to assume Semantica could expose or reconstruct an LLM's internal reasoning. A callout now states explicitly that Semantica explains and audits what the AI *system* did — context fed in, decisions produced, provenance, relationships, policies applied — not the model's private internal reasoning, and moved the note near the top of the README rather than leaving it implicit + ### Fixed - **Every timestamp an export or a provenance record wrote was timezone-naive** (closes #1114) by @fabio-rovai @@ -194,6 +218,59 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New regression coverage in `tests/export/test_distance_exporter.py`: warnings fire on exception for all four helpers, exported sentinel values/shape stay unchanged, and the legitimate "no KG backend" `None` path still logs nothing - Full `tests/export/` suite: 71 passed +- **`explain_violations` rendered hardcoded placeholders (`min_count=1`, `max_count=1`) instead of the SHACL shape's real constraint values, and misused the violation message text as the datatype/class value** (#1094) by @cxzg007 + - `_run_pyshacl` never read `sh:minCount`/`sh:maxCount`/`sh:datatype`/`sh:class` back from the violation's `sh:sourceShape`, so every plain-English explanation was wrong regardless of what the shape actually declared. `SHACLViolation` now carries those four fields (also exposed via `to_dict()`), populated by back-referencing `sh:sourceShape`; `explain_violations` renders the real values, falling back to `"?"` when a value is genuinely absent + - **Known limitation**: `sh:qualifiedMinCount`/`sh:qualifiedMaxCount` are not handled yet and still fall back to the `"?"` placeholder + - New regression tests cover both the rendering path and the `sh:sourceShape` back-reference (skipped when `pyshacl`/`rdflib` are absent) + +- **Entity merging silently dropped `entity_id` aliases, and exact-match entity resolution had three correctness gaps** (#1086, #1026) by @T1mn + - `entity_merger.py`/`merge_strategy.py`/`entity_resolver.py` used inconsistent logic for extracting an entity's id across the merge path, so a merged entity could lose the `entity_id` aliases that let later lookups find it under its old identity. A new `semantica/utils/entity_ids.py` unifies id extraction across all three call sites + - `EntityResolver`'s exact-match path is now honored rather than silently falling through to fuzzy matching in some cases; entities with no identifier are preserved instead of being dropped, and blank exact-match names are ignored rather than matching every other blank name + - New/expanded coverage in `tests/kg/test_entity_pipeline.py` and `tests/kg/test_entity_resolver_exact.py` + +- **`flatten_dict()` silently collided keys when a flattened path from one branch matched a literal key already present at the target depth** (#1062) by @shahzaib-ahmadcs + - Two differently-shaped inputs could flatten to the same output key, with the second write silently overwriting the first — no error, no warning, just a dropped value. Collisions are now detected and handled explicitly instead of overwriting + +- **`ExcelParser.__init__` raised `NameError` on every instantiation — `get_progress_tracker()` was called but never imported** (#1016, closes #1014) by @pravit-amp + - Same defect as the one fixed for `SimilarityCalculator` in #530, this time in `semantica/parse/excel_parser.py`; the existing test imported the class but never constructed it, so nothing caught the missing import. Added construction coverage for every parser exported from `semantica.parse`, driven off `__all__` so future additions are covered automatically, living outside `test_parse_comprehensive.py` (whose `setUp` mocks `get_progress_tracker` into each module and would mock away the exact interaction under test) + +- **Graph analytics (`centrality_calculator.py`, `community_detector.py`, `connectivity_analyzer.py`) dropped isolated nodes and diverged on how each computed its working view of the graph** (#1011) by @T1mn + - Each analyzer had its own ad hoc logic for building the node/edge set it operated over, and none of them included nodes with no edges — a node with zero connections simply vanished from centrality scores, community assignments, and connectivity reports instead of appearing with a zero/singleton value. A new shared `semantica/kg/_graph_view.py` centralizes graph-view construction (including node fallbacks and community payload shaping) for all three analyzers, which are now ~250 lines lighter combined + - New `tests/kg/test_analytics_node_scope.py` covering isolated-node presence across all three analyzers + +- **Explorer fired temporal-bounds and snapshot requests before the graph itself had loaded, tripling failed requests when the backend was down and leaving the timeline scrubber with nothing to scrub** (#1003) by @lakshayxi + - Two new predicate functions gate the temporal effects on the graph having actually loaded (an empty graph still counts as loaded); confirmed against a downed backend that this cuts three failing requests per page load down to one + +- **`SeedDataManager.load_from_database()` never actually reached the database, and connection failures were mislabeled as a missing optional dependency** (#995, closes #973) by @yzxcj797 + - `DBIngestor.execute_query`/`export_table` need the connection string as their first positional argument; `load_from_database()` only passed it into the constructor's config dict, which those methods never read, so every call raised `TypeError` before connecting. Also split the combined `except (ImportError, OSError)` handling apart — a genuine connection failure was reported as `"module not available"`, sending debugging in the wrong direction; `OSError` now propagates as an actual failure, chained via `from e` + +- **SPARQL `CONSTRUCT` detection matched inside a leading `#`-comment, misclassifying `SELECT`/`ASK` queries as `CONSTRUCT` across all four SPARQL backends** (#951) by @pravit-amp + - `CONSTRUCT_QUERY_RE` skipped comments with a bare `\#[^\n]*`, whose backtracking `*` let a `# CONSTRUCT ...` comment line "swallow" the real query-form keyword on the next line for a query like `# CONSTRUCT ...\nSELECT ...`. The mistaken `CONSTRUCT` classification sent `Accept: text/turtle` and tried to parse a SELECT/ASK response body as Turtle, failing with a misleading parse error. The regex now requires a comment to reach a line terminator (LF or CR, per the SPARQL grammar) before matching + +- **`k_shortest_paths` mutated caller-visible graph state during traversal and ignored direction when excluding already-used edges** (#1000) by @T1mn + - `semantica/kg/path_finder.py`'s search left side effects behind after returning, and edge exclusion during Yen's-algorithm-style path removal didn't respect the traversal direction of directed graphs, letting a later search see edges that should have been available. Both fixed; new coverage in `tests/kg/test_path_finder.py` + +- **`trace_decision_causality()` ignored explicitly recorded causal edges, inferring causes only from shared NER entities plus timestamp ordering** (#983) by @hsd2514 + - A `CAUSED`/`INFLUENCED`/`PRECEDENT_FOR` edge added via `add_causal_relationship()` had no effect on the trace — when entity extraction found nothing in common between two decisions, `trace_decision_chain()` came back empty even with an explicit edge stored in the graph. Explicit causal edges are now traversed first as ground truth, with entity/timestamp inference kept as an additive fallback for pairs with no explicit link; edges whose source has no decision record (e.g. a graph restored via `from_dict`) are skipped so a stale edge can't abort the trace + +- **`RepoIngestor`'s module-level DNS resolve cache had no lock, raising `RuntimeError: OrderedDict mutated during iteration` under concurrent `ingest_repository()` calls** (#979) by @manjunathbhaskar + - `_REPO_HOST_RESOLVE_CACHE` is a shared `OrderedDict` read, written, and pruned by every thread with no synchronization — reliably reproduced with 32 threads hammering resolution under a low TTL and small cache cap. Now guarded by a lock + +- **`GraphBuilder` didn't remap relationship endpoints after entity resolution merged nodes, leaving relationships pointing at ids that no longer existed in the resolved graph** (#978) by @T1mn + - New coverage in `tests/kg/test_graph_builder_external.py`; a follow-up commit hardens the remapping against edge cases found during review + +- **Explorer's dev server esbuild target didn't match the browser targets the production build declares**, occasionally producing dev-only syntax errors on older browsers (#966) by @le-czs + - `explorer/vite.config.ts` now sets the dev esbuild target explicitly to match + +- **`normalize`'s number normalizer accepted currency symbols without validating them against the surrounding text, and an earlier fix's currency-code matching wasn't token-bounded** (#940) by @Mr-Neutr0n, reviewed by @ZohaibHassan16 + - Symbol currencies are now validated before being accepted; currency codes are matched on token boundaries so a code embedded inside a longer token no longer false-positives + +- **`ContextGraph.to_dict()` was the one reader on the class that didn't hold `self._lock`, raising `RuntimeError: dictionary changed size during iteration` under a concurrent writer and risking a torn snapshot otherwise** (#929) by @pravit-amp + - Every other reader (`stats()`, `density()`, `find_nodes()`, `find_edges()`, `get_neighbors()`, `get_nodes_by_label()`, `state_at()`, `save_to_file()`) already took the lock after it was introduced; `to_dict()` predated that change and was missed. `save_to_file()` was safe only incidentally, since it builds its payload inline under its own lock rather than delegating to `to_dict()` + +- **`PipelineWithProvenance` had a broken import and no working `run()` method** (#862) by @Karunasagar12 + - `from .pipeline import Pipeline` failed because `Pipeline` lives in `pipeline_builder.py`, not a nonexistent `pipeline.py` — fixed to `from .pipeline_builder import Pipeline`. The class also had no `run()`; it now delegates to `ExecutionEngine.execute_pipeline()`, the intended execution path for a built `Pipeline`. The constructor now accepts a built `Pipeline` instance directly + ### Security - **Tarball restore path traversal, latent SQL injection, DNS-rebinding TOCTOU in the shared SSRF guard, stored XSS in report generation, and unvalidated SPARQL object IRIs in AnzoStore** (#1079) by @KaifAhmad1 @@ -254,6 +331,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Caught by the new gate on its first run**: `python -m pip install -e ".[all]"` pulled in `setuptools==79.0.1`, vulnerable to CVE-2026-59890/GHSA-h35f-9h28-mq5c/PYSEC-2026-3447 (Unicode-normalization bypass of `MANIFEST.in` exclude/prune patterns on macOS APFS/HFS+, letting excluded files leak into a built sdist), fixed in `83.0.0`. `[build-system] requires` had the exact same too-permissive-floor pattern this whole entry is about (`setuptools>=61.0`), and `actions/setup-python`'s baked-in `setuptools` isn't governed by that pin at all since it's outside any isolated build. Bumped `[build-system] requires` to `setuptools>=83.0.0`, and the `Security` workflow now runs `pip install --upgrade pip setuptools` before auditing so the scanned environment can't have a stale ambient copy regardless of what governs it - Full `explorer` suite: 241 passed +- **`SeedDataManager.load_from_api()` made unguarded HTTP requests, with no SSRF protection at all** (#942) by @ZohaibHassan16 + - `load_from_api()` called `requests.get()` directly instead of going through `semantica/ingest/ssrf.py`'s `request_with_ssrf_guard()`, unlike every other ingestor in this module — a caller-supplied `api_url` could target internal/private network addresses with no validation. Now routes through the shared guard, gaining redirect validation and bounded DNS resolution for free + - **Follow-up** (#959, closes #943) by @yunaremaia: added an `allow_private_ips` opt-in (parsed via the shared `parse_bool` helper) for trusted internal deployments that legitimately need to load from a private-network API, while keeping the guard's block-by-default behavior for everyone else + ## [0.6.5] - 2026-08-11 ### Added diff --git a/README.md b/README.md index 49a4c57c..a2646bde 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ compliant = graph.check_decision_rules({"category": "vendor_selection"}) # poli ```bash semantica doctor # Python 3.11.9 pass -# semantica 0.6.5 pass +# semantica 0.6.6 pass # faiss vector store pass # Config file pass ~/.semantica/config.yaml ``` @@ -1466,18 +1466,18 @@ For contributor / dev-server setup: **[explorer/README.md: Local Setup Guide](ex --- -## What's New in v0.6.5 +## What's New in v0.6.6 -**Security release — upgrading is strongly recommended.** Fixes for 5 externally-reported vulnerabilities in the Explorer API and graph/triplet store backends, plus a CodeQL-flagged ReDoS: +**Security release — upgrading is strongly recommended.** Fixes for a privately disclosed batch of vulnerabilities spanning backup/restore, database export, outbound requests, and triplet-store backends, plus SSRF hardening across ingestion: -- **Missing authentication on all Explorer API routes** (GHSA-j4mq-hprp-987v, Critical): every route now requires `SEMANTICA_API_KEY`, fails closed (503) rather than open when unconfigured -- **SSRF via redirect bypass in ontology URL fetching** (GHSA-8c7v-62gr-hj6g, High): redirect targets are now re-validated at every hop and the connection is pinned to the validated address, closing a DNS check-then-use race -- **Cypher injection via unvalidated node labels and property keys** (GHSA-482h-hw99-h62p, Critical): Neptune, Neo4j, and FalkorDB now sanitize every label/relationship-type/property-key interpolation site -- **SPARQL injection via unvalidated triplet IRIs** (GHSA-8vgg-8mr4-r236, Critical): Blazegraph, RDF4J, and Jena now validate subject/predicate/object IRIs before interpolation -- **Missing Origin validation on the WebSocket handshake** (GHSA-4643-wpgq-w329, Moderate, anonymous-mode only): `/ws/graph-updates` now checks `Origin` against the same allowlist `CORSMiddleware` enforces for HTTP -- **Polynomial ReDoS in SPARQL query validation** (CodeQL `py/polynomial-redos`): fixed a backtracking regex in the Explorer's SPARQL route +- **Tarball restore path traversal**: `semantica backup restore` now validates every archive member for path containment and rejects symlink/hardlink escapes before extraction +- **Latent SQL injection in `DataExporter.export_table_data()`**: table/schema names are now identifier-allowlisted and `where`/`order_by` fragments are blocklist-checked +- **DNS-rebinding TOCTOU in the shared SSRF guard**: the resolved IP that passes validation is now the one the connection is pinned to, closing the check-then-use race (also closes the `100.64.0.0/10` CGNAT gap) +- **Stored XSS in HTML report generation** and **unvalidated SPARQL object IRIs in AnzoStore** (SPARQL injection): both now escape/validate before interpolation +- **`Authorization`/`Proxy-Authorization` credential leakage across redirects**, plus **SSRF gaps in `FeedIngestor`/`FeedMonitor`, `RepoIngestor`, and the MCP/public-API ingest paths**: all now route through the shared, redirect-safe SSRF guard +- **HTTP response header injection and an unbounded-memory DoS** in the Explorer API, and a **`fastapi`/`python-multipart` ReDoS** (PYSEC-2024-38): floors raised, inputs sanitized, candidate pools capped -Also includes: embedded Oxigraph backend for `TripletStore`, PROV-O trust/spec completeness for `ProvenanceManager`, and the Altair Anzo triplet store backend. +Also ships: **first-class CrewAI integration** (`semantica[crewai]`, extraction/decision tools + a knowledge source), **`ContextGraph` retraction and purge** (GDPR-style erasure without a full `clear()`), a declared **Semantica RDF vocabulary with deterministic entity/relationship IRIs** (stable, diffable exports), and **timezone-aware timestamps** across `export/` and `provenance/`. → [Full release notes](RELEASE_NOTES.md) · [Changelog](CHANGELOG.md) diff --git a/docs/citation.md b/docs/citation.md index d1077887..45ac350b 100644 --- a/docs/citation.md +++ b/docs/citation.md @@ -17,22 +17,22 @@ icon: "quote-left" author = {Semantica}, year = {2026}, url = {https://github.com/semantica-agi/semantica}, - version = {0.6.5}, + version = {0.6.6}, doi = {10.5281/zenodo.XXXXXXX} } ``` - Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* (Version 0.6.5) \[Computer software\]. https://github.com/semantica-agi/semantica + Semantica. (2026). *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems* (Version 0.6.6) \[Computer software\]. https://github.com/semantica-agi/semantica - Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5, GitHub, 2026, https://github.com/semantica-agi/semantica. + Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.6, GitHub, 2026, https://github.com/semantica-agi/semantica. - Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.5. GitHub, 2026. https://github.com/semantica-agi/semantica. + Semantica. *Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems*. Version 0.6.6. GitHub, 2026. https://github.com/semantica-agi/semantica. - Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," Version 0.6.5, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica + Semantica, "Semantica: Graph-Native Infrastructure for Context and Accountable AI Systems," Version 0.6.6, GitHub, 2026. \[Online\]. Available: https://github.com/semantica-agi/semantica diff --git a/docs/faq.md b/docs/faq.md index 05c708c1..74ec023e 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -17,7 +17,7 @@ icon: "circle-question" | API key required? | Optional: pattern extraction works with no keys | | Works with LangChain / LlamaIndex? | Yes: Semantica is a layer on top, not a replacement | | Production-ready? | Yes: 1,000+ tests, v0.5.0 ships with 12 security fixes | -| Latest version? | **v0.6.5** (August 2026) | +| Latest version? | **v0.6.6** (August 2026) | | Local LLMs? | Yes: Ollama via LiteLLM, HuggingFaceLLM for air-gapped | diff --git a/docs/getting-started.md b/docs/getting-started.md index ee442fed..bd7eb176 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -42,7 +42,7 @@ icon: "rocket" Verify installation: ```python import semantica - print(semantica.__version__) # 0.6.5 + print(semantica.__version__) # 0.6.6 ``` diff --git a/pyproject.toml b/pyproject.toml index 286a26b8..ea0d886e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "semantica" -version = "0.6.5" -description = "Accountability and context layer for AI agents. Context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable." +version = "0.6.6" +description = "Graph-Native Infrastructure for Context and Accountable AI Systems: context graphs, decision intelligence, full provenance tracking, and explainable reasoning engines — every AI decision traceable, every output auditable." readme = "README.md" license = { text = "MIT" } diff --git a/semantica/__init__.py b/semantica/__init__.py index 79f83236..0bc0b384 100644 --- a/semantica/__init__.py +++ b/semantica/__init__.py @@ -10,7 +10,7 @@ Main exports: - Config: Configuration management """ -__version__ = "0.6.5" +__version__ = "0.6.6" __author__ = "Semantica Contributors" __license__ = "MIT" From 861b2bf7571f1eccf95ac689091fe26be3581144 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 08:08:18 +0000 Subject: [PATCH 11/13] security(deps): bump anthropic from 0.121.0 to 0.122.0 Bumps [anthropic](https://github.com/anthropics/anthropic-sdk-python) from 0.121.0 to 0.122.0. - [Release notes](https://github.com/anthropics/anthropic-sdk-python/releases) - [Changelog](https://github.com/anthropics/anthropic-sdk-python/blob/main/CHANGELOG.md) - [Commits](https://github.com/anthropics/anthropic-sdk-python/compare/v0.121.0...v0.122.0) --- updated-dependencies: - dependency-name: anthropic dependency-version: 0.122.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- requirements-ci.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ea0d886e..8ffb9395 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,7 +103,7 @@ Discord = "https://discord.gg/sV34vps5hH" llm-openai = ["openai>=1.0.0"] llm-groq = ["groq>=0.4.0"] llm-gemini = ["google-genai>=0.1.0"] -llm-anthropic = ["anthropic>=0.18.0"] +llm-anthropic = ["anthropic>=0.122.0"] llm-ollama = ["ollama>=0.1.0"] llm-deepseek = ["openai>=1.0.0"] llm-litellm = ["litellm>=1.83.9"] diff --git a/requirements-ci.txt b/requirements-ci.txt index 17df46d4..1126111c 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -159,9 +159,9 @@ annotated-types==0.8.0 \ --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 # via pydantic -anthropic==0.121.0 \ - --hash=sha256:6048713fa441e59e1cba8363171cd2a86273b25bd213e9c7ac70a523af88b011 \ - --hash=sha256:e79d6e08ab3376602fc9a70d4d5ea3540817c76cf7e16658bed790834e1833d6 +anthropic==0.122.0 \ + --hash=sha256:45ec906452ffae6b5f7f0c53d01f50bfb7e4ce878d7ae8e4309d13171e557e67 \ + --hash=sha256:ffec56ae96657c8d19fa575ec96f140f380c353a07ab7d61b92eb18ee6536601 # via semantica (pyproject.toml) antlr4-python3-runtime==4.9.3 \ --hash=sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b From 54c274e02cfc9396e83d99d1b87d324e1717e6fc Mon Sep 17 00:00:00 2001 From: Shubham Srivastava Date: Thu, 20 Aug 2026 13:40:22 +0100 Subject: [PATCH 12/13] test(ingest): track relationship provenance via ProvenanceManager (#1071) * test(ingest): track relationship provenance via ProvenanceManager kg.ProvenanceTracker has no track_relationship and never did, so patch.object raised AttributeError before the test body ran. Closes #1055 * test(ingest): disambiguate relationship keys and pin provenance storage Addresses review feedback on #1071. --------- --- tests/ingest/test_notebook_06.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/ingest/test_notebook_06.py b/tests/ingest/test_notebook_06.py index 61a46201..ec857e64 100644 --- a/tests/ingest/test_notebook_06.py +++ b/tests/ingest/test_notebook_06.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock, patch from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, StreamIngestor, FeedIngestor from semantica.kg import GraphBuilder, EntityResolver, ProvenanceTracker +from semantica.provenance import InMemoryStorage, ProvenanceManager from semantica.conflicts import ConflictDetector pytestmark = pytest.mark.integration @@ -75,13 +76,30 @@ class TestNotebook06MultiSourceIntegration: for entity in all_entities: provenance_tracker.track_entity(entity.get("id"), entity.get("source"), entity) + # Endpoints stay on "source"/"target", which is what GraphBuilder's + # dict normalization expects; the originating document moves to + # "document". The literal previously set "source" twice, so the + # endpoint id was silently overwritten by the document name. relationships = [ - {"source": "e2", "target": "e1", "type": "CEO_of", "source": "file1"} + {"id": "r1", "source": "e2", "target": "e1", + "type": "CEO_of", "document": "file1"} ] - - with patch.object(provenance_tracker, 'track_relationship'): - for rel in relationships: - provenance_tracker.track_relationship(rel.get("source"), rel.get("target"), rel.get("source"), rel) + + # kg.ProvenanceTracker has no track_relationship and never did; that + # lives on ProvenanceManager, which is where ProvenanceTracker's own + # DeprecationWarning points callers. Called for real rather than + # patched, so this step actually exercises something. + # + # Storage is pinned to in-memory: with no argument, ProvenanceManager + # falls back to the mutable class-level _default_storage_path, so an + # earlier test setting it would make this write SQLite to disk and + # turn the result order-dependent. + provenance_manager = ProvenanceManager(storage=InMemoryStorage()) + for rel in relationships: + entry = provenance_manager.track_relationship( + rel["id"], rel["document"], metadata=rel + ) + assert entry is not None # --- Step 5: Build Unified KG --- builder = GraphBuilder() From c5d382ee81edc0727d07907c932970c762ec33bd Mon Sep 17 00:00:00 2001 From: Luan Taraschi <130802253+luantaraschi@users.noreply.github.com> Date: Thu, 20 Aug 2026 09:58:20 -0300 Subject: [PATCH 13/13] test(visualization): isolate optional dependency mocks (#897) * test(visualization): isolate optional dependency mocks * test(visualization): stop requiring Plotly in unit tests Removing the global sys.modules stubs left the tests that patch `...go.Bar`, or call a visualizer, with nothing standing in for the module level `px` and `go` aliases. Those are None when Plotly is missing, so patch resolution and _check_dependencies() both failed. Add a helper that substitutes a double only for the aliases that are None, leaving the real module in place when Plotly is installed. --------- --- tests/visualization/__init__.py | 0 tests/visualization/_plotly_doubles.py | 29 +++ .../test_kg_visualizer_normalize_graph.py | 85 ++++---- .../test_optional_dependencies.py | 188 +++++++----------- tests/visualization/test_visualization.py | 24 +-- .../test_visualization_advanced.py | 55 ++--- .../test_visualization_comprehensive.py | 51 ++--- 7 files changed, 187 insertions(+), 245 deletions(-) create mode 100644 tests/visualization/__init__.py create mode 100644 tests/visualization/_plotly_doubles.py diff --git a/tests/visualization/__init__.py b/tests/visualization/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/visualization/_plotly_doubles.py b/tests/visualization/_plotly_doubles.py new file mode 100644 index 00000000..82465e15 --- /dev/null +++ b/tests/visualization/_plotly_doubles.py @@ -0,0 +1,29 @@ +"""Shared helper for visualization tests. + +The visualization modules treat Plotly as optional: they bind ``px``, ``go`` and +``make_subplots`` to ``None`` when the import fails, and raise ``ProcessingError`` +from ``_check_dependencies()``. Tests that exercise a Plotly-backed path need +those names to be usable, otherwise ``patch("...go.Figure")`` fails on ``None`` +and the visualizers refuse to run. + +``plotly_doubles`` fills in a double for each alias that is ``None``, so the +tests describe their own requirements instead of depending on whether Plotly +happens to be installed. When Plotly is installed the aliases are left alone and +the patches keep asserting against the real attribute names. +""" + +from contextlib import ExitStack, contextmanager +from unittest.mock import MagicMock, patch + +PLOTLY_ALIASES = ("px", "go", "make_subplots") + + +@contextmanager +def plotly_doubles(*modules): + """Stand in for the module-level Plotly aliases that are unavailable.""" + with ExitStack() as stack: + for module in modules: + for alias in PLOTLY_ALIASES: + if getattr(module, alias, "unused") is None: + stack.enter_context(patch.object(module, alias, MagicMock())) + yield diff --git a/tests/visualization/test_kg_visualizer_normalize_graph.py b/tests/visualization/test_kg_visualizer_normalize_graph.py index 055bcdc3..9ff130f5 100644 --- a/tests/visualization/test_kg_visualizer_normalize_graph.py +++ b/tests/visualization/test_kg_visualizer_normalize_graph.py @@ -9,27 +9,16 @@ and must raise a clear ProcessingError for anything else. """ import contextlib -import sys import unittest from dataclasses import dataclass, field from typing import List from unittest.mock import MagicMock, patch -# --------------------------------------------------------------------------- -# Stub out heavy optional deps before importing the module under test -# --------------------------------------------------------------------------- -sys.modules.setdefault("matplotlib", MagicMock()) -sys.modules.setdefault("matplotlib.pyplot", MagicMock()) -sys.modules.setdefault("matplotlib.patches", MagicMock()) -sys.modules.setdefault("plotly", MagicMock()) -sys.modules.setdefault("plotly.express", MagicMock()) -sys.modules.setdefault("plotly.graph_objects", MagicMock()) -sys.modules.setdefault("plotly.subplots", MagicMock()) -sys.modules.setdefault("graphviz", MagicMock()) -sys.modules.setdefault("seaborn", MagicMock()) - from semantica.utils.exceptions import ProcessingError # noqa: E402 +from semantica.visualization import kg_visualizer # noqa: E402 from semantica.visualization.kg_visualizer import KGVisualizer # noqa: E402 +from tests.visualization._plotly_doubles import plotly_doubles # noqa: E402 + # --------------------------------------------------------------------------- # Minimal fixtures @@ -191,10 +180,6 @@ class TestVisualizeNetworkAcceptsKGObject(unittest.TestCase): def _run_visualize_network(self, graph_arg): """Run visualize_network with all Plotly internals mocked.""" mock_fig = MagicMock() - mock_go = sys.modules["plotly.graph_objects"] - mock_go.Figure.return_value = mock_fig - mock_go.Scatter.return_value = MagicMock() - mock_go.Layout.return_value = MagicMock() viz = _make_viz() @@ -207,6 +192,10 @@ class TestVisualizeNetworkAcceptsKGObject(unittest.TestCase): # ColorPalette helpers with ( + plotly_doubles(kg_visualizer), + patch("semantica.visualization.kg_visualizer.go.Figure", return_value=mock_fig), + patch("semantica.visualization.kg_visualizer.go.Scatter"), + patch("semantica.visualization.kg_visualizer.go.Layout"), patch( "semantica.visualization.kg_visualizer.ColorPalette.get_entity_type_colors", return_value={"Person": "#ff0000"}, @@ -254,9 +243,12 @@ class TestAllVisualizeMethodsAcceptKGObject(unittest.TestCase): self.viz._normalize_graph = MagicMock(return_value=GRAPH_DICT) self.viz._visualize_network_plotly = MagicMock(return_value=MagicMock()) communities = {"node_assignments": {"e1": 0, "e2": 1}, "num_communities": 2} - with patch( - "semantica.visualization.kg_visualizer.ColorPalette.get_community_colors", - return_value=["#ff0000", "#00ff00"], + with ( + plotly_doubles(kg_visualizer), + patch( + "semantica.visualization.kg_visualizer.ColorPalette.get_community_colors", + return_value=["#ff0000", "#00ff00"], + ), ): self.viz.visualize_communities(self.kg, communities=communities) self.viz._normalize_graph.assert_called_once_with(self.kg) @@ -264,22 +256,24 @@ class TestAllVisualizeMethodsAcceptKGObject(unittest.TestCase): def test_visualize_centrality_accepts_kg_object(self): self.viz._normalize_graph = MagicMock(return_value=GRAPH_DICT) self.viz._visualize_network_plotly = MagicMock(return_value=MagicMock()) - self.viz.visualize_centrality(self.kg, centrality={"centrality": {}}) + with plotly_doubles(kg_visualizer): + self.viz.visualize_centrality(self.kg, centrality={"centrality": {}}) self.viz._normalize_graph.assert_called_once_with(self.kg) def test_visualize_entity_types_accepts_kg_object(self): self.viz._normalize_graph = MagicMock(return_value=GRAPH_DICT) - mock_px = sys.modules["plotly.express"] - mock_px.bar.return_value = MagicMock() - self.viz.visualize_entity_types(self.kg) + with plotly_doubles(kg_visualizer), patch("semantica.visualization.kg_visualizer.px.bar"): + self.viz.visualize_entity_types(self.kg) self.viz._normalize_graph.assert_called_once_with(self.kg) def test_visualize_relationship_matrix_accepts_kg_object(self): self.viz._normalize_graph = MagicMock(return_value=GRAPH_DICT) - mock_go = sys.modules["plotly.graph_objects"] - mock_go.Figure.return_value = MagicMock() - mock_go.Heatmap.return_value = MagicMock() - self.viz.visualize_relationship_matrix(self.kg) + with ( + plotly_doubles(kg_visualizer), + patch("semantica.visualization.kg_visualizer.go.Figure"), + patch("semantica.visualization.kg_visualizer.go.Heatmap"), + ): + self.viz.visualize_relationship_matrix(self.kg) self.viz._normalize_graph.assert_called_once_with(self.kg) @@ -361,10 +355,6 @@ class TestFormalKnowledgeGraphType(unittest.TestCase): def _run_visualize_network(self, graph_arg): mock_fig = MagicMock() - mock_go = sys.modules["plotly.graph_objects"] - mock_go.Figure.return_value = mock_fig - mock_go.Scatter.return_value = MagicMock() - mock_go.Layout.return_value = MagicMock() viz = _make_viz() fake_pos = {"e1": (0.0, 0.0), "e2": (1.0, 1.0)} viz.force_layout = MagicMock() @@ -372,6 +362,10 @@ class TestFormalKnowledgeGraphType(unittest.TestCase): viz.hierarchical_layout = MagicMock() viz.circular_layout = MagicMock() with ( + plotly_doubles(kg_visualizer), + patch("semantica.visualization.kg_visualizer.go.Figure", return_value=mock_fig), + patch("semantica.visualization.kg_visualizer.go.Scatter"), + patch("semantica.visualization.kg_visualizer.go.Layout"), patch( "semantica.visualization.kg_visualizer.ColorPalette.get_entity_type_colors", return_value={"Person": "#ff0000"}, @@ -392,9 +386,12 @@ class TestFormalKnowledgeGraphType(unittest.TestCase): viz._normalize_graph = MagicMock(return_value=GRAPH_DICT) viz._visualize_network_plotly = MagicMock(return_value=MagicMock()) communities = {"node_assignments": {"e1": 0, "e2": 1}, "num_communities": 2} - with patch( - "semantica.visualization.kg_visualizer.ColorPalette.get_community_colors", - return_value=["#ff0000", "#00ff00"], + with ( + plotly_doubles(kg_visualizer), + patch( + "semantica.visualization.kg_visualizer.ColorPalette.get_community_colors", + return_value=["#ff0000", "#00ff00"], + ), ): viz.visualize_communities(kg, communities=communities) viz._normalize_graph.assert_called_once_with(kg) @@ -404,24 +401,28 @@ class TestFormalKnowledgeGraphType(unittest.TestCase): viz = _make_viz() viz._normalize_graph = MagicMock(return_value=GRAPH_DICT) viz._visualize_network_plotly = MagicMock(return_value=MagicMock()) - viz.visualize_centrality(kg, centrality={"centrality": {}}) + with plotly_doubles(kg_visualizer): + viz.visualize_centrality(kg, centrality={"centrality": {}}) viz._normalize_graph.assert_called_once_with(kg) def test_visualize_entity_types_accepts_knowledge_graph(self): kg = self._make_kg() viz = _make_viz() viz._normalize_graph = MagicMock(return_value=GRAPH_DICT) - sys.modules["plotly.express"].bar.return_value = MagicMock() - viz.visualize_entity_types(kg) + with plotly_doubles(kg_visualizer), patch("semantica.visualization.kg_visualizer.px.bar"): + viz.visualize_entity_types(kg) viz._normalize_graph.assert_called_once_with(kg) def test_visualize_relationship_matrix_accepts_knowledge_graph(self): kg = self._make_kg() viz = _make_viz() viz._normalize_graph = MagicMock(return_value=GRAPH_DICT) - sys.modules["plotly.graph_objects"].Figure.return_value = MagicMock() - sys.modules["plotly.graph_objects"].Heatmap.return_value = MagicMock() - viz.visualize_relationship_matrix(kg) + with ( + plotly_doubles(kg_visualizer), + patch("semantica.visualization.kg_visualizer.go.Figure"), + patch("semantica.visualization.kg_visualizer.go.Heatmap"), + ): + viz.visualize_relationship_matrix(kg) viz._normalize_graph.assert_called_once_with(kg) def test_knowledge_graph_importable_from_kg_module(self): diff --git a/tests/visualization/test_optional_dependencies.py b/tests/visualization/test_optional_dependencies.py index bc390ecc..af41e5ca 100644 --- a/tests/visualization/test_optional_dependencies.py +++ b/tests/visualization/test_optional_dependencies.py @@ -1,149 +1,115 @@ -import unittest -from unittest.mock import MagicMock, patch +import importlib import sys +import unittest +from contextlib import contextmanager +from unittest.mock import patch + import numpy as np -# Helper to mock modules -def mock_module(name): - m = MagicMock() - sys.modules[name] = m - return m +from tests.visualization._plotly_doubles import plotly_doubles + + +@contextmanager +def import_without(module_name, *dependencies): + """Import a module with selected optional dependencies unavailable.""" + package_name, attribute = module_name.rsplit(".", 1) + package = importlib.import_module(package_name) + missing = object() + original_module = sys.modules.pop(module_name, missing) + original_attribute = getattr(package, attribute, missing) + + try: + with patch.dict(sys.modules, {name: None for name in dependencies}): + yield importlib.import_module(module_name) + finally: + sys.modules.pop(module_name, None) + if original_module is not missing: + sys.modules[module_name] = original_module + if original_attribute is missing: + package.__dict__.pop(attribute, None) + else: + setattr(package, attribute, original_attribute) + class TestOptionalDependencies(unittest.TestCase): - - @classmethod - def setUpClass(cls): - # Mock heavy/problematic dependencies globally to prevent environment crashes - # We use a dict to save original modules if they exist, but for this test file - # we generally want to run in a controlled "clean" environment. - cls.modules_to_patch = [ - 'sklearn', 'sklearn.decomposition', 'sklearn.manifold', - 'scipy', 'scipy.optimize', - 'matplotlib', 'matplotlib.pyplot', 'matplotlib.patches', - 'plotly', 'plotly.express', 'plotly.graph_objects', 'plotly.subplots', - 'networkx', 'seaborn' - ] - - cls.original_modules = {} - for mod in cls.modules_to_patch: - if mod in sys.modules: - cls.original_modules[mod] = sys.modules[mod] - sys.modules[mod] = MagicMock() - - @classmethod - def tearDownClass(cls): - # Restore original modules - for mod in cls.modules_to_patch: - if mod in cls.original_modules: - sys.modules[mod] = cls.original_modules[mod] - else: - del sys.modules[mod] - - def setUp(self): - # Clear cached visualization modules to ensure fresh imports - self.viz_modules = [ - 'semantica.visualization.embedding_visualizer', - 'semantica.visualization.ontology_visualizer', - 'semantica.visualization.kg_visualizer', - 'semantica.visualization.utils.export_formats' - ] - for mod in self.viz_modules: - if mod in sys.modules: - del sys.modules[mod] def test_embedding_visualizer_without_umap(self): """Test EmbeddingVisualizer behavior when umap is missing.""" - # Ensure umap is missing - with patch.dict(sys.modules, {'umap': None}): - from semantica.visualization.embedding_visualizer import EmbeddingVisualizer - - # Setup PCA mock to verify fallback - mock_pca_class = sys.modules['sklearn.decomposition'].PCA - mock_pca_instance = mock_pca_class.return_value - # Configure fit_transform to return correct shape (n_samples, 2) - mock_pca_instance.fit_transform.return_value = np.zeros((4, 2)) - - viz = EmbeddingVisualizer() - # Use numpy array! - embeddings = np.array([[0, 1, 2], [1, 0, 3], [0, 0, 0], [1, 1, 1]]) - - # Should fallback to PCA when method="umap" is used but umap is None - # The code logs a warning and uses PCA - viz.visualize_2d_projection(embeddings, method="umap") - - # Verify PCA was called + with import_without( + "semantica.visualization.embedding_visualizer", "umap" + ) as module: + with plotly_doubles(module), patch.object(module, "PCA") as mock_pca_class: + mock_pca_class.return_value.fit_transform.return_value = np.zeros((4, 2)) + + viz = module.EmbeddingVisualizer() + embeddings = np.array([[0, 1, 2], [1, 0, 3], [0, 0, 0], [1, 1, 1]]) + viz.visualize_2d_projection(embeddings, method="umap") + mock_pca_class.assert_called() def test_ontology_visualizer_without_graphviz(self): """Test OntologyVisualizer behavior when graphviz is missing.""" - # Ensure graphviz is missing - with patch.dict(sys.modules, {'graphviz': None}): - from semantica.visualization.ontology_visualizer import OntologyVisualizer, ProcessingError - - viz = OntologyVisualizer() + with import_without( + "semantica.visualization.ontology_visualizer", "graphviz" + ) as module: + viz = module.OntologyVisualizer() ontology = { "classes": [ {"name": "A", "label": "A"}, - {"name": "B", "label": "B", "parent": "A"} + {"name": "B", "label": "B", "parent": "A"}, ] } - - with self.assertRaises(ProcessingError) as cm: + + with self.assertRaises(module.ProcessingError) as cm: viz.visualize_hierarchy(ontology, output="dot", file_path="test.dot") - + self.assertIn("Graphviz is required for DOT export", str(cm.exception)) def test_analytics_visualizer_without_plotly(self): """Test AnalyticsVisualizer behavior when plotly is missing.""" - with patch.dict(sys.modules, {'plotly': None, 'plotly.express': None, 'plotly.graph_objects': None}): - from semantica.visualization.analytics_visualizer import AnalyticsVisualizer, ProcessingError - - # Need to ensure numpy is available for init (it's imported at top level) - # But we are testing plotly missing. - - viz = AnalyticsVisualizer() - - with self.assertRaises(ProcessingError) as cm: - viz.visualize_centrality_rankings({"node1": 1.0}) - - self.assertIn("Plotly is required", str(cm.exception)) + with import_without( + "semantica.visualization.analytics_visualizer", + "plotly", + "plotly.express", + "plotly.graph_objects", + ) as module: + viz = module.AnalyticsVisualizer() - def test_analytics_visualizer_without_plotly(self): - """Test AnalyticsVisualizer behavior when plotly is missing.""" - with patch.dict(sys.modules, {'plotly': None, 'plotly.express': None, 'plotly.graph_objects': None}): - from semantica.visualization.analytics_visualizer import AnalyticsVisualizer, ProcessingError - - viz = AnalyticsVisualizer() - - with self.assertRaises(ProcessingError) as cm: + with self.assertRaises(module.ProcessingError) as cm: viz.visualize_centrality_rankings({}) - + self.assertIn("Plotly is required", str(cm.exception)) def test_semantic_network_visualizer_without_plotly(self): """Test SemanticNetworkVisualizer behavior when plotly is missing.""" - with patch.dict(sys.modules, {'plotly': None, 'plotly.express': None, 'plotly.graph_objects': None}): - from semantica.visualization.semantic_network_visualizer import SemanticNetworkVisualizer, ProcessingError - - viz = SemanticNetworkVisualizer() - - with self.assertRaises(ProcessingError) as cm: + with import_without( + "semantica.visualization.semantic_network_visualizer", + "plotly", + "plotly.express", + "plotly.graph_objects", + ) as module: + viz = module.SemanticNetworkVisualizer() + + with self.assertRaises(module.ProcessingError) as cm: viz.visualize_network({}) - + self.assertIn("Plotly is required", str(cm.exception)) def test_temporal_visualizer_without_plotly(self): """Test TemporalVisualizer behavior when plotly is missing.""" - with patch.dict(sys.modules, {'plotly': None, 'plotly.express': None, 'plotly.graph_objects': None}): - from semantica.visualization.temporal_visualizer import TemporalVisualizer, ProcessingError - - viz = TemporalVisualizer() - - with self.assertRaises(ProcessingError) as cm: + with import_without( + "semantica.visualization.temporal_visualizer", + "plotly", + "plotly.express", + "plotly.graph_objects", + ) as module: + viz = module.TemporalVisualizer() + + with self.assertRaises(module.ProcessingError) as cm: viz.visualize_timeline({"events": []}) - + self.assertIn("Plotly is required", str(cm.exception)) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/visualization/test_visualization.py b/tests/visualization/test_visualization.py index c8288584..424caf0e 100644 --- a/tests/visualization/test_visualization.py +++ b/tests/visualization/test_visualization.py @@ -1,27 +1,5 @@ import unittest -from unittest.mock import MagicMock, patch, ANY -import sys -import types - -# Helper to create a mock package -def mock_package(name): - m = MagicMock() - m.__path__ = [] - sys.modules[name] = m - return m - -# Mock libraries before importing module under test -# We need to ensure matplotlib behaves like a package for seaborn -sys.modules['matplotlib'] = MagicMock() -sys.modules['matplotlib.colors'] = MagicMock() -sys.modules['matplotlib.pyplot'] = MagicMock() -sys.modules['matplotlib.patches'] = MagicMock() -sys.modules['plotly'] = MagicMock() -sys.modules['plotly.express'] = MagicMock() -sys.modules['plotly.graph_objects'] = MagicMock() -sys.modules['plotly.subplots'] = MagicMock() -sys.modules['graphviz'] = MagicMock() -sys.modules['seaborn'] = MagicMock() +from unittest.mock import MagicMock, patch from semantica.visualization.kg_visualizer import KGVisualizer from semantica.visualization.ontology_visualizer import OntologyVisualizer diff --git a/tests/visualization/test_visualization_advanced.py b/tests/visualization/test_visualization_advanced.py index 8d69bd63..555565ef 100644 --- a/tests/visualization/test_visualization_advanced.py +++ b/tests/visualization/test_visualization_advanced.py @@ -1,34 +1,25 @@ import unittest +from contextlib import ExitStack from unittest.mock import MagicMock, patch -import sys import numpy as np -# Mock heavy libraries before importing visualization modules -sys.modules['matplotlib'] = MagicMock() -sys.modules['matplotlib.pyplot'] = MagicMock() -sys.modules['matplotlib.colors'] = MagicMock() -sys.modules['matplotlib.patches'] = MagicMock() -sys.modules['plotly'] = MagicMock() -sys.modules['plotly.express'] = MagicMock() -sys.modules['plotly.graph_objects'] = MagicMock() -sys.modules['plotly.subplots'] = MagicMock() -sys.modules['seaborn'] = MagicMock() -sys.modules['umap'] = MagicMock() -sys.modules['sklearn'] = MagicMock() -sys.modules['sklearn.decomposition'] = MagicMock() -sys.modules['sklearn.manifold'] = MagicMock() - +from semantica.visualization import analytics_visualizer, embedding_visualizer from semantica.visualization.analytics_visualizer import AnalyticsVisualizer from semantica.visualization.embedding_visualizer import EmbeddingVisualizer from semantica.visualization.utils.color_schemes import ColorScheme +from tests.visualization._plotly_doubles import plotly_doubles class TestVisualizationAdvanced(unittest.TestCase): def setUp(self): self.mock_logger = MagicMock() self.mock_tracker = MagicMock() - + + stack = ExitStack() + self.addCleanup(stack.close) + stack.enter_context(plotly_doubles(analytics_visualizer, embedding_visualizer)) + self.patchers = [ patch('semantica.visualization.analytics_visualizer.get_logger', return_value=self.mock_logger), patch('semantica.visualization.analytics_visualizer.get_progress_tracker', return_value=self.mock_tracker), @@ -53,22 +44,17 @@ class TestVisualizationAdvanced(unittest.TestCase): viz = AnalyticsVisualizer() centrality = {"n1": 0.5, "n2": 0.3} - # Access the mock that was injected - import plotly.graph_objects as go - # Reset mock to ensure clean state - go.Bar.reset_mock() - - viz.visualize_centrality_rankings(centrality, output="interactive") - go.Bar.assert_called() + with ( + patch('semantica.visualization.analytics_visualizer.go.Bar') as mock_bar, + patch('semantica.visualization.analytics_visualizer.go.Figure'), + ): + viz.visualize_centrality_rankings(centrality, output="interactive") + mock_bar.assert_called() def test_visualize_community_structure(self): viz = AnalyticsVisualizer() if hasattr(viz, 'visualize_community_structure'): - import plotly.graph_objects as go - # Reset mocks - go.Figure.reset_mock() - graph = MagicMock() communities = {"c1": ["n1", "n2"]} @@ -89,8 +75,6 @@ class TestVisualizationAdvanced(unittest.TestCase): viz = EmbeddingVisualizer() embeddings = np.random.rand(10, 128) - import plotly.graph_objects as go - # Mock UMAP/TSNE/PCA with patch('semantica.visualization.embedding_visualizer.umap') as mock_umap, \ patch('semantica.visualization.embedding_visualizer.TSNE') as mock_tsne, \ @@ -116,12 +100,13 @@ class TestVisualizationAdvanced(unittest.TestCase): viz = EmbeddingVisualizer() embeddings = np.random.rand(5, 5) - import plotly.graph_objects as go - go.Heatmap.reset_mock() - if hasattr(viz, 'visualize_similarity_heatmap'): - viz.visualize_similarity_heatmap(embeddings) - go.Heatmap.assert_called() + with ( + patch('semantica.visualization.embedding_visualizer.go.Heatmap') as mock_heatmap, + patch('semantica.visualization.embedding_visualizer.go.Figure'), + ): + viz.visualize_similarity_heatmap(embeddings) + mock_heatmap.assert_called() if __name__ == '__main__': unittest.main() diff --git a/tests/visualization/test_visualization_comprehensive.py b/tests/visualization/test_visualization_comprehensive.py index 37db07e8..2c3d7577 100644 --- a/tests/visualization/test_visualization_comprehensive.py +++ b/tests/visualization/test_visualization_comprehensive.py @@ -1,26 +1,9 @@ import unittest from unittest.mock import MagicMock, patch -import sys import numpy as np from pathlib import Path import pytest -# Mock heavy libraries before importing visualization modules -sys.modules['matplotlib'] = MagicMock() -sys.modules['matplotlib.pyplot'] = MagicMock() -sys.modules['matplotlib.colors'] = MagicMock() -sys.modules['matplotlib.patches'] = MagicMock() -sys.modules['plotly'] = MagicMock() -sys.modules['plotly.express'] = MagicMock() -sys.modules['plotly.graph_objects'] = MagicMock() -sys.modules['plotly.subplots'] = MagicMock() -sys.modules['seaborn'] = MagicMock() -sys.modules['umap'] = MagicMock() -sys.modules['sklearn'] = MagicMock() -sys.modules['sklearn.decomposition'] = MagicMock() -sys.modules['sklearn.manifold'] = MagicMock() -sys.modules['networkx'] = MagicMock() -sys.modules['graphviz'] = MagicMock() # Import visualizers from semantica.visualization.kg_visualizer import KGVisualizer @@ -52,24 +35,24 @@ class TestVisualizationComprehensive(unittest.TestCase): patch('semantica.visualization.analytics_visualizer.get_progress_tracker', return_value=self.mock_tracker), patch('semantica.visualization.temporal_visualizer.get_logger', return_value=self.mock_logger), patch('semantica.visualization.temporal_visualizer.get_progress_tracker', return_value=self.mock_tracker), - # Mock Layouts - patch('semantica.visualization.kg_visualizer.ForceDirectedLayout', MagicMock()), - patch('semantica.visualization.kg_visualizer.HierarchicalLayout', MagicMock()), - patch('semantica.visualization.kg_visualizer.CircularLayout', MagicMock()), - patch('semantica.visualization.ontology_visualizer.HierarchicalLayout', MagicMock()), - patch('semantica.visualization.semantic_network_visualizer.ForceDirectedLayout', MagicMock()), + patch('semantica.visualization.kg_visualizer.go', MagicMock()), + patch('semantica.visualization.kg_visualizer.px', MagicMock()), + patch('semantica.visualization.ontology_visualizer.go', MagicMock()), + patch('semantica.visualization.ontology_visualizer.make_subplots', MagicMock()), + patch('semantica.visualization.embedding_visualizer.go', MagicMock()), + patch('semantica.visualization.embedding_visualizer.px', MagicMock()), + patch('semantica.visualization.semantic_network_visualizer.go', MagicMock()), + patch('semantica.visualization.semantic_network_visualizer.px', MagicMock()), + patch('semantica.visualization.analytics_visualizer.go', MagicMock()), + patch('semantica.visualization.analytics_visualizer.px', MagicMock()), + patch('semantica.visualization.analytics_visualizer.make_subplots', MagicMock()), + patch('semantica.visualization.temporal_visualizer.go', MagicMock()), + patch('semantica.visualization.temporal_visualizer.px', MagicMock()), ] for p in self.patchers: p.start() - # Reset plotly mocks - import plotly.graph_objects as go - import plotly.express as px - go.Figure.reset_mock() - px.bar.reset_mock() - px.scatter.reset_mock() - def tearDown(self): for p in self.patchers: p.stop() @@ -210,8 +193,8 @@ class TestVisualizationComprehensive(unittest.TestCase): embeddings = np.random.rand(10, 10) # Test visualize_2d_projection (mock UMAP/PCA) - with patch('semantica.visualization.embedding_visualizer.umap.UMAP') as MockUMAP: - MockUMAP.return_value.fit_transform.return_value = np.random.rand(10, 2) + with patch('semantica.visualization.embedding_visualizer.umap', MagicMock()) as mock_umap: + mock_umap.UMAP.return_value.fit_transform.return_value = np.random.rand(10, 2) viz.visualize_2d_projection(embeddings) # Test visualize_similarity_heatmap @@ -219,8 +202,8 @@ class TestVisualizationComprehensive(unittest.TestCase): # Test visualize_clustering clusters = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1] - with patch('semantica.visualization.embedding_visualizer.umap.UMAP') as MockUMAP: - MockUMAP.return_value.fit_transform.return_value = np.random.rand(10, 2) + with patch('semantica.visualization.embedding_visualizer.umap', MagicMock()) as mock_umap: + mock_umap.UMAP.return_value.fit_transform.return_value = np.random.rand(10, 2) viz.visualize_clustering(embeddings, clusters) if __name__ == '__main__':