diff --git a/semantica/export/json_exporter.py b/semantica/export/json_exporter.py index 0c3b492c..e31c39d8 100644 --- a/semantica/export/json_exporter.py +++ b/semantica/export/json_exporter.py @@ -28,12 +28,35 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError, ValidationError -from ..utils.helpers import ensure_directory, utc_now_iso, write_json_file +from ..utils.helpers import ensure_directory, hash_data, utc_now_iso, write_json_file from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri +def _content_iri(prefix: str, payload: Any) -> str: + """Mint a document IRI from what was exported, not when. + + Minting from ``utc_now_iso()`` gave every export of the same graph a new + identity a few microseconds apart, so re-exporting an unchanged graph was + never idempotent and merging exports duplicated every node (#1147). This + mirrors ``mint_entity_iri`` (#1109): identical content hashes to the same + IRI, and any change to the content changes it too. ``default=str`` keeps + the hash defined for values ``json.dumps`` would otherwise reject, such as + ``datetime`` objects a caller may have left in the graph. + + Args: + prefix: IRI prefix the digest is appended to + payload: JSON-serializable value whose content determines the digest + + Returns: + A stable IRI of the form ``{prefix}{16-hex-char digest}`` + """ + canonical = json.dumps(payload, sort_keys=True, default=str) + digest = hash_data(canonical)[:16] + return f"{prefix}{digest}" + + def _is_jsonld_document(data: Dict[str, Any]) -> bool: """ Report whether a dictionary is already a JSON-LD document. @@ -230,7 +253,10 @@ class JSONExporter: - statistics: Statistics dictionary (optional) file_path: Output JSON file path format: Export format - 'json' or 'json-ld' (default: self.format) - **options: Additional options passed to conversion methods + **options: Additional options passed to conversion methods: + - graph_uri: Caller-supplied IRI for the graph node when + format='json-ld', overriding the default content-derived + IRI (see #1147) Example: >>> kg = { @@ -401,7 +427,9 @@ class JSONExporter: data: Data to convert (dict, list, or any value) include_metadata: Whether to include metadata (default: True) include_provenance: Whether to include provenance (default: True) - **options: Additional options passed to knowledge graph conversion + **options: Additional options passed to knowledge graph conversion: + - document_uri: Caller-supplied IRI for the document node, + overriding the default content-derived IRI (see #1147) Returns: Dictionary in JSON-LD format with @context, @graph/@value, and metadata @@ -451,13 +479,17 @@ class JSONExporter: # Add metadata and provenance if requested if include_metadata: - self._attach_document_metadata(jsonld, include_provenance) + self._attach_document_metadata( + jsonld, include_provenance, options.get("document_uri") + ) return jsonld @staticmethod def _attach_document_metadata( - jsonld: Dict[str, Any], include_provenance: bool + jsonld: Dict[str, Any], + include_provenance: bool, + document_uri: Optional[str] = None, ) -> None: """ Attach the export's own metadata without naming the graph. @@ -473,6 +505,9 @@ class JSONExporter: Args: jsonld: Document being built, modified in place include_provenance: Whether to record how and when it was exported + document_uri: Caller-supplied IRI for the document node. Falls back + to a content-derived IRI (#1147) so re-exporting unchanged data + is idempotent instead of minting a new identity every time. """ # A caller may hand us a document that is deliberately a named graph. # That name is theirs to keep, but our own statements must not end up @@ -483,7 +518,10 @@ class JSONExporter: # Do not overwrite an identifier the payload already carries: the # knowledge-graph conversion names its own document node. if "@id" not in jsonld or payload_is_named_graph: - document["@id"] = f"https://semantica.dev/data/{utc_now_iso()}" + content = {key: value for key, value in jsonld.items() if key != "@context"} + document["@id"] = document_uri or _content_iri( + "https://semantica.dev/data/", content + ) if include_provenance: document["semantica:exportedAt"] = utc_now_iso() document["semantica:format"] = "json-ld" @@ -553,7 +591,9 @@ class JSONExporter: - entities: List of entity dictionaries - relationships: List of relationship dictionaries - metadata: Metadata dictionary (optional) - **options: Additional options (unused) + **options: Additional options: + - graph_uri: Caller-supplied IRI for the graph node, + overriding the default content-derived IRI (see #1147) Returns: Dictionary in JSON-LD format with @context, @id, @type, and graph data @@ -566,7 +606,11 @@ class JSONExporter: "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdfs": "http://www.w3.org/2000/01/rdf-schema#", }, - "@id": f"https://semantica.dev/graph/{utc_now_iso()}", + # Minted from the graph's own content rather than the wall clock + # (#1147): re-exporting an unchanged graph must produce the same + # subject, or merging repeated exports duplicates every node. + "@id": options.get("graph_uri") + or _content_iri("https://semantica.dev/graph/", kg), "@type": "semantica:KnowledgeGraph", } diff --git a/tests/export/test_jsonld_document_iri.py b/tests/export/test_jsonld_document_iri.py new file mode 100644 index 00000000..34b2366e --- /dev/null +++ b/tests/export/test_jsonld_document_iri.py @@ -0,0 +1,134 @@ +"""The document IRI of a JSON-LD export must depend on content, not the clock +(issue #1147). + +``_convert_kg_to_jsonld`` minted the graph's ``@id`` from ``utc_now_iso()``, +and the generic ``_attach_document_metadata`` path did the same for a plain +document ``@id``. Exporting an unchanged graph therefore produced a new +subject every time: three exports of one one-entity graph merged into 3 +``semantica:KnowledgeGraph`` nodes and 15 triples for what should have been a +single graph. Neither identifier resolves and the timestamp is already +recorded correctly in ``semantica:exportedAt``, so the fix mints the IRI from +the exported content instead (mirroring ``mint_entity_iri``, #1109), with an +optional caller-supplied override for callers who already name their graphs. +""" + +import json + +from rdflib import RDF, Graph, URIRef + +from semantica.export.json_exporter import JSONExporter + +KG = { + "entities": [{"id": "https://example.org/e1", "text": "Acme Corp", "type": "ORG"}], + "relationships": [], +} + +OTHER_KG = { + "entities": [ + {"id": "https://example.org/e1", "text": "Acme Corp Renamed", "type": "ORG"} + ], + "relationships": [], +} + + +def _export(kg, tmp_path, name="out.jsonld", **options): + path = tmp_path / name + JSONExporter().export_knowledge_graph(kg, path, format="json-ld", **options) + return path + + +def test_reexporting_an_unchanged_graph_is_idempotent(tmp_path): + """The whole point of an identifier: same content, same @id.""" + first = json.loads(_export(KG, tmp_path, "a.jsonld").read_text()) + second = json.loads(_export(KG, tmp_path, "b.jsonld").read_text()) + + assert first["@id"] == second["@id"] + + +def test_a_changed_graph_gets_a_different_id(tmp_path): + unchanged = json.loads(_export(KG, tmp_path, "a.jsonld").read_text()) + changed = json.loads(_export(OTHER_KG, tmp_path, "b.jsonld").read_text()) + + assert unchanged["@id"] != changed["@id"] + + +def test_merging_repeated_exports_yields_one_graph_node(tmp_path): + """Regression for the exact repro in #1147: churn no longer multiplies nodes.""" + merged = Graph() + for i in range(3): + path = _export(KG, tmp_path, f"churn{i}.jsonld") + merged.parse(str(path), format="json-ld") + + # Exactly one subject typed as a KnowledgeGraph, regardless of how many + # times the unchanged graph was exported and merged. + kg_nodes = set( + merged.subjects(RDF.type, URIRef("https://semantica.dev/ns#KnowledgeGraph")) + ) + assert len(kg_nodes) == 1 + + entity_nodes = set( + merged.subjects(RDF.type, URIRef("https://semantica.dev/vocab/ORG")) + ) + assert len(entity_nodes) == 1 + + +def test_exported_at_still_varies_between_exports(tmp_path): + """Identity is now content-derived, but provenance still records each run.""" + first = json.loads(_export(KG, tmp_path, "a.jsonld").read_text()) + second = json.loads(_export(KG, tmp_path, "b.jsonld").read_text()) + + assert first["@id"] == second["@id"] + assert first["semantica:exportedAt"] != second["semantica:exportedAt"] + + +def test_caller_supplied_graph_uri_is_honored(tmp_path): + path = _export(KG, tmp_path, graph_uri="https://example.org/my-graph") + document = json.loads(path.read_text()) + + assert document["@id"] == "https://example.org/my-graph" + + +def _document_node_id(document): + """The generic (non-knowledge-graph) path hangs its own @id off a member + of @graph rather than the top level, to avoid re-creating the named-graph + bug fixed by #1145. Find that member and return its @id.""" + for node in document["@graph"]: + if "semantica:exportedAt" in node: + return node["@id"] + raise AssertionError(f"no document metadata node in @graph: {document}") + + +def test_caller_supplied_document_uri_is_honored_for_a_generic_export(tmp_path): + payload = {"note": "no entities or relationships here"} + path = tmp_path / "generic.jsonld" + JSONExporter().export( + payload, path, format="json-ld", document_uri="https://example.org/my-doc" + ) + document = json.loads(path.read_text()) + + assert _document_node_id(document) == "https://example.org/my-doc" + + +def test_generic_document_id_is_also_content_derived(tmp_path): + """The non-knowledge-graph path (_attach_document_metadata) gets the same fix.""" + payload = {"note": "plain data, no @id of its own"} + + first = tmp_path / "a.jsonld" + second = tmp_path / "b.jsonld" + JSONExporter().export(payload, first, format="json-ld") + JSONExporter().export(dict(payload), second, format="json-ld") + + first_id = _document_node_id(json.loads(first.read_text())) + second_id = _document_node_id(json.loads(second.read_text())) + assert first_id == second_id + + +def test_document_id_still_differs_for_different_generic_payloads(tmp_path): + a = tmp_path / "a.jsonld" + b = tmp_path / "b.jsonld" + JSONExporter().export({"note": "one"}, a, format="json-ld") + JSONExporter().export({"note": "two"}, b, format="json-ld") + + a_id = _document_node_id(json.loads(a.read_text())) + b_id = _document_node_id(json.loads(b.read_text())) + assert a_id != b_id diff --git a/tests/export/test_timestamp_timezones.py b/tests/export/test_timestamp_timezones.py index df34dd29..b97b658e 100644 --- a/tests/export/test_timestamp_timezones.py +++ b/tests/export/test_timestamp_timezones.py @@ -107,20 +107,31 @@ def test_exported_timestamp_survives_a_timezone_qualified_sparql_filter(): assert len(rows) == 1, "the export was dropped by a timezone-qualified filter" -def test_document_iri_carrying_an_offset_is_a_valid_iri(): - """The offset puts '+' and ':' in the @id; both are legal in a path.""" +def test_document_iri_is_a_valid_iri(): + """The graph @id must be a valid IRI regardless of how it is minted. + + Before #1147, this @id was minted from the offset-carrying timestamp + itself (``+00:00`` interpolated straight into the path), so this test + asserted the offset survived without breaking IRI validity. #1147 mints + the @id from the graph's content instead, so the timestamp no longer + appears here at all — it stays in ``semantica:exportedAt`` (still + offset-aware, per ``test_jsonld_export_timestamp_is_offset_aware`` above). + What's left worth guarding is the general case: whatever the @id is + minted from, it has to be a valid IRI that round-trips through RDF. + """ rdflib = pytest.importorskip("rdflib") document_iri = JSONExporter()._convert_kg_to_jsonld(KG)["@id"] - assert "+00:00" in document_iri assert rdflib.term._is_valid_uri(document_iri) graph = rdflib.Graph() - graph.add(( - rdflib.URIRef(document_iri), - rdflib.RDF.type, - rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"), - )) + graph.add( + ( + rdflib.URIRef(document_iri), + rdflib.RDF.type, + rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"), + ) + ) reparsed = rdflib.Graph().parse(data=graph.serialize(format="nt"), format="nt") assert document_iri in {str(s) for s in reparsed.subjects()}