From eb7427d12cfbdc4f8289cadc7491223cdf16c506 Mon Sep 17 00:00:00 2001 From: FABIOTESS Date: Fri, 21 Aug 2026 13:01:40 +0100 Subject: [PATCH 1/3] fix(export): carry metadata through every RDF serialization (#1154) convert_kg_to_rdf copies metadata into the RDF-ready dictionary at rdf_exporter.py:302 and no serializer has ever read it back out. Turtle, N-Triples, RDF/XML and RDFExporter's JSON-LD each write an entity's id, type, text and confidence and nothing else, so an entity keeps its confidence score and loses what produced it. JSONExporter's json-ld path keeps the same fields, which is how one knowledge graph exported two ways carried the user's data through one exporter and none through the other. Measured on e3405ebc with an entity carrying four metadata keys: 3 triples per format, 0 of them metadata. With this change: 7 triples per format, 4 of them metadata, and the same four in all four formats. The keys Semantica itself writes are mapped to declared terms in DEFAULT_METADATA_TERMS and declared in semantica-ns.ttl. A key the caller supplied is not: which namespace an arbitrary key belongs in is #1146, and that issue is open on the maintainer's modelling call, so the exporter warns and skips rather than inventing an IRI. Callers who already know the answer pass metadata_terms={key: iri}. Two keys cannot keep their own name. sem:source is already the ObjectProperty holding the subject of a reified relationship, so the Neo4j loader's "source" is written as sem:sourceSystem and its "uri" as sem:sourceUri, the one term whose value is a node rather than a literal. sem:builtAt and sem:snapshotAt have range xsd:string, not xsd:dateTime. GraphBuilder stamps with a timezone-naive datetime.now(), and #1114 is the demonstration of what typing such a value as xsd:dateTime costs: a timezone-qualified SPARQL filter over it silently drops the row. #1121 swept export and provenance and deliberately left kg/ alone. Graph-level metadata is written only when the caller names the graph with graph_uri, because this serializer has never minted a document node and #1147 is where that default belongs once it lands. The lexical form and datatype of a value are chosen once, in _typed_literal_parts, so the four serializers cannot come to disagree about them the way they disagreed about confidence in #1100. The JSON-LD path writes explicit @value/@type rather than JSON's native numbers, which would have made an integer xsd:double there and xsd:integer everywhere else. 21 tests, asserting on the parsed graph in all four formats. Output is unchanged when no metadata is present. Full-suite failure set is identical to the parent commit: 512 = 512. --- semantica/export/rdf_exporter.py | 311 +++++++++++++++++- .../ontology/vocabulary/semantica-ns.ttl | 84 +++++ tests/export/test_metadata_passthrough.py | 228 +++++++++++++ 3 files changed, 613 insertions(+), 10 deletions(-) create mode 100644 tests/export/test_metadata_passthrough.py diff --git a/semantica/export/rdf_exporter.py b/semantica/export/rdf_exporter.py index de3d0dd0..af5a21c3 100644 --- a/semantica/export/rdf_exporter.py +++ b/semantica/export/rdf_exporter.py @@ -69,6 +69,187 @@ def mint_relationship_iri(index: int, source: Any, target: Any) -> str: return f"{SEMANTICA_NS}rel_{index}_{digest}" +#: The metadata keys Semantica itself produces, and the terms they are written +#: as. GraphBuilder.build_graph writes the first five, create_snapshot writes +#: snapshot_time, and load_from_neo4j writes source / uri / database. These are +#: Semantica's own vocabulary, so they are minted in the declared namespace and +#: declared in semantica-ns.ttl. +#: +#: A key the caller supplied is a different matter. Which namespace an +#: arbitrary metadata key belongs in is issue #1146, and until that is settled +#: the exporter refuses to guess: it warns and skips, and a caller who already +#: knows the answer passes ``metadata_terms``. +#: +#: The map is key -> term rather than key -> namespace because two of the keys +#: cannot keep their own name. ``source`` on a graph loaded from Neo4j is the +#: system it came from, while sem:source is already the ObjectProperty holding +#: the subject of a reified relationship; reusing it would put a string where +#: an entity belongs. +DEFAULT_METADATA_TERMS: Dict[str, str] = { + "num_entities": f"{SEMANTICA_NS}numEntities", + "num_relationships": f"{SEMANTICA_NS}numRelationships", + "temporal_enabled": f"{SEMANTICA_NS}temporalEnabled", + "entity_resolution_applied": f"{SEMANTICA_NS}entityResolutionApplied", + "timestamp": f"{SEMANTICA_NS}builtAt", + "snapshot_time": f"{SEMANTICA_NS}snapshotAt", + "source": f"{SEMANTICA_NS}sourceSystem", + "uri": f"{SEMANTICA_NS}sourceUri", + "database": f"{SEMANTICA_NS}sourceDatabase", +} + +#: Terms whose value is a node rather than a string. Everything else stays a +#: literal: a metadata value that merely looks like a URL is not thereby a +#: reference to one. +IRI_VALUED_METADATA_TERMS: Set[str] = {f"{SEMANTICA_NS}sourceUri"} + +_XSD_NS = "http://www.w3.org/2001/XMLSchema#" + + +def _escape_literal(value: str) -> str: + """Escape a string for a Turtle or N-Triples quoted literal.""" + return ( + value.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + ) + + +def _escape_xml(value: str) -> str: + return value.replace("&", "&").replace("<", "<").replace(">", ">") + + +def _split_iri(iri: str) -> Optional[tuple]: + """Split an IRI into (namespace, local name) for RDF/XML's QName syntax.""" + for sep in ("#", "/"): + index = iri.rfind(sep) + if index != -1 and index + 1 < len(iri): + local = iri[index + 1 :] + if local and not local[0].isdigit(): + return iri[: index + 1], local + return None + + +def _metadata_statements( + metadata: Any, + terms: Dict[str, str], + logger: Any, +) -> List[tuple]: + """Resolve a metadata mapping to a list of (term IRI, value) pairs. + + A key with no term is skipped and reported. Silence is the defect this + fixes, so an unmapped key must be louder than a mapped one, not quieter. + """ + if not isinstance(metadata, dict): + return [] + + statements: List[tuple] = [] + for key, value in metadata.items(): + term = terms.get(key) + if term is None: + logger.warning( + "Metadata key %r has no term and was not exported. Which " + "namespace a caller-supplied key belongs in is issue #1146; " + "pass metadata_terms={%r: ''} to export it now.", + key, + key, + ) + continue + if value is None: + continue + if isinstance(value, (dict, list, tuple, set)): + logger.warning( + "Metadata key %r holds a %s, which has no modelled RDF shape " + "yet, and was not exported.", + key, + type(value).__name__, + ) + continue + statements.append((term, value)) + return statements + + +def _resolve_metadata_terms(overrides: Optional[Dict[str, str]]) -> Dict[str, str]: + if not overrides: + return DEFAULT_METADATA_TERMS + return {**DEFAULT_METADATA_TERMS, **overrides} + + +def _typed_literal_parts(term: str, value: Any) -> tuple: + """Return (kind, lexical, datatype) for one metadata value. + + kind is "iri" or "literal". The lexical form and datatype are chosen once, + here, so that the four serializers cannot disagree about them the way they + disagreed about confidence in #1100. + """ + if term in IRI_VALUED_METADATA_TERMS and isinstance(value, str): + return "iri", value, None + if isinstance(value, bool): + return "literal", "true" if value else "false", f"{_XSD_NS}boolean" + if isinstance(value, int): + return "literal", str(value), f"{_XSD_NS}integer" + if isinstance(value, float): + return "literal", repr(value), f"{_XSD_NS}decimal" + return "literal", str(value), None + + +def _turtle_object(term: str, value: Any) -> str: + kind, lexical, datatype = _typed_literal_parts(term, value) + if kind == "iri": + return f"<{lexical}>" + if datatype is None: + return f'"{_escape_literal(lexical)}"' + return f'"{lexical}"^^<{datatype}>' + + +def _turtle_metadata_clauses(statements: List[tuple]) -> List[str]: + return [f"<{term}> {_turtle_object(term, value)}" for term, value in statements] + + +def _ntriples_metadata_lines(subject: str, statements: List[tuple]) -> List[str]: + return [ + f"<{subject}> <{term}> {_turtle_object(term, value)} ." + for term, value in statements + ] + + +def _rdfxml_metadata_lines(statements: List[tuple], indent: str) -> List[str]: + """RDF/XML needs a QName, so an unprefixed term declares its own prefix.""" + lines: List[str] = [] + for position, (term, value) in enumerate(statements): + split = _split_iri(term) + if split is None: + continue + namespace, local = split + kind, lexical, datatype = _typed_literal_parts(term, value) + prefix = f"md{position}" + opening = f'{indent}<{prefix}:{local} xmlns:{prefix}="{_escape_xml(namespace)}"' + if kind == "iri": + lines.append(f'{opening} rdf:resource="{_escape_xml(lexical)}"/>') + continue + if datatype is not None: + opening += f' rdf:datatype="{_escape_xml(datatype)}"' + lines.append(f"{opening}>{_escape_xml(lexical)}") + return lines + + +def _jsonld_metadata_entries(statements: List[tuple]) -> Dict[str, Any]: + """Absolute IRIs as keys, and explicit @value/@type rather than JSON's own + types: JSON's number is xsd:double, which would make the JSON-LD export + disagree with the other three about the datatype of an integer.""" + entries: Dict[str, Any] = {} + for term, value in statements: + kind, lexical, datatype = _typed_literal_parts(term, value) + if kind == "iri": + entries[term] = {"@id": lexical} + elif datatype is None: + entries[term] = lexical + else: + entries[term] = {"@value": lexical, "@type": datatype} + return entries + + class NamespaceManager: """ RDF namespace management engine. @@ -364,6 +545,8 @@ class RDFSerializer: """ include_temporal: bool = options.pop("include_temporal", False) time_axis: str = options.pop("time_axis", "valid") + metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None)) + graph_uri: Optional[str] = options.pop("graph_uri", None) lines = [] @@ -397,9 +580,23 @@ class RDFSerializer: text = entity.get("text") or entity.get("label", "") confidence = entity.get("confidence", 1.0) - lines.append(f"<{entity_id}> a <{entity_type}> ;") - lines.append(f' semantica:text "{text}" ;') - lines.append(f" semantica:confidence {confidence} .") + clauses = [ + f"a <{entity_type}>", + f'semantica:text "{text}"', + f"semantica:confidence {confidence}", + ] + clauses.extend( + _turtle_metadata_clauses( + _metadata_statements( + entity.get("metadata"), metadata_terms, self.logger + ) + ) + ) + + lines.append(f"<{entity_id}> {clauses[0]} ;") + for clause in clauses[1:-1]: + lines.append(f" {clause} ;") + lines.append(f" {clauses[-1]} .") lines.append("") # Convert relationships to RDF triplets @@ -416,6 +613,30 @@ class RDFSerializer: if owl_lines: lines.extend(owl_lines) + # Graph-level metadata needs a subject, and this serializer has never + # minted a document node. Rather than invent one here, it is written + # only when the caller names the graph; issue #1147 is where the + # default subject comes from once that lands. + graph_clauses = ( + _turtle_metadata_clauses( + _metadata_statements( + rdf_data.get("metadata"), metadata_terms, self.logger + ) + ) + if graph_uri + else [] + ) + if graph_clauses: + lines.append("") + lines.append( + f"<{graph_uri}> {graph_clauses[0]} " + + (";" if len(graph_clauses) > 1 else ".") + ) + for clause in graph_clauses[1:-1]: + lines.append(f" {clause} ;") + if len(graph_clauses) > 1: + lines.append(f" {graph_clauses[-1]} .") + return "\n".join(lines) def _owl_time_triples_for_rel( @@ -510,6 +731,9 @@ class RDFSerializer: ... } >>> rdfxml = serializer.serialize_to_rdfxml(rdf_data) """ + metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None)) + graph_uri: Optional[str] = options.pop("graph_uri", None) + lines = [''] lines.append('{confidence}" ) + lines.extend( + _rdfxml_metadata_lines( + _metadata_statements( + entity.get("metadata"), metadata_terms, self.logger + ), + " ", + ) + ) lines.append(" ") lines.append("") @@ -552,6 +784,22 @@ class RDFSerializer: lines.append(" ") lines.append("") + graph_lines = ( + _rdfxml_metadata_lines( + _metadata_statements( + rdf_data.get("metadata"), metadata_terms, self.logger + ), + " ", + ) + if graph_uri + else [] + ) + if graph_lines: + lines.append(f' ') + lines.extend(graph_lines) + lines.append(" ") + lines.append("") + lines.append("") return "\n".join(lines) @@ -582,6 +830,9 @@ class RDFSerializer: """ import json + metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None)) + graph_uri: Optional[str] = options.pop("graph_uri", None) + # Initialize JSON-LD structure with context jsonld = { "@context": { @@ -608,14 +859,20 @@ class RDFSerializer: # and was dropped in full by a JSON-LD parser, silently. entity_id = entity.get("id") or mint_entity_iri(entity.get("text", "")) - jsonld["@graph"].append( - { - "@id": entity_id, - "@type": entity.get("type", "semantica:Entity"), - "semantica:text": entity.get("text") or entity.get("label", ""), - "semantica:confidence": entity.get("confidence", 1.0), - } + node = { + "@id": entity_id, + "@type": entity.get("type", "semantica:Entity"), + "semantica:text": entity.get("text") or entity.get("label", ""), + "semantica:confidence": entity.get("confidence", 1.0), + } + node.update( + _jsonld_metadata_entries( + _metadata_statements( + entity.get("metadata"), metadata_terms, self.logger + ) + ) ) + jsonld["@graph"].append(node) # Convert relationships to JSON-LD relationships = rdf_data.get("relationships", []) @@ -639,6 +896,18 @@ class RDFSerializer: } ) + graph_entries = ( + _jsonld_metadata_entries( + _metadata_statements( + rdf_data.get("metadata"), metadata_terms, self.logger + ) + ) + if graph_uri + else {} + ) + if graph_entries: + jsonld["@graph"].append({"@id": graph_uri, **graph_entries}) + return json.dumps(jsonld, indent=2, ensure_ascii=False) def serialize_to_ntriples(self, rdf_data: Dict[str, Any], **options) -> str: @@ -655,6 +924,9 @@ class RDFSerializer: Returns: String containing N-Triples serialization """ + metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None)) + graph_uri: Optional[str] = options.pop("graph_uri", None) + lines = [] def expand_uri(uri: str) -> str: @@ -704,6 +976,15 @@ class RDFSerializer: f'{subject} {expand_uri("semantica:confidence")} "{confidence}"^^ .' ) + lines.extend( + _ntriples_metadata_lines( + subject.strip("<>"), + _metadata_statements( + entity.get("metadata"), metadata_terms, self.logger + ), + ) + ) + # Convert relationships relationships = rdf_data.get("relationships", []) for rel in relationships: @@ -716,6 +997,16 @@ class RDFSerializer: f"{expand_uri(source_id)} {expand_uri(rel_type)} {expand_uri(target_id)} ." ) + if graph_uri: + lines.extend( + _ntriples_metadata_lines( + graph_uri, + _metadata_statements( + rdf_data.get("metadata"), metadata_terms, self.logger + ), + ) + ) + return "\n".join(lines) diff --git a/semantica/ontology/vocabulary/semantica-ns.ttl b/semantica/ontology/vocabulary/semantica-ns.ttl index 36e206bc..8dba7fee 100644 --- a/semantica/ontology/vocabulary/semantica-ns.ttl +++ b/semantica/ontology/vocabulary/semantica-ns.ttl @@ -134,6 +134,90 @@ JSONExporter.export_to_jsonld in export/json_exporter.py.""" ; rdfs:range xsd:string ; rdfs:isDefinedBy . +# ── Metadata carried through from the graph builder ────────────────────────── +# +# The keys GraphBuilder and the Neo4j loader write into "metadata". Declared +# here because the RDF serializers emit them (#1154); a caller-supplied key is +# not declared here and is not emitted, because which namespace it belongs in +# is #1146. + +sem:numEntities a owl:DatatypeProperty ; + rdfs:label "number of entities" ; + rdfs:comment """Count of entities in the graph as built, from +GraphBuilder.build_graph. A count of what was built, not a constraint on what +the graph contains: an export filtered after the fact will disagree with it.""" ; + rdfs:range xsd:integer ; + rdfs:isDefinedBy . + +sem:numRelationships a owl:DatatypeProperty ; + rdfs:label "number of relationships" ; + rdfs:comment "Count of relationships in the graph as built." ; + rdfs:range xsd:integer ; + rdfs:isDefinedBy . + +sem:temporalEnabled a owl:DatatypeProperty ; + rdfs:label "temporal enabled" ; + rdfs:comment """True when the builder was configured to track valid time. +False does not mean the graph is untimed; it means no temporal bounds were +recorded for it.""" ; + rdfs:range xsd:boolean ; + rdfs:isDefinedBy . + +sem:entityResolutionApplied a owl:DatatypeProperty ; + rdfs:label "entity resolution applied" ; + rdfs:comment """True when a resolver ran over the extracted entities, so a +consumer knows whether two nodes with the same surface text were ever +considered for merging.""" ; + rdfs:range xsd:boolean ; + rdfs:isDefinedBy . + +sem:builtAt a owl:DatatypeProperty ; + rdfs:label "built at" ; + rdfs:comment """When the graph was built, as GraphBuilder recorded it. + +The range is xsd:string, deliberately, and not xsd:dateTime. GraphBuilder +stamps with a timezone-naive datetime.now(), and #1114 is the demonstration of +what typing such a value as xsd:dateTime costs: a timezone-qualified SPARQL +filter over it raises an indeterminate comparison and silently drops the row. +#1121 swept the export and provenance modules to an explicit UTC offset and +deliberately left kg/ alone, because the context and vector-store modules +compare against naive values already on disk. Until that sweep reaches +GraphBuilder this value is a string that looks like a timestamp, and saying so +is more useful than a type that invites arithmetic it cannot support.""" ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + +sem:snapshotAt a owl:DatatypeProperty ; + rdfs:label "snapshot at" ; + rdfs:comment """The point in time a snapshot represents, from +GraphBuilder.create_snapshot. A string for the same reason as sem:builtAt.""" ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + +sem:sourceSystem a owl:DatatypeProperty ; + rdfs:label "source system" ; + rdfs:comment """The system a graph was loaded from, currently the literal +"neo4j" written by GraphBuilder.load_from_neo4j. + +Named sourceSystem rather than source because sem:source is already the +ObjectProperty carrying the subject of a reified relationship. The metadata key +is still "source"; the exporter maps the key to this term.""" ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + +sem:sourceUri a owl:ObjectProperty ; + rdfs:label "source URI" ; + rdfs:comment """The address of the system a graph was loaded from. The one +metadata term whose value is a node rather than a literal, because it names a +thing rather than describing one.""" ; + rdfs:isDefinedBy . + +sem:sourceDatabase a owl:DatatypeProperty ; + rdfs:label "source database" ; + rdfs:comment "The database within the source system a graph was loaded from." ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + # ── Temporal term (OWL-Time export) ────────────────────────────────────────── sem:openEndedInterval a owl:DatatypeProperty ; diff --git a/tests/export/test_metadata_passthrough.py b/tests/export/test_metadata_passthrough.py new file mode 100644 index 00000000..15fbcb7d --- /dev/null +++ b/tests/export/test_metadata_passthrough.py @@ -0,0 +1,228 @@ +"""Metadata must survive serialization (issue #1154). + +``convert_kg_to_rdf`` copies ``metadata`` into the RDF-ready dictionary at +rdf_exporter.py:302, and no serializer has ever read it back out. Turtle, +N-Triples, RDF/XML and RDFExporter's JSON-LD all write the entity's id, type, +text and confidence, and none of them writes a single metadata statement, so an +entity keeps its confidence score and loses what produced it: the source +document, the page, the extractor, the reviewer. JSONExporter's json-ld path +keeps all of them, which is how the same knowledge graph exported two ways came +to carry ten triples of user data through one exporter and none through the +other. + +The keys Semantica itself produces (GraphBuilder writes num_entities, +num_relationships, temporal_enabled, timestamp and entity_resolution_applied; +the Neo4j loader writes source, uri and database) are Semantica's own +vocabulary, so they are minted in the declared namespace and declared in +semantica-ns.ttl. Keys the caller supplied are not: which namespace those +belong in is issue #1146, and until that is settled the exporter refuses to +guess rather than inventing an IRI, warns, and takes an explicit +``metadata_terms`` mapping from any caller who already knows the answer. +""" + +import json + +import pytest +from rdflib import Graph, Literal, URIRef +from rdflib.namespace import XSD + +from semantica.export.rdf_exporter import ( + DEFAULT_METADATA_TERMS, + RDFSerializer, + SEMANTICA_NS, + mint_entity_iri, +) + +ENTITY_IRI = "https://example.org/e1" + +# The provenance fields the issue names, plus one key Semantica itself writes. +GRAPH_WITH_METADATA = { + "entities": [ + { + "id": ENTITY_IRI, + "type": "https://example.org/Org", + "text": "Acme Corp", + "confidence": 0.91, + "metadata": {"num_entities": 1, "temporal_enabled": True}, + } + ], + "relationships": [], + "metadata": { + "num_entities": 1, + "num_relationships": 0, + "temporal_enabled": False, + "entity_resolution_applied": True, + }, +} + +NUM_ENTITIES = URIRef(f"{SEMANTICA_NS}numEntities") +TEMPORAL_ENABLED = URIRef(f"{SEMANTICA_NS}temporalEnabled") + + +def _parse(text: str, fmt: str) -> Graph: + """Assert on the parsed graph, never on the serialized text.""" + g = Graph() + g.parse(data=text, format=fmt) + return g + + +def _serialize(serializer: RDFSerializer, fmt: str, data, **options) -> Graph: + method, parse_as = { + "turtle": (serializer.serialize_to_turtle, "turtle"), + "ntriples": (serializer.serialize_to_ntriples, "nt"), + "rdfxml": (serializer.serialize_to_rdfxml, "xml"), + "jsonld": (serializer.serialize_to_jsonld, "json-ld"), + }[fmt] + return _parse(method(data, **options), parse_as) + + +FORMATS = ["turtle", "ntriples", "rdfxml", "jsonld"] + + +@pytest.mark.parametrize("fmt", FORMATS) +def test_entity_metadata_reaches_every_serialization(fmt): + """The headline defect: the statement is absent from all four formats.""" + g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA) + assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(1)) in g + + +@pytest.mark.parametrize("fmt", FORMATS) +def test_entity_metadata_booleans_keep_their_datatype(fmt): + g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA) + assert (URIRef(ENTITY_IRI), TEMPORAL_ENABLED, Literal(True)) in g + + +def test_every_format_writes_the_same_metadata_triples(): + """A value must not change datatype with the serializer, as #1100 found.""" + per_format = {} + for fmt in FORMATS: + g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA) + per_format[fmt] = { + (p, o) for s, p, o in g if str(p).startswith(SEMANTICA_NS) and "numEntities" in str(p) + } + assert len(set(map(frozenset, per_format.values()))) == 1, per_format + + +def test_graph_metadata_needs_a_subject_the_caller_named(): + """Graph-level metadata hangs off graph_uri; #1147 owns the default.""" + doc = URIRef("https://example.org/graph/1") + g = _serialize( + RDFSerializer(), + "turtle", + GRAPH_WITH_METADATA, + graph_uri=str(doc), + ) + assert (doc, NUM_ENTITIES, Literal(1)) in g + assert (doc, URIRef(f"{SEMANTICA_NS}entityResolutionApplied"), Literal(True)) in g + + +def test_graph_metadata_is_not_invented_without_a_subject(): + g = _serialize(RDFSerializer(), "turtle", GRAPH_WITH_METADATA) + assert not list(g.subjects(NUM_ENTITIES, Literal(0))) + # the entity keeps its own metadata; only the graph-level block waits + assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(1)) in g + + +def test_an_unknown_key_is_refused_out_loud_not_dropped_in_silence(caplog): + """#1146 owns which namespace a caller's key belongs in. Until then: warn.""" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}} + ], + "relationships": [], + } + with caplog.at_level("WARNING"): + g = _serialize(RDFSerializer(), "turtle", data) + assert not any("reviewed_by" in str(p) for p in g.predicates()) + assert any("reviewed_by" in r.getMessage() for r in caplog.records) + assert any("1146" in r.getMessage() for r in caplog.records) + + +@pytest.mark.parametrize("fmt", FORMATS) +def test_a_caller_who_knows_the_answer_can_supply_the_term(fmt): + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}} + ], + "relationships": [], + } + terms = {"reviewed_by": "http://purl.org/dc/terms/creator"} + g = _serialize(RDFSerializer(), fmt, data, metadata_terms=terms) + assert ( + URIRef(ENTITY_IRI), + URIRef("http://purl.org/dc/terms/creator"), + Literal("fabio"), + ) in g + + +def test_a_literal_with_a_quote_or_newline_still_parses(): + """Metadata is user text; #1098 is the same class of defect one field over.""" + data = { + "entities": [ + { + "id": ENTITY_IRI, + "text": "Acme", + "metadata": {"source": 'the "Q3" report\nsecond line'}, + } + ], + "relationships": [], + } + for fmt in FORMATS: + g = _serialize(RDFSerializer(), fmt, data) + assert ( + URIRef(ENTITY_IRI), + URIRef(f"{SEMANTICA_NS}sourceSystem"), + Literal('the "Q3" report\nsecond line'), + ) in g + + +def test_an_iri_valued_key_is_written_as_a_node_not_a_string(): + """The Neo4j loader's ``uri`` key. Note the term is sem:sourceUri, not + sem:uri: the key names a field, the term names a relation.""" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"uri": "https://example.org/db"}} + ], + "relationships": [], + } + g = _serialize(RDFSerializer(), "turtle", data) + assert ( + URIRef(ENTITY_IRI), + URIRef(f"{SEMANTICA_NS}sourceUri"), + URIRef("https://example.org/db"), + ) in g + + +def test_output_is_unchanged_when_no_metadata_is_present(): + plain = { + "entities": [{"id": ENTITY_IRI, "type": "https://example.org/Org", "text": "Acme"}], + "relationships": [{"source_id": ENTITY_IRI, "target_id": "https://example.org/e2"}], + } + serializer = RDFSerializer() + assert serializer.serialize_to_turtle(plain) == serializer.serialize_to_turtle(plain) + g = _parse(serializer.serialize_to_turtle(plain), "turtle") + assert len(g) == 4 + + +def test_every_default_term_is_declared_in_the_shipped_vocabulary(): + """Drift guard: a term the exporter emits and the vocabulary omits is a bug.""" + from semantica.ontology.vocabulary import vocabulary_path + + vocab = Graph() + vocab.parse(vocabulary_path(), format="turtle") + declared = {str(s) for s in vocab.subjects()} + missing = sorted(set(DEFAULT_METADATA_TERMS.values()) - declared) + assert not missing, f"emitted but undeclared: {missing}" + + +def test_jsonld_metadata_survives_a_real_jsonld_processor(): + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"num_entities": 3}} + ], + "relationships": [], + } + raw = RDFSerializer().serialize_to_jsonld(data) + json.loads(raw) # must be valid JSON before it can be valid JSON-LD + g = _parse(raw, "json-ld") + assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(3)) in g From 1a220da477dc679db45dbcf7c18c9729249f3ba8 Mon Sep 17 00:00:00 2001 From: Fabio Rovai Date: Fri, 21 Aug 2026 14:43:02 +0100 Subject: [PATCH 2/3] fix(export): address the review findings on the metadata pass-through --- semantica/export/rdf_exporter.py | 103 +++++++++++++++-- tests/export/test_metadata_passthrough.py | 135 +++++++++++++++++++++- 2 files changed, 223 insertions(+), 15 deletions(-) diff --git a/semantica/export/rdf_exporter.py b/semantica/export/rdf_exporter.py index 6f07d619..ce64a3a2 100644 --- a/semantica/export/rdf_exporter.py +++ b/semantica/export/rdf_exporter.py @@ -187,16 +187,53 @@ def _escape_literal(value: str) -> str: def _escape_xml(value: str) -> str: - return value.replace("&", "&").replace("<", "<").replace(">", ">") + """Escape a string for either XML element text or an attribute value. + + The quotes matter. This helper feeds `rdf:about`, `rdf:resource` and + `xmlns:` attribute values, which are delimited by double quotes, so a value + carrying one would close the attribute early and produce a document that + does not parse. Escaping them in element text as well is harmless and + means one helper cannot be used in the wrong place. + """ + return ( + value.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + + +def _is_ncname(value: str) -> bool: + """Whether a string can be an XML NCName, which is what RDF/XML requires. + + Checked over the ASCII range rather than the full XML production: the + grammar also admits combining characters and extenders, so this is + deliberately conservative. It refuses names it could have accepted, and it + never accepts one that would produce a document a parser rejects. The + earlier check tested only that the first character was not a digit, which + let through every other way a local name can fail to be a name. + """ + if not value: + return False + if not (value[0].isascii() and (value[0].isalpha() or value[0] == "_")): + return False + return all(c.isascii() and (c.isalnum() or c in "._-") for c in value[1:]) def _split_iri(iri: str) -> Optional[tuple]: - """Split an IRI into (namespace, local name) for RDF/XML's QName syntax.""" + """Split an IRI into (namespace, local name) for RDF/XML's QName syntax. + + Returns None when no split yields a usable local name. RDF/XML is the only + serialization here that cannot write an arbitrary predicate IRI, so this is + the one place a term can be unrepresentable, and the caller reports it + rather than dropping it quietly. + """ for sep in ("#", "/"): index = iri.rfind(sep) if index != -1 and index + 1 < len(iri): local = iri[index + 1 :] - if local and not local[0].isdigit(): + if _is_ncname(local): return iri[: index + 1], local return None @@ -260,7 +297,24 @@ def _typed_literal_parts(term: str, value: Any) -> tuple: if isinstance(value, int): return "literal", str(value), f"{_XSD_NS}integer" if isinstance(value, float): - return "literal", repr(value), f"{_XSD_NS}decimal" + # xsd:double, not xsd:decimal. `repr(1e-05)` is "1e-05" and + # `repr(float("nan"))` is "nan", and xsd:decimal admits neither the + # exponent form nor the special values, so typing a float as decimal + # produced lexicals a strict parser rejects. A Python float is an IEEE + # 754 double; xsd:double has legal lexicals for all of them, and it is + # also the honest claim, since nothing that arrived as a float was ever + # exact. `normalize_confidence` keeps xsd:decimal for confidence + # deliberately: that is a bounded score where exactness is meaningful + # and NaN is not a confidence at all. + if value != value: + lexical = "NaN" + elif value == float("inf"): + lexical = "INF" + elif value == float("-inf"): + lexical = "-INF" + else: + lexical = repr(value) + return "literal", lexical, f"{_XSD_NS}double" return "literal", str(value), None @@ -284,12 +338,28 @@ def _ntriples_metadata_lines(subject: str, statements: List[tuple]) -> List[str] ] -def _rdfxml_metadata_lines(statements: List[tuple], indent: str) -> List[str]: - """RDF/XML needs a QName, so an unprefixed term declares its own prefix.""" +def _rdfxml_metadata_lines( + statements: List[tuple], indent: str, logger: Any = None +) -> List[str]: + """RDF/XML needs a QName, so an unprefixed term declares its own prefix. + + A term with no QName form has no RDF/XML representation at all, and this is + the only serialization with that restriction. Skipping it quietly would + reintroduce, in one format, exactly the silent metadata loss this module + was changed to stop, so it is reported and the other three formats still + carry the statement in full. + """ lines: List[str] = [] for position, (term, value) in enumerate(statements): split = _split_iri(term) if split is None: + if logger is not None: + logger.warning( + "Term %r has no QName form, so it cannot be written in " + "RDF/XML and was omitted from that serialization only. " + "Turtle, N-Triples and JSON-LD carry it in full.", + term, + ) continue namespace, local = split kind, lexical, datatype = _typed_literal_parts(term, value) @@ -881,8 +951,15 @@ class RDFSerializer: confidence = normalize_confidence(entity.get("confidence", 1.0)) # RDF/XML syntax: rdf:Description with rdf:about - lines.append(f' ') - lines.append(f' ') + # Attribute values are delimited by quotes, and both of these + # are caller input. Element text is left alone deliberately: that + # is #1098, and it is being fixed on its own path. + lines.append( + f' ' + ) + lines.append( + f' ' + ) lines.append(f" {text}") if confidence is None: self.logger.warning( @@ -900,6 +977,7 @@ class RDFSerializer: entity.get("metadata"), metadata_terms, self.logger ), " ", + self.logger, ) ) lines.append(" ") @@ -913,8 +991,12 @@ class RDFSerializer: rel_type = rel.get("type", "semantica:related_to") # Relationship as property on source entity - lines.append(f' ') - lines.append(f' <{rel_type} rdf:resource="{target_id}"/>') + lines.append( + f' ' + ) + lines.append( + f' <{rel_type} rdf:resource="{_escape_xml(str(target_id))}"/>' + ) lines.append(" ") lines.append("") @@ -924,6 +1006,7 @@ class RDFSerializer: rdf_data.get("metadata"), metadata_terms, self.logger ), " ", + self.logger, ) if graph_uri else [] diff --git a/tests/export/test_metadata_passthrough.py b/tests/export/test_metadata_passthrough.py index 15fbcb7d..cdb73227 100644 --- a/tests/export/test_metadata_passthrough.py +++ b/tests/export/test_metadata_passthrough.py @@ -98,7 +98,9 @@ def test_every_format_writes_the_same_metadata_triples(): for fmt in FORMATS: g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA) per_format[fmt] = { - (p, o) for s, p, o in g if str(p).startswith(SEMANTICA_NS) and "numEntities" in str(p) + (p, o) + for s, p, o in g + if str(p).startswith(SEMANTICA_NS) and "numEntities" in str(p) } assert len(set(map(frozenset, per_format.values()))) == 1, per_format @@ -181,7 +183,11 @@ def test_an_iri_valued_key_is_written_as_a_node_not_a_string(): sem:uri: the key names a field, the term names a relation.""" data = { "entities": [ - {"id": ENTITY_IRI, "text": "Acme", "metadata": {"uri": "https://example.org/db"}} + { + "id": ENTITY_IRI, + "text": "Acme", + "metadata": {"uri": "https://example.org/db"}, + } ], "relationships": [], } @@ -195,11 +201,17 @@ def test_an_iri_valued_key_is_written_as_a_node_not_a_string(): def test_output_is_unchanged_when_no_metadata_is_present(): plain = { - "entities": [{"id": ENTITY_IRI, "type": "https://example.org/Org", "text": "Acme"}], - "relationships": [{"source_id": ENTITY_IRI, "target_id": "https://example.org/e2"}], + "entities": [ + {"id": ENTITY_IRI, "type": "https://example.org/Org", "text": "Acme"} + ], + "relationships": [ + {"source_id": ENTITY_IRI, "target_id": "https://example.org/e2"} + ], } serializer = RDFSerializer() - assert serializer.serialize_to_turtle(plain) == serializer.serialize_to_turtle(plain) + assert serializer.serialize_to_turtle(plain) == serializer.serialize_to_turtle( + plain + ) g = _parse(serializer.serialize_to_turtle(plain), "turtle") assert len(g) == 4 @@ -226,3 +238,116 @@ def test_jsonld_metadata_survives_a_real_jsonld_processor(): json.loads(raw) # must be valid JSON before it can be valid JSON-LD g = _parse(raw, "json-ld") assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(3)) in g + + +# --- Findings from the Qodo review of PR #1165 ----------------------------- + + +@pytest.mark.parametrize("fmt", FORMATS) +@pytest.mark.parametrize( + "value", [1e-05, 1e300, 0.1, -0.0, float("nan"), float("inf"), float("-inf")] +) +def test_a_float_metadata_value_is_a_double_and_keeps_a_legal_lexical(fmt, value): + """`repr()` of a float is not an xsd:decimal lexical. + + `repr(1e-05)` is "1e-05" and `repr(float("nan"))` is "nan", neither of which + xsd:decimal admits, so typing a float as decimal produced RDF a strict + parser rejects. A Python float is an IEEE 754 double, xsd:double has legal + lexicals for the exponent form and for the three special values, and saying + double is also the honest claim: nothing here was ever exact. + """ + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"num_entities": value}} + ], + "relationships": [], + } + g = _serialize(RDFSerializer(), fmt, data) + objects = list(g.objects(URIRef(ENTITY_IRI), NUM_ENTITIES)) + assert len(objects) == 1, f"{fmt}: {objects}" + (written,) = objects + assert written.datatype == XSD.double, written.datatype + parsed = written.toPython() + if value != value: # NaN + assert parsed != parsed + else: + assert parsed == value + + +def test_every_format_agrees_on_a_float_metadata_value(): + per_format = {} + for fmt in FORMATS: + g = _serialize( + RDFSerializer(), + fmt, + { + "entities": [ + {"id": ENTITY_IRI, "text": "A", "metadata": {"num_entities": 1e-05}} + ], + "relationships": [], + }, + ) + per_format[fmt] = {(p, o) for s, p, o in g if p == NUM_ENTITIES} + assert len(set(map(frozenset, per_format.values()))) == 1, per_format + + +def test_a_term_rdfxml_cannot_name_is_refused_out_loud(caplog): + """RDF/XML needs a QName, and the PR's whole point is no silent drops. + + A term whose local part is not an XML NCName has no RDF/XML form at all. + Skipping it quietly reintroduces, in one format, exactly the loss this + change exists to stop. + """ + unnameable = "http://example.org/ns/123" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}} + ], + "relationships": [], + } + with caplog.at_level("WARNING"): + xml = RDFSerializer().serialize_to_rdfxml( + data, metadata_terms={"reviewed_by": unnameable} + ) + _parse(xml, "xml") # must still be well-formed + messages = " ".join(r.getMessage() for r in caplog.records) + assert unnameable in messages + assert "RDF/XML" in messages + + +@pytest.mark.parametrize("fmt", ["turtle", "ntriples", "jsonld"]) +def test_the_other_formats_still_carry_a_term_rdfxml_cannot_name(fmt): + """Only RDF/XML has the QName restriction; the rest write the full IRI.""" + unnameable = "http://example.org/ns/123" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}} + ], + "relationships": [], + } + g = _serialize( + RDFSerializer(), fmt, data, metadata_terms={"reviewed_by": unnameable} + ) + assert (URIRef(ENTITY_IRI), URIRef(unnameable), Literal("fabio")) in g + + +def test_a_quote_in_an_attribute_value_cannot_break_the_document(): + """`_escape_xml` feeds attribute values, which are delimited by quotes. + + Escaping only &, < and > leaves a caller-supplied value able to close the + attribute early and produce XML that does not parse. + """ + data = { + "entities": [ + { + "id": 'https://example.org/e"1', + "text": "Acme", + "metadata": {"uri": 'https://example.org/db"x'}, + } + ], + "relationships": [], + } + xml = RDFSerializer().serialize_to_rdfxml(data) + from xml.dom.minidom import parseString + + parseString(xml) # well-formedness is the assertion From 220fb10e5c160761587b8398a80713d285420a8a Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Mon, 24 Aug 2026 13:38:07 +0530 Subject: [PATCH 3/3] fix(export): escape IRI-valued metadata to close a Turtle/N-Triples injection gap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _turtle_object() wrote an IRI-valued metadata value (currently only sem:sourceUri, from the "uri" metadata key) straight into `<{value}>` with no escaping. Turtle/N-Triples IRIREFs exclude control characters, space, and <>"{}|^`\ unescaped, so a value shaped like ` .

` closed the reference early and let the rest of the string be parsed as an attacker-chosen extra triple: metadata={"uri": "https://x> . ` delimiter-breaking payload from the report, and a control-character (newline/tab) variant covering the other half of the excluded set. --- semantica/export/rdf_exporter.py | 20 +++++++++- tests/export/test_metadata_passthrough.py | 48 +++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/semantica/export/rdf_exporter.py b/semantica/export/rdf_exporter.py index bcd89140..dffd4951 100644 --- a/semantica/export/rdf_exporter.py +++ b/semantica/export/rdf_exporter.py @@ -193,6 +193,24 @@ def _escape_literal(value: str) -> str: ) +#: Turtle/N-Triples IRIREF grammar excludes these unescaped between `<` and +#: `>`: control characters, space, and <>"{}|^`\. An IRI-valued metadata +#: value (currently only sem:sourceUri, from the caller-controlled "uri" +#: metadata key) is written as `<{value}>` with no other quoting, so a value +#: containing one of these characters — a ">" followed by a full triple, for +#: instance — closes the IRIREF early and lets the rest of the string be +#: parsed as further RDF statements. This is the same shape of defect the +#: entity/relationship IRIs were hardened against; that hardening resolves +#: prefixes as well, which a metadata value never needs, so this stays a +#: narrower, dedicated guard rather than reusing _as_turtle_iri. +_IRI_REF_UNSAFE_RE = re.compile(r'[\x00-\x20<>"{}|^`\\]') + + +def _safe_iri_ref(value: str) -> str: + """Percent-encode the characters an IRIREF may not contain unescaped.""" + return _IRI_REF_UNSAFE_RE.sub(lambda m: quote(m.group(0), safe=""), value) + + def _escape_xml(value: str) -> str: """Escape a string for either XML element text or an attribute value. @@ -328,7 +346,7 @@ def _typed_literal_parts(term: str, value: Any) -> tuple: def _turtle_object(term: str, value: Any) -> str: kind, lexical, datatype = _typed_literal_parts(term, value) if kind == "iri": - return f"<{lexical}>" + return f"<{_safe_iri_ref(lexical)}>" if datatype is None: return f'"{_escape_literal(lexical)}"' return f'"{lexical}"^^<{datatype}>' diff --git a/tests/export/test_metadata_passthrough.py b/tests/export/test_metadata_passthrough.py index cdb73227..ff4c3ddc 100644 --- a/tests/export/test_metadata_passthrough.py +++ b/tests/export/test_metadata_passthrough.py @@ -351,3 +351,51 @@ def test_a_quote_in_an_attribute_value_cannot_break_the_document(): from xml.dom.minidom import parseString parseString(xml) # well-formedness is the assertion + + +# --- Finding from review of PR #1165 ---------------------------------------- + + +@pytest.mark.parametrize("fmt", ["turtle", "ntriples"]) +def test_an_iri_valued_metadata_value_cannot_inject_a_second_triple(fmt): + """`sem:sourceUri` (the "uri" key) is the one metadata term written as a + node, ``<{value}>``, with no other quoting. Turtle/N-Triples IRIREFs + exclude '>' (among other characters) unescaped, so a value shaped like + `` .

`` closed the reference early and let the + rest of the string be parsed as an unrelated, attacker-chosen triple. + """ + payload = ( + "https://evil.example/x> . " + " ' — cover the control-character half of + the grammar, not only the delimiter characters. + """ + payload = "https://evil.example/x\ninjected line\ttabbed" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"uri": payload}} + ], + "relationships": [], + } + g = _serialize(RDFSerializer(), fmt, data) + assert len(g) == 4