diff --git a/semantica/export/rdf_exporter.py b/semantica/export/rdf_exporter.py index 32a09895..dffd4951 100644 --- a/semantica/export/rdf_exporter.py +++ b/semantica/export/rdf_exporter.py @@ -146,6 +146,275 @@ def mint_relationship_iri(index: int, source: Any, target: Any) -> str: return f"{SEMANTICA_NS}rel_{index}_{digest}" +#: The metadata keys Semantica itself produces, and the terms they are written +#: as. GraphBuilder.build_graph writes the first five, create_snapshot writes +#: snapshot_time, and load_from_neo4j writes source / uri / database. These are +#: Semantica's own vocabulary, so they are minted in the declared namespace and +#: declared in semantica-ns.ttl. +#: +#: A key the caller supplied is a different matter. Which namespace an +#: arbitrary metadata key belongs in is issue #1146, and until that is settled +#: the exporter refuses to guess: it warns and skips, and a caller who already +#: knows the answer passes ``metadata_terms``. +#: +#: The map is key -> term rather than key -> namespace because two of the keys +#: cannot keep their own name. ``source`` on a graph loaded from Neo4j is the +#: system it came from, while sem:source is already the ObjectProperty holding +#: the subject of a reified relationship; reusing it would put a string where +#: an entity belongs. +DEFAULT_METADATA_TERMS: Dict[str, str] = { + "num_entities": f"{SEMANTICA_NS}numEntities", + "num_relationships": f"{SEMANTICA_NS}numRelationships", + "temporal_enabled": f"{SEMANTICA_NS}temporalEnabled", + "entity_resolution_applied": f"{SEMANTICA_NS}entityResolutionApplied", + "timestamp": f"{SEMANTICA_NS}builtAt", + "snapshot_time": f"{SEMANTICA_NS}snapshotAt", + "source": f"{SEMANTICA_NS}sourceSystem", + "uri": f"{SEMANTICA_NS}sourceUri", + "database": f"{SEMANTICA_NS}sourceDatabase", +} + +#: Terms whose value is a node rather than a string. Everything else stays a +#: literal: a metadata value that merely looks like a URL is not thereby a +#: reference to one. +IRI_VALUED_METADATA_TERMS: Set[str] = {f"{SEMANTICA_NS}sourceUri"} + +_XSD_NS = "http://www.w3.org/2001/XMLSchema#" + + +def _escape_literal(value: str) -> str: + """Escape a string for a Turtle or N-Triples quoted literal.""" + return ( + value.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + ) + + +#: Turtle/N-Triples IRIREF grammar excludes these unescaped between `<` and +#: `>`: control characters, space, and <>"{}|^`\. An IRI-valued metadata +#: value (currently only sem:sourceUri, from the caller-controlled "uri" +#: metadata key) is written as `<{value}>` with no other quoting, so a value +#: containing one of these characters — a ">" followed by a full triple, for +#: instance — closes the IRIREF early and lets the rest of the string be +#: parsed as further RDF statements. This is the same shape of defect the +#: entity/relationship IRIs were hardened against; that hardening resolves +#: prefixes as well, which a metadata value never needs, so this stays a +#: narrower, dedicated guard rather than reusing _as_turtle_iri. +_IRI_REF_UNSAFE_RE = re.compile(r'[\x00-\x20<>"{}|^`\\]') + + +def _safe_iri_ref(value: str) -> str: + """Percent-encode the characters an IRIREF may not contain unescaped.""" + return _IRI_REF_UNSAFE_RE.sub(lambda m: quote(m.group(0), safe=""), value) + + +def _escape_xml(value: str) -> str: + """Escape a string for either XML element text or an attribute value. + + The quotes matter. This helper feeds `rdf:about`, `rdf:resource` and + `xmlns:` attribute values, which are delimited by double quotes, so a value + carrying one would close the attribute early and produce a document that + does not parse. Escaping them in element text as well is harmless and + means one helper cannot be used in the wrong place. + """ + return ( + value.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + + +def _is_ncname(value: str) -> bool: + """Whether a string can be an XML NCName, which is what RDF/XML requires. + + Checked over the ASCII range rather than the full XML production: the + grammar also admits combining characters and extenders, so this is + deliberately conservative. It refuses names it could have accepted, and it + never accepts one that would produce a document a parser rejects. The + earlier check tested only that the first character was not a digit, which + let through every other way a local name can fail to be a name. + """ + if not value: + return False + if not (value[0].isascii() and (value[0].isalpha() or value[0] == "_")): + return False + return all(c.isascii() and (c.isalnum() or c in "._-") for c in value[1:]) + + +def _split_iri(iri: str) -> Optional[tuple]: + """Split an IRI into (namespace, local name) for RDF/XML's QName syntax. + + Returns None when no split yields a usable local name. RDF/XML is the only + serialization here that cannot write an arbitrary predicate IRI, so this is + the one place a term can be unrepresentable, and the caller reports it + rather than dropping it quietly. + """ + for sep in ("#", "/"): + index = iri.rfind(sep) + if index != -1 and index + 1 < len(iri): + local = iri[index + 1 :] + if _is_ncname(local): + return iri[: index + 1], local + return None + + +def _metadata_statements( + metadata: Any, + terms: Dict[str, str], + logger: Any, +) -> List[tuple]: + """Resolve a metadata mapping to a list of (term IRI, value) pairs. + + A key with no term is skipped and reported. Silence is the defect this + fixes, so an unmapped key must be louder than a mapped one, not quieter. + """ + if not isinstance(metadata, dict): + return [] + + statements: List[tuple] = [] + for key, value in metadata.items(): + term = terms.get(key) + if term is None: + logger.warning( + "Metadata key %r has no term and was not exported. Which " + "namespace a caller-supplied key belongs in is issue #1146; " + "pass metadata_terms={%r: ''} to export it now.", + key, + key, + ) + continue + if value is None: + continue + if isinstance(value, (dict, list, tuple, set)): + logger.warning( + "Metadata key %r holds a %s, which has no modelled RDF shape " + "yet, and was not exported.", + key, + type(value).__name__, + ) + continue + statements.append((term, value)) + return statements + + +def _resolve_metadata_terms(overrides: Optional[Dict[str, str]]) -> Dict[str, str]: + if not overrides: + return DEFAULT_METADATA_TERMS + return {**DEFAULT_METADATA_TERMS, **overrides} + + +def _typed_literal_parts(term: str, value: Any) -> tuple: + """Return (kind, lexical, datatype) for one metadata value. + + kind is "iri" or "literal". The lexical form and datatype are chosen once, + here, so that the four serializers cannot disagree about them the way they + disagreed about confidence in #1100. + """ + if term in IRI_VALUED_METADATA_TERMS and isinstance(value, str): + return "iri", value, None + if isinstance(value, bool): + return "literal", "true" if value else "false", f"{_XSD_NS}boolean" + if isinstance(value, int): + return "literal", str(value), f"{_XSD_NS}integer" + if isinstance(value, float): + # xsd:double, not xsd:decimal. `repr(1e-05)` is "1e-05" and + # `repr(float("nan"))` is "nan", and xsd:decimal admits neither the + # exponent form nor the special values, so typing a float as decimal + # produced lexicals a strict parser rejects. A Python float is an IEEE + # 754 double; xsd:double has legal lexicals for all of them, and it is + # also the honest claim, since nothing that arrived as a float was ever + # exact. `normalize_confidence` keeps xsd:decimal for confidence + # deliberately: that is a bounded score where exactness is meaningful + # and NaN is not a confidence at all. + if value != value: + lexical = "NaN" + elif value == float("inf"): + lexical = "INF" + elif value == float("-inf"): + lexical = "-INF" + else: + lexical = repr(value) + return "literal", lexical, f"{_XSD_NS}double" + return "literal", str(value), None + + +def _turtle_object(term: str, value: Any) -> str: + kind, lexical, datatype = _typed_literal_parts(term, value) + if kind == "iri": + return f"<{_safe_iri_ref(lexical)}>" + if datatype is None: + return f'"{_escape_literal(lexical)}"' + return f'"{lexical}"^^<{datatype}>' + + +def _turtle_metadata_clauses(statements: List[tuple]) -> List[str]: + return [f"<{term}> {_turtle_object(term, value)}" for term, value in statements] + + +def _ntriples_metadata_lines(subject: str, statements: List[tuple]) -> List[str]: + return [ + f"<{subject}> <{term}> {_turtle_object(term, value)} ." + for term, value in statements + ] + + +def _rdfxml_metadata_lines( + statements: List[tuple], indent: str, logger: Any = None +) -> List[str]: + """RDF/XML needs a QName, so an unprefixed term declares its own prefix. + + A term with no QName form has no RDF/XML representation at all, and this is + the only serialization with that restriction. Skipping it quietly would + reintroduce, in one format, exactly the silent metadata loss this module + was changed to stop, so it is reported and the other three formats still + carry the statement in full. + """ + lines: List[str] = [] + for position, (term, value) in enumerate(statements): + split = _split_iri(term) + if split is None: + if logger is not None: + logger.warning( + "Term %r has no QName form, so it cannot be written in " + "RDF/XML and was omitted from that serialization only. " + "Turtle, N-Triples and JSON-LD carry it in full.", + term, + ) + continue + namespace, local = split + kind, lexical, datatype = _typed_literal_parts(term, value) + prefix = f"md{position}" + opening = f'{indent}<{prefix}:{local} xmlns:{prefix}="{_escape_xml(namespace)}"' + if kind == "iri": + lines.append(f'{opening} rdf:resource="{_escape_xml(lexical)}"/>') + continue + if datatype is not None: + opening += f' rdf:datatype="{_escape_xml(datatype)}"' + lines.append(f"{opening}>{_escape_xml(lexical)}") + return lines + + +def _jsonld_metadata_entries(statements: List[tuple]) -> Dict[str, Any]: + """Absolute IRIs as keys, and explicit @value/@type rather than JSON's own + types: JSON's number is xsd:double, which would make the JSON-LD export + disagree with the other three about the datatype of an integer.""" + entries: Dict[str, Any] = {} + for term, value in statements: + kind, lexical, datatype = _typed_literal_parts(term, value) + if kind == "iri": + entries[term] = {"@id": lexical} + elif datatype is None: + entries[term] = lexical + else: + entries[term] = {"@value": lexical, "@type": datatype} + return entries + + class NamespaceManager: """ RDF namespace management engine. @@ -501,6 +770,8 @@ class RDFSerializer: """ include_temporal: bool = options.pop("include_temporal", False) time_axis: str = options.pop("time_axis", "valid") + metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None)) + graph_uri: Optional[str] = options.pop("graph_uri", None) lines = [] @@ -534,21 +805,32 @@ class RDFSerializer: text = entity.get("text") or entity.get("label", "") confidence = normalize_confidence(entity.get("confidence", 1.0)) - lines.append( - f"<{self._as_turtle_iri(entity_id, merged_namespaces)}> a " - f"<{self._as_turtle_iri(entity_type, merged_namespaces)}> ;" - ) + clauses = [ + f"a <{self._as_turtle_iri(entity_type, merged_namespaces)}>", + f'semantica:text "{text}"', + ] if confidence is None: self.logger.warning( f"Entity {entity_id} has a confidence that is not a number " f"({entity.get('confidence')!r}), so no confidence is written" ) - lines.append(f' semantica:text "{text}" .') else: - lines.append(f' semantica:text "{text}" ;') - lines.append( - f' semantica:confidence "{confidence}"^^<{CONFIDENCE_DATATYPE}> .' + clauses.append( + f'semantica:confidence "{confidence}"^^<{CONFIDENCE_DATATYPE}>' ) + clauses.extend( + _turtle_metadata_clauses( + _metadata_statements( + entity.get("metadata"), metadata_terms, self.logger + ) + ) + ) + + entity_iri = self._as_turtle_iri(entity_id, merged_namespaces) + lines.append(f"<{entity_iri}> {clauses[0]} ;") + for clause in clauses[1:-1]: + lines.append(f" {clause} ;") + lines.append(f" {clauses[-1]} .") lines.append("") # Convert relationships to RDF triplets @@ -582,6 +864,31 @@ class RDFSerializer: ) lines.extend(owl_lines) + # Graph-level metadata needs a subject, and this serializer has never + # minted a document node. Rather than invent one here, it is written + # only when the caller names the graph; issue #1147 is where the + # default subject comes from once that lands. + graph_clauses = ( + _turtle_metadata_clauses( + _metadata_statements( + rdf_data.get("metadata"), metadata_terms, self.logger + ) + ) + if graph_uri + else [] + ) + if graph_clauses: + graph_iri = self._as_turtle_iri(graph_uri, merged_namespaces) + lines.append("") + lines.append( + f"<{graph_iri}> {graph_clauses[0]} " + + (";" if len(graph_clauses) > 1 else ".") + ) + for clause in graph_clauses[1:-1]: + lines.append(f" {clause} ;") + if len(graph_clauses) > 1: + lines.append(f" {graph_clauses[-1]} .") + return "\n".join(lines) def _reified_relationship_triples( @@ -730,6 +1037,9 @@ class RDFSerializer: ... } >>> rdfxml = serializer.serialize_to_rdfxml(rdf_data) """ + metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None)) + graph_uri: Optional[str] = options.pop("graph_uri", None) + lines = [''] lines.append('' f"{confidence}" ) + lines.extend( + _rdfxml_metadata_lines( + _metadata_statements( + entity.get("metadata"), metadata_terms, self.logger + ), + " ", + self.logger, + ) + ) lines.append(" ") lines.append("") @@ -795,6 +1117,26 @@ class RDFSerializer: lines.append(" ") lines.append("") + graph_lines = ( + _rdfxml_metadata_lines( + _metadata_statements( + rdf_data.get("metadata"), metadata_terms, self.logger + ), + " ", + self.logger, + ) + if graph_uri + else [] + ) + if graph_lines: + graph_iri = xml_escape( + self._as_turtle_iri(graph_uri, namespaces), quote=True + ) + lines.append(f' ') + lines.extend(graph_lines) + lines.append(" ") + lines.append("") + lines.append("") return "\n".join(lines) @@ -825,6 +1167,9 @@ class RDFSerializer: """ import json + metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None)) + graph_uri: Optional[str] = options.pop("graph_uri", None) + # Initialize JSON-LD structure with context jsonld = { "@context": { @@ -869,6 +1214,13 @@ class RDFSerializer: "@value": confidence, "@type": CONFIDENCE_DATATYPE, } + node.update( + _jsonld_metadata_entries( + _metadata_statements( + entity.get("metadata"), metadata_terms, self.logger + ) + ) + ) jsonld["@graph"].append(node) # Convert relationships to JSON-LD @@ -893,6 +1245,18 @@ class RDFSerializer: } ) + graph_entries = ( + _jsonld_metadata_entries( + _metadata_statements( + rdf_data.get("metadata"), metadata_terms, self.logger + ) + ) + if graph_uri + else {} + ) + if graph_entries: + jsonld["@graph"].append({"@id": graph_uri, **graph_entries}) + return json.dumps(jsonld, indent=2, ensure_ascii=False) def serialize_to_ntriples(self, rdf_data: Dict[str, Any], **options) -> str: @@ -909,6 +1273,9 @@ class RDFSerializer: Returns: String containing N-Triples serialization """ + metadata_terms = _resolve_metadata_terms(options.pop("metadata_terms", None)) + graph_uri: Optional[str] = options.pop("graph_uri", None) + lines = [] namespaces = self.namespace_manager.extract_namespaces(rdf_data) @@ -959,6 +1326,15 @@ class RDFSerializer: f'"{confidence}"^^<{CONFIDENCE_DATATYPE}> .' ) + lines.extend( + _ntriples_metadata_lines( + subject.strip("<>"), + _metadata_statements( + entity.get("metadata"), metadata_terms, self.logger + ), + ) + ) + # Convert relationships relationships = rdf_data.get("relationships", []) for rel in relationships: @@ -971,6 +1347,16 @@ class RDFSerializer: f"{expand_uri(source_id)} {expand_uri(rel_type)} {expand_uri(target_id)} ." ) + if graph_uri: + lines.extend( + _ntriples_metadata_lines( + graph_uri, + _metadata_statements( + rdf_data.get("metadata"), metadata_terms, self.logger + ), + ) + ) + return "\n".join(lines) diff --git a/semantica/ontology/vocabulary/semantica-ns.ttl b/semantica/ontology/vocabulary/semantica-ns.ttl index 42badf60..ffe36df5 100644 --- a/semantica/ontology/vocabulary/semantica-ns.ttl +++ b/semantica/ontology/vocabulary/semantica-ns.ttl @@ -135,6 +135,90 @@ JSONExporter.export_to_jsonld in export/json_exporter.py.""" ; rdfs:range xsd:string ; rdfs:isDefinedBy . +# ── Metadata carried through from the graph builder ────────────────────────── +# +# The keys GraphBuilder and the Neo4j loader write into "metadata". Declared +# here because the RDF serializers emit them (#1154); a caller-supplied key is +# not declared here and is not emitted, because which namespace it belongs in +# is #1146. + +sem:numEntities a owl:DatatypeProperty ; + rdfs:label "number of entities" ; + rdfs:comment """Count of entities in the graph as built, from +GraphBuilder.build_graph. A count of what was built, not a constraint on what +the graph contains: an export filtered after the fact will disagree with it.""" ; + rdfs:range xsd:integer ; + rdfs:isDefinedBy . + +sem:numRelationships a owl:DatatypeProperty ; + rdfs:label "number of relationships" ; + rdfs:comment "Count of relationships in the graph as built." ; + rdfs:range xsd:integer ; + rdfs:isDefinedBy . + +sem:temporalEnabled a owl:DatatypeProperty ; + rdfs:label "temporal enabled" ; + rdfs:comment """True when the builder was configured to track valid time. +False does not mean the graph is untimed; it means no temporal bounds were +recorded for it.""" ; + rdfs:range xsd:boolean ; + rdfs:isDefinedBy . + +sem:entityResolutionApplied a owl:DatatypeProperty ; + rdfs:label "entity resolution applied" ; + rdfs:comment """True when a resolver ran over the extracted entities, so a +consumer knows whether two nodes with the same surface text were ever +considered for merging.""" ; + rdfs:range xsd:boolean ; + rdfs:isDefinedBy . + +sem:builtAt a owl:DatatypeProperty ; + rdfs:label "built at" ; + rdfs:comment """When the graph was built, as GraphBuilder recorded it. + +The range is xsd:string, deliberately, and not xsd:dateTime. GraphBuilder +stamps with a timezone-naive datetime.now(), and #1114 is the demonstration of +what typing such a value as xsd:dateTime costs: a timezone-qualified SPARQL +filter over it raises an indeterminate comparison and silently drops the row. +#1121 swept the export and provenance modules to an explicit UTC offset and +deliberately left kg/ alone, because the context and vector-store modules +compare against naive values already on disk. Until that sweep reaches +GraphBuilder this value is a string that looks like a timestamp, and saying so +is more useful than a type that invites arithmetic it cannot support.""" ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + +sem:snapshotAt a owl:DatatypeProperty ; + rdfs:label "snapshot at" ; + rdfs:comment """The point in time a snapshot represents, from +GraphBuilder.create_snapshot. A string for the same reason as sem:builtAt.""" ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + +sem:sourceSystem a owl:DatatypeProperty ; + rdfs:label "source system" ; + rdfs:comment """The system a graph was loaded from, currently the literal +"neo4j" written by GraphBuilder.load_from_neo4j. + +Named sourceSystem rather than source because sem:source is already the +ObjectProperty carrying the subject of a reified relationship. The metadata key +is still "source"; the exporter maps the key to this term.""" ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + +sem:sourceUri a owl:ObjectProperty ; + rdfs:label "source URI" ; + rdfs:comment """The address of the system a graph was loaded from. The one +metadata term whose value is a node rather than a literal, because it names a +thing rather than describing one.""" ; + rdfs:isDefinedBy . + +sem:sourceDatabase a owl:DatatypeProperty ; + rdfs:label "source database" ; + rdfs:comment "The database within the source system a graph was loaded from." ; + rdfs:range xsd:string ; + rdfs:isDefinedBy . + # ── Temporal term (OWL-Time export) ────────────────────────────────────────── sem:openEndedInterval a owl:DatatypeProperty ; diff --git a/tests/export/test_metadata_passthrough.py b/tests/export/test_metadata_passthrough.py new file mode 100644 index 00000000..ff4c3ddc --- /dev/null +++ b/tests/export/test_metadata_passthrough.py @@ -0,0 +1,401 @@ +"""Metadata must survive serialization (issue #1154). + +``convert_kg_to_rdf`` copies ``metadata`` into the RDF-ready dictionary at +rdf_exporter.py:302, and no serializer has ever read it back out. Turtle, +N-Triples, RDF/XML and RDFExporter's JSON-LD all write the entity's id, type, +text and confidence, and none of them writes a single metadata statement, so an +entity keeps its confidence score and loses what produced it: the source +document, the page, the extractor, the reviewer. JSONExporter's json-ld path +keeps all of them, which is how the same knowledge graph exported two ways came +to carry ten triples of user data through one exporter and none through the +other. + +The keys Semantica itself produces (GraphBuilder writes num_entities, +num_relationships, temporal_enabled, timestamp and entity_resolution_applied; +the Neo4j loader writes source, uri and database) are Semantica's own +vocabulary, so they are minted in the declared namespace and declared in +semantica-ns.ttl. Keys the caller supplied are not: which namespace those +belong in is issue #1146, and until that is settled the exporter refuses to +guess rather than inventing an IRI, warns, and takes an explicit +``metadata_terms`` mapping from any caller who already knows the answer. +""" + +import json + +import pytest +from rdflib import Graph, Literal, URIRef +from rdflib.namespace import XSD + +from semantica.export.rdf_exporter import ( + DEFAULT_METADATA_TERMS, + RDFSerializer, + SEMANTICA_NS, + mint_entity_iri, +) + +ENTITY_IRI = "https://example.org/e1" + +# The provenance fields the issue names, plus one key Semantica itself writes. +GRAPH_WITH_METADATA = { + "entities": [ + { + "id": ENTITY_IRI, + "type": "https://example.org/Org", + "text": "Acme Corp", + "confidence": 0.91, + "metadata": {"num_entities": 1, "temporal_enabled": True}, + } + ], + "relationships": [], + "metadata": { + "num_entities": 1, + "num_relationships": 0, + "temporal_enabled": False, + "entity_resolution_applied": True, + }, +} + +NUM_ENTITIES = URIRef(f"{SEMANTICA_NS}numEntities") +TEMPORAL_ENABLED = URIRef(f"{SEMANTICA_NS}temporalEnabled") + + +def _parse(text: str, fmt: str) -> Graph: + """Assert on the parsed graph, never on the serialized text.""" + g = Graph() + g.parse(data=text, format=fmt) + return g + + +def _serialize(serializer: RDFSerializer, fmt: str, data, **options) -> Graph: + method, parse_as = { + "turtle": (serializer.serialize_to_turtle, "turtle"), + "ntriples": (serializer.serialize_to_ntriples, "nt"), + "rdfxml": (serializer.serialize_to_rdfxml, "xml"), + "jsonld": (serializer.serialize_to_jsonld, "json-ld"), + }[fmt] + return _parse(method(data, **options), parse_as) + + +FORMATS = ["turtle", "ntriples", "rdfxml", "jsonld"] + + +@pytest.mark.parametrize("fmt", FORMATS) +def test_entity_metadata_reaches_every_serialization(fmt): + """The headline defect: the statement is absent from all four formats.""" + g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA) + assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(1)) in g + + +@pytest.mark.parametrize("fmt", FORMATS) +def test_entity_metadata_booleans_keep_their_datatype(fmt): + g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA) + assert (URIRef(ENTITY_IRI), TEMPORAL_ENABLED, Literal(True)) in g + + +def test_every_format_writes_the_same_metadata_triples(): + """A value must not change datatype with the serializer, as #1100 found.""" + per_format = {} + for fmt in FORMATS: + g = _serialize(RDFSerializer(), fmt, GRAPH_WITH_METADATA) + per_format[fmt] = { + (p, o) + for s, p, o in g + if str(p).startswith(SEMANTICA_NS) and "numEntities" in str(p) + } + assert len(set(map(frozenset, per_format.values()))) == 1, per_format + + +def test_graph_metadata_needs_a_subject_the_caller_named(): + """Graph-level metadata hangs off graph_uri; #1147 owns the default.""" + doc = URIRef("https://example.org/graph/1") + g = _serialize( + RDFSerializer(), + "turtle", + GRAPH_WITH_METADATA, + graph_uri=str(doc), + ) + assert (doc, NUM_ENTITIES, Literal(1)) in g + assert (doc, URIRef(f"{SEMANTICA_NS}entityResolutionApplied"), Literal(True)) in g + + +def test_graph_metadata_is_not_invented_without_a_subject(): + g = _serialize(RDFSerializer(), "turtle", GRAPH_WITH_METADATA) + assert not list(g.subjects(NUM_ENTITIES, Literal(0))) + # the entity keeps its own metadata; only the graph-level block waits + assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(1)) in g + + +def test_an_unknown_key_is_refused_out_loud_not_dropped_in_silence(caplog): + """#1146 owns which namespace a caller's key belongs in. Until then: warn.""" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}} + ], + "relationships": [], + } + with caplog.at_level("WARNING"): + g = _serialize(RDFSerializer(), "turtle", data) + assert not any("reviewed_by" in str(p) for p in g.predicates()) + assert any("reviewed_by" in r.getMessage() for r in caplog.records) + assert any("1146" in r.getMessage() for r in caplog.records) + + +@pytest.mark.parametrize("fmt", FORMATS) +def test_a_caller_who_knows_the_answer_can_supply_the_term(fmt): + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}} + ], + "relationships": [], + } + terms = {"reviewed_by": "http://purl.org/dc/terms/creator"} + g = _serialize(RDFSerializer(), fmt, data, metadata_terms=terms) + assert ( + URIRef(ENTITY_IRI), + URIRef("http://purl.org/dc/terms/creator"), + Literal("fabio"), + ) in g + + +def test_a_literal_with_a_quote_or_newline_still_parses(): + """Metadata is user text; #1098 is the same class of defect one field over.""" + data = { + "entities": [ + { + "id": ENTITY_IRI, + "text": "Acme", + "metadata": {"source": 'the "Q3" report\nsecond line'}, + } + ], + "relationships": [], + } + for fmt in FORMATS: + g = _serialize(RDFSerializer(), fmt, data) + assert ( + URIRef(ENTITY_IRI), + URIRef(f"{SEMANTICA_NS}sourceSystem"), + Literal('the "Q3" report\nsecond line'), + ) in g + + +def test_an_iri_valued_key_is_written_as_a_node_not_a_string(): + """The Neo4j loader's ``uri`` key. Note the term is sem:sourceUri, not + sem:uri: the key names a field, the term names a relation.""" + data = { + "entities": [ + { + "id": ENTITY_IRI, + "text": "Acme", + "metadata": {"uri": "https://example.org/db"}, + } + ], + "relationships": [], + } + g = _serialize(RDFSerializer(), "turtle", data) + assert ( + URIRef(ENTITY_IRI), + URIRef(f"{SEMANTICA_NS}sourceUri"), + URIRef("https://example.org/db"), + ) in g + + +def test_output_is_unchanged_when_no_metadata_is_present(): + plain = { + "entities": [ + {"id": ENTITY_IRI, "type": "https://example.org/Org", "text": "Acme"} + ], + "relationships": [ + {"source_id": ENTITY_IRI, "target_id": "https://example.org/e2"} + ], + } + serializer = RDFSerializer() + assert serializer.serialize_to_turtle(plain) == serializer.serialize_to_turtle( + plain + ) + g = _parse(serializer.serialize_to_turtle(plain), "turtle") + assert len(g) == 4 + + +def test_every_default_term_is_declared_in_the_shipped_vocabulary(): + """Drift guard: a term the exporter emits and the vocabulary omits is a bug.""" + from semantica.ontology.vocabulary import vocabulary_path + + vocab = Graph() + vocab.parse(vocabulary_path(), format="turtle") + declared = {str(s) for s in vocab.subjects()} + missing = sorted(set(DEFAULT_METADATA_TERMS.values()) - declared) + assert not missing, f"emitted but undeclared: {missing}" + + +def test_jsonld_metadata_survives_a_real_jsonld_processor(): + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"num_entities": 3}} + ], + "relationships": [], + } + raw = RDFSerializer().serialize_to_jsonld(data) + json.loads(raw) # must be valid JSON before it can be valid JSON-LD + g = _parse(raw, "json-ld") + assert (URIRef(ENTITY_IRI), NUM_ENTITIES, Literal(3)) in g + + +# --- Findings from the Qodo review of PR #1165 ----------------------------- + + +@pytest.mark.parametrize("fmt", FORMATS) +@pytest.mark.parametrize( + "value", [1e-05, 1e300, 0.1, -0.0, float("nan"), float("inf"), float("-inf")] +) +def test_a_float_metadata_value_is_a_double_and_keeps_a_legal_lexical(fmt, value): + """`repr()` of a float is not an xsd:decimal lexical. + + `repr(1e-05)` is "1e-05" and `repr(float("nan"))` is "nan", neither of which + xsd:decimal admits, so typing a float as decimal produced RDF a strict + parser rejects. A Python float is an IEEE 754 double, xsd:double has legal + lexicals for the exponent form and for the three special values, and saying + double is also the honest claim: nothing here was ever exact. + """ + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"num_entities": value}} + ], + "relationships": [], + } + g = _serialize(RDFSerializer(), fmt, data) + objects = list(g.objects(URIRef(ENTITY_IRI), NUM_ENTITIES)) + assert len(objects) == 1, f"{fmt}: {objects}" + (written,) = objects + assert written.datatype == XSD.double, written.datatype + parsed = written.toPython() + if value != value: # NaN + assert parsed != parsed + else: + assert parsed == value + + +def test_every_format_agrees_on_a_float_metadata_value(): + per_format = {} + for fmt in FORMATS: + g = _serialize( + RDFSerializer(), + fmt, + { + "entities": [ + {"id": ENTITY_IRI, "text": "A", "metadata": {"num_entities": 1e-05}} + ], + "relationships": [], + }, + ) + per_format[fmt] = {(p, o) for s, p, o in g if p == NUM_ENTITIES} + assert len(set(map(frozenset, per_format.values()))) == 1, per_format + + +def test_a_term_rdfxml_cannot_name_is_refused_out_loud(caplog): + """RDF/XML needs a QName, and the PR's whole point is no silent drops. + + A term whose local part is not an XML NCName has no RDF/XML form at all. + Skipping it quietly reintroduces, in one format, exactly the loss this + change exists to stop. + """ + unnameable = "http://example.org/ns/123" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}} + ], + "relationships": [], + } + with caplog.at_level("WARNING"): + xml = RDFSerializer().serialize_to_rdfxml( + data, metadata_terms={"reviewed_by": unnameable} + ) + _parse(xml, "xml") # must still be well-formed + messages = " ".join(r.getMessage() for r in caplog.records) + assert unnameable in messages + assert "RDF/XML" in messages + + +@pytest.mark.parametrize("fmt", ["turtle", "ntriples", "jsonld"]) +def test_the_other_formats_still_carry_a_term_rdfxml_cannot_name(fmt): + """Only RDF/XML has the QName restriction; the rest write the full IRI.""" + unnameable = "http://example.org/ns/123" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"reviewed_by": "fabio"}} + ], + "relationships": [], + } + g = _serialize( + RDFSerializer(), fmt, data, metadata_terms={"reviewed_by": unnameable} + ) + assert (URIRef(ENTITY_IRI), URIRef(unnameable), Literal("fabio")) in g + + +def test_a_quote_in_an_attribute_value_cannot_break_the_document(): + """`_escape_xml` feeds attribute values, which are delimited by quotes. + + Escaping only &, < and > leaves a caller-supplied value able to close the + attribute early and produce XML that does not parse. + """ + data = { + "entities": [ + { + "id": 'https://example.org/e"1', + "text": "Acme", + "metadata": {"uri": 'https://example.org/db"x'}, + } + ], + "relationships": [], + } + xml = RDFSerializer().serialize_to_rdfxml(data) + from xml.dom.minidom import parseString + + parseString(xml) # well-formedness is the assertion + + +# --- Finding from review of PR #1165 ---------------------------------------- + + +@pytest.mark.parametrize("fmt", ["turtle", "ntriples"]) +def test_an_iri_valued_metadata_value_cannot_inject_a_second_triple(fmt): + """`sem:sourceUri` (the "uri" key) is the one metadata term written as a + node, ``<{value}>``, with no other quoting. Turtle/N-Triples IRIREFs + exclude '>' (among other characters) unescaped, so a value shaped like + `` .

`` closed the reference early and let the + rest of the string be parsed as an unrelated, attacker-chosen triple. + """ + payload = ( + "https://evil.example/x> . " + " ' — cover the control-character half of + the grammar, not only the delimiter characters. + """ + payload = "https://evil.example/x\ninjected line\ttabbed" + data = { + "entities": [ + {"id": ENTITY_IRI, "text": "Acme", "metadata": {"uri": payload}} + ], + "relationships": [], + } + g = _serialize(RDFSerializer(), fmt, data) + assert len(g) == 4