diff --git a/semantica/export/rdf_exporter.py b/semantica/export/rdf_exporter.py index 71d12f71..32a09895 100644 --- a/semantica/export/rdf_exporter.py +++ b/semantica/export/rdf_exporter.py @@ -29,9 +29,12 @@ Author: Semantica Contributors License: MIT """ +import re from pathlib import Path from decimal import Decimal, InvalidOperation +from html import escape as xml_escape from typing import Any, Dict, List, Optional, Set, Union +from urllib.parse import quote, urlsplit from ..utils.exceptions import ProcessingError, ValidationError from ..utils.helpers import ensure_directory, hash_data @@ -104,7 +107,11 @@ def normalize_confidence(value: Any) -> Optional[str]: # "1e100000000" is eleven characters that expand to a hundred million, and # the export path continues past validation errors, so a single malformed # field could exhaust memory. Nothing near this magnitude is a confidence. - if not -MAX_CONFIDENCE_EXPONENT <= decimal_value.adjusted() <= MAX_CONFIDENCE_EXPONENT: + if ( + not -MAX_CONFIDENCE_EXPONENT + <= decimal_value.adjusted() + <= MAX_CONFIDENCE_EXPONENT + ): return None # `str(Decimal("0.00001"))` gives "0.00001", but a float that has already @@ -395,6 +402,66 @@ class RDFSerializer: # OWL-Time namespace URI _OWL_TIME_NS = "http://www.w3.org/2006/time#" + _SEMANTICA_NS = "https://semantica.dev/ns#" + + # Matches an already-valid percent-escape so it can be passed through + # unchanged instead of being re-encoded into e.g. %2520. + _PERCENT_ESCAPE_RE = re.compile(r"%[0-9A-Fa-f]{2}") + + @classmethod + def _quote_preserving_escapes(cls, value: str, safe: str) -> str: + """quote() that leaves existing valid %XX escapes untouched. + + Blanket-quoting an absolute IRI double-encodes any percent-escape it + already carries (%20 -> %2520), which changes the identity of every + previously-valid IRI containing one. Only the spans between existing + valid escapes are quoted; a bare '%' that isn't part of a valid + escape (e.g. "%zz") still gets encoded to %25, keeping the malformed + case handled. + """ + parts = [] + pos = 0 + for match in cls._PERCENT_ESCAPE_RE.finditer(value): + parts.append(quote(value[pos : match.start()], safe=safe)) + parts.append(match.group(0)) + pos = match.end() + parts.append(quote(value[pos:], safe=safe)) + return "".join(parts) + + def _as_turtle_iri( + self, value: Any, namespaces: Optional[Dict[str, str]] = None + ) -> str: + """Return an absolute, safely encoded IRI for a Turtle resource.""" + value = str(value) + try: + parsed = urlsplit(value) + except ValueError: + parsed = urlsplit("") + if parsed.scheme: + prefix, separator, local_name = value.partition(":") + # Built-in namespaces (semantica:, rdf:, rdfs:, owl:, ...) must + # always be resolvable, not only when the caller passes no + # namespaces of its own — otherwise a value like "semantica:Foo" + # resolves fine with no @context but stops resolving the moment + # any @context is present, since callers pass extract_namespaces() + # (context-only) here without merging in the built-ins. + effective_namespaces = { + **self.namespace_manager.namespaces, + **(namespaces or {}), + } + namespace = effective_namespaces.get(prefix) + if namespace and separator: + return self._quote_preserving_escapes( + namespace + local_name, safe=":/?#[]@!$&'()*+,;=" + ) + # A scheme with at least two characters is an absolute IRI, + # including opaque forms such as mailto:foo and isbn:0451450523. + # Keep one-character schemes as the existing Windows drive-path case. + if len(prefix) >= 2: + return self._quote_preserving_escapes( + value, safe=":/?#[]@!$&'()*+,;=" + ) + return self._SEMANTICA_NS + quote(value, safe="") # Design decision — TemporalBound.OPEN in RDF: # OWL-Time has no standard predicate for "no known end date." We use @@ -467,7 +534,10 @@ class RDFSerializer: text = entity.get("text") or entity.get("label", "") confidence = normalize_confidence(entity.get("confidence", 1.0)) - lines.append(f"<{entity_id}> a <{entity_type}> ;") + lines.append( + f"<{self._as_turtle_iri(entity_id, merged_namespaces)}> a " + f"<{self._as_turtle_iri(entity_type, merged_namespaces)}> ;" + ) if confidence is None: self.logger.warning( f"Entity {entity_id} has a confidence that is not a number " @@ -488,10 +558,16 @@ class RDFSerializer: target_id = rel.get("target_id") or rel.get("target") rel_type = rel.get("type", DEFAULT_RELATION_TYPE) - lines.append(f"<{source_id}> <{rel_type}> <{target_id}> .") + lines.append( + f"<{self._as_turtle_iri(source_id, merged_namespaces)}> " + f"<{self._as_turtle_iri(rel_type, merged_namespaces)}> " + f"<{self._as_turtle_iri(target_id, merged_namespaces)}> ." + ) if include_temporal: - owl_lines = self._owl_time_triples_for_rel(rel, idx, time_axis) + owl_lines = self._owl_time_triples_for_rel( + rel, idx, time_axis, merged_namespaces + ) if owl_lines: # The interval hangs off the relationship's own IRI, and a # relationship written as a single triple has no such node @@ -501,7 +577,7 @@ class RDFSerializer: # export, and every term is declared in the vocabulary. lines.extend( self._reified_relationship_triples( - rel, idx, source_id, target_id, rel_type + rel, idx, source_id, target_id, rel_type, merged_namespaces ) ) lines.extend(owl_lines) @@ -515,6 +591,7 @@ class RDFSerializer: source_id: str, target_id: str, rel_type: str, + namespaces: Optional[Dict[str, str]] = None, ) -> List[str]: """ Emit the reified relationship node that OWL-Time triples hang off. @@ -523,7 +600,11 @@ class RDFSerializer: to, using the same sem:Relationship shape the JSON-LD export already writes, so the two serializations describe relationships the same way. """ - rel_id = rel.get("id") or mint_relationship_iri(idx, source_id or "", target_id or "") + rel_id = self._as_turtle_iri( + rel.get("id") + or mint_relationship_iri(idx, source_id or "", target_id or ""), + namespaces, + ) # The full predicate, not its local name. Truncating to the fragment # made https://a.example/ns#employs and https://b.example/ns#employs the @@ -537,17 +618,25 @@ class RDFSerializer: .replace("\r", "\\r") ) - predicates = [f"a semantica:Relationship"] + predicates = ["a semantica:Relationship"] if source_id: - predicates.append(f"semantica:source <{source_id}>") + predicates.append( + f"semantica:source <{self._as_turtle_iri(source_id, namespaces)}>" + ) if target_id: - predicates.append(f"semantica:target <{target_id}>") + predicates.append( + f"semantica:target <{self._as_turtle_iri(target_id, namespaces)}>" + ) predicates.append(f'semantica:type "{escaped}"') return ["", f"<{rel_id}> " + " ;\n ".join(predicates) + " ."] def _owl_time_triples_for_rel( - self, rel: Dict[str, Any], idx: int, time_axis: str + self, + rel: Dict[str, Any], + idx: int, + time_axis: str, + namespaces: Optional[Dict[str, str]] = None, ) -> List[str]: """ Emit OWL-Time Turtle triples for a relationship that carries temporal metadata. @@ -562,7 +651,7 @@ class RDFSerializer: def _is_open(v: Any) -> bool: if v is None: return False - if hasattr(v, "value"): # TemporalBound enum + if hasattr(v, "value"): # TemporalBound enum return v.value == _OPEN_SENTINEL return str(v).strip().upper() == _OPEN_SENTINEL @@ -579,7 +668,10 @@ class RDFSerializer: # deterministic IRI. source_id = rel.get("source_id") or rel.get("source") or "" target_id = rel.get("target_id") or rel.get("target") or "" - rel_base_id = rel.get("id") or mint_relationship_iri(idx, source_id, target_id) + rel_base_id = self._as_turtle_iri( + rel.get("id") or mint_relationship_iri(idx, source_id, target_id), + namespaces, + ) lines = [""] # blank separator for axis_name, from_val, until_val in axes: @@ -594,9 +686,7 @@ class RDFSerializer: lines.append(f" time:hasBeginning <{begin_id}> ;") if _is_open(until_val): - lines.append( - ' semantica:openEndedInterval "true"^^xsd:boolean .' - ) + lines.append(' semantica:openEndedInterval "true"^^xsd:boolean .') elif until_val is not None: end_id = f"{rel_base_id}__{axis_name}_end" lines.append(f" time:hasEnd <{end_id}> .") @@ -605,7 +695,9 @@ class RDFSerializer: f' time:inXSDDateTimeStamp "{until_val}"^^xsd:dateTimeStamp .' ) else: - lines[-1] = lines[-1].rstrip(" ;") + " ." # close interval without hasEnd + lines[-1] = ( + lines[-1].rstrip(" ;") + " ." + ) # close interval without hasEnd lines.append(f"<{begin_id}> a time:Instant ;") lines.append( @@ -644,6 +736,8 @@ class RDFSerializer: lines.append(' xmlns:semantica="https://semantica.dev/ns#">') lines.append("") + namespaces = self.namespace_manager.extract_namespaces(rdf_data) + # Convert entities to RDF/XML entities = rdf_data.get("entities", []) for entity in entities: @@ -653,13 +747,19 @@ class RDFSerializer: entity_text = entity.get("text", "") entity_id = mint_entity_iri(entity_text) - entity_type = entity.get("type", DEFAULT_ENTITY_TYPE) + entity_type = entity.get("type") or DEFAULT_ENTITY_TYPE text = entity.get("text") or entity.get("label", "") confidence = normalize_confidence(entity.get("confidence", 1.0)) # RDF/XML syntax: rdf:Description with rdf:about - lines.append(f' ') - lines.append(f' ') + entity_iri = xml_escape( + self._as_turtle_iri(entity_id, namespaces), quote=True + ) + entity_type_iri = xml_escape( + self._as_turtle_iri(entity_type, namespaces), quote=True + ) + lines.append(f' ') + lines.append(f' ') lines.append(f" {text}") if confidence is None: self.logger.warning( @@ -679,11 +779,19 @@ class RDFSerializer: for rel in relationships: source_id = rel.get("source_id") or rel.get("source") target_id = rel.get("target_id") or rel.get("target") - rel_type = rel.get("type", "semantica:related_to") + # RDF/XML predicates are emitted as QNames, unlike resource + # attributes which use the shared absolute-IRI normalizer. + rel_type = rel.get("type") or "semantica:related_to" # Relationship as property on source entity - lines.append(f' ') - lines.append(f' <{rel_type} rdf:resource="{target_id}"/>') + source_iri = xml_escape( + self._as_turtle_iri(source_id, namespaces), quote=True + ) + target_iri = xml_escape( + self._as_turtle_iri(target_id, namespaces), quote=True + ) + lines.append(f' ') + lines.append(f' <{rel_type} rdf:resource="{target_iri}"/>') lines.append(" ") lines.append("") @@ -803,20 +911,12 @@ class RDFSerializer: """ lines = [] + namespaces = self.namespace_manager.extract_namespaces(rdf_data) + def expand_uri(uri: str) -> str: if not uri: return "" - if uri.startswith("http"): - return f"<{uri}>" - if uri.startswith("semantica:"): - return f"" - if uri.startswith("rdf:"): - return f"" - if uri.startswith("rdfs:"): - return f"" - if ":" in uri: - return f"<{uri}>" - return f"" + return f"<{self._as_turtle_iri(uri, namespaces)}>" # Convert entities entities = rdf_data.get("entities", []) @@ -830,7 +930,7 @@ class RDFSerializer: subject = expand_uri(entity_id) # Type triple - entity_type = entity.get("type", "semantica:Entity") + entity_type = entity.get("type") or DEFAULT_ENTITY_TYPE lines.append( f"{subject} {expand_uri(entity_type)} ." ) @@ -864,7 +964,7 @@ class RDFSerializer: for rel in relationships: source_id = rel.get("source_id") or rel.get("source") target_id = rel.get("target_id") or rel.get("target") - rel_type = rel.get("type", "semantica:related_to") + rel_type = rel.get("type") or DEFAULT_RELATION_TYPE if source_id and target_id: lines.append( diff --git a/tests/export/test_rdf_exporter_turtle_iris.py b/tests/export/test_rdf_exporter_turtle_iris.py new file mode 100644 index 00000000..bc197a08 --- /dev/null +++ b/tests/export/test_rdf_exporter_turtle_iris.py @@ -0,0 +1,233 @@ +"""Regression tests for valid Turtle IRI generation (issue #1099).""" + +from rdflib import RDF, Graph, URIRef + +from semantica.export import RDFExporter +from semantica.kg.graph_builder import GraphBuilder + + +def test_turtle_normalizes_graph_builder_default_identifiers(): + """Default GraphBuilder labels with spaces become stable absolute IRIs.""" + source = { + "entities": [ + { + "id": "Kochi, Kerala", + "name": "Kochi, Kerala", + "type": "LOCATION", + }, + {"id": "Jane Doe", "name": "Jane Doe", "type": "PERSON"}, + ], + "relationships": [ + { + "source": "Jane Doe", + "target": "Kochi, Kerala", + "type": "located_in", + }, + ], + } + graph_data = GraphBuilder(resolve_conflicts=False).build(sources=[source]) + + turtle = RDFExporter().export_to_rdf(graph_data, format="turtle") + parsed = Graph().parse(data=turtle, format="turtle") + assert "" not in turtle + assert "" not in turtle + + kochi = URIRef("https://semantica.dev/ns#Kochi%2C%20Kerala") + jane = URIRef("https://semantica.dev/ns#Jane%20Doe") + assert ( + kochi, + RDF.type, + URIRef("https://semantica.dev/ns#LOCATION"), + ) in parsed + jane_type = URIRef("https://semantica.dev/ns#PERSON") + assert (jane, RDF.type, jane_type) in parsed + assert ( + jane, + URIRef("https://semantica.dev/ns#located_in"), + kochi, + ) in parsed + + +def test_turtle_preserves_absolute_iris(): + """Already-valid absolute resource IRIs remain unchanged.""" + turtle = RDFExporter().export_to_rdf( + { + "entities": [ + { + "id": "https://example.org/entities/jane", + "text": "Jane", + "type": "urn:example:Person", + } + ], + "relationships": [], + }, + format="turtle", + ) + parsed = Graph().parse(data=turtle, format="turtle") + + assert ( + URIRef("https://example.org/entities/jane"), + RDF.type, + URIRef("urn:example:Person"), + ) in parsed + + +def test_turtle_preserves_opaque_absolute_iris_and_encodes_bad_percent_escapes(): + """Opaque schemes remain absolute and malformed percent escapes are encoded.""" + turtle = RDFExporter().export_to_rdf( + { + "entities": [ + {"id": "mailto:foo", "type": "isbn:0451450523"}, + {"id": "http://example.org/bad%zz", "type": "PERSON"}, + ], + "relationships": [], + }, + format="turtle", + ) + parsed = Graph().parse(data=turtle, format="turtle") + + assert ( + URIRef("mailto:foo"), + RDF.type, + URIRef("isbn:0451450523"), + ) in parsed + assert URIRef("http://example.org/bad%25zz") in parsed.all_nodes() + + +def test_turtle_normalizes_temporal_relationship_endpoints(): + """Temporal relationship metadata uses the same normalized resource IRIs.""" + turtle = RDFExporter().export_to_rdf( + { + "entities": [ + {"id": "Jane Doe", "type": "PERSON"}, + {"id": "Kochi, Kerala", "type": "LOCATION"}, + ], + "relationships": [ + { + "source": "Jane Doe", + "target": "Kochi, Kerala", + "type": "located_in", + "valid_from": "2024-01-01T00:00:00+00:00", + "valid_until": "2024-02-01T00:00:00+00:00", + } + ], + }, + format="turtle", + include_temporal=True, + ) + parsed = Graph().parse(data=turtle, format="turtle") + + assert ( + None, + URIRef("https://semantica.dev/ns#source"), + URIRef("https://semantica.dev/ns#Jane%20Doe"), + ) in parsed + assert ( + None, + URIRef("https://semantica.dev/ns#target"), + URIRef("https://semantica.dev/ns#Kochi%2C%20Kerala"), + ) in parsed + + +def test_turtle_expands_context_prefixes_and_mints_relative_values(): + """Context prefixes expand while bare values use the fallback namespace.""" + turtle = RDFExporter().export_to_rdf( + { + "@context": {"ex": "https://example.org/"}, + "entities": [{"id": "ORG", "type": "ex:Person"}], + "relationships": [], + }, + format="turtle", + ) + parsed = Graph().parse(data=turtle, format="turtle") + + assert ( + URIRef("https://semantica.dev/ns#ORG"), + RDF.type, + URIRef("https://example.org/Person"), + ) in parsed + + +def test_rdfxml_normalizes_resource_iris(): + """RDF/XML resource attributes use the same safe absolute IRIs.""" + data = { + "entities": [ + {"id": "Acme Corp", "type": "Person"}, + {"id": "mailto:foo", "type": "isbn:0451450523"}, + ], + "relationships": [ + {"source": "Jane Doe", "target": "Acme Corp", "type": "knows"} + ], + } + rdfxml = RDFExporter().export_to_rdf(data, format="rdfxml") + parsed = Graph().parse(data=rdfxml, format="xml") + + assert URIRef("https://semantica.dev/ns#Acme%20Corp") in parsed.all_nodes() + assert URIRef("mailto:foo") in parsed.all_nodes() + + +def test_ntriples_normalizes_resource_iris(): + """N-Triples resource IRIs reject neither spaces nor opaque schemes.""" + data = { + "entities": [ + {"id": "Acme Corp", "type": "Person"}, + {"id": "mailto:foo", "type": "isbn:0451450523"}, + ], + "relationships": [ + {"source": "Jane Doe", "target": "Acme Corp", "type": "knows"} + ], + } + ntriples = RDFExporter().export_to_rdf(data, format="ntriples") + parsed = Graph().parse(data=ntriples, format="nt") + + assert URIRef("https://semantica.dev/ns#Acme%20Corp") in parsed.all_nodes() + assert URIRef("mailto:foo") in parsed.all_nodes() + + +def test_turtle_preserves_existing_valid_percent_escapes(): + """A pre-encoded absolute IRI keeps its escape, instead of %20 -> %2520.""" + turtle = RDFExporter().export_to_rdf( + { + "entities": [ + { + "id": "https://example.org/entities/path%20name", + "type": "PERSON", + } + ], + "relationships": [], + }, + format="turtle", + ) + parsed = Graph().parse(data=turtle, format="turtle") + + assert ( + URIRef("https://example.org/entities/path%20name"), + RDF.type, + URIRef("https://semantica.dev/ns#PERSON"), + ) in parsed + assert "%2520" not in turtle + + +def test_ntriples_and_rdfxml_expand_builtin_prefixes_alongside_context(): + """A user @context must not shadow built-in prefixes like semantica:.""" + data = { + "@context": {"ex": "https://example.org/"}, + "entities": [{"id": "ORG", "type": "semantica:Entity"}], + "relationships": [], + } + + ntriples = RDFExporter().export_to_rdf(data, format="ntriples") + nt_parsed = Graph().parse(data=ntriples, format="nt") + assert ( + URIRef("https://semantica.dev/ns#ORG"), + RDF.type, + URIRef("https://semantica.dev/ns#Entity"), + ) in nt_parsed + + rdfxml = RDFExporter().export_to_rdf(data, format="rdfxml") + xml_parsed = Graph().parse(data=rdfxml, format="xml") + assert ( + URIRef("https://semantica.dev/ns#ORG"), + RDF.type, + URIRef("https://semantica.dev/ns#Entity"), + ) in xml_parsed