mengzhx/semantica
1
0
Fork 0
mirror of https://github.com/semantica-agi/semantica.git synced 2026-08-29 04:26:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main' into fix/rdf4j-repository-id

Browse Source
This commit is contained in:
Freakz2z
2026-08-24 09:02:42 +08:00
parent 4c997b5017 6c2ccfd3af
commit 49db007691
2 changed files with 368 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
semantica/export/rdf_exporter.py
+135 -35
View File
@@ -29,9 +29,12 @@ Author: Semantica Contributors
License: MIT
"""
import re
from pathlib import Path
from decimal import Decimal, InvalidOperation
from html import escape as xml_escape
from typing import Any, Dict, List, Optional, Set, Union
from urllib.parse import quote, urlsplit
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.helpers import ensure_directory, hash_data
@@ -104,7 +107,11 @@ def normalize_confidence(value: Any) -> Optional[str]:
# "1e100000000" is eleven characters that expand to a hundred million, and
# the export path continues past validation errors, so a single malformed
# field could exhaust memory. Nothing near this magnitude is a confidence.
if not -MAX_CONFIDENCE_EXPONENT <= decimal_value.adjusted() <= MAX_CONFIDENCE_EXPONENT:
if (
not -MAX_CONFIDENCE_EXPONENT
<= decimal_value.adjusted()
<= MAX_CONFIDENCE_EXPONENT
):
return None
# `str(Decimal("0.00001"))` gives "0.00001", but a float that has already
@@ -395,6 +402,66 @@ class RDFSerializer:
# OWL-Time namespace URI
_OWL_TIME_NS = "http://www.w3.org/2006/time#"
_SEMANTICA_NS = "https://semantica.dev/ns#"
# Matches an already-valid percent-escape so it can be passed through
# unchanged instead of being re-encoded into e.g. %2520.
_PERCENT_ESCAPE_RE = re.compile(r"%[0-9A-Fa-f]{2}")
@classmethod
def _quote_preserving_escapes(cls, value: str, safe: str) -> str:
"""quote() that leaves existing valid %XX escapes untouched.
Blanket-quoting an absolute IRI double-encodes any percent-escape it
already carries (%20 -> %2520), which changes the identity of every
previously-valid IRI containing one. Only the spans between existing
valid escapes are quoted; a bare '%' that isn't part of a valid
escape (e.g. "%zz") still gets encoded to %25, keeping the malformed
case handled.
"""
parts = []
pos = 0
for match in cls._PERCENT_ESCAPE_RE.finditer(value):
parts.append(quote(value[pos : match.start()], safe=safe))
parts.append(match.group(0))
pos = match.end()
parts.append(quote(value[pos:], safe=safe))
return "".join(parts)
def _as_turtle_iri(
self, value: Any, namespaces: Optional[Dict[str, str]] = None
) -> str:
"""Return an absolute, safely encoded IRI for a Turtle resource."""
value = str(value)
try:
parsed = urlsplit(value)
except ValueError:
parsed = urlsplit("")
if parsed.scheme:
prefix, separator, local_name = value.partition(":")
# Built-in namespaces (semantica:, rdf:, rdfs:, owl:, ...) must
# always be resolvable, not only when the caller passes no
# namespaces of its own — otherwise a value like "semantica:Foo"
# resolves fine with no @context but stops resolving the moment
# any @context is present, since callers pass extract_namespaces()
# (context-only) here without merging in the built-ins.
effective_namespaces = {
**self.namespace_manager.namespaces,
**(namespaces or {}),
}
namespace = effective_namespaces.get(prefix)
if namespace and separator:
return self._quote_preserving_escapes(
namespace + local_name, safe=":/?#[]@!$&'()*+,;="
)
# A scheme with at least two characters is an absolute IRI,
# including opaque forms such as mailto:foo and isbn:0451450523.
# Keep one-character schemes as the existing Windows drive-path case.
if len(prefix) >= 2:
return self._quote_preserving_escapes(
value, safe=":/?#[]@!$&'()*+,;="
)
return self._SEMANTICA_NS + quote(value, safe="")
# Design decision — TemporalBound.OPEN in RDF:
# OWL-Time has no standard predicate for "no known end date." We use
@@ -467,7 +534,10 @@ class RDFSerializer:
text = entity.get("text") or entity.get("label", "")
confidence = normalize_confidence(entity.get("confidence", 1.0))
lines.append(f"<{entity_id}> a <{entity_type}> ;")
lines.append(
f"<{self._as_turtle_iri(entity_id, merged_namespaces)}> a "
f"<{self._as_turtle_iri(entity_type, merged_namespaces)}> ;"
)
if confidence is None:
self.logger.warning(
f"Entity {entity_id} has a confidence that is not a number "
@@ -488,10 +558,16 @@ class RDFSerializer:
target_id = rel.get("target_id") or rel.get("target")
rel_type = rel.get("type", DEFAULT_RELATION_TYPE)
lines.append(f"<{source_id}> <{rel_type}> <{target_id}> .")
lines.append(
f"<{self._as_turtle_iri(source_id, merged_namespaces)}> "
f"<{self._as_turtle_iri(rel_type, merged_namespaces)}> "
f"<{self._as_turtle_iri(target_id, merged_namespaces)}> ."
)
if include_temporal:
owl_lines = self._owl_time_triples_for_rel(rel, idx, time_axis)
owl_lines = self._owl_time_triples_for_rel(
rel, idx, time_axis, merged_namespaces
)
if owl_lines:
# The interval hangs off the relationship's own IRI, and a
# relationship written as a single triple has no such node
@@ -501,7 +577,7 @@ class RDFSerializer:
# export, and every term is declared in the vocabulary.
lines.extend(
self._reified_relationship_triples(
rel, idx, source_id, target_id, rel_type
rel, idx, source_id, target_id, rel_type, merged_namespaces
)
)
lines.extend(owl_lines)
@@ -515,6 +591,7 @@ class RDFSerializer:
source_id: str,
target_id: str,
rel_type: str,
namespaces: Optional[Dict[str, str]] = None,
) -> List[str]:
"""
Emit the reified relationship node that OWL-Time triples hang off.
@@ -523,7 +600,11 @@ class RDFSerializer:
to, using the same sem:Relationship shape the JSON-LD export already
writes, so the two serializations describe relationships the same way.
"""
rel_id = rel.get("id") or mint_relationship_iri(idx, source_id or "", target_id or "")
rel_id = self._as_turtle_iri(
rel.get("id")
or mint_relationship_iri(idx, source_id or "", target_id or ""),
namespaces,
)
# The full predicate, not its local name. Truncating to the fragment
# made https://a.example/ns#employs and https://b.example/ns#employs the
@@ -537,17 +618,25 @@ class RDFSerializer:
.replace("\r", "\\r")
)
predicates = [f"a semantica:Relationship"]
predicates = ["a semantica:Relationship"]
if source_id:
predicates.append(f"semantica:source <{source_id}>")
predicates.append(
f"semantica:source <{self._as_turtle_iri(source_id, namespaces)}>"
)
if target_id:
predicates.append(f"semantica:target <{target_id}>")
predicates.append(
f"semantica:target <{self._as_turtle_iri(target_id, namespaces)}>"
)
predicates.append(f'semantica:type "{escaped}"')
return ["", f"<{rel_id}> " + " ;\n ".join(predicates) + " ."]
def _owl_time_triples_for_rel(
self, rel: Dict[str, Any], idx: int, time_axis: str
self,
rel: Dict[str, Any],
idx: int,
time_axis: str,
namespaces: Optional[Dict[str, str]] = None,
) -> List[str]:
"""
Emit OWL-Time Turtle triples for a relationship that carries temporal metadata.
@@ -562,7 +651,7 @@ class RDFSerializer:
def _is_open(v: Any) -> bool:
if v is None:
return False
if hasattr(v, "value"): # TemporalBound enum
if hasattr(v, "value"): # TemporalBound enum
return v.value == _OPEN_SENTINEL
return str(v).strip().upper() == _OPEN_SENTINEL
@@ -579,7 +668,10 @@ class RDFSerializer:
# deterministic IRI.
source_id = rel.get("source_id") or rel.get("source") or ""
target_id = rel.get("target_id") or rel.get("target") or ""
rel_base_id = rel.get("id") or mint_relationship_iri(idx, source_id, target_id)
rel_base_id = self._as_turtle_iri(
rel.get("id") or mint_relationship_iri(idx, source_id, target_id),
namespaces,
)
lines = [""] # blank separator
for axis_name, from_val, until_val in axes:
@@ -594,9 +686,7 @@ class RDFSerializer:
lines.append(f" time:hasBeginning <{begin_id}> ;")
if _is_open(until_val):
lines.append(
' semantica:openEndedInterval "true"^^xsd:boolean .'
)
lines.append(' semantica:openEndedInterval "true"^^xsd:boolean .')
elif until_val is not None:
end_id = f"{rel_base_id}__{axis_name}_end"
lines.append(f" time:hasEnd <{end_id}> .")
@@ -605,7 +695,9 @@ class RDFSerializer:
f' time:inXSDDateTimeStamp "{until_val}"^^xsd:dateTimeStamp .'
)
else:
lines[-1] = lines[-1].rstrip(" ;") + " ." # close interval without hasEnd
lines[-1] = (
lines[-1].rstrip(" ;") + " ."
) # close interval without hasEnd
lines.append(f"<{begin_id}> a time:Instant ;")
lines.append(
@@ -644,6 +736,8 @@ class RDFSerializer:
lines.append(' xmlns:semantica="https://semantica.dev/ns#">')
lines.append("")
namespaces = self.namespace_manager.extract_namespaces(rdf_data)
# Convert entities to RDF/XML
entities = rdf_data.get("entities", [])
for entity in entities:
@@ -653,13 +747,19 @@ class RDFSerializer:
entity_text = entity.get("text", "")
entity_id = mint_entity_iri(entity_text)
entity_type = entity.get("type", DEFAULT_ENTITY_TYPE)
entity_type = entity.get("type") or DEFAULT_ENTITY_TYPE
text = entity.get("text") or entity.get("label", "")
confidence = normalize_confidence(entity.get("confidence", 1.0))
# RDF/XML syntax: rdf:Description with rdf:about
lines.append(f' <rdf:Description rdf:about="{entity_id}">')
lines.append(f' <rdf:type rdf:resource="{entity_type}"/>')
entity_iri = xml_escape(
self._as_turtle_iri(entity_id, namespaces), quote=True
)
entity_type_iri = xml_escape(
self._as_turtle_iri(entity_type, namespaces), quote=True
)
lines.append(f' <rdf:Description rdf:about="{entity_iri}">')
lines.append(f' <rdf:type rdf:resource="{entity_type_iri}"/>')
lines.append(f" <semantica:text>{text}</semantica:text>")
if confidence is None:
self.logger.warning(
@@ -679,11 +779,19 @@ class RDFSerializer:
for rel in relationships:
source_id = rel.get("source_id") or rel.get("source")
target_id = rel.get("target_id") or rel.get("target")
rel_type = rel.get("type", "semantica:related_to")
# RDF/XML predicates are emitted as QNames, unlike resource
# attributes which use the shared absolute-IRI normalizer.
rel_type = rel.get("type") or "semantica:related_to"
# Relationship as property on source entity
lines.append(f' <rdf:Description rdf:about="{source_id}">')
lines.append(f' <{rel_type} rdf:resource="{target_id}"/>')
source_iri = xml_escape(
self._as_turtle_iri(source_id, namespaces), quote=True
)
target_iri = xml_escape(
self._as_turtle_iri(target_id, namespaces), quote=True
)
lines.append(f' <rdf:Description rdf:about="{source_iri}">')
lines.append(f' <{rel_type} rdf:resource="{target_iri}"/>')
lines.append(" </rdf:Description>")
lines.append("")
@@ -803,20 +911,12 @@ class RDFSerializer:
"""
lines = []
namespaces = self.namespace_manager.extract_namespaces(rdf_data)
def expand_uri(uri: str) -> str:
if not uri:
return ""
if uri.startswith("http"):
return f"<{uri}>"
if uri.startswith("semantica:"):
return f"<https://semantica.dev/ns#{uri.split(':', 1)[1]}>"
if uri.startswith("rdf:"):
return f"<http://www.w3.org/1999/02/22-rdf-syntax-ns#{uri.split(':', 1)[1]}>"
if uri.startswith("rdfs:"):
return f"<http://www.w3.org/2000/01/rdf-schema#{uri.split(':', 1)[1]}>"
if ":" in uri:
return f"<{uri}>"
return f"<https://semantica.dev/ns#{uri}>"
return f"<{self._as_turtle_iri(uri, namespaces)}>"
# Convert entities
entities = rdf_data.get("entities", [])
@@ -830,7 +930,7 @@ class RDFSerializer:
subject = expand_uri(entity_id)
# Type triple
entity_type = entity.get("type", "semantica:Entity")
entity_type = entity.get("type") or DEFAULT_ENTITY_TYPE
lines.append(
f"{subject} <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> {expand_uri(entity_type)} ."
)
@@ -864,7 +964,7 @@ class RDFSerializer:
for rel in relationships:
source_id = rel.get("source_id") or rel.get("source")
target_id = rel.get("target_id") or rel.get("target")
rel_type = rel.get("type", "semantica:related_to")
rel_type = rel.get("type") or DEFAULT_RELATION_TYPE
if source_id and target_id:
lines.append(
tests/export/test_rdf_exporter_turtle_iris.py
+233
View File
@@ -0,0 +1,233 @@
"""Regression tests for valid Turtle IRI generation (issue #1099)."""
from rdflib import RDF, Graph, URIRef
from semantica.export import RDFExporter
from semantica.kg.graph_builder import GraphBuilder
def test_turtle_normalizes_graph_builder_default_identifiers():
"""Default GraphBuilder labels with spaces become stable absolute IRIs."""
source = {
"entities": [
{
"id": "Kochi, Kerala",
"name": "Kochi, Kerala",
"type": "LOCATION",
},
{"id": "Jane Doe", "name": "Jane Doe", "type": "PERSON"},
],
"relationships": [
{
"source": "Jane Doe",
"target": "Kochi, Kerala",
"type": "located_in",
},
],
}
graph_data = GraphBuilder(resolve_conflicts=False).build(sources=[source])
turtle = RDFExporter().export_to_rdf(graph_data, format="turtle")
parsed = Graph().parse(data=turtle, format="turtle")
assert "<Jane Doe>" not in turtle
assert "<Kochi, Kerala>" not in turtle
kochi = URIRef("https://semantica.dev/ns#Kochi%2C%20Kerala")
jane = URIRef("https://semantica.dev/ns#Jane%20Doe")
assert (
kochi,
RDF.type,
URIRef("https://semantica.dev/ns#LOCATION"),
) in parsed
jane_type = URIRef("https://semantica.dev/ns#PERSON")
assert (jane, RDF.type, jane_type) in parsed
assert (
jane,
URIRef("https://semantica.dev/ns#located_in"),
kochi,
) in parsed
def test_turtle_preserves_absolute_iris():
"""Already-valid absolute resource IRIs remain unchanged."""
turtle = RDFExporter().export_to_rdf(
{
"entities": [
{
"id": "https://example.org/entities/jane",
"text": "Jane",
"type": "urn:example:Person",
}
],
"relationships": [],
},
format="turtle",
)
parsed = Graph().parse(data=turtle, format="turtle")
assert (
URIRef("https://example.org/entities/jane"),
RDF.type,
URIRef("urn:example:Person"),
) in parsed
def test_turtle_preserves_opaque_absolute_iris_and_encodes_bad_percent_escapes():
"""Opaque schemes remain absolute and malformed percent escapes are encoded."""
turtle = RDFExporter().export_to_rdf(
{
"entities": [
{"id": "mailto:foo", "type": "isbn:0451450523"},
{"id": "http://example.org/bad%zz", "type": "PERSON"},
],
"relationships": [],
},
format="turtle",
)
parsed = Graph().parse(data=turtle, format="turtle")
assert (
URIRef("mailto:foo"),
RDF.type,
URIRef("isbn:0451450523"),
) in parsed
assert URIRef("http://example.org/bad%25zz") in parsed.all_nodes()
def test_turtle_normalizes_temporal_relationship_endpoints():
"""Temporal relationship metadata uses the same normalized resource IRIs."""
turtle = RDFExporter().export_to_rdf(
{
"entities": [
{"id": "Jane Doe", "type": "PERSON"},
{"id": "Kochi, Kerala", "type": "LOCATION"},
],
"relationships": [
{
"source": "Jane Doe",
"target": "Kochi, Kerala",
"type": "located_in",
"valid_from": "2024-01-01T00:00:00+00:00",
"valid_until": "2024-02-01T00:00:00+00:00",
}
],
},
format="turtle",
include_temporal=True,
)
parsed = Graph().parse(data=turtle, format="turtle")
assert (
None,
URIRef("https://semantica.dev/ns#source"),
URIRef("https://semantica.dev/ns#Jane%20Doe"),
) in parsed
assert (
None,
URIRef("https://semantica.dev/ns#target"),
URIRef("https://semantica.dev/ns#Kochi%2C%20Kerala"),
) in parsed
def test_turtle_expands_context_prefixes_and_mints_relative_values():
"""Context prefixes expand while bare values use the fallback namespace."""
turtle = RDFExporter().export_to_rdf(
{
"@context": {"ex": "https://example.org/"},
"entities": [{"id": "ORG", "type": "ex:Person"}],
"relationships": [],
},
format="turtle",
)
parsed = Graph().parse(data=turtle, format="turtle")
assert (
URIRef("https://semantica.dev/ns#ORG"),
RDF.type,
URIRef("https://example.org/Person"),
) in parsed
def test_rdfxml_normalizes_resource_iris():
"""RDF/XML resource attributes use the same safe absolute IRIs."""
data = {
"entities": [
{"id": "Acme Corp", "type": "Person"},
{"id": "mailto:foo", "type": "isbn:0451450523"},
],
"relationships": [
{"source": "Jane Doe", "target": "Acme Corp", "type": "knows"}
],
}
rdfxml = RDFExporter().export_to_rdf(data, format="rdfxml")
parsed = Graph().parse(data=rdfxml, format="xml")
assert URIRef("https://semantica.dev/ns#Acme%20Corp") in parsed.all_nodes()
assert URIRef("mailto:foo") in parsed.all_nodes()
def test_ntriples_normalizes_resource_iris():
"""N-Triples resource IRIs reject neither spaces nor opaque schemes."""
data = {
"entities": [
{"id": "Acme Corp", "type": "Person"},
{"id": "mailto:foo", "type": "isbn:0451450523"},
],
"relationships": [
{"source": "Jane Doe", "target": "Acme Corp", "type": "knows"}
],
}
ntriples = RDFExporter().export_to_rdf(data, format="ntriples")
parsed = Graph().parse(data=ntriples, format="nt")
assert URIRef("https://semantica.dev/ns#Acme%20Corp") in parsed.all_nodes()
assert URIRef("mailto:foo") in parsed.all_nodes()
def test_turtle_preserves_existing_valid_percent_escapes():
"""A pre-encoded absolute IRI keeps its escape, instead of %20 -> %2520."""
turtle = RDFExporter().export_to_rdf(
{
"entities": [
{
"id": "https://example.org/entities/path%20name",
"type": "PERSON",
}
],
"relationships": [],
},
format="turtle",
)
parsed = Graph().parse(data=turtle, format="turtle")
assert (
URIRef("https://example.org/entities/path%20name"),
RDF.type,
URIRef("https://semantica.dev/ns#PERSON"),
) in parsed
assert "%2520" not in turtle
def test_ntriples_and_rdfxml_expand_builtin_prefixes_alongside_context():
"""A user @context must not shadow built-in prefixes like semantica:."""
data = {
"@context": {"ex": "https://example.org/"},
"entities": [{"id": "ORG", "type": "semantica:Entity"}],
"relationships": [],
}
ntriples = RDFExporter().export_to_rdf(data, format="ntriples")
nt_parsed = Graph().parse(data=ntriples, format="nt")
assert (
URIRef("https://semantica.dev/ns#ORG"),
RDF.type,
URIRef("https://semantica.dev/ns#Entity"),
) in nt_parsed
rdfxml = RDFExporter().export_to_rdf(data, format="rdfxml")
xml_parsed = Graph().parse(data=rdfxml, format="xml")
assert (
URIRef("https://semantica.dev/ns#ORG"),
RDF.type,
URIRef("https://semantica.dev/ns#Entity"),
) in xml_parsed