diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cb0fdd7..7ef8e1bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Every timestamp an export or a provenance record wrote was timezone-naive** (closes #1114) by @fabio-rovai + - `semantica/export/` stamped with `datetime.now().isoformat()`, which reads the machine's **local** clock; `semantica/provenance/` stamped with `datetime.utcnow().isoformat()`, which reads **UTC**. Both produce a naive value and both serialize identically, so nothing downstream can tell which zone a given timestamp belongs to — the same string means two different instants depending on which module wrote it + - In RDF the consequence is silent rather than loud. Under XSD 1.1 a value with no timezone compared against one with a timezone is indeterminate whenever the two fall inside the ±14 hour window; SPARQL turns an indeterminate comparison into an error, and `FILTER` discards errors as non-matches. A timezone-qualified query over an Oxigraph store returns an answer with every Semantica-written record quietly absent from it, which is a poor property for `prov:generatedAtTime`, `prov:startedAtTime`, `prov:endedAtTime` and `prov:atTime` to have + - New `utc_now()`/`utc_now_iso()` in `semantica/utils/helpers.py`, exported from `semantica.utils`, and used at all 29 call sites in `export/` (`json_exporter`, `yaml_exporter`, `report_generator`, `export_provenance`) and `provenance/` (`manager`, `schemas`, `bridge_axiom`). Values now read `2026-08-19T14:19:04.229937+00:00`: one unambiguous instant, comparable against any correctly stamped value, and valid `xsd:dateTimeStamp`. `sem:exportedAt`'s range in `semantica/ontology/vocabulary/semantica-ns.ttl` is tightened from `xsd:dateTime` accordingly, and its comment no longer has to explain why the weaker range was necessary + - `datetime.utcnow()` is deprecated as of Python 3.12 and scheduled for removal; constructing a `ProvenanceEntry` under `-W error::DeprecationWarning` on 3.13 raised, and no longer does + - New `tests/export/test_timestamp_timezones.py` and `tests/provenance/test_timestamp_timezones.py`: offset presence on every export and provenance path, PROV-O literals valid as `xsd:dateTimeStamp`, comparison against a timezone-aware instant without `TypeError`, the Oxigraph filter that dropped the naive value (with a bound inside the indeterminate window, so the test cannot pass by accident), and the document `@id` remaining a valid IRI with `+00:00` in it. 11 of the 13 fail on the parent commit + - **Fixed during review** (Qodo): once new entries carry `+00:00` and stored ones do not, `ProvenanceManager.query_recorded_between` and `audit_log` compared ISO timestamps as raw strings, so they ordered by spelling rather than by instant — an inclusive naive bound naming a stored offset-bearing timestamp sorted *below* it and dropped the record, and a bound written in another offset landed wherever its digits fell (`19:45+05:30` is 14:15Z, but sorted after 14:19Z). Both now compare instants through a new `to_utc_datetime()` helper that reads a missing offset as UTC, which is what the values written before this change actually were; a bound that cannot be read as a timestamp keeps the historical string comparison rather than raising on a call that used to work + - The remaining 147 naive call sites are in `context/`, `vector_store/`, `seed/` and elsewhere, where timestamps are compared against values parsed from previously stored naive strings. Converting those without a read-side migration would raise `TypeError: can't compare offset-naive and offset-aware datetimes` on existing data, so they are deliberately left for a separate change + - **`split`/chunking paths bypassed the centralized spaCy model cache, reloading the model on every call** (#1042, closes #998) by @Accute9, reviewed by @Sameer6305 - `semantica/split/methods.py`'s `split_by_sentences()` and `semantica/split/semantic_chunker.py`'s `SemanticChunker.__init__` each called `spacy.load()` directly instead of reusing the process-level cache added in #889/`semantic_extract/methods.py`'s `load_spacy_model()` — every call/construction re-paid the ~120ms model-load cost independently of `NERExtractor`, which already used the cache - Both now route through `load_spacy_model()`, sharing one cached `Language` instance per model name across `split_by_sentences()`, `SemanticChunker`, and `NERExtractor`; a missing model still falls back to regex/paragraph chunking without poisoning the cache for a later successful load diff --git a/semantica/export/export_provenance.py b/semantica/export/export_provenance.py index 551b0f2c..07925d75 100644 --- a/semantica/export/export_provenance.py +++ b/semantica/export/export_provenance.py @@ -14,9 +14,10 @@ License: MIT """ from typing import Any, Optional -from datetime import datetime import uuid +from ..utils.helpers import utc_now_iso + class ExporterWithProvenance: """Base exporter with provenance tracking.""" @@ -45,9 +46,9 @@ class ExporterWithProvenance: def export(self, data: Any, destination: str, **kwargs): """Export data with provenance tracking.""" - activity_started_at_time = datetime.utcnow().isoformat() + activity_started_at_time = utc_now_iso() result = self._exporter.export(data, destination, **kwargs) - activity_ended_at_time = datetime.utcnow().isoformat() + activity_ended_at_time = utc_now_iso() if self.provenance and self._prov_manager: self._prov_manager.track_entity( diff --git a/semantica/export/json_exporter.py b/semantica/export/json_exporter.py index aa1641ae..c9c587de 100644 --- a/semantica/export/json_exporter.py +++ b/semantica/export/json_exporter.py @@ -24,12 +24,11 @@ License: MIT """ import json -from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError, ValidationError -from ..utils.helpers import ensure_directory, write_json_file +from ..utils.helpers import ensure_directory, utc_now_iso, write_json_file from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker from .rdf_exporter import SEMANTICA_NS, mint_entity_iri, mint_relationship_iri @@ -271,7 +270,7 @@ class JSONExporter: }, "entities": entities, "metadata": { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "entity_count": len(entities), **options.get("metadata", {}), }, @@ -304,7 +303,7 @@ class JSONExporter: }, "relationships": relationships, "metadata": { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "relationship_count": len(relationships), **options.get("metadata", {}), }, @@ -342,7 +341,7 @@ class JSONExporter: if include_metadata: if "metadata" not in result: result["metadata"] = {} - result["metadata"]["exported_at"] = datetime.now().isoformat() + result["metadata"]["exported_at"] = utc_now_iso() if include_provenance: result["metadata"]["format"] = "json" @@ -352,7 +351,7 @@ class JSONExporter: "data": data, "count": len(data), "metadata": { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "format": "json" if include_provenance else None, **options.get("metadata", {}), }, @@ -361,7 +360,7 @@ class JSONExporter: # Single value return { "value": data, - "metadata": {"exported_at": datetime.now().isoformat()} + "metadata": {"exported_at": utc_now_iso()} if include_metadata else {}, } @@ -413,9 +412,9 @@ class JSONExporter: # Add metadata and provenance if requested if include_metadata: - jsonld["@id"] = f"https://semantica.dev/data/{datetime.now().isoformat()}" + jsonld["@id"] = f"https://semantica.dev/data/{utc_now_iso()}" if include_provenance: - jsonld["semantica:exportedAt"] = datetime.now().isoformat() + jsonld["semantica:exportedAt"] = utc_now_iso() jsonld["semantica:format"] = "json-ld" return jsonld @@ -447,7 +446,7 @@ class JSONExporter: "nodes": kg.get("nodes", []), "edges": kg.get("edges", []), "metadata": { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), **kg.get("metadata", {}), **options.get("metadata", {}), }, @@ -484,7 +483,7 @@ class JSONExporter: "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdfs": "http://www.w3.org/2000/01/rdf-schema#", }, - "@id": f"https://semantica.dev/graph/{datetime.now().isoformat()}", + "@id": f"https://semantica.dev/graph/{utc_now_iso()}", "@type": "semantica:KnowledgeGraph", } @@ -506,7 +505,7 @@ class JSONExporter: ) # Add metadata - jsonld["semantica:exportedAt"] = datetime.now().isoformat() + jsonld["semantica:exportedAt"] = utc_now_iso() if "metadata" in kg: jsonld["semantica:metadata"] = kg["metadata"] diff --git a/semantica/export/report_generator.py b/semantica/export/report_generator.py index fcee11a1..be9f92be 100644 --- a/semantica/export/report_generator.py +++ b/semantica/export/report_generator.py @@ -25,12 +25,11 @@ License: MIT import html import json -from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError, ValidationError -from ..utils.helpers import ensure_directory +from ..utils.helpers import ensure_directory, utc_now_iso from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -252,7 +251,7 @@ class ReportGenerator: # Build report data with summary report_data = { "title": "Quality Assurance Report", - "generated_at": datetime.now().isoformat(), + "generated_at": utc_now_iso(), "metrics": quality_metrics, "summary": self._generate_quality_summary(quality_metrics), } @@ -278,7 +277,7 @@ class ReportGenerator: """ report_data = { "title": "Analysis Report", - "generated_at": datetime.now().isoformat(), + "generated_at": utc_now_iso(), "analysis": analysis_results, "summary": self._generate_analysis_summary(analysis_results), } @@ -304,7 +303,7 @@ class ReportGenerator: """ report_data = { "title": "Framework Metrics Report", - "generated_at": datetime.now().isoformat(), + "generated_at": utc_now_iso(), "metrics": metrics, "summary": self._generate_metrics_summary(metrics), } diff --git a/semantica/export/yaml_exporter.py b/semantica/export/yaml_exporter.py index ecbd55fa..a9befedb 100644 --- a/semantica/export/yaml_exporter.py +++ b/semantica/export/yaml_exporter.py @@ -22,7 +22,6 @@ License: MIT """ from collections.abc import Mapping -from datetime import datetime from pathlib import Path from typing import Any, Dict, List, Optional, Union @@ -33,6 +32,7 @@ from ..utils.helpers import ( _require_recognized_keys, ensure_directory, normalize_graph_payload, + utc_now_iso, ) from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -216,7 +216,7 @@ class SemanticNetworkYAMLExporter: records = normalize_graph_payload(semantic_network) yaml_data = { "metadata": { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "version": "1.0", **semantic_network.get("metadata", {}), }, @@ -309,7 +309,7 @@ class SemanticNetworkYAMLExporter: if include_metadata: yaml_data["metadata"] = { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "entity_count": len(entities), } @@ -333,7 +333,7 @@ class SemanticNetworkYAMLExporter: if include_properties: yaml_data["metadata"] = { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "relationship_count": len(relationships), } @@ -370,7 +370,7 @@ class SemanticNetworkYAMLExporter: } yaml_data["metadata"] = { - "exported_at": datetime.now().isoformat(), + "exported_at": utc_now_iso(), "triplet_count": len(triplets), } @@ -410,7 +410,7 @@ class SemanticNetworkYAMLExporter: yaml_data = { "pipeline_stage": pipeline_stage, "metadata": { - "extracted_at": datetime.now().isoformat(), + "extracted_at": utc_now_iso(), **extracted_data.get("metadata", {}), }, "semantic_network": semantic_network, diff --git a/semantica/ontology/vocabulary/semantica-ns.ttl b/semantica/ontology/vocabulary/semantica-ns.ttl index c6843a32..36e206bc 100644 --- a/semantica/ontology/vocabulary/semantica-ns.ttl +++ b/semantica/ontology/vocabulary/semantica-ns.ttl @@ -118,10 +118,12 @@ sem:relationships a owl:ObjectProperty ; sem:exportedAt a owl:DatatypeProperty ; rdfs:label "exported at" ; - rdfs:comment """When the export was written. Emitted as an ISO 8601 local -timestamp, so the range is xsd:dateTime rather than xsd:dateTimeStamp: the -values carry no timezone offset.""" ; - rdfs:range xsd:dateTime ; + rdfs:comment """When the export was written, as an ISO 8601 timestamp with +an explicit UTC offset. The range was xsd:dateTime while the exporters stamped +with a naive datetime.now(); with the offset present (#1114) the value is a +determinate instant, comparable against a timestamp written anywhere else, so +the range is the stricter xsd:dateTimeStamp, which requires the offset.""" ; + rdfs:range xsd:dateTimeStamp ; rdfs:isDefinedBy . sem:format a owl:DatatypeProperty ; diff --git a/semantica/provenance/bridge_axiom.py b/semantica/provenance/bridge_axiom.py index e04f054e..e5e96696 100644 --- a/semantica/provenance/bridge_axiom.py +++ b/semantica/provenance/bridge_axiom.py @@ -61,9 +61,10 @@ License: MIT from dataclasses import dataclass, field from typing import Optional, Dict, Any, List -from datetime import datetime import uuid +from ..utils.helpers import utc_now_iso + @dataclass class BridgeAxiom: @@ -280,7 +281,7 @@ class TranslationChain: "type": layer_type, "value": value, "source": source, - "timestamp": datetime.utcnow().isoformat(), + "timestamp": utc_now_iso(), **kwargs } self.layers.append(layer) diff --git a/semantica/provenance/manager.py b/semantica/provenance/manager.py index 8c2d1738..1a2b26f1 100644 --- a/semantica/provenance/manager.py +++ b/semantica/provenance/manager.py @@ -26,7 +26,7 @@ License: MIT from typing import Optional, List, Dict, Any, Union from collections.abc import Mapping -from datetime import datetime +from datetime import datetime, timezone from contextlib import contextmanager import copy import inspect @@ -36,8 +36,13 @@ import threading from .schemas import ProvenanceEntry, SourceReference, AgentRecord, ActivityRecord from .storage import ProvenanceStorage, InMemoryStorage, SQLiteStorage from .integrity import compute_checksum, verify_checksum +from ..utils.helpers import to_utc_datetime, utc_now_iso from ..utils.logging import get_logger +#: Sort key for an entry whose timestamp cannot be read as one, so an +#: unreadable value orders first instead of raising during a sort. +_EPOCH = datetime(1, 1, 1, tzinfo=timezone.utc) + # Issue #825, Part B Tier 3 — configurable base URI for export_prov(), shared # with RDFExporter's NamespaceManager "semantica" entry (semantica/export/ # rdf_exporter.py) so KG-exported and PROV-exported URIs for the same @@ -363,8 +368,8 @@ class ProvenanceManager: source_quote=kwargs.get("source_quote"), confidence=kwargs.get("confidence", 1.0), metadata=metadata or {}, - first_seen=existing.first_seen if existing else datetime.utcnow().isoformat(), - last_updated=datetime.utcnow().isoformat(), + first_seen=existing.first_seen if existing else utc_now_iso(), + last_updated=utc_now_iso(), parent_entity_id=parent_id, used_entities=list(kwargs.get("used_entities", [])), activity_started_at_time=activity_info["activity_started_at_time"], @@ -455,8 +460,8 @@ class ProvenanceManager: source_location=kwargs.get("source_location"), confidence=kwargs.get("confidence", 1.0), metadata=metadata or {}, - first_seen=datetime.utcnow().isoformat(), - last_updated=datetime.utcnow().isoformat(), + first_seen=utc_now_iso(), + last_updated=utc_now_iso(), activity_started_at_time=activity_info["activity_started_at_time"], activity_ended_at_time=activity_info["activity_ended_at_time"], acted_on_behalf_of=kwargs.get("acted_on_behalf_of"), @@ -534,7 +539,7 @@ class ProvenanceManager: # split (issue #825, Part A item 4). derived_from_id=parent_chunk_id, metadata=metadata, - timestamp=datetime.utcnow().isoformat(), + timestamp=utc_now_iso(), activity_started_at_time=activity_info["activity_started_at_time"], activity_ended_at_time=activity_info["activity_ended_at_time"], ) @@ -604,7 +609,7 @@ class ProvenanceManager: **metadata, **source.metadata }, - timestamp=datetime.utcnow().isoformat(), + timestamp=utc_now_iso(), activity_started_at_time=activity_info["activity_started_at_time"], activity_ended_at_time=activity_info["activity_ended_at_time"], ) @@ -959,11 +964,27 @@ class ProvenanceManager: Returns: List of matching entries as dicts, sorted by timestamp ascending. """ - matches = [ - e for e in self.storage.retrieve_all() - if e.timestamp and start <= e.timestamp <= end - ] - matches.sort(key=lambda e: e.timestamp) + # Compare instants, not spellings. Since #1114 new entries carry a + # +00:00 offset while entries written earlier do not, and a raw string + # comparison orders those two by length: an inclusive naive bound equal + # to a stored offset-bearing timestamp would sort below it and drop the + # record. A bound in another offset was mis-ordered the same way. + start_at = to_utc_datetime(start) + end_at = to_utc_datetime(end) + entries = [e for e in self.storage.retrieve_all() if e.timestamp] + + if start_at is None or end_at is None: + # A bound this module cannot read as a timestamp keeps the historical + # string comparison rather than raising on a call that used to work. + matches = [e for e in entries if start <= e.timestamp <= end] + else: + matches = [ + e for e in entries + if (at := to_utc_datetime(e.timestamp)) is not None + and start_at <= at <= end_at + ] + + matches.sort(key=lambda e: (to_utc_datetime(e.timestamp) or _EPOCH, e.timestamp)) return [e.to_dict() for e in matches] def get_all_sources(self, entity_id: str) -> List[Dict[str, Any]]: @@ -1071,7 +1092,7 @@ class ProvenanceManager: entry = copy.deepcopy(existing) entry.invalidated = True - entry.invalidated_at_time = datetime.utcnow().isoformat() + entry.invalidated_at_time = utc_now_iso() entry.invalidated_by = agent_id entry.invalidation_reason = reason entry.previous_version_id = history_id @@ -1163,8 +1184,22 @@ class ProvenanceManager: """ entries = self.storage.retrieve_all() if since: - entries = [e for e in entries if getattr(e, "timestamp", "") >= since] - entries.sort(key=lambda e: getattr(e, "timestamp", "")) + since_at = to_utc_datetime(since) + if since_at is None: + entries = [e for e in entries + if getattr(e, "timestamp", "") >= since] + else: + entries = [ + e for e in entries + if (at := to_utc_datetime(getattr(e, "timestamp", None))) + is not None and at >= since_at + ] + entries.sort( + key=lambda e: ( + to_utc_datetime(getattr(e, "timestamp", None)) or _EPOCH, + getattr(e, "timestamp", ""), + ) + ) if format == "json": return [ diff --git a/semantica/provenance/schemas.py b/semantica/provenance/schemas.py index df888db4..1847b341 100644 --- a/semantica/provenance/schemas.py +++ b/semantica/provenance/schemas.py @@ -30,6 +30,8 @@ from dataclasses import dataclass, field from typing import Optional, List, Dict, Any from datetime import datetime +from ..utils.helpers import utc_now_iso + @dataclass class ProvenanceEntry: @@ -91,7 +93,7 @@ class ProvenanceEntry: source_quote: Optional[str] = None # Temporal tracking (from kg.ProvenanceTracker) - timestamp: str = field(default_factory=lambda: datetime.utcnow().isoformat()) + timestamp: str = field(default_factory=lambda: utc_now_iso()) first_seen: Optional[str] = None last_updated: Optional[str] = None diff --git a/semantica/utils/__init__.py b/semantica/utils/__init__.py index 002a139d..a6d7aa9e 100644 --- a/semantica/utils/__init__.py +++ b/semantica/utils/__init__.py @@ -82,6 +82,9 @@ from .helpers import ( normalize_entities, normalize_graph_payload, parse_timestamp, + to_utc_datetime, + utc_now, + utc_now_iso, read_json_file, retry_on_error, safe_filename, @@ -193,6 +196,9 @@ __all__ = [ "get_file_size", "format_timestamp", "parse_timestamp", + "to_utc_datetime", + "utc_now", + "utc_now_iso", "merge_dicts", "chunk_list", "flatten_dict", diff --git a/semantica/utils/helpers.py b/semantica/utils/helpers.py index 75031fe8..05ece01f 100644 --- a/semantica/utils/helpers.py +++ b/semantica/utils/helpers.py @@ -320,6 +320,65 @@ def format_timestamp( return dt.strftime(format_str) +def utc_now() -> datetime: + """ + Current instant as a timezone-aware UTC datetime. + + ``datetime.now()`` reads the local clock and ``datetime.utcnow()`` reads UTC, + but both return a naive datetime, and the two are indistinguishable once + serialized: a consumer cannot tell which zone the value belongs to, and an + RDF timestamp without an offset is not comparable against one that has an + offset (a SPARQL FILTER drops it rather than reporting an error). Use this + for any timestamp that leaves the process. + + Returns: + Current UTC time, timezone-aware + """ + return datetime.now(timezone.utc) + + +def utc_now_iso() -> str: + """ + Current instant as an ISO 8601 string carrying an explicit UTC offset. + + Returns: + Timestamp string such as ``2026-08-19T14:19:04.229937+00:00``, which is + a valid ``xsd:dateTimeStamp`` and orders correctly against timestamps + written in any other timezone + """ + return utc_now().isoformat() + + +def to_utc_datetime(value: Union[str, datetime, None]) -> Optional[datetime]: + """ + Read an ISO 8601 timestamp as a timezone-aware UTC instant. + + Timestamps written before #1114 carry no offset. They were produced by + ``datetime.utcnow()``, so a missing offset is read as UTC: that keeps a + stored naive value and the same instant written with an offset comparing + equal, instead of ordering by how the timestamp happens to be spelled. + + Args: + value: ISO 8601 string or datetime. ``Z`` is accepted as the offset. + + Returns: + Timezone-aware UTC datetime, or None if the value cannot be read as a + timestamp, so callers can fall back rather than raise on stored data + """ + if value is None: + return None + if isinstance(value, datetime): + parsed = value + else: + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + def parse_timestamp(timestamp_str: str, format_str: Optional[str] = None) -> datetime: """ Parse timestamp string to datetime. diff --git a/tests/export/test_timestamp_timezones.py b/tests/export/test_timestamp_timezones.py new file mode 100644 index 00000000..df34dd29 --- /dev/null +++ b/tests/export/test_timestamp_timezones.py @@ -0,0 +1,141 @@ +"""Timestamps that leave the process must carry a timezone (issue #1114). + +Every timestamp an exporter wrote was naive: ``datetime.now().isoformat()`` +reads the local clock, ``datetime.utcnow().isoformat()`` reads UTC, and the two +serialize identically, so nothing downstream can tell which zone a value belongs +to. In RDF the consequence is not a parse error but a silent one: under XSD 1.1 +a value with no timezone compared against one with a timezone is indeterminate +whenever they fall inside the +/-14 hour window, SPARQL turns that into an error, +and FILTER discards errors as non-matches. A timezone-qualified query therefore +returns an answer with every Semantica-written record quietly missing from it. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from semantica.export.json_exporter import JSONExporter +from semantica.export.report_generator import ReportGenerator +from semantica.export.yaml_exporter import SemanticNetworkYAMLExporter +from semantica.utils.helpers import utc_now, utc_now_iso + +KG = { + "entities": [{"id": "https://example.org/e1", "text": "Bob"}], + "relationships": [], +} + + +def assert_offset_aware(value): + """An ISO 8601 string is only an instant if it says which zone it is in.""" + assert isinstance(value, str), value + parsed = datetime.fromisoformat(value) + assert parsed.tzinfo is not None, f"timezone-naive timestamp: {value!r}" + assert parsed.utcoffset() is not None + + +def test_utc_now_iso_is_offset_aware(): + assert_offset_aware(utc_now_iso()) + assert utc_now().tzinfo is not None + + +def test_jsonld_export_timestamp_is_offset_aware(): + document = JSONExporter()._convert_kg_to_jsonld(KG) + assert_offset_aware(document["semantica:exportedAt"]) + + +def test_json_export_metadata_timestamp_is_offset_aware(tmp_path): + import json + + exporter = JSONExporter() + exporter.export_entities(KG["entities"], tmp_path / "entities.json") + exporter.export_relationships([], tmp_path / "relationships.json") + + for name in ("entities.json", "relationships.json"): + payload = json.loads((tmp_path / name).read_text()) + assert_offset_aware(payload["metadata"]["exported_at"]) + + +def test_yaml_export_timestamp_is_offset_aware(): + yaml = pytest.importorskip("yaml") + + document = SemanticNetworkYAMLExporter().export_entities(KG["entities"]) + payload = yaml.safe_load(document) + assert_offset_aware(payload["metadata"]["exported_at"]) + + +def test_report_timestamp_is_offset_aware(): + import json + + report = json.loads( + ReportGenerator().generate_quality_report({"score": 0.9}, format="json") + ) + assert_offset_aware(report["generated_at"]) + + +def test_exported_timestamp_compares_against_a_timezone_aware_instant(): + """The naive form raised TypeError here, or compared as if it were UTC.""" + exported = datetime.fromisoformat( + JSONExporter()._convert_kg_to_jsonld(KG)["semantica:exportedAt"] + ) + assert exported <= utc_now() + assert exported > datetime(2020, 1, 1, tzinfo=timezone.utc) + + +def test_exported_timestamp_survives_a_timezone_qualified_sparql_filter(): + """The regression in #1114: a strict engine dropped the naive value.""" + pyoxigraph = pytest.importorskip("pyoxigraph") + + exported = JSONExporter()._convert_kg_to_jsonld(KG)["semantica:exportedAt"] + store = pyoxigraph.Store() + store.load( + ( + ' ' + ' ' + f'"{exported}"^^ .' + ).encode(), + format=pyoxigraph.RdfFormat.N_TRIPLES, + ) + # The bound has to sit inside the +/-14 hour window that makes an + # untimezoned comparison indeterminate. A bound years away is determinate + # even for a naive value, and the test would pass without the fix. + bound = (utc_now() + timedelta(hours=1)).isoformat().replace("+00:00", "Z") + rows = list(store.query( + "PREFIX xsd: " + "SELECT ?e WHERE { ?e ?t . " + f'FILTER (?t < "{bound}"^^xsd:dateTime) }}' + )) + assert len(rows) == 1, "the export was dropped by a timezone-qualified filter" + + +def test_document_iri_carrying_an_offset_is_a_valid_iri(): + """The offset puts '+' and ':' in the @id; both are legal in a path.""" + rdflib = pytest.importorskip("rdflib") + + document_iri = JSONExporter()._convert_kg_to_jsonld(KG)["@id"] + assert "+00:00" in document_iri + assert rdflib.term._is_valid_uri(document_iri) + + graph = rdflib.Graph() + graph.add(( + rdflib.URIRef(document_iri), + rdflib.RDF.type, + rdflib.URIRef("https://semantica.dev/ns#KnowledgeGraph"), + )) + reparsed = rdflib.Graph().parse(data=graph.serialize(format="nt"), format="nt") + assert document_iri in {str(s) for s in reparsed.subjects()} + + +def test_vocabulary_range_matches_what_the_exporter_writes(): + """The declared range says the offset is required; the export must carry it.""" + rdflib = pytest.importorskip("rdflib") + from rdflib.namespace import RDFS, XSD + + from semantica.ontology.vocabulary import NAMESPACE, vocabulary_turtle + + graph = rdflib.Graph() + graph.parse(data=vocabulary_turtle(), format="turtle") + declared = graph.value(rdflib.URIRef(f"{NAMESPACE}exportedAt"), RDFS.range) + assert declared == XSD.dateTimeStamp + + exported = JSONExporter()._convert_kg_to_jsonld(KG)["semantica:exportedAt"] + assert datetime.fromisoformat(exported).utcoffset() is not None diff --git a/tests/provenance/test_timestamp_timezones.py b/tests/provenance/test_timestamp_timezones.py new file mode 100644 index 00000000..b2b91959 --- /dev/null +++ b/tests/provenance/test_timestamp_timezones.py @@ -0,0 +1,153 @@ +"""Provenance timestamps must carry a timezone (issue #1114). + +The provenance package stamped every record with ``datetime.utcnow()``, which +returns a naive datetime that happens to hold UTC. The exporters stamped theirs +with ``datetime.now()``, which returns a naive datetime holding local time. Both +serialize identically, so a graph mixing the two cannot be ordered, and the +values reach RDF as ``prov:generatedAtTime``/``startedAtTime``/``endedAtTime`` +typed ``xsd:dateTime``, where a timezone-qualified SPARQL comparison discards +them. ``datetime.utcnow()`` is also deprecated as of Python 3.12. +""" + +import warnings +from datetime import datetime + +import pytest + +from semantica.provenance.manager import ProvenanceManager +from semantica.provenance.schemas import ProvenanceEntry +from semantica.utils.helpers import utc_now + + +def assert_offset_aware(value): + parsed = datetime.fromisoformat(value) + assert parsed.tzinfo is not None, f"timezone-naive timestamp: {value!r}" + + +def test_provenance_entry_default_timestamp_is_offset_aware(): + entry = ProvenanceEntry(entity_id="e1", entity_type="Doc", activity_id="act1") + assert_offset_aware(entry.timestamp) + assert datetime.fromisoformat(entry.timestamp) <= utc_now() + + +def test_creating_an_entry_raises_no_deprecation_warning(): + """datetime.utcnow() is deprecated and scheduled for removal.""" + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + ProvenanceEntry(entity_id="e1", entity_type="Doc", activity_id="act1") + + +def test_tracked_entity_timestamps_are_offset_aware(): + manager = ProvenanceManager() + manager.track_entity("e1", source="doc.pdf") + + entry = manager.storage.retrieve_all()[0] + assert_offset_aware(entry.timestamp) + for field in ("first_seen", "last_updated"): + value = getattr(entry, field, None) + if value: + assert_offset_aware(value) + + +def test_prov_o_export_timestamps_are_offset_aware(): + """The values land in RDF typed xsd:dateTime, so the offset is the contract.""" + rdflib = pytest.importorskip("rdflib") + from rdflib.namespace import XSD + + manager = ProvenanceManager() + manager.track_entity("e_parent", source="doc.pdf") + manager.track_entity( + "e_child", source="doc.pdf", parent_entity_id="e_parent", + used_entities=["e_parent"], activity_id="act_transform", + ) + + graph = rdflib.Graph() + graph.parse(data=manager.export_prov(format="turtle"), format="turtle") + + stamps = [o for o in graph.objects() + if isinstance(o, rdflib.Literal) and o.datatype == XSD.dateTime] + assert stamps, "no xsd:dateTime literals in the PROV-O export" + for stamp in stamps: + assert_offset_aware(str(stamp)) + + +def test_prov_o_timestamps_are_valid_datetimestamp(): + """xsd:dateTimeStamp requires an explicit timezone; these now qualify.""" + rdflib = pytest.importorskip("rdflib") + from rdflib.namespace import XSD + + manager = ProvenanceManager() + manager.track_entity("e1", source="doc.pdf") + graph = rdflib.Graph() + graph.parse(data=manager.export_prov(format="turtle"), format="turtle") + + for stamp in [o for o in graph.objects() + if isinstance(o, rdflib.Literal) and o.datatype == XSD.dateTime]: + assert rdflib.Literal(str(stamp), datatype=XSD.dateTime).ill_typed is False + assert datetime.fromisoformat(str(stamp)).utcoffset() is not None + + +class TestRangeQueriesCompareInstants: + """Range APIs compared ISO strings, so they ordered by spelling (#1121 review). + + Once new entries carry ``+00:00`` and stored ones do not, a raw string + comparison puts an inclusive naive bound *below* the offset-bearing + timestamp it names, dropping the record, and a bound written in another + offset lands wherever its digits fall rather than at its instant. + """ + + @staticmethod + def _manager_with(timestamps): + manager = ProvenanceManager() + for index, stamp in enumerate(timestamps): + manager.storage.store(ProvenanceEntry( + entity_id=f"e{index}", entity_type="Doc", + activity_id="act", timestamp=stamp, + )) + return manager + + def test_inclusive_bound_written_without_an_offset_still_matches(self): + manager = self._manager_with(["2026-08-19T14:19:04.229937+00:00"]) + + found = manager.query_recorded_between( + "2026-08-19T00:00:00", "2026-08-19T14:19:04.229937" + ) + assert [e["entity_id"] for e in found] == ["e0"] + + def test_bound_in_another_offset_selects_by_instant(self): + """19:45+05:30 is 14:15Z: before the entry, though its digits are after.""" + manager = self._manager_with(["2026-08-19T14:19:04+00:00"]) + + assert manager.query_recorded_between( + "2026-08-19T00:00:00Z", "2026-08-19T19:45:00+05:30" + ) == [] + assert len(manager.query_recorded_between( + "2026-08-19T00:00:00Z", "2026-08-19T19:50:00+05:30" + )) == 1 + + def test_legacy_and_offset_bearing_entries_are_both_found_and_ordered(self): + manager = self._manager_with([ + "2026-08-19T14:19:05+00:00", # written after #1114 + "2026-08-19T14:19:04", # written before it, meaning UTC + ]) + + found = manager.query_recorded_between( + "2026-08-19T14:00:00Z", "2026-08-19T15:00:00Z" + ) + assert [e["entity_id"] for e in found] == ["e1", "e0"] + + def test_audit_log_since_reads_a_naive_bound_as_utc(self): + manager = self._manager_with([ + "2026-08-19T14:19:05+00:00", + "2026-08-19T09:00:00", + ]) + + recent = manager.audit_log(since="2026-08-19T14:19:05", format="json") + assert [e["entity_id"] for e in recent] == ["e0"] + + def test_an_unreadable_bound_falls_back_to_the_previous_behaviour(self): + """A call that used to work with a non-timestamp bound must not raise.""" + manager = self._manager_with(["2026-08-19T14:19:04+00:00"]) + + assert manager.query_recorded_between("not-a-date", "also-not") == [] + assert manager.audit_log(since="not-a-date", format="json") == []