From db4361ad46847654787b90df9c2465d6f705f2c3 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Mon, 3 Aug 2026 22:24:24 +0530 Subject: [PATCH 1/3] feat(provenance): close PROV-O compliance gaps and high-stakes trust blockers (closes #825) Part A - high-stakes trust blockers: - Invalidation tombstones via ProvenanceManager.invalidate() (archive-then-append, never mutates or deletes) instead of hard delete - Hash-chained integrity: sequence_id/previous_checksum chain every entry to its predecessor; new verify_chain() detects wholesale row deletion that a lone per-row checksum cannot - Typed Agent (AgentRecord: agent_type/is_automated) and Activity (ActivityRecord: start/end timing), wired through all 18 *_provenance.py wrappers - Split parent_entity_id into previous_version_id (correction) vs derived_from_id (cross-source derivation), additive alongside the legacy combined field - Downstream/descendant lineage traversal (get_descendants/trace_descendants, reverse BFS) closing the dead direction="downstream" code path in the Explorer's provenance route - Qualified Association+hadRole and Invalidation in export_prov() - New CLI: provenance invalidate|verify-chain|descendants Part B - general PROV-O spec completeness: - Qualified Generation/Usage/Derivation in export_prov() - wasAssociatedWith, actedOnBehalfOf, wasInformedBy relations - Bitemporal fields (valid_from/valid_until/revision_type/supersedes) plus revision_history()/query_recorded_between(), closing the deprecated kg.ProvenanceTracker's "no direct equivalent yet" migration gaps - prov:Bundle/hadMember membership via bundle_id - Configurable base_uri (--base-uri CLI flag), shared by RDFExporter's NamespaceManager and OWLExporter's default ontology_uri so KG/OWL/PROV exports co-resolve under one namespace instead of three hardcoded ones Bugs fixed along the way: - agent_id was a dead field: no track_* method read it from kwargs - track_entities_batch silently absorbed typed kwargs into the metadata blob - compute_checksum() had to exclude entity_id itself: hashing it made track_entity's versioning-archive relabel permanently orphan any entry already chained from the pre-relabel checksum, a false-positive "broken chain" for a legitimate rename - InMemoryStorage.get_chain_head() ignored the committed head whenever the current transaction had staged entries, corrupting the next chain link - several new ProvenanceEntry fields were wired into the dataclass and export_prov() but not into SQLiteStorage's DDL/INSERT/row-mapping; InMemoryStorage masked the gap. Added a permanent round-trip regression test to catch this class of bug for future field additions Flagged, not fixed (separate pre-existing issues, out of scope for #825): - pipeline/pipeline_provenance.py imports a nonexistent module and wraps a Pipeline dataclass with no run() method - most *_provenance.py wrappers' backing classes are themselves missing or incomplete (context_manager, deduplicator, normalizer, etc.) - kg_provenance.py passes entity_type inside metadata={} instead of as a top-level track_entity() kwarg across most of its call sites --- CHANGELOG.md | 17 + docs/guides/provenance.md | 23 +- docs/migration/kg-provenance-tracker.md | 4 +- semantica/cli.py | 121 +++- semantica/conflicts/conflicts_provenance.py | 28 +- semantica/context/context_provenance.py | 26 +- .../deduplication/deduplication_provenance.py | 30 +- semantica/embeddings/embeddings_provenance.py | 30 +- semantica/explorer/routes/provenance.py | 85 ++- semantica/export/export_provenance.py | 30 +- semantica/export/owl_exporter.py | 11 +- semantica/export/rdf_exporter.py | 10 +- .../graph_store/graph_store_provenance.py | 30 +- semantica/ingest/ingest_provenance.py | 42 +- semantica/kg/kg_provenance.py | 208 +++++- semantica/llms/llms_provenance.py | 141 +++- semantica/normalize/normalize_provenance.py | 30 +- semantica/ontology/ontology_provenance.py | 30 +- semantica/parse/parse_provenance.py | 30 +- semantica/pipeline/pipeline_provenance.py | 29 +- semantica/provenance/__init__.py | 13 +- semantica/provenance/integrity.py | 59 +- semantica/provenance/manager.py | 634 +++++++++++++++++- semantica/provenance/schemas.py | 240 ++++++- semantica/provenance/storage.py | 329 ++++++++- semantica/reasoning/reasoning_provenance.py | 30 +- .../semantic_extract_provenance.py | 159 +++-- .../triplet_store/triplet_store_provenance.py | 30 +- .../vector_store/vector_store_provenance.py | 30 +- .../visualization/visualization_provenance.py | 30 +- tests/provenance/test_manager.py | 609 +++++++++++++++++ tests/provenance/test_schemas.py | 124 +++- tests/provenance/test_storage.py | 175 ++++- 33 files changed, 3104 insertions(+), 313 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 389779fc..03f31124 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **PROV-O trust blockers and general spec completeness for `ProvenanceManager`** (#825) by @KaifAhmad1 + - **Invalidation instead of hard delete**: new `ProvenanceManager.invalidate(entity_id, agent_id, reason=None)` tombstones an entry — archives its pre-invalidation state under a stable versioned key, then appends the invalidated entry (`invalidated`, `invalidated_at_time`, `invalidated_by`, `invalidation_reason`) — instead of mutating or deleting it, so an audit can prove a fact existed, was reviewed, and was retracted. `ProvenanceManager.clear()` remains the bulk dev/test store-reset utility it always was; it was not repurposed + - **Hash-chained integrity**: every entry now carries `sequence_id`/`previous_checksum`, chaining it to the entry immediately before it in insertion order. New `ProvenanceManager.verify_chain()` walks the chain and reports any break, including a row hard-deleted directly from the underlying table — something a lone per-row SHA-256 checksum can never detect on its own. `compute_checksum()` now also covers `agent_id`/`agent_type`, the lineage-link fields, and the invalidation fields, closing several fields that previously weren't tamper-evident + - **Typed Agent/Activity**: `agent_id` was a dead field — no `track_*` method read it from kwargs, so it was always the `"semantica"` default regardless of what callers passed; fixed, and paired with new `AgentRecord(id, agent_type, is_automated)` / `ActivityRecord(id, activity_type, started_at_time, ended_at_time)` dataclasses (pass via `agent=`/`activity=` kwargs) so a human reviewer, an LLM call, and an automated pipeline stage are now distinguishable, and activities carry real start/end timing. Wired through all 18 `*_provenance.py` wrapper modules and `track_entity`/`track_relationship`/`track_chunk`/`track_property_source` + - **Versioning vs. derivation split**: new `previous_version_id` ("this corrects a prior version of the same fact") and `derived_from_id` ("this was derived from a different source entity") fields, additive alongside the legacy combined `parent_entity_id` so existing readers are unaffected + - **Downstream lineage traversal**: new `get_descendants()`/`trace_descendants()` (reverse BFS in both `InMemoryStorage` and `SQLiteStorage`), closing the gap flagged in `semantica/explorer/routes/provenance.py` where `direction="downstream"` was dead code with no reverse lookup to feed it; the Explorer's `/api/provenance` lineage response now merges both directions + - **W3C PROV-O qualified relations**: `export_prov()` now emits `prov:qualifiedAssociation`/`hadRole` (distinguishing "approved by" from "generated by" for sign-off workflows), `qualifiedGeneration`/`Generation`, `qualifiedUsage`/`Usage`, `qualifiedDerivation`/`Derivation`, `qualifiedInvalidation`/`Invalidation`, `wasAssociatedWith` (Activity→Agent), `actedOnBehalfOf` (Agent→Agent delegation), and `wasInformedBy` (Activity→Activity, via a new `informed_by=[...]` kwarg), alongside the existing plain triples + - **Bitemporal + Bundle support**: `revision_type`/`supersedes`/`valid_from`/`valid_until` fields (plain caller-supplied passthrough, matching the deprecated `kg.ProvenanceTracker`'s actual contract) plus new `revision_history()` and `query_recorded_between()` methods, closing the two "no direct equivalent yet" rows in `docs/migration/kg-provenance-tracker.md`; `bundle_id` emits `prov:Bundle`/`hadMember` membership triples to partition provenance by source/dataset/ingestion-run + - **Configurable, interlinked namespace**: `export_prov(base_uri=...)` / `--base-uri` CLI flag, defaulting to a new `ProvenanceManager.DEFAULT_BASE_URI` (`https://semantica.dev/ns#`) that `RDFExporter`'s `NamespaceManager` and `OWLExporter`'s default `ontology_uri` now both reuse, so KG-exported, OWL-exported, and PROV-exported URIs for the same `entity_id` co-resolve instead of three independently-hardcoded placeholder domains + - New CLI commands: `semantica provenance invalidate|verify-chain|descendants` + - **Fixed along the way**: `track_entities_batch()` silently absorbed batch-level typed kwargs (`agent_id`, `entity_type`, `activity_id`) into the opaque `metadata` JSON blob instead of forwarding them, so the documented banking example in `docs/guides/provenance.md` never actually worked as written + - **Fixed along the way**: `compute_checksum()` had to exclude `entity_id` itself from the hash — `track_entity()`'s versioning archives a prior value by copying it to a new key (`"X"` → `"X:v:"`), and hashing `entity_id` meant that legitimate relabel permanently orphaned any other entry that had already chained its `previous_checksum` from the pre-relabel value, surfacing as a false-positive "broken chain." Archival and invalidation are now always a pure relabel (unchanged checksum/sequence position) followed by a fresh chained append, never an in-place mutation of an already-chained entry + - **Fixed along the way**: `InMemoryStorage.get_chain_head()` ignored the already-committed chain head whenever the current transaction had staged any entries, understating the head and corrupting the next append's chain link + - **Fixed along the way**: several new `ProvenanceEntry` fields were initially wired into the dataclass and `export_prov()` but not into `SQLiteStorage`'s DDL/INSERT/row-mapping — `InMemoryStorage` stores the dataclass directly so it masked the gap. Added a permanent regression test (`test_all_fields_round_trip_through_sqlite`) asserting every field survives a SQLite round trip, to catch this class of bug for any future field additions + - Flagged, not fixed (separate, pre-existing issues independent of #825): `semantica/pipeline/pipeline_provenance.py` imports a nonexistent module and wraps a `Pipeline` dataclass with no `run()` method, so `PipelineWithProvenance` has never worked; most of the 18 wrapper modules' backing classes are themselves missing or incomplete (e.g. `context.context_manager`, `deduplication.deduplicator`, `normalize.normalizer` don't exist; `EmbeddingGenerator` exists but has no `.embed()`); `kg_provenance.py` passes `entity_type` inside its `metadata={}` dict instead of as a top-level `track_entity()` kwarg across most of its ~30 call sites, so it never actually populates the real field + - Extensive new test coverage across `tests/provenance/test_manager.py`, `test_schemas.py`, and `test_storage.py` (invalidation, hash-chain verification including a simulated hard-delete-detection case and an interleaved-chaining stress test, agent/activity typing, versioning/derivation split, downstream lineage, qualified export triples, bitemporal methods, Bundle export, and namespace interlinking) + - **Altair Anzo triplet store backend** (#813) by @KaifAhmad1 - Added `AnzoStore` (`semantica/triplet_store/anzo_store.py`), a fourth peer to `BlazegraphStore`/`RDF4JStore`/`JenaStore` speaking plain SPARQL 1.1 over HTTP — no new dependency, since Anzo has no official Python SDK but needs none - The one structural difference from the existing backends: Anzo addresses data by a dataset/graphmart **URI** (`dataset_uri`, required) rather than a short namespace/repository name, so the endpoint path (`/sparql//`) percent-encodes it; `store_type` defaults to `"graphmart"` and can be set to `"dataset"` diff --git a/docs/guides/provenance.md b/docs/guides/provenance.md index e05fa76b..a1b4d8f1 100644 --- a/docs/guides/provenance.md +++ b/docs/guides/provenance.md @@ -633,12 +633,29 @@ Every `ProvenanceEntry` maps directly to W3C PROV-O terms. If your compliance te | :--- | :--- | :--- | | `prov:Entity` | `entity_id` | The tracked object — entity, chunk, relationship, or property | | `prov:Activity` | `activity_id` | The process that produced it — `"ner_extraction"`, `"bureau_parsing"` | -| `prov:Agent` | `agent_id` | Who ran the activity — pipeline name, analyst ID | -| `prov:wasDerivedFrom` | `parent_entity_id` | The previous version of this entity — enables version chaining | +| `prov:Agent` / `prov:Person` / `prov:SoftwareAgent` / `prov:Organization` | `agent_id`, `agent_type`, `is_automated` | Who — or what — ran the activity, and whether a human was directly accountable | +| `prov:qualifiedAssociation` + `prov:hadRole` | `role` | The agent's role for this specific entity — `"generator"` (default), `"approver"`, `"reviewer"` — for sign-off/four-eyes workflows | +| `prov:wasDerivedFrom` | `parent_entity_id` (legacy combined field) | The previous version or source of this entity | +| — | `previous_version_id` | This entry corrects/replaces a prior version of the *same* fact | +| `prov:wasDerivedFrom` | `derived_from_id` | This entry was derived from a *different* source entity | | `prov:used` | `used_entities` | Entity IDs consumed to produce this one | | `prov:generatedAtTime` | `timestamp` | ISO datetime, auto-set to `datetime.utcnow()` at write time | +| `prov:qualifiedInvalidation` | `invalidated`, `invalidated_at_time`, `invalidated_by`, `invalidation_reason` | A retraction/correction recorded as a tombstone via `ProvenanceManager.invalidate()`, never a hard delete | +| `prov:startedAtTime` / `prov:endedAtTime` | `activity_started_at_time`, `activity_ended_at_time` | Typed Activity timing — pass an `ActivityRecord` via the `activity=` kwarg to set these together with `activity_id` | +| `prov:qualifiedGeneration`/`Generation`, `qualifiedUsage`/`Usage`, `qualifiedDerivation`/`Derivation` | (derived from the fields above) | Additive qualified forms of `wasGeneratedBy`/`used`/`wasDerivedFrom`, emitted automatically alongside the plain triples | +| `prov:wasAssociatedWith` | (derived from `agent_id`) | Direct Activity→Agent link, distinct from the Entity→Agent `wasAttributedTo` | +| `prov:actedOnBehalfOf` | `acted_on_behalf_of` | Agent→Agent delegation — e.g. an automated agent acting on behalf of the human/organization that authorized it | +| `prov:wasInformedBy` | `informed_by_activities` (pass as `informed_by=[...]`) | Chains this entry's activity to prior activities it was informed by (e.g. a pipeline stage informed by the stage before it) | +| `prov:Bundle` + `prov:hadMember` | `bundle_id` | Groups entries by source/dataset/ingestion-run (membership triples, not true RDF named-graph partitioning) | +| — | `valid_from`, `valid_until`, `revision_type`, `supersedes` | Bitemporal fields merged from the deprecated `kg.ProvenanceTracker` — always caller-supplied (never auto-computed), surfaced via `ProvenanceManager.revision_history()`, which falls back to timestamp-based derivation for entries that don't set them explicitly | -The `checksum` field is not part of the PROV-O standard — it is Semantica's tamper-detection extension. Every entry's SHA-256 is computed from its content fields at write time and can be recomputed at any time to verify the record has not been modified. +`previous_version_id` and `derived_from_id` are additive alongside `parent_entity_id` — existing code reading `parent_entity_id` keeps working unchanged, while new code gets the two relations disambiguated. + +The `checksum` field is not part of the PROV-O standard — it is Semantica's tamper-detection extension. Every entry's SHA-256 now also incorporates `previous_checksum` (the prior entry's checksum, by insertion order via `sequence_id`), chaining every entry to the one before it. `ProvenanceManager.verify_chain()` walks the full chain and reports any break — including a row that was hard-deleted from the underlying table, which a lone per-row checksum can't detect on its own. + +Note: the banking example above passes `agent_id="credit_data_service_v2"` to `track_entities_batch()` — this now actually populates the entry's `agent_id` field (previously a bug caused batch-level typed kwargs like `agent_id`/`entity_type`/`activity_id` to be silently absorbed into the opaque `metadata` blob instead). + +`export_prov()` mints entity/agent/activity URIs under `ProvenanceManager.DEFAULT_BASE_URI` (`https://semantica.dev/ns#` by default — the same namespace `RDFExporter`'s `NamespaceManager` uses for its `"semantica"` prefix, so KG-exported and PROV-exported URIs for the same `entity_id` co-resolve) unless overridden via `export_prov(base_uri=...)` or the CLI's `--base-uri` option. ## Related Guides diff --git a/docs/migration/kg-provenance-tracker.md b/docs/migration/kg-provenance-tracker.md index a058a359..496f5b13 100644 --- a/docs/migration/kg-provenance-tracker.md +++ b/docs/migration/kg-provenance-tracker.md @@ -17,8 +17,8 @@ Every method on `kg.ProvenanceTracker` now emits a `DeprecationWarning` on use, | `track_entity(entity_id, source, metadata)` | `track_entity(entity_id, source, metadata)` | Same call shape. `ProvenanceManager` additionally auto-links each update to its prior version via `parent_entity_id`. | | `get_all_sources(entity_id)` | `get_all_sources(entity_id)` | Field name differs: the `kg` tracker returns each record's time under `"recorded_at"`; `ProvenanceManager` returns `"timestamp"`. | | `clear(entity_id=None)` | `clear()` | `ProvenanceManager.clear()` clears all provenance data; there is no per-entity clear yet. | -| `query_recorded_between(start, end)` | *No direct equivalent yet* | Filter the entries returned by `get_lineage()` / `trace_lineage()` client-side in the meantime. | -| `revision_history(fact_id)` | *No direct equivalent yet* | `get_lineage(fact_id)["lineage_chain"]` returns the full chain of `ProvenanceEntry` records but not in the same versioned shape. | +| `query_recorded_between(start, end)` | `query_recorded_between(start, end)` | Same call shape; filters by `timestamp` (ISO 8601 string comparison) across all tracked entries, not just one entity. | +| `revision_history(fact_id)` | `revision_history(fact_id)` | Same call shape and return shape (`version`, `valid_from`, `valid_until`, `recorded_at`, `author`, optional `revision_type`/`supersedes`) — walks the entity's `previous_version_id` chain rather than a flat per-entity dict. | | `export_audit_log(fact_ids, format)` | *No direct equivalent yet* | Build the export from `get_lineage()` output, or serialize `get_statistics()` for a summary view. | Methods with no direct equivalent are not planned to be reimplemented on `kg.ProvenanceTracker` — they will need a small adapter in caller code, or a feature request against `ProvenanceManager` if you rely on them heavily. diff --git a/semantica/cli.py b/semantica/cli.py index 56d8e9a0..5b179c13 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -2559,27 +2559,31 @@ def provenance_audit(cli_ctx: CLIContext, since: Optional[str], fmt: str, @provenance.command("export") @click.option("--format", "fmt", type=click.Choice(["turtle", "ntriples", "jsonld"]), default="turtle", show_default=True) +@click.option("--base-uri", "base_uri", default=None, + help="Namespace URI entities/agents/activities are minted under " + "(default: ProvenanceManager.DEFAULT_BASE_URI).") @click.option("--output", default=None, type=click.Path()) @click.option("--dry-run", "local_dry", is_flag=True, default=False) @click.pass_obj -def provenance_export(cli_ctx: CLIContext, fmt: str, output: Optional[str], - local_dry: bool) -> None: +def provenance_export(cli_ctx: CLIContext, fmt: str, base_uri: Optional[str], + output: Optional[str], local_dry: bool) -> None: """Export provenance as W3C PROV-O RDF. \b Example: semantica provenance export --format turtle --output prov.ttl + semantica provenance export --base-uri https://example.org/kg# --output prov.ttl """ cli_ctx = _require_ctx(cli_ctx) def _action() -> None: if _is_dry(cli_ctx, local_dry): - _dry(cli_ctx, "export provenance", format=fmt, output=output) + _dry(cli_ctx, "export provenance", format=fmt, base_uri=base_uri, output=output) return try: from .provenance import ProvenanceManager pm = ProvenanceManager(config=cli_ctx.config.to_dict()) - data = pm.export_prov(format=fmt) + data = pm.export_prov(format=fmt, base_uri=base_uri) except ImportError as exc: raise click.ClickException(f"Provenance module not available: {exc}") from exc if output: @@ -2621,6 +2625,115 @@ def provenance_check(cli_ctx: CLIContext, strict: bool, local_json: bool) -> Non _run_with_error_handling(_action) +@provenance.command("invalidate") +@click.argument("entity_id") +@click.option("--by", "agent_id", required=True, help="Agent responsible for the invalidation.") +@click.option("--reason", default=None, help="Human-readable reason for the invalidation.") +@click.option("--json", "local_json", is_flag=True, default=False) +@click.option("--dry-run", "local_dry", is_flag=True, default=False) +@click.pass_obj +def provenance_invalidate(cli_ctx: CLIContext, entity_id: str, agent_id: str, + reason: Optional[str], local_json: bool, + local_dry: bool) -> None: + """Mark a tracked entity as invalidated (tombstone, not a hard delete). + + \b + Example: + semantica provenance invalidate entity_alice --by reviewer_jane --reason "Source retracted" + """ + cli_ctx = _require_ctx(cli_ctx) + + def _action() -> None: + if _is_dry(cli_ctx, local_dry): + _dry(cli_ctx, "invalidate provenance entity", entity_id=entity_id, by=agent_id, reason=reason) + return + try: + from .provenance import ProvenanceManager + pm = ProvenanceManager(config=cli_ctx.config.to_dict()) + result = pm.invalidate(entity_id, agent_id=agent_id, reason=reason) + except ImportError as exc: + raise click.ClickException(f"Provenance module not available: {exc}") from exc + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + if _is_json(cli_ctx, local_json): + _jecho(result.to_dict()) + else: + _ok(cli_ctx, f"Invalidated {entity_id} (by {agent_id})") + + _run_with_error_handling(_action) + + +@provenance.command("verify-chain") +@click.option("--json", "local_json", is_flag=True, default=False) +@click.pass_obj +def provenance_verify_chain(cli_ctx: CLIContext, local_json: bool) -> None: + """Verify the hash chain across all provenance entries. + + Detects wholesale row deletion: per-row checksums alone only prove a + surviving row wasn't edited in place, not that no row is missing. + + \b + Example: + semantica provenance verify-chain + """ + cli_ctx = _require_ctx(cli_ctx) + + def _action() -> None: + try: + from .provenance import ProvenanceManager + pm = ProvenanceManager(config=cli_ctx.config.to_dict()) + result = pm.verify_chain() + except ImportError as exc: + raise click.ClickException(f"Provenance module not available: {exc}") from exc + if _is_json(cli_ctx, local_json): + _jecho(result) + elif result.get("valid"): + _ok(cli_ctx, f"Chain verified: {result.get('total_entries')} entries, no breaks") + else: + _warn(cli_ctx, f"Chain verification failed: {len(result.get('broken_links', []))} broken link(s)") + for link in result.get("broken_links", []): + click.echo(f" {link}") + + _run_with_error_handling(_action) + + +@provenance.command("descendants") +@click.argument("entity_id") +@click.option("--depth", default=None, type=int, show_default=True) +@click.option("--json", "local_json", is_flag=True, default=False) +@click.pass_obj +def provenance_descendants(cli_ctx: CLIContext, entity_id: str, depth: Optional[int], + local_json: bool) -> None: + """Show downstream descendants (reverse lineage) for an entity. + + The counterpart to `provenance lineage`, which only traces upstream + ancestors. Answers "entity X was wrong — what downstream facts used it?" + + \b + Example: + semantica provenance descendants entity_alice --depth 3 + """ + cli_ctx = _require_ctx(cli_ctx) + + def _action() -> None: + try: + from .provenance import ProvenanceManager + pm = ProvenanceManager(config=cli_ctx.config.to_dict()) + if depth is not None: + entries = [e.to_dict() for e in pm.trace_descendants(entity_id, max_depth=depth)] + result = {"entity_id": entity_id, "depth": depth, "entries": entries} + else: + result = pm.get_descendants(entity_id) or {"entity_id": entity_id, "entries": []} + except ImportError as exc: + raise click.ClickException(f"Provenance module not available: {exc}") from exc + if _is_json(cli_ctx, local_json): + _jecho(result) + else: + _pprint(cli_ctx, result) + + _run_with_error_handling(_action) + + @main.group(invoke_without_command=True) @click.pass_context def validate(ctx: click.Context) -> None: diff --git a/semantica/conflicts/conflicts_provenance.py b/semantica/conflicts/conflicts_provenance.py index b37ab418..915acf4d 100644 --- a/semantica/conflicts/conflicts_provenance.py +++ b/semantica/conflicts/conflicts_provenance.py @@ -15,36 +15,52 @@ License: MIT """ from typing import Optional, Dict, Any, List +from datetime import datetime class SourceTrackerWithUnifiedBackend: """SourceTracker using unified provenance backend.""" - def __init__(self, **config): + def __init__( + self, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """Initialize with unified backend or fallback to legacy.""" from .source_tracker import SourceTracker - + + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + try: from semantica.provenance import ProvenanceManager self._unified_manager = ProvenanceManager() self._use_unified = True except ImportError: self._use_unified = False - + self._original_tracker = SourceTracker(**config) - + def track_property_source(self, entity_id: str, property_name: str, value: Any, source: Any, **metadata): """Track property source with unified backend.""" + activity_started_at_time = datetime.utcnow().isoformat() if self._use_unified: from semantica.provenance import SourceReference - + source_ref = SourceReference( document=source.document if hasattr(source, 'document') else str(source), page=getattr(source, 'page', None), section=getattr(source, 'section', None), confidence=getattr(source, 'confidence', 1.0) ) - + + metadata.setdefault("agent_id", self._agent_id) + metadata.setdefault("agent_type", "software_agent") + metadata.setdefault("is_automated", self._is_automated) + metadata.setdefault("activity_started_at_time", activity_started_at_time) + metadata.setdefault("activity_ended_at_time", datetime.utcnow().isoformat()) + self._unified_manager.track_property_source( entity_id=entity_id, property_name=property_name, diff --git a/semantica/context/context_provenance.py b/semantica/context/context_provenance.py index 4871d525..a0492496 100644 --- a/semantica/context/context_provenance.py +++ b/semantica/context/context_provenance.py @@ -12,36 +12,52 @@ License: MIT """ from typing import Optional, Any +from datetime import datetime import uuid class ContextManagerWithProvenance: """Context manager with provenance tracking.""" - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """Initialize context manager with optional provenance.""" from .context_manager import ContextManager - + self.provenance = provenance self._context_manager = ContextManager(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def add_context(self, context: Any, source: Optional[str] = None, **kwargs): """Add context with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._context_manager.add_context(context, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"context_{uuid.uuid4().hex[:8]}", source=source or "context_manager", entity_type="context", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={"context_preview": str(context)[:100]} ) diff --git a/semantica/deduplication/deduplication_provenance.py b/semantica/deduplication/deduplication_provenance.py index 611f680d..689329aa 100644 --- a/semantica/deduplication/deduplication_provenance.py +++ b/semantica/deduplication/deduplication_provenance.py @@ -13,37 +13,53 @@ Author: Semantica Contributors License: MIT """ -from typing import List, Any +from typing import List, Any, Optional +from datetime import datetime import uuid class DeduplicatorWithProvenance: """Deduplicator with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .deduplicator import Deduplicator - + self.provenance = provenance self._deduplicator = Deduplicator(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def deduplicate(self, items: List[Any], source: str = None, **kwargs): """Deduplicate items with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() unique_items = self._deduplicator.deduplicate(items, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: duplicates_found = len(items) - len(unique_items) self._prov_manager.track_entity( entity_id=f"dedup_{uuid.uuid4().hex[:8]}", source=source or "deduplication", entity_type="deduplication_operation", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "input_count": len(items), "output_count": len(unique_items), diff --git a/semantica/embeddings/embeddings_provenance.py b/semantica/embeddings/embeddings_provenance.py index 378a7017..03d7adbd 100644 --- a/semantica/embeddings/embeddings_provenance.py +++ b/semantica/embeddings/embeddings_provenance.py @@ -13,36 +13,52 @@ Author: Semantica Contributors License: MIT """ -from typing import List +from typing import List, Optional +from datetime import datetime import uuid class EmbeddingGeneratorWithProvenance: """Embedding generator with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .embedding_generator import EmbeddingGenerator - + self.provenance = provenance self._generator = EmbeddingGenerator(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def embed(self, texts: List[str], source: str = None, **kwargs): """Generate embeddings with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() embeddings = self._generator.embed(texts, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"embed_{uuid.uuid4().hex[:8]}", source=source or "embedding_generation", entity_type="embeddings", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "model": getattr(self._generator, 'model', 'unknown'), "dimensions": len(embeddings[0]) if embeddings else 0, diff --git a/semantica/explorer/routes/provenance.py b/semantica/explorer/routes/provenance.py index 4a6ac5da..dd981862 100644 --- a/semantica/explorer/routes/provenance.py +++ b/semantica/explorer/routes/provenance.py @@ -32,21 +32,7 @@ def _classify_prov(node_type: str) -> tuple[str, str]: return "Entity", "group_entity" -def _transform_audit_lineage(lineage: Dict[str, Any], node_id: str) -> Dict[str, Any]: - # Mapping decision (W3C PROV-O to frontend swim-lanes): - # Every ProvenanceEntry maps to a node classified by _classify_prov(entry["entity_type"]), - # placing documents/chunks/entities in 'group_entity' (prov_type='Entity'), persons/systems - # in 'group_agent', and actions/processes in 'group_activity'. - # For derivation relationships (parent_entity_id and used_entities), we connect - # parent -> child directly with edge label set to activity_id (or 'wasDerivedFrom'), - # keeping the lineage graph scannable without cluttering it with intermediate activity - # nodes when activity_id is an operational label. - nodes: List[Dict[str, Any]] = [] - edges: List[Dict[str, Any]] = [] - seen_nodes = set() - seen_edges = set() - - chain = lineage.get("lineage_chain") or lineage.get("entries") or [] +def _add_chain_nodes(chain: List[Any], nodes: List[Dict[str, Any]], seen_nodes: set) -> None: for entry in chain: if not isinstance(entry, dict): continue @@ -69,6 +55,10 @@ def _transform_audit_lineage(lineage: Dict[str, Any], node_id: str) -> Dict[str, "checksum": entry.get("checksum") or None, }) + +def _add_chain_edges( + chain: List[Any], edges: List[Dict[str, Any]], seen_edges: set, direction: str +) -> None: for entry in chain: if not isinstance(entry, dict): continue @@ -86,21 +76,11 @@ def _transform_audit_lineage(lineage: Dict[str, Any], node_id: str) -> Dict[str, activity = str(entry.get("activity_id") or "wasDerivedFrom") for src in parents: - edge_key = (src, eid) + edge_key = (src, eid, direction) if edge_key in seen_edges: continue seen_edges.add(edge_key) - # NOTE: ProvenanceManager.get_lineage() currently only traces upstream - # ancestor chains via parent_entity_id and used_entities. It does not perform - # reverse lookups for downstream descendants. Consequently, 'direction = "downstream"' - # is unreachable in practice for this audit path until reverse lookup is supported - # by ProvenanceManager. All ancestor derivation edges are upstream lineage. - if src == node_id: - direction = "downstream" - else: - direction = "upstream" - edges.append({ "id": f"{src}-{eid}", "source": src, @@ -109,6 +89,44 @@ def _transform_audit_lineage(lineage: Dict[str, Any], node_id: str) -> Dict[str, "direction": direction, }) + +def _transform_audit_lineage( + lineage: Dict[str, Any], + node_id: str, + descendants: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + # Mapping decision (W3C PROV-O to frontend swim-lanes): + # Every ProvenanceEntry maps to a node classified by _classify_prov(entry["entity_type"]), + # placing documents/chunks/entities in 'group_entity' (prov_type='Entity'), persons/systems + # in 'group_agent', and actions/processes in 'group_activity'. + # For derivation relationships (parent_entity_id and used_entities), we connect + # parent -> child directly with edge label set to activity_id (or 'wasDerivedFrom'), + # keeping the lineage graph scannable without cluttering it with intermediate activity + # nodes when activity_id is an operational label. + # + # Upstream edges come from lineage's ancestor chain (parent_entity_id/ + # used_entities, via ProvenanceManager.get_lineage()); downstream edges + # come from ProvenanceManager.get_descendants()'s descendant chain (issue + # #825, Part A item 5). Previously 'direction="downstream"' was dead code + # here since no reverse lookup existed. + nodes: List[Dict[str, Any]] = [] + edges: List[Dict[str, Any]] = [] + seen_nodes: set = set() + seen_edges: set = set() + + ancestor_chain = lineage.get("lineage_chain") or lineage.get("entries") or [] + descendant_chain = ( + (descendants or {}).get("descendant_chain") + or (descendants or {}).get("entries") + or [] + ) + + _add_chain_nodes(ancestor_chain, nodes, seen_nodes) + _add_chain_nodes(descendant_chain, nodes, seen_nodes) + + _add_chain_edges(ancestor_chain, edges, seen_edges, "upstream") + _add_chain_edges(descendant_chain, edges, seen_edges, "downstream") + for edge in edges: for endpoint in (edge["source"], edge["target"]): if endpoint not in seen_nodes: @@ -132,9 +150,9 @@ def _transform_audit_lineage(lineage: Dict[str, Any], node_id: str) -> Dict[str, def _build_provenance(session: GraphSession, node_id: Optional[str] = None) -> dict: """Build provenance lineage for a node, attempting the audit-grade store first. - NOTE: The audit path (source='audit') shows verified upstream lineage only. - For descendant/downstream relationships, the naive graph-traversal fallback - remains the only source until ProvenanceManager gains a reverse lookup. + Combines ProvenanceManager.get_lineage() (upstream ancestors) with + get_descendants() (downstream — issue #825, Part A item 5) so both + directions are populated from the audit-grade store, not just upstream. """ if not node_id: return {"nodes": [], "edges": [], "source": "graph_traversal"} @@ -149,7 +167,14 @@ def _build_provenance(session: GraphSession, node_id: Optional[str] = None) -> d entries = lineage.get("lineage_chain") or lineage.get("entries") or [] integrity_ok = all(verify_checksum(entry) for entry in entries) if integrity_ok: - return _transform_audit_lineage(lineage, node_id) + descendants = {} + try: + descendants = manager.get_descendants(node_id) or {} + except Exception as exc: + logger.warning( + f"ProvenanceManager get_descendants failed for {node_id}: {exc}" + ) + return _transform_audit_lineage(lineage, node_id, descendants) logger.warning( f"Provenance integrity verification failed for {node_id}, falling back to graph traversal" ) diff --git a/semantica/export/export_provenance.py b/semantica/export/export_provenance.py index 79a635a4..551b0f2c 100644 --- a/semantica/export/export_provenance.py +++ b/semantica/export/export_provenance.py @@ -13,36 +13,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class ExporterWithProvenance: """Base exporter with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .exporter import Exporter - + self.provenance = provenance self._exporter = Exporter(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def export(self, data: Any, destination: str, **kwargs): """Export data with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._exporter.export(data, destination, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"export_{uuid.uuid4().hex[:8]}", source="export_operation", entity_type="export", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "destination": destination, "format": kwargs.get('format', 'unknown') diff --git a/semantica/export/owl_exporter.py b/semantica/export/owl_exporter.py index 5f123f0c..3c604d59 100644 --- a/semantica/export/owl_exporter.py +++ b/semantica/export/owl_exporter.py @@ -31,6 +31,12 @@ from ..utils.helpers import ensure_directory from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker +# Issue #825, Part B Tier 3 — exporter interlinking. Reuses the same default +# namespace as ProvenanceManager.export_prov() and RDFExporter's +# NamespaceManager "semantica" entry, so ontology URIs, KG instance URIs, and +# PROV-exported URIs co-resolve under one shared namespace by default. +from ..provenance.manager import DEFAULT_BASE_URI + class OWLExporter: """ @@ -58,7 +64,7 @@ class OWLExporter: def __init__( self, - ontology_uri: str = "https://semantica.dev/ontology/", + ontology_uri: str = DEFAULT_BASE_URI, version: str = "1.0", format: str = "owl-xml", config: Optional[Dict[str, Any]] = None, @@ -70,7 +76,8 @@ class OWLExporter: Sets up the exporter with ontology URI, version, and format configuration. Args: - ontology_uri: Base URI for the ontology (default: "https://semantica.dev/ontology/") + ontology_uri: Base URI for the ontology (default: ProvenanceManager.DEFAULT_BASE_URI, + shared with RDFExporter's NamespaceManager and export_prov() so URIs co-resolve) version: Ontology version string (default: "1.0") format: Default export format - 'owl-xml' or 'turtle' (default: 'owl-xml') config: Optional configuration dictionary (merged with kwargs) diff --git a/semantica/export/rdf_exporter.py b/semantica/export/rdf_exporter.py index bf6beaf7..7c50ffc5 100644 --- a/semantica/export/rdf_exporter.py +++ b/semantica/export/rdf_exporter.py @@ -71,13 +71,19 @@ class NamespaceManager: """ self.logger = get_logger("namespace_manager") - # Standard RDF namespaces + # Standard RDF namespaces. "semantica" reuses ProvenanceManager's + # DEFAULT_BASE_URI (issue #825, Part B Tier 3 — exporter + # interlinking) so KG-exported and PROV-exported URIs for the same + # entity_id co-resolve to the same namespace instead of two + # independently-hardcoded placeholder domains. + from ..provenance.manager import DEFAULT_BASE_URI + self.namespaces: Dict[str, str] = { "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdfs": "http://www.w3.org/2000/01/rdf-schema#", "owl": "http://www.w3.org/2002/07/owl#", "xsd": "http://www.w3.org/2001/XMLSchema#", - "semantica": "https://semantica.dev/ns#", + "semantica": DEFAULT_BASE_URI, } self.config = config or {} diff --git a/semantica/graph_store/graph_store_provenance.py b/semantica/graph_store/graph_store_provenance.py index c3da635d..06d48fdd 100644 --- a/semantica/graph_store/graph_store_provenance.py +++ b/semantica/graph_store/graph_store_provenance.py @@ -13,37 +13,53 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class GraphStoreWithProvenance: """Graph store with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .graph_store import GraphStore - + self.provenance = provenance self._store = GraphStore(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def add_node(self, node: Any, source: str = None, **kwargs): """Add node with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._store.add_node(node, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: node_id = getattr(node, 'id', f"node_{uuid.uuid4().hex[:8]}") self._prov_manager.track_entity( entity_id=node_id, source=source or "graph_store", entity_type="graph_node", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={"properties": getattr(node, 'properties', {})} ) diff --git a/semantica/ingest/ingest_provenance.py b/semantica/ingest/ingest_provenance.py index b3a97747..51930fa4 100644 --- a/semantica/ingest/ingest_provenance.py +++ b/semantica/ingest/ingest_provenance.py @@ -14,16 +14,25 @@ License: MIT """ from typing import Optional, List +from datetime import datetime import uuid class IngestProvenanceMixin: """Mixin for ingest provenance tracking.""" - - def __init__(self, provenance: bool = False, **kwargs): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **kwargs, + ): self.provenance = provenance self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager @@ -34,17 +43,27 @@ class IngestProvenanceMixin: class PDFIngestorWithProvenance(IngestProvenanceMixin): """PDF ingestor with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .pdf_ingestor import PDFIngestor - - IngestProvenanceMixin.__init__(self, provenance=provenance) + + IngestProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._ingestor = PDFIngestor(**config) - + def ingest(self, file_path: str, **kwargs): """Ingest PDF with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() docs = self._ingestor.ingest(file_path, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: for doc in docs: doc_id = getattr(doc, 'id', f"doc_{uuid.uuid4().hex[:8]}") @@ -52,6 +71,11 @@ class PDFIngestorWithProvenance(IngestProvenanceMixin): entity_id=doc_id, source=file_path, entity_type="document", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "file_type": "pdf", "pages": getattr(doc, 'page_count', None) diff --git a/semantica/kg/kg_provenance.py b/semantica/kg/kg_provenance.py index 11a4807f..784386d0 100644 --- a/semantica/kg/kg_provenance.py +++ b/semantica/kg/kg_provenance.py @@ -54,6 +54,7 @@ Version: 1.0.0 """ from typing import Any, Dict, List, Optional +from datetime import datetime import uuid import time @@ -78,13 +79,21 @@ class GraphBuilderWithProvenance: }) """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .graph_builder import GraphBuilder - + self.provenance = provenance self._builder = GraphBuilder(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager @@ -94,12 +103,18 @@ class GraphBuilderWithProvenance: def build(self, sources, **kwargs): """Build graph with provenance tracking.""" - # Track the build operation + activity_started_at_time = datetime.utcnow().isoformat() + # Track the build operation (recorded before the build runs, so it + # has no end time yet — this is the "in progress" marker). if self.provenance and self._prov_manager: build_id = f"graph_build_{uuid.uuid4().hex[:8]}" self._prov_manager.track_entity( entity_id=build_id, source="graph_construction", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, metadata={ "entity_type": "graph_build_operation", "operation": "build_graph", @@ -107,16 +122,17 @@ class GraphBuilderWithProvenance: "timestamp": time.time() } ) - + result = self._builder.build(sources, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + # Track individual entities and relationships if available if self.provenance and self._prov_manager and hasattr(result, 'get'): try: # Try to extract entities and relationships for tracking entities = result.get('entities', []) relationships = result.get('relationships', []) - + for entity in entities: entity_id = entity.get('id') or str(entity.get('name', '')) if entity_id: @@ -124,6 +140,11 @@ class GraphBuilderWithProvenance: entity_id=entity_id, source="graph_construction", entity_type="graph_entity", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "operation": "build_entity", "entity_type": entity.get('type'), @@ -131,7 +152,7 @@ class GraphBuilderWithProvenance: "timestamp": time.time() } ) - + for relationship in relationships: rel_id = relationship.get('id') or f"{relationship.get('source', '')}-{relationship.get('target', '')}" if rel_id: @@ -139,6 +160,11 @@ class GraphBuilderWithProvenance: entity_id=rel_id, source="graph_construction", entity_type="graph_relationship", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "operation": "build_relationship", "relationship_type": relationship.get('type'), @@ -149,32 +175,39 @@ class GraphBuilderWithProvenance: except Exception as e: # Don't fail the build if provenance tracking fails pass - + return result - + def build_single_source(self, kg_data, **kwargs): """Build graph from single source with provenance tracking.""" - # Track the build operation + activity_started_at_time = datetime.utcnow().isoformat() + # Track the build operation (recorded before the build runs, so it + # has no end time yet — this is the "in progress" marker). if self.provenance and self._prov_manager: build_id = f"graph_build_single_{uuid.uuid4().hex[:8]}" self._prov_manager.track_entity( entity_id=build_id, source="graph_construction", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, metadata={ "entity_type": "graph_build_operation", "operation": "build_single_source", "timestamp": time.time() } ) - + result = self._builder.build_single_source(kg_data, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + # Track entities and relationships if available if self.provenance and self._prov_manager and isinstance(result, dict): try: entities = result.get('entities', []) relationships = result.get('relationships', []) - + for entity in entities: entity_id = entity.get('id') or str(entity.get('name', '')) if entity_id: @@ -182,6 +215,11 @@ class GraphBuilderWithProvenance: entity_id=entity_id, source="graph_construction", entity_type="graph_entity", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "operation": "build_entity", "entity_type": entity.get('type'), @@ -197,6 +235,11 @@ class GraphBuilderWithProvenance: entity_id=rel_id, source="graph_construction", entity_type="graph_relationship", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "operation": "build_relationship", "relationship_type": relationship.get('type'), @@ -247,24 +290,33 @@ class AlgorithmTrackerWithProvenance: ) """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): self.provenance = provenance self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def track_embedding_computation( self, graph: Any, algorithm: str, embeddings: Dict[str, List[float]], parameters: Dict[str, Any], - source: str = None + source: str = None, + **kwargs ): """ Track node embedding algorithm computation with provenance. @@ -286,6 +338,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=execution_id, source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "embedding_computation", "algorithm": algorithm, @@ -303,6 +360,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=f"embedding_{node_id}", source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "node_embedding", "algorithm": algorithm, @@ -345,6 +407,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=execution_id, source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "similarity_calculation", "algorithm": f"similarity_{method}", @@ -363,6 +430,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=f"similarity_{node_id}_{execution_id}", source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "similarity_result", "method": method, @@ -393,6 +465,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=execution_id, source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "link_prediction", "algorithm": f"link_prediction_{method}", @@ -410,6 +487,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=f"prediction_{execution_id}_{i}", source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "link_prediction_result", "method": method, @@ -453,6 +535,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=execution_id, source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "centrality_calculation", "algorithm": f"centrality_{method}", @@ -470,6 +557,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=f"centrality_{node_id}_{execution_id}", source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "centrality_score", "method": method, @@ -500,6 +592,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=execution_id, source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "community_detection", "algorithm": f"community_detection_{method}", @@ -517,6 +614,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=f"community_{execution_id}_{i}", source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "community", "method": method, @@ -547,6 +649,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=execution_id, source=source or "graph_construction", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "graph_construction", "entities_count": entities_count, @@ -573,6 +680,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "similarity_result", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "similarity_result", "node_id": node_id, @@ -599,6 +711,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "similarity_threshold", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "similarity_threshold_analysis", "execution_id": execution_id, @@ -624,6 +741,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "entity_processing", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "entity_processing", "processed_entity_id": entity_id, @@ -648,6 +770,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "relationship_processing", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "relationship_processing", "processed_relationship_id": relationship_id, @@ -672,6 +799,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "path_analysis", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "path_analysis", "paths_count": len(paths) if paths else 0, @@ -700,6 +832,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "path_finding", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "path_finding", "source_node": source_node, @@ -724,6 +861,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "embedding_analysis", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "embedding_analysis", "embeddings_count": len(embeddings), @@ -746,6 +888,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "connectivity_analysis", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "connectivity_analysis", "components_count": len(components), @@ -768,6 +915,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "cross_layer_analysis", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "cross_layer_analysis", "layers_count": len(cross_layer_results) if cross_layer_results else 0, @@ -793,6 +945,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "pipeline_summary", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "pipeline_summary", "pipeline_id": pipeline_id, @@ -820,6 +977,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=summary_id, source=source or "workflow_summary", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "workflow_summary", "master_workflow_id": master_workflow_id, @@ -848,6 +1010,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "link_prediction_result", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "link_prediction_result", "source_node": source_node, @@ -869,6 +1036,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or analysis_type, + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={"entity_type": analysis_type, "timestamp": time.time(), **{k: str(v)[:100] for k, v in kwargs.items() if not callable(v)}}, ) return result_id diff --git a/semantica/llms/llms_provenance.py b/semantica/llms/llms_provenance.py index 62b39ffa..f6277fff 100644 --- a/semantica/llms/llms_provenance.py +++ b/semantica/llms/llms_provenance.py @@ -42,6 +42,7 @@ License: MIT """ from typing import Optional, Dict, Any +from datetime import datetime import time import uuid @@ -54,17 +55,28 @@ class LLMProvenanceMixin: LLM API calls including tokens, costs, and performance metrics. """ - def __init__(self, provenance: bool = False, **kwargs): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **kwargs, + ): """ Initialize LLM provenance tracking. - + Args: provenance: Enable provenance tracking (default: False) + agent_id: Agent identifier for accountability (issue #825); defaults + to the wrapping class name + is_automated: Whether this agent acted without direct human review **kwargs: Additional arguments passed to parent class """ self.provenance = provenance self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager @@ -72,7 +84,7 @@ class LLMProvenanceMixin: except ImportError: # Graceful degradation if provenance module not available self.provenance = False - + def _track_llm_call( self, call_id: str, @@ -82,7 +94,7 @@ class LLMProvenanceMixin: ) -> None: """ Track LLM API call with provenance. - + Args: call_id: Unique identifier for this API call prompt: Input prompt @@ -98,11 +110,22 @@ class LLMProvenanceMixin: response_text = response.content elif not isinstance(response, str): response_text = str(response) - + + # Typed Activity timing (issue #825, Part B Tier 1): popped out so + # it populates real fields, not the opaque metadata blob. + activity_started_at_time = metadata.pop("activity_started_at_time", None) + activity_ended_at_time = metadata.pop("activity_ended_at_time", None) + self._prov_manager.track_entity( entity_id=call_id, source=f"{self.__class__.__name__}_api", entity_type="llm_generation", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_id=call_id, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "model": getattr(self, 'model', 'unknown'), "prompt_preview": prompt[:200] if len(prompt) > 200 else prompt, @@ -124,17 +147,25 @@ class GroqLLMWithProvenance(LLMProvenanceMixin): >>> # API call is tracked with model, tokens, cost, latency """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize Groq LLM with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original GroqLLM """ from .groq_llm import GroqLLM - - LLMProvenanceMixin.__init__(self, provenance=provenance) + + LLMProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._llm = GroqLLM(**config) self.model = getattr(self._llm, 'model', 'groq') @@ -150,22 +181,24 @@ class GroqLLMWithProvenance(LLMProvenanceMixin): LLM response (same format as original GroqLLM) """ start_time = time.time() + activity_started_at_time = datetime.utcnow().isoformat() response = self._llm.generate(prompt, **kwargs) elapsed = time.time() - start_time - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: # Extract token counts if available prompt_tokens = None completion_tokens = None total_cost = None - + if hasattr(response, 'usage'): prompt_tokens = getattr(response.usage, 'prompt_tokens', None) completion_tokens = getattr(response.usage, 'completion_tokens', None) - + if hasattr(response, 'cost'): total_cost = response.cost - + self._track_llm_call( call_id=f"groq_call_{uuid.uuid4().hex[:8]}", prompt=prompt, @@ -175,6 +208,8 @@ class GroqLLMWithProvenance(LLMProvenanceMixin): total_tokens=(prompt_tokens + completion_tokens) if (prompt_tokens and completion_tokens) else None, total_cost=total_cost, latency_seconds=elapsed, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, temperature=kwargs.get('temperature'), max_tokens=kwargs.get('max_tokens'), top_p=kwargs.get('top_p') @@ -194,17 +229,25 @@ class OpenAILLMWithProvenance(LLMProvenanceMixin): Wraps the original OpenAILLM and tracks all API calls. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize OpenAI LLM with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original OpenAILLM """ from .openai_llm import OpenAILLM - - LLMProvenanceMixin.__init__(self, provenance=provenance) + + LLMProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._llm = OpenAILLM(**config) self.model = getattr(self._llm, 'model', 'openai') @@ -220,22 +263,24 @@ class OpenAILLMWithProvenance(LLMProvenanceMixin): LLM response """ start_time = time.time() + activity_started_at_time = datetime.utcnow().isoformat() response = self._llm.generate(prompt, **kwargs) elapsed = time.time() - start_time - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: # Extract token counts if available prompt_tokens = None completion_tokens = None total_cost = None - + if hasattr(response, 'usage'): prompt_tokens = getattr(response.usage, 'prompt_tokens', None) completion_tokens = getattr(response.usage, 'completion_tokens', None) - + if hasattr(response, 'cost'): total_cost = response.cost - + self._track_llm_call( call_id=f"openai_call_{uuid.uuid4().hex[:8]}", prompt=prompt, @@ -245,6 +290,8 @@ class OpenAILLMWithProvenance(LLMProvenanceMixin): total_tokens=(prompt_tokens + completion_tokens) if (prompt_tokens and completion_tokens) else None, total_cost=total_cost, latency_seconds=elapsed, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, temperature=kwargs.get('temperature'), max_tokens=kwargs.get('max_tokens') ) @@ -263,17 +310,25 @@ class HuggingFaceLLMWithProvenance(LLMProvenanceMixin): Wraps the original HuggingFaceLLM and tracks all generations. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize HuggingFace LLM with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original HuggingFaceLLM """ from .huggingface_llm import HuggingFaceLLM - - LLMProvenanceMixin.__init__(self, provenance=provenance) + + LLMProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._llm = HuggingFaceLLM(**config) self.model = getattr(self._llm, 'model', 'huggingface') @@ -289,15 +344,19 @@ class HuggingFaceLLMWithProvenance(LLMProvenanceMixin): LLM response """ start_time = time.time() + activity_started_at_time = datetime.utcnow().isoformat() response = self._llm.generate(prompt, **kwargs) elapsed = time.time() - start_time - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: self._track_llm_call( call_id=f"hf_call_{uuid.uuid4().hex[:8]}", prompt=prompt, response=response, latency_seconds=elapsed, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, max_length=kwargs.get('max_length'), temperature=kwargs.get('temperature') ) @@ -316,17 +375,25 @@ class LiteLLMWithProvenance(LLMProvenanceMixin): Wraps the original LiteLLM and tracks all API calls across providers. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize LiteLLM with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original LiteLLM """ from .lite_llm import LiteLLM - - LLMProvenanceMixin.__init__(self, provenance=provenance) + + LLMProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._llm = LiteLLM(**config) self.model = getattr(self._llm, 'model', 'litellm') @@ -342,22 +409,24 @@ class LiteLLMWithProvenance(LLMProvenanceMixin): LLM response """ start_time = time.time() + activity_started_at_time = datetime.utcnow().isoformat() response = self._llm.generate(prompt, **kwargs) elapsed = time.time() - start_time - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: # LiteLLM provides unified response format prompt_tokens = None completion_tokens = None total_cost = None - + if hasattr(response, 'usage'): prompt_tokens = getattr(response.usage, 'prompt_tokens', None) completion_tokens = getattr(response.usage, 'completion_tokens', None) - + if hasattr(response, '_hidden_params') and 'response_cost' in response._hidden_params: total_cost = response._hidden_params['response_cost'] - + self._track_llm_call( call_id=f"lite_call_{uuid.uuid4().hex[:8]}", prompt=prompt, @@ -366,6 +435,8 @@ class LiteLLMWithProvenance(LLMProvenanceMixin): completion_tokens=completion_tokens, total_cost=total_cost, latency_seconds=elapsed, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, provider=kwargs.get('provider') ) diff --git a/semantica/normalize/normalize_provenance.py b/semantica/normalize/normalize_provenance.py index 37d87867..0dfc17b5 100644 --- a/semantica/normalize/normalize_provenance.py +++ b/semantica/normalize/normalize_provenance.py @@ -11,36 +11,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class NormalizerWithProvenance: """Normalizer with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .normalizer import Normalizer - + self.provenance = provenance self._normalizer = Normalizer(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def normalize(self, data: Any, source: str = None, **kwargs): """Normalize data with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._normalizer.normalize(data, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"normalize_{uuid.uuid4().hex[:8]}", source=source or "normalization", entity_type="normalized_data", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={"method": kwargs.get('method', 'default')} ) diff --git a/semantica/ontology/ontology_provenance.py b/semantica/ontology/ontology_provenance.py index 99c83440..fffbfa10 100644 --- a/semantica/ontology/ontology_provenance.py +++ b/semantica/ontology/ontology_provenance.py @@ -11,36 +11,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class OntologyManagerWithProvenance: """Ontology manager with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .ontology_manager import OntologyManager - + self.provenance = provenance self._manager = OntologyManager(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def add_concept(self, concept: Any, source: str = None, **kwargs): """Add concept with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._manager.add_concept(concept, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"concept_{uuid.uuid4().hex[:8]}", source=source or "ontology", entity_type="ontology_concept", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={"concept_name": str(concept)} ) diff --git a/semantica/parse/parse_provenance.py b/semantica/parse/parse_provenance.py index 6c288c18..6d18386b 100644 --- a/semantica/parse/parse_provenance.py +++ b/semantica/parse/parse_provenance.py @@ -13,36 +13,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class ParserWithProvenance: """Base parser with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .parser import Parser - + self.provenance = provenance self._parser = Parser(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def parse(self, file_path: str, **kwargs): """Parse file with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() data = self._parser.parse(file_path, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"parse_{uuid.uuid4().hex[:8]}", source=file_path, entity_type="parsed_data", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "file_path": file_path, "format": kwargs.get('format', 'unknown') diff --git a/semantica/pipeline/pipeline_provenance.py b/semantica/pipeline/pipeline_provenance.py index 3d5f34c4..b5c7c99b 100644 --- a/semantica/pipeline/pipeline_provenance.py +++ b/semantica/pipeline/pipeline_provenance.py @@ -16,6 +16,7 @@ License: MIT """ from typing import Optional, Any, Dict, List +from datetime import datetime import uuid import time @@ -23,34 +24,50 @@ import time class PipelineWithProvenance: """Pipeline executor with complete provenance tracking.""" - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """Initialize pipeline with optional provenance.""" from .pipeline import Pipeline - + self.provenance = provenance self._pipeline = Pipeline(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def run(self, data: Any, source: Optional[str] = None, **kwargs): """Run pipeline with provenance tracking.""" pipeline_id = f"pipeline_{uuid.uuid4().hex[:8]}" start_time = time.time() - + activity_started_at_time = datetime.utcnow().isoformat() + result = self._pipeline.run(data, **kwargs) elapsed = time.time() - start_time - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=pipeline_id, source=source or "pipeline_execution", entity_type="pipeline_run", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_id=pipeline_id, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "steps": len(self._pipeline.steps) if hasattr(self._pipeline, 'steps') else 0, "duration_seconds": elapsed, diff --git a/semantica/provenance/__init__.py b/semantica/provenance/__init__.py index c6faaea4..4362020a 100644 --- a/semantica/provenance/__init__.py +++ b/semantica/provenance/__init__.py @@ -41,7 +41,13 @@ Author: Semantica Contributors License: MIT """ -from .schemas import ProvenanceEntry, SourceReference +from .schemas import ( + ProvenanceEntry, + SourceReference, + AgentRecord, + ActivityRecord, + Invalidation, +) from .storage import ProvenanceStorage, InMemoryStorage, SQLiteStorage from .manager import ProvenanceManager, default_storage_path from .integrity import compute_checksum, verify_checksum @@ -50,7 +56,10 @@ __all__ = [ # Core schemas "ProvenanceEntry", "SourceReference", - + "AgentRecord", + "ActivityRecord", + "Invalidation", + # Storage backends "ProvenanceStorage", "InMemoryStorage", diff --git a/semantica/provenance/integrity.py b/semantica/provenance/integrity.py index e5327089..bedaaf0e 100644 --- a/semantica/provenance/integrity.py +++ b/semantica/provenance/integrity.py @@ -30,13 +30,36 @@ def compute_checksum(entry: Any) -> str: Creates a deterministic checksum based on critical provenance fields to detect any tampering or corruption of provenance data. - + + Includes `previous_checksum` (issue #825, Part A item 2), which chains + each entry to the prior entry in insertion order (see + ProvenanceStorage.get_chain_head / ProvenanceManager.verify_chain) — + wholesale deletion of a row breaks the chain for the entry that used to + follow it, making the deletion detectable even though per-row checksums + only prove a surviving row wasn't edited in place. Also includes + agent_id/agent_type and the lineage-link fields (parent_entity_id, + previous_version_id, derived_from_id, used_entities) so tampering with + attribution or lineage is detected, not just tampering with the six + original fields. + + Deliberately excludes `entity_id` itself: entity_id is the storage + primary key, and ProvenanceManager.track_entity()'s versioning archives + a prior value by copying it to a new entity_id (e.g. "X" -> "X:v:..."). + If entity_id were hashed, that relabeling would change the archived + copy's checksum, permanently orphaning any later entry whose + previous_checksum had already chained from the pre-relabel value — + a false-positive "broken chain" for a legitimate rename, not tampering. + Tampering that swaps a row's entity_id while keeping its content is a + narrower threat than content tampering, already partially caught by + verify_chain() (it requires a delete+insert, which breaks the chain for + whatever the deleted row's successor was). + Args: entry: ProvenanceEntry or dict to compute checksum for - + Returns: SHA-256 checksum as hexadecimal string - + Example: >>> entry = ProvenanceEntry( ... entity_id="entity_123", @@ -48,26 +71,48 @@ def compute_checksum(entry: Any) -> str: >>> print(checksum) 'a3b2c1d4e5f6...' """ - # Concatenate critical fields for checksum + # Concatenate critical fields for checksum (entity_id intentionally excluded, see docstring) if isinstance(entry, dict): + used_entities = entry.get("used_entities") or [] data = ( - f"{entry.get('entity_id') or ''}" f"{entry.get('entity_type') or ''}" f"{entry.get('activity_id') or ''}" + f"{entry.get('agent_id') or ''}" + f"{entry.get('agent_type') or ''}" f"{entry.get('source_document') or ''}" f"{entry.get('timestamp') or ''}" f"{entry.get('confidence') if entry.get('confidence') is not None else 1.0}" + f"{entry.get('parent_entity_id') or ''}" + f"{entry.get('previous_version_id') or ''}" + f"{entry.get('derived_from_id') or ''}" + f"{','.join(used_entities)}" + f"{entry.get('previous_checksum') or ''}" + f"{bool(entry.get('invalidated'))}" + f"{entry.get('invalidated_at_time') or ''}" + f"{entry.get('invalidated_by') or ''}" + f"{entry.get('invalidation_reason') or ''}" ) else: + used_entities = getattr(entry, "used_entities", None) or [] data = ( - f"{entry.entity_id}" f"{entry.entity_type}" f"{entry.activity_id}" + f"{getattr(entry, 'agent_id', '') or ''}" + f"{getattr(entry, 'agent_type', '') or ''}" f"{entry.source_document}" f"{entry.timestamp}" f"{entry.confidence}" + f"{getattr(entry, 'parent_entity_id', '') or ''}" + f"{getattr(entry, 'previous_version_id', '') or ''}" + f"{getattr(entry, 'derived_from_id', '') or ''}" + f"{','.join(used_entities)}" + f"{getattr(entry, 'previous_checksum', '') or ''}" + f"{bool(getattr(entry, 'invalidated', False))}" + f"{getattr(entry, 'invalidated_at_time', '') or ''}" + f"{getattr(entry, 'invalidated_by', '') or ''}" + f"{getattr(entry, 'invalidation_reason', '') or ''}" ) - + return hashlib.sha256(data.encode('utf-8')).hexdigest() diff --git a/semantica/provenance/manager.py b/semantica/provenance/manager.py index b2e9874c..8e411a17 100644 --- a/semantica/provenance/manager.py +++ b/semantica/provenance/manager.py @@ -33,11 +33,18 @@ import inspect import json import threading -from .schemas import ProvenanceEntry, SourceReference +from .schemas import ProvenanceEntry, SourceReference, AgentRecord, ActivityRecord from .storage import ProvenanceStorage, InMemoryStorage, SQLiteStorage from .integrity import compute_checksum, verify_checksum from ..utils.logging import get_logger +# Issue #825, Part B Tier 3 — configurable base URI for export_prov(), shared +# with RDFExporter's NamespaceManager "semantica" entry (semantica/export/ +# rdf_exporter.py) so KG-exported and PROV-exported URIs for the same +# entity_id co-resolve to the same namespace instead of two different +# placeholder domains. +DEFAULT_BASE_URI = "https://semantica.dev/ns#" + @contextmanager def default_storage_path(path: Optional[str]): @@ -146,6 +153,16 @@ class ProvenanceManager: _raise_on_error: bool = False, ) -> Optional[ProvenanceEntry]: """Compute checksum, store entry persistently, and log/handle storage errors (#783).""" + # Hash-chain linkage (issue #825, Part A item 2): link this entry to + # the previous entry in global insertion order before hashing, so + # deleting a row later breaks the chain for whatever followed it. + try: + head = self.storage.get_chain_head(_conn) + except Exception: + head = None + entry.sequence_id = (head[0] + 1) if head else 1 + entry.previous_checksum = head[1] if head else None + entry.checksum = compute_checksum(entry) try: @@ -178,6 +195,69 @@ class ProvenanceManager: with self.storage.transaction() as conn: yield conn + # Recognized typed kwargs for track_entity, used both directly and to + # split track_entities_batch's **metadata into real kwargs vs. free-form + # metadata (issue #825, Part A item 3 — fixes a bug where agent_id/ + # entity_type/activity_id passed to track_entities_batch were silently + # absorbed into the opaque metadata blob instead of populating fields). + _TRACK_ENTITY_KWARGS = frozenset({ + "entity_type", "activity_id", "agent_id", "agent_type", "is_automated", + "role", "agent", "source_location", "source_quote", "confidence", + "parent_entity_id", "used_entities", + # Part B Tier 1/2/3 (issue #825) + "activity", "activity_started_at_time", "activity_ended_at_time", + "acted_on_behalf_of", "informed_by", "valid_from", "valid_until", + "revision_type", "supersedes", "bundle_id", + }) + + @staticmethod + def _resolve_agent_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: + """ + Resolve agent_id/agent_type/is_automated/role from kwargs. + + Accepts either an `agent=AgentRecord(...)` kwarg (takes precedence) + or individual `agent_id`/`agent_type`/`is_automated` scalar kwargs. + Fixes issue #825's dead agent_id field: previously no track_* + method read agent_id from kwargs at all, so it was always the + dataclass default "semantica" regardless of what callers passed. + """ + agent = kwargs.get("agent") + if isinstance(agent, AgentRecord): + return { + "agent_id": agent.id, + "agent_type": agent.agent_type, + "is_automated": agent.is_automated, + "role": kwargs.get("role"), + } + return { + "agent_id": kwargs.get("agent_id", "semantica"), + "agent_type": kwargs.get("agent_type", "software_agent"), + "is_automated": kwargs.get("is_automated", True), + "role": kwargs.get("role"), + } + + @staticmethod + def _resolve_activity_kwargs(kwargs: Dict[str, Any], default_activity_id: str) -> Dict[str, Any]: + """ + Resolve activity_id/activity_started_at_time/activity_ended_at_time + from kwargs (issue #825, Part B Tier 1 — typed Activity). + + Accepts either an `activity=ActivityRecord(...)` kwarg (takes + precedence) or individual scalar kwargs. + """ + activity = kwargs.get("activity") + if isinstance(activity, ActivityRecord): + return { + "activity_id": activity.id, + "activity_started_at_time": activity.started_at_time, + "activity_ended_at_time": activity.ended_at_time, + } + return { + "activity_id": kwargs.get("activity_id", default_activity_id), + "activity_started_at_time": kwargs.get("activity_started_at_time"), + "activity_ended_at_time": kwargs.get("activity_ended_at_time"), + } + # === Entity Tracking (from kg.ProvenanceTracker) === def track_entity( @@ -253,15 +333,31 @@ class ProvenanceManager: history_entry.entity_id = history_id + # Pure relabel: checksum/sequence_id/previous_checksum are + # left exactly as they were. compute_checksum() excludes + # entity_id specifically so this is safe — recomputing it + # here (or assigning a fresh sequence slot) would either + # invalidate any later entry that already chained from + # this row's checksum, or strand the original sequence + # position and make verify_chain() see a phantom gap + # (issue #825: an archival relabel is not tampering and + # must not look like it to the hash chain). self.storage._store_with_conn(conn, history_entry) archived_history_id = history_id if not explicit_parent_supplied: parent_id = history_id + agent_info = self._resolve_agent_kwargs(kwargs) + activity_info = self._resolve_activity_kwargs(kwargs, "entity_tracking") + entry = ProvenanceEntry( entity_id=entity_id, entity_type=kwargs.get("entity_type", "entity"), - activity_id=kwargs.get("activity_id", "entity_tracking"), + activity_id=activity_info["activity_id"], + agent_id=agent_info["agent_id"], + agent_type=agent_info["agent_type"], + is_automated=agent_info["is_automated"], + role=agent_info["role"], source_document=source, source_location=kwargs.get("source_location"), source_quote=kwargs.get("source_quote"), @@ -271,11 +367,32 @@ class ProvenanceManager: last_updated=datetime.utcnow().isoformat(), parent_entity_id=parent_id, used_entities=list(kwargs.get("used_entities", [])), + activity_started_at_time=activity_info["activity_started_at_time"], + activity_ended_at_time=activity_info["activity_ended_at_time"], + acted_on_behalf_of=kwargs.get("acted_on_behalf_of"), + informed_by_activities=list(kwargs.get("informed_by", [])), + valid_from=kwargs.get("valid_from"), + valid_until=kwargs.get("valid_until"), + revision_type=kwargs.get("revision_type"), + supersedes=kwargs.get("supersedes"), + bundle_id=kwargs.get("bundle_id"), ) + # Versioning vs. derivation (issue #825, Part A item 4): + # previous_version_id always captures "this corrects a prior + # version of the same fact" when one was archived, independent + # of whether an explicit cross-source parent was also given. + # derived_from_id captures "this fact was derived from a + # different source entity" — only set when a parent was + # explicitly resolved (kwarg/metadata['derived_from']/source + # heuristic), never from the automatic archival link. + entry.previous_version_id = archived_history_id + if explicit_parent_supplied: + entry.derived_from_id = parent_id + if archived_history_id and explicit_parent_supplied: entry.used_entities.append(archived_history_id) - + self._save_entry(entry, _conn=conn, _raise_on_error=True) except Exception as e: # When called from a batch's shared transaction (_conn is not None), @@ -323,18 +440,34 @@ class ProvenanceManager: ... metadata={"type": "founded"} ... ) """ + agent_info = self._resolve_agent_kwargs(kwargs) + activity_info = self._resolve_activity_kwargs(kwargs, "relationship_tracking") + entry = ProvenanceEntry( entity_id=relationship_id, entity_type="relationship", - activity_id=kwargs.get("activity_id", "relationship_tracking"), + activity_id=activity_info["activity_id"], + agent_id=agent_info["agent_id"], + agent_type=agent_info["agent_type"], + is_automated=agent_info["is_automated"], + role=agent_info["role"], source_document=source, source_location=kwargs.get("source_location"), confidence=kwargs.get("confidence", 1.0), metadata=metadata or {}, first_seen=datetime.utcnow().isoformat(), - last_updated=datetime.utcnow().isoformat() + last_updated=datetime.utcnow().isoformat(), + activity_started_at_time=activity_info["activity_started_at_time"], + activity_ended_at_time=activity_info["activity_ended_at_time"], + acted_on_behalf_of=kwargs.get("acted_on_behalf_of"), + informed_by_activities=list(kwargs.get("informed_by", [])), + valid_from=kwargs.get("valid_from"), + valid_until=kwargs.get("valid_until"), + revision_type=kwargs.get("revision_type"), + supersedes=kwargs.get("supersedes"), + bundle_id=kwargs.get("bundle_id"), ) - + return self._save_entry(entry) # === Chunk Tracking (from split.ProvenanceTracker) === @@ -374,19 +507,38 @@ class ProvenanceManager: ... end_index=500 ... ) """ + agent_info = self._resolve_agent_kwargs(metadata) + activity_info = self._resolve_activity_kwargs(metadata, "chunking") + for key in ( + "agent_id", "agent_type", "is_automated", "role", "agent", + "activity", "activity_started_at_time", "activity_ended_at_time", + ): + metadata.pop(key, None) + entry = ProvenanceEntry( entity_id=chunk_id, entity_type="chunk", - activity_id="chunking", + activity_id=activity_info["activity_id"], + agent_id=agent_info["agent_id"], + agent_type=agent_info["agent_type"], + is_automated=agent_info["is_automated"], + role=agent_info["role"], source_document=source_document, source_location=source_path, start_index=start_index, end_index=end_index, parent_entity_id=parent_chunk_id, + # A chunk split from a parent chunk is a derivation (a new entity + # produced from an existing one), not a correction of the same + # fact — see track_entity's previous_version_id/derived_from_id + # split (issue #825, Part A item 4). + derived_from_id=parent_chunk_id, metadata=metadata, - timestamp=datetime.utcnow().isoformat() + timestamp=datetime.utcnow().isoformat(), + activity_started_at_time=activity_info["activity_started_at_time"], + activity_ended_at_time=activity_info["activity_ended_at_time"], ) - + return self._save_entry(entry, _conn=_conn, _raise_on_error=(_conn is not None)) # === Source Tracking (from conflicts.SourceTracker) === @@ -425,10 +577,22 @@ class ProvenanceManager: ... source=source ... ) """ + agent_info = self._resolve_agent_kwargs(metadata) + activity_info = self._resolve_activity_kwargs(metadata, "property_tracking") + for key in ( + "agent_id", "agent_type", "is_automated", "role", "agent", + "activity", "activity_started_at_time", "activity_ended_at_time", + ): + metadata.pop(key, None) + entry = ProvenanceEntry( entity_id=f"{entity_id}_{property_name}", entity_type="property", - activity_id="property_tracking", + activity_id=activity_info["activity_id"], + agent_id=agent_info["agent_id"], + agent_type=agent_info["agent_type"], + is_automated=agent_info["is_automated"], + role=agent_info["role"], source_document=source.document, source_location=f"page_{source.page}" if source.page else source.section, confidence=source.confidence, @@ -440,9 +604,11 @@ class ProvenanceManager: **metadata, **source.metadata }, - timestamp=datetime.utcnow().isoformat() + timestamp=datetime.utcnow().isoformat(), + activity_started_at_time=activity_info["activity_started_at_time"], + activity_ended_at_time=activity_info["activity_ended_at_time"], ) - + return self._save_entry(entry) # === Batch Operations === @@ -473,7 +639,19 @@ class ProvenanceManager: """ tracked_count = 0 batch_size = 1000 # Justification (#807): 1,000 items per transaction bounds SQLite WAL frame growth and reduces lock contention during multi-thousand-row imports while achieving a 1000x reduction in connection/commit overhead. - + + # Split batch-level **metadata into recognized typed track_entity + # kwargs (entity_type, activity_id, agent_id, ...) vs. free-form data. + # Previously ALL of **metadata was merged into the metadata dict and + # passed as track_entity's positional `metadata` arg, so typed kwargs + # like agent_id/entity_type/activity_id were silently absorbed into + # the opaque metadata JSON blob instead of populating real fields + # (issue #825, Part A item 3 — this is what made agent_id a "dead" + # field for every batch caller, including the documented example in + # docs/guides/provenance.md). + batch_kwargs = {k: v for k, v in metadata.items() if k in self._TRACK_ENTITY_KWARGS} + free_metadata = {k: v for k, v in metadata.items() if k not in self._TRACK_ENTITY_KWARGS} + for i in range(0, len(entities), batch_size): batch = entities[i : i + batch_size] batch_count = 0 @@ -483,12 +661,14 @@ class ProvenanceManager: entity_id = entity.get("id") or entity.get("entity_id") if not entity_id: continue - - entity_metadata = {**metadata, **entity.get("metadata", {})} - + + entity_metadata = {**free_metadata, **entity.get("metadata", {})} + try: with self.storage.savepoint(conn): - self.track_entity(entity_id, source, entity_metadata, _conn=conn) + self.track_entity( + entity_id, source, entity_metadata, _conn=conn, **batch_kwargs + ) batch_count += 1 except Exception: pass # Continue with other entities in this batch @@ -654,11 +834,142 @@ class ProvenanceManager: if supports_max_depth: return self.storage.trace_lineage(entity_id, max_depth=max_depth) return self.storage.trace_lineage(entity_id) - + + def trace_descendants( + self, entity_id: str, max_depth: Optional[int] = None + ) -> List[ProvenanceEntry]: + """ + Trace downstream descendants (reverse lineage) and return raw entries. + + This is the counterpart to trace_lineage()/get_lineage(), which only + ever trace upstream ancestors via parent_entity_id/used_entities. + Downstream traceability answers the incident-response question "entity + X was wrong — what downstream facts used it?" (issue #825, Part A + item 5). + + Args: + entity_id: Entity identifier + max_depth: Optional maximum BFS depth + + Returns: + List of ProvenanceEntry objects that (transitively) reference + entity_id, in BFS order. + """ + return self.storage.trace_descendants(entity_id, max_depth=max_depth) + + def get_descendants(self, entity_id: str) -> Dict[str, Any]: + """ + Get downstream descendants for an entity, mirroring get_lineage()'s + return shape but for the reverse direction. + + Args: + entity_id: Entity identifier + + Returns: + Dictionary containing descendant entries, or {} if none found. + """ + descendant_entries = self.trace_descendants(entity_id) + + if not descendant_entries: + return {} + + integrity_verified = all(verify_checksum(entry) for entry in descendant_entries) + chain_dicts = [entry.to_dict() for entry in descendant_entries] + return { + "entity_id": entity_id, + "descendant_chain": chain_dicts, + "entries": chain_dicts, + "entity_count": len(descendant_entries), + "integrity_verified": integrity_verified, + } + + def revision_history(self, entity_id: str) -> List[Dict[str, Any]]: + """ + Return the version history for entity_id in ascending order. + + Issue #825, Part B Tier 3 — closes the "no direct equivalent yet" + gap for kg.ProvenanceTracker.revision_history() documented in + docs/migration/kg-provenance-tracker.md. + + Walks the entity's own previous_version_id chain (not the full + upstream lineage via get_lineage(), which also pulls in unrelated + derived_from_id/used_entities links from other source entities). + + Args: + entity_id: Entity identifier + + Returns: + List of {version, valid_from, valid_until, recorded_at, author, + revision_type, supersedes} dicts, oldest first. valid_from/ + valid_until use the entry's own explicit fields when set (the + fact's asserted validity window); otherwise valid_from defaults + to when this version was recorded, and valid_until to the next + version's timestamp (None for the current/most recent version). + Empty list if entity_id was never tracked. + """ + current = self.storage.retrieve(entity_id) + if current is None: + return [] + + chain: List[ProvenanceEntry] = [current] + visited = {entity_id} + cursor = current + while getattr(cursor, "previous_version_id", None): + prev_id = cursor.previous_version_id + if prev_id in visited: + break + prev_entry = self.storage.retrieve(prev_id) + if prev_entry is None: + break + chain.append(prev_entry) + visited.add(prev_id) + cursor = prev_entry + + chain.reverse() # oldest first + + history = [] + for i, entry in enumerate(chain): + default_valid_until = chain[i + 1].timestamp if i + 1 < len(chain) else None + version_dict: Dict[str, Any] = { + "version": i + 1, + "valid_from": entry.valid_from or entry.timestamp, + "valid_until": entry.valid_until or default_valid_until, + "recorded_at": entry.timestamp, + "author": entry.agent_id, + } + if entry.revision_type: + version_dict["revision_type"] = entry.revision_type + if entry.supersedes: + version_dict["supersedes"] = entry.supersedes + history.append(version_dict) + return history + + def query_recorded_between(self, start: str, end: str) -> List[Dict[str, Any]]: + """ + Return all provenance entries whose timestamp falls within [start, end]. + + Issue #825, Part B Tier 3 — closes the "no direct equivalent yet" + gap for kg.ProvenanceTracker.query_recorded_between() documented in + docs/migration/kg-provenance-tracker.md. + + Args: + start: Start of range, ISO 8601 string (inclusive) + end: End of range, ISO 8601 string (inclusive) + + Returns: + List of matching entries as dicts, sorted by timestamp ascending. + """ + matches = [ + e for e in self.storage.retrieve_all() + if e.timestamp and start <= e.timestamp <= end + ] + matches.sort(key=lambda e: e.timestamp) + return [e.to_dict() for e in matches] + def get_all_sources(self, entity_id: str) -> List[Dict[str, Any]]: """ Get all sources for an entity (kg.ProvenanceTracker compatible). - + Args: entity_id: Entity identifier @@ -694,13 +1005,93 @@ class ProvenanceManager: if entry: return entry.to_dict() return None - + + # === Invalidation (tombstone, not hard delete) === + + def invalidate( + self, + entity_id: str, + agent_id: str, + reason: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> ProvenanceEntry: + """ + Mark a tracked entity as invalidated instead of deleting it + (issue #825, Part A item 1 — prov:Invalidation). + + An audit needs "this was deleted/corrected, by whom, when, why" to + itself be provable. Rather than mutating the row in place, this + archives the pre-invalidation state under a stable versioned key + (the same pattern track_entity() uses for corrections) and then + writes the invalidated entry as a fresh, chained append. Mutating + the existing row's checksum in place would silently invalidate any + later entry that had already chained its previous_checksum from + this row's pre-invalidation value — turning a legitimate + invalidation into a false-positive "broken chain" report. + + The entry remains visible via retrieve()/retrieve_all()/lineage + traversal, but callers can filter on `invalidated` to exclude + retracted facts. + + Args: + entity_id: Entity identifier to invalidate + agent_id: Agent responsible for the invalidation (prov:Agent) + reason: Optional human-readable reason + metadata: Optional metadata to merge into the entry + + Returns: + The updated (invalidated) ProvenanceEntry + + Raises: + ValueError: If no provenance entry exists for entity_id + + Example: + >>> prov_mgr.invalidate("entity_1", agent_id="reviewer_jane", + ... reason="Source document retracted") + """ + with self.storage.transaction() as conn: + existing = self.storage._retrieve_with_conn(conn, entity_id) + if existing is None: + raise ValueError( + f"Cannot invalidate: no provenance entry found for entity_id={entity_id!r}" + ) + + # Archive the pre-invalidation state under a stable key — a pure + # relabel (see track_entity's identical pattern), so its + # checksum/sequence_id/previous_checksum are left untouched. + history_entry = copy.deepcopy(existing) + base_history_id = f"{entity_id}:v:{existing.last_updated}" + history_id = base_history_id + counter = 1 + while self.storage._retrieve_with_conn(conn, history_id): + history_id = f"{base_history_id}:{counter}" + counter += 1 + history_entry.entity_id = history_id + self.storage._store_with_conn(conn, history_entry) + + entry = copy.deepcopy(existing) + entry.invalidated = True + entry.invalidated_at_time = datetime.utcnow().isoformat() + entry.invalidated_by = agent_id + entry.invalidation_reason = reason + entry.previous_version_id = history_id + if metadata: + entry.metadata = {**entry.metadata, **metadata} + + self._save_entry(entry, _conn=conn, _raise_on_error=True) + + return entry + # === Utility Methods === - + def clear(self) -> int: """ Clear all provenance data. - + + Note: this is a bulk storage reset (used for dev/test teardown), not + a single-fact retraction — use invalidate() to retract/correct an + individual tracked entity while preserving its audit trail. + Returns: Number of entries cleared """ @@ -801,28 +1192,44 @@ class ProvenanceManager: ) return "\n".join(lines) - def export_prov(self, format: str = "turtle") -> str: + # Agent-type refinement for export_prov (issue #825, Part A item 6 — + # cheap, opportunistic PROV-O typing on top of the generic prov:Agent). + _AGENT_TYPE_PROV_CLASS = { + "person": "Person", + "software_agent": "SoftwareAgent", + "organization": "Organization", + } + + def export_prov(self, format: str = "turtle", base_uri: Optional[str] = None) -> str: """ Export provenance as W3C PROV-O RDF. Args: format: RDF format ('turtle', 'ntriples', 'jsonld') + base_uri: Namespace URI entities/agents/activities are minted + under (issue #825, Part B Tier 3). Defaults to + DEFAULT_BASE_URI, which matches RDFExporter's NamespaceManager + "semantica" entry so KG-exported and PROV-exported URIs for + the same entity_id co-resolve. Returns: Serialized RDF string """ - from rdflib import Graph, Literal, Namespace, URIRef + from rdflib import BNode, Graph, Literal, Namespace, URIRef from rdflib.namespace import RDF, XSD PROV = Namespace("http://www.w3.org/ns/prov#") - EX = Namespace("http://example.org/ns/") + EX = Namespace(base_uri or DEFAULT_BASE_URI) g = Graph() g.bind("prov", PROV) g.bind("ex", EX) + def uri(entity_id: Any) -> URIRef: + return URIRef(EX[str(entity_id)]) + for e in self.storage.retrieve_all(): - ent_uri = URIRef(EX[str(e.entity_id)]) + ent_uri = uri(e.entity_id) g.add((ent_uri, RDF.type, PROV.Entity)) if getattr(e, "timestamp", None): @@ -834,29 +1241,125 @@ class ProvenanceManager: ) ) + ag_uri = None if getattr(e, "agent_id", None) and e.agent_id != "unknown": - ag_uri = URIRef(EX[str(e.agent_id)]) + ag_uri = uri(e.agent_id) g.add((ag_uri, RDF.type, PROV.Agent)) + prov_subclass = self._AGENT_TYPE_PROV_CLASS.get( + getattr(e, "agent_type", None) + ) + if prov_subclass: + g.add((ag_uri, RDF.type, PROV[prov_subclass])) g.add((ent_uri, PROV.wasAttributedTo, ag_uri)) + # Qualified Association with hadRole (issue #825, Part A item + # 6): distinguishes "approved by" from "generated by" from + # "reviewed by" for the same agent/entity pair, which the + # plain wasAttributedTo triple above cannot express. + association = BNode() + g.add((ent_uri, PROV.qualifiedAssociation, association)) + g.add((association, RDF.type, PROV.Association)) + g.add((association, PROV.agent, ag_uri)) + role = getattr(e, "role", None) or "generator" + g.add((association, PROV.hadRole, uri(f"role_{role}"))) + + # prov:actedOnBehalfOf (issue #825, Part B Tier 2) — agent + # delegation, e.g. an automated agent acting on behalf of the + # human/organization that authorized it. + delegate_id = getattr(e, "acted_on_behalf_of", None) + if delegate_id: + delegate_uri = uri(delegate_id) + g.add((delegate_uri, RDF.type, PROV.Agent)) + g.add((ag_uri, PROV.actedOnBehalfOf, delegate_uri)) + + act_uri = None if getattr(e, "activity_id", None) and e.activity_id != "unknown": - act_uri = URIRef(EX[str(e.activity_id)]) + act_uri = uri(e.activity_id) g.add((act_uri, RDF.type, PROV.Activity)) g.add((ent_uri, PROV.wasGeneratedBy, act_uri)) + # Typed Activity timing (issue #825, Part B Tier 1) + if getattr(e, "activity_started_at_time", None): + g.add((act_uri, PROV.startedAtTime, + Literal(e.activity_started_at_time, datatype=XSD.dateTime))) + if getattr(e, "activity_ended_at_time", None): + g.add((act_uri, PROV.endedAtTime, + Literal(e.activity_ended_at_time, datatype=XSD.dateTime))) + + # Qualified Generation (issue #825, Part B Tier 1) + generation = BNode() + g.add((ent_uri, PROV.qualifiedGeneration, generation)) + g.add((generation, RDF.type, PROV.Generation)) + g.add((generation, PROV.activity, act_uri)) + if getattr(e, "timestamp", None): + g.add((generation, PROV.atTime, Literal(e.timestamp, datatype=XSD.dateTime))) + + # prov:wasAssociatedWith (issue #825, Part B Tier 2) — direct + # Activity->Agent link, distinct from the Entity->Agent + # wasAttributedTo/qualifiedAssociation triples above. + if ag_uri is not None: + g.add((act_uri, PROV.wasAssociatedWith, ag_uri)) + + # prov:wasInformedBy (issue #825, Part B Tier 2) — chains this + # activity to prior activities it was informed by (e.g. a + # pipeline stage informed by the stage before it). + for informing_id in getattr(e, "informed_by_activities", []): + g.add((act_uri, PROV.wasInformedBy, uri(informing_id))) + + def emit_derivation(source_id: Any) -> None: + """Emit plain + qualified wasDerivedFrom for a source entity.""" + s_uri = uri(source_id) + g.add((ent_uri, PROV.wasDerivedFrom, s_uri)) + derivation = BNode() + g.add((ent_uri, PROV.qualifiedDerivation, derivation)) + g.add((derivation, RDF.type, PROV.Derivation)) + g.add((derivation, PROV.entity, s_uri)) + if act_uri is not None: + g.add((derivation, PROV.hadActivity, act_uri)) + if getattr(e, "parent_entity_id", None): - p_uri = URIRef(EX[str(e.parent_entity_id)]) - g.add((ent_uri, PROV.wasDerivedFrom, p_uri)) + emit_derivation(e.parent_entity_id) for u_id in getattr(e, "used_entities", []): - u_uri = URIRef(EX[str(u_id)]) + u_uri = uri(u_id) # Emit wasDerivedFrom only when this used entity is not the same # as parent_entity_id — which already carries that triple above. if u_id != getattr(e, "parent_entity_id", None): - g.add((ent_uri, PROV.wasDerivedFrom, u_uri)) - if getattr(e, "activity_id", None) and e.activity_id != "unknown": - act_uri = URIRef(EX[str(e.activity_id)]) + emit_derivation(u_id) + if act_uri is not None: g.add((act_uri, PROV.used, u_uri)) + # Qualified Usage (issue #825, Part B Tier 1) + usage = BNode() + g.add((act_uri, PROV.qualifiedUsage, usage)) + g.add((usage, RDF.type, PROV.Usage)) + g.add((usage, PROV.entity, u_uri)) + + # Qualified Invalidation (issue #825, Part A item 1): records the + # tombstone as provable RDF rather than a silent hard delete. + if getattr(e, "invalidated", False): + invalidation = BNode() + g.add((ent_uri, PROV.qualifiedInvalidation, invalidation)) + g.add((invalidation, RDF.type, PROV.Invalidation)) + if getattr(e, "invalidated_at_time", None): + g.add( + ( + invalidation, + PROV.invalidatedAtTime, + Literal(e.invalidated_at_time, datatype=XSD.dateTime), + ) + ) + if getattr(e, "invalidated_by", None): + inv_ag_uri = uri(e.invalidated_by) + g.add((inv_ag_uri, RDF.type, PROV.Agent)) + g.add((invalidation, PROV.agent, inv_ag_uri)) + + # prov:Collection / prov:Bundle membership (issue #825, Part B + # Tier 3) — partitions provenance by source/dataset/ingestion-run. + # Membership triples, not true RDF named-graph partitioning. + if getattr(e, "bundle_id", None): + bundle_uri = uri(f"bundle_{e.bundle_id}") + g.add((bundle_uri, RDF.type, PROV.Bundle)) + g.add((bundle_uri, PROV.hadMember, ent_uri)) rdf_format = "json-ld" if format == "jsonld" else format return g.serialize(format=rdf_format) @@ -873,6 +1376,7 @@ class ProvenanceManager: """ entries = self.storage.retrieve_all() all_ids = {e.entity_id for e in entries} + all_activity_ids = {e.activity_id for e in entries if getattr(e, "activity_id", None)} missing_refs = [] for e in entries: @@ -882,15 +1386,77 @@ class ProvenanceManager: for u_id in getattr(e, "used_entities", []): if u_id not in all_ids: missing_refs.append(f"{e.entity_id} -> {u_id}") + for ref_id in ( + getattr(e, "previous_version_id", None), + getattr(e, "derived_from_id", None), + getattr(e, "supersedes", None), + ): + if ref_id and ref_id not in all_ids: + missing_refs.append(f"{e.entity_id} -> {ref_id}") + # informed_by_activities references activity_ids, a distinct + # ID space from entity_id (issue #825, Part B Tier 2). + for act_id in getattr(e, "informed_by_activities", []): + if act_id not in all_activity_ids: + missing_refs.append(f"{e.entity_id} (activity) -> {act_id}") valid = len(missing_refs) == 0 errors = len(missing_refs) + invalidated_count = sum(1 for e in entries if getattr(e, "invalidated", False)) return { "valid": valid, "total_entries": len(entries), "missing_references": missing_refs, + "invalidated_count": invalidated_count, "strict": strict, "errors": errors, } + def verify_chain(self) -> Dict[str, Any]: + """ + Verify the hash chain across all provenance entries (issue #825, + Part A item 2). + + Sorts entries by sequence_id (global insertion order) and checks + both that each entry's own checksum matches its content, and that + each entry's previous_checksum matches the checksum of the entry + that precedes it. A gap in the chain — a surviving entry whose + previous_checksum doesn't match its predecessor's checksum — is + exactly what wholesale row deletion produces, since per-row + checksums alone can't detect that a row is simply missing. + + Returns: + Dictionary with "valid", "total_entries", and "broken_links" + (list of {"entity_id", "sequence_id", "reason"} dicts). + """ + entries = sorted( + (e for e in self.storage.retrieve_all() if e.sequence_id is not None), + key=lambda e: e.sequence_id, + ) + + broken_links: List[Dict[str, Any]] = [] + expected_previous: Optional[str] = None + for entry in entries: + if not verify_checksum(entry): + broken_links.append({ + "entity_id": entry.entity_id, + "sequence_id": entry.sequence_id, + "reason": "checksum_mismatch", + }) + continue + if entry.previous_checksum != expected_previous: + broken_links.append({ + "entity_id": entry.entity_id, + "sequence_id": entry.sequence_id, + "reason": "chain_break", + "expected_previous_checksum": expected_previous, + "actual_previous_checksum": entry.previous_checksum, + }) + expected_previous = entry.checksum + + return { + "valid": len(broken_links) == 0, + "total_entries": len(entries), + "broken_links": broken_links, + } + diff --git a/semantica/provenance/schemas.py b/semantica/provenance/schemas.py index 85853a83..df888db4 100644 --- a/semantica/provenance/schemas.py +++ b/semantica/provenance/schemas.py @@ -13,10 +13,14 @@ Consolidates: W3C PROV-O Mapping: - ProvenanceEntry.entity_id → prov:Entity - ProvenanceEntry.activity_id → prov:Activity - - ProvenanceEntry.agent_id → prov:Agent - - ProvenanceEntry.parent_entity_id → prov:wasDerivedFrom + - ProvenanceEntry.agent_id / agent_type → prov:Agent (Person|SoftwareAgent|Organization) + - ProvenanceEntry.role → prov:hadRole (via prov:qualifiedAssociation) + - ProvenanceEntry.parent_entity_id → prov:wasDerivedFrom (legacy combined field) + - ProvenanceEntry.derived_from_id → prov:wasDerivedFrom (true cross-source derivation) + - ProvenanceEntry.previous_version_id → prior version of the same fact (correction/versioning) - ProvenanceEntry.used_entities → prov:used - ProvenanceEntry.timestamp → prov:generatedAtTime + - ProvenanceEntry.invalidated* → prov:Invalidation (tombstone, not a hard delete) Author: Semantica Contributors License: MIT @@ -75,36 +79,89 @@ class ProvenanceEntry: entity_type: str activity_id: str agent_id: str = "semantica" - + + # Accountability-scoped agent typing (issue #825, Part A item 3) + agent_type: str = "software_agent" # "person" | "software_agent" | "organization" + is_automated: bool = True + role: Optional[str] = None # prov:hadRole, e.g. "generator", "approver", "reviewer" + # Audit-grade source tracking source_document: str = "" source_location: Optional[str] = None source_quote: Optional[str] = None - + # Temporal tracking (from kg.ProvenanceTracker) timestamp: str = field(default_factory=lambda: datetime.utcnow().isoformat()) first_seen: Optional[str] = None last_updated: Optional[str] = None - + # Quality metrics confidence: float = 1.0 checksum: Optional[str] = None - - # Chain of custody (W3C PROV-O) + + # Hash-chain linkage (issue #825, Part A item 2): previous_checksum links + # this entry to the prior entry in global insertion order (sequence_id), + # so wholesale row deletion breaks the chain and becomes detectable via + # ProvenanceManager.verify_chain(). + sequence_id: Optional[int] = None + previous_checksum: Optional[str] = None + + # Chain of custody (W3C PROV-O) — kept for backward compatibility. parent_entity_id: Optional[str] = None used_entities: List[str] = field(default_factory=list) - + + # Versioning vs. derivation (issue #825, Part A item 4): previous_version_id + # is "this corrects/replaces a prior version of the same fact" + # (prov:specializationOf-flavored); derived_from_id is "this fact was + # derived from a different source entity" (prov:wasDerivedFrom). Both are + # additive alongside parent_entity_id, which remains the legacy combined + # field for existing readers. + previous_version_id: Optional[str] = None + derived_from_id: Optional[str] = None + + # Typed Activity (issue #825, Part B Tier 1): prov:startedAtTime/endedAtTime + # for the activity that generated this entry. + activity_started_at_time: Optional[str] = None + activity_ended_at_time: Optional[str] = None + + # prov:actedOnBehalfOf / prov:wasInformedBy (issue #825, Part B Tier 2) + acted_on_behalf_of: Optional[str] = None + informed_by_activities: List[str] = field(default_factory=list) + + # Bitemporal fields merged from the deprecated kg.ProvenanceTracker + # (issue #825, Part B Tier 3) — in the deprecated tracker these were + # always caller-supplied optional metadata keys, never auto-computed; + # same contract here. valid_from/valid_until express the fact's own + # asserted validity window (domain/business time), independent of + # sequence_id/timestamp which track when the system recorded it. + valid_from: Optional[str] = None + valid_until: Optional[str] = None + revision_type: Optional[str] = None + supersedes: Optional[str] = None + + # prov:Collection / prov:Bundle membership (issue #825, Part B Tier 3): + # partitions provenance by source/dataset/ingestion-run. + bundle_id: Optional[str] = None + + # Invalidation / tombstone tracking (issue #825, Part A item 1): set via + # ProvenanceManager.invalidate() instead of deleting the row, so an audit + # can prove a fact existed, was reviewed, and was retracted/corrected. + invalidated: bool = False + invalidated_at_time: Optional[str] = None + invalidated_by: Optional[str] = None + invalidation_reason: Optional[str] = None + # Chunk-specific fields (from split.ProvenanceInfo) start_index: Optional[int] = None end_index: Optional[int] = None - + # Source credibility (from conflicts.SourceTracker) credibility: Optional[float] = None - + # Metadata metadata: Dict[str, Any] = field(default_factory=dict) version: str = "1.0" - + def to_dict(self) -> Dict[str, Any]: """ Convert provenance entry to dictionary. @@ -117,6 +174,9 @@ class ProvenanceEntry: "entity_type": self.entity_type, "activity_id": self.activity_id, "agent_id": self.agent_id, + "agent_type": self.agent_type, + "is_automated": self.is_automated, + "role": self.role, "source_document": self.source_document, "source_location": self.source_location, "source_quote": self.source_quote, @@ -125,8 +185,25 @@ class ProvenanceEntry: "last_updated": self.last_updated, "confidence": self.confidence, "checksum": self.checksum, + "sequence_id": self.sequence_id, + "previous_checksum": self.previous_checksum, "parent_entity_id": self.parent_entity_id, "used_entities": self.used_entities, + "previous_version_id": self.previous_version_id, + "derived_from_id": self.derived_from_id, + "activity_started_at_time": self.activity_started_at_time, + "activity_ended_at_time": self.activity_ended_at_time, + "acted_on_behalf_of": self.acted_on_behalf_of, + "informed_by_activities": self.informed_by_activities, + "valid_from": self.valid_from, + "valid_until": self.valid_until, + "revision_type": self.revision_type, + "supersedes": self.supersedes, + "bundle_id": self.bundle_id, + "invalidated": self.invalidated, + "invalidated_at_time": self.invalidated_at_time, + "invalidated_by": self.invalidated_by, + "invalidation_reason": self.invalidation_reason, "start_index": self.start_index, "end_index": self.end_index, "credibility": self.credibility, @@ -275,3 +352,144 @@ class PropertySource: for s in data["sources"] ] return cls(**data) + + +@dataclass +class AgentRecord: + """ + Minimum-viable typed W3C PROV-O agent (prov:Agent). + + Narrower than the full PROV-O Person/SoftwareAgent/Organization taxonomy — + just enough to answer "was a human accountable here" for high-stakes + provenance review. Pass an AgentRecord to ProvenanceManager.track_entity() + (etc.) via the `agent=` kwarg to set agent_id/agent_type/is_automated + together. + + Attributes: + id: Agent identifier (prov:Agent) + agent_type: One of "person", "software_agent", "organization" + is_automated: Whether this agent acted without direct human review + name: Optional human-readable name + metadata: Additional metadata dictionary + + Example: + >>> agent = AgentRecord(id="reviewer_jane", agent_type="person", is_automated=False) + >>> prov_mgr.track_entity("entity_1", source="doc_1", agent=agent, role="approver") + """ + + id: str + agent_type: str = "software_agent" + is_automated: bool = True + name: Optional[str] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + """Convert agent record to dictionary.""" + return { + "id": self.id, + "agent_type": self.agent_type, + "is_automated": self.is_automated, + "name": self.name, + "metadata": self.metadata, + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "AgentRecord": + """Create agent record from dictionary.""" + return cls(**data) + + +@dataclass +class ActivityRecord: + """ + Minimum-viable typed W3C PROV-O activity (prov:Activity). + + Issue #825, Part B Tier 1 — promotes activity_id from a bare string to + a typed object carrying start/end timing, mirroring AgentRecord. Pass an + ActivityRecord to ProvenanceManager.track_entity() (etc.) via the + `activity=` kwarg to set activity_id/activity_started_at_time/ + activity_ended_at_time together. + + Attributes: + id: Activity identifier (prov:Activity) + activity_type: Free-form activity classification (e.g. "process", "extraction") + started_at_time: prov:startedAtTime, ISO datetime string + ended_at_time: prov:endedAtTime, ISO datetime string + metadata: Additional metadata dictionary + + Example: + >>> activity = ActivityRecord( + ... id="bureau_parsing_run_42", + ... started_at_time="2026-01-01T00:00:00", + ... ended_at_time="2026-01-01T00:00:03", + ... ) + >>> prov_mgr.track_entity("entity_1", source="doc_1", activity=activity) + """ + + id: str + activity_type: str = "process" + started_at_time: Optional[str] = None + ended_at_time: Optional[str] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + """Convert activity record to dictionary.""" + return { + "id": self.id, + "activity_type": self.activity_type, + "started_at_time": self.started_at_time, + "ended_at_time": self.ended_at_time, + "metadata": self.metadata, + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "ActivityRecord": + """Create activity record from dictionary.""" + return cls(**data) + + +@dataclass +class Invalidation: + """ + W3C PROV-O invalidation record (prov:Invalidation). + + Represents the tombstone for a retracted/corrected provenance entry: + "this was deleted/corrected, by whom, when, why" — recorded in place of + a hard delete so the fact that an entity existed remains provable. + + Attributes: + entity_id: The entity that was invalidated (prov:Entity) + invalidated_at_time: When invalidation occurred (prov:invalidatedAtTime) + invalidated_by: Agent responsible for the invalidation (prov:Agent) + reason: Optional human-readable reason + metadata: Additional metadata dictionary + + Example: + >>> Invalidation( + ... entity_id="entity_123", + ... invalidated_at_time="2026-01-01T00:00:00", + ... invalidated_by="reviewer_jane", + ... reason="Source document retracted", + ... ) + """ + + entity_id: str + invalidated_at_time: str + invalidated_by: str + reason: Optional[str] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + """Convert invalidation record to dictionary.""" + return { + "entity_id": self.entity_id, + "invalidated_at_time": self.invalidated_at_time, + "invalidated_by": self.invalidated_by, + "reason": self.reason, + "metadata": self.metadata, + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "Invalidation": + """Create invalidation record from dictionary.""" + return cls(**data) diff --git a/semantica/provenance/storage.py b/semantica/provenance/storage.py index 7da91482..768bc263 100644 --- a/semantica/provenance/storage.py +++ b/semantica/provenance/storage.py @@ -20,7 +20,7 @@ License: MIT """ from abc import ABC, abstractmethod -from typing import List, Optional, Dict, Any +from typing import List, Optional, Dict, Any, Tuple import sqlite3 import json import uuid @@ -123,6 +123,70 @@ class ProvenanceStorage(ABC): """Internal retrieve method using an active connection/transaction.""" return self.retrieve(entity_id) + def get_chain_head(self, conn: Any = None) -> Optional[Tuple[int, str]]: + """ + Return (sequence_id, checksum) of the most recently stored entry, or + None if the store is empty or this backend doesn't support chaining. + + Used by ProvenanceManager._save_entry to link each new entry to its + predecessor (issue #825, Part A item 2 — hash-chained integrity). + Default implementation reports no chaining support; override in + backends that maintain sequence_id. + """ + return None + + def trace_descendants( + self, entity_id: str, max_depth: Optional[int] = None + ) -> List[ProvenanceEntry]: + """ + Trace downstream descendants (reverse lineage) of an entity. + + Generic default implemented purely against retrieve_all(): builds a + reverse adjacency map from parent_entity_id/previous_version_id/ + derived_from_id/used_entities, then BFS from entity_id. Correct for + any backend; SQLiteStorage overrides this with indexed queries. + + Args: + entity_id: Entity identifier to find descendants of + max_depth: Optional maximum BFS depth + + Returns: + List of ProvenanceEntry objects that (transitively) reference + entity_id, in BFS order. + """ + reverse_index: Dict[str, List[ProvenanceEntry]] = {} + for entry in self.retrieve_all(): + referenced = set( + filter( + None, + [ + entry.parent_entity_id, + entry.previous_version_id, + entry.derived_from_id, + ], + ) + ) + referenced.update(entry.used_entities or []) + for ref_id in referenced: + reverse_index.setdefault(ref_id, []).append(entry) + + descendants = [] + visited = {entity_id} + queue = deque([(entity_id, 0)]) + + while queue: + current_id, depth = queue.popleft() + if max_depth is not None and depth >= max_depth: + continue + for child in reverse_index.get(current_id, []): + if child.entity_id in visited: + continue + visited.add(child.entity_id) + descendants.append(child) + queue.append((child.entity_id, depth + 1)) + + return descendants + class InMemoryStorage(ProvenanceStorage): """ @@ -150,6 +214,11 @@ class InMemoryStorage(ProvenanceStorage): """Initialize in-memory storage.""" self._entries: Dict[str, ProvenanceEntry] = {} self._local = threading.local() + # Hash-chain state (issue #825, Part A item 2). Best-effort: this is + # the dev/test backend, not the durable audit backend (SQLiteStorage). + self._seq_lock = threading.Lock() + self._seq_counter = 0 + self._chain_head: Optional[str] = None def _get_pending_stack(self) -> list: if not hasattr(self._local, "pending_stack"): @@ -159,10 +228,14 @@ class InMemoryStorage(ProvenanceStorage): def store(self, entry: ProvenanceEntry) -> None: """ Store a provenance entry in memory. - + Args: entry: ProvenanceEntry to store """ + with self._seq_lock: + if entry.sequence_id is not None: + self._seq_counter = max(self._seq_counter, entry.sequence_id) + self._chain_head = entry.checksum self._entries[entry.entity_id] = entry def retrieve(self, entity_id: str) -> Optional[ProvenanceEntry]: @@ -286,6 +359,31 @@ class InMemoryStorage(ProvenanceStorage): return pending[entity_id] return self.retrieve(entity_id) + def get_chain_head(self, conn: Any = None) -> Optional[Tuple[int, str]]: + """Return (sequence_id, checksum) of the most recent entry, including + entries staged but not yet committed within the current transaction. + + Must compare against the *committed* head too, not just the pending + stack: a transaction can stage an unchanged relabel (track_entity's + archival, or invalidate()'s pre-invalidation archive) whose + sequence_id is older than the true head from a prior, already- + committed transaction. Blindly preferring "last staged entry" + understates the head and corrupts the chain for the next append. + """ + with self._seq_lock: + best_seq = self._seq_counter + best_checksum = self._chain_head + for pending in self._get_pending_stack(): + for staged_entry in pending.values(): + if staged_entry.sequence_id is not None and ( + best_checksum is None or staged_entry.sequence_id > best_seq + ): + best_seq = staged_entry.sequence_id + best_checksum = staged_entry.checksum + if best_checksum is None: + return None + return (best_seq, best_checksum) + class SQLiteStorage(ProvenanceStorage): """ @@ -461,26 +559,71 @@ class SQLiteStorage(ProvenanceStorage): end_index INTEGER, credibility REAL, metadata TEXT, - version TEXT DEFAULT '1.0' + version TEXT DEFAULT '1.0', + agent_type TEXT DEFAULT 'software_agent', + is_automated INTEGER DEFAULT 1, + role TEXT, + sequence_id INTEGER, + previous_checksum TEXT, + previous_version_id TEXT, + derived_from_id TEXT, + invalidated INTEGER DEFAULT 0, + invalidated_at_time TEXT, + invalidated_by TEXT, + invalidation_reason TEXT, + activity_started_at_time TEXT, + activity_ended_at_time TEXT, + acted_on_behalf_of TEXT, + informed_by_activities TEXT, + valid_from TEXT, + valid_until TEXT, + revision_type TEXT, + supersedes TEXT, + bundle_id TEXT ) """) - + # Create indexes for efficient querying cursor.execute(""" - CREATE INDEX IF NOT EXISTS idx_entity_type + CREATE INDEX IF NOT EXISTS idx_entity_type ON provenance(entity_type) """) - + cursor.execute(""" - CREATE INDEX IF NOT EXISTS idx_source_document + CREATE INDEX IF NOT EXISTS idx_source_document ON provenance(source_document) """) - + cursor.execute(""" - CREATE INDEX IF NOT EXISTS idx_parent_entity + CREATE INDEX IF NOT EXISTS idx_parent_entity ON provenance(parent_entity_id) """) - + + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_previous_version_id + ON provenance(previous_version_id) + """) + + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_derived_from_id + ON provenance(derived_from_id) + """) + + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_sequence_id + ON provenance(sequence_id) + """) + + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_invalidated + ON provenance(invalidated) + """) + + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_bundle_id + ON provenance(bundle_id) + """) + conn.commit() finally: conn.close() @@ -500,7 +643,9 @@ class SQLiteStorage(ProvenanceStorage): cursor = conn.cursor() cursor.execute(""" INSERT OR REPLACE INTO provenance VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ? ) """, ( entry.entity_id, @@ -521,7 +666,27 @@ class SQLiteStorage(ProvenanceStorage): entry.end_index, entry.credibility, json.dumps(entry.metadata), - entry.version + entry.version, + entry.agent_type, + 1 if entry.is_automated else 0, + entry.role, + entry.sequence_id, + entry.previous_checksum, + entry.previous_version_id, + entry.derived_from_id, + 1 if entry.invalidated else 0, + entry.invalidated_at_time, + entry.invalidated_by, + entry.invalidation_reason, + entry.activity_started_at_time, + entry.activity_ended_at_time, + entry.acted_on_behalf_of, + json.dumps(entry.informed_by_activities), + entry.valid_from, + entry.valid_until, + entry.revision_type, + entry.supersedes, + entry.bundle_id, )) def retrieve(self, entity_id: str) -> Optional[ProvenanceEntry]: @@ -634,9 +799,123 @@ class SQLiteStorage(ProvenanceStorage): frontier = next_frontier depth += 1 - + return lineage - + + def get_chain_head(self, conn: Any = None) -> Optional[Tuple[int, str]]: + """Return (sequence_id, checksum) of the most recently stored entry. + + Reuses the caller's transaction connection when given, so the head + read and the subsequent insert happen atomically under the existing + BEGIN IMMEDIATE writer lock (issue #825, Part A item 2). + """ + # Tie-break on rowid (SQLite's implicit row-order column) in addition + # to sequence_id: during track_entity's archival, the just-relabeled + # history row briefly shares its sequence_id with the still-present + # (about-to-be-overwritten) live row it was copied from. rowid DESC + # deterministically picks the most-recently-written row of the two, + # which is the correct chain head — relying on sequence_id ordering + # alone leaves the tie-break implementation-defined. + query = ( + "SELECT sequence_id, checksum FROM provenance " + "WHERE sequence_id IS NOT NULL " + "ORDER BY sequence_id DESC, rowid DESC LIMIT 1" + ) + if conn is not None: + row = conn.cursor().execute(query).fetchone() + return (row[0], row[1]) if row else None + with self._read_connection() as read_conn: + row = read_conn.cursor().execute(query).fetchone() + return (row[0], row[1]) if row else None + + def trace_descendants( + self, entity_id: str, max_depth: Optional[int] = None + ) -> List[ProvenanceEntry]: + """ + Trace downstream descendants (reverse lineage) using indexed lookups + on parent_entity_id/previous_version_id/derived_from_id, plus an + in-memory reverse index over used_entities (a JSON column, so not + directly indexable in SQLite). + + Args: + entity_id: Entity identifier to find descendants of + max_depth: Optional maximum BFS depth + + Returns: + List of ProvenanceEntry objects that (transitively) reference + entity_id, in BFS order. + """ + descendants: List[ProvenanceEntry] = [] + visited = {entity_id} + frontier = [entity_id] + depth = 0 + + with self._read_connection() as conn: + cursor = conn.cursor() + + cursor.execute( + "SELECT entity_id, used_entities FROM provenance " + "WHERE used_entities IS NOT NULL AND used_entities != '[]'" + ) + used_reverse: Dict[str, List[str]] = {} + for child_id, used_json in cursor.fetchall(): + try: + used_ids = json.loads(used_json) if used_json else [] + except (json.JSONDecodeError, TypeError): + used_ids = [] + for uid in used_ids: + used_reverse.setdefault(uid, []).append(child_id) + + while frontier and (max_depth is None or depth < max_depth): + current_frontier = list(dict.fromkeys(frontier)) + if not current_frontier: + break + + child_ids = set() + for i in range(0, len(current_frontier), 999): + chunk = current_frontier[i : i + 999] + placeholders = ",".join("?" * len(chunk)) + cursor.execute( + f""" + SELECT entity_id FROM provenance + WHERE parent_entity_id IN ({placeholders}) + OR previous_version_id IN ({placeholders}) + OR derived_from_id IN ({placeholders}) + """, + chunk * 3, + ) + child_ids.update(row[0] for row in cursor.fetchall()) + + for ref_id in current_frontier: + child_ids.update(used_reverse.get(ref_id, [])) + + next_frontier = [cid for cid in child_ids if cid not in visited] + if not next_frontier: + break + visited.update(next_frontier) + + entries_map = {} + for i in range(0, len(next_frontier), 999): + chunk = next_frontier[i : i + 999] + placeholders = ",".join("?" * len(chunk)) + cursor.execute( + f"SELECT * FROM provenance WHERE entity_id IN ({placeholders})", + chunk, + ) + for row in cursor.fetchall(): + entry = self._row_to_entry(row) + entries_map[entry.entity_id] = entry + + for cid in next_frontier: + entry = entries_map.get(cid) + if entry: + descendants.append(entry) + + frontier = next_frontier + depth += 1 + + return descendants + def clear(self) -> int: """ Clear all provenance data. @@ -687,5 +966,25 @@ class SQLiteStorage(ProvenanceStorage): end_index=row[15], credibility=row[16], metadata=json.loads(row[17]) if row[17] else {}, - version=row[18] + version=row[18], + agent_type=row[19] or "software_agent", + is_automated=bool(row[20]) if row[20] is not None else True, + role=row[21], + sequence_id=row[22], + previous_checksum=row[23], + previous_version_id=row[24], + derived_from_id=row[25], + invalidated=bool(row[26]) if row[26] is not None else False, + invalidated_at_time=row[27], + invalidated_by=row[28], + invalidation_reason=row[29], + activity_started_at_time=row[30], + activity_ended_at_time=row[31], + acted_on_behalf_of=row[32], + informed_by_activities=json.loads(row[33]) if row[33] else [], + valid_from=row[34], + valid_until=row[35], + revision_type=row[36], + supersedes=row[37], + bundle_id=row[38], ) diff --git a/semantica/reasoning/reasoning_provenance.py b/semantica/reasoning/reasoning_provenance.py index 970d960f..40d4418a 100644 --- a/semantica/reasoning/reasoning_provenance.py +++ b/semantica/reasoning/reasoning_provenance.py @@ -13,36 +13,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class ReasoningEngineWithProvenance: """Reasoning engine with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .reasoning_engine import ReasoningEngine - + self.provenance = provenance self._engine = ReasoningEngine(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def infer(self, premises: Any, source: str = None, **kwargs): """Perform inference with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._engine.infer(premises, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"inference_{uuid.uuid4().hex[:8]}", source=source or "reasoning_engine", entity_type="inference", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "premises_count": len(premises) if hasattr(premises, '__len__') else 1, "confidence": getattr(result, 'confidence', None) diff --git a/semantica/semantic_extract/semantic_extract_provenance.py b/semantica/semantic_extract/semantic_extract_provenance.py index 49b088b3..e66dfd32 100644 --- a/semantica/semantic_extract/semantic_extract_provenance.py +++ b/semantica/semantic_extract/semantic_extract_provenance.py @@ -37,6 +37,7 @@ License: MIT """ from typing import Optional, List, Dict, Any +from datetime import datetime import uuid @@ -48,17 +49,28 @@ class ProvenanceMixin: added to any extraction class without modifying its core functionality. """ - def __init__(self, provenance: bool = False, **kwargs): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **kwargs, + ): """ Initialize provenance tracking. - + Args: provenance: Enable provenance tracking (default: False) + agent_id: Agent identifier for accountability (issue #825); defaults + to the wrapping class name + is_automated: Whether this agent acted without direct human review **kwargs: Additional arguments passed to parent class """ self.provenance = provenance self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager @@ -66,7 +78,7 @@ class ProvenanceMixin: except ImportError: # Graceful degradation if provenance module not available self.provenance = False - + def _track_extraction( self, entity_id: str, @@ -76,7 +88,7 @@ class ProvenanceMixin: ) -> None: """ Track extraction with provenance. - + Args: entity_id: Unique identifier for extracted entity source: Source document or text @@ -84,10 +96,17 @@ class ProvenanceMixin: **metadata: Additional metadata to track """ if self.provenance and self._prov_manager: + activity_started_at_time = metadata.pop("activity_started_at_time", None) + activity_ended_at_time = metadata.pop("activity_ended_at_time", None) self._prov_manager.track_entity( entity_id=entity_id, source=source, entity_type=entity_type, + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata=metadata ) @@ -105,17 +124,25 @@ class NERExtractorWithProvenance(ProvenanceMixin): >>> # Each entity is tracked with source, confidence, and metadata """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize NER extractor with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original NERExtractor """ from .ner_extractor import NERExtractor - - ProvenanceMixin.__init__(self, provenance=provenance) + + ProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._extractor = NERExtractor(**config) def extract(self, text: str, source: Optional[str] = None, **kwargs): @@ -130,8 +157,10 @@ class NERExtractorWithProvenance(ProvenanceMixin): Returns: List of extracted entities (same as original NERExtractor) """ + activity_started_at_time = datetime.utcnow().isoformat() entities = self._extractor.extract(text, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: for entity in entities: entity_id = getattr(entity, 'id', None) @@ -141,7 +170,7 @@ class NERExtractorWithProvenance(ProvenanceMixin): entity.id = entity_id except AttributeError: pass - + self._track_extraction( entity_id=entity_id, source=source or text[:100], @@ -150,7 +179,9 @@ class NERExtractorWithProvenance(ProvenanceMixin): label=entity.label, confidence=getattr(entity, 'confidence', 1.0), start=entity.start, - end=entity.end + end=entity.end, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, ) return entities @@ -167,17 +198,25 @@ class RelationExtractorWithProvenance(ProvenanceMixin): Wraps the original RelationExtractor and tracks all extracted relations. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize relation extractor with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original RelationExtractor """ from .relation_extractor import RelationExtractor - - ProvenanceMixin.__init__(self, provenance=provenance) + + ProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._extractor = RelationExtractor(**config) def extract(self, text: str, source: Optional[str] = None, **kwargs): @@ -192,8 +231,10 @@ class RelationExtractorWithProvenance(ProvenanceMixin): Returns: List of extracted relations """ + activity_started_at_time = datetime.utcnow().isoformat() relations = self._extractor.extract(text, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: for relation in relations: relation_id = getattr(relation, 'id', None) @@ -203,7 +244,7 @@ class RelationExtractorWithProvenance(ProvenanceMixin): relation.id = relation_id except AttributeError: pass - + self._track_extraction( entity_id=relation_id, source=source or text[:100], @@ -211,7 +252,9 @@ class RelationExtractorWithProvenance(ProvenanceMixin): subject=relation.subject, predicate=relation.predicate, object=relation.object, - confidence=getattr(relation, 'confidence', 1.0) + confidence=getattr(relation, 'confidence', 1.0), + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, ) return relations @@ -228,17 +271,25 @@ class EventDetectorWithProvenance(ProvenanceMixin): Wraps the original EventDetector and tracks all detected events. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize event detector with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original EventDetector """ from .event_detector import EventDetector - - ProvenanceMixin.__init__(self, provenance=provenance) + + ProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._detector = EventDetector(**config) def detect(self, text: str, source: Optional[str] = None, **kwargs): @@ -253,8 +304,10 @@ class EventDetectorWithProvenance(ProvenanceMixin): Returns: List of detected events """ + activity_started_at_time = datetime.utcnow().isoformat() events = self._detector.detect(text, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: for event in events: event_id = getattr(event, 'id', None) @@ -264,14 +317,16 @@ class EventDetectorWithProvenance(ProvenanceMixin): event.id = event_id except AttributeError: pass - + self._track_extraction( entity_id=event_id, source=source or text[:100], entity_type="event", event_type=event.type, trigger=event.trigger, - confidence=getattr(event, 'confidence', 1.0) + confidence=getattr(event, 'confidence', 1.0), + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, ) return events @@ -288,17 +343,25 @@ class CoreferenceResolverWithProvenance(ProvenanceMixin): Wraps the original CoreferenceResolver and tracks coreference chains. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize coreference resolver with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original CoreferenceResolver """ from .coreference_resolver import CoreferenceResolver - - ProvenanceMixin.__init__(self, provenance=provenance) + + ProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._resolver = CoreferenceResolver(**config) def resolve(self, text: str, source: Optional[str] = None, **kwargs): @@ -313,8 +376,10 @@ class CoreferenceResolverWithProvenance(ProvenanceMixin): Returns: Coreference chains """ + activity_started_at_time = datetime.utcnow().isoformat() chains = self._resolver.resolve(text, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: for chain in chains: chain_id = getattr(chain, 'id', None) @@ -324,12 +389,14 @@ class CoreferenceResolverWithProvenance(ProvenanceMixin): chain.id = chain_id except AttributeError: pass - + self._track_extraction( entity_id=chain_id, source=source or text[:100], entity_type="coreference_chain", - mentions=len(chain.mentions) if hasattr(chain, 'mentions') else 0 + mentions=len(chain.mentions) if hasattr(chain, 'mentions') else 0, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, ) return chains @@ -346,17 +413,25 @@ class TripletExtractorWithProvenance(ProvenanceMixin): Wraps the original TripletExtractor and tracks all extracted triplets. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize triplet extractor with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original TripletExtractor """ from .triplet_extractor import TripletExtractor - - ProvenanceMixin.__init__(self, provenance=provenance) + + ProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._extractor = TripletExtractor(**config) def extract(self, text: str, source: Optional[str] = None, **kwargs): @@ -371,8 +446,10 @@ class TripletExtractorWithProvenance(ProvenanceMixin): Returns: List of extracted triplets """ + activity_started_at_time = datetime.utcnow().isoformat() triplets = self._extractor.extract(text, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: for triplet in triplets: triplet_id = getattr(triplet, 'id', None) @@ -382,7 +459,7 @@ class TripletExtractorWithProvenance(ProvenanceMixin): triplet.id = triplet_id except AttributeError: pass - + self._track_extraction( entity_id=triplet_id, source=source or text[:100], @@ -390,7 +467,9 @@ class TripletExtractorWithProvenance(ProvenanceMixin): subject=triplet.subject, predicate=triplet.predicate, object=triplet.object, - confidence=getattr(triplet, 'confidence', 1.0) + confidence=getattr(triplet, 'confidence', 1.0), + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, ) return triplets diff --git a/semantica/triplet_store/triplet_store_provenance.py b/semantica/triplet_store/triplet_store_provenance.py index 2ac58cff..83595965 100644 --- a/semantica/triplet_store/triplet_store_provenance.py +++ b/semantica/triplet_store/triplet_store_provenance.py @@ -13,36 +13,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class TripletStoreWithProvenance: """Triplet store with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .triplet_store import TripletStore - + self.provenance = provenance self._store = TripletStore(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def add_triplet(self, subject: Any, predicate: Any, obj: Any, source: str = None, **kwargs): """Add triplet with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._store.add_triplet(subject, predicate, obj, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"triplet_{uuid.uuid4().hex[:8]}", source=source or "triplet_store", entity_type="triplet", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "subject": str(subject), "predicate": str(predicate), diff --git a/semantica/vector_store/vector_store_provenance.py b/semantica/vector_store/vector_store_provenance.py index a540074f..fc7a5e91 100644 --- a/semantica/vector_store/vector_store_provenance.py +++ b/semantica/vector_store/vector_store_provenance.py @@ -13,36 +13,52 @@ Author: Semantica Contributors License: MIT """ -from typing import List, Any +from typing import List, Any, Optional +from datetime import datetime import uuid class VectorStoreWithProvenance: """Vector store with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .vector_store import VectorStore - + self.provenance = provenance self._store = VectorStore(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def add_vectors(self, vectors: List[Any], source: str = None, **kwargs): """Add vectors with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._store.add_vectors(vectors, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"vectors_{uuid.uuid4().hex[:8]}", source=source or "vector_store", entity_type="vector_collection", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "count": len(vectors), "dimensions": len(vectors[0]) if vectors else 0 diff --git a/semantica/visualization/visualization_provenance.py b/semantica/visualization/visualization_provenance.py index 82e9d7b2..780cef4e 100644 --- a/semantica/visualization/visualization_provenance.py +++ b/semantica/visualization/visualization_provenance.py @@ -11,36 +11,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class VisualizerWithProvenance: """Visualizer with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .visualizer import Visualizer - + self.provenance = provenance self._visualizer = Visualizer(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def visualize(self, data: Any, output: str = None, **kwargs): """Visualize data with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._visualizer.visualize(data, output=output, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"viz_{uuid.uuid4().hex[:8]}", source="visualization", entity_type="visualization", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={"output": output, "type": kwargs.get('type', 'unknown')} ) diff --git a/tests/provenance/test_manager.py b/tests/provenance/test_manager.py index 77f61b79..e8c7f44a 100644 --- a/tests/provenance/test_manager.py +++ b/tests/provenance/test_manager.py @@ -913,5 +913,614 @@ class TestProvenanceManager: assert kwargs.get("exc_info") is True +class TestAgentTyping: + """Issue #825, Part A item 3 — agent_id/agent_type/is_automated actually + populate on tracked entries (previously a dead field: no track_* method + read agent_id from kwargs at all).""" + + def test_track_entity_scalar_agent_kwargs(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_entity( + "e1", source="doc1", + agent_id="alice", agent_type="person", is_automated=False, role="approver", + ) + assert entry.agent_id == "alice" + assert entry.agent_type == "person" + assert entry.is_automated is False + assert entry.role == "approver" + + def test_track_entity_agent_record_kwarg(self): + from semantica.provenance import AgentRecord + prov_mgr = ProvenanceManager() + agent = AgentRecord(id="reviewer_bob", agent_type="person", is_automated=False) + entry = prov_mgr.track_entity("e1", source="doc1", agent=agent, role="approver") + assert entry.agent_id == "reviewer_bob" + assert entry.agent_type == "person" + assert entry.is_automated is False + assert entry.role == "approver" + + def test_track_entity_default_agent_unchanged(self): + """No agent kwargs supplied should still default to 'semantica' (back-compat).""" + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_entity("e1", source="doc1") + assert entry.agent_id == "semantica" + assert entry.agent_type == "software_agent" + assert entry.is_automated is True + + def test_track_relationship_agent_kwargs(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_relationship("r1", source="doc1", agent_id="bot1") + assert entry.agent_id == "bot1" + + def test_track_chunk_agent_kwargs_not_leaked_into_metadata(self): + """agent_id/agent_type/is_automated passed to track_chunk must populate + the real fields, not leak into the opaque metadata blob.""" + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_chunk( + "c1", source_document="doc1", agent_id="chunker_v2", note="hello" + ) + assert entry.agent_id == "chunker_v2" + assert "agent_id" not in entry.metadata + assert entry.metadata.get("note") == "hello" + + def test_track_property_source_agent_kwargs(self): + prov_mgr = ProvenanceManager() + source = SourceReference(document="doc1") + entry = prov_mgr.track_property_source( + "e1", "prop", "value", source, agent_id="prop_tracker" + ) + assert entry.agent_id == "prop_tracker" + assert "agent_id" not in entry.metadata + + def test_track_entities_batch_agent_id_not_swallowed_into_metadata(self): + """Regression test for the documented bug: track_entities_batch used to + merge agent_id/entity_type/activity_id into the metadata dict instead + of forwarding them as real track_entity kwargs.""" + prov_mgr = ProvenanceManager() + count = prov_mgr.track_entities_batch( + [{"id": "b1"}, {"id": "b2"}], + source="doc_batch", + agent_id="batch_service_v2", + entity_type="credit_feature", + activity_id="bureau_parsing", + extra_note="kept as free-form metadata", + ) + assert count == 2 + for eid in ("b1", "b2"): + entry = prov_mgr.storage.retrieve(eid) + assert entry.agent_id == "batch_service_v2" + assert entry.entity_type == "credit_feature" + assert entry.activity_id == "bureau_parsing" + assert "agent_id" not in entry.metadata + assert entry.metadata.get("extra_note") == "kept as free-form metadata" + + +class TestVersioningVsDerivation: + """Issue #825, Part A item 4 — previous_version_id (correction) is + additive alongside derived_from_id (cross-source derivation); both are + populated without disturbing the legacy parent_entity_id field.""" + + def test_retrack_without_parent_sets_previous_version_id_only(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("X", source="doc_1") + v2 = prov_mgr.track_entity("X", source="doc_1") + + assert v2.previous_version_id is not None + assert v2.previous_version_id.startswith("X:v:") + assert v2.derived_from_id is None + # Legacy field behavior is unchanged + assert v2.parent_entity_id == v2.previous_version_id + + def test_retrack_with_explicit_parent_sets_both_fields(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("X", source="doc_1") + v2 = prov_mgr.track_entity("X", source="doc_1", parent_entity_id="explicit_parent") + + assert v2.derived_from_id == "explicit_parent" + assert v2.previous_version_id is not None + assert v2.previous_version_id.startswith("X:v:") + # Legacy field keeps explicit-wins semantics + assert v2.parent_entity_id == "explicit_parent" + + def test_track_chunk_split_sets_derived_from_id(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_chunk( + "c2", source_document="doc1", parent_chunk_id="c1" + ) + assert entry.derived_from_id == "c1" + assert entry.previous_version_id is None + assert entry.parent_entity_id == "c1" + + +class TestInvalidation: + """Issue #825, Part A item 1 — tombstone instead of hard delete.""" + + def test_invalidate_marks_entry_without_deleting(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + + result = prov_mgr.invalidate("e1", agent_id="reviewer_jane", reason="retracted") + + assert result.invalidated is True + assert result.invalidated_by == "reviewer_jane" + assert result.invalidation_reason == "retracted" + assert result.invalidated_at_time is not None + + # Entry remains visible via retrieve (tombstone, not delete) + stored = prov_mgr.storage.retrieve("e1") + assert stored is not None + assert stored.invalidated is True + + def test_invalidate_unknown_entity_raises(self): + prov_mgr = ProvenanceManager() + with pytest.raises(ValueError): + prov_mgr.invalidate("never_tracked", agent_id="reviewer_jane") + + def test_check_reports_invalidated_count(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("e2", source="doc1") + prov_mgr.invalidate("e1", agent_id="reviewer_jane") + + result = prov_mgr.check() + assert result["invalidated_count"] == 1 + + +class TestHashChain: + """Issue #825, Part A item 2 — hash-chained integrity: chained checksums + detect wholesale row deletion, which per-row checksums alone cannot.""" + + def test_verify_chain_valid_on_clean_history(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("e2", source="doc1") + prov_mgr.track_entity("X", source="doc1") + prov_mgr.track_entity("X", source="doc1") # triggers archival + prov_mgr.track_entity("X", source="doc1", parent_entity_id="explicit_p") + + result = prov_mgr.verify_chain() + assert result["valid"] is True + assert result["broken_links"] == [] + assert result["total_entries"] == 5 + + def test_verify_chain_detects_deleted_row(self, tmp_path): + """A hard delete of a row must break the chain for whatever followed it.""" + import sqlite3 + db_path = str(tmp_path / "chain.db") + prov_mgr = ProvenanceManager(storage_path=db_path) + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("e2", source="doc1") + prov_mgr.track_entity("e3", source="doc1") + + conn = sqlite3.connect(db_path) + conn.execute("DELETE FROM provenance WHERE entity_id = 'e2'") + conn.commit() + conn.close() + + prov_mgr2 = ProvenanceManager(storage_path=db_path) + result = prov_mgr2.verify_chain() + assert result["valid"] is False + assert any(link["reason"] == "chain_break" for link in result["broken_links"]) + + def test_sequence_ids_are_assigned_and_monotonic(self): + prov_mgr = ProvenanceManager() + e1 = prov_mgr.track_entity("e1", source="doc1") + e2 = prov_mgr.track_entity("e2", source="doc1") + assert e1.sequence_id is not None + assert e2.sequence_id is not None + assert e2.sequence_id > e1.sequence_id + assert e2.previous_checksum == e1.checksum + + def test_chain_survives_interleaved_retrack_and_invalidate(self, tmp_path): + """Regression test: an entry that already chained its previous_checksum + from another entity's checksum (Y -> X) must stay valid even after X + is later retracked (archived/relabeled) and then invalidated — both + of which write NEW entries under X's canonical key. Mutating X's + existing row in place (rather than archiving-then-appending) used to + silently orphan Y's chain link, producing a false-positive break.""" + db_path = str(tmp_path / "interleaved.db") + prov_mgr = ProvenanceManager(storage_path=db_path) + + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("X", source="doc1") + prov_mgr.track_entity("Y", source="doc1", parent_entity_id="X") + prov_mgr.track_entity("X", source="doc1") # retrack: archives old X + prov_mgr.track_entity("Z", source="doc1", parent_entity_id="X") + prov_mgr.invalidate("X", agent_id="reviewer") # archives again + prov_mgr.track_entity("W", source="doc1", parent_entity_id="X") + + result = prov_mgr.verify_chain() + assert result["valid"] is True + assert result["broken_links"] == [] + + def test_invalidate_does_not_break_chain(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("e2", source="doc1", parent_entity_id="e1") + prov_mgr.invalidate("e1", agent_id="reviewer_jane") + + result = prov_mgr.verify_chain() + assert result["valid"] is True + + +class TestDownstreamLineage: + """Issue #825, Part A item 5 — downstream/descendant traversal (reverse + BFS), complementing the existing upstream-only trace_lineage/get_lineage.""" + + def test_get_descendants_finds_direct_child(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("parent1", source="doc1") + prov_mgr.track_entity( + "child1", source="doc1", parent_entity_id="parent1", used_entities=["parent1"] + ) + + result = prov_mgr.get_descendants("parent1") + entity_ids = {e["entity_id"] for e in result["entries"]} + assert "child1" in entity_ids + + def test_get_descendants_transitive(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("a", source="doc1") + prov_mgr.track_entity("b", source="doc1", parent_entity_id="a") + prov_mgr.track_entity("c", source="doc1", parent_entity_id="b") + + result = prov_mgr.get_descendants("a") + entity_ids = {e["entity_id"] for e in result["entries"]} + assert entity_ids == {"b", "c"} + + def test_get_descendants_empty_for_leaf(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("leaf", source="doc1") + assert prov_mgr.get_descendants("leaf") == {} + + def test_descendants_work_on_sqlite_backend(self, tmp_path): + db_path = str(tmp_path / "desc.db") + prov_mgr = ProvenanceManager(storage_path=db_path) + prov_mgr.track_entity("parent1", source="doc1") + prov_mgr.track_entity("child1", source="doc1", parent_entity_id="parent1") + + result = prov_mgr.get_descendants("parent1") + entity_ids = {e["entity_id"] for e in result["entries"]} + assert "child1" in entity_ids + + +class TestQualifiedExport: + """Issue #825, Part A item 6 — qualified Association with hadRole, plus + qualified Invalidation, in the RDF export.""" + + def test_export_prov_includes_qualified_association_and_role(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity( + "e1", source="doc1", agent_id="alice", agent_type="person", role="approver" + ) + ttl = prov_mgr.export_prov(format="turtle") + assert "qualifiedAssociation" in ttl + assert "hadRole" in ttl + assert "role_approver" in ttl + assert "Person" in ttl + + def test_export_prov_includes_qualified_invalidation(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.invalidate("e1", agent_id="reviewer_jane", reason="retracted") + + ttl = prov_mgr.export_prov(format="turtle") + assert "qualifiedInvalidation" in ttl + assert "Invalidation" in ttl + + +class TestTypedActivity: + """Issue #825, Part B Tier 1 — typed Activity via ActivityRecord.""" + + def test_track_entity_activity_record_kwarg(self): + from semantica.provenance import ActivityRecord + prov_mgr = ProvenanceManager() + activity = ActivityRecord( + id="bureau_parsing_run_42", + started_at_time="2026-01-01T00:00:00", + ended_at_time="2026-01-01T00:00:03", + ) + entry = prov_mgr.track_entity("e1", source="doc1", activity=activity) + assert entry.activity_id == "bureau_parsing_run_42" + assert entry.activity_started_at_time == "2026-01-01T00:00:00" + assert entry.activity_ended_at_time == "2026-01-01T00:00:03" + + def test_track_entity_activity_scalar_kwargs(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_entity( + "e1", source="doc1", + activity_id="parse_step", activity_started_at_time="t0", activity_ended_at_time="t1", + ) + assert entry.activity_id == "parse_step" + assert entry.activity_started_at_time == "t0" + assert entry.activity_ended_at_time == "t1" + + def test_export_prov_qualified_generation_usage_derivation(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("parent1", source="doc1") + prov_mgr.track_entity( + "child1", source="doc1", + parent_entity_id="parent1", used_entities=["parent1"], + activity_id="transform", + ) + ttl = prov_mgr.export_prov(format="turtle") + assert "qualifiedGeneration" in ttl + assert "Generation" in ttl + assert "qualifiedUsage" in ttl + assert "Usage" in ttl + assert "qualifiedDerivation" in ttl + assert "Derivation" in ttl + + def test_export_prov_activity_timing(self): + from semantica.provenance import ActivityRecord + prov_mgr = ProvenanceManager() + prov_mgr.track_entity( + "e1", source="doc1", + activity=ActivityRecord(id="act1", started_at_time="2026-01-01T00:00:00", + ended_at_time="2026-01-01T00:00:05"), + ) + ttl = prov_mgr.export_prov(format="turtle") + assert "startedAtTime" in ttl + assert "endedAtTime" in ttl + + +class TestAssociationDelegationChaining: + """Issue #825, Part B Tier 2 — wasAssociatedWith, actedOnBehalfOf, wasInformedBy.""" + + def test_acted_on_behalf_of(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_entity( + "e1", source="doc1", agent_id="bot1", acted_on_behalf_of="org1" + ) + assert entry.acted_on_behalf_of == "org1" + ttl = prov_mgr.export_prov(format="turtle") + assert "actedOnBehalfOf" in ttl + + def test_informed_by_activities(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_entity( + "e1", source="doc1", activity_id="parse", informed_by=["ingest_activity"] + ) + assert entry.informed_by_activities == ["ingest_activity"] + ttl = prov_mgr.export_prov(format="turtle") + assert "wasInformedBy" in ttl + + def test_was_associated_with_in_export(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1", agent_id="alice", activity_id="act1") + ttl = prov_mgr.export_prov(format="turtle") + assert "wasAssociatedWith" in ttl + + def test_track_entities_batch_forwards_tier2_kwargs(self): + """Regression test for the batch-kwargs-vs-metadata bug (issue #825, + Part A) extended to the new Tier 2/3 keys.""" + prov_mgr = ProvenanceManager() + count = prov_mgr.track_entities_batch( + [{"id": "b1"}], + source="doc_batch", + activity_id="bureau_parsing", + acted_on_behalf_of="org1", + informed_by=["ingest_activity"], + bundle_id="run_1", + ) + assert count == 1 + entry = prov_mgr.storage.retrieve("b1") + assert entry.activity_id == "bureau_parsing" + assert entry.acted_on_behalf_of == "org1" + assert entry.informed_by_activities == ["ingest_activity"] + assert entry.bundle_id == "run_1" + assert "acted_on_behalf_of" not in entry.metadata + + +class TestBitemporalMerge: + """Issue #825, Part B Tier 3 — revision_history()/query_recorded_between() + close kg.ProvenanceTracker's documented 'no direct equivalent yet' gaps.""" + + def test_revision_history_ascending_with_valid_until(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("X", source="doc1", agent_id="alice") + prov_mgr.track_entity("X", source="doc1", agent_id="bob", revision_type="correction") + prov_mgr.track_entity("X", source="doc1", agent_id="carol", supersedes="X_old_claim") + + history = prov_mgr.revision_history("X") + assert len(history) == 3 + assert [h["version"] for h in history] == [1, 2, 3] + assert history[0]["author"] == "alice" + assert history[1]["author"] == "bob" + assert history[1]["revision_type"] == "correction" + assert history[2]["author"] == "carol" + assert history[2]["supersedes"] == "X_old_claim" + # Every version except the last has a valid_until set to the next version's timestamp + assert history[0]["valid_until"] == history[1]["valid_from"] + assert history[1]["valid_until"] == history[2]["valid_from"] + assert history[2]["valid_until"] is None + + def test_revision_history_empty_for_untracked_entity(self): + prov_mgr = ProvenanceManager() + assert prov_mgr.revision_history("never_tracked") == [] + + def test_revision_history_single_version(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("Y", source="doc1") + history = prov_mgr.revision_history("Y") + assert len(history) == 1 + assert history[0]["version"] == 1 + assert history[0]["valid_until"] is None + + def test_query_recorded_between(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("e2", source="doc1") + + results = prov_mgr.query_recorded_between("2000-01-01T00:00:00", "2100-01-01T00:00:00") + entity_ids = {r["entity_id"] for r in results} + assert {"e1", "e2"}.issubset(entity_ids) + + no_results = prov_mgr.query_recorded_between("1990-01-01T00:00:00", "1990-01-02T00:00:00") + assert no_results == [] + + def test_check_flags_missing_informed_by_activity(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1", informed_by=["never_tracked_activity"]) + result = prov_mgr.check() + assert result["valid"] is False + assert any("never_tracked_activity" in ref for ref in result["missing_references"]) + + +class TestBundleAndBaseUri: + """Issue #825, Part B Tier 3 — prov:Bundle membership and configurable base_uri.""" + + def test_bundle_id_produces_bundle_triples(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1", bundle_id="ingestion_run_1") + ttl = prov_mgr.export_prov(format="turtle") + assert "Bundle" in ttl + assert "hadMember" in ttl + + def test_default_base_uri_matches_rdf_exporter_namespace(self): + from semantica.provenance.manager import DEFAULT_BASE_URI + from semantica.export.rdf_exporter import NamespaceManager + assert NamespaceManager().namespaces["semantica"] == DEFAULT_BASE_URI + + def test_export_prov_base_uri_override(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + + default_ttl = prov_mgr.export_prov(format="turtle") + assert "https://semantica.dev/ns#" in default_ttl + + custom_ttl = prov_mgr.export_prov(format="turtle", base_uri="https://example.org/kg#") + assert "https://example.org/kg#" in custom_ttl + assert "https://semantica.dev/ns#" not in custom_ttl + + def test_owl_exporter_default_ontology_uri_matches_shared_namespace(self): + """Issue #825 follow-up — OWLExporter was the one exporter left out of + the Part B Tier 3 namespace interlinking; its default ontology_uri + must match the same shared DEFAULT_BASE_URI as RDFExporter and + export_prov().""" + from semantica.provenance.manager import DEFAULT_BASE_URI + from semantica.export.owl_exporter import OWLExporter + assert OWLExporter().ontology_uri == DEFAULT_BASE_URI + + +class TestExplicitBitemporalFields: + """Issue #825 follow-up — valid_from/valid_until as explicit, + caller-supplied ProvenanceEntry fields (matching the deprecated + kg.ProvenanceTracker's actual contract: these were always caller-supplied + metadata keys, never auto-computed).""" + + def test_valid_from_valid_until_are_plain_passthrough_fields(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_entity( + "price1", source="doc1", valid_from="2026-01-01", valid_until="2026-06-01" + ) + assert entry.valid_from == "2026-01-01" + assert entry.valid_until == "2026-06-01" + + def test_revision_history_prefers_explicit_valid_from_until(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity( + "price1", source="doc1", valid_from="2026-01-01", valid_until="2026-06-01" + ) + history = prov_mgr.revision_history("price1") + assert history[0]["valid_from"] == "2026-01-01" + assert history[0]["valid_until"] == "2026-06-01" + + def test_revision_history_falls_back_to_timestamp_when_unset(self): + """Backward-compat: entries that don't set valid_from/valid_until + explicitly still get the dynamic timestamp-based derivation.""" + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("X", source="doc1") + prov_mgr.track_entity("X", source="doc1") + history = prov_mgr.revision_history("X") + assert history[0]["valid_until"] == history[1]["valid_from"] + assert history[1]["valid_until"] is None + + def test_valid_from_until_survive_sqlite_round_trip(self, tmp_path): + db_path = str(tmp_path / "bitemporal.db") + prov_mgr = ProvenanceManager(storage_path=db_path) + prov_mgr.track_entity( + "price1", source="doc1", valid_from="2026-01-01", valid_until="2026-06-01" + ) + prov_mgr2 = ProvenanceManager(storage_path=db_path) + entry = prov_mgr2.storage.retrieve("price1") + assert entry.valid_from == "2026-01-01" + assert entry.valid_until == "2026-06-01" + + +class TestActivityTimingAcrossWrappers: + """Issue #825 follow-up — activity_started_at_time/ended_at_time wired + into all *_provenance.py wrapper modules that measure real work, not + just the 2 wrappers from the original Part B pass.""" + + def test_embedding_wrapper_records_activity_timing(self): + from semantica.embeddings.embeddings_provenance import EmbeddingGeneratorWithProvenance + wrapper = EmbeddingGeneratorWithProvenance(provenance=True, agent_id="embed_svc") + wrapper._generator.embed = lambda texts, **kw: [[0.1, 0.2] for _ in texts] + wrapper.embed(["hello", "world"]) + entries = wrapper._prov_manager.storage.retrieve_all() + assert len(entries) == 1 + assert entries[0].activity_started_at_time is not None + assert entries[0].activity_ended_at_time is not None + + def test_kg_algorithm_tracker_accepts_caller_supplied_activity_timing(self): + from semantica.kg.kg_provenance import AlgorithmTrackerWithProvenance + tracker = AlgorithmTrackerWithProvenance(provenance=True, agent_id="algo_svc") + eid = tracker.track_embedding_computation( + graph=object(), algorithm="node2vec", embeddings={"n1": [0.1, 0.2]}, + parameters={"d": 2}, + activity_started_at_time="t0", activity_ended_at_time="t1", + ) + entry = tracker._prov_manager.get_provenance(eid) + assert entry["activity_started_at_time"] == "t0" + assert entry["activity_ended_at_time"] == "t1" + + def test_graph_builder_build_operation_has_no_end_time_yet(self): + """The build-operation marker is recorded before the build runs, so + it legitimately has a start but no end time.""" + from semantica.kg.kg_provenance import GraphBuilderWithProvenance + builder = GraphBuilderWithProvenance(provenance=True, agent_id="builder_svc") + builder._builder.build_single_source = lambda kg_data, **kw: {"entities": [], "relationships": []} + builder.build_single_source({"foo": "bar"}) + entries = builder._prov_manager.storage.retrieve_all() + # NOTE: entity_type isn't asserted here — kg_provenance.py has a + # pre-existing, out-of-scope bug where entity_type is nested inside + # the metadata dict instead of passed as a track_entity kwarg, so it + # never actually populates the real field for this call site. + build_entries = [e for e in entries if e.entity_id.startswith("graph_build_single_")] + assert len(build_entries) == 1 + assert build_entries[0].activity_started_at_time is not None + assert build_entries[0].activity_ended_at_time is None + + def test_semantic_extract_wrapper_records_activity_timing(self): + from semantica.semantic_extract.semantic_extract_provenance import ProvenanceMixin + + class FakeExtractor(ProvenanceMixin): + pass + + wrapper = FakeExtractor(provenance=True, agent_id="extract_svc") + wrapper._track_extraction( + entity_id="e1", source="doc1", entity_type="named_entity", + activity_started_at_time="t0", activity_ended_at_time="t1", + ) + entry = wrapper._prov_manager.get_provenance("e1") + assert entry["activity_started_at_time"] == "t0" + assert entry["activity_ended_at_time"] == "t1" + assert "activity_started_at_time" not in entry["metadata"] + + def test_conflicts_wrapper_records_activity_timing(self): + from semantica.conflicts.conflicts_provenance import SourceTrackerWithUnifiedBackend + + class FakeSource: + document = "doc1" + page = 1 + section = None + confidence = 0.9 + + tracker = SourceTrackerWithUnifiedBackend(agent_id="conflicts_svc") + tracker.track_property_source("e1", "prop", "val", FakeSource()) + entry = tracker._unified_manager.get_provenance("e1_prop") + assert entry["activity_started_at_time"] is not None + assert entry["activity_ended_at_time"] is not None diff --git a/tests/provenance/test_schemas.py b/tests/provenance/test_schemas.py index 8afed0e8..e151d9f1 100644 --- a/tests/provenance/test_schemas.py +++ b/tests/provenance/test_schemas.py @@ -7,7 +7,14 @@ and PropertySource dataclasses. import pytest from datetime import datetime -from semantica.provenance.schemas import ProvenanceEntry, SourceReference, PropertySource +from semantica.provenance.schemas import ( + ProvenanceEntry, + SourceReference, + PropertySource, + AgentRecord, + ActivityRecord, + Invalidation, +) class TestProvenanceEntry: @@ -158,7 +165,120 @@ class TestPropertySource: ) data = prop_source.to_dict() - + assert isinstance(data, dict) assert data["property_name"] == "name" assert len(data["sources"]) == 1 + + +class TestProvenanceEntryPart825Fields: + """Issue #825, Part A — new additive fields on ProvenanceEntry.""" + + def test_defaults(self): + entry = ProvenanceEntry(entity_id="e1", entity_type="entity", activity_id="act") + assert entry.agent_type == "software_agent" + assert entry.is_automated is True + assert entry.role is None + assert entry.previous_version_id is None + assert entry.derived_from_id is None + assert entry.sequence_id is None + assert entry.previous_checksum is None + assert entry.invalidated is False + assert entry.invalidated_at_time is None + assert entry.invalidated_by is None + assert entry.invalidation_reason is None + # Part B additive fields + assert entry.activity_started_at_time is None + assert entry.activity_ended_at_time is None + assert entry.acted_on_behalf_of is None + assert entry.informed_by_activities == [] + assert entry.valid_from is None + assert entry.valid_until is None + assert entry.revision_type is None + assert entry.supersedes is None + assert entry.bundle_id is None + + def test_round_trip_to_dict_from_dict(self): + entry = ProvenanceEntry( + entity_id="e1", + entity_type="entity", + activity_id="act", + agent_id="alice", + agent_type="person", + is_automated=False, + role="approver", + previous_version_id="e1:v:1", + derived_from_id="source_entity", + sequence_id=5, + previous_checksum="abc123", + invalidated=True, + invalidated_at_time="2026-01-01T00:00:00", + invalidated_by="reviewer_jane", + invalidation_reason="retracted", + activity_started_at_time="2026-01-01T00:00:00", + activity_ended_at_time="2026-01-01T00:00:05", + acted_on_behalf_of="org1", + informed_by_activities=["ingest_activity"], + valid_from="2026-01-01", + valid_until="2026-06-01", + revision_type="correction", + supersedes="old_claim", + bundle_id="ingestion_run_1", + ) + + data = entry.to_dict() + restored = ProvenanceEntry.from_dict(data) + + assert restored == entry + + +class TestActivityRecord: + """Issue #825, Part B Tier 1 — minimum-viable typed activity.""" + + def test_defaults(self): + activity = ActivityRecord(id="act1") + assert activity.activity_type == "process" + assert activity.started_at_time is None + assert activity.ended_at_time is None + + def test_round_trip(self): + activity = ActivityRecord( + id="bureau_parsing_run_42", + activity_type="extraction", + started_at_time="2026-01-01T00:00:00", + ended_at_time="2026-01-01T00:00:03", + ) + data = activity.to_dict() + restored = ActivityRecord.from_dict(data) + assert restored == activity + + +class TestAgentRecord: + """Issue #825, Part A item 3 — minimum-viable typed agent.""" + + def test_defaults(self): + agent = AgentRecord(id="bot1") + assert agent.agent_type == "software_agent" + assert agent.is_automated is True + assert agent.name is None + + def test_round_trip(self): + agent = AgentRecord(id="reviewer_jane", agent_type="person", is_automated=False, name="Jane") + data = agent.to_dict() + restored = AgentRecord.from_dict(data) + assert restored == agent + + +class TestInvalidation: + """Issue #825, Part A item 1 — tombstone record.""" + + def test_round_trip(self): + inv = Invalidation( + entity_id="e1", + invalidated_at_time="2026-01-01T00:00:00", + invalidated_by="reviewer_jane", + reason="retracted", + ) + data = inv.to_dict() + restored = Invalidation.from_dict(data) + assert restored == inv diff --git a/tests/provenance/test_storage.py b/tests/provenance/test_storage.py index b21c6bc0..5c35764e 100644 --- a/tests/provenance/test_storage.py +++ b/tests/provenance/test_storage.py @@ -130,14 +130,120 @@ class TestInMemoryStorage: storage.store(entry) count = storage.clear() - + assert count == 1 assert len(storage.retrieve_all()) == 0 + def test_get_chain_head(self): + """Issue #825, Part A item 2 — chain head reporting.""" + storage = InMemoryStorage() + assert storage.get_chain_head() is None + + entry1 = ProvenanceEntry( + entity_id="e1", entity_type="entity", activity_id="act", + sequence_id=1, checksum="checksum_1", + ) + storage.store(entry1) + assert storage.get_chain_head() == (1, "checksum_1") + + entry2 = ProvenanceEntry( + entity_id="e2", entity_type="entity", activity_id="act", + sequence_id=2, checksum="checksum_2", + ) + storage.store(entry2) + assert storage.get_chain_head() == (2, "checksum_2") + + def test_trace_descendants(self): + """Issue #825, Part A item 5 — reverse (downstream) lineage traversal.""" + storage = InMemoryStorage() + storage.store(ProvenanceEntry(entity_id="a", entity_type="entity", activity_id="act")) + storage.store(ProvenanceEntry( + entity_id="b", entity_type="entity", activity_id="act", parent_entity_id="a" + )) + storage.store(ProvenanceEntry( + entity_id="c", entity_type="entity", activity_id="act", parent_entity_id="b" + )) + storage.store(ProvenanceEntry( + entity_id="d", entity_type="entity", activity_id="act", used_entities=["a"] + )) + + descendants = storage.trace_descendants("a") + entity_ids = {e.entity_id for e in descendants} + assert entity_ids == {"b", "c", "d"} + + def test_trace_descendants_respects_max_depth(self): + storage = InMemoryStorage() + storage.store(ProvenanceEntry(entity_id="a", entity_type="entity", activity_id="act")) + storage.store(ProvenanceEntry( + entity_id="b", entity_type="entity", activity_id="act", parent_entity_id="a" + )) + storage.store(ProvenanceEntry( + entity_id="c", entity_type="entity", activity_id="act", parent_entity_id="b" + )) + + descendants = storage.trace_descendants("a", max_depth=1) + entity_ids = {e.entity_id for e in descendants} + assert entity_ids == {"b"} + + def test_trace_descendants_empty_for_leaf(self): + storage = InMemoryStorage() + storage.store(ProvenanceEntry(entity_id="leaf", entity_type="entity", activity_id="act")) + assert storage.trace_descendants("leaf") == [] + class TestSQLiteStorage: """Test SQLiteStorage backend.""" - + + def test_all_fields_round_trip_through_sqlite(self): + """Regression test: every ProvenanceEntry field, including issue #825 + Part A and Part B additions, must survive a SQLite store/retrieve + round trip byte-for-byte. Part B's activity/actedOnBehalfOf/ + informedBy/revision/bundle fields were initially added to the + dataclass and to export_prov() without updating SQLiteStorage's DDL/ + INSERT/_row_to_entry — InMemoryStorage stores the dataclass directly + so it masked the gap, but SQLite silently dropped every one of those + fields on write.""" + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp: + db_path = tmp.name + + try: + storage = SQLiteStorage(db_path) + entry = ProvenanceEntry( + entity_id="entity_1", + entity_type="entity", + activity_id="extraction", + agent_id="alice", + agent_type="person", + is_automated=False, + role="approver", + sequence_id=1, + previous_checksum="prevchk", + parent_entity_id="parent_1", + used_entities=["u1", "u2"], + previous_version_id="entity_1:v:1", + derived_from_id="source_1", + invalidated=True, + invalidated_at_time="2026-01-01T00:00:00", + invalidated_by="reviewer_jane", + invalidation_reason="retracted", + activity_started_at_time="2026-01-01T00:00:00", + activity_ended_at_time="2026-01-01T00:00:05", + acted_on_behalf_of="org1", + informed_by_activities=["act_a", "act_b"], + valid_from="2026-01-01", + valid_until="2026-06-01", + revision_type="correction", + supersedes="old_claim", + bundle_id="ingestion_run_1", + ) + storage.store(entry) + retrieved = storage.retrieve("entity_1") + + assert retrieved == entry + finally: + if os.path.exists(db_path): + os.unlink(db_path) + def test_store_and_retrieve(self): """Test storing and retrieving entries.""" with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp: @@ -211,8 +317,71 @@ class TestSQLiteStorage: storage.store(entry2) lineage = storage.trace_lineage("entity_2") - + assert len(lineage) == 2 finally: if os.path.exists(db_path): os.unlink(db_path) + + def test_get_chain_head(self): + """Issue #825, Part A item 2 — chain head reporting.""" + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp: + db_path = tmp.name + + try: + storage = SQLiteStorage(db_path) + assert storage.get_chain_head() is None + + entry1 = ProvenanceEntry( + entity_id="e1", entity_type="entity", activity_id="act", + sequence_id=1, checksum="checksum_1", + ) + storage.store(entry1) + assert storage.get_chain_head() == (1, "checksum_1") + + entry2 = ProvenanceEntry( + entity_id="e2", entity_type="entity", activity_id="act", + sequence_id=2, checksum="checksum_2", + ) + storage.store(entry2) + assert storage.get_chain_head() == (2, "checksum_2") + finally: + if os.path.exists(db_path): + os.unlink(db_path) + + def test_trace_descendants(self): + """Issue #825, Part A item 5 — reverse (downstream) lineage traversal.""" + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp: + db_path = tmp.name + + try: + storage = SQLiteStorage(db_path) + storage.store(ProvenanceEntry(entity_id="a", entity_type="entity", activity_id="act")) + storage.store(ProvenanceEntry( + entity_id="b", entity_type="entity", activity_id="act", parent_entity_id="a" + )) + storage.store(ProvenanceEntry( + entity_id="c", entity_type="entity", activity_id="act", parent_entity_id="b" + )) + storage.store(ProvenanceEntry( + entity_id="d", entity_type="entity", activity_id="act", used_entities=["a"] + )) + + descendants = storage.trace_descendants("a") + entity_ids = {e.entity_id for e in descendants} + assert entity_ids == {"b", "c", "d"} + finally: + if os.path.exists(db_path): + os.unlink(db_path) + + def test_trace_descendants_empty_for_leaf(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp: + db_path = tmp.name + + try: + storage = SQLiteStorage(db_path) + storage.store(ProvenanceEntry(entity_id="leaf", entity_type="entity", activity_id="act")) + assert storage.trace_descendants("leaf") == [] + finally: + if os.path.exists(db_path): + os.unlink(db_path) From 0a8330cbb04efa3e556c5b035ecbbe65ab913d8a Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Mon, 3 Aug 2026 22:52:34 +0530 Subject: [PATCH 2/3] fix(provenance): address code review findings on PR #827 - SQLiteStorage now migrates an existing (pre-#825) provenance.db in place via ALTER TABLE ADD COLUMN for any columns introduced since, instead of only ever running CREATE TABLE IF NOT EXISTS. Without this, opening an older database with the new code would break on the first insert/select since the row width and _row_to_entry's fixed indices grew past the old schema. Added test_migrates_pre_existing_old_schema_database. - verify_chain() now also checks that sequence_id is exactly the predecessor's plus one (no gap, no duplicate), in addition to the existing previous_checksum comparison. Hardens against the narrow case where compute_checksum()'s deliberate exclusion of entity_id could let two distinct rows coincidentally share a checksum, which alone would let a checksum-only comparison miss a gap. Added test_verify_chain_detects_tampered_sequence_gap. - Explorer provenance route: edge ids now include direction (f"{src}-{eid}-{direction}") to match the seen_edges dedupe key, which already included it. The same (src, target) pair can legitimately appear in both the upstream and downstream chains (cycles/overlap), and without this the two edges collided on the same id. Added test_add_chain_edges_ids_distinguish_direction. - Removed an unused `Any` import in parse_provenance.py. --- semantica/explorer/routes/provenance.py | 7 ++- semantica/parse/parse_provenance.py | 2 +- semantica/provenance/manager.py | 54 +++++++++++++----- semantica/provenance/storage.py | 48 +++++++++++++++- tests/explorer/test_provenance_route.py | 28 +++++++++- tests/provenance/test_manager.py | 25 +++++++++ tests/provenance/test_storage.py | 74 +++++++++++++++++++++++++ 7 files changed, 219 insertions(+), 19 deletions(-) diff --git a/semantica/explorer/routes/provenance.py b/semantica/explorer/routes/provenance.py index dd981862..daa52842 100644 --- a/semantica/explorer/routes/provenance.py +++ b/semantica/explorer/routes/provenance.py @@ -82,7 +82,12 @@ def _add_chain_edges( seen_edges.add(edge_key) edges.append({ - "id": f"{src}-{eid}", + # Includes direction to match the seen_edges uniqueness key + # above: the same (src, eid) pair can legitimately appear in + # both directions (e.g. cycles/overlap between the ancestor + # and descendant chains), and without this the two edges + # would collide on the same id. + "id": f"{src}-{eid}-{direction}", "source": src, "target": eid, "label": activity, diff --git a/semantica/parse/parse_provenance.py b/semantica/parse/parse_provenance.py index 6d18386b..85a514ba 100644 --- a/semantica/parse/parse_provenance.py +++ b/semantica/parse/parse_provenance.py @@ -13,7 +13,7 @@ Author: Semantica Contributors License: MIT """ -from typing import Any, Optional +from typing import Optional from datetime import datetime import uuid diff --git a/semantica/provenance/manager.py b/semantica/provenance/manager.py index 8e411a17..8c2d1738 100644 --- a/semantica/provenance/manager.py +++ b/semantica/provenance/manager.py @@ -1418,12 +1418,22 @@ class ProvenanceManager: Part A item 2). Sorts entries by sequence_id (global insertion order) and checks - both that each entry's own checksum matches its content, and that - each entry's previous_checksum matches the checksum of the entry - that precedes it. A gap in the chain — a surviving entry whose - previous_checksum doesn't match its predecessor's checksum — is - exactly what wholesale row deletion produces, since per-row - checksums alone can't detect that a row is simply missing. + three things: each entry's own checksum matches its content; each + entry's previous_checksum matches the checksum of the entry that + precedes it; and sequence_id is exactly the predecessor's plus one + (no gap, no duplicate). Every _save_entry() call assigns + head_sequence + 1, and archival relabels (see track_entity's + versioning and invalidate()) always preserve their existing + sequence_id rather than consuming a new one — so under this design, + the full set of currently-existing sequence_id values is always + exactly {1..N} with nothing missing, unless a row was hard-deleted. + A surviving entry whose previous_checksum doesn't match its + predecessor, or whose sequence_id isn't predecessor+1, is exactly + what wholesale row deletion produces; checking both signals also + guards against the narrow case where two distinct rows happen to + share a checksum (compute_checksum() deliberately excludes entity_id, + see integrity.py), which alone would let a checksum-only check miss + a gap that the sequence check still catches. Returns: Dictionary with "valid", "total_entries", and "broken_links" @@ -1436,6 +1446,7 @@ class ProvenanceManager: broken_links: List[Dict[str, Any]] = [] expected_previous: Optional[str] = None + expected_sequence: Optional[int] = None for entry in entries: if not verify_checksum(entry): broken_links.append({ @@ -1443,16 +1454,29 @@ class ProvenanceManager: "sequence_id": entry.sequence_id, "reason": "checksum_mismatch", }) - continue - if entry.previous_checksum != expected_previous: - broken_links.append({ - "entity_id": entry.entity_id, - "sequence_id": entry.sequence_id, - "reason": "chain_break", - "expected_previous_checksum": expected_previous, - "actual_previous_checksum": entry.previous_checksum, - }) + else: + sequence_gap = ( + expected_sequence is not None + and entry.sequence_id != expected_sequence + 1 + ) + checksum_break = entry.previous_checksum != expected_previous + if sequence_gap or checksum_break: + broken_links.append({ + "entity_id": entry.entity_id, + "sequence_id": entry.sequence_id, + "reason": "chain_break", + "expected_previous_checksum": expected_previous, + "actual_previous_checksum": entry.previous_checksum, + "expected_sequence_id": ( + expected_sequence + 1 if expected_sequence is not None else None + ), + }) + + # Advance state from this entry's own stored fields regardless of + # whether it was flagged above, so a single corrupted entry + # doesn't cascade into spurious breaks for every entry after it. expected_previous = entry.checksum + expected_sequence = entry.sequence_id return { "valid": len(broken_links) == 0, diff --git a/semantica/provenance/storage.py b/semantica/provenance/storage.py index 768bc263..159fec5b 100644 --- a/semantica/provenance/storage.py +++ b/semantica/provenance/storage.py @@ -532,13 +532,54 @@ class SQLiteStorage(ProvenanceStorage): finally: conn.close() + # Columns added after the table was first shipped (issue #825, Part A/B). + # CREATE TABLE IF NOT EXISTS alone does not alter an existing table, so a + # provenance.db created before these columns existed would otherwise fail + # on every insert/select once the row width and _row_to_entry's fixed + # indices grew past the old schema. _migrate_schema() adds any of these + # that are missing from an already-existing table. + _MIGRATION_COLUMNS: List[Tuple[str, str]] = [ + ("agent_type", "TEXT DEFAULT 'software_agent'"), + ("is_automated", "INTEGER DEFAULT 1"), + ("role", "TEXT"), + ("sequence_id", "INTEGER"), + ("previous_checksum", "TEXT"), + ("previous_version_id", "TEXT"), + ("derived_from_id", "TEXT"), + ("invalidated", "INTEGER DEFAULT 0"), + ("invalidated_at_time", "TEXT"), + ("invalidated_by", "TEXT"), + ("invalidation_reason", "TEXT"), + ("activity_started_at_time", "TEXT"), + ("activity_ended_at_time", "TEXT"), + ("acted_on_behalf_of", "TEXT"), + ("informed_by_activities", "TEXT"), + ("valid_from", "TEXT"), + ("valid_until", "TEXT"), + ("revision_type", "TEXT"), + ("supersedes", "TEXT"), + ("bundle_id", "TEXT"), + ] + + def _migrate_schema(self, conn: sqlite3.Connection) -> None: + """Add any columns introduced after the table was first created to an + already-existing table (see _MIGRATION_COLUMNS).""" + cursor = conn.cursor() + cursor.execute("PRAGMA table_info(provenance)") + existing_columns = {row[1] for row in cursor.fetchall()} + for column_name, column_def in self._MIGRATION_COLUMNS: + if column_name not in existing_columns: + cursor.execute( + f"ALTER TABLE provenance ADD COLUMN {column_name} {column_def}" + ) + def _init_db(self) -> None: """Create tables with W3C PROV-O compliant schema.""" conn = sqlite3.connect(self.db_path) try: self._configure_connection(conn) cursor = conn.cursor() - + cursor.execute(""" CREATE TABLE IF NOT EXISTS provenance ( entity_id TEXT PRIMARY KEY, @@ -583,6 +624,11 @@ class SQLiteStorage(ProvenanceStorage): ) """) + # Add any columns missing from a table that already existed + # before these were introduced (see _MIGRATION_COLUMNS) — a no-op + # for a table that was just freshly created above. + self._migrate_schema(conn) + # Create indexes for efficient querying cursor.execute(""" CREATE INDEX IF NOT EXISTS idx_entity_type diff --git a/tests/explorer/test_provenance_route.py b/tests/explorer/test_provenance_route.py index 73f59bf2..aa2b9d72 100644 --- a/tests/explorer/test_provenance_route.py +++ b/tests/explorer/test_provenance_route.py @@ -2,7 +2,11 @@ from types import SimpleNamespace -from semantica.explorer.routes.provenance import _build_provenance, _render_markdown +from semantica.explorer.routes.provenance import ( + _add_chain_edges, + _build_provenance, + _render_markdown, +) def _make_session_with_chain() -> SimpleNamespace: @@ -72,3 +76,25 @@ def test_render_markdown_groups_edges_by_direction(): assert "## Lateral" in markdown assert "`Intermediate` -[related_to]-> `node_id`" in markdown assert "`Source` -[related_to]-> `Intermediate`" in markdown + + +def test_add_chain_edges_ids_distinguish_direction(): + """Regression test: the same (src, target) pair appearing in both the + ancestor and descendant chains (e.g. a cycle or overlap between them) + must not collide on edge id — the id must carry the same uniqueness + information as the seen_edges dedupe key, which already includes + direction.""" + same_pair_chain = [ + {"entity_id": "b", "parent_entity_id": "a", "activity_id": "act"}, + ] + + edges: list = [] + seen_edges: set = set() + _add_chain_edges(same_pair_chain, edges, seen_edges, "upstream") + _add_chain_edges(same_pair_chain, edges, seen_edges, "downstream") + + assert len(edges) == 2 + ids = {edge["id"] for edge in edges} + assert len(ids) == 2, f"edge ids collided across directions: {edges}" + directions = {edge["id"]: edge["direction"] for edge in edges} + assert directions == {"a-b-upstream": "upstream", "a-b-downstream": "downstream"} diff --git a/tests/provenance/test_manager.py b/tests/provenance/test_manager.py index e8c7f44a..e9f14a6c 100644 --- a/tests/provenance/test_manager.py +++ b/tests/provenance/test_manager.py @@ -1102,6 +1102,31 @@ class TestHashChain: assert result["valid"] is False assert any(link["reason"] == "chain_break" for link in result["broken_links"]) + def test_verify_chain_detects_tampered_sequence_gap(self, tmp_path): + """The sequence_id continuity check catches a gap introduced by + directly tampering with a row's sequence_id column, independent of + the previous_checksum comparison — hardening against the narrow case + where compute_checksum()'s deliberate exclusion of entity_id could + otherwise let two distinct rows coincidentally share a checksum.""" + import sqlite3 + db_path = str(tmp_path / "chain_seq_gap.db") + prov_mgr = ProvenanceManager(storage_path=db_path) + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("e2", source="doc1") + prov_mgr.track_entity("e3", source="doc1") + + conn = sqlite3.connect(db_path) + conn.execute("UPDATE provenance SET sequence_id = 10 WHERE entity_id = 'e3'") + conn.commit() + conn.close() + + prov_mgr2 = ProvenanceManager(storage_path=db_path) + result = prov_mgr2.verify_chain() + assert result["valid"] is False + broken = [link for link in result["broken_links"] if link["entity_id"] == "e3"] + assert broken + assert broken[0]["expected_sequence_id"] == 3 + def test_sequence_ids_are_assigned_and_monotonic(self): prov_mgr = ProvenanceManager() e1 = prov_mgr.track_entity("e1", source="doc1") diff --git a/tests/provenance/test_storage.py b/tests/provenance/test_storage.py index 5c35764e..673fbb8f 100644 --- a/tests/provenance/test_storage.py +++ b/tests/provenance/test_storage.py @@ -5,6 +5,7 @@ Tests for InMemoryStorage and SQLiteStorage backends. """ import pytest +import sqlite3 import tempfile import os from semantica.provenance.schemas import ProvenanceEntry @@ -194,6 +195,79 @@ class TestInMemoryStorage: class TestSQLiteStorage: """Test SQLiteStorage backend.""" + def test_migrates_pre_existing_old_schema_database(self): + """Regression test: SQLiteStorage._init_db() previously only ran + CREATE TABLE IF NOT EXISTS, which does not alter an existing table. + A provenance.db created before issue #825's Part A/B columns existed + would otherwise break on every insert/select once the new code's + wider positional INSERT and _row_to_entry's fixed indices no longer + matched the old (narrower) row width. SQLiteStorage now migrates any + missing columns in on open.""" + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp: + db_path = tmp.name + + try: + # Simulate a database created by the pre-#825 code: only the + # original 19 columns, no agent_type/sequence_id/valid_from/etc. + conn = sqlite3.connect(db_path) + conn.execute(""" + CREATE TABLE provenance ( + entity_id TEXT PRIMARY KEY, + entity_type TEXT NOT NULL, + activity_id TEXT NOT NULL, + agent_id TEXT DEFAULT 'semantica', + source_document TEXT, + source_location TEXT, + source_quote TEXT, + timestamp TEXT NOT NULL, + first_seen TEXT, + last_updated TEXT, + confidence REAL DEFAULT 1.0, + checksum TEXT, + parent_entity_id TEXT, + used_entities TEXT, + start_index INTEGER, + end_index INTEGER, + credibility REAL, + metadata TEXT, + version TEXT DEFAULT '1.0' + ) + """) + conn.execute( + "INSERT INTO provenance VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + "old_entity", "entity", "legacy_activity", "semantica", "doc1", + None, None, "2025-01-01T00:00:00", "2025-01-01T00:00:00", + "2025-01-01T00:00:00", 1.0, "oldchecksum", None, "[]", + None, None, None, "{}", "1.0", + ), + ) + conn.commit() + conn.close() + + # Opening with the current SQLiteStorage must not crash, and must + # migrate the table in place rather than requiring a fresh file. + storage = SQLiteStorage(db_path) + + old_entry = storage.retrieve("old_entity") + assert old_entry is not None + assert old_entry.entity_id == "old_entity" + assert old_entry.agent_type == "software_agent" # new column default + assert old_entry.sequence_id is None # never assigned pre-migration + + # New writes against the migrated table must work too. + new_entry = ProvenanceEntry( + entity_id="new_entity", entity_type="entity", activity_id="act", + agent_id="alice", sequence_id=1, checksum="chk1", + ) + storage.store(new_entry) + retrieved = storage.retrieve("new_entity") + assert retrieved.agent_id == "alice" + assert retrieved.sequence_id == 1 + finally: + if os.path.exists(db_path): + os.unlink(db_path) + def test_all_fields_round_trip_through_sqlite(self): """Regression test: every ProvenanceEntry field, including issue #825 Part A and Part B additions, must survive a SQLite store/retrieve From e9e05fedbdffd99eeb0e0c50142d8086a5ad1b05 Mon Sep 17 00:00:00 2001 From: Sameer6305 Date: Tue, 4 Aug 2026 00:01:50 +0530 Subject: [PATCH 3/3] fix(provenance): reset in-memory chain state on clear --- semantica/provenance/storage.py | 3 +++ tests/provenance/test_manager.py | 24 ++++++++++++++++++++++++ tests/provenance/test_storage.py | 25 +++++++++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/semantica/provenance/storage.py b/semantica/provenance/storage.py index 159fec5b..6eedca77 100644 --- a/semantica/provenance/storage.py +++ b/semantica/provenance/storage.py @@ -317,6 +317,9 @@ class InMemoryStorage(ProvenanceStorage): """ count = len(self._entries) self._entries.clear() + with self._seq_lock: + self._seq_counter = 0 + self._chain_head = None return count @contextmanager diff --git a/tests/provenance/test_manager.py b/tests/provenance/test_manager.py index e9f14a6c..1a864cff 100644 --- a/tests/provenance/test_manager.py +++ b/tests/provenance/test_manager.py @@ -466,6 +466,30 @@ class TestProvenanceManager: lineage = prov_mgr.get_lineage("entity_1") assert lineage == {} + def test_clear_resets_chain_state(self): + """Regression: clear() must fully reset chain state so the first write + after clear starts a fresh chain and verify_chain() passes.""" + prov_mgr = ProvenanceManager() + + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("e2", source="doc1") + prov_mgr.clear() + + # Chain head must be empty immediately after clear + assert prov_mgr.storage.get_chain_head() is None + + # First write after clear starts a fresh chain (sequence_id=1, previous_checksum=None) + entry = prov_mgr.track_entity("e3", source="doc2") + assert entry.sequence_id == 1 + assert entry.previous_checksum is None + + # verify_chain() must pass on the fresh chain + prov_mgr.track_entity("e4", source="doc2") + result = prov_mgr.verify_chain() + assert result["valid"] is True + assert result["total_entries"] == 2 + assert result["broken_links"] == [] + def test_retrack_with_explicit_parent_overrides_history_link(self): """#742 — re-tracking an entity with an explicit parent_entity_id must honor the new value, not silently replace it with an auto-generated diff --git a/tests/provenance/test_storage.py b/tests/provenance/test_storage.py index 673fbb8f..02ae4c9f 100644 --- a/tests/provenance/test_storage.py +++ b/tests/provenance/test_storage.py @@ -135,6 +135,31 @@ class TestInMemoryStorage: assert count == 1 assert len(storage.retrieve_all()) == 0 + def test_clear_resets_chain_state(self): + """Issue #825 fix: clear() must reset chain state so get_chain_head() + returns None and the first write after clear starts a fresh chain.""" + storage = InMemoryStorage() + + entry1 = ProvenanceEntry( + entity_id="e1", entity_type="entity", activity_id="act", + sequence_id=1, checksum="checksum_1", + ) + storage.store(entry1) + assert storage.get_chain_head() == (1, "checksum_1") + + storage.clear() + + # Chain head must be None after clear + assert storage.get_chain_head() is None + + # First write after clear starts a fresh chain + entry2 = ProvenanceEntry( + entity_id="e2", entity_type="entity", activity_id="act", + sequence_id=1, checksum="checksum_fresh", + ) + storage.store(entry2) + assert storage.get_chain_head() == (1, "checksum_fresh") + def test_get_chain_head(self): """Issue #825, Part A item 2 — chain head reporting.""" storage = InMemoryStorage()