From db4361ad46847654787b90df9c2465d6f705f2c3 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Mon, 3 Aug 2026 22:24:24 +0530 Subject: [PATCH] feat(provenance): close PROV-O compliance gaps and high-stakes trust blockers (closes #825) Part A - high-stakes trust blockers: - Invalidation tombstones via ProvenanceManager.invalidate() (archive-then-append, never mutates or deletes) instead of hard delete - Hash-chained integrity: sequence_id/previous_checksum chain every entry to its predecessor; new verify_chain() detects wholesale row deletion that a lone per-row checksum cannot - Typed Agent (AgentRecord: agent_type/is_automated) and Activity (ActivityRecord: start/end timing), wired through all 18 *_provenance.py wrappers - Split parent_entity_id into previous_version_id (correction) vs derived_from_id (cross-source derivation), additive alongside the legacy combined field - Downstream/descendant lineage traversal (get_descendants/trace_descendants, reverse BFS) closing the dead direction="downstream" code path in the Explorer's provenance route - Qualified Association+hadRole and Invalidation in export_prov() - New CLI: provenance invalidate|verify-chain|descendants Part B - general PROV-O spec completeness: - Qualified Generation/Usage/Derivation in export_prov() - wasAssociatedWith, actedOnBehalfOf, wasInformedBy relations - Bitemporal fields (valid_from/valid_until/revision_type/supersedes) plus revision_history()/query_recorded_between(), closing the deprecated kg.ProvenanceTracker's "no direct equivalent yet" migration gaps - prov:Bundle/hadMember membership via bundle_id - Configurable base_uri (--base-uri CLI flag), shared by RDFExporter's NamespaceManager and OWLExporter's default ontology_uri so KG/OWL/PROV exports co-resolve under one namespace instead of three hardcoded ones Bugs fixed along the way: - agent_id was a dead field: no track_* method read it from kwargs - track_entities_batch silently absorbed typed kwargs into the metadata blob - compute_checksum() had to exclude entity_id itself: hashing it made track_entity's versioning-archive relabel permanently orphan any entry already chained from the pre-relabel checksum, a false-positive "broken chain" for a legitimate rename - InMemoryStorage.get_chain_head() ignored the committed head whenever the current transaction had staged entries, corrupting the next chain link - several new ProvenanceEntry fields were wired into the dataclass and export_prov() but not into SQLiteStorage's DDL/INSERT/row-mapping; InMemoryStorage masked the gap. Added a permanent round-trip regression test to catch this class of bug for future field additions Flagged, not fixed (separate pre-existing issues, out of scope for #825): - pipeline/pipeline_provenance.py imports a nonexistent module and wraps a Pipeline dataclass with no run() method - most *_provenance.py wrappers' backing classes are themselves missing or incomplete (context_manager, deduplicator, normalizer, etc.) - kg_provenance.py passes entity_type inside metadata={} instead of as a top-level track_entity() kwarg across most of its call sites --- CHANGELOG.md | 17 + docs/guides/provenance.md | 23 +- docs/migration/kg-provenance-tracker.md | 4 +- semantica/cli.py | 121 +++- semantica/conflicts/conflicts_provenance.py | 28 +- semantica/context/context_provenance.py | 26 +- .../deduplication/deduplication_provenance.py | 30 +- semantica/embeddings/embeddings_provenance.py | 30 +- semantica/explorer/routes/provenance.py | 85 ++- semantica/export/export_provenance.py | 30 +- semantica/export/owl_exporter.py | 11 +- semantica/export/rdf_exporter.py | 10 +- .../graph_store/graph_store_provenance.py | 30 +- semantica/ingest/ingest_provenance.py | 42 +- semantica/kg/kg_provenance.py | 208 +++++- semantica/llms/llms_provenance.py | 141 +++- semantica/normalize/normalize_provenance.py | 30 +- semantica/ontology/ontology_provenance.py | 30 +- semantica/parse/parse_provenance.py | 30 +- semantica/pipeline/pipeline_provenance.py | 29 +- semantica/provenance/__init__.py | 13 +- semantica/provenance/integrity.py | 59 +- semantica/provenance/manager.py | 634 +++++++++++++++++- semantica/provenance/schemas.py | 240 ++++++- semantica/provenance/storage.py | 329 ++++++++- semantica/reasoning/reasoning_provenance.py | 30 +- .../semantic_extract_provenance.py | 159 +++-- .../triplet_store/triplet_store_provenance.py | 30 +- .../vector_store/vector_store_provenance.py | 30 +- .../visualization/visualization_provenance.py | 30 +- tests/provenance/test_manager.py | 609 +++++++++++++++++ tests/provenance/test_schemas.py | 124 +++- tests/provenance/test_storage.py | 175 ++++- 33 files changed, 3104 insertions(+), 313 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 389779fc..03f31124 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **PROV-O trust blockers and general spec completeness for `ProvenanceManager`** (#825) by @KaifAhmad1 + - **Invalidation instead of hard delete**: new `ProvenanceManager.invalidate(entity_id, agent_id, reason=None)` tombstones an entry — archives its pre-invalidation state under a stable versioned key, then appends the invalidated entry (`invalidated`, `invalidated_at_time`, `invalidated_by`, `invalidation_reason`) — instead of mutating or deleting it, so an audit can prove a fact existed, was reviewed, and was retracted. `ProvenanceManager.clear()` remains the bulk dev/test store-reset utility it always was; it was not repurposed + - **Hash-chained integrity**: every entry now carries `sequence_id`/`previous_checksum`, chaining it to the entry immediately before it in insertion order. New `ProvenanceManager.verify_chain()` walks the chain and reports any break, including a row hard-deleted directly from the underlying table — something a lone per-row SHA-256 checksum can never detect on its own. `compute_checksum()` now also covers `agent_id`/`agent_type`, the lineage-link fields, and the invalidation fields, closing several fields that previously weren't tamper-evident + - **Typed Agent/Activity**: `agent_id` was a dead field — no `track_*` method read it from kwargs, so it was always the `"semantica"` default regardless of what callers passed; fixed, and paired with new `AgentRecord(id, agent_type, is_automated)` / `ActivityRecord(id, activity_type, started_at_time, ended_at_time)` dataclasses (pass via `agent=`/`activity=` kwargs) so a human reviewer, an LLM call, and an automated pipeline stage are now distinguishable, and activities carry real start/end timing. Wired through all 18 `*_provenance.py` wrapper modules and `track_entity`/`track_relationship`/`track_chunk`/`track_property_source` + - **Versioning vs. derivation split**: new `previous_version_id` ("this corrects a prior version of the same fact") and `derived_from_id` ("this was derived from a different source entity") fields, additive alongside the legacy combined `parent_entity_id` so existing readers are unaffected + - **Downstream lineage traversal**: new `get_descendants()`/`trace_descendants()` (reverse BFS in both `InMemoryStorage` and `SQLiteStorage`), closing the gap flagged in `semantica/explorer/routes/provenance.py` where `direction="downstream"` was dead code with no reverse lookup to feed it; the Explorer's `/api/provenance` lineage response now merges both directions + - **W3C PROV-O qualified relations**: `export_prov()` now emits `prov:qualifiedAssociation`/`hadRole` (distinguishing "approved by" from "generated by" for sign-off workflows), `qualifiedGeneration`/`Generation`, `qualifiedUsage`/`Usage`, `qualifiedDerivation`/`Derivation`, `qualifiedInvalidation`/`Invalidation`, `wasAssociatedWith` (Activity→Agent), `actedOnBehalfOf` (Agent→Agent delegation), and `wasInformedBy` (Activity→Activity, via a new `informed_by=[...]` kwarg), alongside the existing plain triples + - **Bitemporal + Bundle support**: `revision_type`/`supersedes`/`valid_from`/`valid_until` fields (plain caller-supplied passthrough, matching the deprecated `kg.ProvenanceTracker`'s actual contract) plus new `revision_history()` and `query_recorded_between()` methods, closing the two "no direct equivalent yet" rows in `docs/migration/kg-provenance-tracker.md`; `bundle_id` emits `prov:Bundle`/`hadMember` membership triples to partition provenance by source/dataset/ingestion-run + - **Configurable, interlinked namespace**: `export_prov(base_uri=...)` / `--base-uri` CLI flag, defaulting to a new `ProvenanceManager.DEFAULT_BASE_URI` (`https://semantica.dev/ns#`) that `RDFExporter`'s `NamespaceManager` and `OWLExporter`'s default `ontology_uri` now both reuse, so KG-exported, OWL-exported, and PROV-exported URIs for the same `entity_id` co-resolve instead of three independently-hardcoded placeholder domains + - New CLI commands: `semantica provenance invalidate|verify-chain|descendants` + - **Fixed along the way**: `track_entities_batch()` silently absorbed batch-level typed kwargs (`agent_id`, `entity_type`, `activity_id`) into the opaque `metadata` JSON blob instead of forwarding them, so the documented banking example in `docs/guides/provenance.md` never actually worked as written + - **Fixed along the way**: `compute_checksum()` had to exclude `entity_id` itself from the hash — `track_entity()`'s versioning archives a prior value by copying it to a new key (`"X"` → `"X:v:"`), and hashing `entity_id` meant that legitimate relabel permanently orphaned any other entry that had already chained its `previous_checksum` from the pre-relabel value, surfacing as a false-positive "broken chain." Archival and invalidation are now always a pure relabel (unchanged checksum/sequence position) followed by a fresh chained append, never an in-place mutation of an already-chained entry + - **Fixed along the way**: `InMemoryStorage.get_chain_head()` ignored the already-committed chain head whenever the current transaction had staged any entries, understating the head and corrupting the next append's chain link + - **Fixed along the way**: several new `ProvenanceEntry` fields were initially wired into the dataclass and `export_prov()` but not into `SQLiteStorage`'s DDL/INSERT/row-mapping — `InMemoryStorage` stores the dataclass directly so it masked the gap. Added a permanent regression test (`test_all_fields_round_trip_through_sqlite`) asserting every field survives a SQLite round trip, to catch this class of bug for any future field additions + - Flagged, not fixed (separate, pre-existing issues independent of #825): `semantica/pipeline/pipeline_provenance.py` imports a nonexistent module and wraps a `Pipeline` dataclass with no `run()` method, so `PipelineWithProvenance` has never worked; most of the 18 wrapper modules' backing classes are themselves missing or incomplete (e.g. `context.context_manager`, `deduplication.deduplicator`, `normalize.normalizer` don't exist; `EmbeddingGenerator` exists but has no `.embed()`); `kg_provenance.py` passes `entity_type` inside its `metadata={}` dict instead of as a top-level `track_entity()` kwarg across most of its ~30 call sites, so it never actually populates the real field + - Extensive new test coverage across `tests/provenance/test_manager.py`, `test_schemas.py`, and `test_storage.py` (invalidation, hash-chain verification including a simulated hard-delete-detection case and an interleaved-chaining stress test, agent/activity typing, versioning/derivation split, downstream lineage, qualified export triples, bitemporal methods, Bundle export, and namespace interlinking) + - **Altair Anzo triplet store backend** (#813) by @KaifAhmad1 - Added `AnzoStore` (`semantica/triplet_store/anzo_store.py`), a fourth peer to `BlazegraphStore`/`RDF4JStore`/`JenaStore` speaking plain SPARQL 1.1 over HTTP — no new dependency, since Anzo has no official Python SDK but needs none - The one structural difference from the existing backends: Anzo addresses data by a dataset/graphmart **URI** (`dataset_uri`, required) rather than a short namespace/repository name, so the endpoint path (`/sparql//`) percent-encodes it; `store_type` defaults to `"graphmart"` and can be set to `"dataset"` diff --git a/docs/guides/provenance.md b/docs/guides/provenance.md index e05fa76b..a1b4d8f1 100644 --- a/docs/guides/provenance.md +++ b/docs/guides/provenance.md @@ -633,12 +633,29 @@ Every `ProvenanceEntry` maps directly to W3C PROV-O terms. If your compliance te | :--- | :--- | :--- | | `prov:Entity` | `entity_id` | The tracked object — entity, chunk, relationship, or property | | `prov:Activity` | `activity_id` | The process that produced it — `"ner_extraction"`, `"bureau_parsing"` | -| `prov:Agent` | `agent_id` | Who ran the activity — pipeline name, analyst ID | -| `prov:wasDerivedFrom` | `parent_entity_id` | The previous version of this entity — enables version chaining | +| `prov:Agent` / `prov:Person` / `prov:SoftwareAgent` / `prov:Organization` | `agent_id`, `agent_type`, `is_automated` | Who — or what — ran the activity, and whether a human was directly accountable | +| `prov:qualifiedAssociation` + `prov:hadRole` | `role` | The agent's role for this specific entity — `"generator"` (default), `"approver"`, `"reviewer"` — for sign-off/four-eyes workflows | +| `prov:wasDerivedFrom` | `parent_entity_id` (legacy combined field) | The previous version or source of this entity | +| — | `previous_version_id` | This entry corrects/replaces a prior version of the *same* fact | +| `prov:wasDerivedFrom` | `derived_from_id` | This entry was derived from a *different* source entity | | `prov:used` | `used_entities` | Entity IDs consumed to produce this one | | `prov:generatedAtTime` | `timestamp` | ISO datetime, auto-set to `datetime.utcnow()` at write time | +| `prov:qualifiedInvalidation` | `invalidated`, `invalidated_at_time`, `invalidated_by`, `invalidation_reason` | A retraction/correction recorded as a tombstone via `ProvenanceManager.invalidate()`, never a hard delete | +| `prov:startedAtTime` / `prov:endedAtTime` | `activity_started_at_time`, `activity_ended_at_time` | Typed Activity timing — pass an `ActivityRecord` via the `activity=` kwarg to set these together with `activity_id` | +| `prov:qualifiedGeneration`/`Generation`, `qualifiedUsage`/`Usage`, `qualifiedDerivation`/`Derivation` | (derived from the fields above) | Additive qualified forms of `wasGeneratedBy`/`used`/`wasDerivedFrom`, emitted automatically alongside the plain triples | +| `prov:wasAssociatedWith` | (derived from `agent_id`) | Direct Activity→Agent link, distinct from the Entity→Agent `wasAttributedTo` | +| `prov:actedOnBehalfOf` | `acted_on_behalf_of` | Agent→Agent delegation — e.g. an automated agent acting on behalf of the human/organization that authorized it | +| `prov:wasInformedBy` | `informed_by_activities` (pass as `informed_by=[...]`) | Chains this entry's activity to prior activities it was informed by (e.g. a pipeline stage informed by the stage before it) | +| `prov:Bundle` + `prov:hadMember` | `bundle_id` | Groups entries by source/dataset/ingestion-run (membership triples, not true RDF named-graph partitioning) | +| — | `valid_from`, `valid_until`, `revision_type`, `supersedes` | Bitemporal fields merged from the deprecated `kg.ProvenanceTracker` — always caller-supplied (never auto-computed), surfaced via `ProvenanceManager.revision_history()`, which falls back to timestamp-based derivation for entries that don't set them explicitly | -The `checksum` field is not part of the PROV-O standard — it is Semantica's tamper-detection extension. Every entry's SHA-256 is computed from its content fields at write time and can be recomputed at any time to verify the record has not been modified. +`previous_version_id` and `derived_from_id` are additive alongside `parent_entity_id` — existing code reading `parent_entity_id` keeps working unchanged, while new code gets the two relations disambiguated. + +The `checksum` field is not part of the PROV-O standard — it is Semantica's tamper-detection extension. Every entry's SHA-256 now also incorporates `previous_checksum` (the prior entry's checksum, by insertion order via `sequence_id`), chaining every entry to the one before it. `ProvenanceManager.verify_chain()` walks the full chain and reports any break — including a row that was hard-deleted from the underlying table, which a lone per-row checksum can't detect on its own. + +Note: the banking example above passes `agent_id="credit_data_service_v2"` to `track_entities_batch()` — this now actually populates the entry's `agent_id` field (previously a bug caused batch-level typed kwargs like `agent_id`/`entity_type`/`activity_id` to be silently absorbed into the opaque `metadata` blob instead). + +`export_prov()` mints entity/agent/activity URIs under `ProvenanceManager.DEFAULT_BASE_URI` (`https://semantica.dev/ns#` by default — the same namespace `RDFExporter`'s `NamespaceManager` uses for its `"semantica"` prefix, so KG-exported and PROV-exported URIs for the same `entity_id` co-resolve) unless overridden via `export_prov(base_uri=...)` or the CLI's `--base-uri` option. ## Related Guides diff --git a/docs/migration/kg-provenance-tracker.md b/docs/migration/kg-provenance-tracker.md index a058a359..496f5b13 100644 --- a/docs/migration/kg-provenance-tracker.md +++ b/docs/migration/kg-provenance-tracker.md @@ -17,8 +17,8 @@ Every method on `kg.ProvenanceTracker` now emits a `DeprecationWarning` on use, | `track_entity(entity_id, source, metadata)` | `track_entity(entity_id, source, metadata)` | Same call shape. `ProvenanceManager` additionally auto-links each update to its prior version via `parent_entity_id`. | | `get_all_sources(entity_id)` | `get_all_sources(entity_id)` | Field name differs: the `kg` tracker returns each record's time under `"recorded_at"`; `ProvenanceManager` returns `"timestamp"`. | | `clear(entity_id=None)` | `clear()` | `ProvenanceManager.clear()` clears all provenance data; there is no per-entity clear yet. | -| `query_recorded_between(start, end)` | *No direct equivalent yet* | Filter the entries returned by `get_lineage()` / `trace_lineage()` client-side in the meantime. | -| `revision_history(fact_id)` | *No direct equivalent yet* | `get_lineage(fact_id)["lineage_chain"]` returns the full chain of `ProvenanceEntry` records but not in the same versioned shape. | +| `query_recorded_between(start, end)` | `query_recorded_between(start, end)` | Same call shape; filters by `timestamp` (ISO 8601 string comparison) across all tracked entries, not just one entity. | +| `revision_history(fact_id)` | `revision_history(fact_id)` | Same call shape and return shape (`version`, `valid_from`, `valid_until`, `recorded_at`, `author`, optional `revision_type`/`supersedes`) — walks the entity's `previous_version_id` chain rather than a flat per-entity dict. | | `export_audit_log(fact_ids, format)` | *No direct equivalent yet* | Build the export from `get_lineage()` output, or serialize `get_statistics()` for a summary view. | Methods with no direct equivalent are not planned to be reimplemented on `kg.ProvenanceTracker` — they will need a small adapter in caller code, or a feature request against `ProvenanceManager` if you rely on them heavily. diff --git a/semantica/cli.py b/semantica/cli.py index 56d8e9a0..5b179c13 100644 --- a/semantica/cli.py +++ b/semantica/cli.py @@ -2559,27 +2559,31 @@ def provenance_audit(cli_ctx: CLIContext, since: Optional[str], fmt: str, @provenance.command("export") @click.option("--format", "fmt", type=click.Choice(["turtle", "ntriples", "jsonld"]), default="turtle", show_default=True) +@click.option("--base-uri", "base_uri", default=None, + help="Namespace URI entities/agents/activities are minted under " + "(default: ProvenanceManager.DEFAULT_BASE_URI).") @click.option("--output", default=None, type=click.Path()) @click.option("--dry-run", "local_dry", is_flag=True, default=False) @click.pass_obj -def provenance_export(cli_ctx: CLIContext, fmt: str, output: Optional[str], - local_dry: bool) -> None: +def provenance_export(cli_ctx: CLIContext, fmt: str, base_uri: Optional[str], + output: Optional[str], local_dry: bool) -> None: """Export provenance as W3C PROV-O RDF. \b Example: semantica provenance export --format turtle --output prov.ttl + semantica provenance export --base-uri https://example.org/kg# --output prov.ttl """ cli_ctx = _require_ctx(cli_ctx) def _action() -> None: if _is_dry(cli_ctx, local_dry): - _dry(cli_ctx, "export provenance", format=fmt, output=output) + _dry(cli_ctx, "export provenance", format=fmt, base_uri=base_uri, output=output) return try: from .provenance import ProvenanceManager pm = ProvenanceManager(config=cli_ctx.config.to_dict()) - data = pm.export_prov(format=fmt) + data = pm.export_prov(format=fmt, base_uri=base_uri) except ImportError as exc: raise click.ClickException(f"Provenance module not available: {exc}") from exc if output: @@ -2621,6 +2625,115 @@ def provenance_check(cli_ctx: CLIContext, strict: bool, local_json: bool) -> Non _run_with_error_handling(_action) +@provenance.command("invalidate") +@click.argument("entity_id") +@click.option("--by", "agent_id", required=True, help="Agent responsible for the invalidation.") +@click.option("--reason", default=None, help="Human-readable reason for the invalidation.") +@click.option("--json", "local_json", is_flag=True, default=False) +@click.option("--dry-run", "local_dry", is_flag=True, default=False) +@click.pass_obj +def provenance_invalidate(cli_ctx: CLIContext, entity_id: str, agent_id: str, + reason: Optional[str], local_json: bool, + local_dry: bool) -> None: + """Mark a tracked entity as invalidated (tombstone, not a hard delete). + + \b + Example: + semantica provenance invalidate entity_alice --by reviewer_jane --reason "Source retracted" + """ + cli_ctx = _require_ctx(cli_ctx) + + def _action() -> None: + if _is_dry(cli_ctx, local_dry): + _dry(cli_ctx, "invalidate provenance entity", entity_id=entity_id, by=agent_id, reason=reason) + return + try: + from .provenance import ProvenanceManager + pm = ProvenanceManager(config=cli_ctx.config.to_dict()) + result = pm.invalidate(entity_id, agent_id=agent_id, reason=reason) + except ImportError as exc: + raise click.ClickException(f"Provenance module not available: {exc}") from exc + except ValueError as exc: + raise click.ClickException(str(exc)) from exc + if _is_json(cli_ctx, local_json): + _jecho(result.to_dict()) + else: + _ok(cli_ctx, f"Invalidated {entity_id} (by {agent_id})") + + _run_with_error_handling(_action) + + +@provenance.command("verify-chain") +@click.option("--json", "local_json", is_flag=True, default=False) +@click.pass_obj +def provenance_verify_chain(cli_ctx: CLIContext, local_json: bool) -> None: + """Verify the hash chain across all provenance entries. + + Detects wholesale row deletion: per-row checksums alone only prove a + surviving row wasn't edited in place, not that no row is missing. + + \b + Example: + semantica provenance verify-chain + """ + cli_ctx = _require_ctx(cli_ctx) + + def _action() -> None: + try: + from .provenance import ProvenanceManager + pm = ProvenanceManager(config=cli_ctx.config.to_dict()) + result = pm.verify_chain() + except ImportError as exc: + raise click.ClickException(f"Provenance module not available: {exc}") from exc + if _is_json(cli_ctx, local_json): + _jecho(result) + elif result.get("valid"): + _ok(cli_ctx, f"Chain verified: {result.get('total_entries')} entries, no breaks") + else: + _warn(cli_ctx, f"Chain verification failed: {len(result.get('broken_links', []))} broken link(s)") + for link in result.get("broken_links", []): + click.echo(f" {link}") + + _run_with_error_handling(_action) + + +@provenance.command("descendants") +@click.argument("entity_id") +@click.option("--depth", default=None, type=int, show_default=True) +@click.option("--json", "local_json", is_flag=True, default=False) +@click.pass_obj +def provenance_descendants(cli_ctx: CLIContext, entity_id: str, depth: Optional[int], + local_json: bool) -> None: + """Show downstream descendants (reverse lineage) for an entity. + + The counterpart to `provenance lineage`, which only traces upstream + ancestors. Answers "entity X was wrong — what downstream facts used it?" + + \b + Example: + semantica provenance descendants entity_alice --depth 3 + """ + cli_ctx = _require_ctx(cli_ctx) + + def _action() -> None: + try: + from .provenance import ProvenanceManager + pm = ProvenanceManager(config=cli_ctx.config.to_dict()) + if depth is not None: + entries = [e.to_dict() for e in pm.trace_descendants(entity_id, max_depth=depth)] + result = {"entity_id": entity_id, "depth": depth, "entries": entries} + else: + result = pm.get_descendants(entity_id) or {"entity_id": entity_id, "entries": []} + except ImportError as exc: + raise click.ClickException(f"Provenance module not available: {exc}") from exc + if _is_json(cli_ctx, local_json): + _jecho(result) + else: + _pprint(cli_ctx, result) + + _run_with_error_handling(_action) + + @main.group(invoke_without_command=True) @click.pass_context def validate(ctx: click.Context) -> None: diff --git a/semantica/conflicts/conflicts_provenance.py b/semantica/conflicts/conflicts_provenance.py index b37ab418..915acf4d 100644 --- a/semantica/conflicts/conflicts_provenance.py +++ b/semantica/conflicts/conflicts_provenance.py @@ -15,36 +15,52 @@ License: MIT """ from typing import Optional, Dict, Any, List +from datetime import datetime class SourceTrackerWithUnifiedBackend: """SourceTracker using unified provenance backend.""" - def __init__(self, **config): + def __init__( + self, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """Initialize with unified backend or fallback to legacy.""" from .source_tracker import SourceTracker - + + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + try: from semantica.provenance import ProvenanceManager self._unified_manager = ProvenanceManager() self._use_unified = True except ImportError: self._use_unified = False - + self._original_tracker = SourceTracker(**config) - + def track_property_source(self, entity_id: str, property_name: str, value: Any, source: Any, **metadata): """Track property source with unified backend.""" + activity_started_at_time = datetime.utcnow().isoformat() if self._use_unified: from semantica.provenance import SourceReference - + source_ref = SourceReference( document=source.document if hasattr(source, 'document') else str(source), page=getattr(source, 'page', None), section=getattr(source, 'section', None), confidence=getattr(source, 'confidence', 1.0) ) - + + metadata.setdefault("agent_id", self._agent_id) + metadata.setdefault("agent_type", "software_agent") + metadata.setdefault("is_automated", self._is_automated) + metadata.setdefault("activity_started_at_time", activity_started_at_time) + metadata.setdefault("activity_ended_at_time", datetime.utcnow().isoformat()) + self._unified_manager.track_property_source( entity_id=entity_id, property_name=property_name, diff --git a/semantica/context/context_provenance.py b/semantica/context/context_provenance.py index 4871d525..a0492496 100644 --- a/semantica/context/context_provenance.py +++ b/semantica/context/context_provenance.py @@ -12,36 +12,52 @@ License: MIT """ from typing import Optional, Any +from datetime import datetime import uuid class ContextManagerWithProvenance: """Context manager with provenance tracking.""" - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """Initialize context manager with optional provenance.""" from .context_manager import ContextManager - + self.provenance = provenance self._context_manager = ContextManager(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def add_context(self, context: Any, source: Optional[str] = None, **kwargs): """Add context with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._context_manager.add_context(context, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"context_{uuid.uuid4().hex[:8]}", source=source or "context_manager", entity_type="context", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={"context_preview": str(context)[:100]} ) diff --git a/semantica/deduplication/deduplication_provenance.py b/semantica/deduplication/deduplication_provenance.py index 611f680d..689329aa 100644 --- a/semantica/deduplication/deduplication_provenance.py +++ b/semantica/deduplication/deduplication_provenance.py @@ -13,37 +13,53 @@ Author: Semantica Contributors License: MIT """ -from typing import List, Any +from typing import List, Any, Optional +from datetime import datetime import uuid class DeduplicatorWithProvenance: """Deduplicator with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .deduplicator import Deduplicator - + self.provenance = provenance self._deduplicator = Deduplicator(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def deduplicate(self, items: List[Any], source: str = None, **kwargs): """Deduplicate items with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() unique_items = self._deduplicator.deduplicate(items, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: duplicates_found = len(items) - len(unique_items) self._prov_manager.track_entity( entity_id=f"dedup_{uuid.uuid4().hex[:8]}", source=source or "deduplication", entity_type="deduplication_operation", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "input_count": len(items), "output_count": len(unique_items), diff --git a/semantica/embeddings/embeddings_provenance.py b/semantica/embeddings/embeddings_provenance.py index 378a7017..03d7adbd 100644 --- a/semantica/embeddings/embeddings_provenance.py +++ b/semantica/embeddings/embeddings_provenance.py @@ -13,36 +13,52 @@ Author: Semantica Contributors License: MIT """ -from typing import List +from typing import List, Optional +from datetime import datetime import uuid class EmbeddingGeneratorWithProvenance: """Embedding generator with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .embedding_generator import EmbeddingGenerator - + self.provenance = provenance self._generator = EmbeddingGenerator(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def embed(self, texts: List[str], source: str = None, **kwargs): """Generate embeddings with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() embeddings = self._generator.embed(texts, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"embed_{uuid.uuid4().hex[:8]}", source=source or "embedding_generation", entity_type="embeddings", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "model": getattr(self._generator, 'model', 'unknown'), "dimensions": len(embeddings[0]) if embeddings else 0, diff --git a/semantica/explorer/routes/provenance.py b/semantica/explorer/routes/provenance.py index 4a6ac5da..dd981862 100644 --- a/semantica/explorer/routes/provenance.py +++ b/semantica/explorer/routes/provenance.py @@ -32,21 +32,7 @@ def _classify_prov(node_type: str) -> tuple[str, str]: return "Entity", "group_entity" -def _transform_audit_lineage(lineage: Dict[str, Any], node_id: str) -> Dict[str, Any]: - # Mapping decision (W3C PROV-O to frontend swim-lanes): - # Every ProvenanceEntry maps to a node classified by _classify_prov(entry["entity_type"]), - # placing documents/chunks/entities in 'group_entity' (prov_type='Entity'), persons/systems - # in 'group_agent', and actions/processes in 'group_activity'. - # For derivation relationships (parent_entity_id and used_entities), we connect - # parent -> child directly with edge label set to activity_id (or 'wasDerivedFrom'), - # keeping the lineage graph scannable without cluttering it with intermediate activity - # nodes when activity_id is an operational label. - nodes: List[Dict[str, Any]] = [] - edges: List[Dict[str, Any]] = [] - seen_nodes = set() - seen_edges = set() - - chain = lineage.get("lineage_chain") or lineage.get("entries") or [] +def _add_chain_nodes(chain: List[Any], nodes: List[Dict[str, Any]], seen_nodes: set) -> None: for entry in chain: if not isinstance(entry, dict): continue @@ -69,6 +55,10 @@ def _transform_audit_lineage(lineage: Dict[str, Any], node_id: str) -> Dict[str, "checksum": entry.get("checksum") or None, }) + +def _add_chain_edges( + chain: List[Any], edges: List[Dict[str, Any]], seen_edges: set, direction: str +) -> None: for entry in chain: if not isinstance(entry, dict): continue @@ -86,21 +76,11 @@ def _transform_audit_lineage(lineage: Dict[str, Any], node_id: str) -> Dict[str, activity = str(entry.get("activity_id") or "wasDerivedFrom") for src in parents: - edge_key = (src, eid) + edge_key = (src, eid, direction) if edge_key in seen_edges: continue seen_edges.add(edge_key) - # NOTE: ProvenanceManager.get_lineage() currently only traces upstream - # ancestor chains via parent_entity_id and used_entities. It does not perform - # reverse lookups for downstream descendants. Consequently, 'direction = "downstream"' - # is unreachable in practice for this audit path until reverse lookup is supported - # by ProvenanceManager. All ancestor derivation edges are upstream lineage. - if src == node_id: - direction = "downstream" - else: - direction = "upstream" - edges.append({ "id": f"{src}-{eid}", "source": src, @@ -109,6 +89,44 @@ def _transform_audit_lineage(lineage: Dict[str, Any], node_id: str) -> Dict[str, "direction": direction, }) + +def _transform_audit_lineage( + lineage: Dict[str, Any], + node_id: str, + descendants: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + # Mapping decision (W3C PROV-O to frontend swim-lanes): + # Every ProvenanceEntry maps to a node classified by _classify_prov(entry["entity_type"]), + # placing documents/chunks/entities in 'group_entity' (prov_type='Entity'), persons/systems + # in 'group_agent', and actions/processes in 'group_activity'. + # For derivation relationships (parent_entity_id and used_entities), we connect + # parent -> child directly with edge label set to activity_id (or 'wasDerivedFrom'), + # keeping the lineage graph scannable without cluttering it with intermediate activity + # nodes when activity_id is an operational label. + # + # Upstream edges come from lineage's ancestor chain (parent_entity_id/ + # used_entities, via ProvenanceManager.get_lineage()); downstream edges + # come from ProvenanceManager.get_descendants()'s descendant chain (issue + # #825, Part A item 5). Previously 'direction="downstream"' was dead code + # here since no reverse lookup existed. + nodes: List[Dict[str, Any]] = [] + edges: List[Dict[str, Any]] = [] + seen_nodes: set = set() + seen_edges: set = set() + + ancestor_chain = lineage.get("lineage_chain") or lineage.get("entries") or [] + descendant_chain = ( + (descendants or {}).get("descendant_chain") + or (descendants or {}).get("entries") + or [] + ) + + _add_chain_nodes(ancestor_chain, nodes, seen_nodes) + _add_chain_nodes(descendant_chain, nodes, seen_nodes) + + _add_chain_edges(ancestor_chain, edges, seen_edges, "upstream") + _add_chain_edges(descendant_chain, edges, seen_edges, "downstream") + for edge in edges: for endpoint in (edge["source"], edge["target"]): if endpoint not in seen_nodes: @@ -132,9 +150,9 @@ def _transform_audit_lineage(lineage: Dict[str, Any], node_id: str) -> Dict[str, def _build_provenance(session: GraphSession, node_id: Optional[str] = None) -> dict: """Build provenance lineage for a node, attempting the audit-grade store first. - NOTE: The audit path (source='audit') shows verified upstream lineage only. - For descendant/downstream relationships, the naive graph-traversal fallback - remains the only source until ProvenanceManager gains a reverse lookup. + Combines ProvenanceManager.get_lineage() (upstream ancestors) with + get_descendants() (downstream — issue #825, Part A item 5) so both + directions are populated from the audit-grade store, not just upstream. """ if not node_id: return {"nodes": [], "edges": [], "source": "graph_traversal"} @@ -149,7 +167,14 @@ def _build_provenance(session: GraphSession, node_id: Optional[str] = None) -> d entries = lineage.get("lineage_chain") or lineage.get("entries") or [] integrity_ok = all(verify_checksum(entry) for entry in entries) if integrity_ok: - return _transform_audit_lineage(lineage, node_id) + descendants = {} + try: + descendants = manager.get_descendants(node_id) or {} + except Exception as exc: + logger.warning( + f"ProvenanceManager get_descendants failed for {node_id}: {exc}" + ) + return _transform_audit_lineage(lineage, node_id, descendants) logger.warning( f"Provenance integrity verification failed for {node_id}, falling back to graph traversal" ) diff --git a/semantica/export/export_provenance.py b/semantica/export/export_provenance.py index 79a635a4..551b0f2c 100644 --- a/semantica/export/export_provenance.py +++ b/semantica/export/export_provenance.py @@ -13,36 +13,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class ExporterWithProvenance: """Base exporter with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .exporter import Exporter - + self.provenance = provenance self._exporter = Exporter(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def export(self, data: Any, destination: str, **kwargs): """Export data with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._exporter.export(data, destination, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"export_{uuid.uuid4().hex[:8]}", source="export_operation", entity_type="export", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "destination": destination, "format": kwargs.get('format', 'unknown') diff --git a/semantica/export/owl_exporter.py b/semantica/export/owl_exporter.py index 5f123f0c..3c604d59 100644 --- a/semantica/export/owl_exporter.py +++ b/semantica/export/owl_exporter.py @@ -31,6 +31,12 @@ from ..utils.helpers import ensure_directory from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker +# Issue #825, Part B Tier 3 — exporter interlinking. Reuses the same default +# namespace as ProvenanceManager.export_prov() and RDFExporter's +# NamespaceManager "semantica" entry, so ontology URIs, KG instance URIs, and +# PROV-exported URIs co-resolve under one shared namespace by default. +from ..provenance.manager import DEFAULT_BASE_URI + class OWLExporter: """ @@ -58,7 +64,7 @@ class OWLExporter: def __init__( self, - ontology_uri: str = "https://semantica.dev/ontology/", + ontology_uri: str = DEFAULT_BASE_URI, version: str = "1.0", format: str = "owl-xml", config: Optional[Dict[str, Any]] = None, @@ -70,7 +76,8 @@ class OWLExporter: Sets up the exporter with ontology URI, version, and format configuration. Args: - ontology_uri: Base URI for the ontology (default: "https://semantica.dev/ontology/") + ontology_uri: Base URI for the ontology (default: ProvenanceManager.DEFAULT_BASE_URI, + shared with RDFExporter's NamespaceManager and export_prov() so URIs co-resolve) version: Ontology version string (default: "1.0") format: Default export format - 'owl-xml' or 'turtle' (default: 'owl-xml') config: Optional configuration dictionary (merged with kwargs) diff --git a/semantica/export/rdf_exporter.py b/semantica/export/rdf_exporter.py index bf6beaf7..7c50ffc5 100644 --- a/semantica/export/rdf_exporter.py +++ b/semantica/export/rdf_exporter.py @@ -71,13 +71,19 @@ class NamespaceManager: """ self.logger = get_logger("namespace_manager") - # Standard RDF namespaces + # Standard RDF namespaces. "semantica" reuses ProvenanceManager's + # DEFAULT_BASE_URI (issue #825, Part B Tier 3 — exporter + # interlinking) so KG-exported and PROV-exported URIs for the same + # entity_id co-resolve to the same namespace instead of two + # independently-hardcoded placeholder domains. + from ..provenance.manager import DEFAULT_BASE_URI + self.namespaces: Dict[str, str] = { "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdfs": "http://www.w3.org/2000/01/rdf-schema#", "owl": "http://www.w3.org/2002/07/owl#", "xsd": "http://www.w3.org/2001/XMLSchema#", - "semantica": "https://semantica.dev/ns#", + "semantica": DEFAULT_BASE_URI, } self.config = config or {} diff --git a/semantica/graph_store/graph_store_provenance.py b/semantica/graph_store/graph_store_provenance.py index c3da635d..06d48fdd 100644 --- a/semantica/graph_store/graph_store_provenance.py +++ b/semantica/graph_store/graph_store_provenance.py @@ -13,37 +13,53 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class GraphStoreWithProvenance: """Graph store with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .graph_store import GraphStore - + self.provenance = provenance self._store = GraphStore(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def add_node(self, node: Any, source: str = None, **kwargs): """Add node with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._store.add_node(node, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: node_id = getattr(node, 'id', f"node_{uuid.uuid4().hex[:8]}") self._prov_manager.track_entity( entity_id=node_id, source=source or "graph_store", entity_type="graph_node", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={"properties": getattr(node, 'properties', {})} ) diff --git a/semantica/ingest/ingest_provenance.py b/semantica/ingest/ingest_provenance.py index b3a97747..51930fa4 100644 --- a/semantica/ingest/ingest_provenance.py +++ b/semantica/ingest/ingest_provenance.py @@ -14,16 +14,25 @@ License: MIT """ from typing import Optional, List +from datetime import datetime import uuid class IngestProvenanceMixin: """Mixin for ingest provenance tracking.""" - - def __init__(self, provenance: bool = False, **kwargs): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **kwargs, + ): self.provenance = provenance self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager @@ -34,17 +43,27 @@ class IngestProvenanceMixin: class PDFIngestorWithProvenance(IngestProvenanceMixin): """PDF ingestor with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .pdf_ingestor import PDFIngestor - - IngestProvenanceMixin.__init__(self, provenance=provenance) + + IngestProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._ingestor = PDFIngestor(**config) - + def ingest(self, file_path: str, **kwargs): """Ingest PDF with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() docs = self._ingestor.ingest(file_path, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: for doc in docs: doc_id = getattr(doc, 'id', f"doc_{uuid.uuid4().hex[:8]}") @@ -52,6 +71,11 @@ class PDFIngestorWithProvenance(IngestProvenanceMixin): entity_id=doc_id, source=file_path, entity_type="document", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "file_type": "pdf", "pages": getattr(doc, 'page_count', None) diff --git a/semantica/kg/kg_provenance.py b/semantica/kg/kg_provenance.py index 11a4807f..784386d0 100644 --- a/semantica/kg/kg_provenance.py +++ b/semantica/kg/kg_provenance.py @@ -54,6 +54,7 @@ Version: 1.0.0 """ from typing import Any, Dict, List, Optional +from datetime import datetime import uuid import time @@ -78,13 +79,21 @@ class GraphBuilderWithProvenance: }) """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .graph_builder import GraphBuilder - + self.provenance = provenance self._builder = GraphBuilder(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager @@ -94,12 +103,18 @@ class GraphBuilderWithProvenance: def build(self, sources, **kwargs): """Build graph with provenance tracking.""" - # Track the build operation + activity_started_at_time = datetime.utcnow().isoformat() + # Track the build operation (recorded before the build runs, so it + # has no end time yet — this is the "in progress" marker). if self.provenance and self._prov_manager: build_id = f"graph_build_{uuid.uuid4().hex[:8]}" self._prov_manager.track_entity( entity_id=build_id, source="graph_construction", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, metadata={ "entity_type": "graph_build_operation", "operation": "build_graph", @@ -107,16 +122,17 @@ class GraphBuilderWithProvenance: "timestamp": time.time() } ) - + result = self._builder.build(sources, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + # Track individual entities and relationships if available if self.provenance and self._prov_manager and hasattr(result, 'get'): try: # Try to extract entities and relationships for tracking entities = result.get('entities', []) relationships = result.get('relationships', []) - + for entity in entities: entity_id = entity.get('id') or str(entity.get('name', '')) if entity_id: @@ -124,6 +140,11 @@ class GraphBuilderWithProvenance: entity_id=entity_id, source="graph_construction", entity_type="graph_entity", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "operation": "build_entity", "entity_type": entity.get('type'), @@ -131,7 +152,7 @@ class GraphBuilderWithProvenance: "timestamp": time.time() } ) - + for relationship in relationships: rel_id = relationship.get('id') or f"{relationship.get('source', '')}-{relationship.get('target', '')}" if rel_id: @@ -139,6 +160,11 @@ class GraphBuilderWithProvenance: entity_id=rel_id, source="graph_construction", entity_type="graph_relationship", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "operation": "build_relationship", "relationship_type": relationship.get('type'), @@ -149,32 +175,39 @@ class GraphBuilderWithProvenance: except Exception as e: # Don't fail the build if provenance tracking fails pass - + return result - + def build_single_source(self, kg_data, **kwargs): """Build graph from single source with provenance tracking.""" - # Track the build operation + activity_started_at_time = datetime.utcnow().isoformat() + # Track the build operation (recorded before the build runs, so it + # has no end time yet — this is the "in progress" marker). if self.provenance and self._prov_manager: build_id = f"graph_build_single_{uuid.uuid4().hex[:8]}" self._prov_manager.track_entity( entity_id=build_id, source="graph_construction", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, metadata={ "entity_type": "graph_build_operation", "operation": "build_single_source", "timestamp": time.time() } ) - + result = self._builder.build_single_source(kg_data, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + # Track entities and relationships if available if self.provenance and self._prov_manager and isinstance(result, dict): try: entities = result.get('entities', []) relationships = result.get('relationships', []) - + for entity in entities: entity_id = entity.get('id') or str(entity.get('name', '')) if entity_id: @@ -182,6 +215,11 @@ class GraphBuilderWithProvenance: entity_id=entity_id, source="graph_construction", entity_type="graph_entity", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "operation": "build_entity", "entity_type": entity.get('type'), @@ -197,6 +235,11 @@ class GraphBuilderWithProvenance: entity_id=rel_id, source="graph_construction", entity_type="graph_relationship", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "operation": "build_relationship", "relationship_type": relationship.get('type'), @@ -247,24 +290,33 @@ class AlgorithmTrackerWithProvenance: ) """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): self.provenance = provenance self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def track_embedding_computation( self, graph: Any, algorithm: str, embeddings: Dict[str, List[float]], parameters: Dict[str, Any], - source: str = None + source: str = None, + **kwargs ): """ Track node embedding algorithm computation with provenance. @@ -286,6 +338,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=execution_id, source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "embedding_computation", "algorithm": algorithm, @@ -303,6 +360,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=f"embedding_{node_id}", source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "node_embedding", "algorithm": algorithm, @@ -345,6 +407,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=execution_id, source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "similarity_calculation", "algorithm": f"similarity_{method}", @@ -363,6 +430,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=f"similarity_{node_id}_{execution_id}", source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "similarity_result", "method": method, @@ -393,6 +465,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=execution_id, source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "link_prediction", "algorithm": f"link_prediction_{method}", @@ -410,6 +487,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=f"prediction_{execution_id}_{i}", source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "link_prediction_result", "method": method, @@ -453,6 +535,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=execution_id, source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "centrality_calculation", "algorithm": f"centrality_{method}", @@ -470,6 +557,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=f"centrality_{node_id}_{execution_id}", source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "centrality_score", "method": method, @@ -500,6 +592,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=execution_id, source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "community_detection", "algorithm": f"community_detection_{method}", @@ -517,6 +614,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=f"community_{execution_id}_{i}", source=source or "algorithm_execution", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "community", "method": method, @@ -547,6 +649,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=execution_id, source=source or "graph_construction", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "graph_construction", "entities_count": entities_count, @@ -573,6 +680,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "similarity_result", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "similarity_result", "node_id": node_id, @@ -599,6 +711,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "similarity_threshold", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "similarity_threshold_analysis", "execution_id": execution_id, @@ -624,6 +741,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "entity_processing", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "entity_processing", "processed_entity_id": entity_id, @@ -648,6 +770,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "relationship_processing", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "relationship_processing", "processed_relationship_id": relationship_id, @@ -672,6 +799,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "path_analysis", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "path_analysis", "paths_count": len(paths) if paths else 0, @@ -700,6 +832,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "path_finding", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "path_finding", "source_node": source_node, @@ -724,6 +861,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "embedding_analysis", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "embedding_analysis", "embeddings_count": len(embeddings), @@ -746,6 +888,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "connectivity_analysis", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "connectivity_analysis", "components_count": len(components), @@ -768,6 +915,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "cross_layer_analysis", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "cross_layer_analysis", "layers_count": len(cross_layer_results) if cross_layer_results else 0, @@ -793,6 +945,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "pipeline_summary", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "pipeline_summary", "pipeline_id": pipeline_id, @@ -820,6 +977,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=summary_id, source=source or "workflow_summary", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "workflow_summary", "master_workflow_id": master_workflow_id, @@ -848,6 +1010,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or "link_prediction_result", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={ "entity_type": "link_prediction_result", "source_node": source_node, @@ -869,6 +1036,11 @@ class AlgorithmTrackerWithProvenance: self._prov_manager.track_entity( entity_id=result_id, source=source or analysis_type, + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=kwargs.get("activity_started_at_time"), + activity_ended_at_time=kwargs.get("activity_ended_at_time"), metadata={"entity_type": analysis_type, "timestamp": time.time(), **{k: str(v)[:100] for k, v in kwargs.items() if not callable(v)}}, ) return result_id diff --git a/semantica/llms/llms_provenance.py b/semantica/llms/llms_provenance.py index 62b39ffa..f6277fff 100644 --- a/semantica/llms/llms_provenance.py +++ b/semantica/llms/llms_provenance.py @@ -42,6 +42,7 @@ License: MIT """ from typing import Optional, Dict, Any +from datetime import datetime import time import uuid @@ -54,17 +55,28 @@ class LLMProvenanceMixin: LLM API calls including tokens, costs, and performance metrics. """ - def __init__(self, provenance: bool = False, **kwargs): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **kwargs, + ): """ Initialize LLM provenance tracking. - + Args: provenance: Enable provenance tracking (default: False) + agent_id: Agent identifier for accountability (issue #825); defaults + to the wrapping class name + is_automated: Whether this agent acted without direct human review **kwargs: Additional arguments passed to parent class """ self.provenance = provenance self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager @@ -72,7 +84,7 @@ class LLMProvenanceMixin: except ImportError: # Graceful degradation if provenance module not available self.provenance = False - + def _track_llm_call( self, call_id: str, @@ -82,7 +94,7 @@ class LLMProvenanceMixin: ) -> None: """ Track LLM API call with provenance. - + Args: call_id: Unique identifier for this API call prompt: Input prompt @@ -98,11 +110,22 @@ class LLMProvenanceMixin: response_text = response.content elif not isinstance(response, str): response_text = str(response) - + + # Typed Activity timing (issue #825, Part B Tier 1): popped out so + # it populates real fields, not the opaque metadata blob. + activity_started_at_time = metadata.pop("activity_started_at_time", None) + activity_ended_at_time = metadata.pop("activity_ended_at_time", None) + self._prov_manager.track_entity( entity_id=call_id, source=f"{self.__class__.__name__}_api", entity_type="llm_generation", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_id=call_id, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "model": getattr(self, 'model', 'unknown'), "prompt_preview": prompt[:200] if len(prompt) > 200 else prompt, @@ -124,17 +147,25 @@ class GroqLLMWithProvenance(LLMProvenanceMixin): >>> # API call is tracked with model, tokens, cost, latency """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize Groq LLM with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original GroqLLM """ from .groq_llm import GroqLLM - - LLMProvenanceMixin.__init__(self, provenance=provenance) + + LLMProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._llm = GroqLLM(**config) self.model = getattr(self._llm, 'model', 'groq') @@ -150,22 +181,24 @@ class GroqLLMWithProvenance(LLMProvenanceMixin): LLM response (same format as original GroqLLM) """ start_time = time.time() + activity_started_at_time = datetime.utcnow().isoformat() response = self._llm.generate(prompt, **kwargs) elapsed = time.time() - start_time - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: # Extract token counts if available prompt_tokens = None completion_tokens = None total_cost = None - + if hasattr(response, 'usage'): prompt_tokens = getattr(response.usage, 'prompt_tokens', None) completion_tokens = getattr(response.usage, 'completion_tokens', None) - + if hasattr(response, 'cost'): total_cost = response.cost - + self._track_llm_call( call_id=f"groq_call_{uuid.uuid4().hex[:8]}", prompt=prompt, @@ -175,6 +208,8 @@ class GroqLLMWithProvenance(LLMProvenanceMixin): total_tokens=(prompt_tokens + completion_tokens) if (prompt_tokens and completion_tokens) else None, total_cost=total_cost, latency_seconds=elapsed, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, temperature=kwargs.get('temperature'), max_tokens=kwargs.get('max_tokens'), top_p=kwargs.get('top_p') @@ -194,17 +229,25 @@ class OpenAILLMWithProvenance(LLMProvenanceMixin): Wraps the original OpenAILLM and tracks all API calls. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize OpenAI LLM with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original OpenAILLM """ from .openai_llm import OpenAILLM - - LLMProvenanceMixin.__init__(self, provenance=provenance) + + LLMProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._llm = OpenAILLM(**config) self.model = getattr(self._llm, 'model', 'openai') @@ -220,22 +263,24 @@ class OpenAILLMWithProvenance(LLMProvenanceMixin): LLM response """ start_time = time.time() + activity_started_at_time = datetime.utcnow().isoformat() response = self._llm.generate(prompt, **kwargs) elapsed = time.time() - start_time - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: # Extract token counts if available prompt_tokens = None completion_tokens = None total_cost = None - + if hasattr(response, 'usage'): prompt_tokens = getattr(response.usage, 'prompt_tokens', None) completion_tokens = getattr(response.usage, 'completion_tokens', None) - + if hasattr(response, 'cost'): total_cost = response.cost - + self._track_llm_call( call_id=f"openai_call_{uuid.uuid4().hex[:8]}", prompt=prompt, @@ -245,6 +290,8 @@ class OpenAILLMWithProvenance(LLMProvenanceMixin): total_tokens=(prompt_tokens + completion_tokens) if (prompt_tokens and completion_tokens) else None, total_cost=total_cost, latency_seconds=elapsed, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, temperature=kwargs.get('temperature'), max_tokens=kwargs.get('max_tokens') ) @@ -263,17 +310,25 @@ class HuggingFaceLLMWithProvenance(LLMProvenanceMixin): Wraps the original HuggingFaceLLM and tracks all generations. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize HuggingFace LLM with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original HuggingFaceLLM """ from .huggingface_llm import HuggingFaceLLM - - LLMProvenanceMixin.__init__(self, provenance=provenance) + + LLMProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._llm = HuggingFaceLLM(**config) self.model = getattr(self._llm, 'model', 'huggingface') @@ -289,15 +344,19 @@ class HuggingFaceLLMWithProvenance(LLMProvenanceMixin): LLM response """ start_time = time.time() + activity_started_at_time = datetime.utcnow().isoformat() response = self._llm.generate(prompt, **kwargs) elapsed = time.time() - start_time - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: self._track_llm_call( call_id=f"hf_call_{uuid.uuid4().hex[:8]}", prompt=prompt, response=response, latency_seconds=elapsed, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, max_length=kwargs.get('max_length'), temperature=kwargs.get('temperature') ) @@ -316,17 +375,25 @@ class LiteLLMWithProvenance(LLMProvenanceMixin): Wraps the original LiteLLM and tracks all API calls across providers. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize LiteLLM with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original LiteLLM """ from .lite_llm import LiteLLM - - LLMProvenanceMixin.__init__(self, provenance=provenance) + + LLMProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._llm = LiteLLM(**config) self.model = getattr(self._llm, 'model', 'litellm') @@ -342,22 +409,24 @@ class LiteLLMWithProvenance(LLMProvenanceMixin): LLM response """ start_time = time.time() + activity_started_at_time = datetime.utcnow().isoformat() response = self._llm.generate(prompt, **kwargs) elapsed = time.time() - start_time - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: # LiteLLM provides unified response format prompt_tokens = None completion_tokens = None total_cost = None - + if hasattr(response, 'usage'): prompt_tokens = getattr(response.usage, 'prompt_tokens', None) completion_tokens = getattr(response.usage, 'completion_tokens', None) - + if hasattr(response, '_hidden_params') and 'response_cost' in response._hidden_params: total_cost = response._hidden_params['response_cost'] - + self._track_llm_call( call_id=f"lite_call_{uuid.uuid4().hex[:8]}", prompt=prompt, @@ -366,6 +435,8 @@ class LiteLLMWithProvenance(LLMProvenanceMixin): completion_tokens=completion_tokens, total_cost=total_cost, latency_seconds=elapsed, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, provider=kwargs.get('provider') ) diff --git a/semantica/normalize/normalize_provenance.py b/semantica/normalize/normalize_provenance.py index 37d87867..0dfc17b5 100644 --- a/semantica/normalize/normalize_provenance.py +++ b/semantica/normalize/normalize_provenance.py @@ -11,36 +11,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class NormalizerWithProvenance: """Normalizer with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .normalizer import Normalizer - + self.provenance = provenance self._normalizer = Normalizer(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def normalize(self, data: Any, source: str = None, **kwargs): """Normalize data with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._normalizer.normalize(data, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"normalize_{uuid.uuid4().hex[:8]}", source=source or "normalization", entity_type="normalized_data", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={"method": kwargs.get('method', 'default')} ) diff --git a/semantica/ontology/ontology_provenance.py b/semantica/ontology/ontology_provenance.py index 99c83440..fffbfa10 100644 --- a/semantica/ontology/ontology_provenance.py +++ b/semantica/ontology/ontology_provenance.py @@ -11,36 +11,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class OntologyManagerWithProvenance: """Ontology manager with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .ontology_manager import OntologyManager - + self.provenance = provenance self._manager = OntologyManager(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def add_concept(self, concept: Any, source: str = None, **kwargs): """Add concept with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._manager.add_concept(concept, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"concept_{uuid.uuid4().hex[:8]}", source=source or "ontology", entity_type="ontology_concept", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={"concept_name": str(concept)} ) diff --git a/semantica/parse/parse_provenance.py b/semantica/parse/parse_provenance.py index 6c288c18..6d18386b 100644 --- a/semantica/parse/parse_provenance.py +++ b/semantica/parse/parse_provenance.py @@ -13,36 +13,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class ParserWithProvenance: """Base parser with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .parser import Parser - + self.provenance = provenance self._parser = Parser(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def parse(self, file_path: str, **kwargs): """Parse file with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() data = self._parser.parse(file_path, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"parse_{uuid.uuid4().hex[:8]}", source=file_path, entity_type="parsed_data", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "file_path": file_path, "format": kwargs.get('format', 'unknown') diff --git a/semantica/pipeline/pipeline_provenance.py b/semantica/pipeline/pipeline_provenance.py index 3d5f34c4..b5c7c99b 100644 --- a/semantica/pipeline/pipeline_provenance.py +++ b/semantica/pipeline/pipeline_provenance.py @@ -16,6 +16,7 @@ License: MIT """ from typing import Optional, Any, Dict, List +from datetime import datetime import uuid import time @@ -23,34 +24,50 @@ import time class PipelineWithProvenance: """Pipeline executor with complete provenance tracking.""" - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """Initialize pipeline with optional provenance.""" from .pipeline import Pipeline - + self.provenance = provenance self._pipeline = Pipeline(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def run(self, data: Any, source: Optional[str] = None, **kwargs): """Run pipeline with provenance tracking.""" pipeline_id = f"pipeline_{uuid.uuid4().hex[:8]}" start_time = time.time() - + activity_started_at_time = datetime.utcnow().isoformat() + result = self._pipeline.run(data, **kwargs) elapsed = time.time() - start_time - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=pipeline_id, source=source or "pipeline_execution", entity_type="pipeline_run", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_id=pipeline_id, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "steps": len(self._pipeline.steps) if hasattr(self._pipeline, 'steps') else 0, "duration_seconds": elapsed, diff --git a/semantica/provenance/__init__.py b/semantica/provenance/__init__.py index c6faaea4..4362020a 100644 --- a/semantica/provenance/__init__.py +++ b/semantica/provenance/__init__.py @@ -41,7 +41,13 @@ Author: Semantica Contributors License: MIT """ -from .schemas import ProvenanceEntry, SourceReference +from .schemas import ( + ProvenanceEntry, + SourceReference, + AgentRecord, + ActivityRecord, + Invalidation, +) from .storage import ProvenanceStorage, InMemoryStorage, SQLiteStorage from .manager import ProvenanceManager, default_storage_path from .integrity import compute_checksum, verify_checksum @@ -50,7 +56,10 @@ __all__ = [ # Core schemas "ProvenanceEntry", "SourceReference", - + "AgentRecord", + "ActivityRecord", + "Invalidation", + # Storage backends "ProvenanceStorage", "InMemoryStorage", diff --git a/semantica/provenance/integrity.py b/semantica/provenance/integrity.py index e5327089..bedaaf0e 100644 --- a/semantica/provenance/integrity.py +++ b/semantica/provenance/integrity.py @@ -30,13 +30,36 @@ def compute_checksum(entry: Any) -> str: Creates a deterministic checksum based on critical provenance fields to detect any tampering or corruption of provenance data. - + + Includes `previous_checksum` (issue #825, Part A item 2), which chains + each entry to the prior entry in insertion order (see + ProvenanceStorage.get_chain_head / ProvenanceManager.verify_chain) — + wholesale deletion of a row breaks the chain for the entry that used to + follow it, making the deletion detectable even though per-row checksums + only prove a surviving row wasn't edited in place. Also includes + agent_id/agent_type and the lineage-link fields (parent_entity_id, + previous_version_id, derived_from_id, used_entities) so tampering with + attribution or lineage is detected, not just tampering with the six + original fields. + + Deliberately excludes `entity_id` itself: entity_id is the storage + primary key, and ProvenanceManager.track_entity()'s versioning archives + a prior value by copying it to a new entity_id (e.g. "X" -> "X:v:..."). + If entity_id were hashed, that relabeling would change the archived + copy's checksum, permanently orphaning any later entry whose + previous_checksum had already chained from the pre-relabel value — + a false-positive "broken chain" for a legitimate rename, not tampering. + Tampering that swaps a row's entity_id while keeping its content is a + narrower threat than content tampering, already partially caught by + verify_chain() (it requires a delete+insert, which breaks the chain for + whatever the deleted row's successor was). + Args: entry: ProvenanceEntry or dict to compute checksum for - + Returns: SHA-256 checksum as hexadecimal string - + Example: >>> entry = ProvenanceEntry( ... entity_id="entity_123", @@ -48,26 +71,48 @@ def compute_checksum(entry: Any) -> str: >>> print(checksum) 'a3b2c1d4e5f6...' """ - # Concatenate critical fields for checksum + # Concatenate critical fields for checksum (entity_id intentionally excluded, see docstring) if isinstance(entry, dict): + used_entities = entry.get("used_entities") or [] data = ( - f"{entry.get('entity_id') or ''}" f"{entry.get('entity_type') or ''}" f"{entry.get('activity_id') or ''}" + f"{entry.get('agent_id') or ''}" + f"{entry.get('agent_type') or ''}" f"{entry.get('source_document') or ''}" f"{entry.get('timestamp') or ''}" f"{entry.get('confidence') if entry.get('confidence') is not None else 1.0}" + f"{entry.get('parent_entity_id') or ''}" + f"{entry.get('previous_version_id') or ''}" + f"{entry.get('derived_from_id') or ''}" + f"{','.join(used_entities)}" + f"{entry.get('previous_checksum') or ''}" + f"{bool(entry.get('invalidated'))}" + f"{entry.get('invalidated_at_time') or ''}" + f"{entry.get('invalidated_by') or ''}" + f"{entry.get('invalidation_reason') or ''}" ) else: + used_entities = getattr(entry, "used_entities", None) or [] data = ( - f"{entry.entity_id}" f"{entry.entity_type}" f"{entry.activity_id}" + f"{getattr(entry, 'agent_id', '') or ''}" + f"{getattr(entry, 'agent_type', '') or ''}" f"{entry.source_document}" f"{entry.timestamp}" f"{entry.confidence}" + f"{getattr(entry, 'parent_entity_id', '') or ''}" + f"{getattr(entry, 'previous_version_id', '') or ''}" + f"{getattr(entry, 'derived_from_id', '') or ''}" + f"{','.join(used_entities)}" + f"{getattr(entry, 'previous_checksum', '') or ''}" + f"{bool(getattr(entry, 'invalidated', False))}" + f"{getattr(entry, 'invalidated_at_time', '') or ''}" + f"{getattr(entry, 'invalidated_by', '') or ''}" + f"{getattr(entry, 'invalidation_reason', '') or ''}" ) - + return hashlib.sha256(data.encode('utf-8')).hexdigest() diff --git a/semantica/provenance/manager.py b/semantica/provenance/manager.py index b2e9874c..8e411a17 100644 --- a/semantica/provenance/manager.py +++ b/semantica/provenance/manager.py @@ -33,11 +33,18 @@ import inspect import json import threading -from .schemas import ProvenanceEntry, SourceReference +from .schemas import ProvenanceEntry, SourceReference, AgentRecord, ActivityRecord from .storage import ProvenanceStorage, InMemoryStorage, SQLiteStorage from .integrity import compute_checksum, verify_checksum from ..utils.logging import get_logger +# Issue #825, Part B Tier 3 — configurable base URI for export_prov(), shared +# with RDFExporter's NamespaceManager "semantica" entry (semantica/export/ +# rdf_exporter.py) so KG-exported and PROV-exported URIs for the same +# entity_id co-resolve to the same namespace instead of two different +# placeholder domains. +DEFAULT_BASE_URI = "https://semantica.dev/ns#" + @contextmanager def default_storage_path(path: Optional[str]): @@ -146,6 +153,16 @@ class ProvenanceManager: _raise_on_error: bool = False, ) -> Optional[ProvenanceEntry]: """Compute checksum, store entry persistently, and log/handle storage errors (#783).""" + # Hash-chain linkage (issue #825, Part A item 2): link this entry to + # the previous entry in global insertion order before hashing, so + # deleting a row later breaks the chain for whatever followed it. + try: + head = self.storage.get_chain_head(_conn) + except Exception: + head = None + entry.sequence_id = (head[0] + 1) if head else 1 + entry.previous_checksum = head[1] if head else None + entry.checksum = compute_checksum(entry) try: @@ -178,6 +195,69 @@ class ProvenanceManager: with self.storage.transaction() as conn: yield conn + # Recognized typed kwargs for track_entity, used both directly and to + # split track_entities_batch's **metadata into real kwargs vs. free-form + # metadata (issue #825, Part A item 3 — fixes a bug where agent_id/ + # entity_type/activity_id passed to track_entities_batch were silently + # absorbed into the opaque metadata blob instead of populating fields). + _TRACK_ENTITY_KWARGS = frozenset({ + "entity_type", "activity_id", "agent_id", "agent_type", "is_automated", + "role", "agent", "source_location", "source_quote", "confidence", + "parent_entity_id", "used_entities", + # Part B Tier 1/2/3 (issue #825) + "activity", "activity_started_at_time", "activity_ended_at_time", + "acted_on_behalf_of", "informed_by", "valid_from", "valid_until", + "revision_type", "supersedes", "bundle_id", + }) + + @staticmethod + def _resolve_agent_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]: + """ + Resolve agent_id/agent_type/is_automated/role from kwargs. + + Accepts either an `agent=AgentRecord(...)` kwarg (takes precedence) + or individual `agent_id`/`agent_type`/`is_automated` scalar kwargs. + Fixes issue #825's dead agent_id field: previously no track_* + method read agent_id from kwargs at all, so it was always the + dataclass default "semantica" regardless of what callers passed. + """ + agent = kwargs.get("agent") + if isinstance(agent, AgentRecord): + return { + "agent_id": agent.id, + "agent_type": agent.agent_type, + "is_automated": agent.is_automated, + "role": kwargs.get("role"), + } + return { + "agent_id": kwargs.get("agent_id", "semantica"), + "agent_type": kwargs.get("agent_type", "software_agent"), + "is_automated": kwargs.get("is_automated", True), + "role": kwargs.get("role"), + } + + @staticmethod + def _resolve_activity_kwargs(kwargs: Dict[str, Any], default_activity_id: str) -> Dict[str, Any]: + """ + Resolve activity_id/activity_started_at_time/activity_ended_at_time + from kwargs (issue #825, Part B Tier 1 — typed Activity). + + Accepts either an `activity=ActivityRecord(...)` kwarg (takes + precedence) or individual scalar kwargs. + """ + activity = kwargs.get("activity") + if isinstance(activity, ActivityRecord): + return { + "activity_id": activity.id, + "activity_started_at_time": activity.started_at_time, + "activity_ended_at_time": activity.ended_at_time, + } + return { + "activity_id": kwargs.get("activity_id", default_activity_id), + "activity_started_at_time": kwargs.get("activity_started_at_time"), + "activity_ended_at_time": kwargs.get("activity_ended_at_time"), + } + # === Entity Tracking (from kg.ProvenanceTracker) === def track_entity( @@ -253,15 +333,31 @@ class ProvenanceManager: history_entry.entity_id = history_id + # Pure relabel: checksum/sequence_id/previous_checksum are + # left exactly as they were. compute_checksum() excludes + # entity_id specifically so this is safe — recomputing it + # here (or assigning a fresh sequence slot) would either + # invalidate any later entry that already chained from + # this row's checksum, or strand the original sequence + # position and make verify_chain() see a phantom gap + # (issue #825: an archival relabel is not tampering and + # must not look like it to the hash chain). self.storage._store_with_conn(conn, history_entry) archived_history_id = history_id if not explicit_parent_supplied: parent_id = history_id + agent_info = self._resolve_agent_kwargs(kwargs) + activity_info = self._resolve_activity_kwargs(kwargs, "entity_tracking") + entry = ProvenanceEntry( entity_id=entity_id, entity_type=kwargs.get("entity_type", "entity"), - activity_id=kwargs.get("activity_id", "entity_tracking"), + activity_id=activity_info["activity_id"], + agent_id=agent_info["agent_id"], + agent_type=agent_info["agent_type"], + is_automated=agent_info["is_automated"], + role=agent_info["role"], source_document=source, source_location=kwargs.get("source_location"), source_quote=kwargs.get("source_quote"), @@ -271,11 +367,32 @@ class ProvenanceManager: last_updated=datetime.utcnow().isoformat(), parent_entity_id=parent_id, used_entities=list(kwargs.get("used_entities", [])), + activity_started_at_time=activity_info["activity_started_at_time"], + activity_ended_at_time=activity_info["activity_ended_at_time"], + acted_on_behalf_of=kwargs.get("acted_on_behalf_of"), + informed_by_activities=list(kwargs.get("informed_by", [])), + valid_from=kwargs.get("valid_from"), + valid_until=kwargs.get("valid_until"), + revision_type=kwargs.get("revision_type"), + supersedes=kwargs.get("supersedes"), + bundle_id=kwargs.get("bundle_id"), ) + # Versioning vs. derivation (issue #825, Part A item 4): + # previous_version_id always captures "this corrects a prior + # version of the same fact" when one was archived, independent + # of whether an explicit cross-source parent was also given. + # derived_from_id captures "this fact was derived from a + # different source entity" — only set when a parent was + # explicitly resolved (kwarg/metadata['derived_from']/source + # heuristic), never from the automatic archival link. + entry.previous_version_id = archived_history_id + if explicit_parent_supplied: + entry.derived_from_id = parent_id + if archived_history_id and explicit_parent_supplied: entry.used_entities.append(archived_history_id) - + self._save_entry(entry, _conn=conn, _raise_on_error=True) except Exception as e: # When called from a batch's shared transaction (_conn is not None), @@ -323,18 +440,34 @@ class ProvenanceManager: ... metadata={"type": "founded"} ... ) """ + agent_info = self._resolve_agent_kwargs(kwargs) + activity_info = self._resolve_activity_kwargs(kwargs, "relationship_tracking") + entry = ProvenanceEntry( entity_id=relationship_id, entity_type="relationship", - activity_id=kwargs.get("activity_id", "relationship_tracking"), + activity_id=activity_info["activity_id"], + agent_id=agent_info["agent_id"], + agent_type=agent_info["agent_type"], + is_automated=agent_info["is_automated"], + role=agent_info["role"], source_document=source, source_location=kwargs.get("source_location"), confidence=kwargs.get("confidence", 1.0), metadata=metadata or {}, first_seen=datetime.utcnow().isoformat(), - last_updated=datetime.utcnow().isoformat() + last_updated=datetime.utcnow().isoformat(), + activity_started_at_time=activity_info["activity_started_at_time"], + activity_ended_at_time=activity_info["activity_ended_at_time"], + acted_on_behalf_of=kwargs.get("acted_on_behalf_of"), + informed_by_activities=list(kwargs.get("informed_by", [])), + valid_from=kwargs.get("valid_from"), + valid_until=kwargs.get("valid_until"), + revision_type=kwargs.get("revision_type"), + supersedes=kwargs.get("supersedes"), + bundle_id=kwargs.get("bundle_id"), ) - + return self._save_entry(entry) # === Chunk Tracking (from split.ProvenanceTracker) === @@ -374,19 +507,38 @@ class ProvenanceManager: ... end_index=500 ... ) """ + agent_info = self._resolve_agent_kwargs(metadata) + activity_info = self._resolve_activity_kwargs(metadata, "chunking") + for key in ( + "agent_id", "agent_type", "is_automated", "role", "agent", + "activity", "activity_started_at_time", "activity_ended_at_time", + ): + metadata.pop(key, None) + entry = ProvenanceEntry( entity_id=chunk_id, entity_type="chunk", - activity_id="chunking", + activity_id=activity_info["activity_id"], + agent_id=agent_info["agent_id"], + agent_type=agent_info["agent_type"], + is_automated=agent_info["is_automated"], + role=agent_info["role"], source_document=source_document, source_location=source_path, start_index=start_index, end_index=end_index, parent_entity_id=parent_chunk_id, + # A chunk split from a parent chunk is a derivation (a new entity + # produced from an existing one), not a correction of the same + # fact — see track_entity's previous_version_id/derived_from_id + # split (issue #825, Part A item 4). + derived_from_id=parent_chunk_id, metadata=metadata, - timestamp=datetime.utcnow().isoformat() + timestamp=datetime.utcnow().isoformat(), + activity_started_at_time=activity_info["activity_started_at_time"], + activity_ended_at_time=activity_info["activity_ended_at_time"], ) - + return self._save_entry(entry, _conn=_conn, _raise_on_error=(_conn is not None)) # === Source Tracking (from conflicts.SourceTracker) === @@ -425,10 +577,22 @@ class ProvenanceManager: ... source=source ... ) """ + agent_info = self._resolve_agent_kwargs(metadata) + activity_info = self._resolve_activity_kwargs(metadata, "property_tracking") + for key in ( + "agent_id", "agent_type", "is_automated", "role", "agent", + "activity", "activity_started_at_time", "activity_ended_at_time", + ): + metadata.pop(key, None) + entry = ProvenanceEntry( entity_id=f"{entity_id}_{property_name}", entity_type="property", - activity_id="property_tracking", + activity_id=activity_info["activity_id"], + agent_id=agent_info["agent_id"], + agent_type=agent_info["agent_type"], + is_automated=agent_info["is_automated"], + role=agent_info["role"], source_document=source.document, source_location=f"page_{source.page}" if source.page else source.section, confidence=source.confidence, @@ -440,9 +604,11 @@ class ProvenanceManager: **metadata, **source.metadata }, - timestamp=datetime.utcnow().isoformat() + timestamp=datetime.utcnow().isoformat(), + activity_started_at_time=activity_info["activity_started_at_time"], + activity_ended_at_time=activity_info["activity_ended_at_time"], ) - + return self._save_entry(entry) # === Batch Operations === @@ -473,7 +639,19 @@ class ProvenanceManager: """ tracked_count = 0 batch_size = 1000 # Justification (#807): 1,000 items per transaction bounds SQLite WAL frame growth and reduces lock contention during multi-thousand-row imports while achieving a 1000x reduction in connection/commit overhead. - + + # Split batch-level **metadata into recognized typed track_entity + # kwargs (entity_type, activity_id, agent_id, ...) vs. free-form data. + # Previously ALL of **metadata was merged into the metadata dict and + # passed as track_entity's positional `metadata` arg, so typed kwargs + # like agent_id/entity_type/activity_id were silently absorbed into + # the opaque metadata JSON blob instead of populating real fields + # (issue #825, Part A item 3 — this is what made agent_id a "dead" + # field for every batch caller, including the documented example in + # docs/guides/provenance.md). + batch_kwargs = {k: v for k, v in metadata.items() if k in self._TRACK_ENTITY_KWARGS} + free_metadata = {k: v for k, v in metadata.items() if k not in self._TRACK_ENTITY_KWARGS} + for i in range(0, len(entities), batch_size): batch = entities[i : i + batch_size] batch_count = 0 @@ -483,12 +661,14 @@ class ProvenanceManager: entity_id = entity.get("id") or entity.get("entity_id") if not entity_id: continue - - entity_metadata = {**metadata, **entity.get("metadata", {})} - + + entity_metadata = {**free_metadata, **entity.get("metadata", {})} + try: with self.storage.savepoint(conn): - self.track_entity(entity_id, source, entity_metadata, _conn=conn) + self.track_entity( + entity_id, source, entity_metadata, _conn=conn, **batch_kwargs + ) batch_count += 1 except Exception: pass # Continue with other entities in this batch @@ -654,11 +834,142 @@ class ProvenanceManager: if supports_max_depth: return self.storage.trace_lineage(entity_id, max_depth=max_depth) return self.storage.trace_lineage(entity_id) - + + def trace_descendants( + self, entity_id: str, max_depth: Optional[int] = None + ) -> List[ProvenanceEntry]: + """ + Trace downstream descendants (reverse lineage) and return raw entries. + + This is the counterpart to trace_lineage()/get_lineage(), which only + ever trace upstream ancestors via parent_entity_id/used_entities. + Downstream traceability answers the incident-response question "entity + X was wrong — what downstream facts used it?" (issue #825, Part A + item 5). + + Args: + entity_id: Entity identifier + max_depth: Optional maximum BFS depth + + Returns: + List of ProvenanceEntry objects that (transitively) reference + entity_id, in BFS order. + """ + return self.storage.trace_descendants(entity_id, max_depth=max_depth) + + def get_descendants(self, entity_id: str) -> Dict[str, Any]: + """ + Get downstream descendants for an entity, mirroring get_lineage()'s + return shape but for the reverse direction. + + Args: + entity_id: Entity identifier + + Returns: + Dictionary containing descendant entries, or {} if none found. + """ + descendant_entries = self.trace_descendants(entity_id) + + if not descendant_entries: + return {} + + integrity_verified = all(verify_checksum(entry) for entry in descendant_entries) + chain_dicts = [entry.to_dict() for entry in descendant_entries] + return { + "entity_id": entity_id, + "descendant_chain": chain_dicts, + "entries": chain_dicts, + "entity_count": len(descendant_entries), + "integrity_verified": integrity_verified, + } + + def revision_history(self, entity_id: str) -> List[Dict[str, Any]]: + """ + Return the version history for entity_id in ascending order. + + Issue #825, Part B Tier 3 — closes the "no direct equivalent yet" + gap for kg.ProvenanceTracker.revision_history() documented in + docs/migration/kg-provenance-tracker.md. + + Walks the entity's own previous_version_id chain (not the full + upstream lineage via get_lineage(), which also pulls in unrelated + derived_from_id/used_entities links from other source entities). + + Args: + entity_id: Entity identifier + + Returns: + List of {version, valid_from, valid_until, recorded_at, author, + revision_type, supersedes} dicts, oldest first. valid_from/ + valid_until use the entry's own explicit fields when set (the + fact's asserted validity window); otherwise valid_from defaults + to when this version was recorded, and valid_until to the next + version's timestamp (None for the current/most recent version). + Empty list if entity_id was never tracked. + """ + current = self.storage.retrieve(entity_id) + if current is None: + return [] + + chain: List[ProvenanceEntry] = [current] + visited = {entity_id} + cursor = current + while getattr(cursor, "previous_version_id", None): + prev_id = cursor.previous_version_id + if prev_id in visited: + break + prev_entry = self.storage.retrieve(prev_id) + if prev_entry is None: + break + chain.append(prev_entry) + visited.add(prev_id) + cursor = prev_entry + + chain.reverse() # oldest first + + history = [] + for i, entry in enumerate(chain): + default_valid_until = chain[i + 1].timestamp if i + 1 < len(chain) else None + version_dict: Dict[str, Any] = { + "version": i + 1, + "valid_from": entry.valid_from or entry.timestamp, + "valid_until": entry.valid_until or default_valid_until, + "recorded_at": entry.timestamp, + "author": entry.agent_id, + } + if entry.revision_type: + version_dict["revision_type"] = entry.revision_type + if entry.supersedes: + version_dict["supersedes"] = entry.supersedes + history.append(version_dict) + return history + + def query_recorded_between(self, start: str, end: str) -> List[Dict[str, Any]]: + """ + Return all provenance entries whose timestamp falls within [start, end]. + + Issue #825, Part B Tier 3 — closes the "no direct equivalent yet" + gap for kg.ProvenanceTracker.query_recorded_between() documented in + docs/migration/kg-provenance-tracker.md. + + Args: + start: Start of range, ISO 8601 string (inclusive) + end: End of range, ISO 8601 string (inclusive) + + Returns: + List of matching entries as dicts, sorted by timestamp ascending. + """ + matches = [ + e for e in self.storage.retrieve_all() + if e.timestamp and start <= e.timestamp <= end + ] + matches.sort(key=lambda e: e.timestamp) + return [e.to_dict() for e in matches] + def get_all_sources(self, entity_id: str) -> List[Dict[str, Any]]: """ Get all sources for an entity (kg.ProvenanceTracker compatible). - + Args: entity_id: Entity identifier @@ -694,13 +1005,93 @@ class ProvenanceManager: if entry: return entry.to_dict() return None - + + # === Invalidation (tombstone, not hard delete) === + + def invalidate( + self, + entity_id: str, + agent_id: str, + reason: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> ProvenanceEntry: + """ + Mark a tracked entity as invalidated instead of deleting it + (issue #825, Part A item 1 — prov:Invalidation). + + An audit needs "this was deleted/corrected, by whom, when, why" to + itself be provable. Rather than mutating the row in place, this + archives the pre-invalidation state under a stable versioned key + (the same pattern track_entity() uses for corrections) and then + writes the invalidated entry as a fresh, chained append. Mutating + the existing row's checksum in place would silently invalidate any + later entry that had already chained its previous_checksum from + this row's pre-invalidation value — turning a legitimate + invalidation into a false-positive "broken chain" report. + + The entry remains visible via retrieve()/retrieve_all()/lineage + traversal, but callers can filter on `invalidated` to exclude + retracted facts. + + Args: + entity_id: Entity identifier to invalidate + agent_id: Agent responsible for the invalidation (prov:Agent) + reason: Optional human-readable reason + metadata: Optional metadata to merge into the entry + + Returns: + The updated (invalidated) ProvenanceEntry + + Raises: + ValueError: If no provenance entry exists for entity_id + + Example: + >>> prov_mgr.invalidate("entity_1", agent_id="reviewer_jane", + ... reason="Source document retracted") + """ + with self.storage.transaction() as conn: + existing = self.storage._retrieve_with_conn(conn, entity_id) + if existing is None: + raise ValueError( + f"Cannot invalidate: no provenance entry found for entity_id={entity_id!r}" + ) + + # Archive the pre-invalidation state under a stable key — a pure + # relabel (see track_entity's identical pattern), so its + # checksum/sequence_id/previous_checksum are left untouched. + history_entry = copy.deepcopy(existing) + base_history_id = f"{entity_id}:v:{existing.last_updated}" + history_id = base_history_id + counter = 1 + while self.storage._retrieve_with_conn(conn, history_id): + history_id = f"{base_history_id}:{counter}" + counter += 1 + history_entry.entity_id = history_id + self.storage._store_with_conn(conn, history_entry) + + entry = copy.deepcopy(existing) + entry.invalidated = True + entry.invalidated_at_time = datetime.utcnow().isoformat() + entry.invalidated_by = agent_id + entry.invalidation_reason = reason + entry.previous_version_id = history_id + if metadata: + entry.metadata = {**entry.metadata, **metadata} + + self._save_entry(entry, _conn=conn, _raise_on_error=True) + + return entry + # === Utility Methods === - + def clear(self) -> int: """ Clear all provenance data. - + + Note: this is a bulk storage reset (used for dev/test teardown), not + a single-fact retraction — use invalidate() to retract/correct an + individual tracked entity while preserving its audit trail. + Returns: Number of entries cleared """ @@ -801,28 +1192,44 @@ class ProvenanceManager: ) return "\n".join(lines) - def export_prov(self, format: str = "turtle") -> str: + # Agent-type refinement for export_prov (issue #825, Part A item 6 — + # cheap, opportunistic PROV-O typing on top of the generic prov:Agent). + _AGENT_TYPE_PROV_CLASS = { + "person": "Person", + "software_agent": "SoftwareAgent", + "organization": "Organization", + } + + def export_prov(self, format: str = "turtle", base_uri: Optional[str] = None) -> str: """ Export provenance as W3C PROV-O RDF. Args: format: RDF format ('turtle', 'ntriples', 'jsonld') + base_uri: Namespace URI entities/agents/activities are minted + under (issue #825, Part B Tier 3). Defaults to + DEFAULT_BASE_URI, which matches RDFExporter's NamespaceManager + "semantica" entry so KG-exported and PROV-exported URIs for + the same entity_id co-resolve. Returns: Serialized RDF string """ - from rdflib import Graph, Literal, Namespace, URIRef + from rdflib import BNode, Graph, Literal, Namespace, URIRef from rdflib.namespace import RDF, XSD PROV = Namespace("http://www.w3.org/ns/prov#") - EX = Namespace("http://example.org/ns/") + EX = Namespace(base_uri or DEFAULT_BASE_URI) g = Graph() g.bind("prov", PROV) g.bind("ex", EX) + def uri(entity_id: Any) -> URIRef: + return URIRef(EX[str(entity_id)]) + for e in self.storage.retrieve_all(): - ent_uri = URIRef(EX[str(e.entity_id)]) + ent_uri = uri(e.entity_id) g.add((ent_uri, RDF.type, PROV.Entity)) if getattr(e, "timestamp", None): @@ -834,29 +1241,125 @@ class ProvenanceManager: ) ) + ag_uri = None if getattr(e, "agent_id", None) and e.agent_id != "unknown": - ag_uri = URIRef(EX[str(e.agent_id)]) + ag_uri = uri(e.agent_id) g.add((ag_uri, RDF.type, PROV.Agent)) + prov_subclass = self._AGENT_TYPE_PROV_CLASS.get( + getattr(e, "agent_type", None) + ) + if prov_subclass: + g.add((ag_uri, RDF.type, PROV[prov_subclass])) g.add((ent_uri, PROV.wasAttributedTo, ag_uri)) + # Qualified Association with hadRole (issue #825, Part A item + # 6): distinguishes "approved by" from "generated by" from + # "reviewed by" for the same agent/entity pair, which the + # plain wasAttributedTo triple above cannot express. + association = BNode() + g.add((ent_uri, PROV.qualifiedAssociation, association)) + g.add((association, RDF.type, PROV.Association)) + g.add((association, PROV.agent, ag_uri)) + role = getattr(e, "role", None) or "generator" + g.add((association, PROV.hadRole, uri(f"role_{role}"))) + + # prov:actedOnBehalfOf (issue #825, Part B Tier 2) — agent + # delegation, e.g. an automated agent acting on behalf of the + # human/organization that authorized it. + delegate_id = getattr(e, "acted_on_behalf_of", None) + if delegate_id: + delegate_uri = uri(delegate_id) + g.add((delegate_uri, RDF.type, PROV.Agent)) + g.add((ag_uri, PROV.actedOnBehalfOf, delegate_uri)) + + act_uri = None if getattr(e, "activity_id", None) and e.activity_id != "unknown": - act_uri = URIRef(EX[str(e.activity_id)]) + act_uri = uri(e.activity_id) g.add((act_uri, RDF.type, PROV.Activity)) g.add((ent_uri, PROV.wasGeneratedBy, act_uri)) + # Typed Activity timing (issue #825, Part B Tier 1) + if getattr(e, "activity_started_at_time", None): + g.add((act_uri, PROV.startedAtTime, + Literal(e.activity_started_at_time, datatype=XSD.dateTime))) + if getattr(e, "activity_ended_at_time", None): + g.add((act_uri, PROV.endedAtTime, + Literal(e.activity_ended_at_time, datatype=XSD.dateTime))) + + # Qualified Generation (issue #825, Part B Tier 1) + generation = BNode() + g.add((ent_uri, PROV.qualifiedGeneration, generation)) + g.add((generation, RDF.type, PROV.Generation)) + g.add((generation, PROV.activity, act_uri)) + if getattr(e, "timestamp", None): + g.add((generation, PROV.atTime, Literal(e.timestamp, datatype=XSD.dateTime))) + + # prov:wasAssociatedWith (issue #825, Part B Tier 2) — direct + # Activity->Agent link, distinct from the Entity->Agent + # wasAttributedTo/qualifiedAssociation triples above. + if ag_uri is not None: + g.add((act_uri, PROV.wasAssociatedWith, ag_uri)) + + # prov:wasInformedBy (issue #825, Part B Tier 2) — chains this + # activity to prior activities it was informed by (e.g. a + # pipeline stage informed by the stage before it). + for informing_id in getattr(e, "informed_by_activities", []): + g.add((act_uri, PROV.wasInformedBy, uri(informing_id))) + + def emit_derivation(source_id: Any) -> None: + """Emit plain + qualified wasDerivedFrom for a source entity.""" + s_uri = uri(source_id) + g.add((ent_uri, PROV.wasDerivedFrom, s_uri)) + derivation = BNode() + g.add((ent_uri, PROV.qualifiedDerivation, derivation)) + g.add((derivation, RDF.type, PROV.Derivation)) + g.add((derivation, PROV.entity, s_uri)) + if act_uri is not None: + g.add((derivation, PROV.hadActivity, act_uri)) + if getattr(e, "parent_entity_id", None): - p_uri = URIRef(EX[str(e.parent_entity_id)]) - g.add((ent_uri, PROV.wasDerivedFrom, p_uri)) + emit_derivation(e.parent_entity_id) for u_id in getattr(e, "used_entities", []): - u_uri = URIRef(EX[str(u_id)]) + u_uri = uri(u_id) # Emit wasDerivedFrom only when this used entity is not the same # as parent_entity_id — which already carries that triple above. if u_id != getattr(e, "parent_entity_id", None): - g.add((ent_uri, PROV.wasDerivedFrom, u_uri)) - if getattr(e, "activity_id", None) and e.activity_id != "unknown": - act_uri = URIRef(EX[str(e.activity_id)]) + emit_derivation(u_id) + if act_uri is not None: g.add((act_uri, PROV.used, u_uri)) + # Qualified Usage (issue #825, Part B Tier 1) + usage = BNode() + g.add((act_uri, PROV.qualifiedUsage, usage)) + g.add((usage, RDF.type, PROV.Usage)) + g.add((usage, PROV.entity, u_uri)) + + # Qualified Invalidation (issue #825, Part A item 1): records the + # tombstone as provable RDF rather than a silent hard delete. + if getattr(e, "invalidated", False): + invalidation = BNode() + g.add((ent_uri, PROV.qualifiedInvalidation, invalidation)) + g.add((invalidation, RDF.type, PROV.Invalidation)) + if getattr(e, "invalidated_at_time", None): + g.add( + ( + invalidation, + PROV.invalidatedAtTime, + Literal(e.invalidated_at_time, datatype=XSD.dateTime), + ) + ) + if getattr(e, "invalidated_by", None): + inv_ag_uri = uri(e.invalidated_by) + g.add((inv_ag_uri, RDF.type, PROV.Agent)) + g.add((invalidation, PROV.agent, inv_ag_uri)) + + # prov:Collection / prov:Bundle membership (issue #825, Part B + # Tier 3) — partitions provenance by source/dataset/ingestion-run. + # Membership triples, not true RDF named-graph partitioning. + if getattr(e, "bundle_id", None): + bundle_uri = uri(f"bundle_{e.bundle_id}") + g.add((bundle_uri, RDF.type, PROV.Bundle)) + g.add((bundle_uri, PROV.hadMember, ent_uri)) rdf_format = "json-ld" if format == "jsonld" else format return g.serialize(format=rdf_format) @@ -873,6 +1376,7 @@ class ProvenanceManager: """ entries = self.storage.retrieve_all() all_ids = {e.entity_id for e in entries} + all_activity_ids = {e.activity_id for e in entries if getattr(e, "activity_id", None)} missing_refs = [] for e in entries: @@ -882,15 +1386,77 @@ class ProvenanceManager: for u_id in getattr(e, "used_entities", []): if u_id not in all_ids: missing_refs.append(f"{e.entity_id} -> {u_id}") + for ref_id in ( + getattr(e, "previous_version_id", None), + getattr(e, "derived_from_id", None), + getattr(e, "supersedes", None), + ): + if ref_id and ref_id not in all_ids: + missing_refs.append(f"{e.entity_id} -> {ref_id}") + # informed_by_activities references activity_ids, a distinct + # ID space from entity_id (issue #825, Part B Tier 2). + for act_id in getattr(e, "informed_by_activities", []): + if act_id not in all_activity_ids: + missing_refs.append(f"{e.entity_id} (activity) -> {act_id}") valid = len(missing_refs) == 0 errors = len(missing_refs) + invalidated_count = sum(1 for e in entries if getattr(e, "invalidated", False)) return { "valid": valid, "total_entries": len(entries), "missing_references": missing_refs, + "invalidated_count": invalidated_count, "strict": strict, "errors": errors, } + def verify_chain(self) -> Dict[str, Any]: + """ + Verify the hash chain across all provenance entries (issue #825, + Part A item 2). + + Sorts entries by sequence_id (global insertion order) and checks + both that each entry's own checksum matches its content, and that + each entry's previous_checksum matches the checksum of the entry + that precedes it. A gap in the chain — a surviving entry whose + previous_checksum doesn't match its predecessor's checksum — is + exactly what wholesale row deletion produces, since per-row + checksums alone can't detect that a row is simply missing. + + Returns: + Dictionary with "valid", "total_entries", and "broken_links" + (list of {"entity_id", "sequence_id", "reason"} dicts). + """ + entries = sorted( + (e for e in self.storage.retrieve_all() if e.sequence_id is not None), + key=lambda e: e.sequence_id, + ) + + broken_links: List[Dict[str, Any]] = [] + expected_previous: Optional[str] = None + for entry in entries: + if not verify_checksum(entry): + broken_links.append({ + "entity_id": entry.entity_id, + "sequence_id": entry.sequence_id, + "reason": "checksum_mismatch", + }) + continue + if entry.previous_checksum != expected_previous: + broken_links.append({ + "entity_id": entry.entity_id, + "sequence_id": entry.sequence_id, + "reason": "chain_break", + "expected_previous_checksum": expected_previous, + "actual_previous_checksum": entry.previous_checksum, + }) + expected_previous = entry.checksum + + return { + "valid": len(broken_links) == 0, + "total_entries": len(entries), + "broken_links": broken_links, + } + diff --git a/semantica/provenance/schemas.py b/semantica/provenance/schemas.py index 85853a83..df888db4 100644 --- a/semantica/provenance/schemas.py +++ b/semantica/provenance/schemas.py @@ -13,10 +13,14 @@ Consolidates: W3C PROV-O Mapping: - ProvenanceEntry.entity_id → prov:Entity - ProvenanceEntry.activity_id → prov:Activity - - ProvenanceEntry.agent_id → prov:Agent - - ProvenanceEntry.parent_entity_id → prov:wasDerivedFrom + - ProvenanceEntry.agent_id / agent_type → prov:Agent (Person|SoftwareAgent|Organization) + - ProvenanceEntry.role → prov:hadRole (via prov:qualifiedAssociation) + - ProvenanceEntry.parent_entity_id → prov:wasDerivedFrom (legacy combined field) + - ProvenanceEntry.derived_from_id → prov:wasDerivedFrom (true cross-source derivation) + - ProvenanceEntry.previous_version_id → prior version of the same fact (correction/versioning) - ProvenanceEntry.used_entities → prov:used - ProvenanceEntry.timestamp → prov:generatedAtTime + - ProvenanceEntry.invalidated* → prov:Invalidation (tombstone, not a hard delete) Author: Semantica Contributors License: MIT @@ -75,36 +79,89 @@ class ProvenanceEntry: entity_type: str activity_id: str agent_id: str = "semantica" - + + # Accountability-scoped agent typing (issue #825, Part A item 3) + agent_type: str = "software_agent" # "person" | "software_agent" | "organization" + is_automated: bool = True + role: Optional[str] = None # prov:hadRole, e.g. "generator", "approver", "reviewer" + # Audit-grade source tracking source_document: str = "" source_location: Optional[str] = None source_quote: Optional[str] = None - + # Temporal tracking (from kg.ProvenanceTracker) timestamp: str = field(default_factory=lambda: datetime.utcnow().isoformat()) first_seen: Optional[str] = None last_updated: Optional[str] = None - + # Quality metrics confidence: float = 1.0 checksum: Optional[str] = None - - # Chain of custody (W3C PROV-O) + + # Hash-chain linkage (issue #825, Part A item 2): previous_checksum links + # this entry to the prior entry in global insertion order (sequence_id), + # so wholesale row deletion breaks the chain and becomes detectable via + # ProvenanceManager.verify_chain(). + sequence_id: Optional[int] = None + previous_checksum: Optional[str] = None + + # Chain of custody (W3C PROV-O) — kept for backward compatibility. parent_entity_id: Optional[str] = None used_entities: List[str] = field(default_factory=list) - + + # Versioning vs. derivation (issue #825, Part A item 4): previous_version_id + # is "this corrects/replaces a prior version of the same fact" + # (prov:specializationOf-flavored); derived_from_id is "this fact was + # derived from a different source entity" (prov:wasDerivedFrom). Both are + # additive alongside parent_entity_id, which remains the legacy combined + # field for existing readers. + previous_version_id: Optional[str] = None + derived_from_id: Optional[str] = None + + # Typed Activity (issue #825, Part B Tier 1): prov:startedAtTime/endedAtTime + # for the activity that generated this entry. + activity_started_at_time: Optional[str] = None + activity_ended_at_time: Optional[str] = None + + # prov:actedOnBehalfOf / prov:wasInformedBy (issue #825, Part B Tier 2) + acted_on_behalf_of: Optional[str] = None + informed_by_activities: List[str] = field(default_factory=list) + + # Bitemporal fields merged from the deprecated kg.ProvenanceTracker + # (issue #825, Part B Tier 3) — in the deprecated tracker these were + # always caller-supplied optional metadata keys, never auto-computed; + # same contract here. valid_from/valid_until express the fact's own + # asserted validity window (domain/business time), independent of + # sequence_id/timestamp which track when the system recorded it. + valid_from: Optional[str] = None + valid_until: Optional[str] = None + revision_type: Optional[str] = None + supersedes: Optional[str] = None + + # prov:Collection / prov:Bundle membership (issue #825, Part B Tier 3): + # partitions provenance by source/dataset/ingestion-run. + bundle_id: Optional[str] = None + + # Invalidation / tombstone tracking (issue #825, Part A item 1): set via + # ProvenanceManager.invalidate() instead of deleting the row, so an audit + # can prove a fact existed, was reviewed, and was retracted/corrected. + invalidated: bool = False + invalidated_at_time: Optional[str] = None + invalidated_by: Optional[str] = None + invalidation_reason: Optional[str] = None + # Chunk-specific fields (from split.ProvenanceInfo) start_index: Optional[int] = None end_index: Optional[int] = None - + # Source credibility (from conflicts.SourceTracker) credibility: Optional[float] = None - + # Metadata metadata: Dict[str, Any] = field(default_factory=dict) version: str = "1.0" - + def to_dict(self) -> Dict[str, Any]: """ Convert provenance entry to dictionary. @@ -117,6 +174,9 @@ class ProvenanceEntry: "entity_type": self.entity_type, "activity_id": self.activity_id, "agent_id": self.agent_id, + "agent_type": self.agent_type, + "is_automated": self.is_automated, + "role": self.role, "source_document": self.source_document, "source_location": self.source_location, "source_quote": self.source_quote, @@ -125,8 +185,25 @@ class ProvenanceEntry: "last_updated": self.last_updated, "confidence": self.confidence, "checksum": self.checksum, + "sequence_id": self.sequence_id, + "previous_checksum": self.previous_checksum, "parent_entity_id": self.parent_entity_id, "used_entities": self.used_entities, + "previous_version_id": self.previous_version_id, + "derived_from_id": self.derived_from_id, + "activity_started_at_time": self.activity_started_at_time, + "activity_ended_at_time": self.activity_ended_at_time, + "acted_on_behalf_of": self.acted_on_behalf_of, + "informed_by_activities": self.informed_by_activities, + "valid_from": self.valid_from, + "valid_until": self.valid_until, + "revision_type": self.revision_type, + "supersedes": self.supersedes, + "bundle_id": self.bundle_id, + "invalidated": self.invalidated, + "invalidated_at_time": self.invalidated_at_time, + "invalidated_by": self.invalidated_by, + "invalidation_reason": self.invalidation_reason, "start_index": self.start_index, "end_index": self.end_index, "credibility": self.credibility, @@ -275,3 +352,144 @@ class PropertySource: for s in data["sources"] ] return cls(**data) + + +@dataclass +class AgentRecord: + """ + Minimum-viable typed W3C PROV-O agent (prov:Agent). + + Narrower than the full PROV-O Person/SoftwareAgent/Organization taxonomy — + just enough to answer "was a human accountable here" for high-stakes + provenance review. Pass an AgentRecord to ProvenanceManager.track_entity() + (etc.) via the `agent=` kwarg to set agent_id/agent_type/is_automated + together. + + Attributes: + id: Agent identifier (prov:Agent) + agent_type: One of "person", "software_agent", "organization" + is_automated: Whether this agent acted without direct human review + name: Optional human-readable name + metadata: Additional metadata dictionary + + Example: + >>> agent = AgentRecord(id="reviewer_jane", agent_type="person", is_automated=False) + >>> prov_mgr.track_entity("entity_1", source="doc_1", agent=agent, role="approver") + """ + + id: str + agent_type: str = "software_agent" + is_automated: bool = True + name: Optional[str] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + """Convert agent record to dictionary.""" + return { + "id": self.id, + "agent_type": self.agent_type, + "is_automated": self.is_automated, + "name": self.name, + "metadata": self.metadata, + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "AgentRecord": + """Create agent record from dictionary.""" + return cls(**data) + + +@dataclass +class ActivityRecord: + """ + Minimum-viable typed W3C PROV-O activity (prov:Activity). + + Issue #825, Part B Tier 1 — promotes activity_id from a bare string to + a typed object carrying start/end timing, mirroring AgentRecord. Pass an + ActivityRecord to ProvenanceManager.track_entity() (etc.) via the + `activity=` kwarg to set activity_id/activity_started_at_time/ + activity_ended_at_time together. + + Attributes: + id: Activity identifier (prov:Activity) + activity_type: Free-form activity classification (e.g. "process", "extraction") + started_at_time: prov:startedAtTime, ISO datetime string + ended_at_time: prov:endedAtTime, ISO datetime string + metadata: Additional metadata dictionary + + Example: + >>> activity = ActivityRecord( + ... id="bureau_parsing_run_42", + ... started_at_time="2026-01-01T00:00:00", + ... ended_at_time="2026-01-01T00:00:03", + ... ) + >>> prov_mgr.track_entity("entity_1", source="doc_1", activity=activity) + """ + + id: str + activity_type: str = "process" + started_at_time: Optional[str] = None + ended_at_time: Optional[str] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + """Convert activity record to dictionary.""" + return { + "id": self.id, + "activity_type": self.activity_type, + "started_at_time": self.started_at_time, + "ended_at_time": self.ended_at_time, + "metadata": self.metadata, + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "ActivityRecord": + """Create activity record from dictionary.""" + return cls(**data) + + +@dataclass +class Invalidation: + """ + W3C PROV-O invalidation record (prov:Invalidation). + + Represents the tombstone for a retracted/corrected provenance entry: + "this was deleted/corrected, by whom, when, why" — recorded in place of + a hard delete so the fact that an entity existed remains provable. + + Attributes: + entity_id: The entity that was invalidated (prov:Entity) + invalidated_at_time: When invalidation occurred (prov:invalidatedAtTime) + invalidated_by: Agent responsible for the invalidation (prov:Agent) + reason: Optional human-readable reason + metadata: Additional metadata dictionary + + Example: + >>> Invalidation( + ... entity_id="entity_123", + ... invalidated_at_time="2026-01-01T00:00:00", + ... invalidated_by="reviewer_jane", + ... reason="Source document retracted", + ... ) + """ + + entity_id: str + invalidated_at_time: str + invalidated_by: str + reason: Optional[str] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + """Convert invalidation record to dictionary.""" + return { + "entity_id": self.entity_id, + "invalidated_at_time": self.invalidated_at_time, + "invalidated_by": self.invalidated_by, + "reason": self.reason, + "metadata": self.metadata, + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "Invalidation": + """Create invalidation record from dictionary.""" + return cls(**data) diff --git a/semantica/provenance/storage.py b/semantica/provenance/storage.py index 7da91482..768bc263 100644 --- a/semantica/provenance/storage.py +++ b/semantica/provenance/storage.py @@ -20,7 +20,7 @@ License: MIT """ from abc import ABC, abstractmethod -from typing import List, Optional, Dict, Any +from typing import List, Optional, Dict, Any, Tuple import sqlite3 import json import uuid @@ -123,6 +123,70 @@ class ProvenanceStorage(ABC): """Internal retrieve method using an active connection/transaction.""" return self.retrieve(entity_id) + def get_chain_head(self, conn: Any = None) -> Optional[Tuple[int, str]]: + """ + Return (sequence_id, checksum) of the most recently stored entry, or + None if the store is empty or this backend doesn't support chaining. + + Used by ProvenanceManager._save_entry to link each new entry to its + predecessor (issue #825, Part A item 2 — hash-chained integrity). + Default implementation reports no chaining support; override in + backends that maintain sequence_id. + """ + return None + + def trace_descendants( + self, entity_id: str, max_depth: Optional[int] = None + ) -> List[ProvenanceEntry]: + """ + Trace downstream descendants (reverse lineage) of an entity. + + Generic default implemented purely against retrieve_all(): builds a + reverse adjacency map from parent_entity_id/previous_version_id/ + derived_from_id/used_entities, then BFS from entity_id. Correct for + any backend; SQLiteStorage overrides this with indexed queries. + + Args: + entity_id: Entity identifier to find descendants of + max_depth: Optional maximum BFS depth + + Returns: + List of ProvenanceEntry objects that (transitively) reference + entity_id, in BFS order. + """ + reverse_index: Dict[str, List[ProvenanceEntry]] = {} + for entry in self.retrieve_all(): + referenced = set( + filter( + None, + [ + entry.parent_entity_id, + entry.previous_version_id, + entry.derived_from_id, + ], + ) + ) + referenced.update(entry.used_entities or []) + for ref_id in referenced: + reverse_index.setdefault(ref_id, []).append(entry) + + descendants = [] + visited = {entity_id} + queue = deque([(entity_id, 0)]) + + while queue: + current_id, depth = queue.popleft() + if max_depth is not None and depth >= max_depth: + continue + for child in reverse_index.get(current_id, []): + if child.entity_id in visited: + continue + visited.add(child.entity_id) + descendants.append(child) + queue.append((child.entity_id, depth + 1)) + + return descendants + class InMemoryStorage(ProvenanceStorage): """ @@ -150,6 +214,11 @@ class InMemoryStorage(ProvenanceStorage): """Initialize in-memory storage.""" self._entries: Dict[str, ProvenanceEntry] = {} self._local = threading.local() + # Hash-chain state (issue #825, Part A item 2). Best-effort: this is + # the dev/test backend, not the durable audit backend (SQLiteStorage). + self._seq_lock = threading.Lock() + self._seq_counter = 0 + self._chain_head: Optional[str] = None def _get_pending_stack(self) -> list: if not hasattr(self._local, "pending_stack"): @@ -159,10 +228,14 @@ class InMemoryStorage(ProvenanceStorage): def store(self, entry: ProvenanceEntry) -> None: """ Store a provenance entry in memory. - + Args: entry: ProvenanceEntry to store """ + with self._seq_lock: + if entry.sequence_id is not None: + self._seq_counter = max(self._seq_counter, entry.sequence_id) + self._chain_head = entry.checksum self._entries[entry.entity_id] = entry def retrieve(self, entity_id: str) -> Optional[ProvenanceEntry]: @@ -286,6 +359,31 @@ class InMemoryStorage(ProvenanceStorage): return pending[entity_id] return self.retrieve(entity_id) + def get_chain_head(self, conn: Any = None) -> Optional[Tuple[int, str]]: + """Return (sequence_id, checksum) of the most recent entry, including + entries staged but not yet committed within the current transaction. + + Must compare against the *committed* head too, not just the pending + stack: a transaction can stage an unchanged relabel (track_entity's + archival, or invalidate()'s pre-invalidation archive) whose + sequence_id is older than the true head from a prior, already- + committed transaction. Blindly preferring "last staged entry" + understates the head and corrupts the chain for the next append. + """ + with self._seq_lock: + best_seq = self._seq_counter + best_checksum = self._chain_head + for pending in self._get_pending_stack(): + for staged_entry in pending.values(): + if staged_entry.sequence_id is not None and ( + best_checksum is None or staged_entry.sequence_id > best_seq + ): + best_seq = staged_entry.sequence_id + best_checksum = staged_entry.checksum + if best_checksum is None: + return None + return (best_seq, best_checksum) + class SQLiteStorage(ProvenanceStorage): """ @@ -461,26 +559,71 @@ class SQLiteStorage(ProvenanceStorage): end_index INTEGER, credibility REAL, metadata TEXT, - version TEXT DEFAULT '1.0' + version TEXT DEFAULT '1.0', + agent_type TEXT DEFAULT 'software_agent', + is_automated INTEGER DEFAULT 1, + role TEXT, + sequence_id INTEGER, + previous_checksum TEXT, + previous_version_id TEXT, + derived_from_id TEXT, + invalidated INTEGER DEFAULT 0, + invalidated_at_time TEXT, + invalidated_by TEXT, + invalidation_reason TEXT, + activity_started_at_time TEXT, + activity_ended_at_time TEXT, + acted_on_behalf_of TEXT, + informed_by_activities TEXT, + valid_from TEXT, + valid_until TEXT, + revision_type TEXT, + supersedes TEXT, + bundle_id TEXT ) """) - + # Create indexes for efficient querying cursor.execute(""" - CREATE INDEX IF NOT EXISTS idx_entity_type + CREATE INDEX IF NOT EXISTS idx_entity_type ON provenance(entity_type) """) - + cursor.execute(""" - CREATE INDEX IF NOT EXISTS idx_source_document + CREATE INDEX IF NOT EXISTS idx_source_document ON provenance(source_document) """) - + cursor.execute(""" - CREATE INDEX IF NOT EXISTS idx_parent_entity + CREATE INDEX IF NOT EXISTS idx_parent_entity ON provenance(parent_entity_id) """) - + + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_previous_version_id + ON provenance(previous_version_id) + """) + + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_derived_from_id + ON provenance(derived_from_id) + """) + + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_sequence_id + ON provenance(sequence_id) + """) + + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_invalidated + ON provenance(invalidated) + """) + + cursor.execute(""" + CREATE INDEX IF NOT EXISTS idx_bundle_id + ON provenance(bundle_id) + """) + conn.commit() finally: conn.close() @@ -500,7 +643,9 @@ class SQLiteStorage(ProvenanceStorage): cursor = conn.cursor() cursor.execute(""" INSERT OR REPLACE INTO provenance VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ? ) """, ( entry.entity_id, @@ -521,7 +666,27 @@ class SQLiteStorage(ProvenanceStorage): entry.end_index, entry.credibility, json.dumps(entry.metadata), - entry.version + entry.version, + entry.agent_type, + 1 if entry.is_automated else 0, + entry.role, + entry.sequence_id, + entry.previous_checksum, + entry.previous_version_id, + entry.derived_from_id, + 1 if entry.invalidated else 0, + entry.invalidated_at_time, + entry.invalidated_by, + entry.invalidation_reason, + entry.activity_started_at_time, + entry.activity_ended_at_time, + entry.acted_on_behalf_of, + json.dumps(entry.informed_by_activities), + entry.valid_from, + entry.valid_until, + entry.revision_type, + entry.supersedes, + entry.bundle_id, )) def retrieve(self, entity_id: str) -> Optional[ProvenanceEntry]: @@ -634,9 +799,123 @@ class SQLiteStorage(ProvenanceStorage): frontier = next_frontier depth += 1 - + return lineage - + + def get_chain_head(self, conn: Any = None) -> Optional[Tuple[int, str]]: + """Return (sequence_id, checksum) of the most recently stored entry. + + Reuses the caller's transaction connection when given, so the head + read and the subsequent insert happen atomically under the existing + BEGIN IMMEDIATE writer lock (issue #825, Part A item 2). + """ + # Tie-break on rowid (SQLite's implicit row-order column) in addition + # to sequence_id: during track_entity's archival, the just-relabeled + # history row briefly shares its sequence_id with the still-present + # (about-to-be-overwritten) live row it was copied from. rowid DESC + # deterministically picks the most-recently-written row of the two, + # which is the correct chain head — relying on sequence_id ordering + # alone leaves the tie-break implementation-defined. + query = ( + "SELECT sequence_id, checksum FROM provenance " + "WHERE sequence_id IS NOT NULL " + "ORDER BY sequence_id DESC, rowid DESC LIMIT 1" + ) + if conn is not None: + row = conn.cursor().execute(query).fetchone() + return (row[0], row[1]) if row else None + with self._read_connection() as read_conn: + row = read_conn.cursor().execute(query).fetchone() + return (row[0], row[1]) if row else None + + def trace_descendants( + self, entity_id: str, max_depth: Optional[int] = None + ) -> List[ProvenanceEntry]: + """ + Trace downstream descendants (reverse lineage) using indexed lookups + on parent_entity_id/previous_version_id/derived_from_id, plus an + in-memory reverse index over used_entities (a JSON column, so not + directly indexable in SQLite). + + Args: + entity_id: Entity identifier to find descendants of + max_depth: Optional maximum BFS depth + + Returns: + List of ProvenanceEntry objects that (transitively) reference + entity_id, in BFS order. + """ + descendants: List[ProvenanceEntry] = [] + visited = {entity_id} + frontier = [entity_id] + depth = 0 + + with self._read_connection() as conn: + cursor = conn.cursor() + + cursor.execute( + "SELECT entity_id, used_entities FROM provenance " + "WHERE used_entities IS NOT NULL AND used_entities != '[]'" + ) + used_reverse: Dict[str, List[str]] = {} + for child_id, used_json in cursor.fetchall(): + try: + used_ids = json.loads(used_json) if used_json else [] + except (json.JSONDecodeError, TypeError): + used_ids = [] + for uid in used_ids: + used_reverse.setdefault(uid, []).append(child_id) + + while frontier and (max_depth is None or depth < max_depth): + current_frontier = list(dict.fromkeys(frontier)) + if not current_frontier: + break + + child_ids = set() + for i in range(0, len(current_frontier), 999): + chunk = current_frontier[i : i + 999] + placeholders = ",".join("?" * len(chunk)) + cursor.execute( + f""" + SELECT entity_id FROM provenance + WHERE parent_entity_id IN ({placeholders}) + OR previous_version_id IN ({placeholders}) + OR derived_from_id IN ({placeholders}) + """, + chunk * 3, + ) + child_ids.update(row[0] for row in cursor.fetchall()) + + for ref_id in current_frontier: + child_ids.update(used_reverse.get(ref_id, [])) + + next_frontier = [cid for cid in child_ids if cid not in visited] + if not next_frontier: + break + visited.update(next_frontier) + + entries_map = {} + for i in range(0, len(next_frontier), 999): + chunk = next_frontier[i : i + 999] + placeholders = ",".join("?" * len(chunk)) + cursor.execute( + f"SELECT * FROM provenance WHERE entity_id IN ({placeholders})", + chunk, + ) + for row in cursor.fetchall(): + entry = self._row_to_entry(row) + entries_map[entry.entity_id] = entry + + for cid in next_frontier: + entry = entries_map.get(cid) + if entry: + descendants.append(entry) + + frontier = next_frontier + depth += 1 + + return descendants + def clear(self) -> int: """ Clear all provenance data. @@ -687,5 +966,25 @@ class SQLiteStorage(ProvenanceStorage): end_index=row[15], credibility=row[16], metadata=json.loads(row[17]) if row[17] else {}, - version=row[18] + version=row[18], + agent_type=row[19] or "software_agent", + is_automated=bool(row[20]) if row[20] is not None else True, + role=row[21], + sequence_id=row[22], + previous_checksum=row[23], + previous_version_id=row[24], + derived_from_id=row[25], + invalidated=bool(row[26]) if row[26] is not None else False, + invalidated_at_time=row[27], + invalidated_by=row[28], + invalidation_reason=row[29], + activity_started_at_time=row[30], + activity_ended_at_time=row[31], + acted_on_behalf_of=row[32], + informed_by_activities=json.loads(row[33]) if row[33] else [], + valid_from=row[34], + valid_until=row[35], + revision_type=row[36], + supersedes=row[37], + bundle_id=row[38], ) diff --git a/semantica/reasoning/reasoning_provenance.py b/semantica/reasoning/reasoning_provenance.py index 970d960f..40d4418a 100644 --- a/semantica/reasoning/reasoning_provenance.py +++ b/semantica/reasoning/reasoning_provenance.py @@ -13,36 +13,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class ReasoningEngineWithProvenance: """Reasoning engine with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .reasoning_engine import ReasoningEngine - + self.provenance = provenance self._engine = ReasoningEngine(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def infer(self, premises: Any, source: str = None, **kwargs): """Perform inference with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._engine.infer(premises, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"inference_{uuid.uuid4().hex[:8]}", source=source or "reasoning_engine", entity_type="inference", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "premises_count": len(premises) if hasattr(premises, '__len__') else 1, "confidence": getattr(result, 'confidence', None) diff --git a/semantica/semantic_extract/semantic_extract_provenance.py b/semantica/semantic_extract/semantic_extract_provenance.py index 49b088b3..e66dfd32 100644 --- a/semantica/semantic_extract/semantic_extract_provenance.py +++ b/semantica/semantic_extract/semantic_extract_provenance.py @@ -37,6 +37,7 @@ License: MIT """ from typing import Optional, List, Dict, Any +from datetime import datetime import uuid @@ -48,17 +49,28 @@ class ProvenanceMixin: added to any extraction class without modifying its core functionality. """ - def __init__(self, provenance: bool = False, **kwargs): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **kwargs, + ): """ Initialize provenance tracking. - + Args: provenance: Enable provenance tracking (default: False) + agent_id: Agent identifier for accountability (issue #825); defaults + to the wrapping class name + is_automated: Whether this agent acted without direct human review **kwargs: Additional arguments passed to parent class """ self.provenance = provenance self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager @@ -66,7 +78,7 @@ class ProvenanceMixin: except ImportError: # Graceful degradation if provenance module not available self.provenance = False - + def _track_extraction( self, entity_id: str, @@ -76,7 +88,7 @@ class ProvenanceMixin: ) -> None: """ Track extraction with provenance. - + Args: entity_id: Unique identifier for extracted entity source: Source document or text @@ -84,10 +96,17 @@ class ProvenanceMixin: **metadata: Additional metadata to track """ if self.provenance and self._prov_manager: + activity_started_at_time = metadata.pop("activity_started_at_time", None) + activity_ended_at_time = metadata.pop("activity_ended_at_time", None) self._prov_manager.track_entity( entity_id=entity_id, source=source, entity_type=entity_type, + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata=metadata ) @@ -105,17 +124,25 @@ class NERExtractorWithProvenance(ProvenanceMixin): >>> # Each entity is tracked with source, confidence, and metadata """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize NER extractor with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original NERExtractor """ from .ner_extractor import NERExtractor - - ProvenanceMixin.__init__(self, provenance=provenance) + + ProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._extractor = NERExtractor(**config) def extract(self, text: str, source: Optional[str] = None, **kwargs): @@ -130,8 +157,10 @@ class NERExtractorWithProvenance(ProvenanceMixin): Returns: List of extracted entities (same as original NERExtractor) """ + activity_started_at_time = datetime.utcnow().isoformat() entities = self._extractor.extract(text, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: for entity in entities: entity_id = getattr(entity, 'id', None) @@ -141,7 +170,7 @@ class NERExtractorWithProvenance(ProvenanceMixin): entity.id = entity_id except AttributeError: pass - + self._track_extraction( entity_id=entity_id, source=source or text[:100], @@ -150,7 +179,9 @@ class NERExtractorWithProvenance(ProvenanceMixin): label=entity.label, confidence=getattr(entity, 'confidence', 1.0), start=entity.start, - end=entity.end + end=entity.end, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, ) return entities @@ -167,17 +198,25 @@ class RelationExtractorWithProvenance(ProvenanceMixin): Wraps the original RelationExtractor and tracks all extracted relations. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize relation extractor with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original RelationExtractor """ from .relation_extractor import RelationExtractor - - ProvenanceMixin.__init__(self, provenance=provenance) + + ProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._extractor = RelationExtractor(**config) def extract(self, text: str, source: Optional[str] = None, **kwargs): @@ -192,8 +231,10 @@ class RelationExtractorWithProvenance(ProvenanceMixin): Returns: List of extracted relations """ + activity_started_at_time = datetime.utcnow().isoformat() relations = self._extractor.extract(text, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: for relation in relations: relation_id = getattr(relation, 'id', None) @@ -203,7 +244,7 @@ class RelationExtractorWithProvenance(ProvenanceMixin): relation.id = relation_id except AttributeError: pass - + self._track_extraction( entity_id=relation_id, source=source or text[:100], @@ -211,7 +252,9 @@ class RelationExtractorWithProvenance(ProvenanceMixin): subject=relation.subject, predicate=relation.predicate, object=relation.object, - confidence=getattr(relation, 'confidence', 1.0) + confidence=getattr(relation, 'confidence', 1.0), + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, ) return relations @@ -228,17 +271,25 @@ class EventDetectorWithProvenance(ProvenanceMixin): Wraps the original EventDetector and tracks all detected events. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize event detector with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original EventDetector """ from .event_detector import EventDetector - - ProvenanceMixin.__init__(self, provenance=provenance) + + ProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._detector = EventDetector(**config) def detect(self, text: str, source: Optional[str] = None, **kwargs): @@ -253,8 +304,10 @@ class EventDetectorWithProvenance(ProvenanceMixin): Returns: List of detected events """ + activity_started_at_time = datetime.utcnow().isoformat() events = self._detector.detect(text, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: for event in events: event_id = getattr(event, 'id', None) @@ -264,14 +317,16 @@ class EventDetectorWithProvenance(ProvenanceMixin): event.id = event_id except AttributeError: pass - + self._track_extraction( entity_id=event_id, source=source or text[:100], entity_type="event", event_type=event.type, trigger=event.trigger, - confidence=getattr(event, 'confidence', 1.0) + confidence=getattr(event, 'confidence', 1.0), + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, ) return events @@ -288,17 +343,25 @@ class CoreferenceResolverWithProvenance(ProvenanceMixin): Wraps the original CoreferenceResolver and tracks coreference chains. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize coreference resolver with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original CoreferenceResolver """ from .coreference_resolver import CoreferenceResolver - - ProvenanceMixin.__init__(self, provenance=provenance) + + ProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._resolver = CoreferenceResolver(**config) def resolve(self, text: str, source: Optional[str] = None, **kwargs): @@ -313,8 +376,10 @@ class CoreferenceResolverWithProvenance(ProvenanceMixin): Returns: Coreference chains """ + activity_started_at_time = datetime.utcnow().isoformat() chains = self._resolver.resolve(text, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: for chain in chains: chain_id = getattr(chain, 'id', None) @@ -324,12 +389,14 @@ class CoreferenceResolverWithProvenance(ProvenanceMixin): chain.id = chain_id except AttributeError: pass - + self._track_extraction( entity_id=chain_id, source=source or text[:100], entity_type="coreference_chain", - mentions=len(chain.mentions) if hasattr(chain, 'mentions') else 0 + mentions=len(chain.mentions) if hasattr(chain, 'mentions') else 0, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, ) return chains @@ -346,17 +413,25 @@ class TripletExtractorWithProvenance(ProvenanceMixin): Wraps the original TripletExtractor and tracks all extracted triplets. """ - def __init__(self, provenance: bool = False, **config): + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): """ Initialize triplet extractor with optional provenance. - + Args: provenance: Enable provenance tracking (default: False) **config: Configuration passed to original TripletExtractor """ from .triplet_extractor import TripletExtractor - - ProvenanceMixin.__init__(self, provenance=provenance) + + ProvenanceMixin.__init__( + self, provenance=provenance, agent_id=agent_id, is_automated=is_automated + ) self._extractor = TripletExtractor(**config) def extract(self, text: str, source: Optional[str] = None, **kwargs): @@ -371,8 +446,10 @@ class TripletExtractorWithProvenance(ProvenanceMixin): Returns: List of extracted triplets """ + activity_started_at_time = datetime.utcnow().isoformat() triplets = self._extractor.extract(text, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance: for triplet in triplets: triplet_id = getattr(triplet, 'id', None) @@ -382,7 +459,7 @@ class TripletExtractorWithProvenance(ProvenanceMixin): triplet.id = triplet_id except AttributeError: pass - + self._track_extraction( entity_id=triplet_id, source=source or text[:100], @@ -390,7 +467,9 @@ class TripletExtractorWithProvenance(ProvenanceMixin): subject=triplet.subject, predicate=triplet.predicate, object=triplet.object, - confidence=getattr(triplet, 'confidence', 1.0) + confidence=getattr(triplet, 'confidence', 1.0), + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, ) return triplets diff --git a/semantica/triplet_store/triplet_store_provenance.py b/semantica/triplet_store/triplet_store_provenance.py index 2ac58cff..83595965 100644 --- a/semantica/triplet_store/triplet_store_provenance.py +++ b/semantica/triplet_store/triplet_store_provenance.py @@ -13,36 +13,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class TripletStoreWithProvenance: """Triplet store with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .triplet_store import TripletStore - + self.provenance = provenance self._store = TripletStore(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def add_triplet(self, subject: Any, predicate: Any, obj: Any, source: str = None, **kwargs): """Add triplet with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._store.add_triplet(subject, predicate, obj, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"triplet_{uuid.uuid4().hex[:8]}", source=source or "triplet_store", entity_type="triplet", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "subject": str(subject), "predicate": str(predicate), diff --git a/semantica/vector_store/vector_store_provenance.py b/semantica/vector_store/vector_store_provenance.py index a540074f..fc7a5e91 100644 --- a/semantica/vector_store/vector_store_provenance.py +++ b/semantica/vector_store/vector_store_provenance.py @@ -13,36 +13,52 @@ Author: Semantica Contributors License: MIT """ -from typing import List, Any +from typing import List, Any, Optional +from datetime import datetime import uuid class VectorStoreWithProvenance: """Vector store with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .vector_store import VectorStore - + self.provenance = provenance self._store = VectorStore(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def add_vectors(self, vectors: List[Any], source: str = None, **kwargs): """Add vectors with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._store.add_vectors(vectors, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"vectors_{uuid.uuid4().hex[:8]}", source=source or "vector_store", entity_type="vector_collection", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={ "count": len(vectors), "dimensions": len(vectors[0]) if vectors else 0 diff --git a/semantica/visualization/visualization_provenance.py b/semantica/visualization/visualization_provenance.py index 82e9d7b2..780cef4e 100644 --- a/semantica/visualization/visualization_provenance.py +++ b/semantica/visualization/visualization_provenance.py @@ -11,36 +11,52 @@ Author: Semantica Contributors License: MIT """ -from typing import Any +from typing import Any, Optional +from datetime import datetime import uuid class VisualizerWithProvenance: """Visualizer with provenance tracking.""" - - def __init__(self, provenance: bool = False, **config): + + def __init__( + self, + provenance: bool = False, + agent_id: Optional[str] = None, + is_automated: bool = True, + **config, + ): from .visualizer import Visualizer - + self.provenance = provenance self._visualizer = Visualizer(**config) self._prov_manager = None - + self._agent_id = agent_id or self.__class__.__name__ + self._is_automated = is_automated + if provenance: try: from semantica.provenance import ProvenanceManager self._prov_manager = ProvenanceManager() except ImportError: self.provenance = False - + def visualize(self, data: Any, output: str = None, **kwargs): """Visualize data with provenance tracking.""" + activity_started_at_time = datetime.utcnow().isoformat() result = self._visualizer.visualize(data, output=output, **kwargs) - + activity_ended_at_time = datetime.utcnow().isoformat() + if self.provenance and self._prov_manager: self._prov_manager.track_entity( entity_id=f"viz_{uuid.uuid4().hex[:8]}", source="visualization", entity_type="visualization", + agent_id=self._agent_id, + agent_type="software_agent", + is_automated=self._is_automated, + activity_started_at_time=activity_started_at_time, + activity_ended_at_time=activity_ended_at_time, metadata={"output": output, "type": kwargs.get('type', 'unknown')} ) diff --git a/tests/provenance/test_manager.py b/tests/provenance/test_manager.py index 77f61b79..e8c7f44a 100644 --- a/tests/provenance/test_manager.py +++ b/tests/provenance/test_manager.py @@ -913,5 +913,614 @@ class TestProvenanceManager: assert kwargs.get("exc_info") is True +class TestAgentTyping: + """Issue #825, Part A item 3 — agent_id/agent_type/is_automated actually + populate on tracked entries (previously a dead field: no track_* method + read agent_id from kwargs at all).""" + + def test_track_entity_scalar_agent_kwargs(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_entity( + "e1", source="doc1", + agent_id="alice", agent_type="person", is_automated=False, role="approver", + ) + assert entry.agent_id == "alice" + assert entry.agent_type == "person" + assert entry.is_automated is False + assert entry.role == "approver" + + def test_track_entity_agent_record_kwarg(self): + from semantica.provenance import AgentRecord + prov_mgr = ProvenanceManager() + agent = AgentRecord(id="reviewer_bob", agent_type="person", is_automated=False) + entry = prov_mgr.track_entity("e1", source="doc1", agent=agent, role="approver") + assert entry.agent_id == "reviewer_bob" + assert entry.agent_type == "person" + assert entry.is_automated is False + assert entry.role == "approver" + + def test_track_entity_default_agent_unchanged(self): + """No agent kwargs supplied should still default to 'semantica' (back-compat).""" + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_entity("e1", source="doc1") + assert entry.agent_id == "semantica" + assert entry.agent_type == "software_agent" + assert entry.is_automated is True + + def test_track_relationship_agent_kwargs(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_relationship("r1", source="doc1", agent_id="bot1") + assert entry.agent_id == "bot1" + + def test_track_chunk_agent_kwargs_not_leaked_into_metadata(self): + """agent_id/agent_type/is_automated passed to track_chunk must populate + the real fields, not leak into the opaque metadata blob.""" + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_chunk( + "c1", source_document="doc1", agent_id="chunker_v2", note="hello" + ) + assert entry.agent_id == "chunker_v2" + assert "agent_id" not in entry.metadata + assert entry.metadata.get("note") == "hello" + + def test_track_property_source_agent_kwargs(self): + prov_mgr = ProvenanceManager() + source = SourceReference(document="doc1") + entry = prov_mgr.track_property_source( + "e1", "prop", "value", source, agent_id="prop_tracker" + ) + assert entry.agent_id == "prop_tracker" + assert "agent_id" not in entry.metadata + + def test_track_entities_batch_agent_id_not_swallowed_into_metadata(self): + """Regression test for the documented bug: track_entities_batch used to + merge agent_id/entity_type/activity_id into the metadata dict instead + of forwarding them as real track_entity kwargs.""" + prov_mgr = ProvenanceManager() + count = prov_mgr.track_entities_batch( + [{"id": "b1"}, {"id": "b2"}], + source="doc_batch", + agent_id="batch_service_v2", + entity_type="credit_feature", + activity_id="bureau_parsing", + extra_note="kept as free-form metadata", + ) + assert count == 2 + for eid in ("b1", "b2"): + entry = prov_mgr.storage.retrieve(eid) + assert entry.agent_id == "batch_service_v2" + assert entry.entity_type == "credit_feature" + assert entry.activity_id == "bureau_parsing" + assert "agent_id" not in entry.metadata + assert entry.metadata.get("extra_note") == "kept as free-form metadata" + + +class TestVersioningVsDerivation: + """Issue #825, Part A item 4 — previous_version_id (correction) is + additive alongside derived_from_id (cross-source derivation); both are + populated without disturbing the legacy parent_entity_id field.""" + + def test_retrack_without_parent_sets_previous_version_id_only(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("X", source="doc_1") + v2 = prov_mgr.track_entity("X", source="doc_1") + + assert v2.previous_version_id is not None + assert v2.previous_version_id.startswith("X:v:") + assert v2.derived_from_id is None + # Legacy field behavior is unchanged + assert v2.parent_entity_id == v2.previous_version_id + + def test_retrack_with_explicit_parent_sets_both_fields(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("X", source="doc_1") + v2 = prov_mgr.track_entity("X", source="doc_1", parent_entity_id="explicit_parent") + + assert v2.derived_from_id == "explicit_parent" + assert v2.previous_version_id is not None + assert v2.previous_version_id.startswith("X:v:") + # Legacy field keeps explicit-wins semantics + assert v2.parent_entity_id == "explicit_parent" + + def test_track_chunk_split_sets_derived_from_id(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_chunk( + "c2", source_document="doc1", parent_chunk_id="c1" + ) + assert entry.derived_from_id == "c1" + assert entry.previous_version_id is None + assert entry.parent_entity_id == "c1" + + +class TestInvalidation: + """Issue #825, Part A item 1 — tombstone instead of hard delete.""" + + def test_invalidate_marks_entry_without_deleting(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + + result = prov_mgr.invalidate("e1", agent_id="reviewer_jane", reason="retracted") + + assert result.invalidated is True + assert result.invalidated_by == "reviewer_jane" + assert result.invalidation_reason == "retracted" + assert result.invalidated_at_time is not None + + # Entry remains visible via retrieve (tombstone, not delete) + stored = prov_mgr.storage.retrieve("e1") + assert stored is not None + assert stored.invalidated is True + + def test_invalidate_unknown_entity_raises(self): + prov_mgr = ProvenanceManager() + with pytest.raises(ValueError): + prov_mgr.invalidate("never_tracked", agent_id="reviewer_jane") + + def test_check_reports_invalidated_count(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("e2", source="doc1") + prov_mgr.invalidate("e1", agent_id="reviewer_jane") + + result = prov_mgr.check() + assert result["invalidated_count"] == 1 + + +class TestHashChain: + """Issue #825, Part A item 2 — hash-chained integrity: chained checksums + detect wholesale row deletion, which per-row checksums alone cannot.""" + + def test_verify_chain_valid_on_clean_history(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("e2", source="doc1") + prov_mgr.track_entity("X", source="doc1") + prov_mgr.track_entity("X", source="doc1") # triggers archival + prov_mgr.track_entity("X", source="doc1", parent_entity_id="explicit_p") + + result = prov_mgr.verify_chain() + assert result["valid"] is True + assert result["broken_links"] == [] + assert result["total_entries"] == 5 + + def test_verify_chain_detects_deleted_row(self, tmp_path): + """A hard delete of a row must break the chain for whatever followed it.""" + import sqlite3 + db_path = str(tmp_path / "chain.db") + prov_mgr = ProvenanceManager(storage_path=db_path) + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("e2", source="doc1") + prov_mgr.track_entity("e3", source="doc1") + + conn = sqlite3.connect(db_path) + conn.execute("DELETE FROM provenance WHERE entity_id = 'e2'") + conn.commit() + conn.close() + + prov_mgr2 = ProvenanceManager(storage_path=db_path) + result = prov_mgr2.verify_chain() + assert result["valid"] is False + assert any(link["reason"] == "chain_break" for link in result["broken_links"]) + + def test_sequence_ids_are_assigned_and_monotonic(self): + prov_mgr = ProvenanceManager() + e1 = prov_mgr.track_entity("e1", source="doc1") + e2 = prov_mgr.track_entity("e2", source="doc1") + assert e1.sequence_id is not None + assert e2.sequence_id is not None + assert e2.sequence_id > e1.sequence_id + assert e2.previous_checksum == e1.checksum + + def test_chain_survives_interleaved_retrack_and_invalidate(self, tmp_path): + """Regression test: an entry that already chained its previous_checksum + from another entity's checksum (Y -> X) must stay valid even after X + is later retracked (archived/relabeled) and then invalidated — both + of which write NEW entries under X's canonical key. Mutating X's + existing row in place (rather than archiving-then-appending) used to + silently orphan Y's chain link, producing a false-positive break.""" + db_path = str(tmp_path / "interleaved.db") + prov_mgr = ProvenanceManager(storage_path=db_path) + + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("X", source="doc1") + prov_mgr.track_entity("Y", source="doc1", parent_entity_id="X") + prov_mgr.track_entity("X", source="doc1") # retrack: archives old X + prov_mgr.track_entity("Z", source="doc1", parent_entity_id="X") + prov_mgr.invalidate("X", agent_id="reviewer") # archives again + prov_mgr.track_entity("W", source="doc1", parent_entity_id="X") + + result = prov_mgr.verify_chain() + assert result["valid"] is True + assert result["broken_links"] == [] + + def test_invalidate_does_not_break_chain(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("e2", source="doc1", parent_entity_id="e1") + prov_mgr.invalidate("e1", agent_id="reviewer_jane") + + result = prov_mgr.verify_chain() + assert result["valid"] is True + + +class TestDownstreamLineage: + """Issue #825, Part A item 5 — downstream/descendant traversal (reverse + BFS), complementing the existing upstream-only trace_lineage/get_lineage.""" + + def test_get_descendants_finds_direct_child(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("parent1", source="doc1") + prov_mgr.track_entity( + "child1", source="doc1", parent_entity_id="parent1", used_entities=["parent1"] + ) + + result = prov_mgr.get_descendants("parent1") + entity_ids = {e["entity_id"] for e in result["entries"]} + assert "child1" in entity_ids + + def test_get_descendants_transitive(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("a", source="doc1") + prov_mgr.track_entity("b", source="doc1", parent_entity_id="a") + prov_mgr.track_entity("c", source="doc1", parent_entity_id="b") + + result = prov_mgr.get_descendants("a") + entity_ids = {e["entity_id"] for e in result["entries"]} + assert entity_ids == {"b", "c"} + + def test_get_descendants_empty_for_leaf(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("leaf", source="doc1") + assert prov_mgr.get_descendants("leaf") == {} + + def test_descendants_work_on_sqlite_backend(self, tmp_path): + db_path = str(tmp_path / "desc.db") + prov_mgr = ProvenanceManager(storage_path=db_path) + prov_mgr.track_entity("parent1", source="doc1") + prov_mgr.track_entity("child1", source="doc1", parent_entity_id="parent1") + + result = prov_mgr.get_descendants("parent1") + entity_ids = {e["entity_id"] for e in result["entries"]} + assert "child1" in entity_ids + + +class TestQualifiedExport: + """Issue #825, Part A item 6 — qualified Association with hadRole, plus + qualified Invalidation, in the RDF export.""" + + def test_export_prov_includes_qualified_association_and_role(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity( + "e1", source="doc1", agent_id="alice", agent_type="person", role="approver" + ) + ttl = prov_mgr.export_prov(format="turtle") + assert "qualifiedAssociation" in ttl + assert "hadRole" in ttl + assert "role_approver" in ttl + assert "Person" in ttl + + def test_export_prov_includes_qualified_invalidation(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.invalidate("e1", agent_id="reviewer_jane", reason="retracted") + + ttl = prov_mgr.export_prov(format="turtle") + assert "qualifiedInvalidation" in ttl + assert "Invalidation" in ttl + + +class TestTypedActivity: + """Issue #825, Part B Tier 1 — typed Activity via ActivityRecord.""" + + def test_track_entity_activity_record_kwarg(self): + from semantica.provenance import ActivityRecord + prov_mgr = ProvenanceManager() + activity = ActivityRecord( + id="bureau_parsing_run_42", + started_at_time="2026-01-01T00:00:00", + ended_at_time="2026-01-01T00:00:03", + ) + entry = prov_mgr.track_entity("e1", source="doc1", activity=activity) + assert entry.activity_id == "bureau_parsing_run_42" + assert entry.activity_started_at_time == "2026-01-01T00:00:00" + assert entry.activity_ended_at_time == "2026-01-01T00:00:03" + + def test_track_entity_activity_scalar_kwargs(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_entity( + "e1", source="doc1", + activity_id="parse_step", activity_started_at_time="t0", activity_ended_at_time="t1", + ) + assert entry.activity_id == "parse_step" + assert entry.activity_started_at_time == "t0" + assert entry.activity_ended_at_time == "t1" + + def test_export_prov_qualified_generation_usage_derivation(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("parent1", source="doc1") + prov_mgr.track_entity( + "child1", source="doc1", + parent_entity_id="parent1", used_entities=["parent1"], + activity_id="transform", + ) + ttl = prov_mgr.export_prov(format="turtle") + assert "qualifiedGeneration" in ttl + assert "Generation" in ttl + assert "qualifiedUsage" in ttl + assert "Usage" in ttl + assert "qualifiedDerivation" in ttl + assert "Derivation" in ttl + + def test_export_prov_activity_timing(self): + from semantica.provenance import ActivityRecord + prov_mgr = ProvenanceManager() + prov_mgr.track_entity( + "e1", source="doc1", + activity=ActivityRecord(id="act1", started_at_time="2026-01-01T00:00:00", + ended_at_time="2026-01-01T00:00:05"), + ) + ttl = prov_mgr.export_prov(format="turtle") + assert "startedAtTime" in ttl + assert "endedAtTime" in ttl + + +class TestAssociationDelegationChaining: + """Issue #825, Part B Tier 2 — wasAssociatedWith, actedOnBehalfOf, wasInformedBy.""" + + def test_acted_on_behalf_of(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_entity( + "e1", source="doc1", agent_id="bot1", acted_on_behalf_of="org1" + ) + assert entry.acted_on_behalf_of == "org1" + ttl = prov_mgr.export_prov(format="turtle") + assert "actedOnBehalfOf" in ttl + + def test_informed_by_activities(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_entity( + "e1", source="doc1", activity_id="parse", informed_by=["ingest_activity"] + ) + assert entry.informed_by_activities == ["ingest_activity"] + ttl = prov_mgr.export_prov(format="turtle") + assert "wasInformedBy" in ttl + + def test_was_associated_with_in_export(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1", agent_id="alice", activity_id="act1") + ttl = prov_mgr.export_prov(format="turtle") + assert "wasAssociatedWith" in ttl + + def test_track_entities_batch_forwards_tier2_kwargs(self): + """Regression test for the batch-kwargs-vs-metadata bug (issue #825, + Part A) extended to the new Tier 2/3 keys.""" + prov_mgr = ProvenanceManager() + count = prov_mgr.track_entities_batch( + [{"id": "b1"}], + source="doc_batch", + activity_id="bureau_parsing", + acted_on_behalf_of="org1", + informed_by=["ingest_activity"], + bundle_id="run_1", + ) + assert count == 1 + entry = prov_mgr.storage.retrieve("b1") + assert entry.activity_id == "bureau_parsing" + assert entry.acted_on_behalf_of == "org1" + assert entry.informed_by_activities == ["ingest_activity"] + assert entry.bundle_id == "run_1" + assert "acted_on_behalf_of" not in entry.metadata + + +class TestBitemporalMerge: + """Issue #825, Part B Tier 3 — revision_history()/query_recorded_between() + close kg.ProvenanceTracker's documented 'no direct equivalent yet' gaps.""" + + def test_revision_history_ascending_with_valid_until(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("X", source="doc1", agent_id="alice") + prov_mgr.track_entity("X", source="doc1", agent_id="bob", revision_type="correction") + prov_mgr.track_entity("X", source="doc1", agent_id="carol", supersedes="X_old_claim") + + history = prov_mgr.revision_history("X") + assert len(history) == 3 + assert [h["version"] for h in history] == [1, 2, 3] + assert history[0]["author"] == "alice" + assert history[1]["author"] == "bob" + assert history[1]["revision_type"] == "correction" + assert history[2]["author"] == "carol" + assert history[2]["supersedes"] == "X_old_claim" + # Every version except the last has a valid_until set to the next version's timestamp + assert history[0]["valid_until"] == history[1]["valid_from"] + assert history[1]["valid_until"] == history[2]["valid_from"] + assert history[2]["valid_until"] is None + + def test_revision_history_empty_for_untracked_entity(self): + prov_mgr = ProvenanceManager() + assert prov_mgr.revision_history("never_tracked") == [] + + def test_revision_history_single_version(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("Y", source="doc1") + history = prov_mgr.revision_history("Y") + assert len(history) == 1 + assert history[0]["version"] == 1 + assert history[0]["valid_until"] is None + + def test_query_recorded_between(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + prov_mgr.track_entity("e2", source="doc1") + + results = prov_mgr.query_recorded_between("2000-01-01T00:00:00", "2100-01-01T00:00:00") + entity_ids = {r["entity_id"] for r in results} + assert {"e1", "e2"}.issubset(entity_ids) + + no_results = prov_mgr.query_recorded_between("1990-01-01T00:00:00", "1990-01-02T00:00:00") + assert no_results == [] + + def test_check_flags_missing_informed_by_activity(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1", informed_by=["never_tracked_activity"]) + result = prov_mgr.check() + assert result["valid"] is False + assert any("never_tracked_activity" in ref for ref in result["missing_references"]) + + +class TestBundleAndBaseUri: + """Issue #825, Part B Tier 3 — prov:Bundle membership and configurable base_uri.""" + + def test_bundle_id_produces_bundle_triples(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1", bundle_id="ingestion_run_1") + ttl = prov_mgr.export_prov(format="turtle") + assert "Bundle" in ttl + assert "hadMember" in ttl + + def test_default_base_uri_matches_rdf_exporter_namespace(self): + from semantica.provenance.manager import DEFAULT_BASE_URI + from semantica.export.rdf_exporter import NamespaceManager + assert NamespaceManager().namespaces["semantica"] == DEFAULT_BASE_URI + + def test_export_prov_base_uri_override(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("e1", source="doc1") + + default_ttl = prov_mgr.export_prov(format="turtle") + assert "https://semantica.dev/ns#" in default_ttl + + custom_ttl = prov_mgr.export_prov(format="turtle", base_uri="https://example.org/kg#") + assert "https://example.org/kg#" in custom_ttl + assert "https://semantica.dev/ns#" not in custom_ttl + + def test_owl_exporter_default_ontology_uri_matches_shared_namespace(self): + """Issue #825 follow-up — OWLExporter was the one exporter left out of + the Part B Tier 3 namespace interlinking; its default ontology_uri + must match the same shared DEFAULT_BASE_URI as RDFExporter and + export_prov().""" + from semantica.provenance.manager import DEFAULT_BASE_URI + from semantica.export.owl_exporter import OWLExporter + assert OWLExporter().ontology_uri == DEFAULT_BASE_URI + + +class TestExplicitBitemporalFields: + """Issue #825 follow-up — valid_from/valid_until as explicit, + caller-supplied ProvenanceEntry fields (matching the deprecated + kg.ProvenanceTracker's actual contract: these were always caller-supplied + metadata keys, never auto-computed).""" + + def test_valid_from_valid_until_are_plain_passthrough_fields(self): + prov_mgr = ProvenanceManager() + entry = prov_mgr.track_entity( + "price1", source="doc1", valid_from="2026-01-01", valid_until="2026-06-01" + ) + assert entry.valid_from == "2026-01-01" + assert entry.valid_until == "2026-06-01" + + def test_revision_history_prefers_explicit_valid_from_until(self): + prov_mgr = ProvenanceManager() + prov_mgr.track_entity( + "price1", source="doc1", valid_from="2026-01-01", valid_until="2026-06-01" + ) + history = prov_mgr.revision_history("price1") + assert history[0]["valid_from"] == "2026-01-01" + assert history[0]["valid_until"] == "2026-06-01" + + def test_revision_history_falls_back_to_timestamp_when_unset(self): + """Backward-compat: entries that don't set valid_from/valid_until + explicitly still get the dynamic timestamp-based derivation.""" + prov_mgr = ProvenanceManager() + prov_mgr.track_entity("X", source="doc1") + prov_mgr.track_entity("X", source="doc1") + history = prov_mgr.revision_history("X") + assert history[0]["valid_until"] == history[1]["valid_from"] + assert history[1]["valid_until"] is None + + def test_valid_from_until_survive_sqlite_round_trip(self, tmp_path): + db_path = str(tmp_path / "bitemporal.db") + prov_mgr = ProvenanceManager(storage_path=db_path) + prov_mgr.track_entity( + "price1", source="doc1", valid_from="2026-01-01", valid_until="2026-06-01" + ) + prov_mgr2 = ProvenanceManager(storage_path=db_path) + entry = prov_mgr2.storage.retrieve("price1") + assert entry.valid_from == "2026-01-01" + assert entry.valid_until == "2026-06-01" + + +class TestActivityTimingAcrossWrappers: + """Issue #825 follow-up — activity_started_at_time/ended_at_time wired + into all *_provenance.py wrapper modules that measure real work, not + just the 2 wrappers from the original Part B pass.""" + + def test_embedding_wrapper_records_activity_timing(self): + from semantica.embeddings.embeddings_provenance import EmbeddingGeneratorWithProvenance + wrapper = EmbeddingGeneratorWithProvenance(provenance=True, agent_id="embed_svc") + wrapper._generator.embed = lambda texts, **kw: [[0.1, 0.2] for _ in texts] + wrapper.embed(["hello", "world"]) + entries = wrapper._prov_manager.storage.retrieve_all() + assert len(entries) == 1 + assert entries[0].activity_started_at_time is not None + assert entries[0].activity_ended_at_time is not None + + def test_kg_algorithm_tracker_accepts_caller_supplied_activity_timing(self): + from semantica.kg.kg_provenance import AlgorithmTrackerWithProvenance + tracker = AlgorithmTrackerWithProvenance(provenance=True, agent_id="algo_svc") + eid = tracker.track_embedding_computation( + graph=object(), algorithm="node2vec", embeddings={"n1": [0.1, 0.2]}, + parameters={"d": 2}, + activity_started_at_time="t0", activity_ended_at_time="t1", + ) + entry = tracker._prov_manager.get_provenance(eid) + assert entry["activity_started_at_time"] == "t0" + assert entry["activity_ended_at_time"] == "t1" + + def test_graph_builder_build_operation_has_no_end_time_yet(self): + """The build-operation marker is recorded before the build runs, so + it legitimately has a start but no end time.""" + from semantica.kg.kg_provenance import GraphBuilderWithProvenance + builder = GraphBuilderWithProvenance(provenance=True, agent_id="builder_svc") + builder._builder.build_single_source = lambda kg_data, **kw: {"entities": [], "relationships": []} + builder.build_single_source({"foo": "bar"}) + entries = builder._prov_manager.storage.retrieve_all() + # NOTE: entity_type isn't asserted here — kg_provenance.py has a + # pre-existing, out-of-scope bug where entity_type is nested inside + # the metadata dict instead of passed as a track_entity kwarg, so it + # never actually populates the real field for this call site. + build_entries = [e for e in entries if e.entity_id.startswith("graph_build_single_")] + assert len(build_entries) == 1 + assert build_entries[0].activity_started_at_time is not None + assert build_entries[0].activity_ended_at_time is None + + def test_semantic_extract_wrapper_records_activity_timing(self): + from semantica.semantic_extract.semantic_extract_provenance import ProvenanceMixin + + class FakeExtractor(ProvenanceMixin): + pass + + wrapper = FakeExtractor(provenance=True, agent_id="extract_svc") + wrapper._track_extraction( + entity_id="e1", source="doc1", entity_type="named_entity", + activity_started_at_time="t0", activity_ended_at_time="t1", + ) + entry = wrapper._prov_manager.get_provenance("e1") + assert entry["activity_started_at_time"] == "t0" + assert entry["activity_ended_at_time"] == "t1" + assert "activity_started_at_time" not in entry["metadata"] + + def test_conflicts_wrapper_records_activity_timing(self): + from semantica.conflicts.conflicts_provenance import SourceTrackerWithUnifiedBackend + + class FakeSource: + document = "doc1" + page = 1 + section = None + confidence = 0.9 + + tracker = SourceTrackerWithUnifiedBackend(agent_id="conflicts_svc") + tracker.track_property_source("e1", "prop", "val", FakeSource()) + entry = tracker._unified_manager.get_provenance("e1_prop") + assert entry["activity_started_at_time"] is not None + assert entry["activity_ended_at_time"] is not None diff --git a/tests/provenance/test_schemas.py b/tests/provenance/test_schemas.py index 8afed0e8..e151d9f1 100644 --- a/tests/provenance/test_schemas.py +++ b/tests/provenance/test_schemas.py @@ -7,7 +7,14 @@ and PropertySource dataclasses. import pytest from datetime import datetime -from semantica.provenance.schemas import ProvenanceEntry, SourceReference, PropertySource +from semantica.provenance.schemas import ( + ProvenanceEntry, + SourceReference, + PropertySource, + AgentRecord, + ActivityRecord, + Invalidation, +) class TestProvenanceEntry: @@ -158,7 +165,120 @@ class TestPropertySource: ) data = prop_source.to_dict() - + assert isinstance(data, dict) assert data["property_name"] == "name" assert len(data["sources"]) == 1 + + +class TestProvenanceEntryPart825Fields: + """Issue #825, Part A — new additive fields on ProvenanceEntry.""" + + def test_defaults(self): + entry = ProvenanceEntry(entity_id="e1", entity_type="entity", activity_id="act") + assert entry.agent_type == "software_agent" + assert entry.is_automated is True + assert entry.role is None + assert entry.previous_version_id is None + assert entry.derived_from_id is None + assert entry.sequence_id is None + assert entry.previous_checksum is None + assert entry.invalidated is False + assert entry.invalidated_at_time is None + assert entry.invalidated_by is None + assert entry.invalidation_reason is None + # Part B additive fields + assert entry.activity_started_at_time is None + assert entry.activity_ended_at_time is None + assert entry.acted_on_behalf_of is None + assert entry.informed_by_activities == [] + assert entry.valid_from is None + assert entry.valid_until is None + assert entry.revision_type is None + assert entry.supersedes is None + assert entry.bundle_id is None + + def test_round_trip_to_dict_from_dict(self): + entry = ProvenanceEntry( + entity_id="e1", + entity_type="entity", + activity_id="act", + agent_id="alice", + agent_type="person", + is_automated=False, + role="approver", + previous_version_id="e1:v:1", + derived_from_id="source_entity", + sequence_id=5, + previous_checksum="abc123", + invalidated=True, + invalidated_at_time="2026-01-01T00:00:00", + invalidated_by="reviewer_jane", + invalidation_reason="retracted", + activity_started_at_time="2026-01-01T00:00:00", + activity_ended_at_time="2026-01-01T00:00:05", + acted_on_behalf_of="org1", + informed_by_activities=["ingest_activity"], + valid_from="2026-01-01", + valid_until="2026-06-01", + revision_type="correction", + supersedes="old_claim", + bundle_id="ingestion_run_1", + ) + + data = entry.to_dict() + restored = ProvenanceEntry.from_dict(data) + + assert restored == entry + + +class TestActivityRecord: + """Issue #825, Part B Tier 1 — minimum-viable typed activity.""" + + def test_defaults(self): + activity = ActivityRecord(id="act1") + assert activity.activity_type == "process" + assert activity.started_at_time is None + assert activity.ended_at_time is None + + def test_round_trip(self): + activity = ActivityRecord( + id="bureau_parsing_run_42", + activity_type="extraction", + started_at_time="2026-01-01T00:00:00", + ended_at_time="2026-01-01T00:00:03", + ) + data = activity.to_dict() + restored = ActivityRecord.from_dict(data) + assert restored == activity + + +class TestAgentRecord: + """Issue #825, Part A item 3 — minimum-viable typed agent.""" + + def test_defaults(self): + agent = AgentRecord(id="bot1") + assert agent.agent_type == "software_agent" + assert agent.is_automated is True + assert agent.name is None + + def test_round_trip(self): + agent = AgentRecord(id="reviewer_jane", agent_type="person", is_automated=False, name="Jane") + data = agent.to_dict() + restored = AgentRecord.from_dict(data) + assert restored == agent + + +class TestInvalidation: + """Issue #825, Part A item 1 — tombstone record.""" + + def test_round_trip(self): + inv = Invalidation( + entity_id="e1", + invalidated_at_time="2026-01-01T00:00:00", + invalidated_by="reviewer_jane", + reason="retracted", + ) + data = inv.to_dict() + restored = Invalidation.from_dict(data) + assert restored == inv diff --git a/tests/provenance/test_storage.py b/tests/provenance/test_storage.py index b21c6bc0..5c35764e 100644 --- a/tests/provenance/test_storage.py +++ b/tests/provenance/test_storage.py @@ -130,14 +130,120 @@ class TestInMemoryStorage: storage.store(entry) count = storage.clear() - + assert count == 1 assert len(storage.retrieve_all()) == 0 + def test_get_chain_head(self): + """Issue #825, Part A item 2 — chain head reporting.""" + storage = InMemoryStorage() + assert storage.get_chain_head() is None + + entry1 = ProvenanceEntry( + entity_id="e1", entity_type="entity", activity_id="act", + sequence_id=1, checksum="checksum_1", + ) + storage.store(entry1) + assert storage.get_chain_head() == (1, "checksum_1") + + entry2 = ProvenanceEntry( + entity_id="e2", entity_type="entity", activity_id="act", + sequence_id=2, checksum="checksum_2", + ) + storage.store(entry2) + assert storage.get_chain_head() == (2, "checksum_2") + + def test_trace_descendants(self): + """Issue #825, Part A item 5 — reverse (downstream) lineage traversal.""" + storage = InMemoryStorage() + storage.store(ProvenanceEntry(entity_id="a", entity_type="entity", activity_id="act")) + storage.store(ProvenanceEntry( + entity_id="b", entity_type="entity", activity_id="act", parent_entity_id="a" + )) + storage.store(ProvenanceEntry( + entity_id="c", entity_type="entity", activity_id="act", parent_entity_id="b" + )) + storage.store(ProvenanceEntry( + entity_id="d", entity_type="entity", activity_id="act", used_entities=["a"] + )) + + descendants = storage.trace_descendants("a") + entity_ids = {e.entity_id for e in descendants} + assert entity_ids == {"b", "c", "d"} + + def test_trace_descendants_respects_max_depth(self): + storage = InMemoryStorage() + storage.store(ProvenanceEntry(entity_id="a", entity_type="entity", activity_id="act")) + storage.store(ProvenanceEntry( + entity_id="b", entity_type="entity", activity_id="act", parent_entity_id="a" + )) + storage.store(ProvenanceEntry( + entity_id="c", entity_type="entity", activity_id="act", parent_entity_id="b" + )) + + descendants = storage.trace_descendants("a", max_depth=1) + entity_ids = {e.entity_id for e in descendants} + assert entity_ids == {"b"} + + def test_trace_descendants_empty_for_leaf(self): + storage = InMemoryStorage() + storage.store(ProvenanceEntry(entity_id="leaf", entity_type="entity", activity_id="act")) + assert storage.trace_descendants("leaf") == [] + class TestSQLiteStorage: """Test SQLiteStorage backend.""" - + + def test_all_fields_round_trip_through_sqlite(self): + """Regression test: every ProvenanceEntry field, including issue #825 + Part A and Part B additions, must survive a SQLite store/retrieve + round trip byte-for-byte. Part B's activity/actedOnBehalfOf/ + informedBy/revision/bundle fields were initially added to the + dataclass and to export_prov() without updating SQLiteStorage's DDL/ + INSERT/_row_to_entry — InMemoryStorage stores the dataclass directly + so it masked the gap, but SQLite silently dropped every one of those + fields on write.""" + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp: + db_path = tmp.name + + try: + storage = SQLiteStorage(db_path) + entry = ProvenanceEntry( + entity_id="entity_1", + entity_type="entity", + activity_id="extraction", + agent_id="alice", + agent_type="person", + is_automated=False, + role="approver", + sequence_id=1, + previous_checksum="prevchk", + parent_entity_id="parent_1", + used_entities=["u1", "u2"], + previous_version_id="entity_1:v:1", + derived_from_id="source_1", + invalidated=True, + invalidated_at_time="2026-01-01T00:00:00", + invalidated_by="reviewer_jane", + invalidation_reason="retracted", + activity_started_at_time="2026-01-01T00:00:00", + activity_ended_at_time="2026-01-01T00:00:05", + acted_on_behalf_of="org1", + informed_by_activities=["act_a", "act_b"], + valid_from="2026-01-01", + valid_until="2026-06-01", + revision_type="correction", + supersedes="old_claim", + bundle_id="ingestion_run_1", + ) + storage.store(entry) + retrieved = storage.retrieve("entity_1") + + assert retrieved == entry + finally: + if os.path.exists(db_path): + os.unlink(db_path) + def test_store_and_retrieve(self): """Test storing and retrieving entries.""" with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp: @@ -211,8 +317,71 @@ class TestSQLiteStorage: storage.store(entry2) lineage = storage.trace_lineage("entity_2") - + assert len(lineage) == 2 finally: if os.path.exists(db_path): os.unlink(db_path) + + def test_get_chain_head(self): + """Issue #825, Part A item 2 — chain head reporting.""" + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp: + db_path = tmp.name + + try: + storage = SQLiteStorage(db_path) + assert storage.get_chain_head() is None + + entry1 = ProvenanceEntry( + entity_id="e1", entity_type="entity", activity_id="act", + sequence_id=1, checksum="checksum_1", + ) + storage.store(entry1) + assert storage.get_chain_head() == (1, "checksum_1") + + entry2 = ProvenanceEntry( + entity_id="e2", entity_type="entity", activity_id="act", + sequence_id=2, checksum="checksum_2", + ) + storage.store(entry2) + assert storage.get_chain_head() == (2, "checksum_2") + finally: + if os.path.exists(db_path): + os.unlink(db_path) + + def test_trace_descendants(self): + """Issue #825, Part A item 5 — reverse (downstream) lineage traversal.""" + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp: + db_path = tmp.name + + try: + storage = SQLiteStorage(db_path) + storage.store(ProvenanceEntry(entity_id="a", entity_type="entity", activity_id="act")) + storage.store(ProvenanceEntry( + entity_id="b", entity_type="entity", activity_id="act", parent_entity_id="a" + )) + storage.store(ProvenanceEntry( + entity_id="c", entity_type="entity", activity_id="act", parent_entity_id="b" + )) + storage.store(ProvenanceEntry( + entity_id="d", entity_type="entity", activity_id="act", used_entities=["a"] + )) + + descendants = storage.trace_descendants("a") + entity_ids = {e.entity_id for e in descendants} + assert entity_ids == {"b", "c", "d"} + finally: + if os.path.exists(db_path): + os.unlink(db_path) + + def test_trace_descendants_empty_for_leaf(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp: + db_path = tmp.name + + try: + storage = SQLiteStorage(db_path) + storage.store(ProvenanceEntry(entity_id="leaf", entity_type="entity", activity_id="act")) + assert storage.trace_descendants("leaf") == [] + finally: + if os.path.exists(db_path): + os.unlink(db_path)