mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-03 04:00:18 +00:00
Compare commits
136
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b1e1c9f0d9 | ||
|
|
b9af181625 | ||
|
|
42899c1416 | ||
|
|
19665b9db2 | ||
|
|
07dc579faf | ||
|
|
96e438c81c | ||
|
|
c483352d7a | ||
|
|
01ba7d113b | ||
|
|
cdd45331fa | ||
|
|
a3a0848577 | ||
|
|
eeda5f5b80 | ||
|
|
62b03d4fa5 | ||
|
|
8a2b07b864 | ||
|
|
0773e24075 | ||
|
|
8de7cc1b6d | ||
|
|
c6dc9d87aa | ||
|
|
781b103436 | ||
|
|
e4f0c8993c | ||
|
|
b2b823a5c3 | ||
|
|
3c863e860e | ||
|
|
4c74da7682 | ||
|
|
de84ab6d06 | ||
|
|
8be16f782e | ||
|
|
9faa5661f8 | ||
|
|
3e0fbf8e95 | ||
|
|
29f5c72533 | ||
|
|
e13ea740cd | ||
|
|
e2b79ada9a | ||
|
|
c6f8f0dd04 | ||
|
|
4147f0ca3b | ||
|
|
ad84cb5897 | ||
|
|
adac6f7a7b | ||
|
|
90e6baa0ec | ||
|
|
ca8a916373 | ||
|
|
0dd5c4b7f1 | ||
|
|
13eed9cf6d | ||
|
|
1064b0bdbe | ||
|
|
ac047f917a | ||
|
|
7b6e74d042 | ||
|
|
ba491d8cba | ||
|
|
54868fea80 | ||
|
|
a0e7e9a1c9 | ||
|
|
e8d71b49fa | ||
|
|
8916200d31 | ||
|
|
290916a6ff | ||
|
|
88bd7d6b05 | ||
|
|
44f817ee71 | ||
|
|
5d6051bee1 | ||
|
|
0b922e77c5 | ||
|
|
e3a0c84b90 | ||
|
|
6bbb8f929f | ||
|
|
d8bbd8877f | ||
|
|
b1e5c9e3c9 | ||
|
|
fe64b8ad8a | ||
|
|
65a00de408 | ||
|
|
e9a2f87325 | ||
|
|
e7a13f7de6 | ||
|
|
fedbd8de8e | ||
|
|
ed27b98c53 | ||
|
|
353a6c605d | ||
|
|
0659509c14 | ||
|
|
b2a2d24b14 | ||
|
|
e315ad849d | ||
|
|
62c7970b32 | ||
|
|
4235840a9e | ||
|
|
e3c33cf23b | ||
|
|
868109fa34 | ||
|
|
2e5ad9d28b | ||
|
|
6c61e34ad4 | ||
|
|
9a6c07417e | ||
|
|
89fe0df40b | ||
|
|
500d0239e0 | ||
|
|
e18e6d1a00 | ||
|
|
753bf18ce7 | ||
|
|
1aee4dfd29 | ||
|
|
572d2da64a | ||
|
|
3f55a34eff | ||
|
|
2dbc502720 | ||
|
|
c5fb2d24fd | ||
|
|
9c09851658 | ||
|
|
2cea2708a6 | ||
|
|
b80c91ccb9 | ||
|
|
ad9ea48d26 | ||
|
|
62f2c0af92 | ||
|
|
24166bbfa9 | ||
|
|
665771f230 | ||
|
|
9d4d682883 | ||
|
|
399416f6ea | ||
|
|
32d0fe105c | ||
|
|
ee7c00f655 | ||
|
|
dcf26dfa99 | ||
|
|
9bb71a45f2 | ||
|
|
4a3b1676d6 | ||
|
|
a6654ba570 | ||
|
|
060ff47826 | ||
|
|
4146fbf277 | ||
|
|
1d1ae398c4 | ||
|
|
99a4db3ece | ||
|
|
de03d05600 | ||
|
|
c077944457 | ||
|
|
b309451398 | ||
|
|
0c1bdc0cee | ||
|
|
c2a6e944fe | ||
|
|
0dc5eb6075 | ||
|
|
5a316a4641 | ||
|
|
39c9fc97b4 | ||
|
|
f043367a73 | ||
|
|
38ec333626 | ||
|
|
2d90bdaad5 | ||
|
|
43a8f823c8 | ||
|
|
7a7e3f9e6b | ||
|
|
c59e33c9d3 | ||
|
|
867ecfda1b | ||
|
|
4103f747c5 | ||
|
|
ad8f24fc6b | ||
|
|
7535e39c56 | ||
|
|
420ccfe45a | ||
|
|
ad72ab9d19 | ||
|
|
a06e029264 | ||
|
|
a4caafbb6d | ||
|
|
9a2b2b9cd1 | ||
|
|
c842af65d0 | ||
|
|
e8d0d7a2cf | ||
|
|
8ffaf6001b | ||
|
|
476267f764 | ||
|
|
7ebdbcc62b | ||
|
|
e8e838829d | ||
|
|
a7e43304fc | ||
|
|
b36f6cd9eb | ||
|
|
af93dd29a8 | ||
|
|
dc8c29a87f | ||
|
|
78c52eb099 | ||
|
|
6b847716b1 | ||
|
|
bcdf3c357a | ||
|
|
2be45a01f1 | ||
|
|
4b1c78372c |
+1
-1
@@ -7,7 +7,7 @@ Check the [docs folder](https://github.com/Hawksight-AI/semantica/tree/main/docs
|
||||
|
||||
### 💬 Community Support
|
||||
- **GitHub Discussions**: [Ask questions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
- **Discord**: Join our [Discord server](https://discord.gg/ggb7vWeP) for real-time chat
|
||||
- **Discord**: Join our [Discord server](https://discord.gg/sV34vps5hH) for real-time chat
|
||||
|
||||
### 💭 Discussions
|
||||
Join the conversation on [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions):
|
||||
|
||||
@@ -2,9 +2,13 @@ name: Semantica Performance Suite
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master]
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- 'mkdocs.yml'
|
||||
- 'requirements-docs.txt'
|
||||
- '**/*.md'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
performance-test:
|
||||
|
||||
@@ -3,8 +3,18 @@ name: CI
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- 'mkdocs.yml'
|
||||
- 'requirements-docs.txt'
|
||||
- '**/*.md'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- 'mkdocs.yml'
|
||||
- 'requirements-docs.txt'
|
||||
- '**/*.md'
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
@@ -8,11 +8,12 @@ on:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'docs/**'
|
||||
- 'semantica/**'
|
||||
- 'mkdocs.yml'
|
||||
- 'requirements-docs.txt'
|
||||
- 'CHANGELOG.md'
|
||||
- 'RELEASE.md'
|
||||
release:
|
||||
types: [published]
|
||||
workflow_dispatch:
|
||||
|
||||
# Permissions needed to deploy to GitHub Pages
|
||||
|
||||
@@ -4,9 +4,19 @@ on:
|
||||
schedule:
|
||||
- cron: '30 1 * * 1,4' # Mon/Thu 7 AM IST
|
||||
push:
|
||||
branches: [ main ]
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- 'mkdocs.yml'
|
||||
- 'requirements-docs.txt'
|
||||
- '**/*.md'
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- 'docs/**'
|
||||
- 'mkdocs.yml'
|
||||
- 'requirements-docs.txt'
|
||||
- '**/*.md'
|
||||
|
||||
jobs:
|
||||
security-scan:
|
||||
@@ -158,7 +168,7 @@ jobs:
|
||||
}
|
||||
|
||||
// Create summary comment
|
||||
const comment = `# 🔒 Security Scan Results\\n\\n${safetyResults}\\n\\n${banditResults}\\n\\n${semgrepResults}\\n\\n---\\n\\n*This security scan runs automatically on every PR and bi-weekly.*\\n\\n📊 **Security Policy**: CI fails on vulnerabilities and HIGH severity issues.`;
|
||||
const comment = `# 🔒 Security Scan Results\\n\\n${safetyResults}\\n\\n${banditResults}\\n\\n${semgrepResults}\\n\\n---\\n\\n*This security scan runs automatically on source-code PRs and bi-weekly (skipped for doc/markdown-only changes).*\\n\\n📊 **Security Policy**: CI fails on vulnerabilities and HIGH severity issues.`;
|
||||
|
||||
// Post comment with error handling
|
||||
try {
|
||||
|
||||
+293
@@ -7,6 +7,299 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
- **Temporal Metadata Extraction from Text** (PR #400 by @KaifAhmad1):
|
||||
- Added `extract_temporal_bounds: bool = False` parameter to `extract_relations_llm()`. When `True`, the LLM prompt is extended with a calibrated confidence scale and four few-shot examples; each returned `Relation` gains `valid_from`, `valid_until`, `temporal_confidence` (0.0–1.0), and `temporal_source_text` in its `metadata` dict. Default `False` preserves 100% backward compatibility.
|
||||
- Confidence scale anchors baked into the prompt: `1.00` = full ISO date, `0.90` = year+month, `0.85` = year only, `0.75` = quarter, `0.65` = named season/approximate range, `0.50` = vague relative with computable anchor, `0.35` = highly vague, `0.00` = no temporal signal. LLMs self-report certainty rather than clustering near 1.0.
|
||||
- Low temporal confidence (< 0.5) with a non-null date logs a `WARNING`; signal is never suppressed — callers decide how to filter.
|
||||
- Cache key now includes the `extract_temporal_bounds` flag to prevent cross-mode cache pollution.
|
||||
- Flag propagated through `_extract_relations_chunked()` so long-text chunked extraction also carries temporal metadata.
|
||||
- Added `RelationWithTemporalOut` and `RelationsWithTemporalResponse` Pydantic schemas in `semantica/semantic_extract/schemas.py`. A separate schema is required because `RelationOut` uses `extra="ignore"`, which silently drops any undeclared field including the four temporal fields.
|
||||
- New `semantica/kg/temporal_normalizer.py` — `TemporalNormalizer` class (zero LLM calls, pure regex + `dateutil` arithmetic):
|
||||
- `normalize(value)` → `(valid_from, valid_until)` UTC `datetime` tuple or `None`. Resolution order: ISO 8601 full parse → partial-date regex (year-only, month+year, YYYY-MM, Q[1-4] YYYY) → ambiguous-slash-date detection → domain phrase map → relative phrase resolution via `relativedelta`.
|
||||
- `normalize_phrase(phrase)` → metadata dict `{"maps_to": ..., "type": ..., "domain": [...]}` or `None` — exact match then regex-pattern keys.
|
||||
- Ambiguous `DD/MM/YYYY`-style inputs issue `TemporalAmbiguityWarning` and return `None` — never silently guesses locale.
|
||||
- Unparseable inputs return `None` with a debug log — never raise.
|
||||
- Relative phrases (`"last year"`, `"three months ago"`, etc.) raise `ValueError` if `reference_date` is `None` rather than guessing.
|
||||
- Default phrase map covers 13 domains: General/Policy (`effective date`, `effective from/as of/beginning`, `in force until`, `retroactive to`, `sunset clause`), Healthcare (`approval date`, `expiry date`, `market authorization`), Cybersecurity (`incident window`, `campaign period`), Supply Chain (`certification valid through`), Finance (`trading halt`), Energy (`commissioned date`, `decommissioned date`).
|
||||
- User-supplied `phrase_map` is merged over defaults at construction (`{**defaults, **user_map}`) — custom entries win without forking the library.
|
||||
- Added `TemporalAmbiguityWarning(UserWarning)` to `semantica/utils/exceptions.py`.
|
||||
- Exported `TemporalNormalizer` from `semantica/kg/__init__.py`.
|
||||
- Added 53 new tests in `tests/semantic_extract/test_temporal_extraction.py`; zero real LLM calls, suite runs in ~3.5 s. All 873 existing tests continue to pass.
|
||||
|
||||
- **Fix: OllamaProvider ignores `base_url`** (PR #408 by @AlexeyMyslin, fixed by @KaifAhmad1):
|
||||
- `OllamaProvider._init_client()` was assigning the raw `ollama` module to `self.client` instead of instantiating `ollama.Client(host=self.base_url)`, causing all requests to silently hit `localhost:11434` regardless of the `base_url` passed by the user
|
||||
- Fixed by replacing `self.client = ollama` with `self.client = ollama.Client(host=self.base_url)` — remote Ollama servers (e.g. `http://192.168.1.3:11434`) are now reachable
|
||||
- Added 3 regression tests: default URL forwarded as host, custom URL forwarded as host, and guard ensuring `self.client` is never the raw module
|
||||
|
||||
- **Temporal Awareness in Context Graph** (PR #399 by @KaifAhmad1):
|
||||
- Added `valid_from` and `valid_until` fields to the `Decision` dataclass and `record_decision()` — decisions now carry explicit validity windows; superseded decisions remain in the graph (history is immutable)
|
||||
- Added `include_superseded=False` and `as_of=None` parameters to `find_precedents_by_scenario()` — defaults exclude expired decisions; `as_of` enables point-in-time precedent queries
|
||||
- Added `ContextGraph.state_at(timestamp)` — returns a serializable point-in-time snapshot of all nodes, edges, and decisions whose validity windows include `timestamp`; source graph is never mutated
|
||||
- Stamped `recorded_at` on causal relationship edges created via `add_causal_relationship()` — enables transaction-time filtering
|
||||
- Added `CausalChainAnalyzer.trace_at_time(event_id, at_time)` — reconstructs a causal chain using only edges recorded up to `at_time` (transaction time); returns an empty list when `at_time` predates all facts, never raises
|
||||
- Added `AgentContext.checkpoint(label)`, `diff_checkpoints(label1, label2)`, and `flush_checkpoint(label)` — named in-memory context snapshots with structured diffs (`decisions_added`, `decisions_removed`, `relationships_added`, `relationships_removed`) and optional persistence via `TemporalVersionManager`
|
||||
- **Review fixes applied in the same PR**:
|
||||
- Fixed `max_depth` error message in `trace_at_time` to match actual bound (1–100)
|
||||
- Fixed Cypher `at_time` query parameter to RFC3339 UTC (`Z` suffix) for unambiguous external DB comparisons
|
||||
- `_normalize_temporal_input` now raises `ValueError` on unparseable strings instead of silently returning raw input
|
||||
- Replaced `datetime.now()` with `datetime.utcnow()` for all `recorded_at` and checkpoint timestamps — aligns with codebase convention and avoids wrong local time on Windows
|
||||
- `flush_checkpoint` wraps `TemporalVersionManager()` construction in a `try/except` and re-raises as `RuntimeError` with a clear actionable message
|
||||
- Added 7 new tests (93 total across context modules, 0 failures)
|
||||
|
||||
- **spaCy Runtime Fallback for NER Benchmarks**:
|
||||
- Hardened `NERExtractor` spaCy initialization so installed-but-broken spaCy environments no longer crash during extractor construction.
|
||||
- Updated ML entity extraction fallback behavior to catch runtime spaCy initialization failures, not just missing-model errors.
|
||||
- Added regression coverage for the "spaCy present but unusable at runtime" initialization path.
|
||||
- **Deterministic Temporal Reasoning Engine** (PR #398 by @KaifAhmad1, implemented and follow-up fixes by OpenAI Codex):
|
||||
- Added `semantica.kg.temporal_reasoning` as the single source of truth for deterministic, LLM-free temporal reasoning with an explicit zero-LLM module contract
|
||||
- Implemented `TemporalInterval`, full Allen interval algebra via `IntervalRelation`, and `TemporalReasoningEngine`
|
||||
- Added deterministic helpers for interval overlap/containment checks, open-ended activity checks, interval merging, gap analysis, coverage calculation, timelines, retroactive coverage, and temporal normalization
|
||||
- Integrated temporal query interval logic with the reasoning engine in `TemporalGraphQuery`
|
||||
- Preserved `semantica.reasoning` access via re-exports without making it the canonical implementation source
|
||||
- Fixed open-ended `query_time_range(..., end_time=None)` handling so temporal range queries no longer crash on `TemporalBound.OPEN`
|
||||
- Restored `temporal_granularity` behavior for point-in-time checks in `query_at_time()`
|
||||
- Eliminated the `semantica.reasoning` / `semantica.kg` circular import risk introduced during the initial module move
|
||||
- Added regression coverage for all 13 Allen relations, open-ended intervals, month-granularity point queries, open-ended range queries, retroactive coverage, and normalization idempotence
|
||||
|
||||
- **Temporal Query Engine: Point-in-Time Correctness** (PR #397 by @KaifAhmad1, implemented and follow-up fixes by OpenAI Codex):
|
||||
- Added `reconstruct_at_time(graph, at_time)` to `TemporalGraphQuery` to build a self-consistent point-in-time subgraph without mutating the input graph
|
||||
- Updated `query_at_time()` to use point-in-time reconstruction internally so returned subgraphs exclude dangling edges when entity lifetimes are available
|
||||
- Added `TemporalConsistencyIssue` and `TemporalConsistencyReport` plus temporal consistency validation for:
|
||||
- inverted relationship intervals
|
||||
- relationships outside entity lifetimes
|
||||
- missing source/target entities
|
||||
- overlapping same-type relationships on the same edge
|
||||
- temporal gaps where a fact ends and restarts later
|
||||
- Added a module-level `validate_temporal_consistency(graph)` API alongside the query-engine method
|
||||
- Implemented sequence and cycle pattern detection with structured outputs containing `pattern_type`, `signature`, `frequency`, and per-occurrence node/edge/time details
|
||||
- Implemented calendar-aligned temporal evolution bucketing based on `temporal_granularity`
|
||||
- Added causal ordering controls to `find_temporal_paths()` via `enforce_causal_ordering` and `ordering_strategy` (`strict`, `overlap`, `loose`)
|
||||
- **Follow-up fixes applied in the same PR**:
|
||||
- Made `validate_temporal_consistency()` non-throwing on malformed temporal fields and return report errors instead of raising
|
||||
- Enforced exclusive `valid_until` semantics for point-in-time checks (`valid_from <= at_time < valid_until`)
|
||||
- Kept `query_time_range(..., temporal_aggregation="evolution")` backward-compatible by returning the flat relationship list plus a new `relationship_buckets` field
|
||||
- Hardened temporal pattern detection for open-ended intervals (`TemporalBound.OPEN`) to avoid datetime arithmetic/comparison crashes
|
||||
- Normalized relationship endpoints during point-in-time reconstruction so mixed-type IDs like `1` and `"1"` do not silently drop valid edges
|
||||
- Added in-code design comments documenting the sequence/cycle output structure required by the checklist
|
||||
- Added and expanded regression coverage for point-in-time reconstruction, exclusive end bounds, non-throwing validation, module-level validator access, pattern detection with gap tolerance/open bounds, evolution bucketing, causal ordering, and mixed-type IDs
|
||||
|
||||
- **Core Temporal Data Model Overhaul** (PR #396 by @KaifAhmad1, implemented and follow-up fixes by OpenAI Codex):
|
||||
- Added `semantica.kg.temporal_model` with shared helpers for parsing, normalizing, serializing, and deserializing temporal relationship fields
|
||||
- Exported `TemporalBound` and `BiTemporalFact` from `semantica.kg` for backward-compatible temporal relationship handling
|
||||
- Updated `TemporalGraphQuery` to use shared temporal parsing/model helpers instead of ad hoc string handling
|
||||
- Added support for `valid`, `transaction`, and `both` time axes in temporal query filtering
|
||||
- Standardized temporal normalization on `timezone.utc` for better cross-version portability
|
||||
- Added `TemporalValidationError` to utils exports and made invalid temporal inputs consistently raise it
|
||||
- Added history-preserving temporal revisions in `TemporalVersionManager.apply_revision()` with provenance metadata and supersession semantics
|
||||
- Added safer snapshot persistence by serializing revision metadata before storage and surfacing storage failures as `ProcessingError`
|
||||
- **Follow-up fixes applied in the same PR**:
|
||||
- Added a default factory for `BiTemporalFact.recorded_at` and preserved legacy transaction-axis behavior by falling back to `valid_from` when `recorded_at` is missing
|
||||
- Treated `TemporalBound.OPEN` as an unbounded value in shared query parsing so open-ended facts do not fail in public APIs like `analyze_evolution()` and path filtering
|
||||
- Recomputed snapshot checksums before persisting revised snapshots and any original snapshot inserted during revision flow
|
||||
- Replaced second-based revision suffixes with collision-resistant revision IDs/labels to avoid duplicate save failures under rapid revisions
|
||||
- Removed warning spam caused by canonical serialized open bounds represented as `None`
|
||||
- Added and expanded regression coverage for UTC normalization, transaction-axis queries, open-ended bounds, revision integrity, checksum verification, and collision-resistant revision identifiers
|
||||
|
||||
- **Audit Trail, Named Tags, and Rollback Protection** (PR #394 by @ZohaibHassan16, reviewed by @KaifAhmad1, follow-up fixes by OpenAI Codex):
|
||||
- Added mutation-level audit tracking for `ContextGraph` node and edge changes via `TemporalVersionManager.attach_to_graph()` and persistent mutation logging backends
|
||||
- Added named version tags in both in-memory and SQLite storage so human-readable tags can point to saved snapshots
|
||||
- Added rollback protection to `restore_snapshot()` so destructive graph restores require explicit confirmation
|
||||
- Added `get_node_history()` for per-entity audit inspection and `diff()` as a Git-like alias over version comparisons
|
||||
- Preserved backward compatibility for snapshot payloads and diff outputs by supporting both `nodes`/`edges` and `entities`/`relationships`
|
||||
- Fixed mixed-schema snapshot comparison and version metadata counts after the audit-trail feature landed on top of PR #393
|
||||
- Fixed restore replay so rollback does not generate synthetic mutation events in the audit log
|
||||
- Added version-label assignment for previously unlabeled mutations when a snapshot is created
|
||||
- Resolved merge conflicts against updated `main` in `managers.py`, `version_storage.py`, `context_graph.py`, and `test_managers.py`
|
||||
- Added and updated regression coverage for audit history, rollback safety, version-label persistence, and snapshot compatibility
|
||||
|
||||
- **Snapshot Schema Compatibility Fix** (PR #393 by @ZohaibHassan16, reviewed by @KaifAhmad1, follow-up fixes by OpenAI Codex):
|
||||
- Fixed silent snapshot restore failures caused by the `ContextGraph` `nodes`/`edges` schema not matching the version manager's legacy `entities`/`relationships` expectations
|
||||
- Updated temporal snapshot handling to accept both `nodes`/`edges` and `entities`/`relationships`
|
||||
- Preserved both schema shapes in stored snapshots to maintain backward compatibility during migration
|
||||
- Fixed temporal diffing and detailed comparison paths so new-format and mixed-format snapshots compare correctly
|
||||
- Fixed version metadata counts so `entity_count` and `relationship_count` remain accurate for both snapshot schemas
|
||||
- Restored ontology snapshot compatibility fields removed during the PR follow-up iteration
|
||||
- Added regression coverage for new-format snapshot creation, metadata counts, and mixed-schema diffing
|
||||
|
||||
- **ContextGraph Traversal Fallbacks for DecisionQuery & DecisionRecorder** (PR #386 by @ZohaibHassan16, reviewed and fixed by @KaifAhmad1):
|
||||
- Added native `ContextGraph` fallback execution paths to all 7 `DecisionQuery` methods (`_find_precedents_basic`, `find_by_category`, `find_by_entity`, `find_by_time_range`, `multi_hop_reasoning`, `trace_decision_path`, `find_similar_exceptions`) — resolves issue #379 where hardcoded Cypher queries broke in-memory usage
|
||||
- Added native `ContextGraph` fallback paths to 4 `DecisionRecorder` methods (`link_entities`, `record_exception`, `link_precedents`, `_store_decision_node`, `_store_exception_node`) using `add_node` / `add_edge` primitives
|
||||
- Implemented undirected BFS in `multi_hop_reasoning` fallback — traverses both outgoing and incoming edges so decisions are reachable from linked entities (matches Cypher `(start)-[*1..N]-(d:Decision)` semantics)
|
||||
- Fixed `isinstance(graph_store, ContextGraph)` guards → `type(graph_store) is ContextGraph` — prevents `Mock(spec=ContextGraph)` from triggering fallback branches and breaking 2 existing tests
|
||||
- Fixed `add_node(properties=metadata)` call in `_store_decision_node` and `_store_exception_node` — changed to `**metadata` so all decision fields are stored flat and remain readable via `_dict_to_decision`; previous form silently nested every field under a `"properties"` key
|
||||
- Fixed spurious `properties={}` keyword argument in all `add_edge` fallback calls — argument did not match the actual `add_edge(**properties)` signature
|
||||
- Fixed tz-aware / naive `datetime` mismatch in `find_by_time_range` fallback — strips `tzinfo` from aware bounds when stored timestamps are naive, preventing `TypeError` at comparison time
|
||||
- Hoisted `find_edges()` calls out of the BFS `while` loop in `trace_decision_path` — edges are now fetched once per call instead of once per visited node, eliminating O(nodes × total_edges) repeated full-graph scans
|
||||
- Removed duplicate `from ..embeddings import EmbeddingGenerator` import in `decision_query.py`
|
||||
- Added `tests/context/test_decision_query_fallback.py` with 14 tests: full integration test covering the complete fallback flow end-to-end, plus 13 targeted unit tests covering each `DecisionQuery` and `DecisionRecorder` fallback method individually, tz-aware/naive datetime mixing, and `Mock` guard correctness
|
||||
|
||||
- **ContextGraph Thread Safety & Pagination** (PR #385, Issues #378 #376 by @ZohaibHassan16, review & fixes by @KaifAhmad1):
|
||||
- `ContextGraph`: added `threading.RLock` (`self._lock`) to `__init__`; all mutation paths (`add_nodes`, `add_edges`, `add_node`, `add_edge`, `save_to_file`, `load_from_file`, `link_graph`) and all read/query paths (`find_nodes`, `find_edges`, `find_node`, `find_active_nodes`, `get_neighbors`, `query`, `stats`, `density`) now protected with `with self._lock:` to prevent race-condition corruption under concurrent FastAPI workers
|
||||
- `find_nodes` and `find_edges` gained native `skip`/`limit` pagination parameters so the explorer layer never loads the full collection into memory to slice it
|
||||
- `GraphSession` (`session.py`): introduced session-level `RLock` wrapping all graph access; all 8 lazy analytics properties (`centrality`, `community`, `connectivity`, `path_finder`, `node_embedder`, `similarity`, `link_predictor`, `validator`) initialised under the lock (thread-safe double-checked); `get_nodes()` and `get_edges()` delegate pagination to the graph layer when no in-memory filter is needed
|
||||
- `pyproject.toml`: removed duplicate entry and added missing comma in the `all` optional-dependency array that caused `ERROR Failed to parse pyproject.toml: Unclosed array` in CI
|
||||
- **Fixes applied post-review (by @KaifAhmad1)**:
|
||||
- Fixed `/api/graph/search` returning empty `content` and `properties` — `ContextGraph.query()` wraps results in `node.to_dict()` which uses a `"properties"` envelope, but `_node_dict_to_response` expected a flat `{id, type, content, metadata}` shape; `session.search()` now normalises the envelope before returning
|
||||
- Fixed edge metadata silently dropped on import — `add_edges()` read only from the `"properties"` key, but edges produced by `find_edges()` and `build_graph_dict()` use `"metadata"`; fixed with `edge.get("properties") or edge.get("metadata", {})` fallback
|
||||
- Fixed `POST /api/enrich/links` blocking the asyncio event loop — the O(n) `score_link` scoring loop ran inline in the `async` handler; wrapped in `asyncio.to_thread(_score_all)`
|
||||
- Removed merge-artifact dead code in `session.py`: duplicate `self.annotations` assignment, duplicate un-locked property set, and double-query logic in `get_nodes()`/`get_edges()` that recomputed results outside the lock and threw away the correctly-paginated result computed inside it
|
||||
- Removed merge-artifact dead code in `enrich.py`: unreachable second `predict_links` implementation block after early `return`, and duplicate `nodes, _` fetch in `detect_duplicates`
|
||||
|
||||
- **Knowledge Explorer API Backend** (PR #384, Issue #377 by @ZohaibHassan16, review & fixes by @KaifAhmad1):
|
||||
- Added `semantica.explorer` package — a full FastAPI backend for the Semantica Knowledge Explorer dashboard
|
||||
- `app.py`: `create_app(session)` factory with CORS middleware, custom exception handlers (`KeyError→404`, `ValueError→422`), and HTML5 static-file fallback routing; generic `Exception` handler correctly re-raises `HTTPException` so dependency-injection 503s are not swallowed
|
||||
- `session.py`: `GraphSession` — thread-safe container wrapping a `ContextGraph` with 8 lazily-initialised analytics components (`CentralityCalculator`, `CommunityDetector`, `ConnectivityAnalyzer`, `PathFinder`, `NodeEmbedder`, `SimilarityCalculator`, `LinkPredictor`, `GraphValidator`); all lazy properties initialised under `RLock` to prevent double-instantiation under concurrent requests; shared `build_graph_dict(node_ids=None)` method eliminates duplication across route files; `from_file(path)` classmethod loads from JSON
|
||||
- `ws.py`: `ConnectionManager` — thread-safe WebSocket manager with `connect()`, `disconnect()`, `broadcast(event_type, data)`, and `send_personal()` support; safe disconnection cleanup during broadcast
|
||||
- `dependencies.py`: `get_session(request)` and `get_ws_manager(request)` FastAPI `Depends`-compatible callables; `get_session` raises `HTTP 503` when no session is attached
|
||||
- 7 modular route files, all using `asyncio.to_thread` for sync graph operations:
|
||||
- `routes/graph.py`: `GET /api/graph/nodes` (type/keyword filter, pagination), `GET /api/graph/node/{id}`, `GET /api/graph/node/{id}/neighbors` (BFS, depth 1–5), `GET /api/graph/edges` (type/source/target filter), `GET /api/graph/node/{id}/path` (BFS or Dijkstra — algorithm param now correctly dispatched), `POST /api/graph/search`, `GET /api/graph/stats`
|
||||
- `routes/analytics.py`: `GET /api/analytics` (centrality, community, connectivity — comma-separated metrics param), `GET /api/analytics/validation`
|
||||
- `routes/decisions.py`: `GET /api/decisions` (category filter, pagination), `GET /api/decisions/{id}`, `GET /api/decisions/{id}/chain` (BFS causal chain up to 5 hops), `GET /api/decisions/{id}/precedents` (category + scenario keyword ranking), `GET /api/decisions/{id}/compliance` (in-graph check over `violates`/`non_compliant`/`breaches` edges — no longer a stub)
|
||||
- `routes/temporal.py`: `GET /api/temporal/snapshot` (ISO-8601 `at` param), `GET /api/temporal/diff` (added/removed node sets between two timestamps), `GET /api/temporal/patterns` (graceful fallback when `TemporalPatternDetector` unavailable, with warning log for unexpected errors)
|
||||
- `routes/enrich.py`: `POST /api/enrich/extract` (NLP entity/relation extraction), `POST /api/enrich/links` (per-node link prediction via `score_link` against all non-adjacent candidates — fixed from broken `predict_links` call), `POST /api/enrich/dedup` (duplicate detection — fixed missing `asyncio.to_thread` that was blocking the event loop), `POST /api/reason` (forward/backward inference via `Reasoner`)
|
||||
- `routes/export_import.py`: `POST /api/export` (12 formats: JSON, Turtle, RDF-XML, N-Triples, CSV, GraphML, GEXF, OWL, Cypher, AQL, YAML — temp file always cleaned up via `try/finally`), `POST /api/import` (JSON/JSON-LD multipart upload with WebSocket progress events)
|
||||
- `routes/annotations.py`: `GET /api/annotations`, `POST /api/annotations` (validates node exists; `add_annotation` mutates dict in-place so no extra roundtrip), `DELETE /api/annotations/{id}`
|
||||
- `schemas.py`: 28 Pydantic v2 request/response models covering all endpoint shapes including pagination, temporal, enrichment, compliance, and annotation types
|
||||
- `__init__.py`: `semantica-explorer` CLI entry point — `--graph`, `--host`, `--port`, `--no-browser` args; validates graph file exists; checks for `uvicorn`; opens browser after 1.5 s delay
|
||||
- `pyproject.toml`: added `[project.optional-dependencies] explorer` group (`fastapi`, `uvicorn[standard]`, `websockets`, `python-multipart`); registered `semantica-explorer` script entry point; fixed missing comma in `all` extra that broke `pip install semantica[all]`
|
||||
- **Fixes applied post-review (by @KaifAhmad1)**:
|
||||
- Fixed `predict_links` endpoint — was calling `predictor.predict_links(graph_dict, node_id, top_n=...)` with wrong type (`dict` as `graph_store`), wrong positional arg (`node_id` as `node_labels`), and wrong kwarg (`top_n` vs `top_k`); rewrote to iterate all non-adjacent candidate nodes and call `predictor.score_link(session.graph, source, candidate)` directly
|
||||
- Fixed `detect_duplicates` endpoint — `session.get_nodes()` was called directly in an `async def` handler without `asyncio.to_thread`, blocking the event loop
|
||||
- Fixed temp file leak in `export_graph` — file was not deleted on exception from `export_fn` or `open()`; wrapped in `try/finally`; moved `import os` to module level
|
||||
- Fixed `pyproject.toml` `all` extra — two consecutive strings with no comma between them caused a TOML syntax error
|
||||
- Fixed generic `Exception` handler swallowing `HTTPException(503)` raised by `get_session`
|
||||
- Fixed compliance endpoint — imported `PolicyEngine` then discarded it, always returning `compliant=True`; replaced with in-graph edge scan
|
||||
- Fixed `temporal_patterns` bare `except Exception` silently hiding bugs — split into `ImportError` (silent graceful) and `Exception` (warning log)
|
||||
- Fixed all 8 lazy analytics properties to initialise under `_lock` (thread-safe double-checked)
|
||||
- Fixed `find_path` ignoring the `algorithm` query param — now dispatches to `dijkstra_shortest_path` or `bfs_shortest_path`
|
||||
- Removed unnecessary `get_annotations()` round-trip in `create_annotation`
|
||||
- Removed `import traceback` unused import in `app.py`
|
||||
- Deduplicated `_build_graph_dict` (was copied identically in `graph.py`, `analytics.py`, `export_import.py`) into `GraphSession.build_graph_dict()`
|
||||
- 49 integration tests in `tests/explorer/test_explorer_api.py` using `starlette.testclient.TestClient` — all passing; covers health, nodes, edges, search, stats, decisions, causal chains, precedents, compliance (including violation detection), temporal snapshots/diff/patterns, analytics, reasoning, entity extraction, link prediction, deduplication, annotations, export (JSON + node-subset), and import (JSON + edges + unsupported format)
|
||||
- **Reasoning Dead Code Removal** (PR #387, Issue #382 by @ZohaibHassan16):
|
||||
- Removed lines 357–358 in `semantica/reasoning/reasoner.py` that silently overwrote the sophisticated `_match_pattern` regex (which handles pre-bound variable embedding, repeated-variable backreferences via `(?P=var)`, and non-greedy named capture groups) with a simpler `re.escape`-based pattern, making all the prior logic unreachable dead code
|
||||
- Removed duplicate unreachable `return None` on line 368 (syntactically dead, appearing immediately after another `return None` in the same branch)
|
||||
- Surfaced `re.error` exceptions instead of swallowing them with `except Exception: pass`, preventing silent failures when malformed patterns were passed to `re.match`
|
||||
- Before this fix, any rule using the same variable twice (e.g. `rel(?x, ?x)`) generated a duplicate named group error that was silently caught, causing the match to return `None` regardless of the fact — breaking transitivity, symmetry, and self-join rule patterns entirely
|
||||
|
||||
- **Agno Agentic Framework Integration** (Issue #249):
|
||||
- Added `AgnoContextStore` — graph-backed agent memory implementing the `agno.memory.db.base.MemoryDb` protocol; wraps `AgentContext` + `VectorStore`; supports `create()`, `table_exists()`, `memory_exists()`, `read_memories()`, `upsert_memory()`, `delete_memory()`, `drop_table()`, `clear()` plus extended `record_decision()`, `find_precedents()`, `retrieve()` methods
|
||||
- Added `AgnoKnowledgeGraph` — multi-hop GraphRAG knowledge base implementing `agno.knowledge.base.AgentKnowledge`; ingests files, directories, URLs, and raw text via NER → relation extraction → graph build → vector index pipeline; `search()` returns `AgnoDocument` objects; `get_graph_context(entity)` returns text summary of entity's graph neighbourhood
|
||||
- Added `AgnoDecisionKit` — Agno `Toolkit` subclass exposing 6 decision-intelligence tools: `record_decision`, `find_precedents`, `trace_causal_chain`, `analyze_impact`, `check_policy`, `get_decision_summary`
|
||||
- Added `AgnoKGToolkit` — Agno `Toolkit` subclass exposing 7 KG pipeline tools: `extract_entities`, `extract_relations`, `add_to_graph`, `query_graph`, `find_related`, `infer_facts`, `export_subgraph`
|
||||
- Added `AgnoSharedContext` — team-level coordinator with a single shared `ContextGraph`; `bind_agent(role)` returns a role-scoped `_AgentScopedStore` with cross-agent memory visibility; thread-safe via `RLock`
|
||||
- All 5 components degrade gracefully when `agno` is not installed (`AGNO_AVAILABLE` flag); importable and functional without agno present
|
||||
- Added `agno = ["agno>=1.0.0"]` optional dependency in `pyproject.toml`; included in `all` extra
|
||||
- 110 integration tests in `tests/integrations/agno/` covering all public APIs, MemoryDb protocol compliance, GraphRAG search, tool registration, shared memory isolation, and thread-safety
|
||||
- 3 cookbook notebooks in `cookbook/integrations/`: `agno_decision_intelligence.ipynb` (loan underwriting), `agno_graphrag_context.ipynb` (regulatory compliance), `agno_multi_agent_shared_context.ipynb` (multi-agent team coordination)
|
||||
- Full reference documentation in `docs/integrations/agno.md`
|
||||
|
||||
- **Novita AI Provider** (PR #374 by @Alex-wuhu):
|
||||
- Added `NovitaProvider` — OpenAI-compatible integration via `https://api.novita.ai/v1`; supports `generate()` and `generate_structured()` (JSON forced format)
|
||||
- Default model: `deepseek/deepseek-v3.2`; configurable via `NOVITA_API_KEY` environment variable
|
||||
- Registered `"novita"` in the built-in provider factory; usable via `create_provider("novita")`
|
||||
- Added integration tests in `tests/test_novita_integration.py` with proper assertions and graceful skip when `NOVITA_API_KEY` is unset
|
||||
|
||||
- **Native Datalog Reasoning Engine** (PR #371, Issue #368 by @ZohaibHassan16, reviewed and fixed by @KaifAhmad1):
|
||||
- Added `DatalogReasoner` to `semantica.reasoning` — a pure-Python, bottom-up semi-naive fixpoint engine with guaranteed termination on finite graphs
|
||||
- Supports recursive Horn clause rules (e.g. `ancestor(X,Y) :- parent(X,Z), ancestor(Z,Y).`) that existing engines loop on indefinitely
|
||||
- Memory-optimized `_unify()` with deferred dict allocation — zero allocation on failed unifications
|
||||
- `O(1)` delta-index lookup per iteration eliminates redundant `O(N)` rule re-evaluations in semi-naive loop
|
||||
- `query("pred(?X, ?Y)")` returns variable-binding dicts; supports both uppercase `?Y` and lowercase `?y` variable syntax
|
||||
- `query(..., bindings={"Y": "val"})` pre-binds variables for exact-match verification
|
||||
- `load_from_graph(ContextGraph)` converts all edges and nodes to Datalog facts in one call; handles both `find_edges`/`find_nodes` and raw `edges`/`nodes` graph APIs
|
||||
- `add_fact()` accepts `"pred(a, b)"` strings and Semantica dicts (`subject/predicate/object`, `source/target/type`, `type/id` shapes); warns on unrecognised dict format instead of silently dropping
|
||||
- `_derived` cache flag — `derive_all()` skips re-evaluation when no facts or rules have changed since last run; `query()` respects the cache
|
||||
- Progress tracking wrapped in `try/finally` — `stop_tracking()` always called even on exception
|
||||
- `DatalogReasoner`, `DatalogFact`, `DatalogRule` exported from `semantica.reasoning`
|
||||
- 18 tests covering recursive rules, multi-hop inference, variable binding, graph integration, idempotency, and edge cases — all passing
|
||||
- **Ontology Diff & Migration** (PR #367 by @ZohaibHassan16, review & fixes by @KaifAhmad1):
|
||||
- `VersionManager.diff_ontologies(base, target)` — structured diff between two ontology dicts using hash-map lookups; handles URI-less items via `name` fallback; deep equality checks for unordered lists; now covers classes, properties, individuals, and axioms
|
||||
- `ChangeLogAnalyzer.analyze(diff)` — classifies each change by semantic impact: removed classes/properties → `CRITICAL/BREAKING`; narrowed domain/range/cardinality → `HIGH/BREAKING`; hierarchy modifications → `MEDIUM/POTENTIALLY_BREAKING`; added elements and annotation updates → `INFO/NON_BREAKING`
|
||||
- `ImpactReport` dataclass and `generate_change_report(diff)` public helper — returns a structured dict with `summary`, `impact_classification` (breaking / potentially_breaking / safe), `recommendations`, and the raw `diff`
|
||||
- `OntologyEngine.compare_versions(base_id, target_id, **options)` — end-to-end orchestrator: loads versions from `VersionManager`, runs `diff_ontologies`, generates impact report; accepts `base_dict`/`target_dict` overrides to bypass version store; `run_validation=True` triggers `OntologyValidator` on the target schema; `graph_data=...` additionally runs `GraphValidator` on instance data against the new schema
|
||||
- `OntologyEngine.get_ontology_version_dict(version_id)` — utility to load a registered version as a plain dict ready for diffing
|
||||
- Documentation added to `docs/reference/change_management.md`: "Ontology Diff & Migration" section with code example and full report format reference
|
||||
- 7 tests added to `tests/change_management/test_managers.py` covering: empty diff, unordered list equality, URI/name fallback, breaking class removal, narrowed domain (HIGH), safe additions and annotation changes, `compare_versions` dict override, version-not-found error path, individuals/axioms diff coverage, null constraint value flagged as breaking
|
||||
- **Fixes applied post-review (by @KaifAhmad1)**:
|
||||
- Fixed typo in `ChangeCategory` enum value: `"potenitally_breaking"` → `"potentially_breaking"`
|
||||
- Fixed missing space in impact description string: `f"New{entity_type}"` → `f"New {entity_type}"`
|
||||
- Added null-value guard in `_analyze_field_changes` — constraint fields with `None` old/new value are now correctly flagged as breaking instead of silently passing the subset check
|
||||
- Made `ChangeLogAnalyzer` stateless — `report` is now a local variable passed into `_generate_recommendations(report)` rather than stored as `self.report`; removes re-entrancy hazard
|
||||
- Removed no-op `__init__` from `ChangeLogAnalyzer`
|
||||
- Replaced non-portable emoji markers in recommendations (`✘✘✘`, `¤¤¤`, `☺☺☺`) with plain-text tags (`[BREAKING]`, `[WARNING]`, `[SAFE]`)
|
||||
- Extended `diff_ontologies` to cover `individuals` and `axioms` — previously only classes and properties were diffed; the public `compare_versions` path now returns all four element types
|
||||
- Fixed exception chaining in `compare_versions`: `raise ProcessingError(...) from e` to preserve original traceback
|
||||
- Removed silent `ImportError` swallow for `GraphValidator` — it is a first-party module; an `ImportError` indicates a broken install, not a graceful skip
|
||||
- Added comment on deferred `VersionManager` import in `OntologyEngine.__init__` explaining the circular-import constraint
|
||||
- Fixed import-before-docstring in `tests/change_management/test_managers.py`
|
||||
- Fixed broken Markdown link syntax in docs JSON example block: `"[http://...](http://...)"` → bare URI string
|
||||
- Updated docs recommendations example to match the new plain-text tag format
|
||||
|
||||
- **Ontology Alignment API** (PR #361 by @ZohaibHassan16, review & fixes by @KaifAhmad1):
|
||||
- Alignment representation using standard RDF predicates: `owl:equivalentClass`, `owl:equivalentProperty`, `owl:sameAs`, `skos:exactMatch`, `skos:closeMatch`, `skos:broadMatch`, `skos:narrowMatch`, `skos:relatedMatch`
|
||||
- `OntologyEngine.create_alignment(source_uri, target_uri, predicate)` — store alignment triples in TripletStore
|
||||
- `OntologyEngine.get_alignments(entity_uri)` — bidirectional retrieval of all alignments for an entity
|
||||
- `OntologyEngine.list_alignments(ontology_uri=None)` — list all alignments, optionally filtered by ontology namespace
|
||||
- `NamespaceManager.get_alignment_predicates()` — expose standard OWL/SKOS alignment URIs as a convenience dict
|
||||
- `ReuseManager.suggest_alignments(target, source)` — O(N+M) hashmap heuristic to suggest alignments based on exact label matches across ontologies
|
||||
- `ReuseManager.merge_ontology_data(..., compute_alignments=True)` — optionally attach suggested alignments to merge output without auto-committing unverified triples
|
||||
- `QueryEngine.expand_entity_uri(uri, store, use_alignments=True)` — bidirectional SPARQL expansion to include aligned equivalents; no-ops when flag is False
|
||||
- `QueryEngine.build_values_clause(variable, uris)` — generate a SPARQL `VALUES` clause for injecting expanded URIs into queries
|
||||
- Alignment-aware queries section added to `docs/reference/triplet_store.md`
|
||||
- Ontology Alignment section added to `docs/reference/ontology.md`
|
||||
- **Fixes applied post-review (by @KaifAhmad1)**:
|
||||
- Fixed progress tracker leak in `expand_entity_uri` — `stop_tracking` was only called inside the `hasattr(execute_sparql)` branch; backends without it silently leaked a tracker entry
|
||||
- Fixed `relatedMatch` predicate gap — `get_alignment_predicates()` exposed `skos:relatedMatch` but all three SPARQL FILTER lists omitted it, making those alignments permanently invisible
|
||||
- Fixed SPARQL injection in `list_alignments` — previously only `"` was escaped; `\`, `{`, and `}` are now also percent-encoded to prevent WHERE block breakout
|
||||
- Fixed SPARQL injection in `build_values_clause` — URIs now run through `_sanitize_uri` before wrapping in angle-bracket literals
|
||||
- Added full-URI validation in `create_alignment` — raises `ProcessingError` if predicate is a CURIE instead of a full URI, preventing silent storage of unqueryable triples
|
||||
- Fixed E2E test `test_end_to_end_cross_ontology_uri_flow` — previously mocked the method under test; now uses a real mock backend with `execute_sparql` to exercise the actual expansion and VALUES clause injection flow
|
||||
- 19 tests added covering: `create_alignment`, `get_alignments`, `suggest_alignments`, merge with alignment computation, `expand_entity_uri` (enabled/disabled), `build_values_clause`, and full E2E cross-ontology query flow
|
||||
- **Context Explainability Output Fixes** (PR pending on `context` by @KaifAhmad1):
|
||||
- Fixed decision-node storage in `ContextGraph` so full human-readable `scenario`, `reasoning`, and decision metadata are preserved on graph nodes instead of degrading into opaque IDs or truncated display text
|
||||
- Fixed causal and precedent reconstruction paths in the context module so returned `Decision` objects prefer readable stored fields over raw node identifiers
|
||||
- Fixed context aggregate outputs to return enriched readable payloads for influence, causality, similarity, policy-impact, and entity-similarity workflows instead of bare UUID lists or tuple-only results
|
||||
- Fixed `PolicyEngine.get_affected_decisions()` so both Cypher and fallback branches return consistent decision metadata including `scenario`, `category`, `outcome`, and `confidence`
|
||||
- Fixed `EntityLinker` similarity flows so enriched similarity results are consumed correctly across internal linking paths and public search aliases
|
||||
- Fixed downstream KG integrations in `node_embeddings`, `link_predictor`, `centrality_calculator`, `path_finder`, and context retrieval fallbacks to normalize enriched neighbor/node outputs without breaking graph algorithms
|
||||
- Added and updated regression tests covering readable decision text preservation, enriched causal/path outputs, policy-impact results, entity similarity payloads, and compatibility with KG consumers
|
||||
|
||||
## [0.3.0] - 2026-03-10
|
||||
|
||||
- **Context Graph Feature Completeness** (by @KaifAhmad1):
|
||||
- Added `valid_from` / `valid_until` temporal validity fields to `ContextNode` and `ContextEdge` dataclasses — both expose `is_active(at_time=None) -> bool`; nodes/edges without these fields are always considered active
|
||||
- Added `add_node(valid_from=..., valid_until=...)` and `add_edge(valid_from=..., valid_until=...)` support — validity windows are extracted from `**properties` and stored as first-class dataclass fields, not in metadata
|
||||
- Added `ContextGraph.find_active_nodes(node_type=None, at_time=None)` — returns only nodes whose validity window includes the given time (defaults to `datetime.utcnow()`); complements `find_nodes()` with temporal filtering
|
||||
- Added `min_weight: float = 0.0` parameter to `ContextGraph.get_neighbors()` — edges with weight below the threshold are skipped during BFS traversal, enabling weighted/confidence-filtered multi-hop navigation; fully backward-compatible (default 0.0 passes all edges)
|
||||
- Added `ContextGraph.link_graph(other_graph, source_node_id, target_node_id, link_type="CROSS_GRAPH") -> str` — creates a navigable bridge between two separate `ContextGraph` instances; records a marker edge internally and returns a `link_id`
|
||||
- Added `ContextGraph.navigate_to(link_id) -> (other_graph, target_node_id)` — resolves a `link_id` to the target graph and its entry node, enabling hierarchical cross-graph traversal (e.g. agent moving from a high-level decision graph into a domain-specific sub-graph)
|
||||
- Added `ContextGraph.resolve_links(registry)` — reconnects cross-graph links after `load_from_file()`; `save_to_file()` now persists a `links` section with `other_graph_id` so navigation survives the full save/load cycle
|
||||
- Added `graph_id` field to `ContextGraph` — stable UUID per instance, persisted to JSON, so separate graphs can identify each other after reload
|
||||
- Fixed `is_active()` on `ContextNode` and `ContextEdge` — tz-aware `datetime` inputs are now normalised to tz-naive UTC before comparison, preventing `TypeError` when callers pass `datetime.now(timezone.utc)`
|
||||
- Fixed `valid_from` / `valid_until` serialisation — `add_nodes()`, `add_edges()`, `to_dict()`, and `from_dict()` all now preserve and restore validity windows; previously these fields were silently lost
|
||||
- Fixed cross-graph link artifact — `link_graph()` now pre-creates a `"cross_graph_link"` typed `ContextNode` for the marker before inserting the marker edge, preventing `_add_internal_edge()` from auto-creating a phantom `"entity"` node
|
||||
- Added 14 tests in `tests/context/test_cross_graph_navigation.py` covering link creation, phantom-node prevention, and full save/load round-trips with `resolve_links()`
|
||||
- Fixed `pipeline_builder.add_step()` return type annotation from `"PipelineBuilder"` to `"PipelineStep"` — implementation was already correct per 0.3.0-beta changelog, only signature and docstring were stale
|
||||
- Fixed `test_hybrid_search_performance` timing computation — accumulated a real `search_times` list and compute true average; raised threshold to `< 5.0s` to account for real `sentence-transformers` (384-dim) latency
|
||||
|
||||
|
||||
|
||||
- **0.3.0 Bug Fixes & Comprehensive Real-World Tests** (by @KaifAhmad1):
|
||||
- Fixed `ProvenanceTracker` missing from `semantica/kg/__init__.py` exports — `from semantica.kg import ProvenanceTracker` now works correctly
|
||||
- Fixed duplicate relation creation in `_parse_relation_result` — orphaned legacy block was appending every relation twice; removed the duplicate block
|
||||
- Added `extraction_method` parameter to `_parse_relation_result`; typed extraction path now correctly sets `"llm_typed"` instead of `"llm"` in relation metadata
|
||||
- Fixed cross-test cache pollution in `tests/semantic_extract/test_retry_logic.py` — module-level `_result_cache` now cleared in `setUp()` to prevent intermittent failures when tests share input text
|
||||
- Added `tests/test_030_realworld_comprehensive.py`: 85 real-world tests covering all 0.3.0-alpha/beta features with real data (tech companies, CEOs, products, investment chains, healthcare scenarios)
|
||||
- ContextGraph basic operations and decision tracking lifecycle
|
||||
- KG algorithms: centrality, community detection, embeddings, path finding, similarity, link prediction, connectivity
|
||||
- PolicyEngine, DecisionQuery, AgentContext, Decision model serialization
|
||||
- ProvenanceTracker with GraphBuilderWithProvenance and AlgorithmTrackerWithProvenance
|
||||
- Deduplication v2 with blocking strategies, RDF/TTL export, Reasoner inference
|
||||
- Pipeline builder/validator/failure handler with retry policies
|
||||
- Multi-hop investment chain (Microsoft→OpenAI, Google→Anthropic) end-to-end
|
||||
- Healthcare entity extraction and knowledge graph construction E2E
|
||||
|
||||
## [0.3.0-beta] - 2026-03-07
|
||||
|
||||
- **Multi-Founder LLM Extraction & Reasoner Inference Fix** (PR #354 by @KaifAhmad1):
|
||||
|
||||
+6
-6
@@ -2,9 +2,9 @@
|
||||
|
||||
Thank you for your interest in contributing! Every contribution, no matter how small, is valuable. 🎉
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/N7WmAuDH)**
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
> **New to contributing?** Start with a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue) or join our [Discord](https://discord.gg/N7WmAuDH) community.
|
||||
> **New to contributing?** Start with a [`good first issue`](https://github.com/Hawksight-AI/semantica/labels/good%20first%20issue) or join our [Discord](https://discord.gg/sV34vps5hH) community.
|
||||
|
||||
---
|
||||
|
||||
@@ -15,7 +15,7 @@ Thank you for your interest in contributing! Every contribution, no matter how s
|
||||
3. Make your changes
|
||||
4. Submit a pull request!
|
||||
|
||||
**Need help?** Join [Discord](https://discord.gg/N7WmAuDH) or [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
**Need help?** Join [Discord](https://discord.gg/sV34vps5hH) or [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
|
||||
---
|
||||
|
||||
@@ -108,7 +108,7 @@ Thank you for your interest in contributing! Every contribution, no matter how s
|
||||
|
||||
**What:** Help others in the community
|
||||
|
||||
**Where:** [Discord](https://discord.gg/N7WmAuDH), [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
**Where:** [Discord](https://discord.gg/sV34vps5hH), [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)
|
||||
|
||||
**Examples:** Answer questions, review PRs, share your projects
|
||||
|
||||
@@ -326,7 +326,7 @@ result = instance.method()
|
||||
|
||||
## 🆘 Getting Help
|
||||
|
||||
- 💬 [Discord](https://discord.gg/N7WmAuDH) - Real-time chat
|
||||
- 💬 [Discord](https://discord.gg/sV34vps5hH) - Real-time chat
|
||||
- 💭 [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions) - Q&A
|
||||
- 🐛 [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) - Bug reports
|
||||
|
||||
@@ -363,4 +363,4 @@ This project follows a [Code of Conduct](CODE_OF_CONDUCT.md). Be respectful and
|
||||
|
||||
Every contribution matters - whether it's a single line of code, a typo fix, a helpful answer, or a bug report. We appreciate you! 🙏
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/N7WmAuDH)**
|
||||
⭐ **Give us a Star** • 🍴 **[Fork Semantica](https://github.com/Hawksight-AI/semantica/fork)** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ Thank you to all the people who have contributed to Semantica! 🎉
|
||||
|
||||
This project follows the [all-contributors](https://allcontributors.org) specification. Contributions of any kind are welcome!
|
||||
|
||||
⭐ **Give us a Star** • 🍴 **Fork us** • 💬 **Join our [Discord](https://discord.gg/N7WmAuDH)**
|
||||
⭐ **Give us a Star** • 🍴 **Fork us** • 💬 **Join our [Discord](https://discord.gg/sV34vps5hH)**
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
# PR Title
|
||||
Harden spaCy NER Fallback in Semantic Extract
|
||||
|
||||
## Summary
|
||||
This PR fixes a runtime failure in Semantic Extract when spaCy is installed but not actually usable at runtime, such as Python 3.12 / Pydantic v2 environments where `spacy.load("en_core_web_sm")` fails during config validation.
|
||||
|
||||
Instead of crashing during `NERExtractor(method="ml")` initialization or ML entity extraction, Semantica now logs the failure and falls back cleanly to non-ML extraction behavior.
|
||||
|
||||
## What Changed
|
||||
|
||||
### spaCy Initialization Hardening
|
||||
Updated `semantica/semantic_extract/ner_extractor.py` so `NERExtractor` no longer crashes if:
|
||||
|
||||
- spaCy is importable
|
||||
- the configured model exists
|
||||
- but `spacy.load(...)` fails at runtime for reasons other than missing files
|
||||
|
||||
This now degrades gracefully by leaving `self.nlp = None` and allowing fallback behavior.
|
||||
|
||||
### ML Extraction Fallback Hardening
|
||||
Updated `semantica/semantic_extract/methods.py` so `extract_entities_ml()` now catches:
|
||||
|
||||
- missing spaCy model errors
|
||||
- generic spaCy runtime initialization failures
|
||||
|
||||
If spaCy cannot initialize, extraction falls back to pattern-based extraction instead of raising.
|
||||
|
||||
### Regression Test
|
||||
Added a regression test in `tests/test_ner_configurations.py` covering the case where:
|
||||
|
||||
- spaCy is available
|
||||
- `spacy.load(...)` raises a runtime exception
|
||||
- `NERExtractor(method="ml")` still initializes safely
|
||||
|
||||
### Changelog
|
||||
Added an `Unreleased` changelog entry documenting the spaCy runtime fallback fix.
|
||||
|
||||
## Why This Matters
|
||||
This fixes benchmark and CI instability caused by spaCy runtime incompatibilities outside Semantica’s control.
|
||||
|
||||
It ensures Semantic Extract remains resilient when spaCy is present in the environment but broken due to dependency mismatches.
|
||||
|
||||
## Validation
|
||||
Tested with:
|
||||
|
||||
```bash
|
||||
pytest tests/test_ner_configurations.py -q -k "spacy_runtime_is_broken"
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```bash
|
||||
1 passed
|
||||
```
|
||||
|
||||
Note:
|
||||
The benchmark failure path was fixed directly, but the local `semantic-extract` branch did not contain the benchmark file path used in CI, so only the targeted regression path was verified locally.
|
||||
|
||||
## Files Changed
|
||||
|
||||
- `semantica/semantic_extract/ner_extractor.py`
|
||||
- `semantica/semantic_extract/methods.py`
|
||||
- `tests/test_ner_configurations.py`
|
||||
- `CHANGELOG.md`
|
||||
@@ -0,0 +1,282 @@
|
||||
# Semantica v0.3.0 — Release Notes
|
||||
|
||||
**Released:** 2026-03-10
|
||||
**PyPI:** `pip install semantica`
|
||||
**Tag:** [v0.3.0](https://github.com/Hawksight-AI/semantica/releases/tag/v0.3.0)
|
||||
**Classification:** Production/Stable
|
||||
|
||||
> First stable, full public release of Semantica. Covers everything shipped across three release stages: 0.3.0-alpha (2026-02-19), 0.3.0-beta (2026-03-07), and 0.3.0 stable (2026-03-10).
|
||||
|
||||
---
|
||||
|
||||
## Contributors
|
||||
|
||||
| Contributor | Role |
|
||||
|------------|------|
|
||||
| [@KaifAhmad1](https://github.com/KaifAhmad1) | Lead maintainer — context graph, decision intelligence, KG algorithms, semantic extraction, pipeline, provenance, bug fixes, release management |
|
||||
| [@ZohaibHassan16](https://github.com/ZohaibHassan16) | Deduplication v2 suite (candidate generation, two-stage scoring, semantic dedup), incremental/delta processing, benchmark suite |
|
||||
| [@Sameer6305](https://github.com/Sameer6305) | Apache AGE backend, PgVector store, Snowflake connector, Apache Arrow export |
|
||||
| [@tibisabau](https://github.com/tibisabau) | ArangoDB AQL export, Apache Parquet export |
|
||||
| [@d4ndr4d3](https://github.com/d4ndr4d3) | ResourceScheduler deadlock fix |
|
||||
|
||||
---
|
||||
|
||||
## v0.3.0 — Stable (2026-03-10)
|
||||
|
||||
### Context Graph Feature Completeness
|
||||
|
||||
**Temporal Validity Windows** (by @KaifAhmad1)
|
||||
|
||||
Nodes and edges now carry first-class `valid_from` / `valid_until` ISO datetime fields. These are stored directly on `ContextNode` and `ContextEdge` dataclasses — not in metadata — and survive full serialisation round-trips through `save_to_file()` / `load_from_file()` and `to_dict()` / `from_dict()`.
|
||||
|
||||
- `ContextNode.is_active(at_time=None)` and `ContextEdge.is_active(at_time=None)` — returns `True` if the node/edge is live at the given time (defaults to now). Handles both tz-aware and tz-naive datetime inputs correctly.
|
||||
- `ContextGraph.find_active_nodes(node_type=None, at_time=None)` — filters the entire graph and returns only nodes within their validity window.
|
||||
- `add_node(valid_from=..., valid_until=...)` and `add_edge(valid_from=..., valid_until=...)` — pass validity fields directly in the call signature.
|
||||
- Bug fix: `is_active()` previously crashed with `TypeError` when passed a tz-aware `datetime` (e.g. `datetime.now(timezone.utc)`). Fixed by normalising all inputs to tz-naive UTC via a new `_parse_iso_dt()` helper.
|
||||
- Bug fix: validity fields were silently lost in `add_nodes()`, `add_edges()`, `to_dict()`, and `from_dict()`. All four paths now correctly preserve and restore them.
|
||||
|
||||
**Weighted Multi-Hop BFS** (by @KaifAhmad1)
|
||||
|
||||
`ContextGraph.get_neighbors(node_id, hops=1, relationship_types=None, min_weight=0.0)` now accepts a `min_weight` threshold. Any edge with weight below the threshold is skipped during BFS traversal, allowing callers to confine multi-hop queries to high-confidence causal links. Default `0.0` is fully backward-compatible.
|
||||
|
||||
**Cross-Graph Navigation** (by @KaifAhmad1)
|
||||
|
||||
Separate `ContextGraph` instances can now be linked and navigated between — hierarchically, like separate knowledge domains that reference each other.
|
||||
|
||||
- `link_graph(other_graph, source_node_id, target_node_id, link_type="CROSS_GRAPH") -> str` — creates a navigable bridge and returns a `link_id`. Records a dedicated `"cross_graph_link"` typed marker node internally (not a phantom `"entity"`) and a marker edge.
|
||||
- `navigate_to(link_id) -> (other_graph, target_node_id)` — jumps to the target graph and entry node for a given link.
|
||||
- `graph_id` field — each `ContextGraph` now carries a stable UUID so instances can identify each other across save/load.
|
||||
- `save_to_file()` — now writes a `links` section alongside nodes and edges, containing `link_id`, `source_node_id`, `target_node_id`, and `other_graph_id` for every cross-graph link.
|
||||
- `load_from_file()` — restores `graph_id` and populates `_unresolved_links` from the `links` section.
|
||||
- `resolve_links(registry: Dict[str, ContextGraph]) -> int` — reconnects unresolved links post-load. Pass `{graph_id: graph_instance}` for each linked graph; returns the count of successfully resolved links. `navigate_to()` raises a clear `KeyError` with a `resolve_links()` hint if called before resolution.
|
||||
- Bug fix: the previous implementation auto-created the synthetic marker target as an `"entity"` node (phantom pollution). Fixed by explicitly pre-creating a `"cross_graph_link"` typed `ContextNode` before the marker edge.
|
||||
- 14 new tests in `tests/context/test_cross_graph_navigation.py` covering all scenarios including full save/load round-trips with partial registry resolution.
|
||||
|
||||
**Other Fixes** (by @KaifAhmad1)
|
||||
|
||||
- `PipelineBuilder.add_step()` return type annotation corrected from `"PipelineBuilder"` to `"PipelineStep"` — the implementation was already correct; only the annotation and docstring were stale.
|
||||
- `test_hybrid_search_performance` timing computation fixed — now accumulates a true `search_times` list instead of reusing the last loop iteration's `start_time`; threshold relaxed to `< 5.0s` for real `sentence-transformers` (384-dim) latency on development machines.
|
||||
|
||||
**Test Coverage Added**
|
||||
|
||||
- 14 cross-graph navigation tests (`tests/context/test_cross_graph_navigation.py`)
|
||||
- **Total: 335 context tests, 886+ tests across all modules — 0 failures**
|
||||
|
||||
---
|
||||
|
||||
## v0.3.0-beta — Beta (2026-03-07)
|
||||
|
||||
### Semantic Extraction Fixes
|
||||
|
||||
**Multi-Founder LLM Extraction & Reasoner Inference Fix** (PR #354, by @KaifAhmad1)
|
||||
|
||||
- `_parse_relation_result` in `methods.py` — unmatched subjects/objects now produce a synthetic `UNKNOWN` entity instead of silently dropping the relation. All co-founders returned by the LLM are preserved in the output.
|
||||
- Duplicate relation fix — an orphaned legacy block that appended every relation twice has been removed.
|
||||
- `extraction_method` parameter added — typed extraction paths now correctly record `"llm_typed"` in relation metadata instead of `"llm"`.
|
||||
- `_match_pattern` in `reasoner.py` rewritten — splits patterns on `?var` placeholders first, then escapes only literal segments. Pre-bound variables resolve to exact literals, repeated variables use backreferences, non-greedy `.+?` prevents over-consumption of separators.
|
||||
- Added `tests/reasoning/test_reasoner.py` (4 tests) and `tests/semantic_extract/test_relation_extractor.py` (6 tests).
|
||||
|
||||
**TTL Export Alias Fix** (PR #355, by @KaifAhmad1)
|
||||
|
||||
- `RDFExporter` now accepts `"ttl"`, `"nt"`, `"xml"`, `"rdf"`, and `"json-ld"` as format aliases in `export_to_rdf()`. Aliases resolve before format validation — zero public API changes.
|
||||
- Added `tests/export/test_rdf_exporter.py` (8 tests).
|
||||
|
||||
### Incremental / Delta Processing
|
||||
|
||||
**Native Delta Computation** (PR #349, by @ZohaibHassan16, reviewed and fixed by @KaifAhmad1)
|
||||
|
||||
- Native SPARQL-based diff between graph snapshots — only changed triples flow through the pipeline.
|
||||
- `delta_mode` configuration in `PipelineBuilder` for near-real-time workloads.
|
||||
- Version snapshot management with graph URI tracking and metadata storage.
|
||||
- `prune_versions()` for automatic snapshot retention cleanup.
|
||||
- Bug fixes: corrected SPARQL variable order, fixed class references, resolved duplicate dictionary keys.
|
||||
|
||||
### Deduplication v2
|
||||
|
||||
**Candidate Generation v2** (PR #338, by @ZohaibHassan16)
|
||||
|
||||
- New opt-in strategies: `blocking_v2` and `hybrid_v2`, replacing O(N²) pair enumeration.
|
||||
- Multi-key blocking with normalised token prefixes, type-aware keys, and optional phonetic (Soundex) blocking.
|
||||
- Deterministic `max_candidates_per_entity` budgeting with stable sorting.
|
||||
- **63.6% faster** in worst-case scenarios (0.259s → 0.094s for 100 entities).
|
||||
|
||||
**Two-Stage Scoring Prefilter** (PR #339, by @ZohaibHassan16)
|
||||
|
||||
- Fast gates for type mismatch, name-length ratio, and token overlap eliminate expensive semantic scoring for obvious non-matches.
|
||||
- Configurable thresholds: `min_length_ratio`, `min_token_overlap_ratio`, `required_shared_token`.
|
||||
- **18–25% faster** batch processing with prefilter enabled (`prefilter_enabled=False` by default).
|
||||
|
||||
**Semantic Relationship Deduplication v2** (PR #340, by @ZohaibHassan16, fixes by @KaifAhmad1)
|
||||
|
||||
- Canonicalisation engine with predicate synonym mapping (`works_for` → `employed_by`).
|
||||
- O(1) hash matching for exact canonical signatures.
|
||||
- Weighted scoring: 60% predicate + 40% object composition with explainable `semantic_match_score`.
|
||||
- **6.98x faster** than legacy mode (83ms vs 579ms).
|
||||
- `dedup_triplets()` infinite recursion bug fixed; function is now a first-class API in `methods.py`.
|
||||
|
||||
**Deduplication v2 Migration Guide** (PR #344, by @ZohaibHassan16, fixes by @KaifAhmad1)
|
||||
|
||||
- Comprehensive `MIGRATION_V2.md` documenting all v2 strategies with code examples.
|
||||
- Full backward compatibility maintained — legacy mode remains the default.
|
||||
|
||||
### Export Formats
|
||||
|
||||
**ArangoDB AQL Export** (PR #342, by @tibisabau)
|
||||
|
||||
- Full AQL INSERT statement generation for vertices and edges.
|
||||
- Configurable collection names with validation and sanitisation; batch processing (default: 1000).
|
||||
- `export_arango()` convenience function; `.aql` auto-detection in the unified exporter.
|
||||
- 17 tests, 100% pass rate.
|
||||
|
||||
**Apache Parquet Export** (PR #343, by @tibisabau)
|
||||
|
||||
- Columnar storage format with configurable compression: snappy, gzip, brotli, zstd, lz4, none.
|
||||
- Explicit Apache Arrow schemas with type safety; field normalisation for varied naming conventions.
|
||||
- `export_parquet()` convenience function; `.parquet` auto-detection.
|
||||
- Analytics-ready for pandas, Spark, Snowflake, BigQuery, Databricks.
|
||||
- 25 tests, 100% pass rate.
|
||||
|
||||
### Bug Fixes & Test Suite Stabilisation
|
||||
|
||||
**Test Suite Fixes** (by @KaifAhmad1)
|
||||
|
||||
Context module:
|
||||
- `retrieve_decision_precedents` — gated entity extraction on `use_hybrid_search=True` correctly.
|
||||
- `_extract_entities_from_query` — now uses `word[0].isupper()` to capture camelCase identifiers like `CreditCard`.
|
||||
- Added missing `expand_context` (BFS traversal) and `_get_decision_query` methods.
|
||||
- Fixed `hybrid_retrieval`, `dynamic_context_traversal`, and `multi_hop_context_assembly` for correct single-pass BFS.
|
||||
- Fixed `_retrieve_from_vector` fallback to `result["metadata"]["content"]` to prevent empty content and negative re-ranking scores.
|
||||
|
||||
KG module:
|
||||
- `calculate_pagerank` — added `alpha`/`max_iter` aliases; return format changed to `{"centrality": scores, "rankings": sorted_list}`.
|
||||
- `community_detector._to_networkx` — now returns a NetworkX graph directly when one is passed (previously lost all edges).
|
||||
- Added 9 domain-specific tracking methods to `AlgorithmTrackerWithProvenance`.
|
||||
- Created `provenance_tracker.py` with `ProvenanceTracker` (`track_entity`, `get_all_sources`, `clear`).
|
||||
|
||||
Pipeline module:
|
||||
- Retry loop fixed — now correctly iterates to `max_retries`.
|
||||
- `RecoveryAction` dataclass and `handle_failure(error, policy, retry_count)` added with LINEAR, EXPONENTIAL, and FIXED strategies.
|
||||
- `add_step()` fixed to return the created `PipelineStep`.
|
||||
- `validate` added as alias for `validate_pipeline` in `PipelineValidator`.
|
||||
|
||||
Other:
|
||||
- Fixed `NameError` for missing `Type` import in `utils/helpers.py`.
|
||||
- Vector store performance threshold relaxed (100ms → 500ms per decision for development machines).
|
||||
- Windows cp1252 encoding fix in test files (emoji → ASCII).
|
||||
- `ProvenanceTracker` added to `semantica/kg/__init__.py` exports.
|
||||
|
||||
**Results: ~840 tests passing, 36 skipped (external services), 0 failed**
|
||||
|
||||
---
|
||||
|
||||
## v0.3.0-alpha — Alpha (2026-02-19)
|
||||
|
||||
### Context & Decision Intelligence
|
||||
|
||||
**Context Engineering Enhancement** (PR #307, by @KaifAhmad1)
|
||||
|
||||
The foundational 0.3.0 feature — complete overhaul of the context module for production-grade decision intelligence:
|
||||
|
||||
- Full decision lifecycle: `record_decision()` → `add_decision()` → `add_causal_relationship()` → `trace_decision_chain()` → `analyze_decision_impact()` → `analyze_decision_influence()` → `find_similar_decisions()`
|
||||
- `AgentContext` unified wrapper with granular feature flags: `decision_tracking`, `kg_algorithms`, `graph_expansion`; methods: `store()`, `retrieve()`, `get_conversation_history()`, `get_statistics()`, `capture_cross_system_inputs()`
|
||||
- `AgentMemory` with working, conversation, and long-term memory tiers
|
||||
- `PolicyEngine` with versioned policy nodes, compliance checking (`check_decision_rules()`), and `PolicyException` model
|
||||
- Hybrid precedent search combining vector, structural, and category similarity with configurable weights
|
||||
- Decision influence analysis via centrality measures and causal chain tracking
|
||||
- GraphStore validation preventing runtime failures; secure logging
|
||||
- 9 critical bug fixes across logging, security, audit trails, API compatibility, Cypher queries, centrality access, validation
|
||||
|
||||
**Context Decision Tracking Fixes** (PR #315, by @KaifAhmad1)
|
||||
|
||||
- Fixed empty/None decision ID handling in `add_decision()`
|
||||
- Fixed None metadata handling preventing `TypeError`
|
||||
- Fixed causal chain depth logic and node exclusion
|
||||
- Fixed nonexistent node handling in `add_causal_relationship()`
|
||||
- Fixed precedent search direction in `find_precedents()`
|
||||
- Added missing `properties` field in `to_dict()`; added `from_dict()` method
|
||||
- Fixed UUID generation across all decision models
|
||||
- All 71 context tests passing
|
||||
|
||||
### Knowledge Graph Algorithms
|
||||
|
||||
**Improved Graph Algorithms** (PR #292, by @KaifAhmad1)
|
||||
|
||||
- 30+ graph algorithms across 7 categories
|
||||
- Node embeddings: Node2Vec, DeepWalk, Word2Vec via `NodeEmbedder`
|
||||
- Similarity: cosine, Euclidean, Manhattan, Correlation via `SimilarityCalculator`
|
||||
- Path finding: Dijkstra, A*, BFS, K-shortest paths via `PathFinder`
|
||||
- Link prediction: preferential attachment, Jaccard, Adamic-Adar via `LinkPredictor`
|
||||
- Centrality: degree, betweenness, closeness, PageRank via `CentralityAnalyzer`
|
||||
- Community detection: Louvain, Leiden, label propagation via `CommunityDetector`
|
||||
- Connectivity: components, bridges, density via `ConnectivityAnalyzer`
|
||||
- `GraphBuilderWithProvenance` and `AlgorithmTrackerWithProvenance` with full execution metadata
|
||||
|
||||
**Improved Vector Store for Decision Tracking** (PR #293, by @KaifAhmad1)
|
||||
|
||||
- `DecisionEmbeddingPipeline` with semantic and structural embeddings
|
||||
- `HybridSimilarityCalculator` with configurable weights (semantic: 0.7, structural: 0.3)
|
||||
- `ContextRetriever` with multi-hop reasoning
|
||||
- Convenience API: `quick_decision()`, `find_precedents()`, `explain()`, `similar_to()`, `batch_decisions()`, `filter_decisions()`
|
||||
- 34+ tests; performance: 0.028s per decision, 0.031s search, ~0.8KB memory per decision
|
||||
|
||||
### Graph Database Backends
|
||||
|
||||
**Apache AGE Backend Security Fixes** (PR #311, by @Sameer6305, fixes by @KaifAhmad1)
|
||||
|
||||
- `AgeStore` class with `GraphStore` API compatibility (openCypher via SQL on PostgreSQL)
|
||||
- SQL injection vulnerabilities fixed with input validation
|
||||
- psycopg2-binary dependency and migration guide added
|
||||
- Fixed parameter replacement and test mock leakage
|
||||
|
||||
**PgVector Store Support** (PR #303, by @Sameer6305, @KaifAhmad1)
|
||||
|
||||
- Native PostgreSQL vector storage using the pgvector extension
|
||||
- Multiple distance metrics: cosine, L2/Euclidean, inner product
|
||||
- HNSW and IVFFlat indexing for approximate nearest neighbour search
|
||||
- JSONB metadata storage with flexible filtering; batch operations
|
||||
- Connection pooling with psycopg3/psycopg2 fallback
|
||||
- SQL injection protection via `psycopg_sql.SQL()`; idempotent index and table management
|
||||
- 36+ tests with Docker integration
|
||||
|
||||
### Infrastructure
|
||||
|
||||
**ResourceScheduler Deadlock Fix** (PR #299, #301, by @d4ndr4d3, @KaifAhmad1)
|
||||
|
||||
- Replaced `threading.Lock()` with `threading.RLock()` to fix nested lock acquisition deadlock in `allocate_resources()`
|
||||
- Added `ValidationError` when no resources can be allocated
|
||||
- Progress tracking updates moved outside lock scope
|
||||
- 6 regression tests for deadlock prevention
|
||||
|
||||
**Security Configuration** (by @KaifAhmad1)
|
||||
|
||||
- Dependabot bi-weekly security updates with manual review
|
||||
- Automated security scans (Bandit, Safety, Semgrep) on schedule
|
||||
- Security-critical package grouping; zero auto-merge policy
|
||||
|
||||
---
|
||||
|
||||
## Summary by the Numbers
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Total tests passing | **886+** |
|
||||
| Test failures | **0** |
|
||||
| Context tests | 335 |
|
||||
| KG tests | ~430 |
|
||||
| Semantic extraction tests | 70 (9 skipped — external LLM APIs) |
|
||||
| Reasoning tests | 19 |
|
||||
| Real-world scenario tests | 85 |
|
||||
| PyPI classifier | Production/Stable |
|
||||
| Python support | 3.8 – 3.12 |
|
||||
|
||||
---
|
||||
|
||||
## Upgrade
|
||||
|
||||
```bash
|
||||
pip install --upgrade semantica
|
||||
```
|
||||
|
||||
No breaking changes. All new parameters have safe defaults and all new methods are additive.
|
||||
|
||||
See [CHANGELOG.md](CHANGELOG.md) for the full line-by-line diff.
|
||||
+1
-1
@@ -27,7 +27,7 @@ Start with our comprehensive documentation:
|
||||
|
||||
**Best for**: Real-time chat and quick questions
|
||||
|
||||
- [Join Discord](https://discord.gg/N7WmAuDH)
|
||||
- [Join Discord](https://discord.gg/sV34vps5hH)
|
||||
|
||||
#### GitHub Issues
|
||||
|
||||
|
||||
@@ -0,0 +1,534 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "title",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Agno × Semantica: Decision Intelligence Agent\n",
|
||||
"\n",
|
||||
"This notebook shows how to wire Semantica's **Decision Intelligence** stack into an Agno agent so it can:\n",
|
||||
"\n",
|
||||
"- Record every decision it makes with full reasoning provenance\n",
|
||||
"- Search historical precedents before acting\n",
|
||||
"- Validate decisions against policy rules\n",
|
||||
"- Trace causal chains across decisions\n",
|
||||
"- Accumulate institutional knowledge that survives across sessions\n",
|
||||
"\n",
|
||||
"**Domain used:** Financial loan underwriting (easily adapted to healthcare, legal, HR, etc.)\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Architecture\n",
|
||||
"\n",
|
||||
"```\n",
|
||||
"Agno Agent\n",
|
||||
" ├── memory=AgnoContextStore ← graph-backed persistent memory\n",
|
||||
" └── tools=[AgnoDecisionKit] ← decision tools the LLM can call\n",
|
||||
" │\n",
|
||||
" ├── record_decision ← Semantica AgentContext.record_decision()\n",
|
||||
" ├── find_precedents ← Semantica AgentContext.find_precedents_advanced()\n",
|
||||
" ├── trace_causal_chain ← Semantica ContextGraph.trace_decision_causality()\n",
|
||||
" ├── analyze_impact ← Semantica AgentContext.analyze_decision_influence()\n",
|
||||
" ├── check_policy ← Semantica PolicyEngine\n",
|
||||
" └── get_decision_summary ← Semantica AgentContext.get_context_insights()\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"## Install\n",
|
||||
"\n",
|
||||
"```bash\n",
|
||||
"pip install semantica[agno]\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "setup-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. Setup — Semantica Backends\n",
|
||||
"\n",
|
||||
"We build the Semantica components first. These are **independent of Agno** — you can swap backends without touching agent code."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "imports",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys, os\n",
|
||||
"sys.path.insert(0, os.path.abspath(\"../../\"))\n",
|
||||
"\n",
|
||||
"# ── Semantica core (not Agno-specific) ──────────────────────────────────────\n",
|
||||
"from semantica.context import AgentContext, ContextGraph\n",
|
||||
"from semantica.context import PolicyEngine, DecisionQuery, CausalChainAnalyzer\n",
|
||||
"from semantica.vector_store import VectorStore\n",
|
||||
"\n",
|
||||
"# ── Agno integration layer ───────────────────────────────────────────────────\n",
|
||||
"from integrations.agno import AgnoContextStore, AgnoDecisionKit, AGNO_AVAILABLE\n",
|
||||
"\n",
|
||||
"print(f\"Semantica imports OK\")\n",
|
||||
"print(f\"Agno installed: {AGNO_AVAILABLE}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "semantica-backends",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ── Vector store (FAISS, no external service needed) ────────────────────────\n",
|
||||
"vector_store = VectorStore(backend=\"faiss\", dimension=768)\n",
|
||||
"print(\"VectorStore ready (FAISS)\")\n",
|
||||
"\n",
|
||||
"# ── In-memory context graph with full analytics ──────────────────────────────\n",
|
||||
"knowledge_graph = ContextGraph(\n",
|
||||
" advanced_analytics=True,\n",
|
||||
" # Switch to neo4j for production:\n",
|
||||
" # backend=\"neo4j\", uri=\"bolt://localhost:7687\"\n",
|
||||
")\n",
|
||||
"print(\"ContextGraph ready (in-memory)\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "seed-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2. Seed Historical Decisions\n",
|
||||
"\n",
|
||||
"Before the agent runs, we pre-load historical decisions using **native Semantica APIs** so the precedent database is warm.\n",
|
||||
"\n",
|
||||
"In production you would ingest from a database or a prior session's graph export."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "seed-decisions",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Build a pure-Semantica AgentContext for seeding historical data\n",
|
||||
"seed_context = AgentContext(\n",
|
||||
" vector_store=vector_store,\n",
|
||||
" knowledge_graph=knowledge_graph,\n",
|
||||
" decision_tracking=True,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"historical_loans = [\n",
|
||||
" dict(\n",
|
||||
" category=\"loan_approval\",\n",
|
||||
" scenario=\"Applicant: credit score 740, income $95k, DTI 28%, down payment 20%\",\n",
|
||||
" reasoning=\"Strong credit history, debt load well below 35% threshold, adequate down payment\",\n",
|
||||
" outcome=\"approved\",\n",
|
||||
" confidence=0.96,\n",
|
||||
" ),\n",
|
||||
" dict(\n",
|
||||
" category=\"loan_approval\",\n",
|
||||
" scenario=\"Applicant: credit score 620, income $45k, DTI 42%, down payment 5%\",\n",
|
||||
" reasoning=\"Credit score below 650 floor, DTI exceeds 40% maximum, insufficient down payment\",\n",
|
||||
" outcome=\"rejected\",\n",
|
||||
" confidence=0.97,\n",
|
||||
" ),\n",
|
||||
" dict(\n",
|
||||
" category=\"loan_approval\",\n",
|
||||
" scenario=\"Applicant: credit score 700, income $72k, DTI 33%, down payment 15%\",\n",
|
||||
" reasoning=\"Adequate credit, moderate DTI within range, down payment slightly below ideal\",\n",
|
||||
" outcome=\"approved_with_conditions\",\n",
|
||||
" confidence=0.82,\n",
|
||||
" ),\n",
|
||||
" dict(\n",
|
||||
" category=\"loan_approval\",\n",
|
||||
" scenario=\"Applicant: credit score 780, income $130k, DTI 22%, down payment 30%\",\n",
|
||||
" reasoning=\"Excellent credit, low debt load, strong down payment — low-risk profile\",\n",
|
||||
" outcome=\"approved\",\n",
|
||||
" confidence=0.99,\n",
|
||||
" ),\n",
|
||||
" dict(\n",
|
||||
" category=\"loan_approval\",\n",
|
||||
" scenario=\"Applicant: credit score 660, income $58k, DTI 38%, down payment 10%\",\n",
|
||||
" reasoning=\"Borderline credit, high DTI, minimal down payment — escalated to senior review\",\n",
|
||||
" outcome=\"escalated\",\n",
|
||||
" confidence=0.70,\n",
|
||||
" ),\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for loan in historical_loans:\n",
|
||||
" did = seed_context.record_decision(**loan)\n",
|
||||
" print(f\" Seeded [{loan['outcome']:25s}] → {did}\")\n",
|
||||
"\n",
|
||||
"print(f\"\\n{len(historical_loans)} historical decisions loaded into Semantica KG\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "policy-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3. Define Policy Rules with Semantica\n",
|
||||
"\n",
|
||||
"We use `PolicyEngine` directly — no Agno involvement here. The `AgnoDecisionKit.check_policy` tool will call this engine during the agent's reasoning loop."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "policy",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"LENDING_POLICY_RULES = [\n",
|
||||
" \"credit_score >= 650\",\n",
|
||||
" \"dti <= 40\",\n",
|
||||
" \"down_payment_pct >= 10\",\n",
|
||||
" \"confidence >= 0.70\",\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# Verify directly with Semantica's PolicyEngine before wiring to Agno\n",
|
||||
"policy_engine = PolicyEngine(graph_store=knowledge_graph)\n",
|
||||
"\n",
|
||||
"test_application = {\"credit_score\": 720, \"dti\": 31, \"down_payment_pct\": 18, \"confidence\": 0.88}\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" result = policy_engine.check_compliance(test_application, LENDING_POLICY_RULES)\n",
|
||||
" print(f\"Policy check result: compliant={getattr(result, 'compliant', 'N/A')}\")\n",
|
||||
" print(f\"Violations: {getattr(result, 'violations', [])}\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"PolicyEngine fallback (expected without full rule engine): {e}\")\n",
|
||||
"\n",
|
||||
"print(\"\\nPolicy rules defined:\", LENDING_POLICY_RULES)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "agent-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 4. Build the Agno Decision-Intelligence Agent\n",
|
||||
"\n",
|
||||
"Now we wire everything into Agno using the integration classes.\n",
|
||||
"\n",
|
||||
"- `AgnoContextStore` gives the agent **persistent graph-backed memory**\n",
|
||||
"- `AgnoDecisionKit` exposes **6 decision tools** the LLM can invoke during reasoning"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "build-agent",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ── AgnoContextStore: wraps AgentContext as Agno MemoryDb ────────────────────\n",
|
||||
"store = AgnoContextStore(\n",
|
||||
" vector_store=vector_store, # Same store — shares seeded decisions\n",
|
||||
" knowledge_graph=knowledge_graph, # Same graph — shares seeded decisions\n",
|
||||
" decision_tracking=True,\n",
|
||||
" graph_expansion=True,\n",
|
||||
" session_id=\"loan_underwriter_v1\",\n",
|
||||
")\n",
|
||||
"print(\"AgnoContextStore ready\")\n",
|
||||
"\n",
|
||||
"# ── AgnoDecisionKit: exposes Semantica decision tools to Agno's LLM ──────────\n",
|
||||
"decision_kit = AgnoDecisionKit(\n",
|
||||
" context=store.context, # Reuse same AgentContext — shared decision history\n",
|
||||
" max_precedents=5,\n",
|
||||
" causal_depth=3,\n",
|
||||
" enable_policy_check=True,\n",
|
||||
")\n",
|
||||
"print(f\"AgnoDecisionKit ready — {len(decision_kit._tools)} tools registered\")\n",
|
||||
"print(\" Tools:\", [fn.__name__ for fn in decision_kit._tools])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "wire-agent",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if AGNO_AVAILABLE:\n",
|
||||
" from agno.agent import Agent\n",
|
||||
" from agno.memory import AgentMemory\n",
|
||||
" from agno.models.openai import OpenAIChat # or any Agno-supported model\n",
|
||||
"\n",
|
||||
" agent = Agent(\n",
|
||||
" name=\"LoanUnderwriter\",\n",
|
||||
" model=OpenAIChat(id=\"gpt-4o\"),\n",
|
||||
" memory=AgentMemory(db=store),\n",
|
||||
" tools=[decision_kit],\n",
|
||||
" show_tool_calls=True,\n",
|
||||
" description=(\n",
|
||||
" \"You are a senior loan underwriter. Before approving or rejecting any application:\"\n",
|
||||
" \" (1) find_precedents for similar past cases,\"\n",
|
||||
" \" (2) check_policy compliance,\"\n",
|
||||
" \" (3) record_decision with full reasoning.\"\n",
|
||||
" \" Always cite precedents and policy rule results in your explanation.\"\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
" print(\"Agno Agent assembled and ready\")\n",
|
||||
"else:\n",
|
||||
" print(\"Agno not installed — demonstrating tool calls directly below\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "demo-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 5. Demonstrate Decision Tools\n",
|
||||
"\n",
|
||||
"We call the decision tools **directly** so the notebook is fully runnable without an OpenAI key. When Agno is wired, the LLM orchestrates these same calls automatically."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "demo-find-precedents",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"# ── 5a. Find Precedents ───────────────────────────────────────────────────────\n",
|
||||
"print(\"=\" * 60)\n",
|
||||
"print(\"TOOL: find_precedents\")\n",
|
||||
"print(\"=\" * 60)\n",
|
||||
"\n",
|
||||
"new_application_scenario = (\n",
|
||||
" \"Applicant: credit score 715, income $82k, DTI 30%, down payment 18%\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"precedents_json = decision_kit.find_precedents(\n",
|
||||
" scenario=new_application_scenario,\n",
|
||||
" category=\"loan_approval\",\n",
|
||||
" limit=3,\n",
|
||||
")\n",
|
||||
"precedents = json.loads(precedents_json)\n",
|
||||
"print(f\"Found {precedents['count']} similar past decisions:\")\n",
|
||||
"for p in precedents['precedents']:\n",
|
||||
" print(f\" [{p.get('outcome','?'):25s}] confidence={p.get('confidence',0):.2f}\")\n",
|
||||
" print(f\" {p.get('scenario','')[:80]}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "demo-policy",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ── 5b. Check Policy ─────────────────────────────────────────────────────────\n",
|
||||
"print(\"=\" * 60)\n",
|
||||
"print(\"TOOL: check_policy\")\n",
|
||||
"print(\"=\" * 60)\n",
|
||||
"\n",
|
||||
"decision_data = json.dumps({\n",
|
||||
" \"credit_score\": 715,\n",
|
||||
" \"dti\": 30,\n",
|
||||
" \"down_payment_pct\": 18,\n",
|
||||
" \"confidence\": 0.88,\n",
|
||||
" \"outcome\": \"approved\",\n",
|
||||
"})\n",
|
||||
"\n",
|
||||
"policy_json = decision_kit.check_policy(\n",
|
||||
" decision_data=decision_data,\n",
|
||||
" policy_rules=json.dumps(LENDING_POLICY_RULES),\n",
|
||||
")\n",
|
||||
"policy_result = json.loads(policy_json)\n",
|
||||
"print(f\"Compliant: {policy_result.get('compliant')}\")\n",
|
||||
"print(f\"Violations: {policy_result.get('violations', [])}\")\n",
|
||||
"print(f\"Warnings: {policy_result.get('warnings', [])}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "demo-record",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ── 5c. Record Decision ──────────────────────────────────────────────────────\n",
|
||||
"print(\"=\" * 60)\n",
|
||||
"print(\"TOOL: record_decision\")\n",
|
||||
"print(\"=\" * 60)\n",
|
||||
"\n",
|
||||
"record_json = decision_kit.record_decision(\n",
|
||||
" category=\"loan_approval\",\n",
|
||||
" scenario=new_application_scenario,\n",
|
||||
" reasoning=(\n",
|
||||
" \"3 similar precedents found — 2 approved, 1 escalated. \"\n",
|
||||
" \"Credit score 715 exceeds 650 floor. DTI 30% well within 40% limit. \"\n",
|
||||
" \"Down payment 18% above 10% minimum. All policy rules satisfied.\"\n",
|
||||
" ),\n",
|
||||
" outcome=\"approved\",\n",
|
||||
" confidence=0.91,\n",
|
||||
" entities=\"loan_applicant, credit_bureau, lending_policy_v2\",\n",
|
||||
")\n",
|
||||
"record_result = json.loads(record_json)\n",
|
||||
"decision_id = record_result['decision_id']\n",
|
||||
"print(f\"Decision recorded: {decision_id}\")\n",
|
||||
"print(f\"Status: {record_result['status']}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "demo-impact",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ── 5d. Analyze Impact ───────────────────────────────────────────────────────\n",
|
||||
"print(\"=\" * 60)\n",
|
||||
"print(\"TOOL: analyze_impact\")\n",
|
||||
"print(\"=\" * 60)\n",
|
||||
"\n",
|
||||
"impact_json = decision_kit.analyze_impact(decision_id=decision_id)\n",
|
||||
"impact = json.loads(impact_json)\n",
|
||||
"print(\"Impact analysis:\")\n",
|
||||
"for k, v in impact.items():\n",
|
||||
" if k != \"decision_id\":\n",
|
||||
" print(f\" {k}: {v}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "demo-summary",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ── 5e. Decision Summary ─────────────────────────────────────────────────────\n",
|
||||
"print(\"=\" * 60)\n",
|
||||
"print(\"TOOL: get_decision_summary\")\n",
|
||||
"print(\"=\" * 60)\n",
|
||||
"\n",
|
||||
"summary_json = decision_kit.get_decision_summary(category=\"loan_approval\")\n",
|
||||
"summary = json.loads(summary_json)\n",
|
||||
"print(\"Decision history summary:\")\n",
|
||||
"for k, v in summary.items():\n",
|
||||
" if k not in (\"category_filter\",):\n",
|
||||
" print(f\" {k}: {v}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "agno-run-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 6. Run the Full Agno Agent (requires API key)\n",
|
||||
"\n",
|
||||
"When `AGNO_AVAILABLE=True` and an OpenAI key is set, the LLM orchestrates all the tool calls automatically."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "run-agent",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"NEW_CASE = (\n",
|
||||
" \"New mortgage application received:\\n\"\n",
|
||||
" \" Credit score: 715, Annual income: $82,000\\n\"\n",
|
||||
" \" Debt-to-income: 30%, Down payment: 18%\\n\"\n",
|
||||
" \" Loan amount: $320,000 for a primary residence in Austin TX\\n\"\n",
|
||||
" \"Should we approve this application?\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"if AGNO_AVAILABLE:\n",
|
||||
" agent.print_response(NEW_CASE)\n",
|
||||
"else:\n",
|
||||
" print(\"[Agno not installed — skipping live agent run]\")\n",
|
||||
" print()\n",
|
||||
" print(\"Expected agent reasoning flow:\")\n",
|
||||
" print(\" 1. find_precedents('credit score 715, DTI 30%, down payment 18%')\")\n",
|
||||
" print(\" → 2 approved, 1 escalated among similar cases\")\n",
|
||||
" print(\" 2. check_policy(credit_score=715, dti=30, down_payment_pct=18)\")\n",
|
||||
" print(\" → compliant=True, violations=[]\")\n",
|
||||
" print(\" 3. record_decision(outcome='approved', confidence=0.91)\")\n",
|
||||
" print(\" → decision_id recorded in Semantica KG\")\n",
|
||||
" print()\n",
|
||||
" print(\" Recommendation: APPROVE — 3 precedents + full policy compliance\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "analytics-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 7. Post-Session Analytics with Semantica\n",
|
||||
"\n",
|
||||
"After the agent session, use **native Semantica APIs** for reporting and causal analysis — no Agno required."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "analytics",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Query decision history directly from Semantica\n",
|
||||
"insights = store.context.get_context_insights()\n",
|
||||
"print(\"Session Insights (Semantica native):\")\n",
|
||||
"if isinstance(insights, dict):\n",
|
||||
" for k, v in insights.items():\n",
|
||||
" print(f\" {k}: {v}\")\n",
|
||||
"else:\n",
|
||||
" print(f\" {insights}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "precedents-direct",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Precedent search directly via Semantica's AgentContext\n",
|
||||
"# (same data, no Agno in the loop)\n",
|
||||
"precedents = store.context.find_precedents_advanced(\n",
|
||||
" scenario=\"borderline mortgage application\",\n",
|
||||
" category=\"loan_approval\",\n",
|
||||
")\n",
|
||||
"print(f\"\\nPrecedent search via Semantica directly → {len(precedents or [])} results\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "summary-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Summary\n",
|
||||
"\n",
|
||||
"| What | How |\n",
|
||||
"|---|---|\n",
|
||||
"| Persistent decision history | `AgnoContextStore` wrapping `AgentContext` + FAISS |\n",
|
||||
"| Tool calls for decision intelligence | `AgnoDecisionKit` (record, find, trace, check, summarise) |\n",
|
||||
"| Historical seeding | Native `AgentContext.record_decision()` — no Agno needed |\n",
|
||||
"| Policy rules | Native `PolicyEngine` — no Agno needed |\n",
|
||||
"| Post-session analytics | Native `AgentContext.get_context_insights()` — no Agno needed |\n",
|
||||
"\n",
|
||||
"The Agno integration is a **thin wrapper** — Semantica's full API remains directly accessible whenever you need finer control."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python",
|
||||
"version": "3.11.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,615 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "title",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Agno × Semantica: GraphRAG Context Agent\n",
|
||||
"\n",
|
||||
"This notebook demonstrates how to give an Agno agent a **relational knowledge graph** instead of a flat document store. The agent retrieves answers via **multi-hop graph traversal** — finding connections that pure vector search misses.\n",
|
||||
"\n",
|
||||
"**Domain:** Regulatory compliance (Basel IV / DORA) — documents are ingested, entities & relations extracted, then the agent answers questions by hopping through the graph.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Architecture\n",
|
||||
"\n",
|
||||
"```\n",
|
||||
"Agno Agent\n",
|
||||
" ├── knowledge=AgnoKnowledgeGraph ← GraphRAG knowledge base\n",
|
||||
" └── tools=[AgnoKGToolkit] ← live graph building/query tools\n",
|
||||
" │\n",
|
||||
" │ Backed by Semantica:\n",
|
||||
" ├── NERExtractor ← named entity recognition\n",
|
||||
" ├── RelationExtractor ← relation extraction\n",
|
||||
" ├── GraphBuilder ← builds ContextGraph from extractions\n",
|
||||
" ├── ContextGraph ← in-memory graph with analytics\n",
|
||||
" └── Reasoner ← rule-based inference\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"## Install\n",
|
||||
"\n",
|
||||
"```bash\n",
|
||||
"pip install semantica[agno]\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "imports-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. Imports — Semantica Core + Agno Integration"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "imports",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys, os, json\n",
|
||||
"sys.path.insert(0, os.path.abspath(\"../../\"))\n",
|
||||
"\n",
|
||||
"# ── Semantica core — used directly for pipeline setup ───────────────────────\n",
|
||||
"from semantica.kg import GraphBuilder\n",
|
||||
"from semantica.context import ContextGraph\n",
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor, TripletExtractor\n",
|
||||
"from semantica.reasoning import Reasoner\n",
|
||||
"from semantica.vector_store import VectorStore\n",
|
||||
"\n",
|
||||
"# ── Agno integration layer ───────────────────────────────────────────────────\n",
|
||||
"from integrations.agno import AgnoKnowledgeGraph, AgnoKGToolkit, AGNO_AVAILABLE\n",
|
||||
"\n",
|
||||
"print(\"Semantica imports OK\")\n",
|
||||
"print(f\"Agno installed: {AGNO_AVAILABLE}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "pipeline-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2. Build the Semantica Extraction Pipeline\n",
|
||||
"\n",
|
||||
"The extraction pipeline (NER → relation extraction → graph build) is pure Semantica. We construct each component explicitly so we can also use them for analysis outside Agno."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "build-pipeline",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# NER — identifies organisations, regulations, dates, amounts, roles\n",
|
||||
"ner = NERExtractor()\n",
|
||||
"\n",
|
||||
"# Relation extractor — finds typed edges between entities\n",
|
||||
"rel_extractor = RelationExtractor(confidence_threshold=0.60)\n",
|
||||
"\n",
|
||||
"# Knowledge graph builder\n",
|
||||
"graph_builder = GraphBuilder(merge_entities=True, temporal_support=True)\n",
|
||||
"\n",
|
||||
"# In-memory context graph (swap to neo4j/falkordb for persistence)\n",
|
||||
"context_graph = ContextGraph(advanced_analytics=True)\n",
|
||||
"\n",
|
||||
"# Reasoner for rule inference over the graph\n",
|
||||
"reasoner = Reasoner()\n",
|
||||
"\n",
|
||||
"print(\"Semantica extraction pipeline assembled\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ingest-raw-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3. Direct Semantica Extraction (Before Agno)\n",
|
||||
"\n",
|
||||
"We first demonstrate extraction using **raw Semantica APIs** so you can see exactly what goes into the graph.\n",
|
||||
"This is the same pipeline `AgnoKnowledgeGraph.load()` runs internally."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "raw-documents",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Regulatory documents (representative snippets)\n",
|
||||
"REGULATORY_DOCS = [\n",
|
||||
" {\n",
|
||||
" \"title\": \"Basel IV — Capital Requirements\",\n",
|
||||
" \"text\": (\n",
|
||||
" \"Basel IV introduces a revised standardised approach for credit risk, \"\n",
|
||||
" \"replacing internal model floors. Banks must maintain a minimum CET1 ratio \"\n",
|
||||
" \"of 4.5% and a total capital ratio of 8%. The BCBS finalised these requirements \"\n",
|
||||
" \"in December 2017 with a phased implementation starting January 2022. \"\n",
|
||||
" \"National regulators including the EBA and FCA are responsible for local \"\n",
|
||||
" \"transposition. Risk-weighted assets under Basel IV are calculated using \"\n",
|
||||
" \"the Output Floor, capping RWA reductions at 72.5%.\"\n",
|
||||
" ),\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"title\": \"DORA — Digital Operational Resilience Act\",\n",
|
||||
" \"text\": (\n",
|
||||
" \"DORA (Regulation EU 2022/2554) applies to financial entities and ICT \"\n",
|
||||
" \"third-party service providers operating in the EU. It mandates ICT risk \"\n",
|
||||
" \"management frameworks, incident classification, and annual operational \"\n",
|
||||
" \"resilience testing. Supervised entities must report major ICT incidents to \"\n",
|
||||
" \"the European Supervisory Authorities (ESAs) within 4 hours of classification. \"\n",
|
||||
" \"Critical ICT providers are subject to direct oversight by the Joint Oversight \"\n",
|
||||
" \"Network led by ESMA, EBA, and EIOPA. DORA became applicable on 17 January 2025.\"\n",
|
||||
" ),\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"title\": \"AML — Anti-Money Laundering Directive VI\",\n",
|
||||
" \"text\": (\n",
|
||||
" \"AMLD6 strengthens the EU's anti-money laundering framework by extending \"\n",
|
||||
" \"criminal liability to 22 predicate offences including cybercrime and \"\n",
|
||||
" \"environmental crime. Financial institutions must apply Customer Due Diligence \"\n",
|
||||
" \"(CDD) at onboarding and Enhanced Due Diligence (EDD) for high-risk customers. \"\n",
|
||||
" \"Suspicious Activity Reports (SARs) are filed with the national Financial \"\n",
|
||||
" \"Intelligence Unit (FIU). Non-compliance carries penalties up to 10% of \"\n",
|
||||
" \"annual global turnover. AMLD6 was transposed into UK law via MLCO 2020.\"\n",
|
||||
" ),\n",
|
||||
" },\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"print(f\"Documents to ingest: {len(REGULATORY_DOCS)}\")\n",
|
||||
"for doc in REGULATORY_DOCS:\n",
|
||||
" print(f\" • {doc['title']}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "run-ner",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ── Run NER directly with Semantica ─────────────────────────────────────────\n",
|
||||
"all_entities = []\n",
|
||||
"for doc in REGULATORY_DOCS:\n",
|
||||
" entities = ner.extract_entities(doc['text']) or []\n",
|
||||
" all_entities.extend(entities)\n",
|
||||
" print(f\"[{doc['title']}] → {len(entities)} entities\")\n",
|
||||
" for e in entities[:4]:\n",
|
||||
" print(f\" {getattr(e,'name','?'):30s} type={getattr(e,'type','?')} conf={getattr(e,'confidence',0):.2f}\")\n",
|
||||
"\n",
|
||||
"print(f\"\\nTotal entities extracted: {len(all_entities)}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "run-rel",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ── Run relation extraction directly with Semantica ──────────────────────────\n",
|
||||
"all_relations = []\n",
|
||||
"for doc in REGULATORY_DOCS:\n",
|
||||
" relations = rel_extractor.extract_relations(doc['text']) or []\n",
|
||||
" all_relations.extend(relations)\n",
|
||||
" print(f\"[{doc['title']}] → {len(relations)} relations\")\n",
|
||||
" for r in relations[:3]:\n",
|
||||
" src = getattr(r, 'source', '?')\n",
|
||||
" rtype = getattr(r, 'type', getattr(r, 'relation', '?'))\n",
|
||||
" tgt = getattr(r, 'target', '?')\n",
|
||||
" conf = getattr(r, 'confidence', 0)\n",
|
||||
" print(f\" {src!s:20s} --[{rtype}]--> {tgt!s:20s} conf={conf:.2f}\")\n",
|
||||
"\n",
|
||||
"print(f\"\\nTotal relations extracted: {len(all_relations)}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "agno-kg-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 4. Build AgnoKnowledgeGraph\n",
|
||||
"\n",
|
||||
"`AgnoKnowledgeGraph` wraps the extraction pipeline and implements Agno's `AgentKnowledge` protocol. It runs the same NER + relation extract + graph build pipeline internally — here we pass our pre-built components so the same instances are used."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "build-agno-kg",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"kg = AgnoKnowledgeGraph(\n",
|
||||
" graph_builder=graph_builder,\n",
|
||||
" ner_extractor=ner,\n",
|
||||
" relation_extractor=rel_extractor,\n",
|
||||
" context_graph=context_graph,\n",
|
||||
" num_documents=5,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Ingest all documents through the integration wrapper\n",
|
||||
"kg.load(texts=[doc['text'] for doc in REGULATORY_DOCS])\n",
|
||||
"\n",
|
||||
"print(f\"AgnoKnowledgeGraph: {len(kg._docs)} documents indexed\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "graphrag-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 5. GraphRAG Search\n",
|
||||
"\n",
|
||||
"The `search()` method implements **multi-hop GraphRAG**:\n",
|
||||
"1. Vector similarity over stored document texts\n",
|
||||
"2. Entity lookup in the context graph\n",
|
||||
"3. Graph hop expansion for entity neighbourhood\n",
|
||||
"4. Context injection into the returned documents"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "graphrag-search",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"queries = [\n",
|
||||
" \"What is the minimum CET1 ratio required under Basel IV?\",\n",
|
||||
" \"Which authorities supervise critical ICT providers under DORA?\",\n",
|
||||
" \"What are the reporting timelines for major ICT incidents?\",\n",
|
||||
" \"How does AMLD6 handle customer due diligence?\",\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for query in queries:\n",
|
||||
" print(f\"\\nQ: {query}\")\n",
|
||||
" results = kg.search(query, num_documents=2)\n",
|
||||
" print(f\" Retrieved {len(results)} document(s)\")\n",
|
||||
" for i, doc in enumerate(results, 1):\n",
|
||||
" content = getattr(doc, 'content', str(doc))\n",
|
||||
" print(f\" [{i}] {content[:120]}...\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "entity-context",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get graph context for a specific entity\n",
|
||||
"entity_contexts = [\"BCBS\", \"EBA\", \"DORA\", \"Basel IV\"]\n",
|
||||
"for entity in entity_contexts:\n",
|
||||
" ctx = kg.get_graph_context(entity)\n",
|
||||
" print(f\"\\nGraph context for '{entity}':\")\n",
|
||||
" print(ctx if ctx else \" (no graph nodes found — depends on NER extraction quality)\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "toolkit-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 6. AgnoKGToolkit — Live Graph Building\n",
|
||||
"\n",
|
||||
"The `AgnoKGToolkit` exposes 7 tools the LLM can call to **actively modify and query the graph** during reasoning."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "build-toolkit",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"toolkit = AgnoKGToolkit(\n",
|
||||
" ner_extractor=ner,\n",
|
||||
" relation_extractor=rel_extractor,\n",
|
||||
" reasoner=reasoner,\n",
|
||||
" context=context_graph, # share same graph as knowledge base\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(f\"AgnoKGToolkit: {len(toolkit._tools)} tools\")\n",
|
||||
"print(\" Tools:\", [fn.__name__ for fn in toolkit._tools])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "demo-extract-entities",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# TOOL: extract_entities\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"print(\"TOOL: extract_entities\")\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"\n",
|
||||
"new_text = (\n",
|
||||
" \"The PRA published a consultation paper requiring UK banks to \"\n",
|
||||
" \"implement DORA-equivalent resilience testing by Q3 2025, \"\n",
|
||||
" \"with Barclays and HSBC named as systemic institutions.\"\n",
|
||||
")\n",
|
||||
"entities_json = toolkit.extract_entities(new_text)\n",
|
||||
"entities_result = json.loads(entities_json)\n",
|
||||
"print(f\"Found {entities_result['count']} entities:\")\n",
|
||||
"for e in entities_result['entities']:\n",
|
||||
" print(f\" {e['name']:30s} type={e['type']:15s} conf={e['confidence']:.2f}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "demo-extract-relations",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# TOOL: extract_relations\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"print(\"TOOL: extract_relations\")\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"\n",
|
||||
"relations_json = toolkit.extract_relations(new_text)\n",
|
||||
"relations_result = json.loads(relations_json)\n",
|
||||
"print(f\"Found {relations_result['count']} relations:\")\n",
|
||||
"for r in relations_result['relations']:\n",
|
||||
" print(f\" {r['source']:20s} --[{r['relation']}]--> {r['target']:20s}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "demo-add-graph",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# TOOL: add_to_graph\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"print(\"TOOL: add_to_graph\")\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"\n",
|
||||
"add_result = json.loads(toolkit.add_to_graph(\n",
|
||||
" entities=json.dumps([\n",
|
||||
" {\"name\": \"PRA\", \"type\": \"REGULATOR\"},\n",
|
||||
" {\"name\": \"Barclays\", \"type\": \"BANK\"},\n",
|
||||
" {\"name\": \"HSBC\", \"type\": \"BANK\"},\n",
|
||||
" ]),\n",
|
||||
" relations=json.dumps([\n",
|
||||
" {\"source\": \"PRA\", \"relation\": \"SUPERVISES\", \"target\": \"Barclays\"},\n",
|
||||
" {\"source\": \"PRA\", \"relation\": \"SUPERVISES\", \"target\": \"HSBC\"},\n",
|
||||
" {\"source\": \"Barclays\", \"relation\": \"SUBJECT_TO\", \"target\": \"DORA\"},\n",
|
||||
" {\"source\": \"HSBC\", \"relation\": \"SUBJECT_TO\", \"target\": \"DORA\"},\n",
|
||||
" ]),\n",
|
||||
"))\n",
|
||||
"print(f\"Added: {add_result['nodes_added']} nodes, {add_result['edges_added']} edges\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "demo-query-graph",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# TOOL: query_graph\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"print(\"TOOL: query_graph\")\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"\n",
|
||||
"query_result = json.loads(toolkit.query_graph(\"PRA\"))\n",
|
||||
"print(f\"Keyword query 'PRA' → {query_result['count']} node(s):\")\n",
|
||||
"for node in query_result['results']:\n",
|
||||
" print(f\" label={node.get('label')} type={node.get('type')}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "demo-find-related",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# TOOL: find_related\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"print(\"TOOL: find_related\")\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"\n",
|
||||
"related_result = json.loads(toolkit.find_related(\"Barclays\", hops=2))\n",
|
||||
"print(f\"Related to 'Barclays' (2 hops): {related_result['count']} entity/entities\")\n",
|
||||
"for name in related_result['related']:\n",
|
||||
" print(f\" → {name}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "demo-infer",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# TOOL: infer_facts — Semantica's Reasoner derives new facts from graph state\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"print(\"TOOL: infer_facts\")\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"\n",
|
||||
"# Rules: regulatory compliance inference\n",
|
||||
"inference_rules = json.dumps([\n",
|
||||
" \"IF BANK(?x) THEN FinancialEntity(?x)\",\n",
|
||||
" \"IF REGULATOR(?x) THEN SupervisoryAuthority(?x)\",\n",
|
||||
" \"IF FinancialEntity(?x) THEN ComplianceSubject(?x)\",\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"infer_result = json.loads(toolkit.infer_facts(rules=inference_rules))\n",
|
||||
"print(f\"Inferred {infer_result['count']} new fact(s):\")\n",
|
||||
"for fact in infer_result['inferred_facts'][:8]:\n",
|
||||
" print(f\" {fact}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "demo-export",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# TOOL: export_subgraph — export knowledge for downstream systems\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"print(\"TOOL: export_subgraph (JSON-LD)\")\n",
|
||||
"print(\"=\" * 55)\n",
|
||||
"\n",
|
||||
"export_result = json.loads(toolkit.export_subgraph(entity=\"DORA\", format=\"json-ld\"))\n",
|
||||
"print(f\"Exported as format='{export_result['format']}'\")\n",
|
||||
"if 'data' in export_result:\n",
|
||||
" preview = str(export_result['data'])[:300]\n",
|
||||
" print(f\"Preview: {preview}...\")\n",
|
||||
"elif 'nodes' in export_result:\n",
|
||||
" print(f\"Graph nodes exported: {len(export_result['nodes'])}\")\n",
|
||||
" for node in export_result['nodes'][:5]:\n",
|
||||
" print(f\" {node}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "agno-run-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 7. Run the Full Agno GraphRAG Agent (requires API key)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "agno-agent",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if AGNO_AVAILABLE:\n",
|
||||
" from agno.agent import Agent\n",
|
||||
" from agno.models.openai import OpenAIChat\n",
|
||||
"\n",
|
||||
" compliance_agent = Agent(\n",
|
||||
" name=\"ComplianceAnalyst\",\n",
|
||||
" model=OpenAIChat(id=\"gpt-4o\"),\n",
|
||||
" knowledge=kg,\n",
|
||||
" search_knowledge=True,\n",
|
||||
" tools=[toolkit],\n",
|
||||
" show_tool_calls=True,\n",
|
||||
" description=(\n",
|
||||
" \"You are a regulatory compliance analyst. Use the knowledge graph \"\n",
|
||||
" \"to answer questions about Basel IV, DORA, and AML regulations. \"\n",
|
||||
" \"When answering, use find_related and query_graph to discover \"\n",
|
||||
" \"connections between regulators, rules, and institutions.\"\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" compliance_agent.print_response(\n",
|
||||
" \"Which supervisory authorities are responsible for overseeing DORA compliance \"\n",
|
||||
" \"for UK banks, and how does this relate to Basel IV capital requirements?\"\n",
|
||||
" )\n",
|
||||
"else:\n",
|
||||
" print(\"[Agno not installed — skipping live agent run]\")\n",
|
||||
" print()\n",
|
||||
" print(\"Expected reasoning flow:\")\n",
|
||||
" print(\" search_knowledge('DORA supervisory authorities UK banks')\")\n",
|
||||
" print(\" → retrieves DORA doc with graph expansion\")\n",
|
||||
" print(\" query_graph('PRA') → finds PRA node\")\n",
|
||||
" print(\" find_related('PRA', hops=2) → PRA → SUPERVISES → Barclays, HSBC\")\n",
|
||||
" print(\" find_related('Basel IV', hops=1) → capital ratio requirements\")\n",
|
||||
" print(\" Answer: PRA supervises UK banks under DORA; Basel IV CET1 requirement is 4.5%\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "semantica-analysis",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 8. Post-Session Graph Analysis with Semantica\n",
|
||||
"\n",
|
||||
"After the agent session, use Semantica's graph analytics directly to explore the accumulated knowledge."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "graph-analytics",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Use Semantica's GraphAnalyzer directly on the same ContextGraph\n",
|
||||
"from semantica.kg import GraphAnalyzer, CentralityCalculator, PathFinder\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" analyzer = GraphAnalyzer()\n",
|
||||
" analysis = analyzer.analyze_graph(context_graph)\n",
|
||||
" print(\"Graph analysis (Semantica native):\")\n",
|
||||
" if isinstance(analysis, dict):\n",
|
||||
" for k, v in list(analysis.items())[:8]:\n",
|
||||
" print(f\" {k}: {v}\")\n",
|
||||
" else:\n",
|
||||
" print(f\" {analysis}\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"GraphAnalyzer: {e}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "centrality",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Centrality — which entities are most connected / influential?\n",
|
||||
"try:\n",
|
||||
" centrality = CentralityCalculator()\n",
|
||||
" scores = centrality.calculate_degree_centrality(context_graph)\n",
|
||||
" print(\"Degree centrality (most connected entities):\")\n",
|
||||
" if isinstance(scores, dict):\n",
|
||||
" top = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:5]\n",
|
||||
" for entity, score in top:\n",
|
||||
" print(f\" {entity:30s} {score:.4f}\")\n",
|
||||
" else:\n",
|
||||
" print(f\" {scores}\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"CentralityCalculator: {e}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "summary-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Summary\n",
|
||||
"\n",
|
||||
"| Component | Role | Library |\n",
|
||||
"|---|---|---|\n",
|
||||
"| `NERExtractor` | Extract regulatory entities from text | Semantica |\n",
|
||||
"| `RelationExtractor` | Extract typed edges between entities | Semantica |\n",
|
||||
"| `GraphBuilder` | Build `ContextGraph` from extractions | Semantica |\n",
|
||||
"| `Reasoner` | Infer new facts from graph state | Semantica |\n",
|
||||
"| `AgnoKnowledgeGraph` | GraphRAG `AgentKnowledge` interface | Agno integration |\n",
|
||||
"| `AgnoKGToolkit` | 7 live graph tools for the Agno LLM | Agno integration |\n",
|
||||
"| `GraphAnalyzer` / `CentralityCalculator` | Post-session analytics | Semantica |\n",
|
||||
"\n",
|
||||
"The Agno integration wraps Semantica components — the full Semantica API is available for pre/post-processing and analytics independently of the agent."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python",
|
||||
"version": "3.11.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,676 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "title",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Agno × Semantica: Multi-Agent Shared Context\n",
|
||||
"\n",
|
||||
"This notebook shows how an Agno **Team** of specialist agents can share a single `ContextGraph` so they:\n",
|
||||
"\n",
|
||||
"- Never make contradictory decisions\n",
|
||||
"- Reuse each other's extracted knowledge without coupling implementations\n",
|
||||
"- Maintain a full causal audit trail across all agents\n",
|
||||
"\n",
|
||||
"**Scenario:** A product strategy team with three specialist agents:\n",
|
||||
"\n",
|
||||
"| Agent | Role | Tools |\n",
|
||||
"|---|---|---|\n",
|
||||
"| `Researcher` | Extracts competitive intelligence from text | `AgnoKGToolkit` |\n",
|
||||
"| `Analyst` | Evaluates opportunities and records decisions | `AgnoDecisionKit` |\n",
|
||||
"| `Strategist` | Synthesises both into a recommendation | both |\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Architecture\n",
|
||||
"\n",
|
||||
"```\n",
|
||||
"AgnoSharedContext (single ContextGraph + VectorStore)\n",
|
||||
" │\n",
|
||||
" ├── bind_agent(\"researcher\") → AgnoContextStore (role-scoped)\n",
|
||||
" ├── bind_agent(\"analyst\") → AgnoContextStore (role-scoped)\n",
|
||||
" └── bind_agent(\"strategist\") → AgnoContextStore (role-scoped)\n",
|
||||
"\n",
|
||||
"Agno Team\n",
|
||||
" ├── Researcher memory=researcher_store tools=[AgnoKGToolkit(context=shared)]\n",
|
||||
" ├── Analyst memory=analyst_store tools=[AgnoDecisionKit(context=shared)]\n",
|
||||
" └── Strategist memory=strategist_store tools=[AgnoKGToolkit, AgnoDecisionKit]\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"## Install\n",
|
||||
"\n",
|
||||
"```bash\n",
|
||||
"pip install semantica[agno]\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "imports-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1. Imports"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "imports",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys, os, json\n",
|
||||
"sys.path.insert(0, os.path.abspath(\"../../\"))\n",
|
||||
"\n",
|
||||
"# ── Semantica core ───────────────────────────────────────────────────────────\n",
|
||||
"from semantica.context import ContextGraph, AgentContext, CausalChainAnalyzer\n",
|
||||
"from semantica.vector_store import VectorStore\n",
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor\n",
|
||||
"from semantica.reasoning import Reasoner\n",
|
||||
"from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator\n",
|
||||
"\n",
|
||||
"# ── Agno integration ─────────────────────────────────────────────────────────\n",
|
||||
"from integrations.agno import (\n",
|
||||
" AgnoSharedContext,\n",
|
||||
" AgnoDecisionKit,\n",
|
||||
" AgnoKGToolkit,\n",
|
||||
" AGNO_AVAILABLE,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"Semantica imports OK\")\n",
|
||||
"print(f\"Agno installed: {AGNO_AVAILABLE}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "shared-context-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2. Build the Shared Semantica Backend\n",
|
||||
"\n",
|
||||
"A single `VectorStore` and `ContextGraph` underpin the entire team. All agents read and write to the same store — role scoping is applied automatically by `AgnoSharedContext`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "build-shared",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ── Single shared backends ───────────────────────────────────────────────────\n",
|
||||
"shared_vector_store = VectorStore(backend=\"faiss\", dimension=768)\n",
|
||||
"shared_graph = ContextGraph(advanced_analytics=True)\n",
|
||||
"\n",
|
||||
"print(\"Shared VectorStore (FAISS) ready\")\n",
|
||||
"print(\"Shared ContextGraph ready\")\n",
|
||||
"\n",
|
||||
"# ── AgnoSharedContext: the team coordinator ───────────────────────────────────\n",
|
||||
"shared = AgnoSharedContext(\n",
|
||||
" vector_store=shared_vector_store,\n",
|
||||
" knowledge_graph=shared_graph,\n",
|
||||
" decision_tracking=True,\n",
|
||||
" session_id=\"product_strategy_team_q1_2026\",\n",
|
||||
")\n",
|
||||
"print(f\"\\nAgnoSharedContext ready — session: {shared.session_id}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "bind-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3. Bind Agent Roles\n",
|
||||
"\n",
|
||||
"Each agent gets a **role-scoped** `AgnoContextStore` via `bind_agent()`. All agents share the same underlying graph, but their writes are tagged with their role for filtering."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "bind-agents",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Bind each agent role — idempotent, can be called multiple times safely\n",
|
||||
"researcher_store = shared.bind_agent(\"researcher\")\n",
|
||||
"analyst_store = shared.bind_agent(\"analyst\")\n",
|
||||
"strategist_store = shared.bind_agent(\"strategist\")\n",
|
||||
"\n",
|
||||
"print(\"Agent roles bound:\")\n",
|
||||
"for role in shared.bound_roles:\n",
|
||||
" store = shared.bind_agent(role)\n",
|
||||
" print(f\" {role:15s} → session={store.session_id}\")\n",
|
||||
"\n",
|
||||
"# Verify all roles see the same underlying knowledge_graph\n",
|
||||
"assert researcher_store._ctx is analyst_store._ctx\n",
|
||||
"print(\"\\nAll agents share the same AgentContext ✓\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "seed-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 4. Pre-Load Competitive Intelligence\n",
|
||||
"\n",
|
||||
"Using **native Semantica APIs**, we load a competitive landscape into the shared graph. This represents knowledge the team has accumulated from prior research sessions."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "seed-intel",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Competitive intelligence documents\n",
|
||||
"COMPETITIVE_INTEL = [\n",
|
||||
" {\n",
|
||||
" \"source\": \"market_research_q4_2025\",\n",
|
||||
" \"text\": (\n",
|
||||
" \"Competitor Alpha launched a new SaaS analytics platform in Q4 2025. \"\n",
|
||||
" \"The product targets mid-market enterprises with annual revenue between \"\n",
|
||||
" \"$50M–$500M and has attracted 200 paying customers within 3 months. \"\n",
|
||||
" \"Pricing is $2,000/seat/year with volume discounts at 50+ seats. \"\n",
|
||||
" \"Alpha raised a $80M Series C led by Sequoia Capital in November 2025.\"\n",
|
||||
" ),\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"source\": \"customer_interviews_q4_2025\",\n",
|
||||
" \"text\": (\n",
|
||||
" \"Customer interviews reveal strong demand for AI-powered anomaly detection \"\n",
|
||||
" \"in financial reporting workflows. 78% of CFOs surveyed cite 'time to insight' \"\n",
|
||||
" \"as the top pain point — currently averaging 14 days per reporting cycle. \"\n",
|
||||
" \"Competitor Alpha scores poorly on integration depth (NPS: 24) while \"\n",
|
||||
" \"our legacy product scores 41. Customers value our data governance features \"\n",
|
||||
" \"but want a modern UI and sub-second query times.\"\n",
|
||||
" ),\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"source\": \"technology_scan_q4_2025\",\n",
|
||||
" \"text\": (\n",
|
||||
" \"Emerging technologies for consideration: LLM-native analytics interfaces \"\n",
|
||||
" \"reduce time-to-insight by 60% in pilot studies (Stanford HAI, 2025). \"\n",
|
||||
" \"Graph-based anomaly detection outperforms time-series approaches for \"\n",
|
||||
" \"multi-entity financial fraud by 34% (ACM SIGMOD 2025). \"\n",
|
||||
" \"Vector database adoption in enterprise analytics grew 120% YoY. \"\n",
|
||||
" \"Apache Arrow and DuckDB emerging as standards for in-process OLAP.\"\n",
|
||||
" ),\n",
|
||||
" },\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# Use Semantica NER + RelationExtractor directly for rich extraction\n",
|
||||
"ner = NERExtractor()\n",
|
||||
"rel_extractor = RelationExtractor(confidence_threshold=0.55)\n",
|
||||
"graph_builder = GraphBuilder(merge_entities=True)\n",
|
||||
"\n",
|
||||
"for doc in COMPETITIVE_INTEL:\n",
|
||||
" text = doc['text']\n",
|
||||
" entities = ner.extract_entities(text) or []\n",
|
||||
" relations = rel_extractor.extract_relations(text) or []\n",
|
||||
" print(f\"[{doc['source']}]\")\n",
|
||||
" print(f\" Entities: {len(entities)}, Relations: {len(relations)}\")\n",
|
||||
" # Store into shared context for all agents to access\n",
|
||||
" shared._context.store(text, conversation_id=doc['source'])\n",
|
||||
"\n",
|
||||
"print(\"\\nCompetitive intelligence loaded into shared context\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "tools-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 5. Build Agent-Specific Tools\n",
|
||||
"\n",
|
||||
"Each toolkit is pointed at the **shared context** so tool calls across agents modify and read the same graph."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "build-tools",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Researcher's KG toolkit — builds knowledge from raw text\n",
|
||||
"researcher_kg_kit = AgnoKGToolkit(\n",
|
||||
" ner_extractor=ner,\n",
|
||||
" relation_extractor=rel_extractor,\n",
|
||||
" reasoner=Reasoner(),\n",
|
||||
" context=shared.knowledge_graph, # shared graph\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Analyst's decision kit — records evaluations and finds precedents\n",
|
||||
"analyst_decision_kit = AgnoDecisionKit(\n",
|
||||
" context=shared._context, # shared AgentContext\n",
|
||||
" max_precedents=5,\n",
|
||||
" causal_depth=3,\n",
|
||||
" enable_policy_check=True,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Strategist gets both\n",
|
||||
"strategist_kg_kit = AgnoKGToolkit(\n",
|
||||
" ner_extractor=ner,\n",
|
||||
" relation_extractor=rel_extractor,\n",
|
||||
" reasoner=Reasoner(),\n",
|
||||
" context=shared.knowledge_graph,\n",
|
||||
")\n",
|
||||
"strategist_decision_kit = AgnoDecisionKit(\n",
|
||||
" context=shared._context,\n",
|
||||
" max_precedents=5,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(f\"Researcher toolkit: {len(researcher_kg_kit._tools)} tools\")\n",
|
||||
"print(f\"Analyst toolkit: {len(analyst_decision_kit._tools)} tools\")\n",
|
||||
"print(f\"Strategist toolkits: {len(strategist_kg_kit._tools)} + {len(strategist_decision_kit._tools)} tools\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "simulate-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 6. Simulate Agent Collaboration\n",
|
||||
"\n",
|
||||
"We simulate the agents' reasoning steps directly, showing how shared context propagates knowledge between roles."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "researcher-turn",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(\"=\" * 65)\n",
|
||||
"print(\"RESEARCHER AGENT TURN\")\n",
|
||||
"print(\"=\" * 65)\n",
|
||||
"\n",
|
||||
"# Researcher extracts entities from new competitive intel\n",
|
||||
"new_intel = (\n",
|
||||
" \"Competitor Beta just closed a strategic partnership with Microsoft Azure, \"\n",
|
||||
" \"integrating their anomaly detection engine natively into Azure Synapse Analytics. \"\n",
|
||||
" \"This gives Beta access to Microsoft's 300,000+ enterprise customer base. \"\n",
|
||||
" \"Beta's CEO Sarah Chen announced the deal at Gartner Data & Analytics Summit.\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Step 1: Extract entities\n",
|
||||
"entities_result = json.loads(researcher_kg_kit.extract_entities(new_intel))\n",
|
||||
"print(f\"\\n[researcher] extracted {entities_result['count']} entities:\")\n",
|
||||
"for e in entities_result['entities']:\n",
|
||||
" print(f\" {e['name']:30s} type={e['type']}\")\n",
|
||||
"\n",
|
||||
"# Step 2: Extract relations\n",
|
||||
"relations_result = json.loads(researcher_kg_kit.extract_relations(new_intel))\n",
|
||||
"print(f\"\\n[researcher] extracted {relations_result['count']} relations\")\n",
|
||||
"\n",
|
||||
"# Step 3: Add to shared graph — now visible to ALL agents\n",
|
||||
"add_result = json.loads(researcher_kg_kit.add_to_graph(\n",
|
||||
" entities=json.dumps([\n",
|
||||
" {\"name\": \"Competitor Beta\", \"type\": \"COMPANY\"},\n",
|
||||
" {\"name\": \"Microsoft Azure\", \"type\": \"COMPANY\"},\n",
|
||||
" {\"name\": \"Azure Synapse Analytics\", \"type\": \"PRODUCT\"},\n",
|
||||
" {\"name\": \"Sarah Chen\", \"type\": \"PERSON\"},\n",
|
||||
" {\"name\": \"Gartner Data & Analytics Summit\", \"type\": \"EVENT\"},\n",
|
||||
" ]),\n",
|
||||
" relations=json.dumps([\n",
|
||||
" {\"source\": \"Competitor Beta\", \"relation\": \"PARTNERSHIP_WITH\", \"target\": \"Microsoft Azure\"},\n",
|
||||
" {\"source\": \"Competitor Beta\", \"relation\": \"INTEGRATES_WITH\", \"target\": \"Azure Synapse Analytics\"},\n",
|
||||
" {\"source\": \"Sarah Chen\", \"relation\": \"CEO_OF\", \"target\": \"Competitor Beta\"},\n",
|
||||
" ]),\n",
|
||||
"))\n",
|
||||
"print(f\"\\n[researcher] added {add_result['nodes_added']} nodes, {add_result['edges_added']} edges to SHARED graph\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "analyst-turn",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(\"=\" * 65)\n",
|
||||
"print(\"ANALYST AGENT TURN (sees researcher's graph additions)\")\n",
|
||||
"print(\"=\" * 65)\n",
|
||||
"\n",
|
||||
"# Analyst queries the graph the researcher just populated\n",
|
||||
"competitor_query = json.loads(analyst_decision_kit.find_precedents(\n",
|
||||
" scenario=\"competitor partnership with cloud hyperscaler threatens market position\",\n",
|
||||
" limit=3,\n",
|
||||
"))\n",
|
||||
"print(f\"\\n[analyst] find_precedents → {competitor_query['count']} similar past strategic responses found\")\n",
|
||||
"\n",
|
||||
"# Analyst records a strategic evaluation decision\n",
|
||||
"eval_json = analyst_decision_kit.record_decision(\n",
|
||||
" category=\"strategic_response\",\n",
|
||||
" scenario=(\n",
|
||||
" \"Competitor Beta + Microsoft Azure partnership gives Beta access to \"\n",
|
||||
" \"300k enterprise customers via Azure Synapse native integration\"\n",
|
||||
" ),\n",
|
||||
" reasoning=(\n",
|
||||
" \"Threat level: HIGH. Beta's Azure native integration removes our \"\n",
|
||||
" \"integration advantage. Existing NPS lead (41 vs 24) remains but \"\n",
|
||||
" \"distribution disadvantage is critical. Recommend accelerated cloud-native \"\n",
|
||||
" \"partnership evaluation, specifically AWS Marketplace + Snowflake Native App.\"\n",
|
||||
" ),\n",
|
||||
" outcome=\"escalate_to_strategy\",\n",
|
||||
" confidence=0.85,\n",
|
||||
" entities=\"Competitor Beta, Microsoft Azure, AWS Marketplace, Snowflake\",\n",
|
||||
")\n",
|
||||
"eval_result = json.loads(eval_json)\n",
|
||||
"analyst_decision_id = eval_result['decision_id']\n",
|
||||
"print(f\"\\n[analyst] recorded evaluation → decision_id: {analyst_decision_id}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "strategist-turn",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(\"=\" * 65)\n",
|
||||
"print(\"STRATEGIST AGENT TURN (sees both researcher + analyst work)\")\n",
|
||||
"print(\"=\" * 65)\n",
|
||||
"\n",
|
||||
"# Strategist queries the graph for the full competitive picture\n",
|
||||
"related = json.loads(strategist_kg_kit.find_related(\"Competitor Beta\", hops=2))\n",
|
||||
"print(f\"\\n[strategist] 'Competitor Beta' 2-hop neighbourhood: {related['count']} entity/entities\")\n",
|
||||
"for entity in related['related']:\n",
|
||||
" print(f\" → {entity}\")\n",
|
||||
"\n",
|
||||
"# Strategist traces what the analyst decided\n",
|
||||
"causal = json.loads(strategist_decision_kit.trace_causal_chain(analyst_decision_id, depth=3))\n",
|
||||
"print(f\"\\n[strategist] causal chain for analyst decision: {causal}\")\n",
|
||||
"\n",
|
||||
"# Strategist records the final strategic recommendation\n",
|
||||
"strategy_json = strategist_decision_kit.record_decision(\n",
|
||||
" category=\"product_strategy\",\n",
|
||||
" scenario=\"Q1 2026 product strategy: respond to Beta+Azure threat\",\n",
|
||||
" reasoning=(\n",
|
||||
" \"Based on researcher's KG (Beta+Azure integration, 300k customer reach) \"\n",
|
||||
" \"and analyst's evaluation (threat level HIGH, escalated decision). \"\n",
|
||||
" \"Strategy: (1) Accelerate AWS Marketplace listing by Q2 2026. \"\n",
|
||||
" \"(2) Launch Snowflake Native App by Q3 2026. \"\n",
|
||||
" \"(3) Invest $2M in UI modernisation to widen NPS lead. \"\n",
|
||||
" \"(4) Fast-track LLM-native analytics interface (60% time-to-insight improvement per HAI study). \"\n",
|
||||
" \"Existing NPS advantage (41 vs 24) provides 18-month window before Beta catches up.\"\n",
|
||||
" ),\n",
|
||||
" outcome=\"approved\",\n",
|
||||
" confidence=0.88,\n",
|
||||
" entities=\"AWS Marketplace, Snowflake, LLM Analytics, Q2 2026, Q3 2026\",\n",
|
||||
")\n",
|
||||
"strategy_result = json.loads(strategy_json)\n",
|
||||
"print(f\"\\n[strategist] final recommendation recorded → {strategy_result['decision_id']}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "shared-pool-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 7. Verify Shared Memory Pool\n",
|
||||
"\n",
|
||||
"Memories written by one agent are readable by all others."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "verify-shared",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from integrations.agno.context_store import _MemoryRow as MemoryRow\n",
|
||||
"\n",
|
||||
"# Researcher writes a memory\n",
|
||||
"researcher_row = MemoryRow(\n",
|
||||
" memory=\"Beta + Azure partnership announced at Gartner Summit — threat level HIGH\",\n",
|
||||
" user_id=\"researcher\",\n",
|
||||
")\n",
|
||||
"researcher_store.upsert_memory(researcher_row)\n",
|
||||
"\n",
|
||||
"# Analyst writes a memory\n",
|
||||
"analyst_row = MemoryRow(\n",
|
||||
" memory=\"NPS advantage (41 vs 24) gives 18-month window — accelerate cloud partnerships\",\n",
|
||||
" user_id=\"analyst\",\n",
|
||||
")\n",
|
||||
"analyst_store.upsert_memory(analyst_row)\n",
|
||||
"\n",
|
||||
"# Strategist reads ALL memories from both agents\n",
|
||||
"strategist_memories = strategist_store.read_memories()\n",
|
||||
"\n",
|
||||
"print(f\"Strategist sees {len(strategist_memories)} shared memory item(s):\")\n",
|
||||
"for m in strategist_memories:\n",
|
||||
" uid = getattr(m, 'user_id', '?')\n",
|
||||
" text = getattr(m, 'memory', str(m))\n",
|
||||
" print(f\" [{uid:12s}] {text[:80]}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "agno-team-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 8. Wire into Agno Team (requires API key)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "agno-team",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if AGNO_AVAILABLE:\n",
|
||||
" from agno.agent import Agent\n",
|
||||
" from agno.team import Team\n",
|
||||
" from agno.memory import AgentMemory\n",
|
||||
" from agno.models.openai import OpenAIChat\n",
|
||||
"\n",
|
||||
" researcher_agent = Agent(\n",
|
||||
" name=\"Researcher\",\n",
|
||||
" model=OpenAIChat(id=\"gpt-4o\"),\n",
|
||||
" memory=AgentMemory(db=researcher_store),\n",
|
||||
" tools=[researcher_kg_kit],\n",
|
||||
" show_tool_calls=True,\n",
|
||||
" description=(\n",
|
||||
" \"You are a competitive intelligence researcher. \"\n",
|
||||
" \"Use extract_entities, extract_relations, and add_to_graph \"\n",
|
||||
" \"to build a structured knowledge graph from market intelligence. \"\n",
|
||||
" \"Always add discoveries to the shared graph.\"\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" analyst_agent = Agent(\n",
|
||||
" name=\"Analyst\",\n",
|
||||
" model=OpenAIChat(id=\"gpt-4o\"),\n",
|
||||
" memory=AgentMemory(db=analyst_store),\n",
|
||||
" tools=[analyst_decision_kit],\n",
|
||||
" show_tool_calls=True,\n",
|
||||
" description=(\n",
|
||||
" \"You are a strategic analyst. Use find_precedents to check historical \"\n",
|
||||
" \"responses to similar threats, then record_decision with your evaluation. \"\n",
|
||||
" \"Always check if a similar situation was handled before acting.\"\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" strategist_agent = Agent(\n",
|
||||
" name=\"Strategist\",\n",
|
||||
" model=OpenAIChat(id=\"gpt-4o\"),\n",
|
||||
" memory=AgentMemory(db=strategist_store),\n",
|
||||
" tools=[strategist_kg_kit, strategist_decision_kit],\n",
|
||||
" show_tool_calls=True,\n",
|
||||
" description=(\n",
|
||||
" \"You are the Chief Strategy Officer. Synthesise the researcher's knowledge \"\n",
|
||||
" \"graph and the analyst's decision record into a concrete product strategy. \"\n",
|
||||
" \"Use find_related to explore the competitive graph, then record_decision \"\n",
|
||||
" \"with the final approved strategy.\"\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" strategy_team = Team(\n",
|
||||
" name=\"Product Strategy Team\",\n",
|
||||
" agents=[researcher_agent, analyst_agent, strategist_agent],\n",
|
||||
" mode=\"coordinate\",\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" strategy_team.print_response(\n",
|
||||
" \"Competitor Beta just announced a native Azure integration. \"\n",
|
||||
" \"Analyse the competitive landscape and recommend our Q1 2026 product strategy.\"\n",
|
||||
" )\n",
|
||||
"else:\n",
|
||||
" print(\"[Agno not installed — skipping live team run]\")\n",
|
||||
" print()\n",
|
||||
" print(\"Expected team coordination flow:\")\n",
|
||||
" print(\" 1. Researcher: extract_entities + add_to_graph (Beta+Azure)\")\n",
|
||||
" print(\" 2. Analyst: find_precedents + record_decision (threat=HIGH, escalate)\")\n",
|
||||
" print(\" 3. Strategist: find_related + trace_causal_chain + record_decision (final strategy)\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "post-session-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 9. Post-Session Analysis with Semantica\n",
|
||||
"\n",
|
||||
"After the team session, use **native Semantica APIs** for cross-agent audit, analytics, and causal chain review."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cross-agent-insights",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Team-level insights from AgnoSharedContext\n",
|
||||
"insights = shared.get_shared_insights()\n",
|
||||
"print(\"Team session insights:\")\n",
|
||||
"if isinstance(insights, dict):\n",
|
||||
" for k, v in insights.items():\n",
|
||||
" print(f\" {k}: {v}\")\n",
|
||||
"else:\n",
|
||||
" print(f\" {insights}\")\n",
|
||||
"\n",
|
||||
"print(f\"\\nBound agent roles: {shared.bound_roles}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "precedent-search",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Find all cross-agent strategic decisions\n",
|
||||
"all_strategic = shared.find_precedents(\n",
|
||||
" scenario=\"cloud partnership competitive response\",\n",
|
||||
" category=\"strategic_response\",\n",
|
||||
")\n",
|
||||
"print(f\"Cross-agent strategic precedents: {len(all_strategic or [])}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "graph-analytics",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Graph analytics on the shared knowledge graph (Semantica native)\n",
|
||||
"try:\n",
|
||||
" analyzer = GraphAnalyzer()\n",
|
||||
" analysis = analyzer.analyze_graph(shared.knowledge_graph)\n",
|
||||
" print(\"Shared knowledge graph analysis:\")\n",
|
||||
" if isinstance(analysis, dict):\n",
|
||||
" for k, v in list(analysis.items())[:6]:\n",
|
||||
" print(f\" {k}: {v}\")\n",
|
||||
" else:\n",
|
||||
" print(f\" {analysis}\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"GraphAnalyzer: {e}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "centrality-analysis",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Which entities are most central in the competitive intelligence graph?\n",
|
||||
"try:\n",
|
||||
" centrality = CentralityCalculator()\n",
|
||||
" scores = centrality.calculate_degree_centrality(shared.knowledge_graph)\n",
|
||||
" print(\"Most central entities in shared graph:\")\n",
|
||||
" if isinstance(scores, dict):\n",
|
||||
" top = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:5]\n",
|
||||
" for entity, score in top:\n",
|
||||
" print(f\" {entity:35s} centrality={score:.4f}\")\n",
|
||||
" else:\n",
|
||||
" print(f\" {scores}\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"CentralityCalculator: {e}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "causal-analysis",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Direct Semantica causal chain analysis (no Agno needed)\n",
|
||||
"try:\n",
|
||||
" causal_analyzer = CausalChainAnalyzer(graph_store=shared.knowledge_graph)\n",
|
||||
" # Query all decisions made during this session\n",
|
||||
" decisions = shared.knowledge_graph.find_precedents(category=\"product_strategy\", limit=10)\n",
|
||||
" print(f\"Product strategy decisions in shared graph: {len(decisions or [])}\")\n",
|
||||
" for d in (decisions or [])[:3]:\n",
|
||||
" scenario = d.get('scenario', '') if isinstance(d, dict) else str(d)\n",
|
||||
" outcome = d.get('outcome', '') if isinstance(d, dict) else ''\n",
|
||||
" print(f\" [{outcome:20s}] {scenario[:70]}\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"CausalChainAnalyzer: {e}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "summary-section",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Summary\n",
|
||||
"\n",
|
||||
"| Pattern | Implementation |\n",
|
||||
"|---|---|\n",
|
||||
"| Single shared knowledge graph | `AgnoSharedContext(vector_store, knowledge_graph)` |\n",
|
||||
"| Role-scoped memory | `shared.bind_agent(\"researcher\")` → `_AgentScopedStore` |\n",
|
||||
"| Cross-agent memory visibility | All stores read from `shared._shared_memories` |\n",
|
||||
"| KG tool sharing | `AgnoKGToolkit(context=shared.knowledge_graph)` |\n",
|
||||
"| Decision tool sharing | `AgnoDecisionKit(context=shared._context)` |\n",
|
||||
"| Thread-safe binding | `AgnoSharedContext._lock` (RLock) |\n",
|
||||
"| Post-session analytics | `GraphAnalyzer`, `CentralityCalculator`, `CausalChainAnalyzer` — all Semantica native |\n",
|
||||
"\n",
|
||||
"**Key design rule:** Every agent writes to the **same underlying graph** via different role-scoped stores. The Agno integration is a thin routing layer — Semantica's full power is available at any point directly."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python",
|
||||
"version": "3.11.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -1018,7 +1018,7 @@ knowledge_graph.apply_resolutions(resolved_data)
|
||||
|
||||
### 💬 Community Support
|
||||
|
||||
- **💬 [Discord Community](https://discord.gg/N7WmAuDH)** - Real-time chat and support
|
||||
- **💬 [Discord Community](https://discord.gg/sV34vps5hH)** - Real-time chat and support
|
||||
- **🐙 [GitHub Discussions](https://github.com/semantica/semantica/discussions)** - Community Q&A
|
||||
- **📧 [Mailing List](https://groups.google.com/g/semantica)** - Announcements and updates
|
||||
- **🐦 [Twitter](https://twitter.com/semantica)** - Latest news and tips
|
||||
@@ -1051,6 +1051,6 @@ This project is licensed under the MIT License - see the [LICENSE](https://githu
|
||||
|
||||
**🚀 Ready to transform your data into intelligent knowledge?**
|
||||
|
||||
[Get Started Now](https://semantica.readthedocs.io/quickstart/) • [View Examples](https://github.com/semantica/examples) • [Join Community](https://discord.gg/N7WmAuDH)
|
||||
[Get Started Now](https://semantica.readthedocs.io/quickstart/) • [View Examples](https://github.com/semantica/examples) • [Join Community](https://discord.gg/sV34vps5hH)
|
||||
|
||||
</div>
|
||||
|
||||
@@ -96,6 +96,6 @@ kg = GraphBuilder().build_graph(entities, relationships)
|
||||
|
||||
## Need Help?
|
||||
|
||||
- **[💬 Discord Community](https://discord.gg/N7WmAuDH)** - Get help from the community
|
||||
- **[💬 Discord Community](https://discord.gg/sV34vps5hH)** - Get help from the community
|
||||
- **[🐛 Issues](https://github.com/Hawksight-AI/semantica/issues)** - Report bugs or request features
|
||||
- **[📖 Documentation](https://semantica.readthedocs.io/)** - Full documentation site
|
||||
|
||||
+2
-2
@@ -9,7 +9,7 @@
|
||||
<a href="https://pypi.org/project/semantica/"><img src="https://img.shields.io/pypi/dm/semantica" alt="Monthly Downloads"></a>
|
||||
<a href="https://pepy.tech/project/semantica"><img src="https://static.pepy.tech/badge/semantica" alt="Total Downloads"></a>
|
||||
<a href="https://semantica.readthedocs.io/"><img src="https://img.shields.io/badge/docs-latest-brightgreen.svg" alt="Documentation"></a>
|
||||
<a href="https://discord.gg/N7WmAuDH"><img src="https://img.shields.io/badge/Discord-Join%20Us-7289da?style=flat&logo=discord&logoColor=white" alt="Discord"></a>
|
||||
<a href="https://discord.gg/sV34vps5hH"><img src="https://img.shields.io/badge/Discord-Join%20Us-7289da?style=flat&logo=discord&logoColor=white" alt="Discord"></a>
|
||||
|
||||
<p><strong>Open-Source Semantic Layer & Knowledge Engineering Framework</strong></p>
|
||||
|
||||
@@ -53,7 +53,7 @@ kg = GraphBuilder().build({"entities": entities, "relationships": []})
|
||||
print(f"Built KG with {len(kg.get('entities', []))} entities")
|
||||
```
|
||||
|
||||
**[📖 Full Quick Start](getting-started.md)** • **[🍳 Cookbook Examples](cookbook.md)** • **[💬 Join Discord](https://discord.gg/N7WmAuDH)** • **[⭐ Star Us](https://github.com/Hawksight-AI/semantica)**
|
||||
**[📖 Full Quick Start](getting-started.md)** • **[🍳 Cookbook Examples](cookbook.md)** • **[💬 Join Discord](https://discord.gg/sV34vps5hH)** • **[⭐ Star Us](https://github.com/Hawksight-AI/semantica)**
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
# Agno Integration
|
||||
|
||||
Semantica's Agno integration (`semantica[agno]`) wires the full Semantica
|
||||
semantic intelligence stack into the [Agno](https://github.com/agno-agi/agno)
|
||||
agentic framework via five focused components.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
# Core integration
|
||||
pip install semantica[agno]
|
||||
|
||||
# With a graph store backend
|
||||
pip install semantica[agno,graph-neo4j]
|
||||
pip install semantica[agno,graph-falkordb]
|
||||
|
||||
# Full stack
|
||||
pip install semantica[agno,graph-neo4j,vectorstore-pgvector]
|
||||
```
|
||||
|
||||
## Components at a Glance
|
||||
|
||||
| Class | Agno Primitive | Semantica Backing |
|
||||
|---|---|---|
|
||||
| `AgnoContextStore` | `AgentMemory(db=…)` | `AgentContext` + `VectorStore` |
|
||||
| `AgnoKnowledgeGraph` | `Agent(knowledge=…)` | `ContextGraph` + KG pipeline |
|
||||
| `AgnoDecisionKit` | `Agent(tools=[…])` | `DecisionQuery`, `CausalChainAnalyzer`, `PolicyEngine` |
|
||||
| `AgnoKGToolkit` | `Agent(tools=[…])` | `NERExtractor`, `RelationExtractor`, `Reasoner` |
|
||||
| `AgnoSharedContext` | Team-level | Shared `ContextGraph` across agents |
|
||||
|
||||
---
|
||||
|
||||
## 1. AgnoContextStore
|
||||
|
||||
Replaces Agno's flat conversation storage with a hybrid **vector + context
|
||||
graph** memory store. Implements `agno.memory.db.base.MemoryDb`.
|
||||
|
||||
```python
|
||||
from agno.agent import Agent
|
||||
from agno.memory import AgentMemory
|
||||
from agno.models.openai import OpenAIChat
|
||||
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.vector_store import VectorStore
|
||||
from integrations.agno import AgnoContextStore
|
||||
|
||||
store = AgnoContextStore(
|
||||
vector_store=VectorStore(backend="faiss"),
|
||||
knowledge_graph=ContextGraph(advanced_analytics=True),
|
||||
decision_tracking=True,
|
||||
graph_expansion=True,
|
||||
session_id="user_session_42",
|
||||
)
|
||||
|
||||
agent = Agent(
|
||||
model=OpenAIChat(id="gpt-4o"),
|
||||
memory=AgentMemory(db=store),
|
||||
description="A financially aware assistant with persistent decision intelligence.",
|
||||
)
|
||||
|
||||
agent.print_response("Recommend a portfolio allocation for a risk-averse investor.")
|
||||
```
|
||||
|
||||
### Key behaviours
|
||||
|
||||
- `upsert_memory()` — stores text in `AgentContext` (vector index + graph node)
|
||||
- `read_memories()` — hybrid retrieval: vector similarity + optional graph hop expansion
|
||||
- `record_decision()` — records a structured decision with reasoning & outcome
|
||||
- `find_precedents()` — returns semantically similar historical decisions
|
||||
|
||||
---
|
||||
|
||||
## 2. AgnoKnowledgeGraph
|
||||
|
||||
Gives Agno agents a queryable `ContextGraph` instead of a flat document store.
|
||||
Ingested documents pass through the full Semantica extraction pipeline.
|
||||
|
||||
```python
|
||||
from agno.agent import Agent
|
||||
from agno.models.openai import OpenAIChat
|
||||
|
||||
from semantica.kg import GraphBuilder
|
||||
from semantica.semantic_extract import NERExtractor, RelationExtractor
|
||||
from integrations.agno import AgnoKnowledgeGraph
|
||||
|
||||
kg = AgnoKnowledgeGraph(
|
||||
graph_builder=GraphBuilder(),
|
||||
ner_extractor=NERExtractor(),
|
||||
relation_extractor=RelationExtractor(),
|
||||
)
|
||||
|
||||
# Ingest local files
|
||||
kg.load("regulatory_docs/", recursive=True)
|
||||
|
||||
# Ingest raw text
|
||||
kg.load(texts=["Basel IV capital requirements apply from January 2026."])
|
||||
|
||||
agent = Agent(
|
||||
model=OpenAIChat(id="gpt-4o"),
|
||||
knowledge=kg,
|
||||
search_knowledge=True,
|
||||
)
|
||||
```
|
||||
|
||||
### Ingestion pipeline
|
||||
|
||||
```
|
||||
parse → NER → relation extract → graph build → vector index
|
||||
```
|
||||
|
||||
### Search: multi-hop GraphRAG
|
||||
|
||||
```
|
||||
vector retrieval → entity lookup → graph hop expansion → context injection
|
||||
```
|
||||
|
||||
### Get entity subgraph
|
||||
|
||||
```python
|
||||
ctx = kg.get_graph_context("Basel IV")
|
||||
# Returns a text summary of the entity's immediate neighbourhood in the graph
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. AgnoDecisionKit
|
||||
|
||||
Exposes Semantica's decision intelligence as native Agno tools.
|
||||
|
||||
```python
|
||||
from agno.agent import Agent
|
||||
from agno.models.openai import OpenAIChat
|
||||
|
||||
from semantica.context import AgentContext
|
||||
from integrations.agno import AgnoDecisionKit
|
||||
|
||||
ctx = AgentContext(decision_tracking=True)
|
||||
|
||||
agent = Agent(
|
||||
model=OpenAIChat(id="gpt-4o"),
|
||||
tools=[AgnoDecisionKit(context=ctx)],
|
||||
show_tool_calls=True,
|
||||
)
|
||||
|
||||
agent.print_response("Should we approve this mortgage application?")
|
||||
```
|
||||
|
||||
### Tools
|
||||
|
||||
| Tool | Description | Key Parameters |
|
||||
|---|---|---|
|
||||
| `record_decision` | Record decision with reasoning and outcome | `category`, `scenario`, `reasoning`, `outcome`, `confidence`, `entities` |
|
||||
| `find_precedents` | Search for similar past decisions | `scenario`, `category`, `limit` |
|
||||
| `trace_causal_chain` | Trace causal chain of a decision | `decision_id`, `depth` |
|
||||
| `analyze_impact` | Assess downstream influence of a decision | `decision_id` |
|
||||
| `check_policy` | Validate decision against policy rules | `decision_data`, `policy_rules` |
|
||||
| `get_decision_summary` | Summarise decision history by category | `category`, `since`, `limit` |
|
||||
|
||||
### Example agent turn
|
||||
|
||||
```
|
||||
User: Should we approve this mortgage application?
|
||||
|
||||
Agent [tool: find_precedents] → 12 similar mortgage approvals found
|
||||
Agent [tool: check_policy] → complies with lending policy v2.3
|
||||
Agent [tool: record_decision] → recorded: loan_approval / approved / confidence=0.94
|
||||
Agent: Based on 12 historical precedents and full policy compliance, I recommend
|
||||
approval. Credit score 740, 22% down payment, DTI 31% — all within thresholds.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. AgnoKGToolkit
|
||||
|
||||
Lets agents actively build and query the context graph during reasoning.
|
||||
|
||||
```python
|
||||
from agno.agent import Agent
|
||||
from agno.models.openai import OpenAIChat
|
||||
|
||||
from integrations.agno import AgnoKGToolkit
|
||||
|
||||
agent = Agent(
|
||||
model=OpenAIChat(id="gpt-4o"),
|
||||
tools=[AgnoKGToolkit()],
|
||||
show_tool_calls=True,
|
||||
)
|
||||
|
||||
agent.print_response(
|
||||
"Extract entities and relationships from this article and store them in the knowledge graph."
|
||||
)
|
||||
```
|
||||
|
||||
### Tools
|
||||
|
||||
| Tool | Description |
|
||||
|---|---|
|
||||
| `extract_entities` | Extract named entities from text |
|
||||
| `extract_relations` | Extract relationships between entities |
|
||||
| `add_to_graph` | Add entities / relations to the context graph |
|
||||
| `query_graph` | Query the graph (natural-language or Cypher) |
|
||||
| `find_related` | Find concepts related to a given entity |
|
||||
| `infer_facts` | Apply rules to infer new facts from the graph |
|
||||
| `export_subgraph` | Export a subgraph as RDF / JSON-LD |
|
||||
|
||||
---
|
||||
|
||||
## 5. AgnoSharedContext
|
||||
|
||||
A single `ContextGraph` shared across an Agno `Team`. Each agent gets a
|
||||
**role-scoped view** via `bind_agent()`.
|
||||
|
||||
```python
|
||||
from agno.agent import Agent
|
||||
from agno.team import Team
|
||||
from agno.models.openai import OpenAIChat
|
||||
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.vector_store import VectorStore
|
||||
from integrations.agno import AgnoSharedContext, AgnoDecisionKit, AgnoKGToolkit
|
||||
|
||||
shared = AgnoSharedContext(
|
||||
vector_store=VectorStore(backend="faiss"),
|
||||
knowledge_graph=ContextGraph(advanced_analytics=True),
|
||||
decision_tracking=True,
|
||||
)
|
||||
|
||||
research_agent = Agent(
|
||||
name="Researcher",
|
||||
model=OpenAIChat(id="gpt-4o"),
|
||||
memory=shared.bind_agent("researcher"),
|
||||
tools=[AgnoKGToolkit(context=shared)],
|
||||
)
|
||||
|
||||
decision_agent = Agent(
|
||||
name="Analyst",
|
||||
model=OpenAIChat(id="gpt-4o"),
|
||||
memory=shared.bind_agent("analyst"),
|
||||
tools=[AgnoDecisionKit(context=shared)],
|
||||
)
|
||||
|
||||
team = Team(
|
||||
name="Research & Decision Team",
|
||||
agents=[research_agent, decision_agent],
|
||||
mode="coordinate",
|
||||
)
|
||||
|
||||
team.print_response(
|
||||
"Analyse the competitive landscape and recommend our product strategy."
|
||||
)
|
||||
```
|
||||
|
||||
### Shared memory pool
|
||||
|
||||
Memories written by one agent are immediately visible to all other agents in the
|
||||
team. Each agent's writes are tagged with their role so they can be filtered
|
||||
independently.
|
||||
|
||||
### Shared decisions
|
||||
|
||||
```python
|
||||
# Record a team-level decision
|
||||
decision_id = shared.record_decision(
|
||||
category="strategy",
|
||||
scenario="Expand to EU market",
|
||||
reasoning="Strong demand signals from Q1 survey",
|
||||
outcome="approved",
|
||||
confidence=0.87,
|
||||
agent_role="cfo",
|
||||
)
|
||||
|
||||
# Query precedents across all agents' history
|
||||
precedents = shared.find_precedents("market expansion")
|
||||
|
||||
# Get cross-agent analytics
|
||||
insights = shared.get_shared_insights()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Use Cases
|
||||
|
||||
### Regulated Industry Agents (Finance, Healthcare, Legal)
|
||||
|
||||
Agents that log every decision with full provenance, reasoning chain, and policy
|
||||
compliance check for audit trails.
|
||||
|
||||
```python
|
||||
kit = AgnoDecisionKit(context=ctx)
|
||||
# Every agent turn: find_precedents → check_policy → record_decision
|
||||
```
|
||||
|
||||
### Long-Running Research Agents
|
||||
|
||||
Agents that accumulate a persistent `ContextGraph` over days or weeks, enabling
|
||||
multi-hop reasoning over a growing knowledge base.
|
||||
|
||||
```python
|
||||
kg = AgnoKnowledgeGraph(graph_builder=GraphBuilder(), ...)
|
||||
# Agents load new documents continuously; search benefits from the growing graph
|
||||
```
|
||||
|
||||
### Enterprise Multi-Agent Coordination
|
||||
|
||||
Teams using `AgnoSharedContext` to prevent contradictory decisions and share
|
||||
structured knowledge across specialist agents.
|
||||
|
||||
### GraphRAG Customer Support
|
||||
|
||||
Support agents that retrieve answers via graph traversal, providing more
|
||||
contextually grounded responses than flat vector search.
|
||||
|
||||
### Explainable AI Pipelines
|
||||
|
||||
Every agent step, entity reference, and causal chain is traceable back to a
|
||||
source document or prior decision.
|
||||
|
||||
---
|
||||
|
||||
## API Reference
|
||||
|
||||
```python
|
||||
from integrations.agno import (
|
||||
AgnoContextStore, # MemoryDb implementation
|
||||
AgnoKnowledgeGraph, # AgentKnowledge implementation
|
||||
AgnoDecisionKit, # Decision intelligence Toolkit
|
||||
AgnoKGToolkit, # Knowledge graph Toolkit
|
||||
AgnoSharedContext, # Team-level shared context
|
||||
AGNO_AVAILABLE, # bool — True if agno is installed
|
||||
)
|
||||
```
|
||||
|
||||
All five classes are usable **without** `agno` installed — they carry the full
|
||||
Semantica API and degrade gracefully when passed to Agno constructors.
|
||||
@@ -391,3 +391,79 @@ prod_manager = TemporalVersionManager(
|
||||
for version in prod_manager.list_versions():
|
||||
print(f"{version['timestamp']}: {version['description']} by {version['author']}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Ontology Diff & Migration
|
||||
|
||||
Semantica allows you to treat ontology schema changes with the same rigor as database migrations. By comparing two versions, you can generate a machine-readable diff and a structured impact report to catch breaking changes before they reach production.
|
||||
|
||||
|
||||
### Comparing Versions
|
||||
|
||||
The `OntologyEngine` provides a high-level API to orchestrate the comparison of two schema versions.
|
||||
|
||||
```python
|
||||
from semantica.ontology.engine import OntologyEngine
|
||||
|
||||
engine = OntologyEngine()
|
||||
|
||||
# Generate a migration impact report between v1.0 and v2.0
|
||||
report = engine.compare_versions(
|
||||
base_id="v1.0",
|
||||
target_id="v2.0"
|
||||
)
|
||||
|
||||
print(f"Total changes detected: {report['summary']['total_changes']}")
|
||||
```
|
||||
---
|
||||
|
||||
### Understanding the Report Format
|
||||
|
||||
The `compare_versions` method returns a comprehensive dictionary containing both a machine-readable diff and a human-readable impact analysis.
|
||||
|
||||
|
||||
|
||||
Here is the exact structure of the returned report:
|
||||
|
||||
```json
|
||||
{
|
||||
"summary": {
|
||||
"total_changes": 12
|
||||
},
|
||||
"impact_classification": {
|
||||
"breaking": [
|
||||
{
|
||||
"entity_uri": "http://example.org/Person",
|
||||
"severity": "critical",
|
||||
"description": "Class Person removed.",
|
||||
"mitigation": "Migrate orphaned instances."
|
||||
}
|
||||
],
|
||||
"potentially_breaking": [],
|
||||
"safe": []
|
||||
},
|
||||
"recommendations": [
|
||||
"[BREAKING] Schedule downtime or validate existing data."
|
||||
],
|
||||
"diff": {
|
||||
"added_classes": [],
|
||||
"removed_classes": [],
|
||||
"changed_classes": [],
|
||||
"added_properties": [],
|
||||
"removed_properties": [],
|
||||
"changed_properties": []
|
||||
},
|
||||
"validation_results": {
|
||||
"valid": true,
|
||||
"consistent": true,
|
||||
"satisfiable": true,
|
||||
"errors": [],
|
||||
"warnings": []
|
||||
},
|
||||
"graph_validation": {
|
||||
"valid": false,
|
||||
"errors": ["Instance data violates new domain constraint"],
|
||||
"warnings": []
|
||||
}
|
||||
}
|
||||
|
||||
@@ -250,6 +250,56 @@ ontology:
|
||||
|
||||
---
|
||||
|
||||
## Ontology Alignment
|
||||
|
||||
Semantica supports mapping and connecting different ontologies to unify data across systems, standards, and domains. This enables cross-system interoperability, allowing a single semantic layer to span multiple standards (e.g., internal models and industry standards).
|
||||
|
||||
Alignments are represented using standard RDF predicates such as `owl:equivalentClass`, `owl:equivalentProperty`, and `skos:exactMatch`.
|
||||
|
||||
### Creating and Managing Alignments
|
||||
|
||||
You can create and query alignments programmatically using the `OntologyEngine`:
|
||||
|
||||
```python
|
||||
from semantica.ontology.engine import OntologyEngine
|
||||
from semantica.triplet_store.triplet_store import TripletStore
|
||||
|
||||
# Setup the store and engine (using Blazegraph as an example)
|
||||
my_triplet_store = TripletStore(backend="blazegraph")
|
||||
engine = OntologyEngine(store=my_triplet_store)
|
||||
|
||||
# Create an alignment between an internal class and a standard schema
|
||||
engine.create_alignment(
|
||||
source_uri="http://internal.org/ontology/Employee",
|
||||
target_uri="http://schema.org/Person",
|
||||
predicate="http://www.w3.org/2002/07/owl#equivalentClass"
|
||||
)
|
||||
|
||||
# Retrieve all bidirectional alignments for a specific entity
|
||||
alignments = engine.get_alignments("http://internal.org/ontology/Employee")
|
||||
```
|
||||
### Automated Alignment Suggestions
|
||||
|
||||
When importing or merging external ontologies, the ReuseManager can automatically suggest alignments based on heuristic matching (such as identical labels with differing URIs).
|
||||
|
||||
```python
|
||||
from semantica.ontology.reuse_manager import ReuseManager
|
||||
|
||||
manager = ReuseManager()
|
||||
|
||||
# Merge ontologies and auto-compute alignment suggestions
|
||||
merged_ontology = manager.merge_ontology_data(
|
||||
target=internal_ontology,
|
||||
source=industry_ontology,
|
||||
compute_alignments=True
|
||||
)
|
||||
|
||||
# Suggestions are stored in merged_ontology["suggested_alignments"]
|
||||
```
|
||||
|
||||
For executing SPARQL queries that utilize these alignments to retrieve cross-ontology results, see the [Triplet Store Alignment-Aware Queries](triplet_store.md#alignment-aware-queries)
|
||||
|
||||
|
||||
## Integration Examples
|
||||
|
||||
### Schema-First Knowledge Graph
|
||||
@@ -312,4 +362,4 @@ Interactive tutorials to learn ontology generation and management:
|
||||
- **[Unstructured to Ontology](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/12_Unstructured_to_Ontology.ipynb)**: Generate ontologies automatically from unstructured data
|
||||
- **Topics**: Automatic ontology generation, 6-stage pipeline, OWL validation
|
||||
- **Difficulty**: Advanced
|
||||
- **Use Cases**: Domain modeling, automatic schema generation
|
||||
- **Use Cases**: Domain modeling, automatic schema generation
|
||||
@@ -734,4 +734,4 @@ MIT License - See [LICENSE](../../LICENSE) for details.
|
||||
|
||||
## Support
|
||||
|
||||
For issues or questions, please open an issue on GitHub or join our [Discord](https://discord.gg/N7WmAuDH).
|
||||
For issues or questions, please open an issue on GitHub or join our [Discord](https://discord.gg/sV34vps5hH).
|
||||
|
||||
@@ -152,6 +152,8 @@ SPARQL query execution and optimization engine.
|
||||
|--------|-------------|-----------|
|
||||
| `execute(query)` | Execute SPARQL query | Query execution |
|
||||
| `optimize(query)` | Optimize SPARQL query | Query rewriting |
|
||||
| `expand_entity_uri(uri, store, ...)` | Expand aligned entity URIs | Bidirectional SPARQL lookup |
|
||||
| `build_values_clause(var, uris)` | Generate VALUES clause | String formatting |
|
||||
|
||||
---
|
||||
|
||||
@@ -204,3 +206,40 @@ LIMIT 10
|
||||
"""
|
||||
results = store.execute_query(query)
|
||||
```
|
||||
### Alignment-Aware Queries
|
||||
|
||||
In complex enterprise environments with multiple data sources, you may want queries to seamlessly retrieve instances across aligned classes. For example, retrieving all http://schema.org/Person instances when querying for your internal http://internal.org/ontology/Employee class.
|
||||
|
||||
The QueryEngine provides helper methods to expand entity URIs based on stored alignments (e.g., owl:equivalentClass, owl:sameAs, skos:exactMatch) and safely inject them into your queries using SPARQL VALUES clauses.
|
||||
|
||||
Expanding URIs in Queries
|
||||
You can expand a URI and build an alignment-aware query dynamically:
|
||||
|
||||
```python
|
||||
from semantica.triplet_store.query_engine import QueryEngine
|
||||
|
||||
engine = QueryEngine()
|
||||
|
||||
# i) Expand the base URI to include all aligned equivalents
|
||||
expanded_uris = engine.expand_entity_uri(
|
||||
entity_uri="[http://internal.org/ontology/Employee](http://internal.org/ontology/Employee)",
|
||||
store_backend=store_backend,
|
||||
use_alignments=True
|
||||
)
|
||||
|
||||
# ii) Build a SPARQL VALUES clause
|
||||
values_clause = engine.build_values_clause("entity_class", expanded_uris)
|
||||
# Result: VALUES ?entity_class { [http://internal.org/ontology/Employee](http://internal.org/ontology/Employee) [http://schema.org/Person](http://schema.org/Person) }
|
||||
|
||||
# iii) Inject the clause into your query template
|
||||
query = f"""
|
||||
SELECT ?instance ?name WHERE {{
|
||||
{values_clause}
|
||||
?instance a ?entity_class .
|
||||
?instance [http://schema.org/name](http://schema.org/name) ?name .
|
||||
}}
|
||||
"""
|
||||
|
||||
# Execute the query to retrieve results across all aligned ontologies
|
||||
results = engine.execute_query(query, store_backend)
|
||||
```
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
"""
|
||||
Semantica × Agno Integration
|
||||
=============================
|
||||
|
||||
First-class integration between the Semantica semantic intelligence stack and
|
||||
the `Agno <https://github.com/agno-agi/agno>`_ agentic framework.
|
||||
|
||||
Public surface
|
||||
--------------
|
||||
AgnoContextStore — Graph-backed ``MemoryDb`` (drop-in for ``AgentMemory(db=…)``)
|
||||
AgnoKnowledgeGraph — Relational ``AgentKnowledge`` with multi-hop GraphRAG
|
||||
AgnoDecisionKit — Agno ``Toolkit`` exposing decision-intelligence tools
|
||||
AgnoKGToolkit — Agno ``Toolkit`` exposing KG construction/query tools
|
||||
AgnoSharedContext — Team-level shared ``ContextGraph`` with per-agent scoping
|
||||
|
||||
Quick start
|
||||
-----------
|
||||
pip install semantica[agno]
|
||||
|
||||
>>> from integrations.agno import (
|
||||
... AgnoContextStore,
|
||||
... AgnoKnowledgeGraph,
|
||||
... AgnoDecisionKit,
|
||||
... AgnoKGToolkit,
|
||||
... AgnoSharedContext,
|
||||
... )
|
||||
|
||||
Compatibility
|
||||
-------------
|
||||
Requires ``agno >= 1.0``. All five classes degrade gracefully when ``agno``
|
||||
is not installed — they are still importable and carry the full Semantica API,
|
||||
but cannot be passed directly to Agno ``Agent`` / ``Team`` constructors.
|
||||
"""
|
||||
|
||||
from .context_store import AGNO_AVAILABLE, AgnoContextStore
|
||||
from .decision_kit import AgnoDecisionKit
|
||||
from .kg_toolkit import AgnoKGToolkit
|
||||
from .knowledge_graph import AgnoKnowledgeGraph
|
||||
from .shared_context import AgnoSharedContext
|
||||
|
||||
__all__ = [
|
||||
"AgnoContextStore",
|
||||
"AgnoKnowledgeGraph",
|
||||
"AgnoDecisionKit",
|
||||
"AgnoKGToolkit",
|
||||
"AgnoSharedContext",
|
||||
"AGNO_AVAILABLE",
|
||||
]
|
||||
|
||||
__version__ = "0.3.0"
|
||||
@@ -0,0 +1,378 @@
|
||||
"""
|
||||
AgnoContextStore — Graph-backed agent memory storage for Agno.
|
||||
|
||||
Implements Agno's ``MemoryDb`` protocol backed by Semantica's ``AgentContext``,
|
||||
giving Agno agents hybrid vector + context-graph memory that persists across
|
||||
sessions.
|
||||
|
||||
Key behaviours
|
||||
--------------
|
||||
- ``upsert_memory()`` → stores text in ``AgentContext`` (vector + graph) and
|
||||
extracts entities into the knowledge graph
|
||||
- ``read_memories()`` → hybrid retrieval: vector similarity + graph expansion
|
||||
- ``delete_memory()`` → removes from cache and calls ``AgentContext.forget()``
|
||||
- ``record_decision()`` → records a structured decision with reasoning & outcome
|
||||
- ``find_precedents()`` → returns semantically similar historical decisions
|
||||
- ``get_context_for_prompt()`` → formats precedents for system-prompt injection
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[agno]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from semantica.context import ContextGraph
|
||||
>>> from semantica.vector_store import VectorStore
|
||||
>>> from integrations.agno import AgnoContextStore
|
||||
>>> store = AgnoContextStore(
|
||||
... vector_store=VectorStore(backend="faiss"),
|
||||
... knowledge_graph=ContextGraph(advanced_analytics=True),
|
||||
... decision_tracking=True,
|
||||
... session_id="user_session_42",
|
||||
... )
|
||||
>>> from agno.agent import Agent
|
||||
>>> from agno.memory import AgentMemory
|
||||
>>> agent = Agent(memory=AgentMemory(db=store))
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: Agno MemoryDb base class
|
||||
# ---------------------------------------------------------------------------
|
||||
AGNO_AVAILABLE = False
|
||||
AGNO_IMPORT_ERROR: Optional[str] = None
|
||||
|
||||
_MemoryDbBase: Any = object # fallback when agno is absent
|
||||
|
||||
try:
|
||||
from agno.memory.db.base import MemoryDb as _AgnoMemoryDb # type: ignore
|
||||
from agno.memory.db.row import MemoryRow as _AgnoMemoryRow # type: ignore
|
||||
|
||||
_MemoryDbBase = _AgnoMemoryDb
|
||||
AGNO_AVAILABLE = True
|
||||
except ImportError as exc:
|
||||
AGNO_IMPORT_ERROR = str(exc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lightweight memory row when agno is not installed
|
||||
# ---------------------------------------------------------------------------
|
||||
class _MemoryRow:
|
||||
"""Minimal stand-in for ``agno.memory.db.row.MemoryRow``."""
|
||||
|
||||
__slots__ = ("id", "memory", "user_id", "topics", "input", "last_updated")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
memory: str,
|
||||
id: Optional[str] = None,
|
||||
user_id: Optional[str] = None,
|
||||
topics: Optional[List[str]] = None,
|
||||
input: Optional[str] = None,
|
||||
) -> None:
|
||||
self.id = id or str(uuid.uuid4())
|
||||
self.memory = memory
|
||||
self.user_id = user_id
|
||||
self.topics = topics or []
|
||||
self.input = input
|
||||
self.last_updated = time.time()
|
||||
|
||||
|
||||
MemoryRow = _AgnoMemoryRow if AGNO_AVAILABLE else _MemoryRow # type: ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AgnoContextStore
|
||||
# ---------------------------------------------------------------------------
|
||||
class AgnoContextStore(_MemoryDbBase): # type: ignore[misc]
|
||||
"""
|
||||
Graph-backed agent memory store that implements Agno's ``MemoryDb`` protocol.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
vector_store:
|
||||
A ``semantica.vector_store.VectorStore`` instance (or ``None`` to use
|
||||
an in-memory FAISS store created automatically).
|
||||
knowledge_graph:
|
||||
A ``semantica.context.ContextGraph`` instance (or ``None`` for a fresh
|
||||
in-memory graph).
|
||||
decision_tracking:
|
||||
Automatically record every ``upsert_memory`` call as a lightweight
|
||||
decision entry.
|
||||
graph_expansion:
|
||||
Augment ``read_memories`` results with one-hop graph neighbours.
|
||||
session_id:
|
||||
Logical session identifier used for node scoping in the context graph.
|
||||
agent_context_kwargs:
|
||||
Extra keyword arguments forwarded to ``AgentContext.__init__``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vector_store: Any = None,
|
||||
knowledge_graph: Any = None,
|
||||
decision_tracking: bool = True,
|
||||
graph_expansion: bool = True,
|
||||
session_id: Optional[str] = None,
|
||||
**agent_context_kwargs: Any,
|
||||
) -> None:
|
||||
# Call agno's base init only when the real base class is available.
|
||||
if AGNO_AVAILABLE:
|
||||
super().__init__() # type: ignore[call-arg]
|
||||
|
||||
self.decision_tracking = decision_tracking
|
||||
self.graph_expansion = graph_expansion
|
||||
self.session_id = session_id or str(uuid.uuid4())
|
||||
self._memories: Dict[str, Any] = {} # id → MemoryRow (in-process cache)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Build AgentContext from provided components
|
||||
# ------------------------------------------------------------------
|
||||
from semantica.context import AgentContext, ContextGraph # lazy import
|
||||
from semantica.vector_store import VectorStore # lazy import
|
||||
|
||||
if knowledge_graph is None:
|
||||
knowledge_graph = ContextGraph()
|
||||
|
||||
if vector_store is None:
|
||||
vector_store = VectorStore(backend="faiss")
|
||||
|
||||
self._context = AgentContext(
|
||||
vector_store=vector_store,
|
||||
knowledge_graph=knowledge_graph,
|
||||
decision_tracking=decision_tracking,
|
||||
**agent_context_kwargs,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"AgnoContextStore initialised",
|
||||
extra={"session_id": self.session_id, "decision_tracking": decision_tracking},
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# MemoryDb protocol
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def create(self) -> None:
|
||||
"""Initialise storage (no-op for in-memory graph)."""
|
||||
logger.debug("AgnoContextStore.create() called — in-memory graph ready")
|
||||
|
||||
def table_exists(self) -> bool:
|
||||
return True
|
||||
|
||||
def memory_exists(self, memory: Any) -> bool:
|
||||
mem_id = getattr(memory, "id", None)
|
||||
return mem_id is not None and mem_id in self._memories
|
||||
|
||||
def read_memories(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
sort: Optional[str] = None,
|
||||
) -> List[Any]:
|
||||
"""
|
||||
Return stored memories, optionally filtered by ``user_id``.
|
||||
|
||||
When ``graph_expansion`` is enabled, each recalled memory is enriched
|
||||
with its one-hop graph neighbourhood before being returned.
|
||||
"""
|
||||
rows = list(self._memories.values())
|
||||
|
||||
if user_id:
|
||||
rows = [r for r in rows if getattr(r, "user_id", None) == user_id]
|
||||
|
||||
# Sort: newest first by default
|
||||
reverse = sort != "asc"
|
||||
rows.sort(key=lambda r: getattr(r, "last_updated", 0), reverse=reverse)
|
||||
|
||||
if limit is not None:
|
||||
rows = rows[:limit]
|
||||
|
||||
return rows
|
||||
|
||||
def upsert_memory(self, memory: Any) -> Optional[Any]:
|
||||
"""
|
||||
Persist ``memory`` into both the vector store and the context graph.
|
||||
|
||||
Entity extraction is performed so the knowledge graph is populated
|
||||
with nodes for the stored content. If ``decision_tracking`` is enabled
|
||||
a lightweight decision entry is also recorded.
|
||||
"""
|
||||
mem_id = getattr(memory, "id", None) or str(uuid.uuid4())
|
||||
mem_text = getattr(memory, "memory", str(memory))
|
||||
user_id = getattr(memory, "user_id", None)
|
||||
|
||||
# Persist in AgentContext (vector + graph)
|
||||
try:
|
||||
self._context.store(
|
||||
mem_text,
|
||||
conversation_id=user_id or self.session_id,
|
||||
)
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.warning("AgentContext.store() failed: %s", exc)
|
||||
|
||||
# Extract entities and index them into the knowledge graph
|
||||
try:
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
ner = NERExtractor()
|
||||
entities = ner.extract_entities(mem_text) or []
|
||||
kg = getattr(self._context, "knowledge_graph", None)
|
||||
if kg is not None:
|
||||
for ent in entities:
|
||||
name = getattr(ent, "name", str(ent))
|
||||
ntype = getattr(ent, "type", "Entity")
|
||||
try:
|
||||
kg.add_node(node_id=name, node_type=ntype)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.debug("NER/graph indexing skipped: %s", exc)
|
||||
|
||||
# Optional decision tracking
|
||||
if self.decision_tracking:
|
||||
try:
|
||||
self._context.record_decision(
|
||||
category="memory",
|
||||
scenario=mem_text[:200],
|
||||
reasoning="Stored via AgnoContextStore.upsert_memory()",
|
||||
outcome="stored",
|
||||
confidence=1.0,
|
||||
)
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.debug("Decision tracking skipped: %s", exc)
|
||||
|
||||
# Update in-process cache
|
||||
if hasattr(memory, "id"):
|
||||
memory.id = mem_id
|
||||
self._memories[mem_id] = memory
|
||||
logger.debug("upsert_memory id=%s", mem_id)
|
||||
return memory
|
||||
|
||||
def delete_memory(self, id: str) -> None:
|
||||
self._memories.pop(id, None)
|
||||
try:
|
||||
self._context.forget(memory_id=id)
|
||||
except Exception as exc:
|
||||
logger.debug("forget(%s) failed: %s", id, exc)
|
||||
logger.debug("delete_memory id=%s", id)
|
||||
|
||||
def drop_table(self) -> None:
|
||||
self._memories.clear()
|
||||
try:
|
||||
self._context.forget()
|
||||
except Exception as exc:
|
||||
logger.debug("drop_table forget() failed: %s", exc)
|
||||
logger.debug("AgnoContextStore: all memories dropped")
|
||||
|
||||
def clear(self) -> bool:
|
||||
self._memories.clear()
|
||||
try:
|
||||
self._context.forget()
|
||||
except Exception as exc:
|
||||
logger.debug("clear forget() failed: %s", exc)
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Extended Semantica API (usable from application code directly)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def record_decision(
|
||||
self,
|
||||
category: str,
|
||||
scenario: str,
|
||||
reasoning: str,
|
||||
outcome: str,
|
||||
confidence: float = 0.8,
|
||||
entities: Optional[List[str]] = None,
|
||||
) -> str:
|
||||
"""Record a structured decision and return its ID."""
|
||||
return self._context.record_decision(
|
||||
category=category,
|
||||
scenario=scenario,
|
||||
reasoning=reasoning,
|
||||
outcome=outcome,
|
||||
confidence=confidence,
|
||||
entities=entities,
|
||||
)
|
||||
|
||||
def find_precedents(
|
||||
self,
|
||||
scenario: str,
|
||||
category: Optional[str] = None,
|
||||
limit: int = 5,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search for similar historical decisions."""
|
||||
try:
|
||||
return self._context.find_precedents_advanced(
|
||||
scenario=scenario,
|
||||
category=category,
|
||||
limit=limit,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("find_precedents failed: %s", exc)
|
||||
return []
|
||||
|
||||
def retrieve(self, query: str, limit: int = 5) -> List[Dict[str, Any]]:
|
||||
"""Hybrid retrieval: vector similarity + optional graph expansion."""
|
||||
try:
|
||||
return self._context.retrieve(query, max_results=limit)
|
||||
except Exception as exc:
|
||||
logger.warning("retrieve failed: %s", exc)
|
||||
return []
|
||||
|
||||
def get_context_for_prompt(self, scenario: str, max_precedents: int = 3) -> str:
|
||||
"""
|
||||
Return formatted precedents suitable for injection into a system prompt.
|
||||
|
||||
Call this before each LLM invocation to surface relevant past decisions
|
||||
automatically.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
scenario:
|
||||
Description of the current situation.
|
||||
max_precedents:
|
||||
Maximum number of precedents to include.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Multi-line string ready to prepend to a system prompt, or an
|
||||
empty string when no relevant precedents exist.
|
||||
"""
|
||||
try:
|
||||
precedents = self.find_precedents(scenario, limit=max_precedents)
|
||||
if not precedents:
|
||||
return ""
|
||||
lines = ["Relevant past decisions:"]
|
||||
for i, p in enumerate(precedents[:max_precedents], 1):
|
||||
if isinstance(p, dict):
|
||||
sc = p.get("scenario", "")
|
||||
outcome = p.get("outcome", "")
|
||||
conf = p.get("confidence", "")
|
||||
else:
|
||||
sc = getattr(p, "scenario", str(p))
|
||||
outcome = getattr(p, "outcome", "")
|
||||
conf = getattr(p, "confidence", "")
|
||||
lines.append(
|
||||
f"{i}. Scenario: {sc} → Outcome: {outcome}"
|
||||
+ (f" (confidence: {conf})" if conf != "" else "")
|
||||
)
|
||||
return "\n".join(lines)
|
||||
except Exception as exc:
|
||||
logger.warning("get_context_for_prompt failed: %s", exc)
|
||||
return ""
|
||||
|
||||
@property
|
||||
def context(self) -> Any:
|
||||
"""Direct access to the underlying ``AgentContext``."""
|
||||
return self._context
|
||||
@@ -0,0 +1,425 @@
|
||||
"""
|
||||
AgnoDecisionKit — Decision Intelligence Toolkit for Agno agents.
|
||||
|
||||
Exposes Semantica's decision intelligence as native Agno tools so that agents
|
||||
can actively record, query, and validate decisions during their reasoning loop.
|
||||
|
||||
Follows Agno's ``Toolkit`` pattern — each method decorated with ``@register``
|
||||
(or manually registered via ``self.register()``) becomes a tool the LLM can
|
||||
call.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[agno]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from semantica.context import AgentContext
|
||||
>>> from integrations.agno import AgnoDecisionKit
|
||||
>>> ctx = AgentContext(decision_tracking=True)
|
||||
>>> from agno.agent import Agent
|
||||
>>> agent = Agent(tools=[AgnoDecisionKit(context=ctx)], show_tool_calls=True)
|
||||
|
||||
Tools exposed
|
||||
-------------
|
||||
record_decision — Record a decision with reasoning and outcome
|
||||
find_precedents — Search for similar past decisions
|
||||
trace_causal_chain — Trace causal chain of a decision node
|
||||
analyze_impact — Assess downstream influence of a decision
|
||||
check_policy — Validate a decision against policy rules
|
||||
get_decision_summary — Summarise decision history by category
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: Agno Toolkit base class
|
||||
# ---------------------------------------------------------------------------
|
||||
AGNO_AVAILABLE = False
|
||||
AGNO_IMPORT_ERROR: Optional[str] = None
|
||||
|
||||
_ToolkitBase: Any = object
|
||||
|
||||
try:
|
||||
from agno.tools.toolkit import Toolkit as _AgnoToolkit # type: ignore
|
||||
|
||||
_ToolkitBase = _AgnoToolkit
|
||||
AGNO_AVAILABLE = True
|
||||
except ImportError as exc:
|
||||
AGNO_IMPORT_ERROR = str(exc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AgnoDecisionKit
|
||||
# ---------------------------------------------------------------------------
|
||||
class AgnoDecisionKit(_ToolkitBase): # type: ignore[misc]
|
||||
"""
|
||||
Agno Toolkit that surfaces Semantica's decision intelligence as agent tools.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
context:
|
||||
A ``semantica.context.AgentContext`` (or ``AgentContext``-compatible
|
||||
object with ``record_decision``, ``find_precedents_advanced``,
|
||||
``analyze_decision_influence`` methods). A fresh in-memory context is
|
||||
created when ``None``.
|
||||
max_precedents:
|
||||
Default number of precedents returned by ``find_precedents``.
|
||||
causal_depth:
|
||||
Default chain depth used by ``trace_causal_chain``.
|
||||
enable_policy_check:
|
||||
Register the ``check_policy`` tool (default: ``True``).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
context: Any = None,
|
||||
max_precedents: int = 5,
|
||||
causal_depth: int = 3,
|
||||
enable_policy_check: bool = True,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if AGNO_AVAILABLE:
|
||||
super().__init__(name="decision_kit", **kwargs) # type: ignore[call-arg]
|
||||
|
||||
# Always initialise _tools so the attribute exists regardless of agno
|
||||
if not hasattr(self, "_tools"):
|
||||
self._tools: list = []
|
||||
|
||||
self.max_precedents = max_precedents
|
||||
self.causal_depth = causal_depth
|
||||
|
||||
# Build or reuse AgentContext
|
||||
if context is None:
|
||||
from semantica.context import AgentContext
|
||||
from semantica.vector_store import VectorStore
|
||||
|
||||
context = AgentContext(
|
||||
vector_store=VectorStore(backend="faiss"),
|
||||
decision_tracking=True,
|
||||
)
|
||||
self._ctx = context
|
||||
|
||||
# Register tools.
|
||||
# _tools is always kept as a plain list so callers can inspect registered
|
||||
# tools regardless of whether agno is installed. When agno IS available
|
||||
# we also call Toolkit.register() so the real agno runtime picks them up.
|
||||
tools_to_register = [
|
||||
self.record_decision,
|
||||
self.find_precedents,
|
||||
self.trace_causal_chain,
|
||||
self.analyze_impact,
|
||||
self.get_decision_summary,
|
||||
]
|
||||
if enable_policy_check:
|
||||
tools_to_register.append(self.check_policy)
|
||||
|
||||
for fn in tools_to_register:
|
||||
self._tools.append(fn)
|
||||
if AGNO_AVAILABLE:
|
||||
try:
|
||||
self.register(fn)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.info("AgnoDecisionKit initialised")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Tools
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def record_decision(
|
||||
self,
|
||||
category: str,
|
||||
scenario: str,
|
||||
reasoning: str,
|
||||
outcome: str,
|
||||
confidence: float = 0.8,
|
||||
entities: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Record a decision with its reasoning and outcome.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
category:
|
||||
Domain category, e.g. ``"loan_approval"``, ``"content_moderation"``.
|
||||
scenario:
|
||||
Short description of the situation being decided.
|
||||
reasoning:
|
||||
Why this outcome was chosen.
|
||||
outcome:
|
||||
The decision result, e.g. ``"approved"``, ``"rejected"``.
|
||||
confidence:
|
||||
Confidence score in [0, 1].
|
||||
entities:
|
||||
Comma-separated list of entity names relevant to the decision.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
JSON with ``{"decision_id": "<id>", "status": "recorded"}``.
|
||||
"""
|
||||
entity_list: Optional[List[str]] = None
|
||||
if entities:
|
||||
entity_list = [e.strip() for e in entities.split(",") if e.strip()]
|
||||
|
||||
try:
|
||||
decision_id = self._ctx.record_decision(
|
||||
category=category,
|
||||
scenario=scenario,
|
||||
reasoning=reasoning,
|
||||
outcome=outcome,
|
||||
confidence=float(confidence),
|
||||
entities=entity_list,
|
||||
)
|
||||
result = {"decision_id": str(decision_id), "status": "recorded"}
|
||||
logger.info("record_decision → %s", decision_id)
|
||||
except Exception as exc:
|
||||
result = {"error": str(exc), "status": "failed"}
|
||||
logger.warning("record_decision failed: %s", exc)
|
||||
|
||||
return json.dumps(result)
|
||||
|
||||
def find_precedents(
|
||||
self,
|
||||
scenario: str,
|
||||
category: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Search for past decisions similar to the given scenario.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
scenario:
|
||||
Description of the current situation.
|
||||
category:
|
||||
Optional category filter.
|
||||
limit:
|
||||
Maximum number of precedents to return.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
JSON list of precedent summaries.
|
||||
"""
|
||||
k = limit or self.max_precedents
|
||||
try:
|
||||
precedents = self._ctx.find_precedents_advanced(
|
||||
scenario=scenario,
|
||||
category=category,
|
||||
)
|
||||
# Normalise to a serialisable list
|
||||
out: List[Dict[str, Any]] = []
|
||||
for p in (precedents or [])[:k]:
|
||||
if isinstance(p, dict):
|
||||
out.append(p)
|
||||
else:
|
||||
out.append(
|
||||
{
|
||||
"scenario": getattr(p, "scenario", str(p)),
|
||||
"outcome": getattr(p, "outcome", ""),
|
||||
"confidence": getattr(p, "confidence", 0.0),
|
||||
"category": getattr(p, "category", ""),
|
||||
}
|
||||
)
|
||||
logger.info("find_precedents('%s') → %d results", scenario, len(out))
|
||||
return json.dumps({"precedents": out, "count": len(out)})
|
||||
except Exception as exc:
|
||||
logger.warning("find_precedents failed: %s", exc)
|
||||
return json.dumps({"precedents": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def trace_causal_chain(
|
||||
self,
|
||||
decision_id: str,
|
||||
depth: Optional[int] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Trace the causal chain starting from a decision node.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
decision_id:
|
||||
Identifier of the decision to trace.
|
||||
depth:
|
||||
Maximum chain depth to traverse.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
JSON representation of the causal chain.
|
||||
"""
|
||||
max_depth = depth or self.causal_depth
|
||||
try:
|
||||
chain = self._ctx.knowledge_graph.trace_decision_causality( # type: ignore[attr-defined]
|
||||
decision_id, depth=max_depth
|
||||
)
|
||||
return json.dumps({"causal_chain": chain, "decision_id": decision_id})
|
||||
except AttributeError:
|
||||
# Fallback if the graph doesn't expose trace_decision_causality
|
||||
try:
|
||||
chain = self._ctx.knowledge_graph.find_precedents( # type: ignore[attr-defined]
|
||||
category="decision", limit=max_depth
|
||||
)
|
||||
return json.dumps({"causal_chain": chain, "decision_id": decision_id})
|
||||
except Exception as exc:
|
||||
return json.dumps({"error": str(exc), "decision_id": decision_id})
|
||||
except Exception as exc:
|
||||
logger.warning("trace_causal_chain failed: %s", exc)
|
||||
return json.dumps({"error": str(exc), "decision_id": decision_id})
|
||||
|
||||
def analyze_impact(self, decision_id: str) -> str:
|
||||
"""
|
||||
Assess the downstream influence of a decision using graph centrality.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
decision_id:
|
||||
Identifier of the decision to analyse.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
JSON with influence metrics.
|
||||
"""
|
||||
try:
|
||||
influence = self._ctx.analyze_decision_influence(decision_id)
|
||||
if not isinstance(influence, dict):
|
||||
influence = {"influence": str(influence)}
|
||||
influence["decision_id"] = decision_id
|
||||
return json.dumps(influence)
|
||||
except Exception as exc:
|
||||
logger.warning("analyze_impact failed: %s", exc)
|
||||
return json.dumps({"error": str(exc), "decision_id": decision_id})
|
||||
|
||||
def check_policy(
|
||||
self,
|
||||
decision_data: str,
|
||||
policy_rules: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Validate a proposed decision against policy rules.
|
||||
|
||||
Rules are evaluated inline using simple comparison expressions. This
|
||||
avoids misuse of ``PolicyEngine.check_compliance`` (which requires a
|
||||
stored ``Decision`` + ``policy_id``) and ensures exceptions never
|
||||
silently return ``compliant=True``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
decision_data:
|
||||
JSON string describing the decision (must include ``category``,
|
||||
``outcome``, ``confidence`` keys at minimum).
|
||||
policy_rules:
|
||||
JSON list of rule strings, e.g.
|
||||
``'["confidence >= 0.7", "category != \\"test\\""]'``.
|
||||
Each rule is a simple comparison: ``<field> <op> <value>``
|
||||
where op is one of ``>=``, ``<=``, ``!=``, ``==``, ``>``, ``<``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
JSON with ``{"compliant": bool, "violations": [...], "warnings": [...]}``
|
||||
"""
|
||||
try:
|
||||
data = json.loads(decision_data) if isinstance(decision_data, str) else decision_data
|
||||
except json.JSONDecodeError as exc:
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": False,
|
||||
"violations": [f"Invalid decision_data JSON: {exc}"],
|
||||
"warnings": [],
|
||||
}
|
||||
)
|
||||
|
||||
rules: List[str] = []
|
||||
if policy_rules:
|
||||
try:
|
||||
rules = json.loads(policy_rules)
|
||||
except json.JSONDecodeError:
|
||||
rules = [r.strip() for r in policy_rules.split(",") if r.strip()]
|
||||
|
||||
violations: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
for rule in rules:
|
||||
try:
|
||||
if not self._eval_rule(rule, data):
|
||||
violations.append(f"Rule violated: {rule}")
|
||||
except Exception as exc:
|
||||
warnings.append(f"Could not evaluate rule '{rule}': {exc}")
|
||||
|
||||
compliant = len(violations) == 0
|
||||
logger.debug("check_policy: compliant=%s, violations=%d", compliant, len(violations))
|
||||
return json.dumps(
|
||||
{
|
||||
"compliant": compliant,
|
||||
"violations": violations,
|
||||
"warnings": warnings,
|
||||
}
|
||||
)
|
||||
|
||||
def _eval_rule(self, rule: str, data: Dict[str, Any]) -> bool:
|
||||
"""Evaluate a simple comparison rule (``field op value``) against data."""
|
||||
m = re.match(r"(\w+)\s*(>=|<=|!=|==|>|<)\s*(.+)", rule.strip())
|
||||
if not m:
|
||||
return True # unrecognised format — pass through
|
||||
field, op, val_str = m.group(1), m.group(2), m.group(3).strip().strip("\"'")
|
||||
actual = data.get(field)
|
||||
if actual is None:
|
||||
return True # field absent — cannot evaluate
|
||||
try:
|
||||
val: Any = type(actual)(val_str)
|
||||
except (ValueError, TypeError):
|
||||
val = val_str
|
||||
ops = {
|
||||
">=": lambda a, b: a >= b,
|
||||
"<=": lambda a, b: a <= b,
|
||||
"!=": lambda a, b: a != b,
|
||||
"==": lambda a, b: a == b,
|
||||
">": lambda a, b: a > b,
|
||||
"<": lambda a, b: a < b,
|
||||
}
|
||||
return ops[op](actual, val)
|
||||
|
||||
def get_decision_summary(
|
||||
self,
|
||||
category: Optional[str] = None,
|
||||
since: Optional[str] = None,
|
||||
limit: int = 10,
|
||||
) -> str:
|
||||
"""
|
||||
Summarise the decision history, optionally filtered by category.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
category:
|
||||
Filter to a specific decision category.
|
||||
since:
|
||||
ISO-8601 timestamp — only include decisions after this time.
|
||||
limit:
|
||||
Maximum number of decisions to include.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
JSON summary of recent decisions.
|
||||
"""
|
||||
try:
|
||||
insights = self._ctx.get_context_insights()
|
||||
if not isinstance(insights, dict):
|
||||
insights = {"raw": str(insights)}
|
||||
insights["category_filter"] = category
|
||||
return json.dumps(insights)
|
||||
except Exception as exc:
|
||||
logger.warning("get_decision_summary failed: %s", exc)
|
||||
return json.dumps({"error": str(exc)})
|
||||
@@ -0,0 +1,465 @@
|
||||
"""
|
||||
AgnoKGToolkit — Knowledge Graph Toolkit for Agno agents.
|
||||
|
||||
Lets agents actively build and query the context graph as part of their
|
||||
reasoning loop. Backed by Semantica's ``NERExtractor``, ``RelationExtractor``,
|
||||
``Reasoner``, and ``ContextGraph``.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[agno]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.agno import AgnoKGToolkit
|
||||
>>> from agno.agent import Agent
|
||||
>>> agent = Agent(tools=[AgnoKGToolkit()], show_tool_calls=True)
|
||||
|
||||
Tools exposed
|
||||
-------------
|
||||
extract_entities — Extract named entities from text
|
||||
extract_relations — Extract relationships between entities
|
||||
add_to_graph — Add entities / relations to the context graph
|
||||
query_graph — Query the graph (natural-language keyword or Cypher)
|
||||
find_related — Find concepts related to a given entity
|
||||
infer_facts — Apply rules to infer new facts from the graph
|
||||
export_subgraph — Export a subgraph as JSON-LD / RDF Turtle
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: Agno Toolkit base class
|
||||
# ---------------------------------------------------------------------------
|
||||
AGNO_AVAILABLE = False
|
||||
AGNO_IMPORT_ERROR: Optional[str] = None
|
||||
|
||||
_ToolkitBase: Any = object
|
||||
|
||||
try:
|
||||
from agno.tools.toolkit import Toolkit as _AgnoToolkit # type: ignore
|
||||
|
||||
_ToolkitBase = _AgnoToolkit
|
||||
AGNO_AVAILABLE = True
|
||||
except ImportError as exc:
|
||||
AGNO_IMPORT_ERROR = str(exc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AgnoKGToolkit
|
||||
# ---------------------------------------------------------------------------
|
||||
class AgnoKGToolkit(_ToolkitBase): # type: ignore[misc]
|
||||
"""
|
||||
Agno Toolkit that surfaces Semantica's KG pipeline as agent tools.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph_store_backend:
|
||||
Storage backend for the internal ``ContextGraph``. One of
|
||||
``"inmemory"`` (default), ``"neo4j"``, ``"falkordb"``.
|
||||
ner_extractor:
|
||||
A ``semantica.semantic_extract.NERExtractor`` instance; auto-created
|
||||
when ``None``.
|
||||
relation_extractor:
|
||||
A ``semantica.semantic_extract.RelationExtractor`` instance; auto-
|
||||
created when ``None``.
|
||||
reasoner:
|
||||
A ``semantica.reasoning.Reasoner`` instance; auto-created when
|
||||
``None``.
|
||||
context:
|
||||
An existing ``AgentContext`` or ``ContextGraph`` to attach to. A
|
||||
fresh in-memory ``ContextGraph`` is used when ``None``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph_store_backend: str = "inmemory",
|
||||
ner_extractor: Any = None,
|
||||
relation_extractor: Any = None,
|
||||
reasoner: Any = None,
|
||||
context: Any = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if AGNO_AVAILABLE:
|
||||
super().__init__(name="kg_toolkit", **kwargs) # type: ignore[call-arg]
|
||||
|
||||
# Always initialise _tools so the attribute exists regardless of agno
|
||||
if not hasattr(self, "_tools"):
|
||||
self._tools: list = []
|
||||
|
||||
# Lazy imports
|
||||
from semantica.context import ContextGraph
|
||||
from semantica.reasoning import Reasoner
|
||||
from semantica.semantic_extract import NERExtractor, RelationExtractor
|
||||
|
||||
if context is not None:
|
||||
self._graph = getattr(context, "knowledge_graph", context)
|
||||
else:
|
||||
self._graph = ContextGraph()
|
||||
|
||||
self._ner = ner_extractor or NERExtractor()
|
||||
self._rel = relation_extractor or RelationExtractor()
|
||||
self._reasoner = reasoner or Reasoner()
|
||||
|
||||
# Register tools.
|
||||
# _tools is always kept as a plain list so callers can inspect registered
|
||||
# tools regardless of whether agno is installed. When agno IS available
|
||||
# we also call Toolkit.register() so the real agno runtime picks them up.
|
||||
tools_to_register = [
|
||||
self.extract_entities,
|
||||
self.extract_relations,
|
||||
self.add_to_graph,
|
||||
self.query_graph,
|
||||
self.find_related,
|
||||
self.infer_facts,
|
||||
self.export_subgraph,
|
||||
]
|
||||
for fn in tools_to_register:
|
||||
self._tools.append(fn)
|
||||
if AGNO_AVAILABLE:
|
||||
try:
|
||||
self.register(fn)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.info("AgnoKGToolkit initialised (backend=%s)", graph_store_backend)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Tools
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def extract_entities(self, text: str) -> str:
|
||||
"""
|
||||
Extract named entities from the given text.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
text:
|
||||
Input text to analyse.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
JSON list of ``{"name": str, "type": str, "confidence": float}``.
|
||||
"""
|
||||
try:
|
||||
raw = self._ner.extract_entities(text) or []
|
||||
entities = [
|
||||
{
|
||||
"name": getattr(e, "name", str(e)),
|
||||
"type": getattr(e, "type", ""),
|
||||
"confidence": round(float(getattr(e, "confidence", 1.0)), 4),
|
||||
}
|
||||
for e in raw
|
||||
]
|
||||
logger.debug("extract_entities → %d entities", len(entities))
|
||||
return json.dumps({"entities": entities, "count": len(entities)})
|
||||
except Exception as exc:
|
||||
logger.warning("extract_entities failed: %s", exc)
|
||||
return json.dumps({"entities": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def extract_relations(self, text: str, entities: Optional[str] = None) -> str:
|
||||
"""
|
||||
Extract relationships between entities in the given text.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
text:
|
||||
Input text to analyse.
|
||||
entities:
|
||||
Optional JSON list of entity names to restrict extraction to.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
JSON list of ``{"source": str, "relation": str, "target": str, "confidence": float}``.
|
||||
"""
|
||||
entity_list: Optional[List[str]] = None
|
||||
if entities:
|
||||
try:
|
||||
entity_list = json.loads(entities)
|
||||
except json.JSONDecodeError:
|
||||
entity_list = [e.strip() for e in entities.split(",") if e.strip()]
|
||||
|
||||
try:
|
||||
raw = self._rel.extract_relations(text, entities=entity_list) or []
|
||||
relations = [
|
||||
{
|
||||
"source": getattr(r, "source", ""),
|
||||
"relation": getattr(r, "type", getattr(r, "relation", "")),
|
||||
"target": getattr(r, "target", ""),
|
||||
"confidence": round(float(getattr(r, "confidence", 1.0)), 4),
|
||||
}
|
||||
for r in raw
|
||||
]
|
||||
logger.debug("extract_relations → %d relations", len(relations))
|
||||
return json.dumps({"relations": relations, "count": len(relations)})
|
||||
except Exception as exc:
|
||||
logger.warning("extract_relations failed: %s", exc)
|
||||
return json.dumps({"relations": [], "count": 0, "error": str(exc)})
|
||||
|
||||
def add_to_graph(
|
||||
self,
|
||||
entities: Optional[str] = None,
|
||||
relations: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Add entities and/or relations to the active context graph.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
entities:
|
||||
JSON list of ``{"name": str, "type": str}`` objects.
|
||||
relations:
|
||||
JSON list of ``{"source": str, "relation": str, "target": str}`` objects.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
JSON summary of nodes and edges added.
|
||||
"""
|
||||
nodes_added = 0
|
||||
edges_added = 0
|
||||
|
||||
if entities:
|
||||
try:
|
||||
ent_list = json.loads(entities) if isinstance(entities, str) else entities
|
||||
for ent in ent_list:
|
||||
name = ent.get("name", str(ent))
|
||||
ntype = ent.get("type", "Entity")
|
||||
try:
|
||||
# ContextGraph.add_node(node_id, node_type, content=None, **props)
|
||||
self._graph.add_node(node_id=name, node_type=ntype) # type: ignore[attr-defined]
|
||||
nodes_added += 1
|
||||
except Exception:
|
||||
pass
|
||||
except (json.JSONDecodeError, AttributeError) as exc:
|
||||
logger.debug("add_to_graph entities parse error: %s", exc)
|
||||
|
||||
if relations:
|
||||
try:
|
||||
rel_list = json.loads(relations) if isinstance(relations, str) else relations
|
||||
for rel in rel_list:
|
||||
src = rel.get("source", "")
|
||||
tgt = rel.get("target", "")
|
||||
rel_type = rel.get("relation", "related_to")
|
||||
try:
|
||||
# ContextGraph.add_edge(source_id, target_id, edge_type, **props)
|
||||
self._graph.add_edge(source_id=src, target_id=tgt, edge_type=rel_type) # type: ignore[attr-defined]
|
||||
edges_added += 1
|
||||
except Exception:
|
||||
pass
|
||||
except (json.JSONDecodeError, AttributeError) as exc:
|
||||
logger.debug("add_to_graph relations parse error: %s", exc)
|
||||
|
||||
logger.debug("add_to_graph: +%d nodes, +%d edges", nodes_added, edges_added)
|
||||
return json.dumps({"nodes_added": nodes_added, "edges_added": edges_added})
|
||||
|
||||
def query_graph(self, query: str) -> str:
|
||||
"""
|
||||
Query the context graph in natural language or Cypher.
|
||||
|
||||
For natural-language queries all nodes are retrieved and filtered by
|
||||
whether ``query`` appears in their ``node_id``. Pass a string starting
|
||||
with ``"MATCH"`` for raw Cypher execution (requires a Neo4j / FalkorDB
|
||||
backend).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
query:
|
||||
Search query string.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
JSON list of matching nodes / records.
|
||||
"""
|
||||
try:
|
||||
if query.strip().upper().startswith("MATCH"):
|
||||
# Cypher path
|
||||
try:
|
||||
result = self._graph.execute_query(query) # type: ignore[attr-defined]
|
||||
records = result if isinstance(result, list) else [str(result)]
|
||||
return json.dumps({"results": records, "query_type": "cypher"})
|
||||
except AttributeError:
|
||||
return json.dumps(
|
||||
{
|
||||
"error": "Cypher queries require a Neo4j/FalkorDB backend",
|
||||
"query_type": "cypher",
|
||||
}
|
||||
)
|
||||
else:
|
||||
# Natural-language keyword lookup — ContextGraph.find_nodes() → List[Dict]
|
||||
all_nodes = self._graph.find_nodes() # type: ignore[attr-defined]
|
||||
q_lower = query.lower()
|
||||
out = []
|
||||
for n in (all_nodes or []):
|
||||
if isinstance(n, dict):
|
||||
node_id = n.get("node_id", "")
|
||||
node_type = n.get("node_type", "")
|
||||
else:
|
||||
node_id = getattr(n, "id", getattr(n, "label", str(n)))
|
||||
node_type = getattr(n, "node_type", "")
|
||||
if q_lower in node_id.lower() or q_lower in node_type.lower():
|
||||
out.append({"label": node_id, "type": node_type, "id": node_id})
|
||||
return json.dumps({"results": out, "count": len(out), "query_type": "keyword"})
|
||||
except Exception as exc:
|
||||
logger.warning("query_graph failed: %s", exc)
|
||||
return json.dumps({"results": [], "error": str(exc)})
|
||||
|
||||
def find_related(self, entity: str, hops: int = 1) -> str:
|
||||
"""
|
||||
Find concepts related to ``entity`` within ``hops`` graph hops.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
entity:
|
||||
The entity name to start from.
|
||||
hops:
|
||||
Maximum number of relationship hops to traverse.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
JSON list of related entity names.
|
||||
"""
|
||||
try:
|
||||
related: List[str] = []
|
||||
frontier = [entity]
|
||||
visited = {entity}
|
||||
|
||||
for _ in range(max(1, hops)):
|
||||
next_frontier: List[str] = []
|
||||
for e in frontier:
|
||||
try:
|
||||
# ContextGraph.get_neighbors(node_id, hops=1, ...) → List[Dict]
|
||||
neighbours = self._graph.get_neighbors(node_id=e, hops=1) # type: ignore[attr-defined]
|
||||
for n in (neighbours or []):
|
||||
if isinstance(n, dict):
|
||||
label = n.get("node_id", "")
|
||||
else:
|
||||
label = getattr(n, "label", str(n))
|
||||
if label and label not in visited:
|
||||
visited.add(label)
|
||||
next_frontier.append(label)
|
||||
related.append(label)
|
||||
except Exception:
|
||||
pass
|
||||
frontier = next_frontier
|
||||
|
||||
logger.debug("find_related('%s', hops=%d) → %d", entity, hops, len(related))
|
||||
return json.dumps({"entity": entity, "related": related, "count": len(related)})
|
||||
except Exception as exc:
|
||||
logger.warning("find_related failed: %s", exc)
|
||||
return json.dumps({"entity": entity, "related": [], "error": str(exc)})
|
||||
|
||||
def infer_facts(self, rules: str, facts: Optional[str] = None) -> str:
|
||||
"""
|
||||
Apply inference rules to the graph and return newly derived facts.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
rules:
|
||||
JSON list of rule strings, e.g.
|
||||
``'["IF Person(?x) THEN Human(?x)"]'``
|
||||
facts:
|
||||
Optional JSON list of additional fact strings to load before
|
||||
inference. When ``None``, the current graph state is used.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
JSON list of inferred fact strings.
|
||||
"""
|
||||
try:
|
||||
rule_list: List[str] = json.loads(rules) if rules else []
|
||||
except json.JSONDecodeError:
|
||||
rule_list = [r.strip() for r in rules.split(",") if r.strip()]
|
||||
|
||||
fact_list: List[str] = []
|
||||
if facts:
|
||||
try:
|
||||
fact_list = json.loads(facts)
|
||||
except json.JSONDecodeError:
|
||||
fact_list = [f.strip() for f in facts.split(",") if f.strip()]
|
||||
|
||||
if not fact_list:
|
||||
# Derive facts from graph nodes via the public API
|
||||
try:
|
||||
all_nodes = self._graph.find_nodes() # type: ignore[attr-defined]
|
||||
for node in (all_nodes or [])[:50]:
|
||||
if isinstance(node, dict):
|
||||
label = node.get("node_id", "")
|
||||
ntype = node.get("node_type", "Entity")
|
||||
else:
|
||||
label = getattr(node, "label", str(node))
|
||||
ntype = getattr(node, "node_type", "Entity")
|
||||
if label:
|
||||
fact_list.append(f"{ntype}({label})")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
result = self._reasoner.infer_facts(fact_list, rule_list)
|
||||
inferred = getattr(result, "inferred_facts", []) or []
|
||||
inferred_strs = [str(f) for f in inferred]
|
||||
logger.debug("infer_facts → %d new facts", len(inferred_strs))
|
||||
return json.dumps({"inferred_facts": inferred_strs, "count": len(inferred_strs)})
|
||||
except Exception as exc:
|
||||
logger.warning("infer_facts failed: %s", exc)
|
||||
return json.dumps({"inferred_facts": [], "error": str(exc)})
|
||||
|
||||
def export_subgraph(
|
||||
self,
|
||||
entity: Optional[str] = None,
|
||||
format: str = "json-ld",
|
||||
) -> str:
|
||||
"""
|
||||
Export a subgraph centred on ``entity`` as RDF / JSON-LD.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
entity:
|
||||
Root entity of the subgraph. The whole graph is exported when
|
||||
``None``.
|
||||
format:
|
||||
Output format: ``"json-ld"`` (default), ``"turtle"`` / ``"ttl"``,
|
||||
``"xml"``, ``"nt"``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Serialised subgraph in the requested format (JSON string wrapper).
|
||||
"""
|
||||
try:
|
||||
from semantica.export import RDFExporter # lazy import
|
||||
|
||||
exporter = RDFExporter()
|
||||
rdf_format = {"ttl": "turtle", "json-ld": "json-ld", "xml": "xml", "nt": "nt"}.get(
|
||||
format, format
|
||||
)
|
||||
output = exporter.export_to_rdf(self._graph, format=rdf_format) # type: ignore[arg-type]
|
||||
return json.dumps({"format": rdf_format, "data": output})
|
||||
except Exception as exc:
|
||||
logger.warning("export_subgraph failed: %s", exc)
|
||||
# Fallback: return graph nodes via the public API
|
||||
try:
|
||||
all_nodes = self._graph.find_nodes() # type: ignore[attr-defined]
|
||||
nodes = []
|
||||
for n in (all_nodes or []):
|
||||
if isinstance(n, dict):
|
||||
nodes.append({"id": n.get("node_id", ""), "label": n.get("node_id", "")})
|
||||
else:
|
||||
nodes.append(
|
||||
{"id": getattr(n, "id", ""), "label": getattr(n, "label", "")}
|
||||
)
|
||||
return json.dumps({"format": "json", "nodes": nodes, "note": str(exc)})
|
||||
except Exception:
|
||||
return json.dumps({"format": format, "data": "", "error": str(exc)})
|
||||
@@ -0,0 +1,475 @@
|
||||
"""
|
||||
AgnoKnowledgeGraph — Relational agent knowledge backed by Semantica's KG pipeline.
|
||||
|
||||
Implements Agno's ``AgentKnowledge`` protocol so that Agno agents can query a
|
||||
structured ``ContextGraph`` instead of a flat vector document store.
|
||||
|
||||
Ingested documents pass through the full Semantica extraction pipeline:
|
||||
|
||||
parse → split → NER → relation extract → graph build
|
||||
|
||||
and search uses multi-hop GraphRAG: vector retrieval + graph traversal +
|
||||
context injection.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[agno]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from integrations.agno import AgnoKnowledgeGraph
|
||||
>>> from semantica.kg import GraphBuilder
|
||||
>>> from semantica.semantic_extract import NERExtractor, RelationExtractor
|
||||
>>> kg = AgnoKnowledgeGraph(
|
||||
... graph_builder=GraphBuilder(),
|
||||
... ner_extractor=NERExtractor(),
|
||||
... relation_extractor=RelationExtractor(),
|
||||
... )
|
||||
>>> kg.load("regulatory_docs/", recursive=True)
|
||||
>>> from agno.agent import Agent
|
||||
>>> agent = Agent(knowledge=kg, search_knowledge=True)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterator, List, Optional, Union
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional: Agno AgentKnowledge base class
|
||||
# ---------------------------------------------------------------------------
|
||||
AGNO_AVAILABLE = False
|
||||
AGNO_IMPORT_ERROR: Optional[str] = None
|
||||
|
||||
_KnowledgeBase: Any = object
|
||||
|
||||
try:
|
||||
from agno.knowledge.base import AgentKnowledge as _AgnoAgentKnowledge # type: ignore
|
||||
|
||||
_KnowledgeBase = _AgnoAgentKnowledge
|
||||
AGNO_AVAILABLE = True
|
||||
except ImportError as exc:
|
||||
AGNO_IMPORT_ERROR = str(exc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lightweight document stand-in (used when agno is absent)
|
||||
# ---------------------------------------------------------------------------
|
||||
class _Document:
|
||||
"""Minimal stand-in for ``agno.document.Document``."""
|
||||
|
||||
__slots__ = ("id", "content", "meta_data", "name")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
content: str,
|
||||
id: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
meta_data: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
self.id = id
|
||||
self.content = content
|
||||
self.name = name
|
||||
self.meta_data = meta_data or {}
|
||||
|
||||
|
||||
try:
|
||||
from agno.document.base import Document as AgnoDocument # type: ignore
|
||||
except ImportError:
|
||||
AgnoDocument = _Document # type: ignore
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AgnoKnowledgeGraph
|
||||
# ---------------------------------------------------------------------------
|
||||
class AgnoKnowledgeGraph(_KnowledgeBase): # type: ignore[misc]
|
||||
"""
|
||||
Relational agent knowledge store backed by Semantica's KG pipeline.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
graph_builder:
|
||||
A ``semantica.kg.GraphBuilder`` instance. Created automatically if
|
||||
``None``.
|
||||
ner_extractor:
|
||||
A ``semantica.semantic_extract.NERExtractor`` instance. Created
|
||||
automatically if ``None``.
|
||||
relation_extractor:
|
||||
A ``semantica.semantic_extract.RelationExtractor`` instance. Created
|
||||
automatically if ``None``.
|
||||
context_graph:
|
||||
An existing ``semantica.context.ContextGraph`` to use as the backing
|
||||
store. A fresh in-memory graph is created when ``None``.
|
||||
graph_store_backend:
|
||||
Passed to ``ContextGraph`` when ``context_graph`` is ``None``.
|
||||
Supported values: ``"inmemory"`` (default), ``"neo4j"``,
|
||||
``"falkordb"``.
|
||||
graph_store_uri:
|
||||
Connection URI for the chosen graph store backend.
|
||||
num_documents:
|
||||
Default number of documents returned by ``search()``.
|
||||
chunk_size:
|
||||
Maximum characters per text chunk during ingestion.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph_builder: Any = None,
|
||||
ner_extractor: Any = None,
|
||||
relation_extractor: Any = None,
|
||||
context_graph: Any = None,
|
||||
graph_store_backend: str = "inmemory",
|
||||
graph_store_uri: Optional[str] = None,
|
||||
num_documents: int = 5,
|
||||
chunk_size: int = 1000,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if AGNO_AVAILABLE:
|
||||
super().__init__(**kwargs) # type: ignore[call-arg]
|
||||
|
||||
self.num_documents = num_documents
|
||||
self.chunk_size = chunk_size
|
||||
self._graph_store_backend = graph_store_backend
|
||||
|
||||
# Lazy imports to keep semantica core optional at import time
|
||||
from semantica.context import AgentContext, ContextGraph
|
||||
from semantica.kg import GraphBuilder
|
||||
from semantica.semantic_extract import NERExtractor, RelationExtractor
|
||||
from semantica.vector_store import VectorStore
|
||||
|
||||
self._graph = context_graph or ContextGraph()
|
||||
|
||||
# Connect GraphBuilder to the ContextGraph so build() persists content.
|
||||
self._graph_builder = graph_builder or GraphBuilder()
|
||||
self._graph_builder.graph_store = self._graph
|
||||
|
||||
self._ner = ner_extractor or NERExtractor()
|
||||
self._rel = relation_extractor or RelationExtractor()
|
||||
|
||||
# Internal AgentContext for vector-based retrieval (shares same graph).
|
||||
self._agent_context = AgentContext(
|
||||
vector_store=VectorStore(backend="faiss"),
|
||||
knowledge_graph=self._graph,
|
||||
decision_tracking=False,
|
||||
)
|
||||
|
||||
# In-process document store for keyword-search fallback
|
||||
self._docs: List[Dict[str, Any]] = []
|
||||
|
||||
logger.info(
|
||||
"AgnoKnowledgeGraph initialised",
|
||||
extra={"backend": graph_store_backend},
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# AgentKnowledge protocol
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
num_documents: Optional[int] = None,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
) -> List[Any]:
|
||||
"""
|
||||
Multi-hop GraphRAG search.
|
||||
|
||||
1. Vector retrieval via ``AgentContext.retrieve()``.
|
||||
2. Graph hop expansion for entities found in top results.
|
||||
3. Returns a list of Agno ``Document`` objects.
|
||||
|
||||
Falls back to keyword scoring over the in-process ``_docs`` cache
|
||||
when vector retrieval is unavailable.
|
||||
"""
|
||||
k = num_documents or self.num_documents
|
||||
results: List[Any] = []
|
||||
|
||||
# Primary: vector similarity retrieval
|
||||
try:
|
||||
retrieved = self._agent_context.retrieve(query, max_results=k)
|
||||
for item in retrieved:
|
||||
if isinstance(item, dict):
|
||||
content = item.get("content", item.get("text", str(item)))
|
||||
entities = item.get("entities", [])
|
||||
meta = {k2: v for k2, v in item.items() if k2 not in ("content", "text")}
|
||||
else:
|
||||
content = str(item)
|
||||
entities = []
|
||||
meta = {}
|
||||
extra = self._graph_context_for(entities) if entities else ""
|
||||
if extra:
|
||||
content = content + "\n\n[Graph context]\n" + extra
|
||||
results.append(AgnoDocument(content=content, meta_data=meta))
|
||||
if results:
|
||||
logger.debug("search('%s') → %d documents (vector)", query, len(results))
|
||||
return results
|
||||
except Exception as exc:
|
||||
logger.debug("Vector retrieval failed, using keyword fallback: %s", exc)
|
||||
|
||||
# Fallback: keyword / substring scoring over in-process cache
|
||||
q_lower = query.lower()
|
||||
scored = [
|
||||
(doc, sum(1 for w in q_lower.split() if w in doc["text"].lower()))
|
||||
for doc in self._docs
|
||||
]
|
||||
scored.sort(key=lambda t: t[1], reverse=True)
|
||||
top = [d for d, _ in scored[:k]]
|
||||
|
||||
for doc in top:
|
||||
extra = self._graph_context_for(doc.get("entities", []))
|
||||
content = doc["text"]
|
||||
if extra:
|
||||
content += "\n\n[Graph context]\n" + extra
|
||||
results.append(
|
||||
AgnoDocument(
|
||||
content=content,
|
||||
id=doc.get("id"),
|
||||
name=doc.get("source"),
|
||||
meta_data=doc.get("metadata", {}),
|
||||
)
|
||||
)
|
||||
|
||||
logger.debug("search('%s') → %d documents (keyword)", query, len(results))
|
||||
return results
|
||||
|
||||
def load(
|
||||
self,
|
||||
path: Union[str, Path, None] = None,
|
||||
urls: Optional[List[str]] = None,
|
||||
texts: Optional[List[str]] = None,
|
||||
recursive: bool = False,
|
||||
recreate: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Ingest documents into the knowledge graph.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path:
|
||||
A file path, directory path, or glob pattern.
|
||||
urls:
|
||||
List of URLs to fetch and ingest.
|
||||
texts:
|
||||
Raw text strings to ingest directly.
|
||||
recursive:
|
||||
When ``path`` points to a directory, walk subdirectories.
|
||||
recreate:
|
||||
Drop all previously loaded documents before ingesting.
|
||||
"""
|
||||
if recreate:
|
||||
self._docs.clear()
|
||||
|
||||
if texts:
|
||||
for text in texts:
|
||||
self._ingest_text(text, source="<inline>")
|
||||
|
||||
if path is not None:
|
||||
self._ingest_path(Path(path), recursive=recursive)
|
||||
|
||||
if urls:
|
||||
self.load_urls(urls)
|
||||
|
||||
def load_urls(self, urls: List[str]) -> None:
|
||||
"""Fetch each URL and ingest the response body.
|
||||
|
||||
Only ``http`` and ``https`` schemes are permitted to prevent SSRF.
|
||||
"""
|
||||
import urllib.request
|
||||
from urllib.parse import urlparse
|
||||
|
||||
for url in urls:
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
logger.warning(
|
||||
"Skipping URL with disallowed scheme '%s': %s",
|
||||
parsed.scheme,
|
||||
url,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=10) as resp: # noqa: S310
|
||||
text = resp.read().decode("utf-8", errors="replace")
|
||||
self._ingest_text(text, source=url)
|
||||
logger.info("Loaded URL: %s", url)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to fetch %s: %s", url, exc)
|
||||
|
||||
# AgentKnowledge also expects `load_documents`
|
||||
def load_documents(
|
||||
self,
|
||||
documents: List[Any],
|
||||
upsert: bool = False,
|
||||
) -> None:
|
||||
"""Ingest a list of Agno ``Document`` objects."""
|
||||
for doc in documents:
|
||||
text = getattr(doc, "content", None) or getattr(doc, "text", str(doc))
|
||||
source = getattr(doc, "name", None) or getattr(doc, "id", "<document>")
|
||||
self._ingest_text(text, source=source)
|
||||
|
||||
def get_graph_context(self, entity: str) -> str:
|
||||
"""
|
||||
Return a structured text representation of an entity's subgraph
|
||||
(neighbours and edge types), suitable for structured reasoning.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
entity:
|
||||
Root entity name (must have been added to the graph).
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Multi-line text with nodes and labelled edge types.
|
||||
"""
|
||||
lines = [f"Entity: {entity}"]
|
||||
try:
|
||||
neighbours = self._graph.get_neighbors(node_id=entity, hops=1)
|
||||
for n in (neighbours or [])[:10]:
|
||||
if isinstance(n, dict):
|
||||
node_id = n.get("node_id", "")
|
||||
ntype = n.get("node_type", "")
|
||||
edge_type = n.get("edge_type", "related_to")
|
||||
suffix = f" (type: {ntype})" if ntype else ""
|
||||
lines.append(f" --[{edge_type}]--> {node_id}{suffix}")
|
||||
else:
|
||||
lines.append(f" --> {getattr(n, 'label', str(n))}")
|
||||
except Exception:
|
||||
pass
|
||||
return "\n".join(lines)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _chunk_text(self, text: str) -> List[str]:
|
||||
"""Split text into chunks at paragraph boundaries."""
|
||||
paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
|
||||
if not paragraphs:
|
||||
return [text] if text.strip() else []
|
||||
|
||||
chunks: List[str] = []
|
||||
current: List[str] = []
|
||||
current_len = 0
|
||||
|
||||
for para in paragraphs:
|
||||
if current_len + len(para) > self.chunk_size and current:
|
||||
chunks.append("\n\n".join(current))
|
||||
current = []
|
||||
current_len = 0
|
||||
current.append(para)
|
||||
current_len += len(para)
|
||||
|
||||
if current:
|
||||
chunks.append("\n\n".join(current))
|
||||
|
||||
return chunks or [text]
|
||||
|
||||
def _ingest_text(self, text: str, source: str = "<text>") -> None:
|
||||
"""Run the full extraction pipeline and store in graph + doc list."""
|
||||
import uuid
|
||||
|
||||
chunks = self._chunk_text(text)
|
||||
all_entities: List[str] = []
|
||||
all_relations: List[Any] = []
|
||||
|
||||
for chunk in chunks:
|
||||
# NER
|
||||
ner_result: List[Any] = []
|
||||
try:
|
||||
ner_result = self._ner.extract_entities(chunk) or []
|
||||
chunk_entities = [getattr(e, "name", str(e)) for e in ner_result]
|
||||
all_entities.extend(chunk_entities)
|
||||
except Exception as exc:
|
||||
logger.debug("NER failed for chunk in '%s': %s", source, exc)
|
||||
|
||||
# Relation extraction
|
||||
try:
|
||||
chunk_relations = self._rel.extract_relations(chunk, entities=ner_result) or []
|
||||
all_relations.extend(chunk_relations)
|
||||
except Exception as exc:
|
||||
logger.debug("RelationExtractor failed for chunk in '%s': %s", source, exc)
|
||||
|
||||
# Graph build — graph_store is wired to self._graph in __init__
|
||||
try:
|
||||
sources = [
|
||||
{
|
||||
"text": text,
|
||||
"entities": all_entities,
|
||||
"relations": all_relations,
|
||||
"source": source,
|
||||
}
|
||||
]
|
||||
self._graph_builder.build(sources)
|
||||
except Exception as exc:
|
||||
logger.debug("GraphBuilder.build() failed for '%s': %s", source, exc)
|
||||
|
||||
# Vector index for AgentContext.retrieve()
|
||||
try:
|
||||
self._agent_context.store(text, conversation_id=source)
|
||||
except Exception as exc:
|
||||
logger.debug("AgentContext.store() failed for '%s': %s", source, exc)
|
||||
|
||||
# Cache document for keyword-search fallback
|
||||
self._docs.append(
|
||||
{
|
||||
"id": str(uuid.uuid4()),
|
||||
"text": text,
|
||||
"source": source,
|
||||
"entities": all_entities,
|
||||
"metadata": {"source": source},
|
||||
}
|
||||
)
|
||||
logger.debug(
|
||||
"Ingested '%s' — %d entities, %d relations, %d chunks",
|
||||
source,
|
||||
len(all_entities),
|
||||
len(all_relations),
|
||||
len(chunks),
|
||||
)
|
||||
|
||||
def _ingest_path(self, path: Path, recursive: bool = False) -> None:
|
||||
"""Walk a file or directory and ingest all text files."""
|
||||
if path.is_file():
|
||||
self._ingest_file(path)
|
||||
elif path.is_dir():
|
||||
pattern = "**/*" if recursive else "*"
|
||||
for child in path.glob(pattern):
|
||||
if child.is_file():
|
||||
self._ingest_file(child)
|
||||
else:
|
||||
logger.warning("Path not found: %s", path)
|
||||
|
||||
def _ingest_file(self, filepath: Path) -> None:
|
||||
try:
|
||||
text = filepath.read_text(encoding="utf-8", errors="replace")
|
||||
self._ingest_text(text, source=str(filepath))
|
||||
except Exception as exc:
|
||||
logger.warning("Could not read %s: %s", filepath, exc)
|
||||
|
||||
def _graph_context_for(self, entities: List[str]) -> str:
|
||||
"""Build a short text summary of graph neighbours for a set of entities."""
|
||||
if not entities:
|
||||
return ""
|
||||
lines: List[str] = []
|
||||
for entity in entities[:3]: # limit to avoid context bloat
|
||||
try:
|
||||
neighbours = self._graph.get_neighbors(node_id=entity, hops=1)
|
||||
for n in (neighbours or [])[:3]:
|
||||
if isinstance(n, dict):
|
||||
node_id = n.get("node_id", "")
|
||||
ntype = n.get("node_type", "")
|
||||
edge_type = n.get("edge_type", "related_to")
|
||||
lines.append(
|
||||
f"- {entity} --[{edge_type}]--> {node_id}"
|
||||
+ (f" ({ntype})" if ntype else "")
|
||||
)
|
||||
else:
|
||||
lines.append(f"- {entity} --> {getattr(n, 'label', str(n))}")
|
||||
except Exception:
|
||||
pass
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,292 @@
|
||||
"""
|
||||
AgnoSharedContext — Shared ContextGraph for Agno multi-agent teams.
|
||||
|
||||
A single ``ContextGraph`` is shared across all agents in an Agno ``Team``.
|
||||
Each agent gets a **role-scoped view** via ``bind_agent()``, which returns an
|
||||
``AgnoContextStore`` namespaced to that agent's role. This prevents
|
||||
contradictory decisions and enables knowledge reuse without coupling agent
|
||||
implementations.
|
||||
|
||||
Install
|
||||
-------
|
||||
pip install semantica[agno]
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> from semantica.context import ContextGraph
|
||||
>>> from semantica.vector_store import VectorStore
|
||||
>>> from integrations.agno import AgnoSharedContext, AgnoDecisionKit, AgnoKGToolkit
|
||||
>>> shared = AgnoSharedContext(
|
||||
... vector_store=VectorStore(backend="faiss"),
|
||||
... knowledge_graph=ContextGraph(advanced_analytics=True),
|
||||
... decision_tracking=True,
|
||||
... )
|
||||
>>> from agno.agent import Agent
|
||||
>>> from agno.team import Team
|
||||
>>> researcher = Agent(
|
||||
... name="Researcher",
|
||||
... memory=shared.bind_agent("researcher"),
|
||||
... tools=[AgnoKGToolkit(context=shared)],
|
||||
... )
|
||||
>>> analyst = Agent(
|
||||
... name="Analyst",
|
||||
... memory=shared.bind_agent("analyst"),
|
||||
... tools=[AgnoDecisionKit(context=shared)],
|
||||
... )
|
||||
>>> team = Team(agents=[researcher, analyst], mode="coordinate")
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from semantica.utils.logging import get_logger
|
||||
|
||||
from .context_store import AgnoContextStore
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class _AgentScopedStore(AgnoContextStore):
|
||||
"""
|
||||
An ``AgnoContextStore`` bound to a specific agent role.
|
||||
|
||||
All operations are delegated to the parent ``AgnoSharedContext``'s
|
||||
``AgentContext`` but tagged with the agent's ``role`` for filtering.
|
||||
"""
|
||||
|
||||
def __init__(self, shared: "AgnoSharedContext", role: str) -> None:
|
||||
# Re-use the parent's context rather than creating a new one.
|
||||
# We skip the normal __init__ and wire all required parent attributes
|
||||
# directly so that inherited methods (record_decision, find_precedents,
|
||||
# retrieve, get_context_for_prompt) work correctly via self._context.
|
||||
self._role = role
|
||||
self._shared = shared
|
||||
self._memories: Dict[str, Any] = {}
|
||||
self.decision_tracking = shared.decision_tracking
|
||||
self.graph_expansion = shared.graph_expansion
|
||||
self.session_id = f"{shared.session_id}::{role}"
|
||||
# Use the attribute name the parent class expects.
|
||||
self._context = shared._context # type: ignore[attr-defined]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Override upsert / record to tag with role
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def upsert_memory(self, memory: Any) -> Optional[Any]: # type: ignore[override]
|
||||
import uuid
|
||||
|
||||
mem_id = getattr(memory, "id", None) or str(uuid.uuid4())
|
||||
mem_text = getattr(memory, "memory", str(memory))
|
||||
|
||||
try:
|
||||
self._context.store(mem_text, conversation_id=self.session_id)
|
||||
except Exception as exc:
|
||||
logger.warning("[%s] store failed: %s", self._role, exc)
|
||||
|
||||
if self.decision_tracking:
|
||||
try:
|
||||
self._context.record_decision(
|
||||
category=f"memory:{self._role}",
|
||||
scenario=mem_text[:200],
|
||||
reasoning=f"Stored by agent role='{self._role}'",
|
||||
outcome="stored",
|
||||
confidence=1.0,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if hasattr(memory, "id"):
|
||||
memory.id = mem_id
|
||||
self._memories[mem_id] = memory
|
||||
|
||||
# Also push into the shared registry so all agents can read it
|
||||
self._shared._shared_memories[mem_id] = memory
|
||||
|
||||
return memory
|
||||
|
||||
def read_memories( # type: ignore[override]
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
sort: Optional[str] = None,
|
||||
) -> List[Any]:
|
||||
# Return own memories + shared memories from all agents
|
||||
combined = dict(self._shared._shared_memories)
|
||||
combined.update(self._memories)
|
||||
|
||||
rows = list(combined.values())
|
||||
if user_id:
|
||||
rows = [r for r in rows if getattr(r, "user_id", None) == user_id]
|
||||
|
||||
reverse = sort != "asc"
|
||||
rows.sort(key=lambda r: getattr(r, "last_updated", 0), reverse=reverse)
|
||||
|
||||
if limit is not None:
|
||||
rows = rows[:limit]
|
||||
return rows
|
||||
|
||||
|
||||
class AgnoSharedContext:
|
||||
"""
|
||||
Shared context graph coordinator for Agno multi-agent teams.
|
||||
|
||||
Maintains a single ``AgentContext`` and ``ContextGraph`` that all agents
|
||||
access concurrently. Thread-safety is ensured via a reentrant lock.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
vector_store:
|
||||
Shared ``semantica.vector_store.VectorStore`` instance.
|
||||
knowledge_graph:
|
||||
Shared ``semantica.context.ContextGraph`` instance.
|
||||
decision_tracking:
|
||||
Enable decision recording for all bound agents.
|
||||
graph_expansion:
|
||||
Enable graph-hop expansion in all bound agents' ``read_memories``.
|
||||
session_id:
|
||||
Team-level session identifier (auto-generated when ``None``).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vector_store: Any = None,
|
||||
knowledge_graph: Any = None,
|
||||
decision_tracking: bool = True,
|
||||
graph_expansion: bool = True,
|
||||
session_id: Optional[str] = None,
|
||||
**agent_context_kwargs: Any,
|
||||
) -> None:
|
||||
import uuid
|
||||
|
||||
from semantica.context import AgentContext, ContextGraph
|
||||
from semantica.vector_store import VectorStore
|
||||
|
||||
self.decision_tracking = decision_tracking
|
||||
self.graph_expansion = graph_expansion
|
||||
self.session_id = session_id or str(uuid.uuid4())
|
||||
|
||||
if knowledge_graph is None:
|
||||
knowledge_graph = ContextGraph(advanced_analytics=True)
|
||||
|
||||
if vector_store is None:
|
||||
vector_store = VectorStore(backend="faiss")
|
||||
|
||||
self._context = AgentContext(
|
||||
vector_store=vector_store,
|
||||
knowledge_graph=knowledge_graph,
|
||||
decision_tracking=decision_tracking,
|
||||
**agent_context_kwargs,
|
||||
)
|
||||
self._knowledge_graph = knowledge_graph
|
||||
|
||||
# Shared memory pool (all agents read from this)
|
||||
self._shared_memories: Dict[str, Any] = {}
|
||||
self._lock = threading.RLock()
|
||||
self._bound_agents: Dict[str, _AgentScopedStore] = {}
|
||||
|
||||
logger.info(
|
||||
"AgnoSharedContext initialised (session=%s, decision_tracking=%s)",
|
||||
self.session_id,
|
||||
decision_tracking,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def bind_agent(self, role: str) -> _AgentScopedStore:
|
||||
"""
|
||||
Return a role-scoped ``AgnoContextStore`` for the given agent role.
|
||||
|
||||
Multiple calls with the same ``role`` return the **same** store
|
||||
instance (idempotent).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
role:
|
||||
Agent role name, e.g. ``"researcher"``, ``"analyst"``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
_AgentScopedStore
|
||||
An ``AgnoContextStore`` scoped to ``role`` backed by this shared
|
||||
context.
|
||||
"""
|
||||
with self._lock:
|
||||
if role not in self._bound_agents:
|
||||
store = _AgentScopedStore(shared=self, role=role)
|
||||
self._bound_agents[role] = store
|
||||
logger.info("Bound agent role='%s' to shared context", role)
|
||||
return self._bound_agents[role]
|
||||
|
||||
def record_decision(
|
||||
self,
|
||||
category: str,
|
||||
scenario: str,
|
||||
reasoning: str,
|
||||
outcome: str,
|
||||
confidence: float = 0.8,
|
||||
entities: Optional[List[str]] = None,
|
||||
agent_role: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Record a decision into the shared context graph.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
agent_role:
|
||||
If provided, the decision is tagged with this agent's role.
|
||||
"""
|
||||
tagged_category = f"{category}:{agent_role}" if agent_role else category
|
||||
with self._lock:
|
||||
return self._context.record_decision(
|
||||
category=tagged_category,
|
||||
scenario=scenario,
|
||||
reasoning=reasoning,
|
||||
outcome=outcome,
|
||||
confidence=confidence,
|
||||
entities=entities,
|
||||
)
|
||||
|
||||
def find_precedents(
|
||||
self,
|
||||
scenario: str,
|
||||
category: Optional[str] = None,
|
||||
limit: int = 5,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search all agents' decision history for similar precedents."""
|
||||
try:
|
||||
return self._context.find_precedents_advanced(
|
||||
scenario=scenario,
|
||||
category=category,
|
||||
limit=limit,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("find_precedents failed: %s", exc)
|
||||
return []
|
||||
|
||||
def get_shared_insights(self) -> Dict[str, Any]:
|
||||
"""Return analytics over the full shared decision graph."""
|
||||
try:
|
||||
return self._context.get_context_insights()
|
||||
except Exception as exc:
|
||||
logger.warning("get_shared_insights failed: %s", exc)
|
||||
return {}
|
||||
|
||||
@property
|
||||
def knowledge_graph(self) -> Any:
|
||||
"""Direct access to the shared ``ContextGraph``."""
|
||||
return self._knowledge_graph
|
||||
|
||||
@property
|
||||
def bound_roles(self) -> List[str]:
|
||||
"""List of agent roles currently bound to this shared context."""
|
||||
return list(self._bound_agents.keys())
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover
|
||||
return (
|
||||
f"AgnoSharedContext(session={self.session_id!r}, "
|
||||
f"agents={self.bound_roles})"
|
||||
)
|
||||
@@ -138,6 +138,7 @@ nav:
|
||||
- examples.md
|
||||
- glossary.md
|
||||
- Integrations:
|
||||
- Agno: integrations/agno.md
|
||||
- Docling: integrations/docling.md
|
||||
- Snowflake: integrations/snowflake.md
|
||||
- Cookbook: cookbook.md
|
||||
|
||||
+27
-7
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "semantica"
|
||||
version = "0.3.0-beta"
|
||||
version = "0.3.0"
|
||||
description = "🧠 Semantica - An Open Source Framework for building Semantic Layers and Knowledge Engineering"
|
||||
readme = "README.md"
|
||||
license = { text = "MIT" }
|
||||
@@ -15,7 +15,7 @@ maintainers = [{ name = "Hawksight AI", email = "semantica-dev@users.noreply.git
|
||||
requires-python = ">=3.8"
|
||||
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"Intended Audience :: Developers",
|
||||
"Intended Audience :: Science/Research",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
@@ -156,8 +156,8 @@ monitoring = [
|
||||
"prometheus-client>=0.14.0",
|
||||
"opentelemetry-api>=1.30.0,<2.0.0",
|
||||
"opentelemetry-sdk>=1.30.0,<2.0.0",
|
||||
"opentelemetry-semantic-conventions>=0.58b0,<0.61b0",
|
||||
"opentelemetry-instrumentation>=0.58b0,<0.61b0"
|
||||
"opentelemetry-semantic-conventions>=0.58b0,<0.62",
|
||||
"opentelemetry-instrumentation>=0.58b0,<0.62"
|
||||
]
|
||||
|
||||
# ---- Visualization ----
|
||||
@@ -173,6 +173,9 @@ gpu = [
|
||||
"cupy>=10.0.0"
|
||||
]
|
||||
|
||||
# ---- Agentic Framework Integrations ----
|
||||
agno = ["agno>=1.0.0"]
|
||||
|
||||
# ---- Splitting / Chunking ----
|
||||
split-tiktoken = ["tiktoken>=0.5.0"]
|
||||
split-community = ["python-louvain>=0.16"]
|
||||
@@ -196,9 +199,22 @@ dev = [
|
||||
"ipykernel>=6.15.0"
|
||||
]
|
||||
|
||||
# ---- Everything ----
|
||||
# Explorer Dashboard
|
||||
explorer = [
|
||||
"fastapi>=0.100.0",
|
||||
"uvicorn[standard]>=0.22.0",
|
||||
"websockets>=11.0",
|
||||
"python-multipart>=0.0.6"
|
||||
]
|
||||
explorer-lite = [
|
||||
"streamlit>=1.25.0",
|
||||
"streamlit-agraph>=0.0.45"
|
||||
]
|
||||
|
||||
# Everything
|
||||
all = [
|
||||
"semantica[dev,viz,gpu,infra,cloud,monitoring,llm-all,models-huggingface,split-all,graph-all,vectorstore-all,parse-docling]"
|
||||
"semantica[dev,viz,gpu,infra,cloud,monitoring,llm-all,models-huggingface,split-all,graph-all,vectorstore-all,parse-docling,explorer]",
|
||||
"semantica[dev,viz,gpu,infra,cloud,monitoring,llm-all,models-huggingface,split-all,graph-all,vectorstore-all,parse-docling,agno]"
|
||||
]
|
||||
|
||||
# ---------------- ENTRYPOINTS ----------------
|
||||
@@ -206,11 +222,12 @@ all = [
|
||||
semantica = "semantica.cli:main"
|
||||
semantica-server = "semantica.server:main"
|
||||
semantica-worker = "semantica.worker:main"
|
||||
semantica-explorer = "semantica.explorer:main"
|
||||
|
||||
# ---------------- TOOLING ----------------
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["semantica*"]
|
||||
include = ["semantica*", "integrations*"]
|
||||
|
||||
[tool.black]
|
||||
line-length = 88
|
||||
@@ -220,3 +237,6 @@ profile = "black"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
markers = [
|
||||
"integration: marks tests that require external services or API keys (deselect with '-m not integration')",
|
||||
]
|
||||
|
||||
@@ -10,7 +10,7 @@ Main exports:
|
||||
- Config: Configuration management
|
||||
"""
|
||||
|
||||
__version__ = "0.3.0-beta"
|
||||
__version__ = "0.3.0"
|
||||
__author__ = "Semantica Contributors"
|
||||
__license__ = "MIT"
|
||||
|
||||
|
||||
@@ -9,12 +9,14 @@ Key Features:
|
||||
- Email validation for authors
|
||||
- Timestamp handling in ISO 8601 format
|
||||
- Optional change linking and tracking
|
||||
- Granular MutationRecord for per-entity audit trails
|
||||
|
||||
Main Classes:
|
||||
- ChangeLogEntry: Standard metadata for version changes
|
||||
- MutationRecord: Granular log of node/edge level changes
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.common.change_log import ChangeLogEntry
|
||||
>>> from semantica.change_management.change_log import ChangeLogEntry
|
||||
>>> entry = ChangeLogEntry(
|
||||
... timestamp="2024-01-15T10:30:00Z",
|
||||
... author="alice@company.com",
|
||||
@@ -28,7 +30,8 @@ License: MIT
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
from typing import List, Optional, Any, Dict, Tuple, Union, Set
|
||||
from enum import Enum
|
||||
|
||||
from ..utils.exceptions import ValidationError
|
||||
|
||||
@@ -106,3 +109,171 @@ class ChangeLogEntry:
|
||||
change_id=change_id,
|
||||
related_changes=related_changes or []
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MutationRecord:
|
||||
"""
|
||||
Granular record of a single change to a graph entity (node or edge).
|
||||
|
||||
Attributes:
|
||||
timestamp: ISO 8601 timestamp of the mutation
|
||||
operation: 'ADD_NODE', 'UPDATE_NODE', 'REMOVE_NODE', 'ADD_EDGE', 'UPDATE_EDGE', 'REMOVE_EDGE'
|
||||
entity_id: The ID of the affected node or edge
|
||||
payload: The state of the entity after the mutation
|
||||
version_label: Optional association with a specific saved snapshot version
|
||||
"""
|
||||
timestamp: str
|
||||
operation: str
|
||||
entity_id: str
|
||||
payload: Dict[str, Any]
|
||||
version_label: Optional[str] = None
|
||||
|
||||
class Severity(Enum):
|
||||
CRITICAL = "critical"
|
||||
HIGH = "high"
|
||||
MEDIUM = "medium"
|
||||
LOW = "low"
|
||||
INFO = "info"
|
||||
|
||||
class ChangeCategory(Enum):
|
||||
BREAKING = "breaking"
|
||||
POTENTIALLY_BREAKING = "potentially_breaking"
|
||||
NON_BREAKING = "non_breaking"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
@dataclass
|
||||
class ImpactReport:
|
||||
""" Structured impact analysis report."""
|
||||
timestamp: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
summary: Dict[str, Any] = field(default_factory=dict)
|
||||
breaking_changes: List[Dict[str, Any]] = field(default_factory=list)
|
||||
potentially_breaking: List[Dict[str, Any]] = field(default_factory=list)
|
||||
safe_changes: List[Dict[str, Any]] = field(default_factory=list)
|
||||
recommendations: List[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"timestamp": self.timestamp,
|
||||
"summary": self.summary,
|
||||
"impact_classification": {
|
||||
"breaking": self.breaking_changes,
|
||||
"potentially_breaking": self.potentially_breaking,
|
||||
"safe": self.safe_changes
|
||||
},
|
||||
"recommendations": self.recommendations
|
||||
}
|
||||
|
||||
class ChangeLogAnalyzer:
|
||||
"""
|
||||
Analyzes ontology diffs and classifies impact severity.
|
||||
"""
|
||||
|
||||
VALIDITY_CONSTRAINTS = {'domain', 'range', 'cardinality', 'max_cardinality'}
|
||||
STRUCTURAL_FIELDS = {'subclasses', 'superclasses', 'equivalent_to', 'disjoint_with'}
|
||||
|
||||
def analyze(self, diff: Dict[str, Any]) -> ImpactReport:
|
||||
report = ImpactReport()
|
||||
if not diff:
|
||||
report.summary = {"error": "Empty diff provided"}
|
||||
return report
|
||||
|
||||
all_changes = []
|
||||
|
||||
for key, entity_type, change_type in [
|
||||
("added_classes", "class", "added"), ("added_properties", "property", "added"),
|
||||
("removed_classes", "class", "removed"), ("removed_properties", "property", "removed"),
|
||||
("changed_classes", "class", "modified"), ("changed_properties", "property", "modified")
|
||||
]:
|
||||
for item in diff.get(key, []):
|
||||
all_changes.append({
|
||||
"uri": item.get("uri", item.get("name", "unknown")),
|
||||
"entity_type": entity_type,
|
||||
"change_type": change_type,
|
||||
"changes": item.get("changes", {})
|
||||
})
|
||||
|
||||
report.summary = {"total_changes": len(all_changes)}
|
||||
|
||||
for change in all_changes:
|
||||
severity, category, description, mitigation = self._classify_change(change)
|
||||
entry = {
|
||||
"entity_uri": change['uri'],
|
||||
"entity_type": change['entity_type'],
|
||||
"change_type": change['change_type'],
|
||||
"description": description,
|
||||
"severity": severity.value,
|
||||
"mitigation": mitigation
|
||||
}
|
||||
|
||||
if category == ChangeCategory.BREAKING:
|
||||
report.breaking_changes.append(entry)
|
||||
elif category == ChangeCategory.POTENTIALLY_BREAKING:
|
||||
report.potentially_breaking.append(entry)
|
||||
else:
|
||||
report.safe_changes.append(entry)
|
||||
|
||||
self._generate_recommendations(report)
|
||||
return report
|
||||
|
||||
|
||||
def _classify_change(self, change: Dict[str, Any]) -> Tuple[Severity, ChangeCategory, str, str]:
|
||||
change_type = change.get('change_type')
|
||||
entity_type = change.get('entity_type')
|
||||
uri = change.get('uri')
|
||||
|
||||
if change_type == 'removed':
|
||||
if entity_type == 'class':
|
||||
return (Severity.CRITICAL, ChangeCategory.BREAKING, f"Class {uri} removed.", "Migrate orphaned instances.")
|
||||
return (Severity.CRITICAL, ChangeCategory.BREAKING, f"Property {uri} removed.", "Migrate property values.")
|
||||
|
||||
if change_type == 'added':
|
||||
return (Severity.INFO, ChangeCategory.NON_BREAKING, f"New {entity_type} {uri} added.", "No action required.")
|
||||
|
||||
if change_type == 'modified':
|
||||
return self._analyze_field_changes(uri, change.get('changes', {}))
|
||||
|
||||
return (Severity.LOW, ChangeCategory.UNKNOWN, f"Unknown change for {uri}", "Manual review required.")
|
||||
|
||||
def _analyze_field_changes(self, uri: str, field_changes: Dict[str, Any]) -> Tuple[Severity, ChangeCategory, str, str]:
|
||||
has_restriction = False
|
||||
has_structural = False
|
||||
|
||||
for field, vals in field_changes.items():
|
||||
if field in self.VALIDITY_CONSTRAINTS:
|
||||
old_val, new_val = vals.get("old"), vals.get("new")
|
||||
|
||||
if old_val is None or new_val is None:
|
||||
has_restriction = True
|
||||
continue
|
||||
|
||||
# if new constraint is smaller, it is a restriction
|
||||
old_set = set(old_val) if isinstance(old_val, list) else {old_val}
|
||||
new_set = set(new_val) if isinstance(new_val, list) else {new_val}
|
||||
|
||||
if new_set < old_set:
|
||||
has_restriction = True
|
||||
|
||||
elif field in self.STRUCTURAL_FIELDS:
|
||||
has_structural = True
|
||||
|
||||
if has_restriction:
|
||||
return (Severity.HIGH, ChangeCategory.BREAKING, f"Domain/range restricted on {uri}", "Validate existing data against new constraints.")
|
||||
if has_structural:
|
||||
return (Severity.MEDIUM, ChangeCategory.POTENTIALLY_BREAKING, f"Hierarchy modified for {uri}", "Check dependent reasoning chains.")
|
||||
|
||||
return (Severity.LOW, ChangeCategory.NON_BREAKING, f"Safe annotations updated for {uri}", "No action required.")
|
||||
|
||||
def _generate_recommendations(self, report: ImpactReport) -> None:
|
||||
if report.breaking_changes:
|
||||
report.recommendations.append("[BREAKING] Schedule downtime or validate existing data.")
|
||||
if report.potentially_breaking:
|
||||
report.recommendations.append("[WARNING] Run full regression tests on queries.")
|
||||
if not report.breaking_changes and not report.potentially_breaking:
|
||||
report.recommendations.append("[SAFE] Minor version bump sufficient.")
|
||||
|
||||
|
||||
def generate_change_report(diff: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Public API for generating impact reports from diffs."""
|
||||
analyzer = ChangeLogAnalyzer()
|
||||
return analyzer.analyze(diff).to_dict()
|
||||
|
||||
@@ -1041,4 +1041,4 @@ manager = TemporalVersionManager(storage_path="large_data.db")
|
||||
For questions or issues:
|
||||
- GitHub Issues: https://github.com/Hawksight-AI/semantica/issues
|
||||
- Documentation: https://semantica.readthedocs.io
|
||||
- Community: https://discord.gg/N7WmAuDH
|
||||
- Community: https://discord.gg/sV34vps5hH
|
||||
|
||||
@@ -114,6 +114,7 @@ class TemporalVersionManager(BaseVersionManager):
|
||||
"""
|
||||
super().__init__(storage_path)
|
||||
self.config = config
|
||||
self._attached_graphs: List[Any] = []
|
||||
|
||||
def create_snapshot(
|
||||
self,
|
||||
@@ -127,7 +128,8 @@ class TemporalVersionManager(BaseVersionManager):
|
||||
Create and store snapshot with checksum and metadata.
|
||||
|
||||
Args:
|
||||
graph: Knowledge graph dict with "entities" and "relationships"
|
||||
graph: Knowledge graph dict with "nodes"/"edges" or
|
||||
legacy "entities"/"relationships"
|
||||
version_label: Version string (e.g., "v1.0")
|
||||
author: Email address of the change author
|
||||
description: Change description (max 500 chars)
|
||||
@@ -140,31 +142,56 @@ class TemporalVersionManager(BaseVersionManager):
|
||||
ValidationError: If input validation fails
|
||||
ProcessingError: If storage operation fails
|
||||
"""
|
||||
# Validate inputs
|
||||
change_entry = ChangeLogEntry(
|
||||
timestamp=datetime.now().isoformat(), author=author, description=description
|
||||
)
|
||||
entities, relationships = self._extract_graph_collections(graph)
|
||||
|
||||
# Create snapshot
|
||||
snapshot = {
|
||||
"label": version_label,
|
||||
"timestamp": change_entry.timestamp,
|
||||
"author": change_entry.author,
|
||||
"description": change_entry.description,
|
||||
"entities": graph.get("entities", []).copy(),
|
||||
"relationships": graph.get("relationships", []).copy(),
|
||||
# Store both key shapes during the migration window so older
|
||||
# readers and newer ContextGraph restore paths both work.
|
||||
"nodes": entities.copy(),
|
||||
"edges": relationships.copy(),
|
||||
"entities": entities.copy(),
|
||||
"relationships": relationships.copy(),
|
||||
"metadata": options.get("metadata", {}),
|
||||
}
|
||||
|
||||
# Compute and add checksum
|
||||
snapshot["checksum"] = compute_checksum(snapshot)
|
||||
|
||||
# Store snapshot
|
||||
self.storage.save(snapshot)
|
||||
|
||||
self.storage.assign_version_to_unlabeled_mutations(version_label)
|
||||
self.logger.info(f"Created snapshot '{version_label}' by {author}")
|
||||
return snapshot
|
||||
|
||||
def _extract_graph_collections(
|
||||
self, graph: Dict[str, Any]
|
||||
) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
||||
"""Normalize graph payloads to entity/relationship collections."""
|
||||
if not isinstance(graph, dict):
|
||||
raise ValidationError("Graph must be provided as a dictionary")
|
||||
|
||||
has_node_schema = "nodes" in graph or "edges" in graph
|
||||
has_legacy_schema = "entities" in graph or "relationships" in graph
|
||||
if not (has_node_schema or has_legacy_schema):
|
||||
raise ValidationError(
|
||||
"Graph dictionary must contain 'nodes'/'edges' or "
|
||||
"'entities'/'relationships'"
|
||||
)
|
||||
|
||||
entities = graph.get("nodes")
|
||||
if entities is None:
|
||||
entities = graph.get("entities", [])
|
||||
|
||||
relationships = graph.get("edges")
|
||||
if relationships is None:
|
||||
relationships = graph.get("relationships", [])
|
||||
|
||||
return list(entities or []), list(relationships or [])
|
||||
|
||||
def compare_versions(
|
||||
self,
|
||||
v1_label_or_dict,
|
||||
@@ -184,7 +211,6 @@ class TemporalVersionManager(BaseVersionManager):
|
||||
Returns:
|
||||
dict: Detailed version comparison results
|
||||
"""
|
||||
# Handle both label strings and snapshot dictionaries
|
||||
if isinstance(v1_label_or_dict, str):
|
||||
version1 = self.storage.get(v1_label_or_dict)
|
||||
if not version1:
|
||||
@@ -199,10 +225,8 @@ class TemporalVersionManager(BaseVersionManager):
|
||||
else:
|
||||
version2 = v2_label_or_dict
|
||||
|
||||
# Compute detailed diff
|
||||
detailed_diff = self._compute_detailed_diff(version1, version2)
|
||||
|
||||
# Maintain backward compatibility with summary
|
||||
summary = {
|
||||
"entities_added": len(detailed_diff["entities_added"]),
|
||||
"entities_removed": len(detailed_diff["entities_removed"]),
|
||||
@@ -210,6 +234,12 @@ class TemporalVersionManager(BaseVersionManager):
|
||||
"relationships_added": len(detailed_diff["relationships_added"]),
|
||||
"relationships_removed": len(detailed_diff["relationships_removed"]),
|
||||
"relationships_modified": len(detailed_diff["relationships_modified"]),
|
||||
"nodes_added": len(detailed_diff["nodes_added"]),
|
||||
"nodes_removed": len(detailed_diff["nodes_removed"]),
|
||||
"nodes_modified": len(detailed_diff["nodes_modified"]),
|
||||
"edges_added": len(detailed_diff["edges_added"]),
|
||||
"edges_removed": len(detailed_diff["edges_removed"]),
|
||||
"edges_modified": len(detailed_diff["edges_modified"]),
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -232,71 +262,63 @@ class TemporalVersionManager(BaseVersionManager):
|
||||
Returns:
|
||||
Dict with detailed diff information
|
||||
"""
|
||||
entities1 = {
|
||||
e.get("id", str(i)): e for i, e in enumerate(version1.get("entities", []))
|
||||
}
|
||||
entities2 = {
|
||||
e.get("id", str(i)): e for i, e in enumerate(version2.get("entities", []))
|
||||
}
|
||||
version1_entities, version1_relationships = self._extract_graph_collections(
|
||||
version1
|
||||
)
|
||||
version2_entities, version2_relationships = self._extract_graph_collections(
|
||||
version2
|
||||
)
|
||||
|
||||
relationships1 = {
|
||||
self._relationship_key(r): r for r in version1.get("relationships", [])
|
||||
}
|
||||
relationships2 = {
|
||||
self._relationship_key(r): r for r in version2.get("relationships", [])
|
||||
}
|
||||
nodes1 = {n.get("id", str(i)): n for i, n in enumerate(version1_entities)}
|
||||
nodes2 = {n.get("id", str(i)): n for i, n in enumerate(version2_entities)}
|
||||
|
||||
# Entity differences
|
||||
entity_ids1 = set(entities1.keys())
|
||||
entity_ids2 = set(entities2.keys())
|
||||
edges1 = {self._relationship_key(e): e for e in version1_relationships}
|
||||
edges2 = {self._relationship_key(e): e for e in version2_relationships}
|
||||
|
||||
entities_added = [entities2[eid] for eid in entity_ids2 - entity_ids1]
|
||||
entities_removed = [entities1[eid] for eid in entity_ids1 - entity_ids2]
|
||||
|
||||
entities_modified = []
|
||||
for eid in entity_ids1 & entity_ids2:
|
||||
if entities1[eid] != entities2[eid]:
|
||||
changes = self._compute_entity_changes(entities1[eid], entities2[eid])
|
||||
entities_modified.append(
|
||||
{
|
||||
"id": eid,
|
||||
"before": entities1[eid],
|
||||
"after": entities2[eid],
|
||||
"changes": changes,
|
||||
}
|
||||
)
|
||||
node_ids1 = set(nodes1.keys())
|
||||
node_ids2 = set(nodes2.keys())
|
||||
|
||||
# Relationship differences
|
||||
rel_keys1 = set(relationships1.keys())
|
||||
rel_keys2 = set(relationships2.keys())
|
||||
nodes_added = [nodes2[nid] for nid in node_ids2 - node_ids1]
|
||||
nodes_removed = [nodes1[nid] for nid in node_ids1 - node_ids2]
|
||||
nodes_modified = []
|
||||
for nid in node_ids1 & node_ids2:
|
||||
if nodes1[nid] != nodes2[nid]:
|
||||
changes = self._compute_entity_changes(nodes1[nid], nodes2[nid])
|
||||
nodes_modified.append({
|
||||
"id": nid, "before": nodes1[nid], "after": nodes2[nid], "changes": changes
|
||||
})
|
||||
|
||||
relationships_added = [relationships2[key] for key in rel_keys2 - rel_keys1]
|
||||
relationships_removed = [relationships1[key] for key in rel_keys1 - rel_keys2]
|
||||
|
||||
relationships_modified = []
|
||||
for key in rel_keys1 & rel_keys2:
|
||||
if relationships1[key] != relationships2[key]:
|
||||
changes = self._compute_relationship_changes(
|
||||
relationships1[key], relationships2[key]
|
||||
)
|
||||
relationships_modified.append(
|
||||
{
|
||||
"key": key,
|
||||
"before": relationships1[key],
|
||||
"after": relationships2[key],
|
||||
"changes": changes,
|
||||
}
|
||||
)
|
||||
edge_keys1 = set(edges1.keys())
|
||||
edge_keys2 = set(edges2.keys())
|
||||
|
||||
edges_added = [edges2[k] for k in edge_keys2 - edge_keys1]
|
||||
edges_removed = [edges1[k] for k in edge_keys1 - edge_keys2]
|
||||
edges_modified = []
|
||||
for k in edge_keys1 & edge_keys2:
|
||||
if edges1[k] != edges2[k]:
|
||||
changes = self._compute_relationship_changes(edges1[k], edges2[k])
|
||||
edges_modified.append({
|
||||
"key": k, "before": edges1[k], "after": edges2[k], "changes": changes
|
||||
})
|
||||
|
||||
return {
|
||||
"entities_added": entities_added,
|
||||
"entities_removed": entities_removed,
|
||||
"entities_modified": entities_modified,
|
||||
"relationships_added": relationships_added,
|
||||
"relationships_removed": relationships_removed,
|
||||
"relationships_modified": relationships_modified,
|
||||
"entities_added": nodes_added,
|
||||
"entities_removed": nodes_removed,
|
||||
"entities_modified": nodes_modified,
|
||||
"relationships_added": edges_added,
|
||||
"relationships_removed": edges_removed,
|
||||
"relationships_modified": edges_modified,
|
||||
"nodes_added": nodes_added,
|
||||
"nodes_removed": nodes_removed,
|
||||
"nodes_modified": nodes_modified,
|
||||
"edges_added": edges_added,
|
||||
"edges_removed": edges_removed,
|
||||
"edges_modified": edges_modified,
|
||||
}
|
||||
|
||||
|
||||
|
||||
def _relationship_key(self, relationship: Dict[str, Any]) -> str:
|
||||
"""Generate a unique key for a relationship."""
|
||||
source = relationship.get("source", "")
|
||||
@@ -377,11 +399,110 @@ class TemporalVersionManager(BaseVersionManager):
|
||||
"pruned_versions": deleted_labels,
|
||||
"retained_count": len(all_versions) - len(deleted_labels)
|
||||
}
|
||||
|
||||
# Git-like audit trails
|
||||
|
||||
def attach_to_graph(self, graph: Any) -> None:
|
||||
"""
|
||||
Attach this manager to a ContextGraph to enable mutation tracking.
|
||||
Injects the mutation callback into the graph to capture all node/edge changes.
|
||||
"""
|
||||
graph.mutation_callback = self.record_mutation
|
||||
if graph not in self._attached_graphs:
|
||||
self._attached_graphs.append(graph)
|
||||
self.logger.info(f"Attached mutation tracking to graph: {getattr(graph, 'graph_id', 'unknown')}")
|
||||
|
||||
def record_mutation(
|
||||
self,
|
||||
operation: str,
|
||||
entity_id: str,
|
||||
payload: Dict[str, Any],
|
||||
version_label: Optional[str] = None
|
||||
) -> None:
|
||||
"""
|
||||
Callback triggered by the graph to record granular mutations.
|
||||
"""
|
||||
from .change_log import MutationRecord
|
||||
record = MutationRecord(
|
||||
timestamp=datetime.now().isoformat(),
|
||||
operation=operation,
|
||||
entity_id=entity_id,
|
||||
payload=payload,
|
||||
version_label=version_label
|
||||
)
|
||||
|
||||
mutation_dict = {
|
||||
"timestamp": record.timestamp,
|
||||
"operation": record.operation,
|
||||
"entity_id": record.entity_id,
|
||||
"payload": record.payload,
|
||||
"version_label": record.version_label
|
||||
}
|
||||
self.storage.save_mutation(mutation_dict)
|
||||
self.logger.debug(f"Recorded mutation: {operation} on {entity_id}")
|
||||
def tag_version(self, version_label: str, tag_name: str) -> None:
|
||||
"""
|
||||
Create a named tag (e.g., 'v1.0-approved') for a specific version.
|
||||
"""
|
||||
if not self.storage.exists(version_label):
|
||||
raise ValidationError(f"Cannot tag non-existent version: '{version_label}'")
|
||||
|
||||
self.storage.save_tag(tag_name, version_label)
|
||||
self.logger.info(f"Tagged version '{version_label}' as '{tag_name}'")
|
||||
|
||||
def list_tags(self) -> Dict[str, str]:
|
||||
"""
|
||||
Return a mapping of all tag names to their version labels.
|
||||
"""
|
||||
return self.storage.list_tags()
|
||||
|
||||
def diff(self, version_a: str, version_b: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Git-like alias for compare_versions. Computes added/removed/modified entities.
|
||||
"""
|
||||
return self.compare_versions(version_a, version_b)
|
||||
|
||||
def get_node_history(self, node_id: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Retrieve the complete chronological mutation history for a specific node.
|
||||
"""
|
||||
return self.storage.get_entity_history(node_id)
|
||||
|
||||
def restore_snapshot(self, graph: Any, target_version: str, require_confirmation: bool = True) -> bool:
|
||||
"""
|
||||
Restore the graph to a specific version snapshot.
|
||||
|
||||
Args:
|
||||
graph: The ContextGraph instance to restore.
|
||||
target_version: The version label to restore to.
|
||||
require_confirmation: If True, raises ProcessingError to prevent accidental data loss.
|
||||
"""
|
||||
if require_confirmation:
|
||||
raise ProcessingError(
|
||||
"Rollback protection active. Explicitly set require_confirmation=False "
|
||||
"to overwrite the current graph state."
|
||||
)
|
||||
|
||||
snapshot = self.storage.get(target_version)
|
||||
if not snapshot:
|
||||
raise ValidationError(f"Version '{target_version}' not found in storage.")
|
||||
|
||||
self.logger.warning(f"Restoring graph to version '{target_version}' - clearing current state.")
|
||||
|
||||
entities, relationships = self._extract_graph_collections(snapshot)
|
||||
graph_payload = {"nodes": entities, "edges": relationships}
|
||||
|
||||
previous_state = getattr(graph, "_suspend_mutation_callback", False)
|
||||
graph._suspend_mutation_callback = True
|
||||
try:
|
||||
graph.from_dict(graph_payload)
|
||||
finally:
|
||||
graph._suspend_mutation_callback = previous_state
|
||||
|
||||
self.logger.info(f"Successfully restored graph to version '{target_version}'.")
|
||||
return True
|
||||
|
||||
|
||||
|
||||
|
||||
class OntologyVersionManager(BaseVersionManager):
|
||||
"""
|
||||
Version management for ontologies with structural comparison.
|
||||
@@ -430,12 +551,10 @@ class OntologyVersionManager(BaseVersionManager):
|
||||
Returns:
|
||||
dict: Ontology version snapshot
|
||||
"""
|
||||
# Validate inputs
|
||||
change_entry = ChangeLogEntry(
|
||||
timestamp=datetime.now().isoformat(), author=author, description=description
|
||||
)
|
||||
|
||||
# Create snapshot
|
||||
snapshot = {
|
||||
"label": version_label,
|
||||
"timestamp": change_entry.timestamp,
|
||||
@@ -447,15 +566,9 @@ class OntologyVersionManager(BaseVersionManager):
|
||||
"metadata": options.get("metadata", {}),
|
||||
}
|
||||
|
||||
# Compute and add checksum
|
||||
snapshot["checksum"] = compute_checksum(snapshot)
|
||||
|
||||
# Store snapshot
|
||||
self.storage.save(snapshot)
|
||||
|
||||
# Also store in memory for compatibility
|
||||
self.versions[version_label] = snapshot
|
||||
|
||||
self.logger.info(f"Created ontology snapshot '{version_label}' by {author}")
|
||||
return snapshot
|
||||
|
||||
|
||||
@@ -295,6 +295,98 @@ class VersionManager:
|
||||
"axioms_removed": len(axioms_removed)
|
||||
}
|
||||
}
|
||||
|
||||
def diff_ontologies(self, base: Dict[str, Any], target: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Computes a structured diff between two ontology versions.
|
||||
"""
|
||||
def _compute_section_diff(base_list, target_list):
|
||||
|
||||
base_map = {}
|
||||
for item in base_list:
|
||||
if isinstance(item, dict):
|
||||
key = item.get("uri") or item.get("name")
|
||||
if key:
|
||||
base_map[key] = item
|
||||
elif isinstance(item, str):
|
||||
base_map[item] = {"uri": item}
|
||||
|
||||
target_map = {}
|
||||
for item in target_list:
|
||||
if isinstance(item, dict):
|
||||
key = item.get("uri") or item.get("name")
|
||||
if key:
|
||||
target_map[key] = item
|
||||
elif isinstance(item, str):
|
||||
target_map[item] = {"uri": item}
|
||||
|
||||
added, removed, changed = [], [], []
|
||||
|
||||
# Find Added and Changed
|
||||
for key, t_item in target_map.items():
|
||||
if key not in base_map:
|
||||
added.append(t_item)
|
||||
else:
|
||||
b_item = base_map[key]
|
||||
changes = {}
|
||||
|
||||
all_fields = set(b_item.keys()).union(t_item.keys())
|
||||
for field in all_fields:
|
||||
if field in ["uri", "name"]:
|
||||
continue
|
||||
|
||||
b_val = b_item.get(field)
|
||||
t_val = t_item.get(field)
|
||||
|
||||
# Deep equality check for lists
|
||||
if isinstance(b_val, list) and isinstance(t_val, list):
|
||||
if set(str(x) for x in b_val) != set(str(x) for x in t_val):
|
||||
changes[field] = {"old": b_val, "new": t_val}
|
||||
elif b_val != t_val:
|
||||
changes[field] = {"old": b_val, "new": t_val}
|
||||
|
||||
if changes:
|
||||
changed.append({
|
||||
"uri": t_item.get("uri") or key,
|
||||
"name": t_item.get("name") or key,
|
||||
"changes": changes
|
||||
})
|
||||
|
||||
# Find deleted
|
||||
for key, b_item in base_map.items():
|
||||
if key not in target_map:
|
||||
removed.append(b_item)
|
||||
|
||||
return added, removed, changed
|
||||
|
||||
|
||||
classes_added, classes_removed, classes_changed = _compute_section_diff(
|
||||
base.get("classes", []), target.get("classes", [])
|
||||
)
|
||||
props_added, props_removed, props_changed = _compute_section_diff(
|
||||
base.get("properties", []), target.get("properties", [])
|
||||
)
|
||||
inds_added, inds_removed, inds_changed = _compute_section_diff(
|
||||
base.get("individuals", []), target.get("individuals", [])
|
||||
)
|
||||
axioms_added, axioms_removed, axioms_changed = _compute_section_diff(
|
||||
base.get("axioms", []), target.get("axioms", [])
|
||||
)
|
||||
|
||||
return {
|
||||
"added_classes": classes_added,
|
||||
"removed_classes": classes_removed,
|
||||
"changed_classes": classes_changed,
|
||||
"added_properties": props_added,
|
||||
"removed_properties": props_removed,
|
||||
"changed_properties": props_changed,
|
||||
"added_individuals": inds_added,
|
||||
"removed_individuals": inds_removed,
|
||||
"changed_individuals": inds_changed,
|
||||
"added_axioms": axioms_added,
|
||||
"removed_axioms": axioms_removed,
|
||||
"changed_axioms": axioms_changed,
|
||||
}
|
||||
|
||||
def get_version(self, version: str) -> Optional[OntologyVersion]:
|
||||
"""Get version by version string."""
|
||||
|
||||
@@ -10,6 +10,7 @@ Key Features:
|
||||
- SQLite-based persistent storage implementation
|
||||
- Checksum computation and validation
|
||||
- Thread-safe operations
|
||||
- Tagging and Granular Mutation Logging (Audit Trail)
|
||||
|
||||
Main Classes:
|
||||
- VersionStorage: Abstract base class for storage backends
|
||||
@@ -17,7 +18,7 @@ Main Classes:
|
||||
- SQLiteVersionStorage: SQLite-based persistent storage
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.common.version_storage import SQLiteVersionStorage
|
||||
>>> from semantica.change_management.version_storage import SQLiteVersionStorage
|
||||
>>> storage = SQLiteVersionStorage("versions.db")
|
||||
>>> storage.save(snapshot)
|
||||
>>> versions = storage.list_all()
|
||||
@@ -39,6 +40,17 @@ from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
|
||||
def _snapshot_collections(snapshot: Dict[str, Any]) -> tuple[List[Any], List[Any]]:
|
||||
"""Normalize snapshot collections for compatibility-aware reads."""
|
||||
entities = snapshot.get("entities")
|
||||
if entities is None:
|
||||
entities = snapshot.get("nodes", [])
|
||||
|
||||
relationships = snapshot.get("relationships")
|
||||
if relationships is None:
|
||||
relationships = snapshot.get("edges", [])
|
||||
|
||||
return list(entities or []), list(relationships or [])
|
||||
|
||||
def create_graph_snapshot_record(
|
||||
version_id: str,
|
||||
@@ -82,65 +94,58 @@ class VersionStorage(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def save(self, snapshot: Dict[str, Any]) -> None:
|
||||
"""
|
||||
Save a version snapshot.
|
||||
|
||||
Args:
|
||||
snapshot: Version snapshot dictionary with metadata
|
||||
|
||||
Raises:
|
||||
ValidationError: If snapshot data is invalid
|
||||
ProcessingError: If save operation fails
|
||||
"""
|
||||
"""Save a version snapshot."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get(self, label: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Retrieve a version snapshot by label.
|
||||
|
||||
Args:
|
||||
label: Version label to retrieve
|
||||
|
||||
Returns:
|
||||
Snapshot dictionary or None if not found
|
||||
"""
|
||||
"""Retrieve a version snapshot by label."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def list_all(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
List all version snapshots.
|
||||
|
||||
Returns:
|
||||
List of snapshot metadata dictionaries
|
||||
"""
|
||||
"""List all version snapshots."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def exists(self, label: str) -> bool:
|
||||
"""
|
||||
Check if a version exists.
|
||||
|
||||
Args:
|
||||
label: Version label to check
|
||||
|
||||
Returns:
|
||||
True if version exists, False otherwise
|
||||
"""
|
||||
"""Check if a version exists."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def delete(self, label: str) -> bool:
|
||||
"""
|
||||
Delete a version snapshot.
|
||||
"""Delete a version snapshot."""
|
||||
pass
|
||||
|
||||
Args:
|
||||
label: Version label to delete
|
||||
|
||||
Returns:
|
||||
True if deleted, False if not found
|
||||
"""
|
||||
@abstractmethod
|
||||
def save_tag(self, tag_name: str, version_label: str) -> None:
|
||||
"""Save a named tag pointing to a specific version."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_tag(self, tag_name: str) -> Optional[str]:
|
||||
"""Retrieve the version label associated with a tag."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def list_tags(self) -> Dict[str, str]:
|
||||
"""List all tags as a mapping of tag_name -> version_label."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def save_mutation(self, mutation: Dict[str, Any]) -> None:
|
||||
"""Save a granular mutation record for the audit trail."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_entity_history(self, entity_id: str) -> List[Dict[str, Any]]:
|
||||
"""Retrieve the chronological mutation history for a specific entity."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def assign_version_to_unlabeled_mutations(self, version_label: str) -> None:
|
||||
"""Attach a version label to unlabeled mutations recorded since the last snapshot."""
|
||||
pass
|
||||
|
||||
|
||||
@@ -155,6 +160,8 @@ class InMemoryVersionStorage(VersionStorage):
|
||||
def __init__(self):
|
||||
"""Initialize in-memory storage."""
|
||||
self._storage: Dict[str, Dict[str, Any]] = {}
|
||||
self._tags: Dict[str, str] = {}
|
||||
self._mutations: List[Dict[str, Any]] = []
|
||||
self._lock = threading.RLock()
|
||||
self.logger = get_logger("in_memory_storage")
|
||||
|
||||
@@ -187,6 +194,7 @@ class InMemoryVersionStorage(VersionStorage):
|
||||
# Return metadata only (without full graph data)
|
||||
metadata_list = []
|
||||
for label, snapshot in self._storage.items():
|
||||
entities, relationships = _snapshot_collections(snapshot)
|
||||
metadata = {
|
||||
"label": snapshot.get("label"),
|
||||
"version_id": snapshot.get("version_id", snapshot.get("label")),
|
||||
@@ -195,8 +203,8 @@ class InMemoryVersionStorage(VersionStorage):
|
||||
"author": snapshot.get("author"),
|
||||
"description": snapshot.get("description"),
|
||||
"checksum": snapshot.get("checksum"),
|
||||
"entity_count": len(snapshot.get("entities", [])),
|
||||
"relationship_count": len(snapshot.get("relationships", [])),
|
||||
"entity_count": len(entities),
|
||||
"relationship_count": len(relationships),
|
||||
}
|
||||
metadata_list.append(metadata)
|
||||
return metadata_list
|
||||
@@ -215,6 +223,32 @@ class InMemoryVersionStorage(VersionStorage):
|
||||
return True
|
||||
return False
|
||||
|
||||
def save_tag(self, tag_name: str, version_label: str) -> None:
|
||||
with self._lock:
|
||||
self._tags[tag_name] = version_label
|
||||
|
||||
def get_tag(self, tag_name: str) -> Optional[str]:
|
||||
with self._lock:
|
||||
return self._tags.get(tag_name)
|
||||
|
||||
def list_tags(self) -> Dict[str, str]:
|
||||
with self._lock:
|
||||
return self._tags.copy()
|
||||
|
||||
def save_mutation(self, mutation: Dict[str, Any]) -> None:
|
||||
with self._lock:
|
||||
self._mutations.append(json.loads(json.dumps(mutation)))
|
||||
|
||||
def get_entity_history(self, entity_id: str) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
return [m for m in self._mutations if m.get("entity_id") == entity_id]
|
||||
|
||||
def assign_version_to_unlabeled_mutations(self, version_label: str) -> None:
|
||||
with self._lock:
|
||||
for mutation in self._mutations:
|
||||
if mutation.get("version_label") is None:
|
||||
mutation["version_label"] = version_label
|
||||
|
||||
|
||||
class SQLiteVersionStorage(VersionStorage):
|
||||
"""
|
||||
@@ -258,6 +292,23 @@ class SQLiteVersionStorage(VersionStorage):
|
||||
created_at TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS version_tags (
|
||||
tag_name TEXT PRIMARY KEY,
|
||||
version_label TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS mutation_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT,
|
||||
operation TEXT,
|
||||
entity_id TEXT,
|
||||
payload TEXT,
|
||||
version_label TEXT
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
self.logger.debug(f"Initialized SQLite database at {self.storage_path}")
|
||||
finally:
|
||||
@@ -342,6 +393,7 @@ class SQLiteVersionStorage(VersionStorage):
|
||||
metadata_list = []
|
||||
for row in cursor.fetchall():
|
||||
snapshot = json.loads(row[0])
|
||||
entities, relationships = _snapshot_collections(snapshot)
|
||||
|
||||
metadata = {
|
||||
"label": snapshot.get("label"),
|
||||
@@ -351,8 +403,8 @@ class SQLiteVersionStorage(VersionStorage):
|
||||
"author": snapshot.get("author"),
|
||||
"description": snapshot.get("description"),
|
||||
"checksum": snapshot.get("checksum"),
|
||||
"entity_count": len(snapshot.get("entities", [])),
|
||||
"relationship_count": len(snapshot.get("relationships", [])),
|
||||
"entity_count": len(entities),
|
||||
"relationship_count": len(relationships),
|
||||
}
|
||||
metadata_list.append(metadata)
|
||||
|
||||
@@ -396,6 +448,124 @@ class SQLiteVersionStorage(VersionStorage):
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def save_tag(self, tag_name: str, version_label: str) -> None:
|
||||
with self._lock:
|
||||
conn = sqlite3.connect(str(self.storage_path))
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO version_tags (tag_name, version_label)
|
||||
VALUES (?, ?)
|
||||
""",
|
||||
(tag_name, version_label)
|
||||
)
|
||||
conn.commit()
|
||||
except sqlite3.Error as e:
|
||||
raise ProcessingError(f"Failed to save tag: {e}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def get_tag(self, tag_name: str) -> Optional[str]:
|
||||
with self._lock:
|
||||
conn = sqlite3.connect(str(self.storage_path))
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT version_label FROM version_tags WHERE tag_name = ?", (tag_name,))
|
||||
row = cursor.fetchone()
|
||||
return row[0] if row else None
|
||||
except sqlite3.Error as e:
|
||||
raise ProcessingError(f"Failed to get tag: {e}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def list_tags(self) -> Dict[str, str]:
|
||||
with self._lock:
|
||||
conn = sqlite3.connect(str(self.storage_path))
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT tag_name, version_label FROM version_tags")
|
||||
return {row[0]: row[1] for row in cursor.fetchall()}
|
||||
except sqlite3.Error as e:
|
||||
raise ProcessingError(f"Failed to list tags: {e}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def save_mutation(self, mutation: Dict[str, Any]) -> None:
|
||||
with self._lock:
|
||||
conn = sqlite3.connect(str(self.storage_path))
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"""
|
||||
INSERT INTO mutation_log
|
||||
(timestamp, operation, entity_id, payload, version_label)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
mutation.get("timestamp"),
|
||||
mutation.get("operation"),
|
||||
mutation.get("entity_id"),
|
||||
json.dumps(mutation.get("payload", {})),
|
||||
mutation.get("version_label")
|
||||
)
|
||||
)
|
||||
conn.commit()
|
||||
except sqlite3.Error as e:
|
||||
raise ProcessingError(f"Failed to save mutation: {e}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def get_entity_history(self, entity_id: str) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
conn = sqlite3.connect(str(self.storage_path))
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT timestamp, operation, entity_id, payload, version_label
|
||||
FROM mutation_log
|
||||
WHERE entity_id = ?
|
||||
ORDER BY id ASC
|
||||
""",
|
||||
(entity_id,)
|
||||
)
|
||||
history = []
|
||||
for row in cursor.fetchall():
|
||||
history.append({
|
||||
"timestamp": row[0],
|
||||
"operation": row[1],
|
||||
"entity_id": row[2],
|
||||
"payload": json.loads(row[3]) if row[3] else {},
|
||||
"version_label": row[4]
|
||||
})
|
||||
return history
|
||||
except sqlite3.Error as e:
|
||||
raise ProcessingError(f"Failed to get entity history: {e}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def assign_version_to_unlabeled_mutations(self, version_label: str) -> None:
|
||||
with self._lock:
|
||||
conn = sqlite3.connect(str(self.storage_path))
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"""
|
||||
UPDATE mutation_log
|
||||
SET version_label = ?
|
||||
WHERE version_label IS NULL
|
||||
""",
|
||||
(version_label,),
|
||||
)
|
||||
conn.commit()
|
||||
except sqlite3.Error as e:
|
||||
raise ProcessingError(
|
||||
f"Failed to assign version label to mutations: {e}"
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def compute_checksum(data: Dict[str, Any]) -> str:
|
||||
"""
|
||||
|
||||
@@ -83,6 +83,7 @@ from .decision_recorder import DecisionRecorder
|
||||
from .decision_query import DecisionQuery
|
||||
from .causal_analyzer import CausalChainAnalyzer
|
||||
from .policy_engine import PolicyEngine
|
||||
from ..change_management import TemporalVersionManager
|
||||
|
||||
|
||||
class AgentContext:
|
||||
@@ -166,6 +167,8 @@ class AgentContext:
|
||||
|
||||
self.vector_store = vector_store
|
||||
self.knowledge_graph = knowledge_graph
|
||||
self._checkpoints: Dict[str, Dict[str, Any]] = {}
|
||||
self._temporal_version_manager = kwargs.get("temporal_version_manager")
|
||||
|
||||
# Store advanced feature flags
|
||||
self.config = {
|
||||
@@ -1559,7 +1562,9 @@ class AgentContext:
|
||||
confidence: float,
|
||||
entities: Optional[List[str]] = None,
|
||||
cross_system_context: Optional[Dict[str, Any]] = None,
|
||||
decision_maker: Optional[str] = "ai_agent"
|
||||
decision_maker: Optional[str] = "ai_agent",
|
||||
valid_from: Optional[Union[str, datetime]] = None,
|
||||
valid_until: Optional[Union[str, datetime]] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Record decision (wrapper for DecisionRecorder).
|
||||
@@ -1594,7 +1599,9 @@ class AgentContext:
|
||||
outcome=outcome,
|
||||
confidence=confidence,
|
||||
timestamp=datetime.now(),
|
||||
decision_maker=decision_maker or "ai_agent"
|
||||
decision_maker=decision_maker or "ai_agent",
|
||||
valid_from=valid_from,
|
||||
valid_until=valid_until,
|
||||
)
|
||||
|
||||
entities = entities or []
|
||||
@@ -1624,6 +1631,8 @@ class AgentContext:
|
||||
confidence=confidence,
|
||||
entities=entities,
|
||||
decision_maker=decision_maker,
|
||||
valid_from=valid_from,
|
||||
valid_until=valid_until,
|
||||
metadata={"cross_system_context": cross_system_context} if cross_system_context else None
|
||||
)
|
||||
|
||||
@@ -1636,7 +1645,9 @@ class AgentContext:
|
||||
limit: int = 10,
|
||||
use_hybrid_search: bool = True,
|
||||
max_hops: int = 3,
|
||||
include_context: bool = True
|
||||
include_context: bool = True,
|
||||
include_superseded: bool = False,
|
||||
as_of: Optional[Union[str, datetime]] = None,
|
||||
) -> List[Decision]:
|
||||
"""
|
||||
Find similar decisions with user controls.
|
||||
@@ -1665,7 +1676,9 @@ class AgentContext:
|
||||
scenario=scenario,
|
||||
category=category,
|
||||
limit=limit,
|
||||
use_semantic_search=use_hybrid_search
|
||||
use_semantic_search=use_hybrid_search,
|
||||
include_superseded=include_superseded,
|
||||
as_of=as_of,
|
||||
)
|
||||
# Convert to Decision objects if needed
|
||||
from .decision_models import Decision
|
||||
@@ -1684,6 +1697,8 @@ class AgentContext:
|
||||
confidence=decision_data["confidence"],
|
||||
timestamp=datetime.fromtimestamp(decision_data["timestamp"]),
|
||||
decision_maker=decision_data.get("decision_maker"),
|
||||
valid_from=decision_data.get("valid_from"),
|
||||
valid_until=decision_data.get("valid_until"),
|
||||
metadata=metadata,
|
||||
)
|
||||
decisions.append(decision)
|
||||
@@ -1780,6 +1795,93 @@ class AgentContext:
|
||||
|
||||
return results[:limit]
|
||||
|
||||
def checkpoint(self, label: str) -> Dict[str, Any]:
|
||||
"""Capture the current context state under a label."""
|
||||
snapshot = self._capture_checkpoint_state()
|
||||
self._checkpoints[label] = snapshot
|
||||
return snapshot
|
||||
|
||||
def diff_checkpoints(self, label1: str, label2: str) -> Dict[str, Any]:
|
||||
"""Return a structured diff between two named checkpoints."""
|
||||
missing = [label for label in (label1, label2) if label not in self._checkpoints]
|
||||
if missing:
|
||||
raise KeyError(f"Unknown checkpoint label(s): {', '.join(missing)}")
|
||||
|
||||
first = self._checkpoints[label1]
|
||||
second = self._checkpoints[label2]
|
||||
|
||||
first_decisions = {decision.get("id"): decision for decision in first.get("decisions", [])}
|
||||
second_decisions = {decision.get("id"): decision for decision in second.get("decisions", [])}
|
||||
first_relationships = {self._relationship_key(rel): rel for rel in first.get("relationships", [])}
|
||||
second_relationships = {self._relationship_key(rel): rel for rel in second.get("relationships", [])}
|
||||
|
||||
return {
|
||||
"decisions_added": [second_decisions[key] for key in sorted(set(second_decisions) - set(first_decisions))],
|
||||
"decisions_removed": [first_decisions[key] for key in sorted(set(first_decisions) - set(second_decisions))],
|
||||
"relationships_added": [second_relationships[key] for key in sorted(set(second_relationships) - set(first_relationships))],
|
||||
"relationships_removed": [first_relationships[key] for key in sorted(set(first_relationships) - set(second_relationships))],
|
||||
}
|
||||
|
||||
def flush_checkpoint(self, label: str) -> Dict[str, Any]:
|
||||
"""Persist a named checkpoint via ``TemporalVersionManager``."""
|
||||
if label not in self._checkpoints:
|
||||
raise KeyError(f"Unknown checkpoint label: {label}")
|
||||
|
||||
if self._temporal_version_manager is None:
|
||||
try:
|
||||
self._temporal_version_manager = TemporalVersionManager()
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
"flush_checkpoint requires a TemporalVersionManager. "
|
||||
"Pass one via temporal_version_manager= at construction time."
|
||||
) from exc
|
||||
|
||||
return self._temporal_version_manager.create_snapshot(
|
||||
self._checkpoints[label],
|
||||
version_label=label,
|
||||
author=str(self.config.get("checkpoint_author", "agent_context@local.test")),
|
||||
description=f"Checkpoint '{label}'",
|
||||
)
|
||||
|
||||
def _capture_checkpoint_state(self) -> Dict[str, Any]:
|
||||
"""Capture a serializable snapshot of the current graph state."""
|
||||
if self.knowledge_graph and hasattr(self.knowledge_graph, "state_at"):
|
||||
return self.knowledge_graph.state_at(datetime.utcnow())
|
||||
if self.knowledge_graph and hasattr(self.knowledge_graph, "to_dict"):
|
||||
graph_dict = self.knowledge_graph.to_dict()
|
||||
return {
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"nodes": graph_dict.get("nodes", []),
|
||||
"edges": graph_dict.get("edges", []),
|
||||
"entities": graph_dict.get("nodes", []),
|
||||
"relationships": graph_dict.get("edges", []),
|
||||
"decisions": [
|
||||
{
|
||||
"id": node.get("id"),
|
||||
"category": (node.get("properties", {}) or {}).get("category", ""),
|
||||
"scenario": node.get("content", ""),
|
||||
}
|
||||
for node in graph_dict.get("nodes", [])
|
||||
if str(node.get("type", "")).lower() == "decision"
|
||||
],
|
||||
}
|
||||
return {
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"nodes": [],
|
||||
"edges": [],
|
||||
"entities": [],
|
||||
"relationships": [],
|
||||
"decisions": [],
|
||||
}
|
||||
|
||||
def _relationship_key(self, relationship: Dict[str, Any]) -> tuple:
|
||||
"""Build a stable comparison key for checkpoint relationship diffs."""
|
||||
return (
|
||||
relationship.get("source_id", relationship.get("source")),
|
||||
relationship.get("target_id", relationship.get("target")),
|
||||
relationship.get("type"),
|
||||
)
|
||||
|
||||
def get_causal_chain(
|
||||
self,
|
||||
decision_id: str,
|
||||
@@ -2073,7 +2175,7 @@ class AgentContext:
|
||||
|
||||
def find_similar_entities(
|
||||
self, entity_id: str, similarity_type: str = "content", top_k: int = 10
|
||||
) -> List[Tuple[str, float]]:
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Find similar entities using advanced similarity measures.
|
||||
|
||||
@@ -2083,7 +2185,7 @@ class AgentContext:
|
||||
top_k: Number of similar entities to return
|
||||
|
||||
Returns:
|
||||
List of (entity_id, similarity_score) tuples
|
||||
List of dicts with entity ID, content, type, and similarity score
|
||||
"""
|
||||
if not self._graph_builder:
|
||||
return []
|
||||
|
||||
@@ -59,7 +59,7 @@ Production Use Cases:
|
||||
- Policy: Trace policy decision consequences
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
from collections import deque
|
||||
|
||||
@@ -151,6 +151,50 @@ class CausalChainAnalyzer:
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to get causal chain: {e}")
|
||||
raise
|
||||
|
||||
def trace_at_time(
|
||||
self,
|
||||
event_id: str,
|
||||
at_time: Any,
|
||||
direction: str = "upstream",
|
||||
max_depth: int = 10,
|
||||
) -> List[Decision]:
|
||||
"""Trace a causal chain using only facts recorded up to ``at_time``."""
|
||||
cutoff = self._normalize_at_time(at_time)
|
||||
|
||||
if direction not in ["upstream", "downstream"]:
|
||||
raise ValueError("Direction must be 'upstream' or 'downstream'")
|
||||
if not (1 <= max_depth <= 100):
|
||||
raise ValueError("max_depth must be between 1 and 100")
|
||||
|
||||
if hasattr(self.graph_store, "nodes") and hasattr(self.graph_store, "edges"):
|
||||
return self._trace_at_time_from_context_graph(event_id, cutoff, direction, max_depth)
|
||||
|
||||
if hasattr(self.graph_store, "execute_query"):
|
||||
rel_pattern = "<-[rel:CAUSED|INFLUENCED|PRECEDENT_FOR]-" if direction == "upstream" else "-[rel:CAUSED|INFLUENCED|PRECEDENT_FOR]->"
|
||||
query = f"""
|
||||
MATCH (start:Decision {{decision_id: $decision_id}})
|
||||
MATCH path = (start){rel_pattern}{{1,{max_depth}}}(end:Decision)
|
||||
WHERE ALL(rel IN relationships(path) WHERE rel.recorded_at IS NOT NULL AND rel.recorded_at <= $at_time)
|
||||
RETURN DISTINCT end, length(path) as distance
|
||||
ORDER BY distance, end.timestamp
|
||||
"""
|
||||
results = self.graph_store.execute_query(
|
||||
query,
|
||||
{"decision_id": event_id, "at_time": cutoff.strftime("%Y-%m-%dT%H:%M:%S") + "Z"},
|
||||
)
|
||||
records = self._extract_records(results)
|
||||
decisions: List[Decision] = []
|
||||
for record in records:
|
||||
decision_data = record.get("end") if isinstance(record, dict) else None
|
||||
if not isinstance(decision_data, dict):
|
||||
decision_data = record if isinstance(record, dict) else {}
|
||||
decision = self._dict_to_decision(decision_data)
|
||||
decision.metadata["causal_distance"] = record.get("distance", 0)
|
||||
decisions.append(decision)
|
||||
return decisions
|
||||
|
||||
return []
|
||||
|
||||
def get_influenced_decisions(
|
||||
self,
|
||||
@@ -258,7 +302,8 @@ class CausalChainAnalyzer:
|
||||
WHERE ALL(i IN range(0, length(path)-2) |
|
||||
path[i].decision_id <> path[i+1].decision_id)
|
||||
RETURN d1.decision_id as decision_id,
|
||||
[node in nodes(path) | node.decision_id] as loop_path,
|
||||
d1.scenario as decision_scenario,
|
||||
[node in nodes(path) | {{decision_id: node.decision_id, scenario: node.scenario, category: node.category}}] as loop_path,
|
||||
length(path) as loop_length
|
||||
ORDER BY loop_length
|
||||
"""
|
||||
@@ -557,3 +602,78 @@ class CausalChainAnalyzer:
|
||||
if isinstance(results, list):
|
||||
return results
|
||||
return []
|
||||
|
||||
def _normalize_at_time(self, value: Any) -> datetime:
|
||||
"""Normalize supported ``at_time`` inputs."""
|
||||
if isinstance(value, datetime):
|
||||
return value.replace(tzinfo=None) if value.tzinfo is None else value.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
if isinstance(value, str):
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
return parsed.replace(tzinfo=None) if parsed.tzinfo is None else parsed.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
raise ValueError("at_time must be a datetime or ISO datetime string")
|
||||
|
||||
def _trace_at_time_from_context_graph(
|
||||
self,
|
||||
event_id: str,
|
||||
cutoff: datetime,
|
||||
direction: str,
|
||||
max_depth: int,
|
||||
) -> List[Decision]:
|
||||
"""Trace transaction-time causal chains against an in-memory ContextGraph."""
|
||||
eligible_edges = []
|
||||
for edge in getattr(self.graph_store, "edges", []):
|
||||
if edge.edge_type not in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]:
|
||||
continue
|
||||
recorded_at = (edge.metadata or {}).get("recorded_at")
|
||||
if recorded_at is None:
|
||||
continue
|
||||
try:
|
||||
edge_recorded_at = self._normalize_at_time(recorded_at)
|
||||
except ValueError:
|
||||
continue
|
||||
if edge_recorded_at <= cutoff:
|
||||
eligible_edges.append(edge)
|
||||
|
||||
if not eligible_edges:
|
||||
return []
|
||||
|
||||
visited = set()
|
||||
queue = deque([(event_id, 0)])
|
||||
decisions: List[Decision] = []
|
||||
|
||||
while queue:
|
||||
current_id, depth = queue.popleft()
|
||||
if current_id in visited or depth > max_depth:
|
||||
continue
|
||||
visited.add(current_id)
|
||||
|
||||
if current_id != event_id:
|
||||
node = getattr(self.graph_store, "nodes", {}).get(current_id)
|
||||
if node and getattr(node, "node_type", "").lower() == "decision":
|
||||
decision = self._dict_to_decision(
|
||||
{
|
||||
"id": node.node_id,
|
||||
"category": node.properties.get("category", ""),
|
||||
"scenario": node.properties.get("scenario", node.content),
|
||||
"reasoning": node.properties.get("reasoning", ""),
|
||||
"outcome": node.properties.get("outcome", ""),
|
||||
"confidence": node.properties.get("confidence", 0.0),
|
||||
"timestamp": node.properties.get("timestamp"),
|
||||
"decision_maker": node.properties.get("decision_maker", ""),
|
||||
"metadata": {},
|
||||
}
|
||||
)
|
||||
decision.metadata["causal_distance"] = depth
|
||||
decisions.append(decision)
|
||||
|
||||
for edge in eligible_edges:
|
||||
if direction == "upstream" and edge.target_id == current_id and depth < max_depth:
|
||||
queue.append((edge.source_id, depth + 1))
|
||||
if direction == "downstream" and edge.source_id == current_id and depth < max_depth:
|
||||
queue.append((edge.target_id, depth + 1))
|
||||
|
||||
if direction == "upstream":
|
||||
decisions.sort(key=lambda d: d.metadata.get("causal_distance", 0), reverse=True)
|
||||
else:
|
||||
decisions.sort(key=lambda d: d.metadata.get("causal_distance", 0))
|
||||
return decisions
|
||||
|
||||
+882
-265
File diff suppressed because it is too large
Load Diff
@@ -2023,7 +2023,11 @@ Answer:"""
|
||||
elif hasattr(self.knowledge_graph, "get_neighbor_ids"):
|
||||
return list(self.knowledge_graph.get_neighbor_ids(node))
|
||||
elif hasattr(self.knowledge_graph, "neighbors"):
|
||||
return list(self.knowledge_graph.neighbors(node))
|
||||
return [
|
||||
n.get("id") if isinstance(n, dict) else n
|
||||
for n in self.knowledge_graph.neighbors(node)
|
||||
if n
|
||||
]
|
||||
return []
|
||||
|
||||
visited: set = {entity_name}
|
||||
@@ -2111,7 +2115,11 @@ Answer:"""
|
||||
# Simplified centrality calculation
|
||||
if hasattr(self.knowledge_graph, 'get_neighbors'):
|
||||
if hasattr(self.knowledge_graph, "neighbors"):
|
||||
neighbor_ids = list(self.knowledge_graph.neighbors(entity_name))
|
||||
neighbor_ids = [
|
||||
n.get("id") if isinstance(n, dict) else n
|
||||
for n in self.knowledge_graph.neighbors(entity_name)
|
||||
if n
|
||||
]
|
||||
elif hasattr(self.knowledge_graph, "get_neighbor_ids"):
|
||||
neighbor_ids = self.knowledge_graph.get_neighbor_ids(entity_name)
|
||||
else:
|
||||
@@ -2162,8 +2170,13 @@ Answer:"""
|
||||
if hasattr(self.knowledge_graph, 'get_nodes_by_label'):
|
||||
policy_nodes = self.knowledge_graph.get_nodes_by_label("Policy")
|
||||
for policy in policy_nodes[:5]: # Limit results
|
||||
policy_name = (
|
||||
policy.get("content")
|
||||
or policy.get("metadata", {}).get("name", "")
|
||||
or policy.get("id", "")
|
||||
) if isinstance(policy, dict) else policy
|
||||
policies.append({
|
||||
"name": policy,
|
||||
"name": policy_name,
|
||||
"type": "policy",
|
||||
"source": "policy_search",
|
||||
"related_category": category
|
||||
@@ -2482,17 +2495,20 @@ Answer:"""
|
||||
decision_nodes = self.knowledge_graph.get_nodes_by_label("Decision")
|
||||
|
||||
for node_data in decision_nodes[:limit]:
|
||||
metadata = {}
|
||||
if isinstance(node_data, dict):
|
||||
metadata = node_data.get("metadata") or node_data.get("properties") or {}
|
||||
# Convert to Decision object
|
||||
decision = Decision(
|
||||
decision_id=node_data.get("id", ""),
|
||||
category=node_data.get("properties", {}).get("category", ""),
|
||||
scenario=node_data.get("content", ""),
|
||||
reasoning=node_data.get("properties", {}).get("reasoning", ""),
|
||||
outcome=node_data.get("properties", {}).get("outcome", ""),
|
||||
confidence=node_data.get("properties", {}).get("confidence", 0.0),
|
||||
decision_id=node_data.get("id", "") if isinstance(node_data, dict) else "",
|
||||
category=metadata.get("category", ""),
|
||||
scenario=node_data.get("content", "") if isinstance(node_data, dict) else "",
|
||||
reasoning=metadata.get("reasoning", ""),
|
||||
outcome=metadata.get("outcome", ""),
|
||||
confidence=metadata.get("confidence", 0.0),
|
||||
timestamp=datetime.now(),
|
||||
decision_maker=node_data.get("properties", {}).get("decision_maker", ""),
|
||||
metadata=node_data.get("properties", {})
|
||||
decision_maker=metadata.get("decision_maker", ""),
|
||||
metadata=metadata
|
||||
)
|
||||
|
||||
# Filter by category if specified
|
||||
|
||||
@@ -623,13 +623,22 @@ def analyze_decision_impact(
|
||||
# Get root causes
|
||||
root_causes = analyzer.find_root_causes(decision_id, max_depth=5)
|
||||
|
||||
def _decision_dict(d) -> Dict[str, Any]:
|
||||
return {
|
||||
"decision_id": d.decision_id,
|
||||
"scenario": d.scenario,
|
||||
"category": d.category,
|
||||
"outcome": d.outcome,
|
||||
"confidence": d.confidence,
|
||||
}
|
||||
|
||||
return {
|
||||
"decision_id": decision_id,
|
||||
"impact_score": impact_score,
|
||||
"influenced_decisions": len(influenced),
|
||||
"root_causes": len(root_causes),
|
||||
"influenced_decision_ids": [d.decision_id for d in influenced],
|
||||
"root_cause_ids": [d.decision_id for d in root_causes],
|
||||
"influenced_decisions": [_decision_dict(d) for d in influenced],
|
||||
"root_causes": [_decision_dict(d) for d in root_causes],
|
||||
"total_influenced": len(influenced),
|
||||
"total_root_causes": len(root_causes),
|
||||
"analysis_timestamp": datetime.now().isoformat()
|
||||
}
|
||||
|
||||
|
||||
@@ -97,6 +97,8 @@ class Decision:
|
||||
decision_maker: str
|
||||
reasoning_embedding: Optional[List[float]] = None
|
||||
node2vec_embedding: Optional[List[float]] = None
|
||||
valid_from: Optional[str] = None
|
||||
valid_until: Optional[str] = None
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self, auto_generate_id: bool = True):
|
||||
@@ -121,6 +123,8 @@ class Decision:
|
||||
"decision_maker": self.decision_maker,
|
||||
"reasoning_embedding": self.reasoning_embedding,
|
||||
"node2vec_embedding": self.node2vec_embedding,
|
||||
"valid_from": self.valid_from,
|
||||
"valid_until": self.valid_until,
|
||||
"metadata": self.metadata
|
||||
}
|
||||
|
||||
|
||||
@@ -83,9 +83,10 @@ import numpy as np
|
||||
from ..embeddings import EmbeddingGenerator
|
||||
from ..graph_store import GraphStore
|
||||
from ..utils.logging import get_logger
|
||||
from .context_graph import ContextGraph
|
||||
from .decision_models import Decision, PolicyException
|
||||
|
||||
# Optional imports for advanced features
|
||||
|
||||
try:
|
||||
from ..kg import (
|
||||
CentralityCalculator, CommunityDetector, PathFinder,
|
||||
@@ -238,12 +239,10 @@ class DecisionQuery:
|
||||
"""Find precedents using vector store hybrid search."""
|
||||
hybrid_search = self.vector_components["hybrid_search"]
|
||||
|
||||
# Build filters
|
||||
filters = {}
|
||||
if category:
|
||||
filters["category"] = category
|
||||
|
||||
# Search vector store
|
||||
results = hybrid_search.search(
|
||||
query=scenario,
|
||||
filters=filters,
|
||||
@@ -328,7 +327,46 @@ class DecisionQuery:
|
||||
if self.embedding_generator:
|
||||
query_embedding = self.embedding_generator.generate(scenario)
|
||||
|
||||
# Build base query
|
||||
# Native ContextGraph flow
|
||||
if type(self.graph_store) is ContextGraph:
|
||||
nodes = self.graph_store.find_nodes(node_type="Decision")
|
||||
decisions = []
|
||||
for node in nodes:
|
||||
metadata = node.get("metadata", {}).get("properties", node.get("metadata", {}))
|
||||
if category and metadata.get("category") != category:
|
||||
continue
|
||||
|
||||
# Format to conform to _dict_to_decision
|
||||
core_fields = {
|
||||
"category", "scenario", "reasoning", "outcome", "confidence",
|
||||
"timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding"
|
||||
}
|
||||
custom_metadata = {k: v for k, v in metadata.items() if k not in core_fields}
|
||||
decision_data = {"id": node.get("id"), "metadata": custom_metadata}
|
||||
decision_data.update({k: v for k, v in metadata.items() if k in core_fields})
|
||||
|
||||
try:
|
||||
decision = self._dict_to_decision(decision_data)
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
if query_embedding and decision.reasoning_embedding:
|
||||
similarity = self._cosine_similarity(
|
||||
query_embedding, decision.reasoning_embedding
|
||||
)
|
||||
decision.metadata["similarity_score"] = similarity
|
||||
decisions.append(decision)
|
||||
|
||||
if query_embedding:
|
||||
decisions.sort(
|
||||
key=lambda d: d.metadata.get("similarity_score", 0),
|
||||
reverse=True
|
||||
)
|
||||
|
||||
self.logger.info(f"Found {min(len(decisions), limit)} precedents for scenario")
|
||||
return decisions[:limit]
|
||||
|
||||
# Build base Cypher query
|
||||
query_parts = ["MATCH (d:Decision)"]
|
||||
where_conditions = []
|
||||
params = {"limit": limit}
|
||||
@@ -385,6 +423,29 @@ class DecisionQuery:
|
||||
List of decisions in the category
|
||||
"""
|
||||
try:
|
||||
if type(self.graph_store) is ContextGraph:
|
||||
nodes = self.graph_store.find_nodes("Decision")
|
||||
decisions = []
|
||||
for node in nodes:
|
||||
metadata = node.get("metadata", {}).get("properties", node.get("metadata", {}))
|
||||
if metadata.get("category") == category:
|
||||
core_fields = {
|
||||
"category", "scenario", "reasoning", "outcome", "confidence",
|
||||
"timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding"
|
||||
}
|
||||
custom_metadata = {k: v for k, v in metadata.items() if k not in core_fields}
|
||||
decision_data = {"id": node.get("id"), "metadata": custom_metadata}
|
||||
decision_data.update({k: v for k, v in metadata.items() if k in core_fields})
|
||||
|
||||
try:
|
||||
decisions.append(self._dict_to_decision(decision_data))
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
decisions.sort(key=lambda d: d.timestamp, reverse=True)
|
||||
self.logger.info(f"Found {min(len(decisions), limit)} decisions in category {category}")
|
||||
return decisions[:limit]
|
||||
|
||||
query = """
|
||||
MATCH (d:Decision {category: $category})
|
||||
RETURN d
|
||||
@@ -423,6 +484,35 @@ class DecisionQuery:
|
||||
List of decisions about the entity
|
||||
"""
|
||||
try:
|
||||
if type(self.graph_store) is ContextGraph:
|
||||
edges = self.graph_store.find_edges(edge_type="ABOUT")
|
||||
decision_ids = {
|
||||
e["source"] for e in edges
|
||||
if e["target"] == entity_id
|
||||
}
|
||||
|
||||
decisions = []
|
||||
for d_id in decision_ids:
|
||||
node = self.graph_store.find_node(d_id)
|
||||
if node and node.get("type") == "Decision":
|
||||
metadata = node.get("metadata", {}).get("properties", node.get("metadata", {}))
|
||||
core_fields = {
|
||||
"category", "scenario", "reasoning", "outcome", "confidence",
|
||||
"timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding"
|
||||
}
|
||||
custom_metadata = {k: v for k, v in metadata.items() if k not in core_fields}
|
||||
decision_data = {"id": node.get("id"), "metadata": custom_metadata}
|
||||
decision_data.update({k: v for k, v in metadata.items() if k in core_fields})
|
||||
|
||||
try:
|
||||
decisions.append(self._dict_to_decision(decision_data))
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
decisions.sort(key=lambda d: d.timestamp, reverse=True)
|
||||
self.logger.info(f"Found {min(len(decisions), limit)} decisions about entity {entity_id}")
|
||||
return decisions[:limit]
|
||||
|
||||
query = """
|
||||
MATCH (d:Decision)-[:ABOUT]->(e)
|
||||
WHERE e.id = $entity_id OR e.entity_id = $entity_id
|
||||
@@ -470,6 +560,49 @@ class DecisionQuery:
|
||||
if end <= start:
|
||||
raise ValueError("End time must be after start time")
|
||||
try:
|
||||
if type(self.graph_store) is ContextGraph:
|
||||
nodes = self.graph_store.find_nodes("Decision")
|
||||
decisions = []
|
||||
for node in nodes:
|
||||
metadata = node.get("metadata", {}).get("properties", node.get("metadata", {}))
|
||||
ts_val = metadata.get("timestamp")
|
||||
if not ts_val:
|
||||
continue
|
||||
|
||||
if isinstance(ts_val, str):
|
||||
try:
|
||||
dt = datetime.fromisoformat(ts_val)
|
||||
except ValueError:
|
||||
continue
|
||||
elif isinstance(ts_val, datetime):
|
||||
dt = ts_val
|
||||
else:
|
||||
continue
|
||||
|
||||
# Ensure dt is naive or both are aware
|
||||
if start.tzinfo is not None and dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=start.tzinfo)
|
||||
elif start.tzinfo is None and dt.tzinfo is not None:
|
||||
dt = dt.replace(tzinfo=None)
|
||||
|
||||
if start <= dt <= end:
|
||||
core_fields = {
|
||||
"category", "scenario", "reasoning", "outcome", "confidence",
|
||||
"timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding"
|
||||
}
|
||||
custom_metadata = {k: v for k, v in metadata.items() if k not in core_fields}
|
||||
decision_data = {"id": node.get("id"), "metadata": custom_metadata}
|
||||
decision_data.update({k: v for k, v in metadata.items() if k in core_fields})
|
||||
|
||||
try:
|
||||
decisions.append(self._dict_to_decision(decision_data))
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
decisions.sort(key=lambda d: d.timestamp, reverse=True)
|
||||
self.logger.info(f"Found {min(len(decisions), limit)} decisions in time range")
|
||||
return decisions[:limit]
|
||||
|
||||
query = """
|
||||
MATCH (d:Decision)
|
||||
WHERE d.timestamp >= $start AND d.timestamp <= $end
|
||||
@@ -518,6 +651,57 @@ class DecisionQuery:
|
||||
if not (1 <= max_hops <= 10):
|
||||
raise ValueError("max_hops must be between 1 and 10")
|
||||
try:
|
||||
if type(self.graph_store) is ContextGraph:
|
||||
from collections import deque
|
||||
# Use BFS to find all Decisions within max_hops
|
||||
queue = deque([(start_entity, 0)])
|
||||
visited = {start_entity}
|
||||
decisions = []
|
||||
|
||||
while queue:
|
||||
current_node_id, current_hop = queue.popleft()
|
||||
|
||||
if current_hop > 0:
|
||||
node = self.graph_store.find_node(current_node_id)
|
||||
if node and node.get("type") == "Decision":
|
||||
metadata = node.get("metadata", {}).get("properties", node.get("metadata", {}))
|
||||
core_fields = {
|
||||
"category", "scenario", "reasoning", "outcome", "confidence",
|
||||
"timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding"
|
||||
}
|
||||
custom_metadata = {k: v for k, v in metadata.items() if k not in core_fields}
|
||||
decision_data = {"id": node.get("id"), "metadata": custom_metadata}
|
||||
decision_data.update({k: v for k, v in metadata.items() if k in core_fields})
|
||||
|
||||
try:
|
||||
decision = self._dict_to_decision(decision_data)
|
||||
decision.metadata["hop_count"] = current_hop
|
||||
decisions.append(decision)
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
if current_hop < max_hops:
|
||||
# Undirected traversal: Get both incoming and outgoing neighbors
|
||||
neighbors = []
|
||||
# Outgoing edges
|
||||
for edge in self.graph_store._adjacency.get(current_node_id, []):
|
||||
neighbors.append(edge.target_id)
|
||||
# Incoming edges
|
||||
for src_id, edges in self.graph_store._adjacency.items():
|
||||
for edge in edges:
|
||||
if edge.target_id == current_node_id:
|
||||
neighbors.append(src_id)
|
||||
|
||||
for n_id in neighbors:
|
||||
if n_id not in visited:
|
||||
visited.add(n_id)
|
||||
queue.append((n_id, current_hop + 1))
|
||||
|
||||
# Sort by hop_count then timestamp desc
|
||||
decisions.sort(key=lambda d: (d.metadata.get("hop_count", 0), -d.timestamp.timestamp()))
|
||||
self.logger.info(f"Found {len(decisions)} decisions via multi-hop reasoning")
|
||||
return decisions
|
||||
|
||||
# Build multi-hop query
|
||||
query = f"""
|
||||
MATCH (start {{id: $start_entity}})
|
||||
@@ -567,26 +751,84 @@ class DecisionQuery:
|
||||
List of path information
|
||||
"""
|
||||
try:
|
||||
if type(self.graph_store) is ContextGraph:
|
||||
from collections import deque
|
||||
|
||||
# Check root node
|
||||
root = self.graph_store.find_node(decision_id)
|
||||
if not root or root.get("type") != "Decision":
|
||||
return []
|
||||
|
||||
paths = []
|
||||
# Queue stores (current_node_id, current_path_nodes, current_path_rels)
|
||||
root_meta = root.get("metadata", {}).get("properties", root.get("metadata", {}))
|
||||
root_info = {"decision_id": root.get("id"), "scenario": root_meta.get("scenario", ""), "category": root_meta.get("category", "")}
|
||||
queue = deque([(decision_id, [root_info], [])])
|
||||
|
||||
# Simple BFS path tracing matching cypher `MATCH path = (d)-[:REL*]-(related)`
|
||||
# We limit depth to prevent infinite loops in cyclic graphs
|
||||
max_depth = 5
|
||||
|
||||
# Fetch all relevant edges once before BFS to avoid O(nodes * edges) fetches
|
||||
all_edges = []
|
||||
for rel_type in relationship_types:
|
||||
all_edges.extend(self.graph_store.find_edges(edge_type=rel_type))
|
||||
|
||||
while queue:
|
||||
curr_id, path_nodes, path_rels = queue.popleft()
|
||||
|
||||
if len(path_rels) > 0:
|
||||
paths.append({
|
||||
"path_length": len(path_rels),
|
||||
"nodes": path_nodes,
|
||||
"relationships": path_rels
|
||||
})
|
||||
|
||||
if len(path_rels) >= max_depth:
|
||||
continue
|
||||
|
||||
for edge in all_edges:
|
||||
if edge["source"] == curr_id or edge["target"] == curr_id:
|
||||
next_id = edge["target"] if edge["source"] == curr_id else edge["source"]
|
||||
|
||||
# Avoid simple cycles in individual paths
|
||||
if any(n["decision_id"] == next_id for n in path_nodes):
|
||||
continue
|
||||
|
||||
next_node = self.graph_store.find_node(next_id)
|
||||
if next_node:
|
||||
next_meta = next_node.get("metadata", {}).get("properties", next_node.get("metadata", {}))
|
||||
next_info = {"decision_id": next_node.get("id"), "scenario": next_meta.get("scenario", ""), "category": next_meta.get("category", "")}
|
||||
rel_info = {"from": edge["source"], "to": edge["target"], "type": edge["type"]}
|
||||
queue.append((next_id, path_nodes + [next_info], path_rels + [rel_info]))
|
||||
|
||||
paths.sort(key=lambda x: x["path_length"])
|
||||
self.logger.info(f"Traced {len(paths)} paths from decision {decision_id}")
|
||||
return paths
|
||||
|
||||
# Build relationship filter
|
||||
rel_filter = "|".join(relationship_types)
|
||||
|
||||
query = f"""
|
||||
MATCH (d:Decision {{decision_id: $decision_id}})
|
||||
MATCH path = (d)-[:{rel_filter}*]-(related)
|
||||
RETURN path, length(path) as path_length
|
||||
RETURN [node in nodes(path) | {{decision_id: node.decision_id, scenario: node.scenario, category: node.category}}] as path_nodes,
|
||||
[r in relationships(path) | {{from: startNode(r).decision_id, to: endNode(r).decision_id, type: type(r)}}] as path_rels,
|
||||
length(path) as path_length
|
||||
ORDER BY path_length
|
||||
"""
|
||||
|
||||
|
||||
results = self.graph_store.execute_query(query, {
|
||||
"decision_id": decision_id
|
||||
})
|
||||
results = self._extract_records(results)
|
||||
|
||||
|
||||
paths = []
|
||||
for record in results:
|
||||
path_info = {
|
||||
"path": record.get("path"),
|
||||
"path_length": record.get("path_length", 0)
|
||||
"path_length": record.get("path_length", 0),
|
||||
"nodes": record.get("path_nodes", []),
|
||||
"relationships": record.get("path_rels", []),
|
||||
}
|
||||
paths.append(path_info)
|
||||
|
||||
@@ -618,6 +860,40 @@ class DecisionQuery:
|
||||
if self.embedding_generator:
|
||||
query_embedding = self.embedding_generator.generate(exception_reason)
|
||||
|
||||
if type(self.graph_store) is ContextGraph:
|
||||
nodes = self.graph_store.find_nodes("Exception")
|
||||
exceptions = []
|
||||
for node in nodes:
|
||||
metadata = node.get("metadata", {}).get("properties", node.get("metadata", {}))
|
||||
core_fields = {
|
||||
"decision_id", "policy_id", "reason", "approver",
|
||||
"approval_timestamp", "justification"
|
||||
}
|
||||
custom_metadata = {k: v for k, v in metadata.items() if k not in core_fields}
|
||||
exception_data = {"id": node.get("id"), "metadata": custom_metadata}
|
||||
exception_data.update({k: v for k, v in metadata.items() if k in core_fields})
|
||||
|
||||
try:
|
||||
exception = self._dict_to_exception(exception_data)
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
if query_embedding:
|
||||
reason_embedding = self.embedding_generator.generate(exception.reason)
|
||||
similarity = self._cosine_similarity(query_embedding, reason_embedding)
|
||||
exception.metadata["similarity_score"] = similarity
|
||||
|
||||
exceptions.append(exception)
|
||||
|
||||
if query_embedding:
|
||||
exceptions.sort(
|
||||
key=lambda e: e.metadata.get("similarity_score", 0),
|
||||
reverse=True
|
||||
)
|
||||
|
||||
self.logger.info(f"Found {min(len(exceptions), limit)} similar exceptions")
|
||||
return exceptions[:limit]
|
||||
|
||||
# Find exceptions with similar reasons
|
||||
query = """
|
||||
MATCH (e:Exception)
|
||||
@@ -948,10 +1224,11 @@ class DecisionQuery:
|
||||
|
||||
for measure_type, measure_data in centrality_measures.items():
|
||||
if isinstance(measure_data, dict) and 'centrality' in measure_data:
|
||||
decision_measures[measure_type] = measure_data['centrality'].get(decision_id, 0.0)
|
||||
|
||||
val = measure_data['centrality'].get(decision_id, 0.0)
|
||||
decision_measures[measure_type] = val if isinstance(val, (int, float)) else 0.0
|
||||
|
||||
analysis["centrality_measures"] = decision_measures
|
||||
|
||||
|
||||
# Calculate overall influence score
|
||||
measures = analysis["centrality_measures"]
|
||||
analysis["influence_score"] = (
|
||||
|
||||
@@ -82,6 +82,7 @@ from .decision_models import (
|
||||
Decision, DecisionContext, Policy, PolicyException,
|
||||
Precedent, ApprovalChain
|
||||
)
|
||||
from .context_graph import ContextGraph
|
||||
|
||||
|
||||
class DecisionRecorder:
|
||||
@@ -161,6 +162,13 @@ class DecisionRecorder:
|
||||
entities: List of entity IDs to link
|
||||
"""
|
||||
try:
|
||||
if type(self.graph_store) is ContextGraph:
|
||||
for entity_id in entities:
|
||||
self.graph_store.add_node(node_id=entity_id, node_type="Entity")
|
||||
self.graph_store.add_edge(source_id=decision_id, target_id=entity_id, edge_type="ABOUT")
|
||||
self.logger.info(f"Linked decision {decision_id} to {len(entities)} entities")
|
||||
return
|
||||
|
||||
for entity_id in entities:
|
||||
# Create ABOUT relationship between decision and entity
|
||||
query = """
|
||||
@@ -297,6 +305,13 @@ class DecisionRecorder:
|
||||
# Store exception in graph
|
||||
self._store_exception_node(exception)
|
||||
|
||||
if type(self.graph_store) is ContextGraph:
|
||||
self.graph_store.add_node(node_id=policy_id, node_type="Policy")
|
||||
self.graph_store.add_edge(source_id=decision_id, target_id=exception.exception_id, edge_type="GRANTED_EXCEPTION")
|
||||
self.graph_store.add_edge(source_id=exception.exception_id, target_id=policy_id, edge_type="OVERRIDDEN_POLICY")
|
||||
self.logger.info(f"Recorded exception: {exception.exception_id}")
|
||||
return exception.exception_id
|
||||
|
||||
# Create relationships
|
||||
query = """
|
||||
MATCH (d:Decision {decision_id: $decision_id})
|
||||
@@ -426,6 +441,12 @@ class DecisionRecorder:
|
||||
if len(precedent_ids) != len(relationship_types):
|
||||
raise ValueError("Precedent IDs and relationship types must have same length")
|
||||
|
||||
if type(self.graph_store) is ContextGraph:
|
||||
for precedent_id, relationship_type in zip(precedent_ids, relationship_types):
|
||||
self.graph_store.add_edge(source_id=decision_id, target_id=precedent_id, edge_type=relationship_type)
|
||||
self.logger.info(f"Linked {len(precedent_ids)} precedents to decision {decision_id}")
|
||||
return
|
||||
|
||||
for precedent_id, relationship_type in zip(precedent_ids, relationship_types):
|
||||
# Create precedent relationship
|
||||
query = """
|
||||
@@ -447,6 +468,27 @@ class DecisionRecorder:
|
||||
|
||||
def _store_decision_node(self, decision: Decision) -> None:
|
||||
"""Store decision node in graph database."""
|
||||
metadata = decision.metadata.copy() if decision.metadata else {}
|
||||
metadata.update({
|
||||
"category": decision.category,
|
||||
"scenario": decision.scenario,
|
||||
"reasoning": decision.reasoning,
|
||||
"outcome": decision.outcome,
|
||||
"confidence": decision.confidence,
|
||||
"timestamp": decision.timestamp.isoformat() if decision.timestamp else None,
|
||||
"decision_maker": decision.decision_maker,
|
||||
"reasoning_embedding": decision.reasoning_embedding,
|
||||
"node2vec_embedding": decision.node2vec_embedding
|
||||
})
|
||||
|
||||
if type(self.graph_store) is ContextGraph:
|
||||
self.graph_store.add_node(
|
||||
node_id=decision.decision_id,
|
||||
node_type="Decision",
|
||||
**metadata
|
||||
)
|
||||
return
|
||||
|
||||
query = """
|
||||
CREATE (d:Decision {
|
||||
decision_id: $decision_id,
|
||||
@@ -478,6 +520,24 @@ class DecisionRecorder:
|
||||
|
||||
def _store_exception_node(self, exception: PolicyException) -> None:
|
||||
"""Store exception node in graph database."""
|
||||
metadata = exception.metadata.copy() if exception.metadata else {}
|
||||
metadata.update({
|
||||
"decision_id": exception.decision_id,
|
||||
"policy_id": exception.policy_id,
|
||||
"reason": exception.reason,
|
||||
"approver": exception.approver,
|
||||
"approval_timestamp": exception.approval_timestamp.isoformat() if exception.approval_timestamp else None,
|
||||
"justification": exception.justification
|
||||
})
|
||||
|
||||
if type(self.graph_store) is ContextGraph:
|
||||
self.graph_store.add_node(
|
||||
node_id=exception.exception_id,
|
||||
node_type="Exception",
|
||||
**metadata
|
||||
)
|
||||
return
|
||||
|
||||
query = """
|
||||
CREATE (e:Exception {
|
||||
exception_id: $exception_id,
|
||||
|
||||
@@ -366,7 +366,7 @@ class EntityLinker:
|
||||
entity_text: str,
|
||||
entity_type: Optional[str] = None,
|
||||
threshold: Optional[float] = None,
|
||||
) -> List[Tuple[str, float]]:
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Find similar entities in knowledge graph.
|
||||
|
||||
@@ -376,7 +376,7 @@ class EntityLinker:
|
||||
threshold: Similarity threshold (uses default if None)
|
||||
|
||||
Returns:
|
||||
List of (entity_id, similarity_score) tuples
|
||||
List of dicts with entity_id, text, type, uri, and similarity
|
||||
"""
|
||||
threshold = threshold or self.similarity_threshold
|
||||
|
||||
@@ -404,10 +404,16 @@ class EntityLinker:
|
||||
if similarity >= threshold:
|
||||
entity_id = entity.get("id") or entity.get("entity_id")
|
||||
if entity_id:
|
||||
similar_entities.append((entity_id, similarity))
|
||||
similar_entities.append({
|
||||
"entity_id": entity_id,
|
||||
"text": entity_text2,
|
||||
"type": entity.get("type", ""),
|
||||
"uri": self.entity_registry.get(entity_id, ""),
|
||||
"similarity": similarity,
|
||||
})
|
||||
|
||||
# Sort by similarity
|
||||
similar_entities.sort(key=lambda x: x[1], reverse=True)
|
||||
similar_entities.sort(key=lambda x: x["similarity"], reverse=True)
|
||||
|
||||
return similar_entities
|
||||
|
||||
@@ -425,7 +431,11 @@ class EntityLinker:
|
||||
# Find similar entities in knowledge graph
|
||||
if self.knowledge_graph:
|
||||
similar = self.find_similar_entities(entity_text, entity_type)
|
||||
for similar_id, similarity in similar:
|
||||
for similar_entity in similar:
|
||||
similar_id = similar_entity.get("entity_id")
|
||||
similarity = similar_entity.get("similarity", 0.0)
|
||||
if not similar_id:
|
||||
continue
|
||||
if similar_id != entity_id:
|
||||
links.append(
|
||||
EntityLink(
|
||||
@@ -578,7 +588,11 @@ class EntityLinker:
|
||||
)
|
||||
|
||||
linked_entities = []
|
||||
for similar_id, similarity in similar:
|
||||
for similar_entity in similar:
|
||||
similar_id = similar_entity.get("entity_id")
|
||||
similarity = similar_entity.get("similarity", 0.0)
|
||||
if not similar_id:
|
||||
continue
|
||||
linked_entities.append(
|
||||
EntityLink(
|
||||
source_entity_id=entity.get("id", ""),
|
||||
@@ -604,7 +618,7 @@ class EntityLinker:
|
||||
# Search Methods
|
||||
def find_similar(
|
||||
self, entity: Union[str, EntityDict], threshold: float = 0.8
|
||||
) -> List[Tuple[str, float]]:
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Find similar entities.
|
||||
|
||||
@@ -613,7 +627,7 @@ class EntityLinker:
|
||||
threshold: Similarity threshold (default: 0.8)
|
||||
|
||||
Returns:
|
||||
List of (entity_id, similarity) tuples
|
||||
List of dicts with entity_id, text, type, uri, and similarity
|
||||
|
||||
Example:
|
||||
>>> similar = linker.find_similar("Python", threshold=0.8)
|
||||
|
||||
@@ -601,7 +601,7 @@ class PolicyEngine:
|
||||
to_version: New version
|
||||
|
||||
Returns:
|
||||
List of affected decision IDs
|
||||
List of affected decisions with readable metadata
|
||||
"""
|
||||
try:
|
||||
if self._supports_cypher:
|
||||
@@ -610,16 +610,23 @@ class PolicyEngine:
|
||||
policy_id: $policy_id,
|
||||
version: $from_version
|
||||
})
|
||||
RETURN d.decision_id as decision_id
|
||||
RETURN d.decision_id as decision_id,
|
||||
d.scenario as scenario,
|
||||
d.category as category,
|
||||
d.outcome as outcome,
|
||||
d.confidence as confidence
|
||||
"""
|
||||
results = self.graph_store.execute_query(query, {
|
||||
results = self._extract_records(self.graph_store.execute_query(query, {
|
||||
"policy_id": policy_id,
|
||||
"from_version": from_version
|
||||
})
|
||||
|
||||
decisions = []
|
||||
for record in results:
|
||||
decisions.append(record if isinstance(record, dict) else {"decision_id": record})
|
||||
}))
|
||||
|
||||
decisions = [
|
||||
self._enrich_affected_decision(
|
||||
record if isinstance(record, dict) else {"decision_id": record}
|
||||
)
|
||||
for record in results
|
||||
]
|
||||
|
||||
self.logger.info(f"Found {len(decisions)} decisions affected by policy change")
|
||||
return decisions
|
||||
@@ -627,15 +634,79 @@ class PolicyEngine:
|
||||
if not hasattr(self.graph_store, "find_edges"):
|
||||
return []
|
||||
policy_node_id = f"{policy_id}:{from_version}"
|
||||
decision_ids: List[str] = []
|
||||
decisions: List[Dict[str, Any]] = []
|
||||
for edge in self.graph_store.find_edges(edge_type="APPLIED_POLICY"):
|
||||
if edge.get("target") == policy_node_id:
|
||||
decision_ids.append(edge.get("source"))
|
||||
return decision_ids
|
||||
decisions.append(
|
||||
self._enrich_affected_decision(
|
||||
{"decision_id": edge.get("source")}
|
||||
)
|
||||
)
|
||||
return decisions
|
||||
|
||||
except Exception as e:
|
||||
self.logger.exception("Failed to get affected decisions")
|
||||
raise
|
||||
|
||||
def _enrich_affected_decision(self, record: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Normalize an affected decision record with readable fields."""
|
||||
decision_id = record.get("decision_id") or record.get("source") or ""
|
||||
node_record = self._get_decision_node_record(decision_id)
|
||||
|
||||
enriched = {
|
||||
"decision_id": decision_id,
|
||||
"scenario": record.get("scenario", node_record.get("scenario", "")),
|
||||
"category": record.get("category", node_record.get("category", "")),
|
||||
"outcome": record.get("outcome", node_record.get("outcome", "")),
|
||||
"confidence": record.get("confidence", node_record.get("confidence", 0.0)),
|
||||
}
|
||||
|
||||
for key, value in record.items():
|
||||
if key not in enriched:
|
||||
enriched[key] = value
|
||||
|
||||
return enriched
|
||||
|
||||
def _get_decision_node_record(self, decision_id: str) -> Dict[str, Any]:
|
||||
"""Best-effort lookup of a decision node from the backing graph store."""
|
||||
if not decision_id:
|
||||
return {}
|
||||
|
||||
if hasattr(self.graph_store, "nodes"):
|
||||
nodes = getattr(self.graph_store, "nodes", {})
|
||||
if isinstance(nodes, dict):
|
||||
node = nodes.get(decision_id)
|
||||
if node:
|
||||
properties = getattr(node, "properties", {}) or {}
|
||||
content = getattr(node, "content", "") or ""
|
||||
return {
|
||||
"scenario": properties.get("scenario", content),
|
||||
"category": properties.get("category", ""),
|
||||
"outcome": properties.get("outcome", ""),
|
||||
"confidence": properties.get("confidence", 0.0),
|
||||
}
|
||||
|
||||
get_node = getattr(self.graph_store, "get_node", None)
|
||||
if callable(get_node):
|
||||
try:
|
||||
node = get_node(decision_id)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
if isinstance(node, dict):
|
||||
properties = node.get("properties", {}) or {}
|
||||
return {
|
||||
"scenario": (
|
||||
properties.get("scenario")
|
||||
or node.get("content")
|
||||
or node.get("scenario", "")
|
||||
),
|
||||
"category": properties.get("category", node.get("category", "")),
|
||||
"outcome": properties.get("outcome", node.get("outcome", "")),
|
||||
"confidence": properties.get("confidence", node.get("confidence", 0.0)),
|
||||
}
|
||||
|
||||
return {}
|
||||
|
||||
def analyze_policy_impact(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""
|
||||
Semantica Knowledge Explorer : CLI Entry Point
|
||||
|
||||
Provides the ``semantica-explorer`` command that loads a graph from a
|
||||
JSON file, starts a FastAPI server, and optionally opens the browser.
|
||||
|
||||
Usage::
|
||||
|
||||
semantica-explorer --graph my_graph.json --port 8000
|
||||
python -m semantica.explorer --graph my_graph.json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import webbrowser
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
"""CLI entry point for the Knowledge Explorer server."""
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="semantica-explorer",
|
||||
description="Semantica Knowledge Explorer — interactive dashboard for KG exploration",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--graph", "-g",
|
||||
required=True,
|
||||
help="Path to a ContextGraph JSON file to load.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port", "-p",
|
||||
type=int,
|
||||
default=8000,
|
||||
help="Port to bind the server to (default: 8000).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
default="127.0.0.1",
|
||||
help="Host to bind the server to (default: 127.0.0.1).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-browser",
|
||||
action="store_true",
|
||||
help="Do not open the browser automatically.",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
|
||||
import os
|
||||
if not os.path.isfile(args.graph):
|
||||
print(f"Error: graph file not found: {args.graph}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
import uvicorn
|
||||
except ImportError:
|
||||
print(
|
||||
"Error: uvicorn is required. Install with:\n"
|
||||
" pip install semantica[explorer]",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
from .session import GraphSession
|
||||
from .app import create_app
|
||||
|
||||
print(f"Loading graph from {args.graph} ...")
|
||||
session = GraphSession.from_file(args.graph)
|
||||
stats = session.get_stats()
|
||||
print(
|
||||
f"Graph loaded — {stats.get('node_count', 0)} nodes, "
|
||||
f"{stats.get('edge_count', 0)} edges"
|
||||
)
|
||||
|
||||
|
||||
app = create_app(session=session)
|
||||
|
||||
|
||||
url = f"http://{args.host}:{args.port}"
|
||||
if not args.no_browser:
|
||||
import threading
|
||||
threading.Timer(1.5, lambda: webbrowser.open(url)).start()
|
||||
|
||||
print(f"Starting explorer at {url}")
|
||||
print(f" API docs: {url}/docs")
|
||||
print(f" Health: {url}/api/health")
|
||||
|
||||
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
Semantica Explorer — FastAPI Application Factory
|
||||
|
||||
Creates and configures the FastAPI app with CORS, error handling,
|
||||
static file serving, route registration, and WebSocket support.
|
||||
"""
|
||||
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from .. import __version__
|
||||
from .session import GraphSession
|
||||
from .ws import ConnectionManager
|
||||
|
||||
|
||||
def create_app(session: Optional[GraphSession] = None) -> FastAPI:
|
||||
"""
|
||||
Build a fully-configured FastAPI application.
|
||||
|
||||
Args:
|
||||
session: Pre-built ``GraphSession``. If ``None`` the caller must
|
||||
attach one to ``app.state.session`` before the first
|
||||
request arrives.
|
||||
"""
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
if session is not None:
|
||||
app.state.session = session
|
||||
app.state.ws_manager = ConnectionManager()
|
||||
yield
|
||||
|
||||
app = FastAPI(
|
||||
title="Semantica Knowledge Explorer",
|
||||
description="Interactive dashboard API for exploring Semantica knowledge graphs.",
|
||||
version=__version__,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
cors_origins = os.environ.get("EXPLORER_CORS_ORIGINS", "*")
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=cors_origins.split(","),
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
@app.exception_handler(KeyError)
|
||||
async def key_error_handler(request: Request, exc: KeyError):
|
||||
return JSONResponse(
|
||||
status_code=404,
|
||||
content={"detail": f"Not found: {exc}"},
|
||||
)
|
||||
|
||||
@app.exception_handler(ValueError)
|
||||
async def value_error_handler(request: Request, exc: ValueError):
|
||||
return JSONResponse(
|
||||
status_code=422,
|
||||
content={"detail": str(exc)},
|
||||
)
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def generic_error_handler(request: Request, exc: Exception):
|
||||
# Let FastAPI's built-in HTTPException handler take precedence so that
|
||||
# responses from dependency injection (e.g. 503 from get_session) are
|
||||
# not swallowed and converted to 500.
|
||||
if isinstance(exc, HTTPException):
|
||||
raise exc
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"detail": "Internal Server Error"},
|
||||
)
|
||||
|
||||
from .routes.graph import router as graph_router
|
||||
from .routes.analytics import router as analytics_router
|
||||
from .routes.decisions import router as decisions_router
|
||||
from .routes.temporal import router as temporal_router
|
||||
from .routes.enrich import router as enrich_router
|
||||
from .routes.export_import import router as export_import_router
|
||||
from .routes.annotations import router as annotations_router
|
||||
|
||||
app.include_router(graph_router)
|
||||
app.include_router(analytics_router)
|
||||
app.include_router(decisions_router)
|
||||
app.include_router(temporal_router)
|
||||
app.include_router(enrich_router)
|
||||
app.include_router(export_import_router)
|
||||
app.include_router(annotations_router)
|
||||
|
||||
from fastapi import WebSocket, WebSocketDisconnect
|
||||
|
||||
@app.websocket("/ws/graph-updates")
|
||||
async def websocket_endpoint(websocket: WebSocket):
|
||||
manager: ConnectionManager = app.state.ws_manager
|
||||
await manager.connect(websocket)
|
||||
try:
|
||||
while True:
|
||||
await websocket.receive_text()
|
||||
except WebSocketDisconnect:
|
||||
manager.disconnect(websocket)
|
||||
|
||||
@app.get("/api/health")
|
||||
async def health():
|
||||
return {"status": "healthy"}
|
||||
|
||||
@app.get("/api/info")
|
||||
async def info():
|
||||
return {
|
||||
"name": "Semantica Knowledge Explorer",
|
||||
"version": __version__,
|
||||
"status": "active",
|
||||
}
|
||||
|
||||
static_dir = Path(__file__).resolve().parent.parent / "static"
|
||||
if static_dir.is_dir():
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
app.mount("/", StaticFiles(directory=str(static_dir), html=True), name="static")
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,27 @@
|
||||
"""
|
||||
Semantica Explorer : FastAPI Dependencies
|
||||
|
||||
Provides ``Depends()``-compatible callables for injecting the
|
||||
current ``GraphSession`` and ``ConnectionManager`` into route handlers.
|
||||
"""
|
||||
|
||||
from fastapi import Request
|
||||
from fastapi import Request, HTTPException, status
|
||||
|
||||
from .session import GraphSession
|
||||
from .ws import ConnectionManager
|
||||
|
||||
|
||||
def get_session(request: Request) -> GraphSession:
|
||||
"""Retrieve the GraphSession stored on ``app.state``."""
|
||||
if not hasattr(request.app.state, "session") or request.app.state.session is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="GraphSession not initialized."
|
||||
)
|
||||
return request.app.state.session
|
||||
|
||||
|
||||
def get_ws_manager(request: Request) -> ConnectionManager:
|
||||
"""Retrieve the ConnectionManager stored on ``app.state``."""
|
||||
return request.app.state.ws_manager
|
||||
@@ -0,0 +1 @@
|
||||
"""Route package for the Semantica Knowledge Explorer API."""
|
||||
@@ -0,0 +1,123 @@
|
||||
"""
|
||||
Analytics routes : centrality, community, connectivity, validation.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from ..dependencies import get_session
|
||||
from ..schemas import AnalyticsResponse, ValidationIssue, ValidationReportResponse
|
||||
from ..session import GraphSession
|
||||
|
||||
router = APIRouter(prefix="/api/analytics", tags=["Analytics"])
|
||||
|
||||
|
||||
def _build_graph_dict(session: GraphSession) -> dict:
|
||||
"""Build the entity/relationship dict expected by KG analysers."""
|
||||
nodes, _ = session.get_nodes(skip=0, limit=999_999)
|
||||
edges, _ = session.get_edges(skip=0, limit=999_999)
|
||||
return {
|
||||
"entities": [
|
||||
{"id": n.get("id"), "type": n.get("type", "entity"),
|
||||
"text": n.get("content", n.get("id", "")), "metadata": n.get("metadata", {})}
|
||||
for n in nodes
|
||||
],
|
||||
"relationships": [
|
||||
{"source": e.get("source"), "target": e.get("target"),
|
||||
"type": e.get("type", "related_to"), "metadata": e.get("metadata", {})}
|
||||
for e in edges
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@router.get("", response_model=AnalyticsResponse)
|
||||
async def get_analytics(
|
||||
metrics: Optional[str] = Query(
|
||||
None,
|
||||
description="Comma-separated metrics to compute: centrality,community,connectivity",
|
||||
),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Compute graph analytics (centrality, community, connectivity)."""
|
||||
requested = set((metrics or "centrality,community,connectivity").split(","))
|
||||
graph_dict = await asyncio.to_thread(_build_graph_dict, session)
|
||||
graph_dict = await asyncio.to_thread(session.build_graph_dict)
|
||||
result: dict = {}
|
||||
|
||||
if "centrality" in requested and session.centrality is not None:
|
||||
try:
|
||||
centrality = await asyncio.to_thread(
|
||||
session.centrality.calculate_degree_centrality, graph_dict
|
||||
)
|
||||
result["centrality"] = centrality
|
||||
except Exception as exc:
|
||||
result["centrality"] = {"error": str(exc)}
|
||||
|
||||
if "community" in requested and session.community is not None:
|
||||
try:
|
||||
community = await asyncio.to_thread(
|
||||
session.community.detect_communities, graph_dict
|
||||
)
|
||||
result["community"] = community
|
||||
except Exception as exc:
|
||||
result["community"] = {"error": str(exc)}
|
||||
|
||||
if "connectivity" in requested and session.connectivity is not None:
|
||||
try:
|
||||
connectivity = await asyncio.to_thread(
|
||||
session.connectivity.analyze_connectivity, graph_dict
|
||||
)
|
||||
result["connectivity"] = connectivity
|
||||
except Exception as exc:
|
||||
result["connectivity"] = {"error": str(exc)}
|
||||
|
||||
return AnalyticsResponse(**result)
|
||||
|
||||
|
||||
@router.get("/validation", response_model=ValidationReportResponse)
|
||||
async def validate_graph(
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Run graph validation and return a pass/fail report."""
|
||||
validator = session.validator
|
||||
if validator is None:
|
||||
return ValidationReportResponse(valid=True, error_count=0, warning_count=0, issues=[])
|
||||
|
||||
graph_dict = await asyncio.to_thread(_build_graph_dict, session)
|
||||
graph_dict = await asyncio.to_thread(session.build_graph_dict)
|
||||
|
||||
try:
|
||||
report = await asyncio.to_thread(validator.validate, graph_dict)
|
||||
except Exception as exc:
|
||||
return ValidationReportResponse(
|
||||
valid=False,
|
||||
error_count=1,
|
||||
issues=[ValidationIssue(severity="error", message=str(exc))],
|
||||
)
|
||||
|
||||
|
||||
if isinstance(report, dict):
|
||||
valid = report.get("valid", True)
|
||||
errors = report.get("errors", [])
|
||||
warnings = report.get("warnings", [])
|
||||
else:
|
||||
valid = getattr(report, "valid", True)
|
||||
errors = getattr(report, "errors", [])
|
||||
warnings = getattr(report, "warnings", [])
|
||||
|
||||
issues = []
|
||||
for e in (errors or []):
|
||||
msg = e if isinstance(e, str) else str(e)
|
||||
issues.append(ValidationIssue(severity="error", message=msg))
|
||||
for w in (warnings or []):
|
||||
msg = w if isinstance(w, str) else str(w)
|
||||
issues.append(ValidationIssue(severity="warning", message=msg))
|
||||
|
||||
return ValidationReportResponse(
|
||||
valid=valid,
|
||||
error_count=len(errors or []),
|
||||
warning_count=len(warnings or []),
|
||||
issues=issues,
|
||||
)
|
||||
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
Annotation routes — CRUD for collaborative annotations on graph nodes.
|
||||
|
||||
Annotations are stored in-memory on the ``GraphSession`` and do not
|
||||
modify Semantica core.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from ..dependencies import get_session
|
||||
from ..schemas import AnnotationCreate, AnnotationResponse
|
||||
from ..session import GraphSession
|
||||
|
||||
router = APIRouter(prefix="/api/annotations", tags=["Annotations"])
|
||||
|
||||
|
||||
@router.get("", response_model=list[AnnotationResponse])
|
||||
async def list_annotations(
|
||||
node_id: Optional[str] = Query(None, description="Filter by node ID"),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""List annotations, optionally filtered by node_id."""
|
||||
anns = await asyncio.to_thread(session.get_annotations, node_id)
|
||||
return [AnnotationResponse(**a) for a in anns]
|
||||
|
||||
|
||||
@router.post("", response_model=AnnotationResponse, status_code=201)
|
||||
async def create_annotation(
|
||||
body: AnnotationCreate,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Create a new annotation on a node."""
|
||||
node = await asyncio.to_thread(session.get_node, body.node_id)
|
||||
if node is None:
|
||||
raise KeyError(body.node_id)
|
||||
|
||||
ann_data = body.model_dump()
|
||||
ann_id = await asyncio.to_thread(session.add_annotation, ann_data)
|
||||
|
||||
|
||||
anns = await asyncio.to_thread(session.get_annotations)
|
||||
for a in anns:
|
||||
if a.get("annotation_id") == ann_id:
|
||||
return AnnotationResponse(**a)
|
||||
|
||||
return AnnotationResponse(
|
||||
annotation_id=ann_id,
|
||||
node_id=body.node_id,
|
||||
content=body.content,
|
||||
tags=body.tags,
|
||||
visibility=body.visibility,
|
||||
)
|
||||
# add_annotation mutates ann_data in-place, adding annotation_id and created_at.
|
||||
await asyncio.to_thread(session.add_annotation, ann_data)
|
||||
return AnnotationResponse(**ann_data)
|
||||
|
||||
|
||||
@router.delete("/{annotation_id}", status_code=204)
|
||||
async def delete_annotation(
|
||||
annotation_id: str,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Delete an annotation by ID."""
|
||||
deleted = await asyncio.to_thread(session.delete_annotation, annotation_id)
|
||||
if not deleted:
|
||||
raise KeyError(annotation_id)
|
||||
return None
|
||||
@@ -0,0 +1,184 @@
|
||||
"""
|
||||
Decision routes : decision listing, causal chains, precedents, compliance.
|
||||
|
||||
Uses ContextGraph-native queries so it works without a Neo4j/FalkorDB backend.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from ..dependencies import get_session
|
||||
from ..schemas import (
|
||||
CausalChainResponse,
|
||||
ComplianceResponse,
|
||||
DecisionResponse,
|
||||
)
|
||||
from ..session import GraphSession
|
||||
|
||||
router = APIRouter(prefix="/api/decisions", tags=["Decisions"])
|
||||
|
||||
|
||||
def _node_to_decision(n: dict) -> DecisionResponse:
|
||||
"""Map a ContextGraph node dict to a DecisionResponse."""
|
||||
meta = n.get("metadata", {})
|
||||
return DecisionResponse(
|
||||
decision_id=n.get("id", ""),
|
||||
category=meta.get("category", ""),
|
||||
scenario=meta.get("scenario", ""),
|
||||
reasoning=meta.get("reasoning", ""),
|
||||
outcome=meta.get("outcome", ""),
|
||||
confidence=float(meta.get("confidence", 0.0)),
|
||||
timestamp=meta.get("timestamp"),
|
||||
metadata=meta,
|
||||
)
|
||||
|
||||
|
||||
@router.get("", response_model=list[DecisionResponse])
|
||||
async def list_decisions(
|
||||
category: Optional[str] = Query(None),
|
||||
skip: int = Query(0, ge=0),
|
||||
limit: int = Query(50, ge=1, le=500),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""List decision nodes (type='decision') with optional category filter."""
|
||||
nodes, _ = await asyncio.to_thread(
|
||||
session.get_nodes, node_type="decision", skip=0, limit=999_999
|
||||
)
|
||||
|
||||
if category:
|
||||
nodes = [
|
||||
n for n in nodes
|
||||
if n.get("metadata", {}).get("category", "").lower() == category.lower()
|
||||
]
|
||||
|
||||
page = nodes[skip: skip + limit]
|
||||
return [_node_to_decision(n) for n in page]
|
||||
|
||||
|
||||
@router.get("/{decision_id}", response_model=DecisionResponse)
|
||||
async def get_decision(
|
||||
decision_id: str,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Get a single decision by ID."""
|
||||
node = await asyncio.to_thread(session.get_node, decision_id)
|
||||
if node is None:
|
||||
raise KeyError(decision_id)
|
||||
return _node_to_decision(node)
|
||||
|
||||
|
||||
@router.get("/{decision_id}/chain", response_model=CausalChainResponse)
|
||||
async def get_causal_chain(
|
||||
decision_id: str,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""
|
||||
Trace the causal chain for a decision.
|
||||
|
||||
Uses BFS neighbour traversal over ``caused_by`` / ``influences``
|
||||
relationship types as a lightweight, backend-agnostic fallback.
|
||||
"""
|
||||
node = await asyncio.to_thread(session.get_node, decision_id)
|
||||
if node is None:
|
||||
raise KeyError(decision_id)
|
||||
|
||||
# Walk outbound causal edges (up to 5 hops)
|
||||
neighbors = await asyncio.to_thread(session.get_neighbors, decision_id, depth=5)
|
||||
chain = [
|
||||
{
|
||||
"id": nb.get("id"),
|
||||
"type": nb.get("type"),
|
||||
"relationship": nb.get("relationship"),
|
||||
"hop": nb.get("hop"),
|
||||
"content": nb.get("content", ""),
|
||||
}
|
||||
for nb in neighbors
|
||||
]
|
||||
return CausalChainResponse(decision_id=decision_id, chain=chain)
|
||||
|
||||
|
||||
@router.get("/{decision_id}/precedents", response_model=list[DecisionResponse])
|
||||
async def get_precedents(
|
||||
decision_id: str,
|
||||
limit: int = Query(10, ge=1, le=100),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""
|
||||
Find precedent decisions similar to the given decision.
|
||||
|
||||
Lightweight: looks for other decision-type nodes and ranks by shared
|
||||
category and keyword overlap.
|
||||
"""
|
||||
node = await asyncio.to_thread(session.get_node, decision_id)
|
||||
if node is None:
|
||||
raise KeyError(decision_id)
|
||||
|
||||
meta = node.get("metadata", {})
|
||||
category = meta.get("category", "")
|
||||
scenario_words = set(meta.get("scenario", "").lower().split())
|
||||
|
||||
all_decisions, _ = await asyncio.to_thread(
|
||||
session.get_nodes, node_type="decision", skip=0, limit=999_999
|
||||
)
|
||||
|
||||
scored = []
|
||||
for d in all_decisions:
|
||||
if d.get("id") == decision_id:
|
||||
continue
|
||||
d_meta = d.get("metadata", {})
|
||||
score = 0.0
|
||||
if d_meta.get("category", "").lower() == category.lower() and category:
|
||||
score += 0.5
|
||||
d_words = set(d_meta.get("scenario", "").lower().split())
|
||||
if scenario_words and d_words:
|
||||
overlap = len(scenario_words & d_words) / max(len(scenario_words | d_words), 1)
|
||||
score += 0.5 * overlap
|
||||
scored.append((score, d))
|
||||
|
||||
scored.sort(key=lambda x: x[0], reverse=True)
|
||||
return [_node_to_decision(d) for _, d in scored[:limit]]
|
||||
|
||||
|
||||
@router.get("/{decision_id}/compliance", response_model=ComplianceResponse)
|
||||
async def check_compliance(
|
||||
decision_id: str,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""
|
||||
Check policy compliance for a decision.
|
||||
|
||||
Returns a stub result when no PolicyEngine is wired up.
|
||||
Inspects edges of type ``violates``, ``non_compliant``, or ``breaches``
|
||||
originating from the decision node. Returns ``compliant=True`` when no
|
||||
such edges are found, which is the correct result for graphs that have
|
||||
no policy-violation edges defined.
|
||||
"""
|
||||
node = await asyncio.to_thread(session.get_node, decision_id)
|
||||
if node is None:
|
||||
raise KeyError(decision_id)
|
||||
|
||||
|
||||
edges, _ = await asyncio.to_thread(session.get_edges, skip=0, limit=999_999)
|
||||
|
||||
_VIOLATION_TYPES = {"violates", "non_compliant", "breaches"}
|
||||
violation_edges = [
|
||||
e for e in edges
|
||||
if e.get("source") == decision_id and e.get("type") in _VIOLATION_TYPES
|
||||
]
|
||||
|
||||
violations = [
|
||||
{
|
||||
"policy_id": e.get("target"),
|
||||
"type": e.get("type"),
|
||||
"metadata": e.get("metadata", {}),
|
||||
}
|
||||
for e in violation_edges
|
||||
]
|
||||
|
||||
return ComplianceResponse(
|
||||
decision_id=decision_id,
|
||||
compliant=len(violations) == 0,
|
||||
violations=violations,
|
||||
)
|
||||
@@ -0,0 +1,175 @@
|
||||
"""
|
||||
Enrichment & reasoning routes — extraction, link prediction, dedup, reasoning.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from ..dependencies import get_session
|
||||
from ..schemas import (
|
||||
DedupRequest,
|
||||
DedupResponse,
|
||||
EnrichExtractRequest,
|
||||
EnrichExtractResponse,
|
||||
LinkPredictionRequest,
|
||||
LinkPredictionResponse,
|
||||
ReasoningRequest,
|
||||
ReasoningResponse,
|
||||
)
|
||||
from ..session import GraphSession
|
||||
|
||||
router = APIRouter(tags=["Enrichment"])
|
||||
|
||||
|
||||
@router.post("/api/enrich/extract", response_model=EnrichExtractResponse)
|
||||
async def extract_entities(
|
||||
body: EnrichExtractRequest,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Extract entities and relations from free text."""
|
||||
try:
|
||||
from ...semantic_extract.methods import extract_entities as _extract_entities
|
||||
from ...semantic_extract.methods import extract_relations as _extract_relations
|
||||
|
||||
entities = await asyncio.to_thread(_extract_entities, body.text)
|
||||
relations = await asyncio.to_thread(_extract_relations, body.text)
|
||||
|
||||
ent_list = entities if isinstance(entities, list) else getattr(entities, "entities", [])
|
||||
rel_list = relations if isinstance(relations, list) else getattr(relations, "relations", [])
|
||||
|
||||
return EnrichExtractResponse(
|
||||
entities=[_safe_dict(e) for e in ent_list],
|
||||
relations=[_safe_dict(r) for r in rel_list],
|
||||
)
|
||||
except ImportError:
|
||||
raise ValueError(
|
||||
"semantic_extract module not available. "
|
||||
"Ensure spacy and transformers are installed."
|
||||
)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"Extraction failed: {exc}")
|
||||
|
||||
|
||||
@router.post("/api/enrich/links", response_model=LinkPredictionResponse)
|
||||
async def predict_links(
|
||||
body: LinkPredictionRequest,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Predict likely new edges for a node."""
|
||||
predictor = session.link_predictor
|
||||
if predictor is None:
|
||||
raise ValueError("LinkPredictor not available — KG extras may not be installed.")
|
||||
|
||||
node = await asyncio.to_thread(session.get_node, body.node_id)
|
||||
if node is None:
|
||||
raise KeyError(body.node_id)
|
||||
|
||||
nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=999_999)
|
||||
edges, _ = await asyncio.to_thread(session.get_edges, skip=0, limit=999_999)
|
||||
|
||||
# Pre-compute already-connected node IDs so we skip them.
|
||||
# (LinkPredictor._edge_exists returns False for ContextGraph since it has no
|
||||
# has_edge/get_edge method, so we handle exclusion here instead.)
|
||||
existing_neighbours = {
|
||||
e.get("target") for e in edges if e.get("source") == body.node_id
|
||||
} | {
|
||||
e.get("source") for e in edges if e.get("target") == body.node_id
|
||||
}
|
||||
|
||||
# Score each candidate via score_link (which works with ContextGraph because
|
||||
# it falls back to has_node / get_neighbors).
|
||||
# Run in a thread so the CPU-bound scoring loop never blocks the event loop.
|
||||
def _score_all() -> list:
|
||||
results = []
|
||||
for n in nodes:
|
||||
candidate = n.get("id")
|
||||
if not candidate or candidate == body.node_id or candidate in existing_neighbours:
|
||||
continue
|
||||
try:
|
||||
score = predictor.score_link(session.graph, body.node_id, candidate)
|
||||
if score > 0:
|
||||
results.append(
|
||||
{"target": candidate, "score": score, "type": n.get("type", "entity")}
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
results.sort(key=lambda x: x["score"], reverse=True)
|
||||
return results
|
||||
|
||||
scored = await asyncio.to_thread(_score_all)
|
||||
|
||||
return LinkPredictionResponse(
|
||||
node_id=body.node_id,
|
||||
predictions=scored[: body.top_n],
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/enrich/dedup", response_model=DedupResponse)
|
||||
async def detect_duplicates(
|
||||
body: DedupRequest,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Run a deduplication scan over graph entities."""
|
||||
try:
|
||||
from ...deduplication import DuplicateDetector
|
||||
|
||||
detector = DuplicateDetector()
|
||||
# Use asyncio.to_thread — get_nodes acquires an RLock and must not block
|
||||
# the event loop.
|
||||
nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=999_999)
|
||||
|
||||
entities = [
|
||||
{
|
||||
"id": n.get("id"),
|
||||
"text": n.get("content", n.get("id", "")),
|
||||
"type": n.get("type", "entity"),
|
||||
}
|
||||
for n in nodes
|
||||
]
|
||||
|
||||
dups = await asyncio.to_thread(
|
||||
detector.detect_duplicates, entities, threshold=body.threshold
|
||||
)
|
||||
dup_list = dups if isinstance(dups, list) else getattr(dups, "duplicates", [])
|
||||
return DedupResponse(
|
||||
duplicates=[_safe_dict(d) for d in dup_list],
|
||||
total_flagged=len(dup_list),
|
||||
)
|
||||
except ImportError:
|
||||
raise ValueError("Deduplication module not available.")
|
||||
except Exception as exc:
|
||||
raise ValueError(f"Dedup scan failed: {exc}")
|
||||
|
||||
|
||||
@router.post("/api/reason", response_model=ReasoningResponse)
|
||||
async def run_reasoning(
|
||||
body: ReasoningRequest,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Run inference rules over facts."""
|
||||
try:
|
||||
from ...reasoning.reasoner import Reasoner
|
||||
|
||||
reasoner = Reasoner()
|
||||
inferred = await asyncio.to_thread(
|
||||
reasoner.infer_facts, body.facts, body.rules
|
||||
)
|
||||
|
||||
return ReasoningResponse(
|
||||
inferred_facts=inferred if isinstance(inferred, list) else [],
|
||||
rules_fired=len(inferred) if isinstance(inferred, list) else 0,
|
||||
)
|
||||
except ImportError:
|
||||
raise ValueError("Reasoning module not available.")
|
||||
except Exception as exc:
|
||||
raise ValueError(f"Reasoning failed: {exc}")
|
||||
|
||||
|
||||
def _safe_dict(obj) -> dict:
|
||||
"""Convert an object to a JSON-safe dict."""
|
||||
if isinstance(obj, dict):
|
||||
return obj
|
||||
if hasattr(obj, "__dict__"):
|
||||
return {k: v for k, v in obj.__dict__.items() if not k.startswith("_")}
|
||||
return {"value": str(obj)}
|
||||
@@ -0,0 +1,235 @@
|
||||
"""
|
||||
Export & import routes.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, File, UploadFile
|
||||
from fastapi.responses import Response
|
||||
|
||||
from ..dependencies import get_session, get_ws_manager
|
||||
from ..schemas import ExportRequest
|
||||
from ..session import GraphSession
|
||||
from ..ws import ConnectionManager
|
||||
|
||||
router = APIRouter(tags=["Export / Import"])
|
||||
|
||||
|
||||
_FORMAT_MAP = {
|
||||
"json": ("export_json", "application/json", ".json"),
|
||||
"json-ld": ("export_json", "application/ld+json", ".jsonld"),
|
||||
"turtle": ("export_rdf", "text/turtle", ".ttl"),
|
||||
"rdf-xml": ("export_rdf", "application/rdf+xml", ".rdf"),
|
||||
"n-triples": ("export_rdf", "application/n-triples", ".nt"),
|
||||
"csv": ("export_csv", "text/csv", ".csv"),
|
||||
"graphml": ("export_graph", "application/xml", ".graphml"),
|
||||
"gexf": ("export_graph", "application/xml", ".gexf"),
|
||||
"owl": ("export_owl", "application/rdf+xml", ".owl"),
|
||||
"cypher": ("export_lpg", "text/plain", ".cypher"),
|
||||
"aql": ("export_arango", "text/plain", ".aql"),
|
||||
"yaml": ("export_yaml", "text/yaml", ".yaml"),
|
||||
}
|
||||
|
||||
|
||||
def _build_kg_dict(session: GraphSession, node_ids: Optional[list] = None) -> dict:
|
||||
"""Build the knowledge-graph dict that exporters expect."""
|
||||
nodes, _ = session.get_nodes(skip=0, limit=999_999)
|
||||
edges, _ = session.get_edges(skip=0, limit=999_999)
|
||||
|
||||
if node_ids:
|
||||
id_set = set(node_ids)
|
||||
nodes = [n for n in nodes if n.get("id") in id_set]
|
||||
edges = [
|
||||
e for e in edges
|
||||
if e.get("source") in id_set and e.get("target") in id_set
|
||||
]
|
||||
|
||||
return {
|
||||
"entities": [
|
||||
{
|
||||
"id": n.get("id"),
|
||||
"type": n.get("type", "entity"),
|
||||
"text": n.get("content", n.get("id", "")),
|
||||
"metadata": n.get("metadata", {}),
|
||||
}
|
||||
for n in nodes
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source": e.get("source"),
|
||||
"target": e.get("target"),
|
||||
"type": e.get("type", "related_to"),
|
||||
"metadata": e.get("metadata", {}),
|
||||
}
|
||||
for e in edges
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/export")
|
||||
async def export_graph(
|
||||
body: ExportRequest,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Export the current graph in the requested format."""
|
||||
fmt = body.format.lower()
|
||||
if fmt not in _FORMAT_MAP:
|
||||
raise ValueError(
|
||||
f"Unsupported format '{fmt}'. Supported: {', '.join(sorted(_FORMAT_MAP))}"
|
||||
)
|
||||
|
||||
func_name, content_type, ext = _FORMAT_MAP[fmt]
|
||||
|
||||
kg = await asyncio.to_thread(_build_kg_dict, session, body.node_ids)
|
||||
|
||||
kg = await asyncio.to_thread(session.build_graph_dict, body.node_ids)
|
||||
|
||||
try:
|
||||
from ...export.methods import (
|
||||
export_json, export_rdf, export_csv, export_graph as export_graph_fn,
|
||||
export_owl, export_lpg, export_arango, export_yaml,
|
||||
)
|
||||
fn_map = {
|
||||
"export_json": export_json,
|
||||
"export_rdf": export_rdf,
|
||||
"export_csv": export_csv,
|
||||
"export_graph": export_graph_fn,
|
||||
"export_owl": export_owl,
|
||||
"export_lpg": export_lpg,
|
||||
"export_arango": export_arango,
|
||||
"export_yaml": export_yaml,
|
||||
}
|
||||
export_fn = fn_map.get(func_name)
|
||||
if export_fn is None:
|
||||
raise ValueError(f"Export function {func_name} not found.")
|
||||
|
||||
# Write to a temp file, read back content.
|
||||
with tempfile.NamedTemporaryFile(suffix=ext, delete=False, mode="w") as tmp:
|
||||
tmp_path = tmp.name
|
||||
|
||||
await asyncio.to_thread(export_fn, kg, tmp_path)
|
||||
|
||||
with open(tmp_path, "r", encoding="utf-8", errors="replace") as fh:
|
||||
content = fh.read()
|
||||
|
||||
import os
|
||||
os.unlink(tmp_path)
|
||||
|
||||
except ImportError:
|
||||
|
||||
# Write to a temp file; always clean up even if export or read fails.
|
||||
tmp_path = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=ext, delete=False, mode="w") as tmp:
|
||||
tmp_path = tmp.name
|
||||
|
||||
await asyncio.to_thread(export_fn, kg, tmp_path)
|
||||
|
||||
with open(tmp_path, "r", encoding="utf-8", errors="replace") as fh:
|
||||
content = fh.read()
|
||||
finally:
|
||||
if tmp_path and os.path.exists(tmp_path):
|
||||
os.unlink(tmp_path)
|
||||
|
||||
except ImportError:
|
||||
content = json.dumps(kg, indent=2, default=str)
|
||||
content_type = "application/json"
|
||||
ext = ".json"
|
||||
|
||||
filename = f"semantica_export{ext}"
|
||||
return Response(
|
||||
content=content,
|
||||
media_type=content_type,
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/import")
|
||||
async def import_file(
|
||||
file: UploadFile = File(...),
|
||||
session: GraphSession = Depends(get_session),
|
||||
ws: ConnectionManager = Depends(get_ws_manager),
|
||||
):
|
||||
"""
|
||||
Import entities from an uploaded file (JSON or CSV).
|
||||
|
||||
For JSON files the expected shape is ``{"nodes": [...], "edges": [...]}``.
|
||||
"""
|
||||
content = await file.read()
|
||||
filename = file.filename or "upload"
|
||||
|
||||
await ws.broadcast("import_started", {"filename": filename})
|
||||
|
||||
try:
|
||||
if filename.endswith(".json") or filename.endswith(".jsonld"):
|
||||
data = json.loads(content)
|
||||
|
||||
|
||||
raw_nodes = data.get("nodes", data.get("entities", []))
|
||||
raw_edges = data.get("edges", data.get("relationships", []))
|
||||
|
||||
|
||||
# KG export uses {id, type, text, metadata}
|
||||
# ContextGraph.add_nodes expects {id, type, properties: {content, ...}}
|
||||
nodes = []
|
||||
for n in raw_nodes:
|
||||
if "properties" in n:
|
||||
nodes.append(n)
|
||||
else:
|
||||
nodes.append({
|
||||
"id": n.get("id"),
|
||||
"type": n.get("type", "entity"),
|
||||
"properties": {
|
||||
"content": n.get("text", n.get("content", n.get("id", ""))),
|
||||
**(n.get("metadata") or {}),
|
||||
},
|
||||
})
|
||||
|
||||
# KG export uses {source, target, type, metadata}
|
||||
# ContextGraph.add_edges expects {source_id, target_id, type, weight, properties}
|
||||
edges = []
|
||||
for r in raw_edges:
|
||||
src = r.get("source_id", r.get("source"))
|
||||
tgt = r.get("target_id", r.get("target"))
|
||||
if not src or not tgt:
|
||||
continue
|
||||
edges.append({
|
||||
"source_id": src,
|
||||
"target_id": tgt,
|
||||
"type": r.get("type", "related_to"),
|
||||
"weight": r.get("weight", 1.0),
|
||||
"properties": r.get("metadata") or r.get("properties") or {},
|
||||
})
|
||||
nodes = data.get("nodes", data.get("entities", []))
|
||||
edges = data.get("edges", data.get("relationships", []))
|
||||
|
||||
for edge in edges:
|
||||
if "source" in edge and "source_id" not in edge:
|
||||
edge["source_id"] = edge["source"]
|
||||
if "target" in edge and "target_id" not in edge:
|
||||
edge["target_id"] = edge["target"]
|
||||
|
||||
added_nodes = await asyncio.to_thread(session.add_nodes, nodes)
|
||||
added_edges = await asyncio.to_thread(session.add_edges, edges)
|
||||
|
||||
result = {
|
||||
"status": "success",
|
||||
"nodes_added": added_nodes,
|
||||
"edges_added": added_edges,
|
||||
}
|
||||
else:
|
||||
result = {
|
||||
"status": "unsupported",
|
||||
"detail": f"File type not supported yet: {filename}",
|
||||
}
|
||||
except Exception as exc:
|
||||
result = {"status": "error", "detail": str(exc)}
|
||||
|
||||
await ws.broadcast("import_completed", result)
|
||||
return result
|
||||
@@ -0,0 +1,222 @@
|
||||
"""
|
||||
Graph routes — node / edge / path / search endpoints.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from ..dependencies import get_session
|
||||
from ..schemas import (
|
||||
EdgeListResponse,
|
||||
EdgeResponse,
|
||||
GraphStatsResponse,
|
||||
NeighborResponse,
|
||||
NodeListResponse,
|
||||
NodeResponse,
|
||||
PathResponse,
|
||||
SearchRequest,
|
||||
SearchResultItem,
|
||||
SearchResultResponse,
|
||||
)
|
||||
from ..session import GraphSession
|
||||
|
||||
router = APIRouter(prefix="/api/graph", tags=["Graph"])
|
||||
|
||||
|
||||
def _node_dict_to_response(n: dict) -> NodeResponse:
|
||||
"""Convert a ContextGraph node dict to a NodeResponse."""
|
||||
meta = n.get("metadata", {})
|
||||
return NodeResponse(
|
||||
id=n.get("id", ""),
|
||||
type=n.get("type", "entity"),
|
||||
content=n.get("content", meta.get("content", "")),
|
||||
properties=meta,
|
||||
valid_from=meta.get("valid_from"),
|
||||
valid_until=meta.get("valid_until"),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@router.get("/nodes", response_model=NodeListResponse)
|
||||
async def list_nodes(
|
||||
type: Optional[str] = Query(None, description="Filter by node type"),
|
||||
search: Optional[str] = Query(None, description="Keyword search over node content"),
|
||||
skip: int = Query(0, ge=0),
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""List nodes with optional filtering and pagination."""
|
||||
nodes, total = await asyncio.to_thread(
|
||||
session.get_nodes, node_type=type, search=search, skip=skip, limit=limit
|
||||
)
|
||||
return NodeListResponse(
|
||||
nodes=[_node_dict_to_response(n) for n in nodes],
|
||||
total=total,
|
||||
skip=skip,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/node/{node_id}", response_model=NodeResponse)
|
||||
async def get_node(
|
||||
node_id: str,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Get a single node by ID."""
|
||||
node = await asyncio.to_thread(session.get_node, node_id)
|
||||
if node is None:
|
||||
raise KeyError(node_id)
|
||||
return _node_dict_to_response(node)
|
||||
|
||||
|
||||
@router.get("/node/{node_id}/neighbors", response_model=list[NeighborResponse])
|
||||
async def get_neighbors(
|
||||
node_id: str,
|
||||
depth: int = Query(1, ge=1, le=5),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Get neighbours of a node via BFS traversal."""
|
||||
neighbors = await asyncio.to_thread(session.get_neighbors, node_id, depth)
|
||||
return [
|
||||
NeighborResponse(
|
||||
id=nb.get("id", ""),
|
||||
type=nb.get("type", ""),
|
||||
content=nb.get("content", ""),
|
||||
relationship=nb.get("relationship", ""),
|
||||
weight=nb.get("weight", 1.0),
|
||||
hop=nb.get("hop", 1),
|
||||
)
|
||||
for nb in neighbors
|
||||
]
|
||||
|
||||
|
||||
|
||||
|
||||
@router.get("/edges", response_model=EdgeListResponse)
|
||||
async def list_edges(
|
||||
type: Optional[str] = Query(None, description="Filter by edge type"),
|
||||
source: Optional[str] = Query(None, description="Filter by source node ID"),
|
||||
target: Optional[str] = Query(None, description="Filter by target node ID"),
|
||||
skip: int = Query(0, ge=0),
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""List edges with optional filtering and pagination."""
|
||||
edges, total = await asyncio.to_thread(
|
||||
session.get_edges, edge_type=type, source=source, target=target, skip=skip, limit=limit
|
||||
)
|
||||
return EdgeListResponse(
|
||||
edges=[
|
||||
EdgeResponse(
|
||||
source=e.get("source", ""),
|
||||
target=e.get("target", ""),
|
||||
type=e.get("type", ""),
|
||||
weight=e.get("weight", 1.0),
|
||||
properties=e.get("metadata", {}),
|
||||
)
|
||||
for e in edges
|
||||
],
|
||||
total=total,
|
||||
skip=skip,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
@router.get("/node/{node_id}/path", response_model=PathResponse)
|
||||
async def find_path(
|
||||
node_id: str,
|
||||
target: str = Query(..., description="Target node ID"),
|
||||
algorithm: str = Query("bfs", description="Algorithm: bfs, dijkstra"),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Find a path between two nodes."""
|
||||
pf = session.path_finder
|
||||
if pf is None:
|
||||
raise ValueError("PathFinder not available — KG extras may not be installed.")
|
||||
|
||||
graph_data = await asyncio.to_thread(_build_graph_dict, session)
|
||||
|
||||
result = await asyncio.to_thread(
|
||||
pf.find_shortest_path, graph_data, node_id, target
|
||||
)
|
||||
|
||||
path_nodes = result.get("path", []) if isinstance(result, dict) else []
|
||||
graph_data = await asyncio.to_thread(session.build_graph_dict)
|
||||
|
||||
# Select algorithm: dijkstra for weighted shortest path, bfs otherwise.
|
||||
if algorithm.lower() == "dijkstra":
|
||||
path_fn = pf.dijkstra_shortest_path
|
||||
else:
|
||||
path_fn = pf.bfs_shortest_path
|
||||
|
||||
result = await asyncio.to_thread(path_fn, graph_data, node_id, target)
|
||||
|
||||
path_nodes = result.get("path", []) if isinstance(result, dict) else (result or [])
|
||||
total_weight = result.get("total_weight", 0.0) if isinstance(result, dict) else 0.0
|
||||
|
||||
return PathResponse(
|
||||
source=node_id,
|
||||
target=target,
|
||||
algorithm=algorithm,
|
||||
path=path_nodes,
|
||||
total_weight=total_weight,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@router.post("/search", response_model=SearchResultResponse)
|
||||
async def search_nodes(
|
||||
body: SearchRequest,
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Keyword search over graph nodes."""
|
||||
results = await asyncio.to_thread(session.search, body.query, body.limit)
|
||||
items = [
|
||||
SearchResultItem(
|
||||
node=_node_dict_to_response(r.get("node", {})),
|
||||
score=r.get("score", 0.0),
|
||||
)
|
||||
for r in results
|
||||
]
|
||||
return SearchResultResponse(results=items, total=len(items), query=body.query)
|
||||
|
||||
|
||||
|
||||
@router.get("/stats", response_model=GraphStatsResponse)
|
||||
async def graph_stats(
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""Get graph-level statistics."""
|
||||
stats = await asyncio.to_thread(session.get_stats)
|
||||
return GraphStatsResponse(**stats)
|
||||
|
||||
|
||||
|
||||
def _build_graph_dict(session: GraphSession) -> dict:
|
||||
"""Build a graph dict for analytics helpers (entities + relationships)."""
|
||||
nodes, _ = session.get_nodes(skip=0, limit=999_999)
|
||||
edges, _ = session.get_edges(skip=0, limit=999_999)
|
||||
return {
|
||||
"entities": [
|
||||
{
|
||||
"id": n.get("id"),
|
||||
"type": n.get("type", "entity"),
|
||||
"text": n.get("content", n.get("id", "")),
|
||||
"metadata": n.get("metadata", {}),
|
||||
}
|
||||
for n in nodes
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source": e.get("source"),
|
||||
"target": e.get("target"),
|
||||
"type": e.get("type", "related_to"),
|
||||
"metadata": e.get("metadata", {}),
|
||||
}
|
||||
for e in edges
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
"""
|
||||
Temporal routes — snapshot, diff, patterns.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from ..dependencies import get_session
|
||||
from ..schemas import (
|
||||
NodeResponse,
|
||||
TemporalDiffResponse,
|
||||
TemporalPatternResponse,
|
||||
TemporalSnapshotResponse,
|
||||
)
|
||||
from ..session import GraphSession
|
||||
|
||||
router = APIRouter(prefix="/api/temporal", tags=["Temporal"])
|
||||
|
||||
|
||||
def _node_dict_to_response(n: dict) -> NodeResponse:
|
||||
meta = n.get("metadata", {})
|
||||
return NodeResponse(
|
||||
id=n.get("id", ""),
|
||||
type=n.get("type", "entity"),
|
||||
content=n.get("content", meta.get("content", "")),
|
||||
properties=meta,
|
||||
valid_from=meta.get("valid_from"),
|
||||
valid_until=meta.get("valid_until"),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/snapshot", response_model=TemporalSnapshotResponse)
|
||||
async def temporal_snapshot(
|
||||
at: Optional[str] = Query(
|
||||
None, description="ISO-8601 datetime; defaults to now."
|
||||
),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""
|
||||
Return the graph as it existed at a given timestamp.
|
||||
|
||||
Only nodes whose ``valid_from`` / ``valid_until`` window includes
|
||||
the requested time are returned.
|
||||
"""
|
||||
if at:
|
||||
ts_str = at.replace("Z", "+00:00")
|
||||
at_time = datetime.fromisoformat(ts_str)
|
||||
else:
|
||||
at_time = datetime.now(timezone.utc)
|
||||
|
||||
active = await asyncio.to_thread(session.get_active_nodes, at_time=at_time)
|
||||
return TemporalSnapshotResponse(
|
||||
timestamp=at_time.isoformat(),
|
||||
active_nodes=[_node_dict_to_response(n) for n in active],
|
||||
active_node_count=len(active),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/diff", response_model=TemporalDiffResponse)
|
||||
async def temporal_diff(
|
||||
from_time: str = Query(..., description="Start ISO-8601 datetime"),
|
||||
to_time: str = Query(..., description="End ISO-8601 datetime"),
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""
|
||||
Diff the graph between two points in time.
|
||||
|
||||
Returns node IDs that were added (active at ``to_time`` but not
|
||||
``from_time``) and removed (active at ``from_time`` but not
|
||||
``to_time``).
|
||||
"""
|
||||
t1 = datetime.fromisoformat(from_time.replace("Z", "+00:00"))
|
||||
t2 = datetime.fromisoformat(to_time.replace("Z", "+00:00"))
|
||||
|
||||
active_t1 = await asyncio.to_thread(session.get_active_nodes, at_time=t1)
|
||||
active_t2 = await asyncio.to_thread(session.get_active_nodes, at_time=t2)
|
||||
|
||||
ids_t1 = {n.get("id") for n in active_t1}
|
||||
ids_t2 = {n.get("id") for n in active_t2}
|
||||
|
||||
return TemporalDiffResponse(
|
||||
from_time=t1.isoformat(),
|
||||
to_time=t2.isoformat(),
|
||||
added_nodes=sorted(ids_t2 - ids_t1),
|
||||
removed_nodes=sorted(ids_t1 - ids_t2),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/patterns", response_model=TemporalPatternResponse)
|
||||
async def temporal_patterns(
|
||||
session: GraphSession = Depends(get_session),
|
||||
):
|
||||
"""
|
||||
Detect temporal patterns (trends, cycles, anomalies).
|
||||
|
||||
Falls back to a stub when ``TemporalPatternDetector`` is not available.
|
||||
"""
|
||||
try:
|
||||
from ...kg import TemporalPatternDetector
|
||||
detector = TemporalPatternDetector()
|
||||
|
||||
|
||||
nodes, _ = await asyncio.to_thread(session.get_nodes, skip=0, limit=999_999)
|
||||
edges, _ = await asyncio.to_thread(session.get_edges, skip=0, limit=999_999)
|
||||
graph_dict = {
|
||||
"entities": [
|
||||
{"id": n.get("id"), "type": n.get("type"), "metadata": n.get("metadata", {})}
|
||||
for n in nodes
|
||||
],
|
||||
"relationships": [
|
||||
{"source": e.get("source"), "target": e.get("target"),
|
||||
"type": e.get("type"), "metadata": e.get("metadata", {})}
|
||||
for e in edges
|
||||
],
|
||||
}
|
||||
|
||||
patterns = await asyncio.to_thread(detector.detect_patterns, graph_dict)
|
||||
if isinstance(patterns, dict):
|
||||
patterns = patterns.get("patterns", [])
|
||||
return TemporalPatternResponse(patterns=patterns if isinstance(patterns, list) else [])
|
||||
except ImportError:
|
||||
# TemporalPatternDetector is an optional KG extra; return empty gracefully.
|
||||
return TemporalPatternResponse(patterns=[])
|
||||
except Exception as exc:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning("temporal_patterns failed: %s", exc, exc_info=True)
|
||||
return TemporalPatternResponse(patterns=[])
|
||||
@@ -0,0 +1,257 @@
|
||||
"""
|
||||
Semantica Explorer : Pydantic Schemas
|
||||
|
||||
All request/response models for the Knowledge Explorer REST API.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
|
||||
class ErrorResponse(BaseModel):
|
||||
"""Standard error envelope."""
|
||||
detail: str
|
||||
status_code: int = 500
|
||||
|
||||
|
||||
|
||||
class NodeResponse(BaseModel):
|
||||
"""Single node representation."""
|
||||
id: str
|
||||
type: str
|
||||
content: str = ""
|
||||
properties: Dict[str, Any] = Field(default_factory=dict)
|
||||
valid_from: Optional[str] = None
|
||||
valid_until: Optional[str] = None
|
||||
|
||||
|
||||
class EdgeResponse(BaseModel):
|
||||
"""Single edge representation."""
|
||||
source: str
|
||||
target: str
|
||||
type: str
|
||||
weight: float = 1.0
|
||||
properties: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class NodeListResponse(BaseModel):
|
||||
"""Paginated node list."""
|
||||
nodes: List[NodeResponse]
|
||||
total: int
|
||||
skip: int = 0
|
||||
limit: int = 100
|
||||
|
||||
|
||||
class EdgeListResponse(BaseModel):
|
||||
"""Paginated edge list."""
|
||||
edges: List[EdgeResponse]
|
||||
total: int
|
||||
skip: int = 0
|
||||
limit: int = 100
|
||||
|
||||
|
||||
class NeighborResponse(BaseModel):
|
||||
"""Neighbor node with relationship info."""
|
||||
id: str
|
||||
type: str
|
||||
content: str = ""
|
||||
relationship: str = ""
|
||||
weight: float = 1.0
|
||||
hop: int = 1
|
||||
|
||||
|
||||
class PathResponse(BaseModel):
|
||||
"""Path between two nodes."""
|
||||
source: str
|
||||
target: str
|
||||
algorithm: str
|
||||
path: List[str]
|
||||
total_weight: float = 0.0
|
||||
|
||||
|
||||
class GraphStatsResponse(BaseModel):
|
||||
"""Graph-level statistics."""
|
||||
node_count: int
|
||||
edge_count: int
|
||||
node_types: Dict[str, int] = Field(default_factory=dict)
|
||||
edge_types: Dict[str, int] = Field(default_factory=dict)
|
||||
density: float = 0.0
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
"""Search request body."""
|
||||
query: str
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
limit: int = 20
|
||||
|
||||
|
||||
class SearchResultItem(BaseModel):
|
||||
"""Single search result."""
|
||||
node: NodeResponse
|
||||
score: float = 0.0
|
||||
|
||||
|
||||
class SearchResultResponse(BaseModel):
|
||||
"""Search results."""
|
||||
results: List[SearchResultItem]
|
||||
total: int
|
||||
query: str
|
||||
|
||||
|
||||
|
||||
|
||||
class AnalyticsResponse(BaseModel):
|
||||
"""Analytics results."""
|
||||
centrality: Optional[Dict[str, Any]] = None
|
||||
community: Optional[Dict[str, Any]] = None
|
||||
connectivity: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class ValidationIssue(BaseModel):
|
||||
"""Single validation error or warning."""
|
||||
severity: str # "error" or "warning"
|
||||
message: str
|
||||
node_id: Optional[str] = None
|
||||
edge_source: Optional[str] = None
|
||||
edge_target: Optional[str] = None
|
||||
|
||||
|
||||
class ValidationReportResponse(BaseModel):
|
||||
"""Graph validation report."""
|
||||
valid: bool
|
||||
error_count: int = 0
|
||||
warning_count: int = 0
|
||||
issues: List[ValidationIssue] = Field(default_factory=list)
|
||||
|
||||
|
||||
|
||||
class DecisionResponse(BaseModel):
|
||||
"""Single decision."""
|
||||
decision_id: str
|
||||
category: str = ""
|
||||
scenario: str = ""
|
||||
reasoning: str = ""
|
||||
outcome: str = ""
|
||||
confidence: float = 0.0
|
||||
timestamp: Optional[str] = None
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class CausalChainResponse(BaseModel):
|
||||
"""Causal chain for a decision."""
|
||||
decision_id: str
|
||||
chain: List[Dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ComplianceResponse(BaseModel):
|
||||
"""Policy compliance check result."""
|
||||
decision_id: str
|
||||
compliant: bool = True
|
||||
violations: List[Dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
|
||||
class TemporalSnapshotResponse(BaseModel):
|
||||
"""Graph state at a point in time."""
|
||||
timestamp: str
|
||||
active_nodes: List[NodeResponse]
|
||||
active_node_count: int
|
||||
|
||||
|
||||
class TemporalDiffResponse(BaseModel):
|
||||
"""Diff between two temporal snapshots."""
|
||||
from_time: str
|
||||
to_time: str
|
||||
added_nodes: List[str] = Field(default_factory=list)
|
||||
removed_nodes: List[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class TemporalPatternResponse(BaseModel):
|
||||
"""Detected temporal patterns."""
|
||||
patterns: List[Dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
class EnrichExtractRequest(BaseModel):
|
||||
"""Entity/relation extraction from text."""
|
||||
text: str
|
||||
|
||||
|
||||
class EnrichExtractResponse(BaseModel):
|
||||
"""Extraction results."""
|
||||
entities: List[Dict[str, Any]] = Field(default_factory=list)
|
||||
relations: List[Dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
class LinkPredictionRequest(BaseModel):
|
||||
"""Link prediction request."""
|
||||
node_id: str
|
||||
top_n: int = 10
|
||||
|
||||
|
||||
class LinkPredictionResponse(BaseModel):
|
||||
"""Link prediction results."""
|
||||
node_id: str
|
||||
predictions: List[Dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
class DedupRequest(BaseModel):
|
||||
"""Deduplication scan request."""
|
||||
threshold: float = 0.8
|
||||
|
||||
|
||||
class DedupResponse(BaseModel):
|
||||
"""Deduplication results."""
|
||||
duplicates: List[Dict[str, Any]] = Field(default_factory=list)
|
||||
total_flagged: int = 0
|
||||
|
||||
|
||||
class ReasoningRequest(BaseModel):
|
||||
"""Reasoning request."""
|
||||
facts: List[str]
|
||||
rules: List[str]
|
||||
mode: str = "forward" # forward, backward, rete
|
||||
|
||||
|
||||
class ReasoningResponse(BaseModel):
|
||||
"""Reasoning results."""
|
||||
inferred_facts: List[str] = Field(default_factory=list)
|
||||
rules_fired: int = 0
|
||||
|
||||
|
||||
|
||||
|
||||
class ExportRequest(BaseModel):
|
||||
"""Export request."""
|
||||
format: str = "json"
|
||||
node_ids: Optional[List[str]] = None
|
||||
|
||||
|
||||
class ExportResponse(BaseModel):
|
||||
"""Export result metadata."""
|
||||
format: str
|
||||
content_type: str
|
||||
filename: str
|
||||
size_bytes: int = 0
|
||||
|
||||
|
||||
|
||||
|
||||
class AnnotationCreate(BaseModel):
|
||||
"""Create an annotation."""
|
||||
node_id: str
|
||||
content: str
|
||||
tags: List[str] = Field(default_factory=list)
|
||||
visibility: str = "public"
|
||||
|
||||
|
||||
class AnnotationResponse(BaseModel):
|
||||
"""Single annotation."""
|
||||
annotation_id: str
|
||||
node_id: str
|
||||
content: str
|
||||
tags: List[str] = Field(default_factory=list)
|
||||
visibility: str = "public"
|
||||
created_at: str = ""
|
||||
@@ -0,0 +1,316 @@
|
||||
"""
|
||||
Semantica Explorer : Graph Session
|
||||
|
||||
Holds a loaded ContextGraph together with lazily-initialized analytics
|
||||
components. One session is created at server startup and shared across
|
||||
all API requests via FastAPI's dependency injection.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..context.context_graph import ContextGraph
|
||||
|
||||
_KG_AVAILABLE = False
|
||||
try:
|
||||
from ..kg import (
|
||||
CentralityCalculator,
|
||||
CommunityDetector,
|
||||
ConnectivityAnalyzer,
|
||||
GraphValidator,
|
||||
LinkPredictor,
|
||||
NodeEmbedder,
|
||||
PathFinder,
|
||||
SimilarityCalculator,
|
||||
)
|
||||
_KG_AVAILABLE = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
class GraphSession:
|
||||
"""
|
||||
Holds a loaded graph and its associated analytics components.
|
||||
|
||||
Thread safety: all mutations to the graph or the annotations store
|
||||
must go through methods on this class, which are protected by an
|
||||
``RLock``. Lazy analytics properties are also initialised under the
|
||||
same lock to prevent double-instantiation under concurrent requests.
|
||||
"""
|
||||
|
||||
def __init__(self, graph: ContextGraph) -> None:
|
||||
self.graph = graph
|
||||
self._lock = threading.RLock()
|
||||
|
||||
self.annotations: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
self._centrality: Any = None
|
||||
self._community: Any = None
|
||||
self._connectivity: Any = None
|
||||
self._path_finder: Any = None
|
||||
self._node_embedder: Any = None
|
||||
self._similarity: Any = None
|
||||
self._link_predictor: Any = None
|
||||
self._validator: Any = None
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: str) -> "GraphSession":
|
||||
"""Load a ContextGraph from a JSON file and wrap it in a session."""
|
||||
graph = ContextGraph()
|
||||
graph.load_from_file(path)
|
||||
return cls(graph)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lazy analytics properties (thread-safe double-checked locking)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def centrality(self) -> Any:
|
||||
with self._lock:
|
||||
if self._centrality is None and _KG_AVAILABLE:
|
||||
self._centrality = CentralityCalculator()
|
||||
return self._centrality
|
||||
|
||||
@property
|
||||
def community(self) -> Any:
|
||||
with self._lock:
|
||||
if self._community is None and _KG_AVAILABLE:
|
||||
self._community = CommunityDetector()
|
||||
return self._community
|
||||
|
||||
@property
|
||||
def connectivity(self) -> Any:
|
||||
with self._lock:
|
||||
if self._connectivity is None and _KG_AVAILABLE:
|
||||
self._connectivity = ConnectivityAnalyzer()
|
||||
return self._connectivity
|
||||
|
||||
@property
|
||||
def path_finder(self) -> Any:
|
||||
with self._lock:
|
||||
if self._path_finder is None and _KG_AVAILABLE:
|
||||
self._path_finder = PathFinder()
|
||||
return self._path_finder
|
||||
|
||||
@property
|
||||
def node_embedder(self) -> Any:
|
||||
with self._lock:
|
||||
if self._node_embedder is None and _KG_AVAILABLE:
|
||||
self._node_embedder = NodeEmbedder()
|
||||
return self._node_embedder
|
||||
|
||||
@property
|
||||
def similarity(self) -> Any:
|
||||
with self._lock:
|
||||
if self._similarity is None and _KG_AVAILABLE:
|
||||
self._similarity = SimilarityCalculator()
|
||||
return self._similarity
|
||||
|
||||
@property
|
||||
def link_predictor(self) -> Any:
|
||||
with self._lock:
|
||||
if self._link_predictor is None and _KG_AVAILABLE:
|
||||
self._link_predictor = LinkPredictor()
|
||||
return self._link_predictor
|
||||
|
||||
@property
|
||||
def validator(self) -> Any:
|
||||
with self._lock:
|
||||
if self._validator is None and _KG_AVAILABLE:
|
||||
self._validator = GraphValidator()
|
||||
return self._validator
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Graph read helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_node(self, node_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get a single node by ID, or ``None``."""
|
||||
with self._lock:
|
||||
return self.graph.find_node(node_id)
|
||||
|
||||
def get_nodes(
|
||||
self,
|
||||
node_type: Optional[str] = None,
|
||||
search: Optional[str] = None,
|
||||
skip: int = 0,
|
||||
limit: int = 100,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Return a paginated slice of nodes and the total count."""
|
||||
with self._lock:
|
||||
if search:
|
||||
# Must load all nodes to apply the in-memory keyword filter.
|
||||
all_nodes = self.graph.find_nodes(node_type=node_type)
|
||||
search_lower = search.lower()
|
||||
all_nodes = [
|
||||
n for n in all_nodes
|
||||
if search_lower in n.get("id", "").lower()
|
||||
or search_lower in n.get("content", "").lower()
|
||||
or search_lower in str(n.get("metadata", {})).lower()
|
||||
]
|
||||
total = len(all_nodes)
|
||||
page = all_nodes[skip: skip + limit]
|
||||
else:
|
||||
# Delegate pagination to the graph layer to avoid loading
|
||||
# the full node list into memory unnecessarily.
|
||||
stats = self.graph.stats()
|
||||
total = (
|
||||
stats.get("node_types", {}).get(node_type, 0)
|
||||
if node_type
|
||||
else stats.get("node_count", 0)
|
||||
)
|
||||
page = self.graph.find_nodes(node_type=node_type, skip=skip, limit=limit)
|
||||
return page, total
|
||||
|
||||
def get_edges(
|
||||
self,
|
||||
edge_type: Optional[str] = None,
|
||||
source: Optional[str] = None,
|
||||
target: Optional[str] = None,
|
||||
skip: int = 0,
|
||||
limit: int = 100,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Return a paginated slice of edges and the total count."""
|
||||
with self._lock:
|
||||
if source or target:
|
||||
# Must load all edges to apply source/target filters in memory.
|
||||
all_edges = self.graph.find_edges(edge_type=edge_type)
|
||||
if source:
|
||||
all_edges = [e for e in all_edges if e.get("source") == source]
|
||||
if target:
|
||||
all_edges = [e for e in all_edges if e.get("target") == target]
|
||||
total = len(all_edges)
|
||||
page = all_edges[skip: skip + limit]
|
||||
else:
|
||||
stats = self.graph.stats()
|
||||
total = (
|
||||
stats.get("edge_types", {}).get(edge_type, 0)
|
||||
if edge_type
|
||||
else stats.get("edge_count", 0)
|
||||
)
|
||||
page = self.graph.find_edges(edge_type=edge_type, skip=skip, limit=limit)
|
||||
return page, total
|
||||
|
||||
def get_neighbors(self, node_id: str, depth: int = 1) -> List[Dict[str, Any]]:
|
||||
"""Get neighbours for a node (BFS). Returns [] for unknown nodes."""
|
||||
with self._lock:
|
||||
return self.graph.get_neighbors(node_id, hops=depth)
|
||||
|
||||
def search(self, query: str, limit: int = 20) -> List[Dict[str, Any]]:
|
||||
"""Keyword search across node content.
|
||||
|
||||
``ContextGraph.query`` returns ``{"node": node.to_dict(), "score": …}``
|
||||
where ``node.to_dict()`` uses a ``"properties"`` envelope. We normalise
|
||||
each result to the flat ``{"id", "type", "content", "metadata"}`` shape
|
||||
that the rest of the session/route layer expects.
|
||||
"""
|
||||
with self._lock:
|
||||
raw = self.graph.query(query)[:limit]
|
||||
|
||||
normalised = []
|
||||
for r in raw:
|
||||
node = r.get("node", {})
|
||||
props = node.get("properties", {})
|
||||
flat_node = {
|
||||
"id": node.get("id", ""),
|
||||
"type": node.get("type", "entity"),
|
||||
"content": props.get("content", node.get("content", "")),
|
||||
"metadata": {k: v for k, v in props.items() if k != "content"},
|
||||
}
|
||||
normalised.append({"node": flat_node, "score": r.get("score", 0.0)})
|
||||
return normalised
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""Graph-level statistics."""
|
||||
with self._lock:
|
||||
return self.graph.stats()
|
||||
|
||||
def get_active_nodes(
|
||||
self, at_time: Optional[datetime] = None, node_type: Optional[str] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Nodes active at a given point in time."""
|
||||
with self._lock:
|
||||
return self.graph.find_active_nodes(node_type=node_type, at_time=at_time)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Annotation CRUD
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_annotation(self, annotation: Dict[str, Any]) -> str:
|
||||
"""Add an annotation (mutates the dict in-place) and return its ID."""
|
||||
ann_id = str(uuid.uuid4())
|
||||
annotation["annotation_id"] = ann_id
|
||||
annotation["created_at"] = datetime.utcnow().isoformat()
|
||||
with self._lock:
|
||||
self.annotations[ann_id] = annotation
|
||||
return ann_id
|
||||
|
||||
def get_annotations(self, node_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""List annotations, optionally filtered by node_id."""
|
||||
with self._lock:
|
||||
anns = list(self.annotations.values())
|
||||
if node_id:
|
||||
anns = [a for a in anns if a.get("node_id") == node_id]
|
||||
return anns
|
||||
|
||||
def delete_annotation(self, annotation_id: str) -> bool:
|
||||
"""Delete an annotation. Returns True if found and deleted."""
|
||||
with self._lock:
|
||||
return self.annotations.pop(annotation_id, None) is not None
|
||||
|
||||
def build_graph_dict(self, node_ids: Optional[list] = None) -> dict:
|
||||
"""
|
||||
Build the ``{entities, relationships}`` dict consumed by KG analytics
|
||||
helpers, exporters, and path-finders.
|
||||
|
||||
Args:
|
||||
node_ids: Optional list of node IDs to include. When given, only
|
||||
nodes in the list and edges between them are returned.
|
||||
"""
|
||||
nodes, _ = self.get_nodes(skip=0, limit=999_999)
|
||||
edges, _ = self.get_edges(skip=0, limit=999_999)
|
||||
|
||||
if node_ids:
|
||||
id_set = set(node_ids)
|
||||
nodes = [n for n in nodes if n.get("id") in id_set]
|
||||
edges = [
|
||||
e for e in edges
|
||||
if e.get("source") in id_set and e.get("target") in id_set
|
||||
]
|
||||
|
||||
return {
|
||||
"entities": [
|
||||
{
|
||||
"id": n.get("id"),
|
||||
"type": n.get("type", "entity"),
|
||||
"text": n.get("content", n.get("id", "")),
|
||||
"metadata": n.get("metadata", {}),
|
||||
}
|
||||
for n in nodes
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source": e.get("source"),
|
||||
"target": e.get("target"),
|
||||
"type": e.get("type", "related_to"),
|
||||
"metadata": e.get("metadata", {}),
|
||||
}
|
||||
for e in edges
|
||||
],
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Graph mutation helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_nodes(self, nodes: List[Dict[str, Any]]) -> int:
|
||||
"""Thread-safe node addition."""
|
||||
with self._lock:
|
||||
return self.graph.add_nodes(nodes)
|
||||
|
||||
def add_edges(self, edges: List[Dict[str, Any]]) -> int:
|
||||
"""Thread-safe edge addition."""
|
||||
with self._lock:
|
||||
return self.graph.add_edges(edges)
|
||||
@@ -0,0 +1,91 @@
|
||||
"""
|
||||
Semantica Explorer : WebSocket Connection Manager
|
||||
|
||||
Manages WebSocket connections for real-time graph updates,
|
||||
import progress events, and mutation broadcasts.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Set
|
||||
|
||||
from fastapi import WebSocket
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
"""
|
||||
Thread-safe WebSocket connection manager.
|
||||
|
||||
Maintains a set of active WebSocket connections and provides
|
||||
methods to broadcast events to all connected clients.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._active_connections: Set[WebSocket] = set()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
async def connect(self, websocket: WebSocket) -> None:
|
||||
"""Accept a WebSocket connection and add it to the active set."""
|
||||
await websocket.accept()
|
||||
with self._lock:
|
||||
self._active_connections.add(websocket)
|
||||
|
||||
def disconnect(self, websocket: WebSocket) -> None:
|
||||
"""Remove a WebSocket connection from the active set."""
|
||||
with self._lock:
|
||||
self._active_connections.discard(websocket)
|
||||
|
||||
@property
|
||||
def active_count(self) -> int:
|
||||
"""Number of active connections."""
|
||||
with self._lock:
|
||||
return len(self._active_connections)
|
||||
|
||||
async def broadcast(self, event_type: str, data: Any = None) -> None:
|
||||
"""
|
||||
Broadcast a JSON message to all connected clients.
|
||||
|
||||
Args:
|
||||
event_type: Event type string (e.g., "node_added", "import_progress").
|
||||
data: Arbitrary JSON-serialisable payload.
|
||||
"""
|
||||
message = json.dumps(
|
||||
{
|
||||
"event": event_type,
|
||||
"data": data,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
default=str,
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
connections = set(self._active_connections)
|
||||
|
||||
|
||||
disconnected: list[WebSocket] = []
|
||||
for ws in connections:
|
||||
try:
|
||||
await ws.send_text(message)
|
||||
except Exception:
|
||||
disconnected.append(ws)
|
||||
|
||||
if disconnected:
|
||||
with self._lock:
|
||||
for ws in disconnected:
|
||||
self._active_connections.discard(ws)
|
||||
|
||||
async def send_personal(
|
||||
self, websocket: WebSocket, event_type: str, data: Any = None
|
||||
) -> None:
|
||||
"""Send a message to a single client."""
|
||||
message = json.dumps(
|
||||
{
|
||||
"event": event_type,
|
||||
"data": data,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
},
|
||||
default=str,
|
||||
)
|
||||
await websocket.send_text(message)
|
||||
@@ -34,10 +34,6 @@ from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import sqlalchemy
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
@@ -101,13 +97,13 @@ class DatabaseConnector:
|
||||
self.logger = get_logger("database_connector")
|
||||
self.db_type = db_type.lower() if db_type else ""
|
||||
self.config = config
|
||||
self.engine: Optional[Engine] = None
|
||||
self.engine: Optional[Any] = None
|
||||
|
||||
self.logger.debug(
|
||||
f"Database connector initialized: db_type={db_type or 'auto-detect'}"
|
||||
)
|
||||
|
||||
def connect(self, connection_string: str) -> Engine:
|
||||
def connect(self, connection_string: str) -> Any:
|
||||
"""
|
||||
Establish database connection.
|
||||
|
||||
@@ -129,6 +125,14 @@ class DatabaseConnector:
|
||||
ProcessingError: If connection fails or database type is unsupported
|
||||
"""
|
||||
try:
|
||||
try:
|
||||
from sqlalchemy import create_engine, text
|
||||
except ImportError:
|
||||
raise ProcessingError(
|
||||
"sqlalchemy is required for database ingestion. "
|
||||
"Install with: pip install sqlalchemy"
|
||||
)
|
||||
|
||||
# Parse connection string to detect database type
|
||||
parsed = urlparse(connection_string)
|
||||
|
||||
@@ -188,6 +192,7 @@ class DatabaseConnector:
|
||||
bool: True if connection successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
from sqlalchemy import create_engine, text
|
||||
engine = create_engine(connection_string)
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("SELECT 1"))
|
||||
@@ -226,7 +231,7 @@ class DataExporter:
|
||||
|
||||
def export_table_data(
|
||||
self,
|
||||
connection: Engine,
|
||||
connection: Any,
|
||||
table_name: str,
|
||||
schema: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
@@ -264,6 +269,7 @@ class DataExporter:
|
||||
ProcessingError: If table export fails
|
||||
"""
|
||||
try:
|
||||
from sqlalchemy import inspect
|
||||
inspector = inspect(connection)
|
||||
|
||||
# Get column information
|
||||
@@ -379,7 +385,7 @@ class DataExporter:
|
||||
return transformed
|
||||
|
||||
def export_schema(
|
||||
self, connection: Engine, schema: Optional[str] = None
|
||||
self, connection: Any, schema: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Export database schema information.
|
||||
@@ -406,6 +412,7 @@ class DataExporter:
|
||||
ProcessingError: If schema export fails
|
||||
"""
|
||||
try:
|
||||
from sqlalchemy import inspect
|
||||
inspector = inspect(connection)
|
||||
|
||||
schema_info = {"tables": [], "views": [], "foreign_keys": []}
|
||||
@@ -591,6 +598,7 @@ class DBIngestor:
|
||||
schema = self.analyze_schema(connection_string)
|
||||
|
||||
# Get all table names
|
||||
from sqlalchemy import inspect
|
||||
inspector = inspect(engine)
|
||||
all_tables = inspector.get_table_names()
|
||||
|
||||
|
||||
@@ -117,6 +117,7 @@ from .link_predictor import LinkPredictor
|
||||
from .node_embeddings import NodeEmbedder
|
||||
from .path_finder import PathFinder
|
||||
from .kg_provenance import GraphBuilderWithProvenance, AlgorithmTrackerWithProvenance
|
||||
from .provenance_tracker import ProvenanceTracker
|
||||
from .registry import MethodRegistry, method_registry, AlgorithmRegistry, algorithm_registry
|
||||
from .seed_manager import SeedManager
|
||||
from .similarity_calculator import SimilarityCalculator
|
||||
@@ -125,6 +126,8 @@ from .temporal_query import (
|
||||
TemporalPatternDetector,
|
||||
TemporalVersionManager,
|
||||
)
|
||||
from .temporal_model import BiTemporalFact, TemporalBound
|
||||
from .temporal_normalizer import TemporalNormalizer
|
||||
|
||||
__all__ = [
|
||||
# Core Classes
|
||||
@@ -136,7 +139,11 @@ __all__ = [
|
||||
"TemporalGraphQuery",
|
||||
"TemporalPatternDetector",
|
||||
"TemporalVersionManager",
|
||||
"TemporalBound",
|
||||
"BiTemporalFact",
|
||||
"TemporalNormalizer",
|
||||
"AlgorithmTrackerWithProvenance",
|
||||
"ProvenanceTracker",
|
||||
# Enhanced Graph Algorithms
|
||||
"NodeEmbedder",
|
||||
"SimilarityCalculator",
|
||||
|
||||
@@ -757,7 +757,8 @@ class CentralityCalculator:
|
||||
) -> List[str]:
|
||||
"""Get neighbors filtered by relationship types."""
|
||||
if hasattr(graph, 'neighbors'):
|
||||
neighbors = list(graph.neighbors(node))
|
||||
_raw = list(graph.neighbors(node))
|
||||
neighbors = [n.get("id") if isinstance(n, dict) else n for n in _raw]
|
||||
elif hasattr(graph, 'get_neighbors'):
|
||||
neighbors = graph.get_neighbors(node)
|
||||
if neighbors and isinstance(neighbors[0], dict):
|
||||
|
||||
@@ -398,7 +398,11 @@ class LinkPredictor:
|
||||
if hasattr(graph_store, 'get_nodes_by_label') and callable(graph_store.get_nodes_by_label):
|
||||
result = graph_store.get_nodes_by_label(label)
|
||||
if isinstance(result, list):
|
||||
nodes.extend(result)
|
||||
nodes.extend(
|
||||
item.get("id") if isinstance(item, dict) else item
|
||||
for item in result
|
||||
if item and (not isinstance(item, dict) or item.get("id"))
|
||||
)
|
||||
else:
|
||||
# Fallback - get all nodes and filter by label if possible
|
||||
all_nodes = self._get_all_nodes(graph_store)
|
||||
@@ -500,7 +504,7 @@ class LinkPredictor:
|
||||
return neighbors
|
||||
if hasattr(graph_store, 'neighbors') and callable(graph_store.neighbors):
|
||||
try:
|
||||
raw = list(graph_store.neighbors(node_id))
|
||||
raw = [n.get("id") if isinstance(n, dict) else n for n in graph_store.neighbors(node_id)]
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
if relationship_types and hasattr(graph_store, 'get_edge_data') and callable(graph_store.get_edge_data):
|
||||
|
||||
@@ -356,7 +356,13 @@ class NodeEmbedder:
|
||||
nodes = []
|
||||
if hasattr(graph_store, 'get_nodes_by_label'):
|
||||
for label in node_labels:
|
||||
nodes.extend(graph_store.get_nodes_by_label(label))
|
||||
for node in graph_store.get_nodes_by_label(label):
|
||||
if isinstance(node, dict):
|
||||
node_id = node.get("id")
|
||||
if node_id:
|
||||
nodes.append(node_id)
|
||||
elif node:
|
||||
nodes.append(node)
|
||||
else:
|
||||
# Fallback for different graph store implementations
|
||||
nodes = list(graph_store.nodes())
|
||||
@@ -365,7 +371,14 @@ class NodeEmbedder:
|
||||
for node in nodes:
|
||||
if hasattr(graph_store, 'get_neighbors'):
|
||||
try:
|
||||
neighbor_details = graph_store.get_neighbors(node, relationship_types)
|
||||
try:
|
||||
neighbor_details = graph_store.get_neighbors(
|
||||
node,
|
||||
relationship_types=relationship_types,
|
||||
)
|
||||
except TypeError:
|
||||
neighbor_details = graph_store.get_neighbors(node, relationship_types)
|
||||
|
||||
if isinstance(neighbor_details, list):
|
||||
adjacency[node] = [
|
||||
n.get("id") if isinstance(n, dict) else n
|
||||
@@ -380,7 +393,11 @@ class NodeEmbedder:
|
||||
adjacency[node] = list(graph_store.get_neighbor_ids(node, relationship_types))
|
||||
elif hasattr(graph_store, 'neighbors'):
|
||||
try:
|
||||
adjacency[node] = list(graph_store.neighbors(node))
|
||||
adjacency[node] = [
|
||||
n.get("id") if isinstance(n, dict) else n
|
||||
for n in graph_store.neighbors(node)
|
||||
if n
|
||||
]
|
||||
except TypeError:
|
||||
adjacency[node] = []
|
||||
else:
|
||||
|
||||
@@ -569,7 +569,8 @@ class PathFinder:
|
||||
neighbors = []
|
||||
|
||||
if hasattr(graph, 'neighbors'):
|
||||
for neighbor in graph.neighbors(node):
|
||||
for _raw in graph.neighbors(node):
|
||||
neighbor = _raw.get("id") if isinstance(_raw, dict) else _raw
|
||||
edge_data = self._get_edge_data(graph, node, neighbor)
|
||||
neighbors.append((neighbor, edge_data))
|
||||
elif hasattr(graph, 'get_neighbors'):
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
"""
|
||||
Temporal data model helpers for knowledge graph relationships.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from ..utils.exceptions import TemporalValidationError
|
||||
|
||||
|
||||
class TemporalBound(Enum):
|
||||
"""Sentinel bounds for open-ended temporal intervals."""
|
||||
|
||||
OPEN = "OPEN"
|
||||
|
||||
|
||||
def _default_recorded_at() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BiTemporalFact:
|
||||
"""
|
||||
Backward-compatible wrapper around existing relationship dictionaries.
|
||||
|
||||
Design note:
|
||||
Facts continue to live as plain relationship dicts in the graph. This wrapper
|
||||
is only used internally for normalization so existing callers can keep
|
||||
reading and writing `valid_from` / `valid_until` directly.
|
||||
"""
|
||||
|
||||
valid_from: Optional[datetime]
|
||||
valid_until: Optional[datetime | TemporalBound]
|
||||
recorded_at: datetime = field(default_factory=_default_recorded_at)
|
||||
superseded_at: datetime | TemporalBound = TemporalBound.OPEN
|
||||
|
||||
@classmethod
|
||||
def from_relationship(cls, relationship: Dict[str, Any]) -> "BiTemporalFact":
|
||||
valid_until_raw = relationship.get("valid_until", TemporalBound.OPEN)
|
||||
if valid_until_raw is None:
|
||||
valid_until_raw = TemporalBound.OPEN
|
||||
|
||||
valid_from = parse_temporal_value(relationship.get("valid_from"))
|
||||
recorded_at_raw = relationship.get("recorded_at")
|
||||
superseded_at_raw = relationship.get("superseded_at", TemporalBound.OPEN)
|
||||
|
||||
return cls(
|
||||
valid_from=valid_from,
|
||||
valid_until=parse_temporal_bound(valid_until_raw),
|
||||
recorded_at=parse_temporal_value(recorded_at_raw) if recorded_at_raw is not None else (valid_from or _default_recorded_at()),
|
||||
superseded_at=parse_temporal_bound(superseded_at_raw, default=TemporalBound.OPEN),
|
||||
)
|
||||
|
||||
def to_relationship_fields(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"valid_from": serialize_temporal_value(self.valid_from),
|
||||
"valid_until": serialize_temporal_bound(self.valid_until),
|
||||
"recorded_at": serialize_temporal_value(self.recorded_at),
|
||||
"superseded_at": serialize_temporal_bound(self.superseded_at),
|
||||
}
|
||||
|
||||
|
||||
def _coerce_iso_like_string(value: str) -> str:
|
||||
match = re.match(
|
||||
r"^(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})(?P<rest>.*)$",
|
||||
value.strip(),
|
||||
)
|
||||
if not match:
|
||||
return value.strip()
|
||||
|
||||
month = int(match.group("month"))
|
||||
day = int(match.group("day"))
|
||||
rest = match.group("rest")
|
||||
return f"{match.group('year')}-{month:02d}-{day:02d}{rest}"
|
||||
|
||||
|
||||
def parse_temporal_value(value: Any) -> Optional[datetime]:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
dt = value
|
||||
elif isinstance(value, (int, float)):
|
||||
dt = datetime.fromtimestamp(value, timezone.utc)
|
||||
elif isinstance(value, str):
|
||||
normalized = _coerce_iso_like_string(value)
|
||||
if normalized.endswith("Z"):
|
||||
normalized = normalized[:-1] + "+00:00"
|
||||
try:
|
||||
dt = datetime.fromisoformat(normalized)
|
||||
except ValueError as exc:
|
||||
raise TemporalValidationError(
|
||||
"Invalid temporal value",
|
||||
temporal_context={"value": value},
|
||||
) from exc
|
||||
else:
|
||||
raise TemporalValidationError(
|
||||
"Unsupported temporal value type",
|
||||
temporal_context={"value": value, "type": type(value).__name__},
|
||||
)
|
||||
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def parse_temporal_bound(
|
||||
value: Any,
|
||||
*,
|
||||
default: Optional[datetime | TemporalBound] = None,
|
||||
) -> Optional[datetime | TemporalBound]:
|
||||
if value is None:
|
||||
return default
|
||||
if value == TemporalBound.OPEN or value == TemporalBound.OPEN.value:
|
||||
return TemporalBound.OPEN
|
||||
return parse_temporal_value(value)
|
||||
|
||||
|
||||
def serialize_temporal_value(value: Optional[datetime]) -> Optional[str]:
|
||||
if value is None:
|
||||
return None
|
||||
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def serialize_temporal_bound(value: Optional[datetime | TemporalBound]) -> Optional[str]:
|
||||
if value in (None, TemporalBound.OPEN):
|
||||
return None
|
||||
return serialize_temporal_value(value)
|
||||
|
||||
|
||||
def deserialize_relationship_temporal_fields(relationship: Dict[str, Any]) -> Dict[str, Any]:
|
||||
normalized = dict(relationship)
|
||||
fact = BiTemporalFact.from_relationship(normalized)
|
||||
normalized.update(fact.to_relationship_fields())
|
||||
if fact.valid_until is TemporalBound.OPEN:
|
||||
normalized["valid_until"] = TemporalBound.OPEN
|
||||
if fact.superseded_at is TemporalBound.OPEN:
|
||||
normalized["superseded_at"] = TemporalBound.OPEN
|
||||
return normalized
|
||||
|
||||
|
||||
def relationship_to_json_ready(relationship: Dict[str, Any]) -> Dict[str, Any]:
|
||||
json_ready = dict(relationship)
|
||||
for field in ("valid_from", "recorded_at"):
|
||||
if field in json_ready:
|
||||
json_ready[field] = serialize_temporal_value(parse_temporal_value(json_ready[field]))
|
||||
for field in ("valid_until", "superseded_at"):
|
||||
if field in json_ready:
|
||||
json_ready[field] = serialize_temporal_bound(parse_temporal_bound(json_ready[field]))
|
||||
return json_ready
|
||||
|
||||
|
||||
def temporal_structure_to_json_ready(value: Any) -> Any:
|
||||
"""Recursively convert temporal values into JSON-safe primitives."""
|
||||
if isinstance(value, dict):
|
||||
return {key: temporal_structure_to_json_ready(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [temporal_structure_to_json_ready(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return [temporal_structure_to_json_ready(item) for item in value]
|
||||
if value is TemporalBound.OPEN:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return serialize_temporal_value(value)
|
||||
return value
|
||||
|
||||
|
||||
def dumps_relationship_json(relationship: Dict[str, Any]) -> str:
|
||||
return json.dumps(relationship_to_json_ready(relationship))
|
||||
@@ -0,0 +1,391 @@
|
||||
"""
|
||||
Temporal Normalizer
|
||||
|
||||
Deterministic resolution of temporal phrases extracted from text into
|
||||
UTC datetime intervals. Zero LLM calls — pure regex and date arithmetic.
|
||||
|
||||
Usage::
|
||||
|
||||
from semantica.kg import TemporalNormalizer
|
||||
from datetime import datetime, timezone
|
||||
|
||||
tn = TemporalNormalizer(reference_date=datetime(2025, 3, 25, tzinfo=timezone.utc))
|
||||
start, end = tn.normalize("Q2 2021") # → (2021-04-01, 2021-06-30)
|
||||
start, end = tn.normalize("last year") # → (2024-01-01, 2024-12-31)
|
||||
info = tn.normalize_phrase("expiry date") # → {"maps_to": "valid_until", ...}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import calendar
|
||||
import logging
|
||||
import re
|
||||
import warnings
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Callable, Dict, Optional, Tuple
|
||||
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
from ..utils.exceptions import TemporalAmbiguityWarning
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Compiled regex patterns for structured date formats
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_RE_YEAR_ONLY = re.compile(r"^\s*(\d{4})\s*$")
|
||||
_RE_MONTH_YEAR_WORD = re.compile(
|
||||
r"^\s*(january|february|march|april|may|june|july|august|september|october|november|december|"
|
||||
r"jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+(\d{4})\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_RE_YEAR_MONTH_ISO = re.compile(r"^\s*(\d{4})-(\d{1,2})\s*$")
|
||||
_RE_QUARTER = re.compile(r"^\s*Q([1-4])\s+(\d{4})\s*$", re.IGNORECASE)
|
||||
_RE_AMBIGUOUS_SLASH = re.compile(r"^\s*\d{1,2}/\d{1,2}/\d{4}\s*$")
|
||||
|
||||
_MONTH_NAMES: Dict[str, int] = {
|
||||
"january": 1, "jan": 1,
|
||||
"february": 2, "feb": 2,
|
||||
"march": 3, "mar": 3,
|
||||
"april": 4, "apr": 4,
|
||||
"may": 5,
|
||||
"june": 6, "jun": 6,
|
||||
"july": 7, "jul": 7,
|
||||
"august": 8, "aug": 8,
|
||||
"september": 9, "sep": 9,
|
||||
"october": 10, "oct": 10,
|
||||
"november": 11, "nov": 11,
|
||||
"december": 12, "dec": 12,
|
||||
}
|
||||
|
||||
_QUARTER_BOUNDS: Dict[int, Tuple[int, int, int, int]] = {
|
||||
# quarter → (from_month, from_day, until_month, until_day)
|
||||
1: (1, 1, 3, 31),
|
||||
2: (4, 1, 6, 30),
|
||||
3: (7, 1, 9, 30),
|
||||
4: (10, 1, 12, 31),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Small date-arithmetic helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _utc(year: int, month: int, day: int) -> datetime:
|
||||
return datetime(year, month, day, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _last_day_of_month(year: int, month: int) -> int:
|
||||
return calendar.monthrange(year, month)[1]
|
||||
|
||||
|
||||
def _this_quarter(ref: datetime) -> Tuple[datetime, datetime]:
|
||||
q = (ref.month - 1) // 3 + 1
|
||||
fm, fd, um, ud = _QUARTER_BOUNDS[q]
|
||||
return _utc(ref.year, fm, fd), _utc(ref.year, um, ud)
|
||||
|
||||
|
||||
def _last_quarter(ref: datetime) -> Tuple[datetime, datetime]:
|
||||
q = (ref.month - 1) // 3 + 1
|
||||
prev_q = q - 1 if q > 1 else 4
|
||||
year = ref.year if q > 1 else ref.year - 1
|
||||
fm, fd, um, ud = _QUARTER_BOUNDS[prev_q]
|
||||
return _utc(year, fm, fd), _utc(year, um, ud)
|
||||
|
||||
|
||||
def _last_month(ref: datetime) -> Tuple[datetime, datetime]:
|
||||
first = ref.replace(day=1) - relativedelta(months=1)
|
||||
last_day = _last_day_of_month(first.year, first.month)
|
||||
return _utc(first.year, first.month, 1), _utc(first.year, first.month, last_day)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default phrase map
|
||||
# ---------------------------------------------------------------------------
|
||||
# Keys: lowercase canonical phrases (or regex patterns prefixed with "r:").
|
||||
# Values: callables (ref: datetime) → (valid_from, valid_until).
|
||||
#
|
||||
# Domain-specific terms that carry no self-contained date (e.g. "approval date")
|
||||
# return (ref, ref) as a placeholder so callers can distinguish
|
||||
# "known temporal term, date needs context" from "unrecognised phrase".
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _phrase_entry(maps_to: str, type_: str, **extra: Any) -> Dict[str, Any]:
|
||||
return {"maps_to": maps_to, "type": type_, **extra}
|
||||
|
||||
|
||||
# Phrase map entries also carry metadata for normalize_phrase()
|
||||
_DEFAULT_PHRASE_META: Dict[str, Dict[str, Any]] = {
|
||||
# ── Relative references ─────────────────────────────────────────────
|
||||
"last year": _phrase_entry("valid_from", "relative"),
|
||||
"this year": _phrase_entry("valid_from", "relative"),
|
||||
"last quarter": _phrase_entry("valid_from", "relative"),
|
||||
"this quarter": _phrase_entry("valid_from", "relative"),
|
||||
"last month": _phrase_entry("valid_from", "relative"),
|
||||
"this month": _phrase_entry("valid_from", "relative"),
|
||||
"three months ago": _phrase_entry("valid_from", "relative"),
|
||||
"six months ago": _phrase_entry("valid_from", "relative"),
|
||||
"two years ago": _phrase_entry("valid_from", "relative"),
|
||||
# ── General / Policy ────────────────────────────────────────────────
|
||||
"r:effective\\s+(as\\s+of|from|beginning|date)":
|
||||
_phrase_entry("valid_from", "start", domain=["General", "Policy"]),
|
||||
"in force until":
|
||||
_phrase_entry("valid_until", "end", domain=["Policy", "Regulatory"]),
|
||||
"retroactive to":
|
||||
_phrase_entry("valid_from", "start", retroactive=True, domain=["Regulatory", "Finance"]),
|
||||
"sunset clause":
|
||||
_phrase_entry("valid_until", "sunset", domain=["Policy"]),
|
||||
# ── Healthcare / Drug Discovery ──────────────────────────────────────
|
||||
"approval date":
|
||||
_phrase_entry("valid_from", "start", domain=["Healthcare", "Drug Discovery"]),
|
||||
"expiry date":
|
||||
_phrase_entry("valid_until", "end", domain=["Healthcare", "Supply Chain"]),
|
||||
"market authorization":
|
||||
_phrase_entry("valid_from", "start", domain=["Drug Discovery", "Healthcare"]),
|
||||
# ── Cybersecurity ────────────────────────────────────────────────────
|
||||
"incident window":
|
||||
_phrase_entry("window", "window", domain=["Cybersecurity"]),
|
||||
"campaign period":
|
||||
_phrase_entry("window", "window", domain=["Cybersecurity"]),
|
||||
# ── Supply Chain ─────────────────────────────────────────────────────
|
||||
"certification valid through":
|
||||
_phrase_entry("valid_until", "end", domain=["Supply Chain"]),
|
||||
# ── Finance ──────────────────────────────────────────────────────────
|
||||
"trading halt":
|
||||
_phrase_entry("window", "window", domain=["Finance"]),
|
||||
# ── Energy ───────────────────────────────────────────────────────────
|
||||
"commissioned date":
|
||||
_phrase_entry("valid_from", "start", domain=["Energy"]),
|
||||
"decommissioned date":
|
||||
_phrase_entry("valid_until", "end", domain=["Energy"]),
|
||||
}
|
||||
|
||||
# Separate callable map for date resolution (subset of the above)
|
||||
def _build_default_callable_map() -> Dict[str, Callable[[datetime], Tuple[datetime, datetime]]]:
|
||||
return {
|
||||
"last year": lambda ref: (
|
||||
_utc(ref.year - 1, 1, 1),
|
||||
_utc(ref.year - 1, 12, 31),
|
||||
),
|
||||
"this year": lambda ref: (
|
||||
_utc(ref.year, 1, 1),
|
||||
_utc(ref.year, 12, 31),
|
||||
),
|
||||
"last quarter": _last_quarter,
|
||||
"this quarter": _this_quarter,
|
||||
"last month": _last_month,
|
||||
"this month": lambda ref: (
|
||||
_utc(ref.year, ref.month, 1),
|
||||
_utc(ref.year, ref.month, _last_day_of_month(ref.year, ref.month)),
|
||||
),
|
||||
"three months ago": lambda ref: (
|
||||
(ref - relativedelta(months=3)).replace(day=1, hour=0, minute=0, second=0, microsecond=0),
|
||||
((ref - relativedelta(months=3)).replace(day=1) + relativedelta(months=1) - relativedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0),
|
||||
),
|
||||
"six months ago": lambda ref: (
|
||||
(ref - relativedelta(months=6)).replace(day=1, hour=0, minute=0, second=0, microsecond=0),
|
||||
((ref - relativedelta(months=6)).replace(day=1) + relativedelta(months=1) - relativedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0),
|
||||
),
|
||||
"two years ago": lambda ref: (
|
||||
_utc(ref.year - 2, 1, 1),
|
||||
_utc(ref.year - 2, 12, 31),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TemporalNormalizer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TemporalNormalizer:
|
||||
"""
|
||||
Deterministic resolution of temporal phrases into UTC datetime intervals.
|
||||
|
||||
Zero LLM calls. All resolution is done via regex patterns and Python
|
||||
date arithmetic (``dateutil.relativedelta``).
|
||||
|
||||
Args:
|
||||
reference_date: Anchor for relative phrases like "last year". When
|
||||
``None`` and a relative phrase is encountered, :meth:`normalize`
|
||||
raises :class:`ValueError`.
|
||||
phrase_map: Optional dict that extends or overrides the default
|
||||
domain phrase map. Keys are lowercase phrases (or regex patterns
|
||||
prefixed with ``"r:"``). Values are callables
|
||||
``(reference_date: datetime) -> (start: datetime, end: datetime)``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
reference_date: Optional[datetime] = None,
|
||||
phrase_map: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
self.reference_date = reference_date
|
||||
# Build the callable resolution map (relative dates + user overrides)
|
||||
self._callable_map: Dict[str, Callable[[datetime], Tuple[datetime, datetime]]] = (
|
||||
_build_default_callable_map()
|
||||
)
|
||||
if phrase_map:
|
||||
self._callable_map.update(phrase_map)
|
||||
# Phrase metadata map (for normalize_phrase)
|
||||
self._phrase_meta: Dict[str, Dict[str, Any]] = {**_DEFAULT_PHRASE_META}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def normalize(self, value: Optional[str]) -> Optional[Tuple[datetime, datetime]]:
|
||||
"""
|
||||
Resolve a temporal string to a ``(valid_from, valid_until)`` interval.
|
||||
|
||||
Resolution order:
|
||||
1. ``None`` / empty → ``None``
|
||||
2. ISO 8601 full datetime / date → point interval ``(dt, dt)``
|
||||
3. Partial date patterns: year-only, month+year, quarter+year
|
||||
4. Ambiguous slash-date (``DD/MM/YYYY`` vs ``MM/DD/YYYY``) →
|
||||
issues :class:`~semantica.utils.exceptions.TemporalAmbiguityWarning`
|
||||
and returns ``None``
|
||||
5. Phrase map / domain phrase lookup
|
||||
6. Relative phrase via callable map (requires ``reference_date``)
|
||||
7. Unparseable → ``None`` (debug log, never raises)
|
||||
|
||||
Returns:
|
||||
Tuple of UTC datetimes ``(start, end)`` or ``None``.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
value_stripped = value.strip()
|
||||
if not value_stripped:
|
||||
return None
|
||||
|
||||
# 1. ISO 8601 parse
|
||||
iso_result = self._try_iso(value_stripped)
|
||||
if iso_result is not None:
|
||||
return iso_result
|
||||
|
||||
# 2. Partial date patterns
|
||||
partial_result = self._try_partial_date(value_stripped)
|
||||
if partial_result is not None:
|
||||
return partial_result
|
||||
|
||||
# 3. Ambiguous slash date — warn, return None
|
||||
if _RE_AMBIGUOUS_SLASH.match(value_stripped):
|
||||
warnings.warn(
|
||||
f"Temporal expression {value_stripped!r} is ambiguous (day/month ordering unknown). "
|
||||
"Provide locale or use ISO 8601 format (YYYY-MM-DD).",
|
||||
TemporalAmbiguityWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return None
|
||||
|
||||
# 4. Relative phrase / callable map
|
||||
callable_result = self._try_callable(value_stripped)
|
||||
if callable_result is not None:
|
||||
return callable_result
|
||||
|
||||
logger.debug("Could not parse temporal value: %r", value_stripped)
|
||||
return None
|
||||
|
||||
def normalize_phrase(self, phrase: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Look up a temporal phrase in the domain phrase map.
|
||||
|
||||
Checks exact match first, then regex patterns (keys prefixed with
|
||||
``"r:"``). Returns the metadata dict if matched, ``None`` otherwise.
|
||||
|
||||
Args:
|
||||
phrase: Lowercase phrase to look up (case-insensitive internally).
|
||||
|
||||
Returns:
|
||||
Dict with at minimum ``{"maps_to": ..., "type": ...}`` or ``None``.
|
||||
"""
|
||||
normalized = phrase.strip().lower()
|
||||
|
||||
# Exact match
|
||||
if normalized in self._phrase_meta:
|
||||
return self._phrase_meta[normalized]
|
||||
|
||||
# Regex pattern match (keys prefixed with "r:")
|
||||
for key, meta in self._phrase_meta.items():
|
||||
if key.startswith("r:"):
|
||||
pattern = key[2:]
|
||||
if re.search(pattern, normalized, re.IGNORECASE):
|
||||
return meta
|
||||
|
||||
logger.debug("Unrecognized temporal phrase: %r", phrase)
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _try_iso(self, value: str) -> Optional[Tuple[datetime, datetime]]:
|
||||
"""Attempt ISO 8601 parse. Returns point interval on success."""
|
||||
normalized = value
|
||||
if normalized.endswith("Z"):
|
||||
normalized = normalized[:-1] + "+00:00"
|
||||
try:
|
||||
dt = datetime.fromisoformat(normalized)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return (dt, dt)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def _try_partial_date(self, value: str) -> Optional[Tuple[datetime, datetime]]:
|
||||
"""Try partial date patterns: YYYY, Month YYYY, YYYY-MM, Q[1-4] YYYY."""
|
||||
# Year only
|
||||
m = _RE_YEAR_ONLY.match(value)
|
||||
if m:
|
||||
year = int(m.group(1))
|
||||
return _utc(year, 1, 1), _utc(year, 12, 31)
|
||||
|
||||
# Month YYYY (word)
|
||||
m = _RE_MONTH_YEAR_WORD.match(value)
|
||||
if m:
|
||||
month = _MONTH_NAMES[m.group(1).lower()]
|
||||
year = int(m.group(2))
|
||||
last = _last_day_of_month(year, month)
|
||||
return _utc(year, month, 1), _utc(year, month, last)
|
||||
|
||||
# YYYY-MM (ISO partial)
|
||||
m = _RE_YEAR_MONTH_ISO.match(value)
|
||||
if m:
|
||||
year, month = int(m.group(1)), int(m.group(2))
|
||||
if 1 <= month <= 12:
|
||||
last = _last_day_of_month(year, month)
|
||||
return _utc(year, month, 1), _utc(year, month, last)
|
||||
|
||||
# Q[1-4] YYYY
|
||||
m = _RE_QUARTER.match(value)
|
||||
if m:
|
||||
q, year = int(m.group(1)), int(m.group(2))
|
||||
fm, fd, um, ud = _QUARTER_BOUNDS[q]
|
||||
return _utc(year, fm, fd), _utc(year, um, ud)
|
||||
|
||||
return None
|
||||
|
||||
def _try_callable(self, value: str) -> Optional[Tuple[datetime, datetime]]:
|
||||
"""Try the relative phrase callable map."""
|
||||
key = value.lower()
|
||||
|
||||
# Exact match
|
||||
if key in self._callable_map:
|
||||
if self.reference_date is None:
|
||||
raise ValueError(
|
||||
f"reference_date is required to resolve relative temporal expression: {value!r}"
|
||||
)
|
||||
return self._callable_map[key](self.reference_date)
|
||||
|
||||
# Regex pattern match (keys prefixed with "r:")
|
||||
for map_key, fn in self._callable_map.items():
|
||||
if map_key.startswith("r:"):
|
||||
pattern = map_key[2:]
|
||||
if re.search(pattern, key, re.IGNORECASE):
|
||||
if self.reference_date is None:
|
||||
raise ValueError(
|
||||
f"reference_date is required to resolve relative temporal expression: {value!r}"
|
||||
)
|
||||
return fn(self.reference_date)
|
||||
|
||||
return None
|
||||
+749
-89
@@ -1,36 +1,41 @@
|
||||
"""
|
||||
Temporal Query Module
|
||||
|
||||
This module provides comprehensive time-aware querying capabilities for the
|
||||
Semantica framework, enabling temporal queries and analysis on knowledge
|
||||
graphs with temporal information.
|
||||
|
||||
Key Features:
|
||||
- Time-point queries (query graph at specific time)
|
||||
- Time-range queries (query within time intervals)
|
||||
- Temporal pattern detection (sequences, cycles, trends)
|
||||
- Graph evolution analysis
|
||||
- Temporal path finding
|
||||
- Temporal version management
|
||||
|
||||
Main Classes:
|
||||
- TemporalGraphQuery: Main temporal query engine
|
||||
- TemporalPatternDetector: Temporal pattern detection engine
|
||||
- TemporalVersionManager: Temporal version/snapshot management
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.kg import TemporalGraphQuery
|
||||
>>> query_engine = TemporalGraphQuery()
|
||||
>>> result = query_engine.query_at_time(graph, query, at_time="2024-01-01")
|
||||
>>> evolution = query_engine.analyze_evolution(graph, start_time="2024-01-01")
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
import copy
|
||||
import warnings
|
||||
from collections import defaultdict
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Literal, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .temporal_model import (
|
||||
BiTemporalFact,
|
||||
TemporalBound,
|
||||
deserialize_relationship_temporal_fields,
|
||||
parse_temporal_bound,
|
||||
parse_temporal_value,
|
||||
relationship_to_json_ready,
|
||||
serialize_temporal_value,
|
||||
temporal_structure_to_json_ready,
|
||||
)
|
||||
from .temporal_reasoning import IntervalRelation, TemporalInterval, TemporalReasoningEngine
|
||||
from ..utils.exceptions import ProcessingError, TemporalValidationError
|
||||
|
||||
|
||||
@dataclass
|
||||
class TemporalConsistencyIssue:
|
||||
message: str
|
||||
fact_id: str
|
||||
issue_type: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class TemporalConsistencyReport:
|
||||
errors: List[Dict[str, str]]
|
||||
warnings: List[Dict[str, str]]
|
||||
|
||||
|
||||
class TemporalGraphQuery:
|
||||
@@ -97,6 +102,7 @@ class TemporalGraphQuery:
|
||||
self.pattern_detector = TemporalPatternDetector(
|
||||
**kwargs.get("pattern_detection", {})
|
||||
)
|
||||
self.reasoning_engine = TemporalReasoningEngine()
|
||||
|
||||
def query_at_time(
|
||||
self,
|
||||
@@ -105,6 +111,7 @@ class TemporalGraphQuery:
|
||||
at_time: Any,
|
||||
include_history: bool = False,
|
||||
temporal_precision: Optional[str] = None,
|
||||
time_axis: str = "valid",
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
@@ -136,24 +143,15 @@ class TemporalGraphQuery:
|
||||
|
||||
# Parse time
|
||||
query_time = self._parse_time(at_time)
|
||||
|
||||
# Filter relationships valid at query time
|
||||
relationships = []
|
||||
if "relationships" in graph:
|
||||
for rel in graph.get("relationships", []):
|
||||
valid_from = self._parse_time(rel.get("valid_from"))
|
||||
valid_until = self._parse_time(rel.get("valid_until"))
|
||||
|
||||
# Check if relationship is valid at query time
|
||||
if valid_from and self._compare_times(query_time, valid_from) < 0:
|
||||
continue
|
||||
if valid_until and self._compare_times(query_time, valid_until) > 0:
|
||||
continue
|
||||
|
||||
relationships.append(rel)
|
||||
reconstructed_graph = self.reconstruct_at_time(
|
||||
graph,
|
||||
query_time,
|
||||
time_axis=time_axis,
|
||||
)
|
||||
|
||||
# Get entities
|
||||
entities = graph.get("entities", [])
|
||||
entities = reconstructed_graph.get("entities", [])
|
||||
relationships = reconstructed_graph.get("relationships", [])
|
||||
|
||||
# Include history if requested
|
||||
if include_history:
|
||||
@@ -169,6 +167,195 @@ class TemporalGraphQuery:
|
||||
"num_relationships": len(relationships),
|
||||
}
|
||||
|
||||
def reconstruct_at_time(
|
||||
self,
|
||||
graph: Any,
|
||||
at_time: Any,
|
||||
*,
|
||||
time_axis: str = "valid",
|
||||
) -> Dict[str, Any]:
|
||||
"""Return a self-consistent subgraph for a single point in time."""
|
||||
query_time = at_time if isinstance(at_time, datetime) else self._parse_time(at_time)
|
||||
reconstructed = copy.deepcopy(graph)
|
||||
entity_list = graph.get("entities", [])
|
||||
|
||||
if not entity_list:
|
||||
reconstructed["entities"] = []
|
||||
reconstructed["relationships"] = [
|
||||
copy.deepcopy(relationship)
|
||||
for relationship in graph.get("relationships", [])
|
||||
if self._relationship_active_at_time(relationship, query_time, time_axis=time_axis)
|
||||
]
|
||||
return reconstructed
|
||||
|
||||
entity_index = {
|
||||
self._entity_id(entity): entity
|
||||
for entity in entity_list
|
||||
if self._entity_active_at_time(entity, query_time, time_axis=time_axis)
|
||||
}
|
||||
|
||||
relationships = []
|
||||
for relationship in graph.get("relationships", []):
|
||||
if not self._relationship_active_at_time(relationship, query_time, time_axis=time_axis):
|
||||
continue
|
||||
source = self._entity_id({"id": relationship.get("source")})
|
||||
target = self._entity_id({"id": relationship.get("target")})
|
||||
if source not in entity_index or target not in entity_index:
|
||||
continue
|
||||
relationships.append(copy.deepcopy(relationship))
|
||||
|
||||
reconstructed["entities"] = list(entity_index.values())
|
||||
reconstructed["relationships"] = relationships
|
||||
return reconstructed
|
||||
|
||||
def validate_temporal_consistency(self, graph: Any) -> TemporalConsistencyReport:
|
||||
errors: List[Dict[str, str]] = []
|
||||
warnings_list: List[Dict[str, str]] = []
|
||||
|
||||
entities = {
|
||||
self._entity_id(entity): entity
|
||||
for entity in graph.get("entities", [])
|
||||
}
|
||||
rel_groups: Dict[tuple[str, str, str], List[Dict[str, Any]]] = defaultdict(list)
|
||||
|
||||
for relationship in graph.get("relationships", []):
|
||||
rel_id = relationship.get("id") or self._relationship_key(relationship)
|
||||
rel_groups[
|
||||
(
|
||||
relationship.get("source", ""),
|
||||
relationship.get("type", relationship.get("relationship", "")),
|
||||
relationship.get("target", ""),
|
||||
)
|
||||
].append(relationship)
|
||||
|
||||
try:
|
||||
start, end = self._get_axis_bounds(relationship, "valid")
|
||||
except TemporalValidationError as exc:
|
||||
errors.append(
|
||||
asdict(
|
||||
TemporalConsistencyIssue(
|
||||
message=f"Unable to parse temporal fields: {exc}",
|
||||
fact_id=rel_id,
|
||||
issue_type="invalid_temporal_fields",
|
||||
)
|
||||
)
|
||||
)
|
||||
continue
|
||||
if start and isinstance(end, datetime) and self._compare_times(start, end) > 0:
|
||||
errors.append(
|
||||
asdict(
|
||||
TemporalConsistencyIssue(
|
||||
message="Relationship has an inverted validity interval.",
|
||||
fact_id=rel_id,
|
||||
issue_type="inverted_interval",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
for endpoint in ("source", "target"):
|
||||
entity = entities.get(relationship.get(endpoint))
|
||||
if entity is None:
|
||||
errors.append(
|
||||
asdict(
|
||||
TemporalConsistencyIssue(
|
||||
message=f"Relationship references missing {endpoint} entity.",
|
||||
fact_id=rel_id,
|
||||
issue_type=f"missing_{endpoint}_entity",
|
||||
)
|
||||
)
|
||||
)
|
||||
continue
|
||||
try:
|
||||
within_lifetime = self._window_within_entity_lifetime(entity, start, end)
|
||||
except TemporalValidationError as exc:
|
||||
errors.append(
|
||||
asdict(
|
||||
TemporalConsistencyIssue(
|
||||
message=f"Unable to parse {endpoint} entity lifetime: {exc}",
|
||||
fact_id=rel_id,
|
||||
issue_type=f"invalid_{endpoint}_temporal_fields",
|
||||
)
|
||||
)
|
||||
)
|
||||
continue
|
||||
if not within_lifetime:
|
||||
errors.append(
|
||||
asdict(
|
||||
TemporalConsistencyIssue(
|
||||
message=f"Relationship validity falls outside {endpoint} entity lifetime.",
|
||||
fact_id=rel_id,
|
||||
issue_type=f"{endpoint}_lifetime_mismatch",
|
||||
)
|
||||
)
|
||||
)
|
||||
for relationships in rel_groups.values():
|
||||
safe_relationships = []
|
||||
for relationship in relationships:
|
||||
rel_id = relationship.get("id") or self._relationship_key(relationship)
|
||||
try:
|
||||
start, end = self._get_axis_bounds(relationship, "valid")
|
||||
except TemporalValidationError as exc:
|
||||
errors.append(
|
||||
asdict(
|
||||
TemporalConsistencyIssue(
|
||||
message=f"Unable to parse temporal fields: {exc}",
|
||||
fact_id=rel_id,
|
||||
issue_type="invalid_temporal_fields",
|
||||
)
|
||||
)
|
||||
)
|
||||
continue
|
||||
safe_relationships.append((relationship, start, end))
|
||||
ordered = sorted(
|
||||
safe_relationships,
|
||||
key=lambda item: self._sort_key(item[1]),
|
||||
)
|
||||
for index, (current, current_start, current_end) in enumerate(ordered):
|
||||
current_id = current.get("id") or self._relationship_key(current)
|
||||
if index > 0:
|
||||
_, previous_start, previous_end = ordered[index - 1]
|
||||
if self._range_overlaps_bounds(
|
||||
current_start or datetime.min.replace(tzinfo=timezone.utc),
|
||||
current_end if isinstance(current_end, datetime) else datetime.max.replace(tzinfo=timezone.utc),
|
||||
previous_start,
|
||||
previous_end,
|
||||
):
|
||||
warnings_list.append(
|
||||
asdict(
|
||||
TemporalConsistencyIssue(
|
||||
message="Relationship overlaps another relationship with the same edge and type.",
|
||||
fact_id=current_id,
|
||||
issue_type="overlapping_same_edge",
|
||||
)
|
||||
)
|
||||
)
|
||||
elif (
|
||||
isinstance(previous_end, datetime)
|
||||
and current_start
|
||||
and self._compare_times(current_start, previous_end) > 0
|
||||
):
|
||||
warnings_list.append(
|
||||
asdict(
|
||||
TemporalConsistencyIssue(
|
||||
message="Relationship restarts after a temporal gap.",
|
||||
fact_id=current_id,
|
||||
issue_type="gap_after_restart",
|
||||
)
|
||||
)
|
||||
)
|
||||
elif current_start and previous_start and self._compare_times(current_start, previous_start) == 0:
|
||||
warnings_list.append(
|
||||
asdict(
|
||||
TemporalConsistencyIssue(
|
||||
message="Relationship interval overlaps another interval on the same edge.",
|
||||
fact_id=current_id,
|
||||
issue_type="overlapping_same_edge",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
return TemporalConsistencyReport(errors=errors, warnings=warnings_list)
|
||||
|
||||
def query_time_range(
|
||||
self,
|
||||
graph: Any,
|
||||
@@ -177,6 +364,7 @@ class TemporalGraphQuery:
|
||||
end_time: Any,
|
||||
temporal_aggregation: str = "union",
|
||||
include_intervals: bool = True,
|
||||
time_axis: str = "valid",
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
@@ -209,23 +397,26 @@ class TemporalGraphQuery:
|
||||
self.logger.info(f"Querying graph in time range: {start_time} to {end_time}")
|
||||
|
||||
# Parse times
|
||||
start = self._parse_time(start_time)
|
||||
end = self._parse_time(end_time)
|
||||
normalized_range = self.reasoning_engine.normalize_interval(
|
||||
start_time,
|
||||
end_time,
|
||||
self.temporal_granularity,
|
||||
)
|
||||
start = normalized_range.start
|
||||
end = normalized_range.end
|
||||
|
||||
# Filter relationships valid in time range
|
||||
relationships = []
|
||||
relationship_buckets = None
|
||||
if "relationships" in graph:
|
||||
for rel in graph.get("relationships", []):
|
||||
valid_from = self._parse_time(rel.get("valid_from"))
|
||||
valid_until = self._parse_time(rel.get("valid_until"))
|
||||
|
||||
# Check if relationship overlaps with time range
|
||||
if valid_from and self._compare_times(end, valid_from) < 0:
|
||||
continue
|
||||
if valid_until and self._compare_times(start, valid_until) > 0:
|
||||
continue
|
||||
|
||||
relationships.append(rel)
|
||||
if self._relationship_overlaps_range(
|
||||
rel,
|
||||
start,
|
||||
end,
|
||||
time_axis=time_axis,
|
||||
):
|
||||
relationships.append(rel)
|
||||
|
||||
# Aggregate based on strategy
|
||||
if temporal_aggregation == "intersection":
|
||||
@@ -233,21 +424,18 @@ class TemporalGraphQuery:
|
||||
relationships = [
|
||||
rel
|
||||
for rel in relationships
|
||||
if self._parse_time(rel.get("valid_from")) <= start
|
||||
and (
|
||||
not rel.get("valid_until")
|
||||
or self._parse_time(rel.get("valid_until")) >= end
|
||||
)
|
||||
if self._relationship_covers_range(rel, start, end, time_axis=time_axis)
|
||||
]
|
||||
elif temporal_aggregation == "evolution":
|
||||
# Group by time periods
|
||||
relationships = self._group_by_time_periods(relationships, start, end)
|
||||
relationship_buckets = self._group_by_time_periods(relationships, start, end)
|
||||
|
||||
return {
|
||||
"query": query,
|
||||
"start_time": start,
|
||||
"end_time": end,
|
||||
"relationships": relationships,
|
||||
"relationship_buckets": relationship_buckets,
|
||||
"num_relationships": len(relationships),
|
||||
"aggregation": temporal_aggregation,
|
||||
}
|
||||
@@ -414,6 +602,8 @@ class TemporalGraphQuery:
|
||||
end_time: Optional[Any] = None,
|
||||
max_path_length: Optional[int] = None,
|
||||
temporal_constraints: Optional[Dict[str, Any]] = None,
|
||||
enforce_causal_ordering: bool = True,
|
||||
ordering_strategy: Literal["strict", "overlap", "loose"] = "strict",
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
@@ -448,6 +638,8 @@ class TemporalGraphQuery:
|
||||
# Build adjacency with temporal constraints
|
||||
adjacency = {}
|
||||
relationships = graph.get("relationships", [])
|
||||
parsed_start_time = self._parse_time(start_time) if start_time else None
|
||||
parsed_end_time = self._parse_time(end_time) if end_time else None
|
||||
|
||||
for rel in relationships:
|
||||
s = rel.get("source")
|
||||
@@ -459,15 +651,15 @@ class TemporalGraphQuery:
|
||||
valid_until = self._parse_time(rel.get("valid_until"))
|
||||
|
||||
if (
|
||||
start_time
|
||||
parsed_start_time
|
||||
and valid_until
|
||||
and self._compare_times(valid_until, start_time) < 0
|
||||
and self._compare_times(valid_until, parsed_start_time) < 0
|
||||
):
|
||||
continue
|
||||
if (
|
||||
end_time
|
||||
parsed_end_time
|
||||
and valid_from
|
||||
and self._compare_times(valid_from, end_time) > 0
|
||||
and self._compare_times(valid_from, parsed_end_time) > 0
|
||||
):
|
||||
continue
|
||||
|
||||
@@ -499,7 +691,13 @@ class TemporalGraphQuery:
|
||||
|
||||
for neighbor, rel in adjacency.get(node, []):
|
||||
if neighbor not in path: # Avoid cycles
|
||||
queue.append((neighbor, path + [neighbor], edges + [rel]))
|
||||
next_edges = edges + [rel]
|
||||
if self._path_respects_causal_order(
|
||||
next_edges,
|
||||
enforce_causal_ordering=enforce_causal_ordering,
|
||||
ordering_strategy=ordering_strategy,
|
||||
):
|
||||
queue.append((neighbor, path + [neighbor], next_edges))
|
||||
|
||||
return {
|
||||
"source": source,
|
||||
@@ -509,30 +707,205 @@ class TemporalGraphQuery:
|
||||
}
|
||||
|
||||
def _parse_time(self, time_value):
|
||||
"""Parse time value."""
|
||||
from datetime import datetime
|
||||
|
||||
if time_value is None:
|
||||
"""Parse time value into a UTC-normalized datetime."""
|
||||
if time_value in (None, TemporalBound.OPEN, TemporalBound.OPEN.value):
|
||||
return None
|
||||
|
||||
if isinstance(time_value, str):
|
||||
return time_value
|
||||
|
||||
if isinstance(time_value, datetime):
|
||||
return time_value.isoformat()
|
||||
|
||||
return str(time_value)
|
||||
try:
|
||||
return parse_temporal_value(time_value)
|
||||
except TemporalValidationError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise TemporalValidationError(
|
||||
"Invalid temporal value",
|
||||
temporal_context={"value": time_value},
|
||||
) from exc
|
||||
|
||||
def _compare_times(self, time1, time2):
|
||||
"""Compare two time strings."""
|
||||
"""Compare two UTC datetimes after granularity truncation."""
|
||||
if time1 is None or time2 is None:
|
||||
return 0
|
||||
time1 = self._truncate_to_granularity(time1)
|
||||
time2 = self._truncate_to_granularity(time2)
|
||||
return (time1 > time2) - (time1 < time2)
|
||||
|
||||
def _truncate_to_granularity(self, value: datetime) -> datetime:
|
||||
granularity = getattr(self, "temporal_granularity", "second")
|
||||
return self.reasoning_engine.normalize_timestamp(value, granularity)
|
||||
|
||||
def _get_axis_bounds(self, relationship: Dict[str, Any], axis: str):
|
||||
normalized = deserialize_relationship_temporal_fields(relationship)
|
||||
fact = BiTemporalFact.from_relationship(normalized)
|
||||
if axis == "valid":
|
||||
return fact.valid_from, fact.valid_until
|
||||
if axis == "transaction":
|
||||
return fact.recorded_at, fact.superseded_at
|
||||
raise ValueError(f"Unsupported time axis: {axis}")
|
||||
|
||||
def _is_point_in_bounds(self, point: datetime, start: Optional[datetime], end: Optional[datetime | TemporalBound]) -> bool:
|
||||
if start is None:
|
||||
start = datetime.min.replace(tzinfo=timezone.utc)
|
||||
return self.reasoning_engine.active_at(
|
||||
TemporalInterval(start=start, end=end or TemporalBound.OPEN),
|
||||
point,
|
||||
granularity=self.temporal_granularity,
|
||||
)
|
||||
|
||||
def _range_overlaps_bounds(self, query_start: datetime, query_end: datetime | TemporalBound, start: Optional[datetime], end: Optional[datetime | TemporalBound]) -> bool:
|
||||
query_end_value = datetime.max.replace(tzinfo=timezone.utc) if query_end is TemporalBound.OPEN else query_end
|
||||
if query_end_value < query_start:
|
||||
return False
|
||||
if isinstance(end, datetime) and start is not None and end < start:
|
||||
return False
|
||||
candidate = TemporalInterval(
|
||||
start=start or datetime.min.replace(tzinfo=timezone.utc),
|
||||
end=end or TemporalBound.OPEN,
|
||||
)
|
||||
query_interval = TemporalInterval(start=query_start, end=query_end)
|
||||
relation = self.reasoning_engine.relation(candidate, query_interval)
|
||||
return relation not in {
|
||||
IntervalRelation.BEFORE,
|
||||
IntervalRelation.AFTER,
|
||||
}
|
||||
|
||||
def _range_covered_by_bounds(self, query_start: datetime, query_end: datetime | TemporalBound, start: Optional[datetime], end: Optional[datetime | TemporalBound]) -> bool:
|
||||
query_end_value = datetime.max.replace(tzinfo=timezone.utc) if query_end is TemporalBound.OPEN else query_end
|
||||
if query_end_value < query_start:
|
||||
return False
|
||||
if isinstance(end, datetime) and start is not None and end < start:
|
||||
return False
|
||||
candidate = TemporalInterval(
|
||||
start=start or datetime.min.replace(tzinfo=timezone.utc),
|
||||
end=end or TemporalBound.OPEN,
|
||||
)
|
||||
return self.reasoning_engine.contains(candidate, TemporalInterval(start=query_start, end=query_end))
|
||||
|
||||
def _relationship_active_at_time(self, relationship: Dict[str, Any], query_time: datetime, *, time_axis: str) -> bool:
|
||||
axes = ["valid", "transaction"] if time_axis == "both" else [time_axis]
|
||||
return all(
|
||||
self._is_point_in_bounds(query_time, *self._get_axis_bounds(relationship, axis))
|
||||
for axis in axes
|
||||
)
|
||||
|
||||
def _relationship_overlaps_range(self, relationship: Dict[str, Any], start: datetime, end: datetime | TemporalBound, *, time_axis: str) -> bool:
|
||||
axes = ["valid", "transaction"] if time_axis == "both" else [time_axis]
|
||||
return all(
|
||||
self._range_overlaps_bounds(start, end, *self._get_axis_bounds(relationship, axis))
|
||||
for axis in axes
|
||||
)
|
||||
|
||||
def _relationship_covers_range(self, relationship: Dict[str, Any], start: datetime, end: datetime | TemporalBound, *, time_axis: str) -> bool:
|
||||
axes = ["valid", "transaction"] if time_axis == "both" else [time_axis]
|
||||
return all(
|
||||
self._range_covered_by_bounds(start, end, *self._get_axis_bounds(relationship, axis))
|
||||
for axis in axes
|
||||
)
|
||||
|
||||
def _entity_id(self, entity: Dict[str, Any]) -> str:
|
||||
return str(entity.get("id", entity.get("name", "")))
|
||||
|
||||
def _entity_active_at_time(self, entity: Dict[str, Any], query_time: datetime, *, time_axis: str) -> bool:
|
||||
relationship_like = {
|
||||
"valid_from": entity.get("valid_from"),
|
||||
"valid_until": entity.get("valid_until", TemporalBound.OPEN),
|
||||
"recorded_at": entity.get("recorded_at", entity.get("valid_from")),
|
||||
"superseded_at": entity.get("superseded_at", TemporalBound.OPEN),
|
||||
}
|
||||
return self._relationship_active_at_time(relationship_like, query_time, time_axis=time_axis)
|
||||
|
||||
def _window_within_entity_lifetime(
|
||||
self,
|
||||
entity: Dict[str, Any],
|
||||
rel_start: Optional[datetime],
|
||||
rel_end: Optional[datetime | TemporalBound],
|
||||
) -> bool:
|
||||
entity_start = self._parse_time(entity.get("valid_from")) if entity.get("valid_from") is not None else None
|
||||
entity_end = parse_temporal_bound(entity.get("valid_until"), default=TemporalBound.OPEN)
|
||||
if rel_start and entity_start and self._compare_times(rel_start, entity_start) < 0:
|
||||
return False
|
||||
if isinstance(entity_end, datetime):
|
||||
if rel_start and self._compare_times(rel_start, entity_end) > 0:
|
||||
return False
|
||||
if isinstance(rel_end, datetime) and self._compare_times(rel_end, entity_end) > 0:
|
||||
return False
|
||||
if rel_end is TemporalBound.OPEN:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _sort_key(self, value: Optional[datetime]) -> datetime:
|
||||
return value or datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
def _period_floor(self, value: datetime) -> datetime:
|
||||
return self._truncate_to_granularity(value)
|
||||
|
||||
def _next_period(self, value: datetime) -> datetime:
|
||||
if self.temporal_granularity == "day":
|
||||
return value + timedelta(days=1)
|
||||
if self.temporal_granularity == "week":
|
||||
return value + timedelta(weeks=1)
|
||||
if self.temporal_granularity == "month":
|
||||
if value.month == 12:
|
||||
return value.replace(year=value.year + 1, month=1)
|
||||
return value.replace(month=value.month + 1)
|
||||
if self.temporal_granularity == "year":
|
||||
return value.replace(year=value.year + 1)
|
||||
return value + timedelta(days=1)
|
||||
|
||||
def _period_label(self, value: datetime) -> str:
|
||||
if self.temporal_granularity == "week":
|
||||
return f"{value.isocalendar().year}-W{value.isocalendar().week:02d}"
|
||||
if self.temporal_granularity == "month":
|
||||
return value.strftime("%Y-%m")
|
||||
if self.temporal_granularity == "year":
|
||||
return value.strftime("%Y")
|
||||
return value.strftime("%Y-%m-%d")
|
||||
|
||||
def _path_respects_causal_order(
|
||||
self,
|
||||
edges: List[Dict[str, Any]],
|
||||
*,
|
||||
enforce_causal_ordering: bool,
|
||||
ordering_strategy: Literal["strict", "overlap", "loose"],
|
||||
) -> bool:
|
||||
if not enforce_causal_ordering or ordering_strategy == "loose" or len(edges) < 2:
|
||||
return True
|
||||
|
||||
previous = edges[-2]
|
||||
current = edges[-1]
|
||||
previous_start, previous_end = self._get_axis_bounds(previous, "valid")
|
||||
current_start, _ = self._get_axis_bounds(current, "valid")
|
||||
|
||||
if ordering_strategy == "strict":
|
||||
if previous_start and current_start and self._compare_times(current_start, previous_start) < 0:
|
||||
return False
|
||||
return True
|
||||
if ordering_strategy == "overlap":
|
||||
if current_start is None:
|
||||
return True
|
||||
if isinstance(previous_end, datetime):
|
||||
return self._compare_times(current_start, previous_end) <= 0
|
||||
return True
|
||||
return True
|
||||
|
||||
def _group_by_time_periods(self, relationships, start, end):
|
||||
"""Group relationships by time periods."""
|
||||
# Simplified grouping
|
||||
return relationships
|
||||
"""Group relationships into calendar-aligned buckets."""
|
||||
buckets: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
|
||||
current = self._period_floor(start)
|
||||
end_bound = self._period_floor(end)
|
||||
|
||||
while current <= end_bound:
|
||||
bucket_end = self._next_period(current)
|
||||
label = self._period_label(current)
|
||||
for relationship in relationships:
|
||||
if self._relationship_overlaps_range(
|
||||
relationship,
|
||||
current,
|
||||
bucket_end,
|
||||
time_axis="valid",
|
||||
):
|
||||
buckets[label].append(relationship)
|
||||
current = bucket_end
|
||||
|
||||
return dict(buckets)
|
||||
|
||||
|
||||
class TemporalPatternDetector:
|
||||
@@ -601,6 +974,8 @@ class TemporalPatternDetector:
|
||||
list: List of detected pattern dictionaries
|
||||
"""
|
||||
self.logger.info(f"Detecting temporal patterns: {pattern_type}")
|
||||
if "gap_tolerance" in options:
|
||||
self.config["gap_tolerance"] = options["gap_tolerance"]
|
||||
|
||||
relationships = graph.get("relationships", [])
|
||||
|
||||
@@ -620,13 +995,148 @@ class TemporalPatternDetector:
|
||||
|
||||
def _find_sequences(self, relationships, min_frequency):
|
||||
"""Find sequential patterns."""
|
||||
# Simplified sequence detection
|
||||
return []
|
||||
# Pattern output design:
|
||||
# - sequences return dicts with pattern_type/signature/frequency/occurrences
|
||||
# - each occurrence stores ordered nodes, ordered edges, and start/end timestamps
|
||||
gap_tolerance = self._resolve_gap_tolerance()
|
||||
outgoing: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
|
||||
for relationship in relationships:
|
||||
outgoing[relationship.get("source")].append(relationship)
|
||||
|
||||
occurrences = defaultdict(list)
|
||||
for relationship in relationships:
|
||||
sequence = [relationship]
|
||||
current = relationship
|
||||
while True:
|
||||
next_candidates = []
|
||||
_, current_end = self._get_axis_bounds(current)
|
||||
if current_end is TemporalBound.OPEN:
|
||||
break
|
||||
for candidate in outgoing.get(current.get("target"), []):
|
||||
if candidate is current:
|
||||
continue
|
||||
candidate_start, _ = self._get_axis_bounds(candidate)
|
||||
if not isinstance(current_end, datetime) or candidate_start is None:
|
||||
continue
|
||||
gap = candidate_start - current_end
|
||||
if gap < timedelta(0) or gap > gap_tolerance:
|
||||
continue
|
||||
next_candidates.append((candidate_start, candidate))
|
||||
if not next_candidates:
|
||||
break
|
||||
next_candidates.sort(key=lambda item: item[0])
|
||||
current = next_candidates[0][1]
|
||||
sequence.append(current)
|
||||
|
||||
if len(sequence) < 2:
|
||||
continue
|
||||
|
||||
signature = tuple(
|
||||
[sequence[0].get("source")] + [edge.get("target") for edge in sequence]
|
||||
)
|
||||
occurrences[signature].append(
|
||||
{
|
||||
"nodes": list(signature),
|
||||
"edges": [copy.deepcopy(edge) for edge in sequence],
|
||||
"start_time": serialize_temporal_value(self._get_axis_bounds(sequence[0])[0]),
|
||||
"end_time": serialize_temporal_value(self._sequence_end(sequence)),
|
||||
}
|
||||
)
|
||||
|
||||
return [
|
||||
{
|
||||
"pattern_type": "sequence",
|
||||
"signature": signature,
|
||||
"frequency": len(items),
|
||||
"occurrences": items,
|
||||
}
|
||||
for signature, items in occurrences.items()
|
||||
if len(items) >= min_frequency
|
||||
]
|
||||
|
||||
def _find_cycles(self, relationships, min_frequency):
|
||||
"""Find cyclic patterns."""
|
||||
# Simplified cycle detection
|
||||
return []
|
||||
# Pattern output design mirrors sequences:
|
||||
# - cycles return pattern_type/signature/frequency/occurrences
|
||||
# - each occurrence records the ordered cycle nodes, edges, and time span
|
||||
outgoing: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
|
||||
for relationship in relationships:
|
||||
outgoing[relationship.get("source")].append(relationship)
|
||||
|
||||
occurrences = defaultdict(list)
|
||||
for relationship in relationships:
|
||||
start_node = relationship.get("source")
|
||||
path_edges = [relationship]
|
||||
visited_nodes = [start_node, relationship.get("target")]
|
||||
current = relationship
|
||||
|
||||
while len(path_edges) <= 6:
|
||||
if visited_nodes[-1] == start_node:
|
||||
signature = tuple(visited_nodes)
|
||||
occurrences[signature].append(
|
||||
{
|
||||
"nodes": visited_nodes[:],
|
||||
"edges": [copy.deepcopy(edge) for edge in path_edges],
|
||||
"start_time": serialize_temporal_value(self._get_axis_bounds(path_edges[0])[0]),
|
||||
"end_time": serialize_temporal_value(self._sequence_end(path_edges)),
|
||||
}
|
||||
)
|
||||
break
|
||||
|
||||
next_candidates = []
|
||||
_, current_end = self._get_axis_bounds(current)
|
||||
for candidate in outgoing.get(visited_nodes[-1], []):
|
||||
candidate_start, _ = self._get_axis_bounds(candidate)
|
||||
if isinstance(current_end, datetime) and candidate_start and self._compare_times(candidate_start, current_end) < 0:
|
||||
continue
|
||||
next_candidates.append((self._sort_key(candidate_start), candidate))
|
||||
if not next_candidates:
|
||||
break
|
||||
next_candidates.sort(key=lambda item: item[0])
|
||||
current = next_candidates[0][1]
|
||||
next_target = current.get("target")
|
||||
if next_target in visited_nodes[1:-1] and next_target != start_node:
|
||||
break
|
||||
path_edges.append(current)
|
||||
visited_nodes.append(next_target)
|
||||
|
||||
return [
|
||||
{
|
||||
"pattern_type": "cycle",
|
||||
"signature": signature,
|
||||
"frequency": len(items),
|
||||
"occurrences": items,
|
||||
}
|
||||
for signature, items in occurrences.items()
|
||||
if len(items) >= min_frequency
|
||||
]
|
||||
|
||||
def _resolve_gap_tolerance(self) -> timedelta:
|
||||
gap_tolerance = self.config.get("gap_tolerance", timedelta(days=0))
|
||||
if isinstance(gap_tolerance, timedelta):
|
||||
return gap_tolerance
|
||||
if isinstance(gap_tolerance, (int, float)):
|
||||
return timedelta(days=gap_tolerance)
|
||||
return timedelta(days=0)
|
||||
|
||||
def _sequence_end(self, edges: List[Dict[str, Any]]) -> Optional[datetime]:
|
||||
last_start, last_end = self._get_axis_bounds(edges[-1])
|
||||
if isinstance(last_end, datetime):
|
||||
return last_end
|
||||
return last_start
|
||||
|
||||
def _get_axis_bounds(self, relationship: Dict[str, Any]) -> tuple[Optional[datetime], Optional[datetime | TemporalBound]]:
|
||||
normalized = deserialize_relationship_temporal_fields(relationship)
|
||||
fact = BiTemporalFact.from_relationship(normalized)
|
||||
return fact.valid_from, fact.valid_until
|
||||
|
||||
def _compare_times(self, time1: Optional[datetime], time2: Optional[datetime]) -> int:
|
||||
if time1 is None or time2 is None:
|
||||
return 0
|
||||
return (time1 > time2) - (time1 < time2)
|
||||
|
||||
def _sort_key(self, value: Optional[datetime]) -> datetime:
|
||||
return value or datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
class TemporalVersionManager:
|
||||
@@ -848,7 +1358,10 @@ class TemporalVersionManager:
|
||||
"author": change_entry.author,
|
||||
"description": change_entry.description,
|
||||
"entities": graph.get("entities", []).copy(),
|
||||
"relationships": graph.get("relationships", []).copy(),
|
||||
"relationships": [
|
||||
relationship_to_json_ready(rel)
|
||||
for rel in graph.get("relationships", []).copy()
|
||||
],
|
||||
"metadata": options.get("metadata", {})
|
||||
}
|
||||
|
||||
@@ -856,10 +1369,116 @@ class TemporalVersionManager:
|
||||
snapshot["checksum"] = compute_checksum(snapshot)
|
||||
|
||||
# Store snapshot
|
||||
self.storage.save(snapshot)
|
||||
try:
|
||||
self.storage.save(snapshot)
|
||||
except Exception as exc:
|
||||
raise ProcessingError(
|
||||
"Failed to persist snapshot",
|
||||
processing_context={"label": version_label, "author": author},
|
||||
) from exc
|
||||
|
||||
self.logger.info(f"Created snapshot '{version_label}' by {author}")
|
||||
return snapshot
|
||||
|
||||
def apply_revision(self, snapshot: Dict[str, Any], revision: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Apply a temporal revision without deleting the original facts.
|
||||
|
||||
Design note:
|
||||
Overlapping retroactive revisions against the same fact are handled by
|
||||
superseding the latest matching version and emitting a warning when the
|
||||
newly requested valid window overlaps a sibling fact on the same edge
|
||||
and relationship type. This preserves all prior versions instead of
|
||||
trying to collapse them into a single mutable record.
|
||||
"""
|
||||
from ..change_management import compute_checksum
|
||||
|
||||
revision_time = datetime.now(timezone.utc)
|
||||
revision_suffix = self._generate_revision_suffix(revision_time)
|
||||
relationships = copy.deepcopy(snapshot.get("relationships", []))
|
||||
fact_ids = set(revision.get("fact_ids", []))
|
||||
new_valid_from = parse_temporal_value(revision.get("new_valid_from"))
|
||||
new_valid_until = parse_temporal_bound(revision.get("new_valid_until"), default=TemporalBound.OPEN)
|
||||
revision_type = revision.get("revision_type", "correction")
|
||||
|
||||
revised_relationships = []
|
||||
provenance_event = {
|
||||
"type": "temporal_revision",
|
||||
"revision_type": revision_type,
|
||||
"author": revision.get("author"),
|
||||
"reason": revision.get("reason"),
|
||||
"recorded_at": serialize_temporal_value(revision_time),
|
||||
"fact_ids": list(fact_ids),
|
||||
}
|
||||
|
||||
for rel in relationships:
|
||||
rel_id = rel.get("id") or self._relationship_key(rel)
|
||||
if rel_id not in fact_ids:
|
||||
revised_relationships.append(rel)
|
||||
continue
|
||||
|
||||
original = deserialize_relationship_temporal_fields(rel)
|
||||
original["id"] = rel_id
|
||||
original["superseded_at"] = serialize_temporal_value(revision_time)
|
||||
original.setdefault("provenance", []).append(
|
||||
{
|
||||
**provenance_event,
|
||||
"role": "superseded",
|
||||
}
|
||||
)
|
||||
revised_relationships.append(original)
|
||||
|
||||
replacement = copy.deepcopy(rel)
|
||||
replacement["id"] = f"{rel_id}__rev__{revision_suffix}"
|
||||
replacement["valid_from"] = serialize_temporal_value(new_valid_from)
|
||||
replacement["valid_until"] = (
|
||||
TemporalBound.OPEN if new_valid_until is TemporalBound.OPEN else serialize_temporal_value(new_valid_until)
|
||||
)
|
||||
replacement["recorded_at"] = serialize_temporal_value(revision_time)
|
||||
replacement["superseded_at"] = TemporalBound.OPEN
|
||||
replacement.setdefault("provenance", []).append(
|
||||
{
|
||||
**provenance_event,
|
||||
"role": "replacement",
|
||||
"replaces": rel_id,
|
||||
}
|
||||
)
|
||||
self._warn_on_retroactive_overlap(replacement, relationships, revision_type)
|
||||
revised_relationships.append(replacement)
|
||||
|
||||
revised_snapshot = copy.deepcopy(snapshot)
|
||||
revised_snapshot["relationships"] = [
|
||||
relationship_to_json_ready(rel) for rel in revised_relationships
|
||||
]
|
||||
base_label = snapshot.get("label", "snapshot")
|
||||
original_label = base_label
|
||||
revised_label = f"{base_label}__revision__{revision_suffix}"
|
||||
|
||||
original_snapshot = copy.deepcopy(snapshot)
|
||||
original_snapshot["label"] = original_label
|
||||
original_snapshot_inserted = False
|
||||
if self.storage.get(original_label) is None:
|
||||
original_snapshot["checksum"] = compute_checksum(
|
||||
{k: v for k, v in original_snapshot.items() if k != "checksum"}
|
||||
)
|
||||
self.storage.save(original_snapshot)
|
||||
original_snapshot_inserted = True
|
||||
|
||||
revised_snapshot["label"] = revised_label
|
||||
revised_snapshot.setdefault("metadata", {})["revision_event"] = temporal_structure_to_json_ready(provenance_event)
|
||||
revised_snapshot["checksum"] = compute_checksum(
|
||||
{k: v for k, v in revised_snapshot.items() if k != "checksum"}
|
||||
)
|
||||
try:
|
||||
self.storage.save(revised_snapshot)
|
||||
except Exception as exc:
|
||||
if original_snapshot_inserted:
|
||||
self.storage.delete(original_label)
|
||||
raise ProcessingError(
|
||||
"Failed to persist revised snapshot",
|
||||
processing_context={"original_label": original_label, "revised_label": revised_label},
|
||||
) from exc
|
||||
return revised_snapshot
|
||||
|
||||
def list_versions(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
@@ -894,7 +1513,7 @@ class TemporalVersionManager:
|
||||
"""
|
||||
from ..change_management import verify_checksum
|
||||
return verify_checksum(snapshot)
|
||||
|
||||
|
||||
def _compute_detailed_diff(self, version1: Dict[str, Any], version2: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Compute detailed entity and relationship differences between versions.
|
||||
@@ -971,6 +1590,42 @@ class TemporalVersionManager:
|
||||
target = relationship.get("target", "")
|
||||
rel_type = relationship.get("type", relationship.get("relationship", ""))
|
||||
return f"{source}|{rel_type}|{target}"
|
||||
|
||||
def _generate_revision_suffix(self, revision_time: datetime) -> str:
|
||||
timestamp_part = revision_time.strftime("%Y%m%dT%H%M%S%fZ")
|
||||
return f"{timestamp_part}_{uuid4().hex[:8]}"
|
||||
|
||||
def _warn_on_retroactive_overlap(
|
||||
self,
|
||||
replacement: Dict[str, Any],
|
||||
relationships: List[Dict[str, Any]],
|
||||
revision_type: str,
|
||||
) -> None:
|
||||
if revision_type != "retroactive":
|
||||
return
|
||||
|
||||
query = TemporalGraphQuery(temporal_granularity="second")
|
||||
candidate_start, candidate_end = query._get_axis_bounds(replacement, "valid")
|
||||
for sibling in relationships:
|
||||
if sibling.get("source") != replacement.get("source"):
|
||||
continue
|
||||
if sibling.get("target") != replacement.get("target"):
|
||||
continue
|
||||
if sibling.get("type") != replacement.get("type"):
|
||||
continue
|
||||
sibling_start, sibling_end = query._get_axis_bounds(sibling, "valid")
|
||||
if query._range_overlaps_bounds(
|
||||
candidate_start or datetime.min.replace(tzinfo=timezone.utc),
|
||||
candidate_end if isinstance(candidate_end, datetime) else datetime.max.replace(tzinfo=timezone.utc),
|
||||
sibling_start,
|
||||
sibling_end,
|
||||
):
|
||||
warnings.warn(
|
||||
"Retroactive revision overlaps an existing fact on the same edge.",
|
||||
UserWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return
|
||||
|
||||
def _compute_entity_changes(self, entity1: Dict[str, Any], entity2: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
@@ -1021,3 +1676,8 @@ class TemporalVersionManager:
|
||||
changes[key] = {"from": val1, "to": val2}
|
||||
|
||||
return changes
|
||||
|
||||
|
||||
def validate_temporal_consistency(graph: Any) -> TemporalConsistencyReport:
|
||||
"""Module-level validator entry point required by the temporal query API."""
|
||||
return TemporalGraphQuery().validate_temporal_consistency(graph)
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
"""
|
||||
Deterministic temporal reasoning primitives for Semantica.
|
||||
|
||||
This module is the single source of truth for interval math across temporal KG
|
||||
features. It performs zero LLM calls: extraction may happen upstream, but all
|
||||
temporal reasoning here is pure Python and fully deterministic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import calendar
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from .temporal_model import BiTemporalFact, TemporalBound, parse_temporal_bound, parse_temporal_value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TemporalInterval:
|
||||
start: datetime
|
||||
end: datetime | TemporalBound
|
||||
label: Optional[str] = None
|
||||
|
||||
|
||||
class IntervalRelation(Enum):
|
||||
BEFORE = "before"
|
||||
AFTER = "after"
|
||||
MEETS = "meets"
|
||||
MET_BY = "met_by"
|
||||
OVERLAPS = "overlaps"
|
||||
OVERLAPPED_BY = "overlapped_by"
|
||||
STARTS = "starts"
|
||||
STARTED_BY = "started_by"
|
||||
DURING = "during"
|
||||
CONTAINS = "contains"
|
||||
FINISHES = "finishes"
|
||||
FINISHED_BY = "finished_by"
|
||||
EQUALS = "equals"
|
||||
|
||||
|
||||
class TemporalReasoningEngine:
|
||||
"""Pure-Python temporal reasoning engine with Allen interval algebra."""
|
||||
|
||||
SUPPORTED_GRANULARITIES = {"second", "minute", "hour", "day", "week", "month", "year"}
|
||||
|
||||
def relation(self, a: TemporalInterval, b: TemporalInterval) -> IntervalRelation:
|
||||
self._validate_interval(a)
|
||||
self._validate_interval(b)
|
||||
|
||||
a_end = self._end_value(a.end)
|
||||
b_end = self._end_value(b.end)
|
||||
|
||||
if a_end < b.start:
|
||||
return IntervalRelation.BEFORE
|
||||
if a.start > b_end:
|
||||
return IntervalRelation.AFTER
|
||||
if a_end == b.start:
|
||||
return IntervalRelation.MEETS
|
||||
if a.start == b_end:
|
||||
return IntervalRelation.MET_BY
|
||||
if a.start == b.start and a_end == b_end:
|
||||
return IntervalRelation.EQUALS
|
||||
if a.start == b.start and a_end < b_end:
|
||||
return IntervalRelation.STARTS
|
||||
if a.start == b.start and a_end > b_end:
|
||||
return IntervalRelation.STARTED_BY
|
||||
if a_end == b_end and a.start > b.start:
|
||||
return IntervalRelation.FINISHES
|
||||
if a_end == b_end and a.start < b.start:
|
||||
return IntervalRelation.FINISHED_BY
|
||||
if a.start < b.start and a_end > b.start and a_end < b_end:
|
||||
return IntervalRelation.OVERLAPS
|
||||
if a.start > b.start and a.start < b_end and a_end > b_end:
|
||||
return IntervalRelation.OVERLAPPED_BY
|
||||
if a.start > b.start and a_end < b_end:
|
||||
return IntervalRelation.DURING
|
||||
return IntervalRelation.CONTAINS
|
||||
|
||||
def overlaps(self, a: TemporalInterval, b: TemporalInterval) -> bool:
|
||||
relation = self.relation(a, b)
|
||||
return relation not in {
|
||||
IntervalRelation.BEFORE,
|
||||
IntervalRelation.AFTER,
|
||||
IntervalRelation.MEETS,
|
||||
IntervalRelation.MET_BY,
|
||||
}
|
||||
|
||||
def contains(self, outer: TemporalInterval, inner: TemporalInterval) -> bool:
|
||||
self._validate_interval(outer)
|
||||
self._validate_interval(inner)
|
||||
return outer.start <= inner.start and self._end_value(outer.end) >= self._end_value(inner.end)
|
||||
|
||||
def active_at(
|
||||
self,
|
||||
interval: TemporalInterval,
|
||||
timestamp: Any,
|
||||
*,
|
||||
granularity: Optional[str] = None,
|
||||
) -> bool:
|
||||
self._validate_interval(interval)
|
||||
point = parse_temporal_value(timestamp)
|
||||
start = interval.start
|
||||
end = interval.end
|
||||
|
||||
if granularity is not None:
|
||||
point = self.normalize_timestamp(point, granularity)
|
||||
start = self.normalize_timestamp(start, granularity)
|
||||
if isinstance(end, datetime):
|
||||
end = self.normalize_timestamp(end, granularity)
|
||||
|
||||
return start <= point and (end is TemporalBound.OPEN or point < self._coerce_datetime(end))
|
||||
|
||||
def merge_intervals(self, intervals: Iterable[TemporalInterval]) -> List[TemporalInterval]:
|
||||
ordered = sorted((self._validated_copy(i) for i in intervals), key=lambda item: item.start)
|
||||
if not ordered:
|
||||
return []
|
||||
|
||||
merged: List[TemporalInterval] = [ordered[0]]
|
||||
for interval in ordered[1:]:
|
||||
current = merged[-1]
|
||||
if self._touches_or_overlaps(current, interval):
|
||||
new_end = self._max_end(current.end, interval.end)
|
||||
merged[-1] = TemporalInterval(start=current.start, end=new_end, label=current.label)
|
||||
else:
|
||||
merged.append(interval)
|
||||
return merged
|
||||
|
||||
def gap_analysis(
|
||||
self,
|
||||
intervals: Iterable[TemporalInterval],
|
||||
domain_start: Any,
|
||||
domain_end: Any,
|
||||
) -> List[TemporalInterval]:
|
||||
domain = self._make_interval(domain_start, domain_end, label="domain")
|
||||
clipped = self._clip_to_domain(intervals, domain)
|
||||
merged = self.merge_intervals(clipped)
|
||||
|
||||
gaps: List[TemporalInterval] = []
|
||||
cursor = domain.start
|
||||
for interval in merged:
|
||||
if cursor < interval.start:
|
||||
gaps.append(TemporalInterval(start=cursor, end=interval.start, label="gap"))
|
||||
cursor = self._max_datetime(cursor, self._end_as_datetime(interval.end, domain.end))
|
||||
if cursor < self._coerce_datetime(domain.end):
|
||||
gaps.append(TemporalInterval(start=cursor, end=self._coerce_datetime(domain.end), label="gap"))
|
||||
return gaps
|
||||
|
||||
def coverage_percentage(
|
||||
self,
|
||||
intervals: Iterable[TemporalInterval],
|
||||
domain_start: Any,
|
||||
domain_end: Any,
|
||||
) -> float:
|
||||
domain = self._make_interval(domain_start, domain_end, label="domain")
|
||||
domain_duration = (self._coerce_datetime(domain.end) - domain.start).total_seconds()
|
||||
if domain_duration <= 0:
|
||||
return 0.0
|
||||
covered = 0.0
|
||||
for interval in self.merge_intervals(self._clip_to_domain(intervals, domain)):
|
||||
covered += (self._end_as_datetime(interval.end, domain.end) - interval.start).total_seconds()
|
||||
return max(0.0, min(1.0, covered / domain_duration))
|
||||
|
||||
def timeline_of(self, entity_id: Any, graph: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
entity_key = str(entity_id)
|
||||
events: List[Dict[str, Any]] = []
|
||||
|
||||
for fact in graph.get("entities", []):
|
||||
if str(fact.get("id", fact.get("name", ""))) != entity_key:
|
||||
continue
|
||||
events.extend(self._events_for_fact(fact))
|
||||
|
||||
for fact in graph.get("relationships", []):
|
||||
if str(fact.get("source")) != entity_key and str(fact.get("target")) != entity_key:
|
||||
continue
|
||||
events.extend(self._events_for_fact(fact))
|
||||
|
||||
return sorted(events, key=lambda item: (item["timestamp"], item["change_type"]))
|
||||
|
||||
def retroactive_coverage(
|
||||
self,
|
||||
revision: BiTemporalFact | Dict[str, Any],
|
||||
original_facts: Iterable[BiTemporalFact | Dict[str, Any]],
|
||||
) -> Dict[str, List[BiTemporalFact | Dict[str, Any]]]:
|
||||
revision_fact = self._coerce_fact(revision)
|
||||
revision_interval = self._fact_interval(revision_fact)
|
||||
result = {"affected": [], "partial": [], "unaffected": []}
|
||||
|
||||
for fact in original_facts:
|
||||
coerced = self._coerce_fact(fact)
|
||||
original_interval = self._fact_interval(coerced)
|
||||
if self.contains(original_interval, revision_interval):
|
||||
result["affected"].append(fact)
|
||||
elif self.overlaps(original_interval, revision_interval):
|
||||
result["partial"].append(fact)
|
||||
else:
|
||||
result["unaffected"].append(fact)
|
||||
return result
|
||||
|
||||
def normalize_timestamp(self, timestamp: Any, granularity: str) -> datetime:
|
||||
granularity = self._validate_granularity(granularity)
|
||||
value = parse_temporal_value(timestamp)
|
||||
if granularity == "second":
|
||||
return value.replace(microsecond=0)
|
||||
if granularity == "minute":
|
||||
return value.replace(second=0, microsecond=0)
|
||||
if granularity == "hour":
|
||||
return value.replace(minute=0, second=0, microsecond=0)
|
||||
if granularity == "day":
|
||||
return value.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
if granularity == "week":
|
||||
start_of_week = value - timedelta(days=value.weekday())
|
||||
return start_of_week.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
if granularity == "month":
|
||||
return value.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
||||
return value.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
def normalize_interval(self, start: Any, end: Any, granularity: str) -> TemporalInterval:
|
||||
granularity = self._validate_granularity(granularity)
|
||||
normalized_start = self.normalize_timestamp(start, granularity)
|
||||
parsed_end = parse_temporal_bound(end, default=TemporalBound.OPEN)
|
||||
if parsed_end is TemporalBound.OPEN:
|
||||
return TemporalInterval(start=normalized_start, end=TemporalBound.OPEN)
|
||||
end_dt = parse_temporal_value(parsed_end)
|
||||
normalized_end = self._expand_end(end_dt, granularity)
|
||||
return TemporalInterval(start=normalized_start, end=normalized_end)
|
||||
|
||||
def _events_for_fact(self, fact: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
events: List[Dict[str, Any]] = []
|
||||
start = parse_temporal_value(fact.get("valid_from")) if fact.get("valid_from") is not None else None
|
||||
end = parse_temporal_bound(fact.get("valid_until"), default=TemporalBound.OPEN)
|
||||
recorded_at = parse_temporal_value(fact.get("recorded_at")) if fact.get("recorded_at") is not None else None
|
||||
superseded_at = parse_temporal_bound(fact.get("superseded_at"), default=TemporalBound.OPEN)
|
||||
|
||||
if start is not None:
|
||||
events.append({"timestamp": start, "change_type": "added", "fact": fact})
|
||||
if isinstance(recorded_at, datetime) and (start is None or recorded_at != start):
|
||||
events.append({"timestamp": recorded_at, "change_type": "modified", "fact": fact})
|
||||
if isinstance(superseded_at, datetime):
|
||||
events.append({"timestamp": superseded_at, "change_type": "modified", "fact": fact})
|
||||
if isinstance(end, datetime):
|
||||
events.append({"timestamp": end, "change_type": "removed", "fact": fact})
|
||||
return events
|
||||
|
||||
def _coerce_fact(self, fact: BiTemporalFact | Dict[str, Any]) -> BiTemporalFact:
|
||||
if isinstance(fact, BiTemporalFact):
|
||||
return fact
|
||||
return BiTemporalFact.from_relationship(dict(fact))
|
||||
|
||||
def _fact_interval(self, fact: BiTemporalFact) -> TemporalInterval:
|
||||
start = fact.valid_from or datetime.min.replace(tzinfo=timezone.utc)
|
||||
end = fact.valid_until if fact.valid_until is not None else TemporalBound.OPEN
|
||||
return TemporalInterval(start=start, end=end)
|
||||
|
||||
def _clip_to_domain(
|
||||
self,
|
||||
intervals: Iterable[TemporalInterval],
|
||||
domain: TemporalInterval,
|
||||
) -> List[TemporalInterval]:
|
||||
clipped: List[TemporalInterval] = []
|
||||
domain_end = self._coerce_datetime(domain.end)
|
||||
for interval in intervals:
|
||||
candidate = self._validated_copy(interval)
|
||||
if not self.overlaps(candidate, domain) and candidate.end != domain.start and candidate.start != domain_end:
|
||||
continue
|
||||
start = max(candidate.start, domain.start)
|
||||
end_dt = min(self._end_as_datetime(candidate.end, domain.end), domain_end)
|
||||
if start < end_dt:
|
||||
clipped.append(TemporalInterval(start=start, end=end_dt, label=candidate.label))
|
||||
return clipped
|
||||
|
||||
def _touches_or_overlaps(self, left: TemporalInterval, right: TemporalInterval) -> bool:
|
||||
left_end = self._end_value(left.end)
|
||||
return right.start <= left_end
|
||||
|
||||
def _make_interval(self, start: Any, end: Any, *, label: Optional[str] = None) -> TemporalInterval:
|
||||
interval = TemporalInterval(
|
||||
start=parse_temporal_value(start),
|
||||
end=parse_temporal_bound(end, default=TemporalBound.OPEN),
|
||||
label=label,
|
||||
)
|
||||
return self._validated_copy(interval)
|
||||
|
||||
def _validated_copy(self, interval: TemporalInterval) -> TemporalInterval:
|
||||
normalized = TemporalInterval(
|
||||
start=parse_temporal_value(interval.start),
|
||||
end=parse_temporal_bound(interval.end, default=TemporalBound.OPEN),
|
||||
label=interval.label,
|
||||
)
|
||||
self._validate_interval(normalized)
|
||||
return normalized
|
||||
|
||||
def _validate_interval(self, interval: TemporalInterval) -> None:
|
||||
if isinstance(interval.end, datetime) and interval.start > interval.end:
|
||||
raise ValueError("Temporal intervals must satisfy start <= end.")
|
||||
|
||||
def _end_value(self, value: datetime | TemporalBound) -> datetime:
|
||||
if value is TemporalBound.OPEN:
|
||||
return datetime.max.replace(tzinfo=timezone.utc)
|
||||
return self._coerce_datetime(value)
|
||||
|
||||
def _end_as_datetime(self, value: datetime | TemporalBound, fallback: datetime | TemporalBound) -> datetime:
|
||||
if value is TemporalBound.OPEN:
|
||||
return self._coerce_datetime(fallback)
|
||||
return self._coerce_datetime(value)
|
||||
|
||||
def _coerce_datetime(self, value: Any) -> datetime:
|
||||
return parse_temporal_value(value)
|
||||
|
||||
def _max_end(self, left: datetime | TemporalBound, right: datetime | TemporalBound) -> datetime | TemporalBound:
|
||||
if left is TemporalBound.OPEN or right is TemporalBound.OPEN:
|
||||
return TemporalBound.OPEN
|
||||
return max(self._coerce_datetime(left), self._coerce_datetime(right))
|
||||
|
||||
def _max_datetime(self, left: datetime, right: datetime) -> datetime:
|
||||
return left if left >= right else right
|
||||
|
||||
def _validate_granularity(self, granularity: str) -> str:
|
||||
if granularity not in self.SUPPORTED_GRANULARITIES:
|
||||
raise ValueError(f"Unsupported temporal granularity: {granularity}")
|
||||
return granularity
|
||||
|
||||
def _expand_end(self, value: datetime, granularity: str) -> datetime:
|
||||
floor = self.normalize_timestamp(value, granularity)
|
||||
if granularity == "second":
|
||||
return floor + timedelta(seconds=1) - timedelta(microseconds=1)
|
||||
if granularity == "minute":
|
||||
return floor + timedelta(minutes=1) - timedelta(microseconds=1)
|
||||
if granularity == "hour":
|
||||
return floor + timedelta(hours=1) - timedelta(microseconds=1)
|
||||
if granularity == "day":
|
||||
return floor + timedelta(days=1) - timedelta(microseconds=1)
|
||||
if granularity == "week":
|
||||
return floor + timedelta(weeks=1) - timedelta(microseconds=1)
|
||||
if granularity == "month":
|
||||
_, days = calendar.monthrange(floor.year, floor.month)
|
||||
return floor.replace(day=days, hour=23, minute=59, second=59, microsecond=999999)
|
||||
return floor.replace(month=12, day=31, hour=23, minute=59, second=59, microsecond=999999)
|
||||
@@ -10,6 +10,7 @@ from .owl_generator import OWLGenerator
|
||||
from .ontology_evaluator import OntologyEvaluator
|
||||
from .ontology_validator import OntologyValidator
|
||||
from .llm_generator import LLMOntologyGenerator
|
||||
from ..semantic_extract.triplet_extractor import Triplet
|
||||
|
||||
|
||||
class OntologyEngine:
|
||||
@@ -25,6 +26,11 @@ class OntologyEngine:
|
||||
self.evaluator = OntologyEvaluator(**config)
|
||||
self.validator = OntologyValidator(**config)
|
||||
self.llm = LLMOntologyGenerator(**config)
|
||||
self.store = config.get("store")
|
||||
|
||||
# Deferred to avoid circular import: change_management → ontology → change_management
|
||||
from ..change_management.ontology_version_manager import VersionManager
|
||||
self.version_manager = config.get("version_manager") or VersionManager(**config)
|
||||
|
||||
def from_data(self, data: Dict[str, Any], **options) -> Dict[str, Any]:
|
||||
tracking_id = self.progress.start_tracking(
|
||||
@@ -56,7 +62,135 @@ class OntologyEngine:
|
||||
**options,
|
||||
) -> List[Dict[str, Any]]:
|
||||
return self.propgen.infer_properties(entities, relationships, classes, **options)
|
||||
|
||||
def _sanitize_uri(self, uri: str) -> str:
|
||||
"""Prevent SPARQL injection by percent-encoding dangerous characters."""
|
||||
if not isinstance(uri, str):
|
||||
return ""
|
||||
return uri.replace("<", "%3C").replace(">", "%3E")
|
||||
|
||||
def create_alignment(self, source_uri: str, target_uri: str, predicate: str, **options) -> None:
|
||||
"""
|
||||
Creates an alignment between two ontology entities and stores it.
|
||||
"""
|
||||
if not self.store:
|
||||
raise ProcessingError("TripletStore instance not configured in OntologyEngine.")
|
||||
|
||||
if not predicate.startswith(("http://", "https://")):
|
||||
raise ProcessingError(
|
||||
f"predicate must be a full URI (e.g. 'http://www.w3.org/2002/07/owl#equivalentClass'), "
|
||||
f"not a CURIE: '{predicate}'"
|
||||
)
|
||||
|
||||
tracking_id = self.progress.start_tracking(
|
||||
module="ontology",
|
||||
submodule="OntologyEngine",
|
||||
message=f"Creating alignment: {source_uri} -> {target_uri}"
|
||||
)
|
||||
try:
|
||||
triplet = Triplet(subject=source_uri, predicate=predicate, object=target_uri)
|
||||
self.store.add_triplet(triplet, **options)
|
||||
|
||||
self.progress.stop_tracking(tracking_id, status="completed", message="Alignment created")
|
||||
except Exception as e:
|
||||
self.progress.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
self.logger.error(f"Failed to create alignment: {e}")
|
||||
raise ProcessingError(f"Alignment creation failed: {e}")
|
||||
|
||||
def get_alignments(self, entity_uri: str, **options) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Retrieves all alignments for a specific entity URI (bidirectional).
|
||||
"""
|
||||
if not self.store:
|
||||
raise ProcessingError("TripletStore instance not configured in OntologyEngine.")
|
||||
|
||||
safe_uri = self._sanitize_uri(entity_uri)
|
||||
|
||||
query = f"""
|
||||
SELECT ?s ?p ?o WHERE {{
|
||||
{{ <{safe_uri}> ?p ?o . BIND(<{safe_uri}> AS ?s) }}
|
||||
UNION
|
||||
{{ ?s ?p <{safe_uri}> . BIND(<{safe_uri}> AS ?o) }}
|
||||
|
||||
FILTER (?p IN (
|
||||
<http://www.w3.org/2002/07/owl#equivalentClass>,
|
||||
<http://www.w3.org/2002/07/owl#equivalentProperty>,
|
||||
<http://www.w3.org/2002/07/owl#sameAs>,
|
||||
<http://www.w3.org/2004/02/skos/core#exactMatch>,
|
||||
<http://www.w3.org/2004/02/skos/core#closeMatch>,
|
||||
<http://www.w3.org/2004/02/skos/core#broadMatch>,
|
||||
<http://www.w3.org/2004/02/skos/core#narrowMatch>,
|
||||
<http://www.w3.org/2004/02/skos/core#relatedMatch>
|
||||
))
|
||||
}}
|
||||
"""
|
||||
try:
|
||||
results = self.store.execute_query(query, **options)
|
||||
|
||||
alignments = []
|
||||
if hasattr(results, 'bindings'):
|
||||
for b in results.bindings:
|
||||
alignments.append({
|
||||
"source": b.get("s", {}).get("value") if isinstance(b.get("s"), dict) else b.get("s"),
|
||||
"predicate": b.get("p", {}).get("value") if isinstance(b.get("p"), dict) else b.get("p"),
|
||||
"target": b.get("o", {}).get("value") if isinstance(b.get("o"), dict) else b.get("o")
|
||||
})
|
||||
return alignments
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to get alignments for {entity_uri}: {e}")
|
||||
raise ProcessingError(f"Failed to get alignments: {e}")
|
||||
|
||||
def list_alignments(self, ontology_uri: Optional[str] = None, **options) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Lists all alignments, optionally filtered by an ontology URI.
|
||||
"""
|
||||
if not self.store:
|
||||
raise ProcessingError("TripletStore instance not configured in OntologyEngine.")
|
||||
|
||||
filter_clause = ""
|
||||
if ontology_uri:
|
||||
# Sanitize characters that could break out of the SPARQL string literal or WHERE block
|
||||
safe_ontology_uri = (
|
||||
ontology_uri
|
||||
.replace("\\", "%5C")
|
||||
.replace('"', '%22')
|
||||
.replace("{", "%7B")
|
||||
.replace("}", "%7D")
|
||||
)
|
||||
filter_clause = f'FILTER(STRSTARTS(STR(?s), "{safe_ontology_uri}") || STRSTARTS(STR(?o), "{safe_ontology_uri}"))'
|
||||
|
||||
query = f"""
|
||||
SELECT ?s ?p ?o WHERE {{
|
||||
?s ?p ?o .
|
||||
FILTER (?p IN (
|
||||
<http://www.w3.org/2002/07/owl#equivalentClass>,
|
||||
<http://www.w3.org/2002/07/owl#equivalentProperty>,
|
||||
<http://www.w3.org/2002/07/owl#sameAs>,
|
||||
<http://www.w3.org/2004/02/skos/core#exactMatch>,
|
||||
<http://www.w3.org/2004/02/skos/core#closeMatch>,
|
||||
<http://www.w3.org/2004/02/skos/core#broadMatch>,
|
||||
<http://www.w3.org/2004/02/skos/core#narrowMatch>,
|
||||
<http://www.w3.org/2004/02/skos/core#relatedMatch>
|
||||
))
|
||||
{filter_clause}
|
||||
}}
|
||||
"""
|
||||
try:
|
||||
results = self.store.execute_query(query, **options)
|
||||
|
||||
alignments = []
|
||||
if hasattr(results, 'bindings'):
|
||||
for b in results.bindings:
|
||||
alignments.append({
|
||||
"source": b.get("s", {}).get("value") if isinstance(b.get("s"), dict) else b.get("s"),
|
||||
"predicate": b.get("p", {}).get("value") if isinstance(b.get("p"), dict) else b.get("p"),
|
||||
"target": b.get("o", {}).get("value") if isinstance(b.get("o"), dict) else b.get("o")
|
||||
})
|
||||
return alignments
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to list alignments: {e}")
|
||||
raise ProcessingError(f"Failed to list alignments: {e}")
|
||||
|
||||
def evaluate(self, ontology: Dict[str, Any], **options):
|
||||
return self.evaluator.evaluate_ontology(ontology, **options)
|
||||
|
||||
@@ -69,3 +203,70 @@ class OntologyEngine:
|
||||
def export_owl(self, ontology: Dict[str, Any], path: str, format: str = "turtle"):
|
||||
return self.owl.export_owl(ontology, path, format=format)
|
||||
|
||||
def get_ontology_version_dict(self, version_id: str) -> Dict[str, Any]:
|
||||
"""Utility to load an ontology version as plain dict ready for diffing."""
|
||||
version_record = self.version_manager.get_version(version_id)
|
||||
if not version_record:
|
||||
raise ProcessingError(f"Version {version_id} not found.")
|
||||
return version_record.metadata.get("structure", {"classes": [], "properties": []})
|
||||
|
||||
def compare_versions(self, base_id: str, target_id: str, **options) -> Dict[str, Any]:
|
||||
"""
|
||||
Orchestrates version loading, diff computation, and report generation.
|
||||
|
||||
Args:
|
||||
base_id: Version ID of the old ontology
|
||||
target_id: Version ID of the new ontology
|
||||
**options: Can pass 'base_dict' and 'target_dict' directly to bypass loading.
|
||||
Can pass 'run_validation=True' to validate schema.
|
||||
Can pass 'graph_data' to validate instances against new schema.
|
||||
|
||||
Returns:
|
||||
A structured dictionary containing the impact report and machine-readable diff.
|
||||
"""
|
||||
tracking_id = self.progress.start_tracking(
|
||||
module="ontology",
|
||||
submodule="OntologyEngine",
|
||||
message=f"Comparing ontology versions: {base_id} -> {target_id}"
|
||||
)
|
||||
|
||||
try:
|
||||
# Deferred to avoid circular import
|
||||
from ..change_management.change_log import generate_change_report
|
||||
from ..kg.graph_validator import GraphValidator
|
||||
|
||||
base_dict = options.get("base_dict") or self.get_ontology_version_dict(base_id)
|
||||
target_dict = options.get("target_dict") or self.get_ontology_version_dict(target_id)
|
||||
|
||||
diff_result = self.version_manager.diff_ontologies(base_dict, target_dict)
|
||||
report = generate_change_report(diff_result)
|
||||
report["diff"] = diff_result
|
||||
|
||||
if options.get("run_validation"):
|
||||
self.progress.update_tracking(tracking_id, message="Running validation on target schema...")
|
||||
val_res = self.validate(target_dict, **options)
|
||||
report["validation_results"] = {
|
||||
"valid": getattr(val_res, "valid", getattr(val_res, "is_valid", False)),
|
||||
"consistent": getattr(val_res, "consistent", True),
|
||||
"satisfiable": getattr(val_res, "satisfiable", True),
|
||||
"errors": getattr(val_res, "errors", []),
|
||||
"warnings": getattr(val_res, "warnings", [])
|
||||
}
|
||||
|
||||
if "graph_data" in options:
|
||||
self.progress.update_tracking(tracking_id, message="Running graph data validation...")
|
||||
kg_validator = GraphValidator(**self.config)
|
||||
kg_res = kg_validator.validate(options["graph_data"], ontology=target_dict, **options)
|
||||
report["graph_validation"] = {
|
||||
"valid": getattr(kg_res, "valid", getattr(kg_res, "is_valid", False)),
|
||||
"errors": getattr(kg_res, "errors", []),
|
||||
"warnings": getattr(kg_res, "warnings", [])
|
||||
}
|
||||
|
||||
self.progress.stop_tracking(tracking_id, status="completed", message="Comparison complete")
|
||||
return report
|
||||
|
||||
except Exception as e:
|
||||
self.progress.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
self.logger.error(f"Failed to compare versions: {e}")
|
||||
raise ProcessingError(f"Version comparison failed: {e}") from e
|
||||
|
||||
@@ -206,6 +206,31 @@ class NamespaceManager:
|
||||
Dictionary of prefix -> URI mappings
|
||||
"""
|
||||
return dict(self.namespaces)
|
||||
|
||||
def get_alignment_predicates(self) -> Dict[str, str]:
|
||||
"""
|
||||
Get standard alignment predicates for ontology mapping.
|
||||
|
||||
Returns:
|
||||
Dictionary mapping common alignment types to their full URIs.
|
||||
"""
|
||||
owl_ns = self.get_namespace("owl")
|
||||
skos_ns = self.get_namespace("skos")
|
||||
|
||||
return {
|
||||
#OWL alignments
|
||||
"equivalentClass": f"{owl_ns}equivalentClass",
|
||||
"equivalentProperty": f"{owl_ns}equivalentProperty",
|
||||
"sameAs": f"{owl_ns}sameAs",
|
||||
#SKOS alignments
|
||||
"exactMatch": f"{skos_ns}exactMatch",
|
||||
"closeMatch": f"{skos_ns}closeMatch",
|
||||
"broadMatch": f"{skos_ns}broadMatch",
|
||||
"narrowMatch": f"{skos_ns}narrowMatch",
|
||||
"relatedMatch": f"{skos_ns}relatedMatch",
|
||||
|
||||
}
|
||||
|
||||
|
||||
def _to_pascal_case(self, name: str) -> str:
|
||||
"""Convert name to PascalCase."""
|
||||
|
||||
@@ -363,6 +363,54 @@ class ReuseManager:
|
||||
def list_known_ontologies(self) -> List[str]:
|
||||
"""List known ontology URIs."""
|
||||
return list(self.known_ontologies.keys())
|
||||
|
||||
def suggest_alignments(
|
||||
self, target: Dict[str, Any], source: Dict[str, Any], **options
|
||||
) -> List[Dict[str, str]]:
|
||||
"""
|
||||
Suggest alignments between a source and target ontology based on heuristics.
|
||||
"""
|
||||
suggestions = []
|
||||
|
||||
# Nested function for DRY
|
||||
|
||||
def find_matches(target_items, source_items, entity_type):
|
||||
predicate = (
|
||||
"http://www.w3.org/2002/07/owl#equivalentClass"
|
||||
if entity_type == "class"
|
||||
else "http://www.w3.org/2002/07/owl#equivalentProperty"
|
||||
)
|
||||
|
||||
# Build hash map of target entities by normalized name
|
||||
target_map = {}
|
||||
for t_item in target_items:
|
||||
t_uri = t_item.get("uri")
|
||||
t_name = t_item.get("name", "").strip().lower()
|
||||
if t_uri and t_name:
|
||||
target_map.setdefault(t_name, []).append(t_uri)
|
||||
|
||||
# Single pass through source items checking the hash map
|
||||
for s_item in source_items:
|
||||
s_uri = s_item.get("uri")
|
||||
s_name = s_item.get("name", "").strip().lower()
|
||||
if not s_uri or not s_name:
|
||||
continue
|
||||
|
||||
if s_name in target_map:
|
||||
for t_uri in target_map[s_name]:
|
||||
if s_uri != t_uri:
|
||||
suggestions.append({
|
||||
"source_uri": s_uri,
|
||||
"target_uri": t_uri,
|
||||
"predicate": predicate,
|
||||
"reason": f"Exact label match for {entity_type}: '{s_item.get('name')}'"
|
||||
})
|
||||
|
||||
find_matches(target.get("classes", []), source.get("classes", []), "class")
|
||||
find_matches(target.get("properties", []), source.get("properties", []), "property")
|
||||
|
||||
return suggestions
|
||||
|
||||
|
||||
def merge_ontology_data(
|
||||
self, target: Dict[str, Any], source: Dict[str, Any], **options
|
||||
@@ -437,6 +485,16 @@ class ReuseManager:
|
||||
for imp in source["imports"]:
|
||||
if imp not in target["imports"]:
|
||||
target["imports"].append(imp)
|
||||
|
||||
if options.get("compute_alignments", False):
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Computing suggested alignments..."
|
||||
)
|
||||
suggested = self.suggest_alignments(target, source, **options)
|
||||
if suggested:
|
||||
if "suggested_alignments" not in target:
|
||||
target["suggested_alignments"] = []
|
||||
target["suggested_alignments"].extend(suggested)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
|
||||
@@ -33,9 +33,6 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import pdfplumber
|
||||
from PIL import Image
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
@@ -119,6 +116,13 @@ class PDFParser:
|
||||
raise ValidationError(f"File is not a PDF: {file_path}")
|
||||
|
||||
try:
|
||||
try:
|
||||
import pdfplumber
|
||||
except ImportError:
|
||||
raise ProcessingError(
|
||||
"pdfplumber is required for PDF parsing. "
|
||||
"Install with: pip install pdfplumber"
|
||||
)
|
||||
with pdfplumber.open(str(file_path)) as pdf:
|
||||
# Extract metadata
|
||||
metadata = self._extract_metadata(pdf)
|
||||
|
||||
@@ -32,8 +32,6 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from pptx import Presentation
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
@@ -97,6 +95,13 @@ class PPTXParser:
|
||||
raise ValidationError(f"File is not a PPTX: {file_path}")
|
||||
|
||||
try:
|
||||
try:
|
||||
from pptx import Presentation
|
||||
except ImportError:
|
||||
raise ProcessingError(
|
||||
"python-pptx is required for PPTX parsing. "
|
||||
"Install with: pip install python-pptx"
|
||||
)
|
||||
prs = Presentation(str(file_path))
|
||||
|
||||
# Extract metadata
|
||||
@@ -220,7 +225,7 @@ class PPTXParser:
|
||||
images=images,
|
||||
)
|
||||
|
||||
def _extract_metadata(self, prs: Presentation) -> Dict[str, Any]:
|
||||
def _extract_metadata(self, prs: Any) -> Dict[str, Any]:
|
||||
"""Extract presentation metadata."""
|
||||
metadata = {}
|
||||
|
||||
|
||||
@@ -116,7 +116,7 @@ class PipelineBuilder:
|
||||
self.step_registry: Dict[str, Callable] = {}
|
||||
self.pipeline_config: Dict[str, Any] = {}
|
||||
|
||||
def add_step(self, step_name: str, step_type: str, **config) -> "PipelineBuilder":
|
||||
def add_step(self, step_name: str, step_type: str, **config) -> "PipelineStep":
|
||||
"""
|
||||
Add step to pipeline.
|
||||
|
||||
@@ -126,7 +126,7 @@ class PipelineBuilder:
|
||||
**config: Step configuration
|
||||
|
||||
Returns:
|
||||
Self for method chaining
|
||||
Created PipelineStep object
|
||||
"""
|
||||
delta_mode = config.pop("delta_mode", False)
|
||||
base_version_id = config.pop("base_version_id", None)
|
||||
|
||||
@@ -3,7 +3,8 @@ Reasoning Module
|
||||
|
||||
This module provides reasoning and inference capabilities for knowledge graph
|
||||
analysis and query answering, supporting multiple reasoning strategies including
|
||||
rule-based inference via Rete, SPARQL reasoning, abductive and deductive reasoning.
|
||||
rule-based inference via Rete, SPARQL reasoning, abductive and deductive reasoning,
|
||||
and native Datalog evaluation.
|
||||
"""
|
||||
|
||||
from .reasoner import Reasoner, InferenceResult, Rule, Fact, RuleType
|
||||
@@ -25,6 +26,9 @@ from .rete_engine import (
|
||||
)
|
||||
from .sparql_reasoner import SPARQLQueryResult, SPARQLReasoner
|
||||
|
||||
from .datalog_reasoner import DatalogReasoner, DatalogFact, DatalogRule
|
||||
from .temporal_reasoning import IntervalRelation, TemporalInterval, TemporalReasoningEngine
|
||||
|
||||
__all__ = [
|
||||
# Reasoner facade
|
||||
"Reasoner",
|
||||
@@ -43,6 +47,13 @@ __all__ = [
|
||||
# SPARQL reasoning
|
||||
"SPARQLReasoner",
|
||||
"SPARQLQueryResult",
|
||||
# Datalog reasoning
|
||||
"DatalogReasoner",
|
||||
"DatalogFact",
|
||||
"DatalogRule",
|
||||
"TemporalInterval",
|
||||
"IntervalRelation",
|
||||
"TemporalReasoningEngine",
|
||||
# Explanation
|
||||
"ExplanationGenerator",
|
||||
"Explanation",
|
||||
|
||||
@@ -0,0 +1,432 @@
|
||||
"""
|
||||
Datalog reasoner module
|
||||
|
||||
This module provides a native Datalog engine using bottom-up semi-naive fixpoint evaluation.
|
||||
It supports recursive rules, multi-hop inference, and guarantees termination on finite graphs.
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple, Union
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
# data structs
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatalogFact:
|
||||
"""Represents a ground truth fact."""
|
||||
predicate: str
|
||||
args: Tuple[str, ...]
|
||||
|
||||
class BodyAtom(NamedTuple):
|
||||
"""Represents a single predicate condition in a rule's body."""
|
||||
predicate: str
|
||||
args: Tuple[str, ...]
|
||||
|
||||
@dataclass
|
||||
class DatalogRule:
|
||||
"""Represents a Horn clause rule."""
|
||||
head_predicate: str
|
||||
head_args: Tuple[str, ...]
|
||||
body: List[BodyAtom]
|
||||
|
||||
|
||||
# datalog reasoner
|
||||
|
||||
class DatalogReasoner:
|
||||
"""
|
||||
Datalog reasoning engine supporting recursive rule evaluation via semi-naive
|
||||
bottom-up fixpoint computation.
|
||||
"""
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None, **kwargs):
|
||||
self.logger = get_logger("datalog_reasoner")
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self._fact_index: Dict[str, Set[DatalogFact]] = defaultdict(set)
|
||||
self._all_facts: Set[DatalogFact] = set()
|
||||
|
||||
self._rules: List[DatalogRule] = []
|
||||
self._derived: bool = False
|
||||
|
||||
self._delta_old: Set[DatalogFact] = set()
|
||||
self._delta_new: Set[DatalogFact] = set()
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all facts and rules from the engine."""
|
||||
self._fact_index.clear()
|
||||
self._all_facts.clear()
|
||||
self._rules.clear()
|
||||
self._derived = False
|
||||
self._delta_old.clear()
|
||||
self._delta_new.clear()
|
||||
|
||||
def add_fact(self, fact: Any) -> None:
|
||||
"""
|
||||
Add a ground fact to the engine.
|
||||
Accepts strings like "parent(tom, bob)" or standard Semantica Dicts.
|
||||
"""
|
||||
parsed_fact = None
|
||||
|
||||
if isinstance(fact, str):
|
||||
parsed_fact = self._parse_fact_string(fact)
|
||||
elif isinstance(fact, dict):
|
||||
if "subject" in fact and "predicate" in fact and "object" in fact:
|
||||
parsed_fact = DatalogFact(
|
||||
predicate=str(fact["predicate"]).replace(' ', '_').lower(),
|
||||
args=(str(fact["subject"]).replace(' ', '_').lower(), str(fact["object"]).replace(' ', '_').lower())
|
||||
)
|
||||
elif "source" in fact or "source_id" in fact or "source_name" in fact:
|
||||
source = fact.get("source", fact.get("source_name", fact.get("source_id")))
|
||||
target = fact.get("target", fact.get("target_name", fact.get("target_id")))
|
||||
rtype = fact.get("type", fact.get("relation", "connected_to"))
|
||||
if source and target:
|
||||
parsed_fact = DatalogFact(
|
||||
predicate=str(rtype).replace(' ', '_').lower(),
|
||||
args=(str(source).replace(' ', '_').lower(), str(target).replace(' ', '_').lower())
|
||||
)
|
||||
|
||||
elif "type" in fact and ("id" in fact or "name" in fact):
|
||||
name = fact.get("id", fact.get("name"))
|
||||
etype = fact.get("type", "entity")
|
||||
if name:
|
||||
parsed_fact = DatalogFact(
|
||||
predicate=str(etype).replace(' ', '_').lower(),
|
||||
args=(str(name).replace(' ', '_').lower(),)
|
||||
)
|
||||
|
||||
|
||||
if parsed_fact:
|
||||
for arg in parsed_fact.args:
|
||||
if not arg:
|
||||
raise ValueError("Facts cannot contain empty arguments")
|
||||
if arg[0].isupper():
|
||||
raise ValueError(f"Facts must be constants only. Found variable '{arg}' in {fact}")
|
||||
|
||||
if parsed_fact is None and isinstance(fact, dict):
|
||||
self.logger.warning(f"Unrecognised dict fact format, skipping: {fact}")
|
||||
return
|
||||
|
||||
if parsed_fact and parsed_fact not in self._all_facts:
|
||||
self._all_facts.add(parsed_fact)
|
||||
self._fact_index[parsed_fact.predicate].add(parsed_fact)
|
||||
self._derived = False
|
||||
|
||||
def add_rule(self, rule_str: str) -> None:
|
||||
""" Add a Datalog rule using Horn clause syntax."""
|
||||
rule = self._parse_rule_string(rule_str)
|
||||
self._rules.append(rule)
|
||||
self._derived = False
|
||||
|
||||
# Parsing helpers
|
||||
|
||||
def _parse_fact_string(self, s: str) -> DatalogFact:
|
||||
"""Parse 'predicate(arg1, arg2)' into a DatalogFact."""
|
||||
match = re.match(r'^\s*([a-zA-Z0-9_]+)\s*\(\s*([^)]+)\s*\)\s*\.?\s*$', s.strip())
|
||||
if not match:
|
||||
raise ValueError(f"Invalid fact syntax: {s}")
|
||||
|
||||
predicate = match.group(1)
|
||||
args_str = match.group(2)
|
||||
args = tuple(arg.strip() for arg in args_str.split(','))
|
||||
|
||||
for arg in args:
|
||||
if not arg:
|
||||
raise ValueError(f"Empty argument found in fact: {s}")
|
||||
if arg[0].isupper():
|
||||
raise ValueError(f"Facts must be constants only (no variables). Found variable '{arg}' in {s}")
|
||||
|
||||
return DatalogFact(predicate, args)
|
||||
|
||||
def _parse_rule_string(self, s: str) -> DatalogRule:
|
||||
"""Parse 'head(X, Y) :- body1(X, Z), body2(Z, Y).' into a DatalogRule."""
|
||||
s = s.strip()
|
||||
if ":-" not in s:
|
||||
raise ValueError(f"Invalid rule syntax (missing ':-'): {s}")
|
||||
|
||||
head_str, body_str = s.split(":-", 1)
|
||||
head_str = head_str.strip()
|
||||
body_str = body_str.strip().rstrip('.')
|
||||
|
||||
# Parse head
|
||||
head_match = re.match(r'^([a-zA-Z0-9_]+)\s*\(\s*([^)]+)\s*\)$', head_str)
|
||||
if not head_match:
|
||||
raise ValueError(f"Invalid rule head syntax: {head_str}")
|
||||
|
||||
head_pred = head_match.group(1)
|
||||
head_args = tuple(arg.strip() for arg in head_match.group(2).split(','))
|
||||
|
||||
# Parse body atoms
|
||||
body = []
|
||||
atom_matches = re.findall(r'([a-zA-Z0-9_]+)\s*\(\s*([^)]+)\s*\)', body_str)
|
||||
if not atom_matches:
|
||||
raise ValueError(f"No valid body atoms found in rule: {s}")
|
||||
|
||||
for pred, args_str in atom_matches:
|
||||
args = tuple(arg.strip() for arg in args_str.split(','))
|
||||
body.append(BodyAtom(pred, args))
|
||||
|
||||
return DatalogRule(head_pred, head_args, body)
|
||||
|
||||
# Unification & Instantiation
|
||||
|
||||
|
||||
def _is_variable(self, term: str) -> bool:
|
||||
"""Variables strictly start with an uppercase letter."""
|
||||
return bool(term and term[0].isupper())
|
||||
|
||||
def _unify(
|
||||
self,
|
||||
pattern_args: Tuple[str, ...],
|
||||
fact_args: Tuple[str, ...],
|
||||
bindings: Dict[str, str]
|
||||
) -> Optional[Dict[str, str]]:
|
||||
"""
|
||||
Unifies a rule atom's pattern with a concrete fact.
|
||||
Optimized to prevent unnecessary dictionary allocations.
|
||||
"""
|
||||
if len(pattern_args) != len(fact_args):
|
||||
return None
|
||||
|
||||
new_additions = {}
|
||||
|
||||
for p_arg, f_arg in zip(pattern_args, fact_args):
|
||||
if self._is_variable(p_arg):
|
||||
if p_arg in bindings:
|
||||
if bindings[p_arg] != f_arg:
|
||||
return None
|
||||
elif p_arg in new_additions:
|
||||
if new_additions[p_arg] != f_arg:
|
||||
return None
|
||||
else:
|
||||
new_additions[p_arg] = f_arg
|
||||
else:
|
||||
if p_arg != f_arg:
|
||||
return None
|
||||
|
||||
if new_additions:
|
||||
return {**bindings, **new_additions}
|
||||
return bindings
|
||||
|
||||
def _instantiate(self, args: Tuple[str, ...], bindings: Dict[str, str]) -> Optional[Tuple[str, ...]]:
|
||||
"""Replaces variables in a tuple with their bound values."""
|
||||
result = []
|
||||
for arg in args:
|
||||
if self._is_variable(arg):
|
||||
if arg not in bindings:
|
||||
return None
|
||||
result.append(bindings[arg])
|
||||
else:
|
||||
result.append(arg)
|
||||
|
||||
return tuple(result)
|
||||
|
||||
def _instantiate_fact(
|
||||
self, predicate: str, args: Tuple[str, ...], bindings: Dict[str, str]
|
||||
) -> Optional[DatalogFact]:
|
||||
"""Creates a concrete DatalogFact from a predicate, arguments, and bindings."""
|
||||
ground_args = self._instantiate(args, bindings)
|
||||
if ground_args is None:
|
||||
return None
|
||||
return DatalogFact(predicate, ground_args)
|
||||
|
||||
# Semi-Naive Fixpoint Evaluation
|
||||
|
||||
|
||||
def derive_all(self) -> List[str]:
|
||||
"""
|
||||
Executes bottom-up semi-naive evaluation until fixpoint is reached.
|
||||
Returns a list of all derived facts as strings.
|
||||
"""
|
||||
if self._derived:
|
||||
return [f"{f.predicate}({', '.join(f.args)})" for f in self._all_facts]
|
||||
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="reasoning",
|
||||
submodule="DatalogReasoner",
|
||||
message="Starting semi-naive fixpoint evaluation"
|
||||
)
|
||||
|
||||
iteration = 0
|
||||
newly_derived_count = 0
|
||||
|
||||
try:
|
||||
self._delta_new = self._all_facts.copy()
|
||||
|
||||
while self._delta_new:
|
||||
iteration += 1
|
||||
|
||||
# Shift deltas
|
||||
self._delta_old = self._delta_new
|
||||
self._delta_new = set()
|
||||
|
||||
delta_index = defaultdict(set)
|
||||
for f in self._delta_old:
|
||||
delta_index[f.predicate].add(f)
|
||||
|
||||
for rule in self._rules:
|
||||
new_facts = self._apply_rule(rule, delta_index)
|
||||
|
||||
for fact in new_facts:
|
||||
if fact not in self._all_facts:
|
||||
self._delta_new.add(fact)
|
||||
self._all_facts.add(fact)
|
||||
self._fact_index[fact.predicate].add(fact)
|
||||
newly_derived_count += 1
|
||||
|
||||
self.logger.debug(f"Datalog Iteration {iteration}: derived {len(self._delta_new)} new facts")
|
||||
|
||||
self._derived = True
|
||||
finally:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Fixpoint reached in {iteration} iterations. {newly_derived_count} new facts derived."
|
||||
)
|
||||
|
||||
return [f"{f.predicate}({', '.join(f.args)})" for f in self._all_facts]
|
||||
|
||||
def _apply_rule(
|
||||
self, rule: DatalogRule, delta_index: Optional[Dict[str, Set[DatalogFact]]] = None
|
||||
) -> Set[DatalogFact]:
|
||||
"""
|
||||
Evaluates a single rule.
|
||||
Uses semi-naive strategy if delta_index is provided, otherwise falls back to naive evaluation.
|
||||
"""
|
||||
results = set()
|
||||
|
||||
if not rule.body:
|
||||
fact = self._instantiate_fact(rule.head_predicate, rule.head_args, {})
|
||||
if fact:
|
||||
results.add(fact)
|
||||
return results
|
||||
|
||||
is_seminaive = delta_index is not None
|
||||
evaluation_paths = range(len(rule.body)) if is_seminaive else [0]
|
||||
|
||||
for delta_index_pos in evaluation_paths:
|
||||
bindings_list = [{}]
|
||||
|
||||
for i, atom in enumerate(rule.body):
|
||||
new_bindings_list = []
|
||||
|
||||
if is_seminaive and i == delta_index_pos:
|
||||
candidate_facts = delta_index.get(atom.predicate, set())
|
||||
else:
|
||||
candidate_facts = self._fact_index.get(atom.predicate, set())
|
||||
|
||||
for bindings in bindings_list:
|
||||
for fact in candidate_facts:
|
||||
merged_bindings = self._unify(atom.args, fact.args, bindings)
|
||||
if merged_bindings is not None:
|
||||
new_bindings_list.append(merged_bindings)
|
||||
|
||||
bindings_list = new_bindings_list
|
||||
if not bindings_list:
|
||||
break
|
||||
|
||||
for final_bindings in bindings_list:
|
||||
head_fact = self._instantiate_fact(rule.head_predicate, rule.head_args, final_bindings)
|
||||
if head_fact:
|
||||
results.add(head_fact)
|
||||
|
||||
return results
|
||||
|
||||
# Query & ContextGraph Integration
|
||||
|
||||
|
||||
def query(self, pattern: str, bindings: dict = None) -> List[dict]:
|
||||
"""
|
||||
Queries the derived fact set. Automatically runs derive_all() if rules exist.
|
||||
Syntax: "ancestor(tom, ?Y)" or "ancestor(tom, ?y)"
|
||||
Returns: [{"Y": "bob"}] or [{"y": "bob"}]
|
||||
"""
|
||||
if self._rules and not self._derived:
|
||||
self.derive_all()
|
||||
|
||||
match = re.match(r'^\s*([a-zA-Z0-9_]+)\s*\(\s*([^)]+)\s*\)\s*\.?\s*$', pattern.strip())
|
||||
if not match:
|
||||
raise ValueError(f"Invalid query syntax: {pattern}")
|
||||
|
||||
pred = match.group(1)
|
||||
raw_args = tuple(arg.strip() for arg in match.group(2).split(','))
|
||||
|
||||
query_vars = {}
|
||||
pattern_args = []
|
||||
|
||||
for i, arg in enumerate(raw_args):
|
||||
if arg.startswith('?'):
|
||||
var_name = arg[1:]
|
||||
if not var_name:
|
||||
raise ValueError("Empty variable name after '?'")
|
||||
internal_var = var_name[0].upper() + var_name[1:]
|
||||
query_vars[i] = (var_name, internal_var)
|
||||
pattern_args.append(internal_var)
|
||||
elif self._is_variable(arg):
|
||||
query_vars[i] = (arg, arg)
|
||||
pattern_args.append(arg)
|
||||
else:
|
||||
pattern_args.append(arg)
|
||||
|
||||
initial_bindings = {}
|
||||
for k, v in (bindings or {}).items():
|
||||
internal_k = k[0].upper() + k[1:] if k and not k[0].isupper() else k
|
||||
initial_bindings[internal_k] = v
|
||||
|
||||
for i, arg in enumerate(pattern_args):
|
||||
if self._is_variable(arg) and arg in initial_bindings:
|
||||
pattern_args[i] = initial_bindings[arg]
|
||||
|
||||
results = []
|
||||
candidates = self._fact_index.get(pred, set())
|
||||
|
||||
for fact in candidates:
|
||||
match_bindings = self._unify(tuple(pattern_args), fact.args, {})
|
||||
if match_bindings is not None:
|
||||
result_row = {}
|
||||
for idx, (orig_var, internal_var) in query_vars.items():
|
||||
if internal_var in match_bindings:
|
||||
result_row[orig_var] = match_bindings[internal_var]
|
||||
elif internal_var in initial_bindings:
|
||||
result_row[orig_var] = initial_bindings[internal_var]
|
||||
|
||||
if result_row and result_row not in results:
|
||||
results.append(result_row)
|
||||
|
||||
return results
|
||||
|
||||
def load_from_graph(self, graph: Any) -> int:
|
||||
"""
|
||||
Loads a ContextGraph into Datalog facts using central add_fact validation.
|
||||
"""
|
||||
initial_count = len(self._all_facts)
|
||||
|
||||
if hasattr(graph, 'find_edges') and hasattr(graph, 'find_nodes'):
|
||||
for edge_dict in graph.find_edges():
|
||||
self.add_fact(edge_dict)
|
||||
for node_dict in graph.find_nodes():
|
||||
self.add_fact(node_dict)
|
||||
else:
|
||||
if hasattr(graph, 'edges'):
|
||||
edges = graph.edges() if callable(graph.edges) else graph.edges
|
||||
for edge in edges:
|
||||
self.add_fact(edge if isinstance(edge, dict) else edge.__dict__)
|
||||
|
||||
if hasattr(graph, 'nodes'):
|
||||
nodes = graph.nodes() if callable(graph.nodes) else graph.nodes
|
||||
if isinstance(nodes, dict):
|
||||
nodes = nodes.values()
|
||||
for node in nodes:
|
||||
self.add_fact(node if isinstance(node, dict) else node.__dict__)
|
||||
|
||||
facts_added = len(self._all_facts) - initial_count
|
||||
self.logger.info(f"Loaded {facts_added} facts from ContextGraph.")
|
||||
return facts_added
|
||||
|
||||
|
||||
@@ -354,10 +354,6 @@ class Reasoner:
|
||||
|
||||
# Simple regex-based matcher for patterns like "Person(?x)" and facts like "Person(John)"
|
||||
|
||||
p_regex = re.escape(pattern)
|
||||
p_regex = re.sub(r"\\\?(\w+)", r"(?P<\1>.+)", p_regex)
|
||||
p_regex = f"^{p_regex}$"
|
||||
|
||||
try:
|
||||
match = re.match(p_regex, fact)
|
||||
if match:
|
||||
@@ -365,12 +361,10 @@ class Reasoner:
|
||||
for var, value in match.groupdict().items():
|
||||
if var in new_bindings and new_bindings[var] != value:
|
||||
return None # Binding conflict
|
||||
return None # Binding conflict
|
||||
new_bindings[var] = value
|
||||
return new_bindings
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Error matching pattern '{pattern}' (regex: '{p_regex}') against fact '{fact}': {e}")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
"""
|
||||
Reasoning-layer re-export for deterministic temporal reasoning.
|
||||
|
||||
The canonical implementation lives in ``semantica.kg.temporal_reasoning``.
|
||||
"""
|
||||
|
||||
from ..kg.temporal_reasoning import IntervalRelation, TemporalInterval, TemporalReasoningEngine
|
||||
|
||||
__all__ = ["TemporalInterval", "IntervalRelation", "TemporalReasoningEngine"]
|
||||
@@ -6,7 +6,7 @@ supporting multiple configuration sources including environment variables, confi
|
||||
and programmatic configuration.
|
||||
|
||||
Supported Configuration Sources:
|
||||
- Environment variables: OPENAI_API_KEY, GEMINI_API_KEY, GROQ_API_KEY, etc.
|
||||
- Environment variables: OPENAI_API_KEY, GEMINI_API_KEY, GROQ_API_KEY, NOVITA_API_KEY, etc.
|
||||
- Config files: YAML, JSON, TOML formats
|
||||
- Programmatic: Python API for setting provider configurations
|
||||
|
||||
@@ -97,7 +97,7 @@ class Config:
|
||||
def _load_env_vars(self):
|
||||
"""Load configuration from environment variables."""
|
||||
# Common environment variable patterns
|
||||
providers = ["openai", "gemini", "groq", "anthropic", "ollama"]
|
||||
providers = ["openai", "gemini", "groq", "anthropic", "ollama", "novita"]
|
||||
for provider in providers:
|
||||
env_key = f"{provider.upper()}_API_KEY"
|
||||
api_key = os.getenv(env_key)
|
||||
|
||||
@@ -122,7 +122,12 @@ from .cache import ExtractionCache
|
||||
from .config import config
|
||||
|
||||
try:
|
||||
from .schemas import EntitiesResponse, RelationsResponse, TripletsResponse
|
||||
from .schemas import (
|
||||
EntitiesResponse,
|
||||
RelationsResponse,
|
||||
RelationsWithTemporalResponse,
|
||||
TripletsResponse,
|
||||
)
|
||||
SCHEMAS_AVAILABLE = True
|
||||
except ImportError:
|
||||
SCHEMAS_AVAILABLE = False
|
||||
@@ -683,6 +688,19 @@ def extract_entities_ml(
|
||||
"spaCy model not available, falling back to pattern extraction"
|
||||
)
|
||||
return extract_entities_pattern(text, **kwargs)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"spaCy fallback triggered because the default model failed to initialize. Falling back to pattern extraction.",
|
||||
exc_info=True,
|
||||
)
|
||||
return extract_entities_pattern(text, **kwargs)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"spaCy model %s failed to initialize, falling back to pattern extraction.",
|
||||
model,
|
||||
exc_info=True,
|
||||
)
|
||||
return extract_entities_pattern(text, **kwargs)
|
||||
|
||||
doc = nlp(text)
|
||||
entities = []
|
||||
@@ -1651,11 +1669,12 @@ def extract_relations_llm(
|
||||
max_text_length: Optional[int] = None,
|
||||
structured_output_mode: str = "typed",
|
||||
max_retries: int = 3,
|
||||
extract_temporal_bounds: bool = False,
|
||||
**kwargs,
|
||||
) -> List[Relation]:
|
||||
"""
|
||||
LLM-based relation extraction.
|
||||
|
||||
|
||||
Args:
|
||||
text: Input text
|
||||
entities: Pre-extracted entities
|
||||
@@ -1664,6 +1683,10 @@ def extract_relations_llm(
|
||||
silent_fail: If True, return empty list on error. If False (default), raise exception.
|
||||
max_text_length: Maximum text length before auto-chunking. None = provider default.
|
||||
max_retries: Maximum number of retries for LLM calls (default: 3)
|
||||
extract_temporal_bounds: If True, extend the prompt to extract temporal validity
|
||||
per relation. Each relation's metadata gains: valid_from, valid_until,
|
||||
temporal_confidence (0.0–1.0), and temporal_source_text. Low confidence (<0.5)
|
||||
produces a warning log but is not suppressed. Default False.
|
||||
**kwargs: Additional options
|
||||
"""
|
||||
# Support llm_model parameter to disambiguate from ML model
|
||||
@@ -1678,6 +1701,7 @@ def extract_relations_llm(
|
||||
"structured_output_mode": structured_output_mode,
|
||||
"max_retries": max_retries,
|
||||
"relation_types": kwargs.get("relation_types"),
|
||||
"extract_temporal_bounds": extract_temporal_bounds,
|
||||
# Include entities hash/str in cache key implicitly via **cache_params
|
||||
"entities_hash": hash(tuple(sorted([e.text for e in entities]))) if entities else 0
|
||||
}
|
||||
@@ -1746,9 +1770,10 @@ def extract_relations_llm(
|
||||
if len(text) > max_text_length:
|
||||
logger.info(f"Text length ({len(text)}) exceeds limit for relations. Chunking...")
|
||||
return _extract_relations_chunked(
|
||||
text, entities, provider=provider, model=model,
|
||||
silent_fail=silent_fail, max_text_length=max_text_length,
|
||||
text, entities, provider=provider, model=model,
|
||||
silent_fail=silent_fail, max_text_length=max_text_length,
|
||||
max_retries=max_retries,
|
||||
extract_temporal_bounds=extract_temporal_bounds,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
@@ -1764,7 +1789,7 @@ def extract_relations_llm(
|
||||
)
|
||||
|
||||
entities_str = ", ".join([f"{e.text} ({e.label})" for e in prompt_entities])
|
||||
|
||||
|
||||
# Use custom relation types if provided
|
||||
relation_types = kwargs.get("relation_types")
|
||||
if relation_types:
|
||||
@@ -1777,16 +1802,18 @@ If a relation doesn't fit any of the preferred types, use the most appropriate t
|
||||
relation_types_instruction = """
|
||||
Extract meaningful relationships between entities. Use appropriate relation types that accurately describe how entities are connected.
|
||||
Common relation types include: related_to, part_of, located_in, created_by, uses, depends_on, interacts_with, and similar variations."""
|
||||
|
||||
|
||||
verbose_mode = kwargs.get("verbose", False)
|
||||
if verbose_mode:
|
||||
import sys
|
||||
print(f" [methods.extract_relations_llm] Constructing prompt for {len(prompt_entities)} entities...", flush=True, file=sys.stdout)
|
||||
|
||||
|
||||
if not SCHEMAS_AVAILABLE:
|
||||
raise ImportError("Pydantic schemas not available. Install pydantic/instructor to use LLM extraction.")
|
||||
|
||||
prompt = f"""Extract relations between entities from the provided text.
|
||||
# ── Base prompt (always included) ───────────────────────────────────────
|
||||
if not extract_temporal_bounds:
|
||||
prompt = f"""Extract relations between entities from the provided text.
|
||||
Return the result as a JSON object with a "relations" key containing the list of relations.
|
||||
Each relation must have 'subject', 'predicate', and 'object' fields.
|
||||
|
||||
@@ -1807,115 +1834,59 @@ Instructions:
|
||||
Text to extract from:
|
||||
{text}
|
||||
Entities found in text: {entities_str}"""
|
||||
|
||||
|
||||
if not entities:
|
||||
error_msg = "No entities provided for relation extraction. Relations require existing entities."
|
||||
logger.error(error_msg)
|
||||
if not silent_fail:
|
||||
raise ProcessingError(error_msg)
|
||||
return []
|
||||
|
||||
# Pass api_key if provided in kwargs
|
||||
provider_kwargs = kwargs.copy()
|
||||
|
||||
# Check if api_key is provided but empty, or not provided at all
|
||||
if "api_key" not in provider_kwargs or not provider_kwargs["api_key"]:
|
||||
import os
|
||||
env_key = f"{provider.upper()}_API_KEY"
|
||||
api_key = os.getenv(env_key)
|
||||
if api_key:
|
||||
provider_kwargs["api_key"] = api_key
|
||||
|
||||
# Remove None/empty API key if still present to avoid provider errors
|
||||
if "api_key" in provider_kwargs and not provider_kwargs["api_key"]:
|
||||
del provider_kwargs["api_key"]
|
||||
|
||||
# 2. PROVIDER VALIDATION
|
||||
try:
|
||||
llm = create_provider(provider, model=model, **provider_kwargs)
|
||||
if not llm.is_available():
|
||||
error_msg = f"{provider} provider not available for relation extraction (key missing?)."
|
||||
logger.error(error_msg)
|
||||
if not silent_fail:
|
||||
raise ProcessingError(error_msg)
|
||||
return []
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to create {provider} provider for relations: {e}"
|
||||
logger.error(error_msg)
|
||||
if not silent_fail:
|
||||
raise ProcessingError(error_msg) from e
|
||||
return []
|
||||
|
||||
# 3. TEXT LENGTH CHECK AND CHUNKING
|
||||
if max_text_length is None:
|
||||
# Default limits for chunking only - NOT for LLM generation
|
||||
max_text_length = {
|
||||
"groq": 64000,
|
||||
"openai": 64000,
|
||||
"gemini": 64000,
|
||||
"anthropic": 64000,
|
||||
"deepseek": 64000,
|
||||
}.get(provider.lower(), 32000)
|
||||
|
||||
if len(text) > max_text_length:
|
||||
logger.info(f"Text length ({len(text)}) exceeds limit for relations. Chunking...")
|
||||
return _extract_relations_chunked(
|
||||
text, entities, provider=provider, model=model,
|
||||
silent_fail=silent_fail, max_text_length=max_text_length,
|
||||
max_retries=max_retries,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
original_entities = entities
|
||||
# Use a fixed internal default for prompt entity cap (do not accept overrides from kwargs)
|
||||
max_entities_prompt = 80
|
||||
prompt_entities = original_entities
|
||||
if max_entities_prompt > 0 and len(original_entities) > max_entities_prompt:
|
||||
prompt_entities = filter_entities_for_text(
|
||||
text,
|
||||
original_entities,
|
||||
max_keep=max_entities_prompt,
|
||||
)
|
||||
|
||||
entities_str = ", ".join([f"{e.text} ({e.label})" for e in prompt_entities])
|
||||
|
||||
# Use custom relation types if provided
|
||||
relation_types = kwargs.get("relation_types")
|
||||
if relation_types:
|
||||
relation_types_str = ", ".join(relation_types)
|
||||
relation_types_instruction = f"""
|
||||
Preferred relation types: {relation_types_str}.
|
||||
You may also use related or similar relation types if they better capture the relationship (e.g., variations, synonyms, or domain-specific relations).
|
||||
If a relation doesn't fit any of the preferred types, use the most appropriate type from the preferred list or a closely related type that accurately describes the relationship."""
|
||||
else:
|
||||
relation_types_instruction = """
|
||||
Extract meaningful relationships between entities. Use appropriate relation types that accurately describe how entities are connected.
|
||||
Common relation types include: related_to, part_of, located_in, created_by, uses, depends_on, interacts_with, and similar variations."""
|
||||
|
||||
verbose_mode = kwargs.get("verbose", False)
|
||||
if verbose_mode:
|
||||
import sys
|
||||
print(f" [methods.extract_relations_llm] Constructing prompt for {len(prompt_entities)} entities...", flush=True, file=sys.stdout)
|
||||
|
||||
if not SCHEMAS_AVAILABLE:
|
||||
raise ImportError("Pydantic schemas not available. Install pydantic/instructor to use LLM extraction.")
|
||||
# ── Temporal-extended prompt ─────────────────────────────────────────
|
||||
prompt = f"""Extract relations between entities from the provided text, along with temporal validity information for each relation.
|
||||
Return the result as a JSON object with a "relations" key. Each relation must have:
|
||||
'subject', 'predicate', 'object', 'confidence', 'valid_from', 'valid_until', 'temporal_confidence', 'temporal_source_text'.
|
||||
|
||||
prompt = f"""Extract relations between entities from the provided text.
|
||||
Return the result as a JSON object with a "relations" key containing the list of relations.
|
||||
Each relation must have 'subject', 'predicate', and 'object' fields.
|
||||
TEMPORAL EXTRACTION RULES:
|
||||
- valid_from: ISO 8601 date or exact phrase from the text for when this relation became valid. Set to null if no temporal signal is present.
|
||||
- valid_until: ISO 8601 date or exact phrase for when this relation ceased. Set to null if open-ended or absent.
|
||||
- temporal_confidence (float 0.0–1.0) — calibrated as follows:
|
||||
1.00 = full ISO date ("2022-03-15", "March 15, 2022")
|
||||
0.90 = explicit year + month ("March 2022", "2022-03")
|
||||
0.85 = explicit year only ("in 2022", "since 2021", "from 2019")
|
||||
0.75 = quarter ("Q3 2023", "Q2 2021")
|
||||
0.65 = named season or approximate range ("summer 2022", "early 2020s", "mid-2022")
|
||||
0.50 = vague relative with computable anchor ("last year", "three months ago")
|
||||
0.35 = highly vague relative ("recently", "years ago", "in the past")
|
||||
0.00 = no temporal signal present for this relation
|
||||
- temporal_source_text: the EXACT verbatim substring from the source text that contains the temporal signal. Set to null when temporal_confidence is 0.0.
|
||||
|
||||
Example output (JSON format only):
|
||||
IMPORTANT: Do NOT invent or guess dates. If the text contains no temporal signal for a relation, set valid_from and valid_until to null and temporal_confidence to 0.0.
|
||||
|
||||
Few-shot examples (do NOT include these in your output):
|
||||
Text: "Apple acquired Beats in May 2014."
|
||||
→ valid_from: "2014-05-01", valid_until: null, temporal_confidence: 0.90, temporal_source_text: "May 2014"
|
||||
|
||||
Text: "The CEO has led the company since Q3 2020."
|
||||
→ valid_from: "Q3 2020", valid_until: null, temporal_confidence: 0.75, temporal_source_text: "since Q3 2020"
|
||||
|
||||
Text: "Last year, Google partnered with Samsung."
|
||||
→ valid_from: "last year", valid_until: null, temporal_confidence: 0.50, temporal_source_text: "Last year"
|
||||
|
||||
Text: "The firm was under enhanced supervision between Q2 and Q4 2021."
|
||||
→ valid_from: "Q2 2021", valid_until: "Q4 2021", temporal_confidence: 0.75, temporal_source_text: "between Q2 and Q4 2021"
|
||||
|
||||
Text: "Microsoft develops Windows."
|
||||
→ valid_from: null, valid_until: null, temporal_confidence: 0.00, temporal_source_text: null
|
||||
|
||||
Example JSON output format:
|
||||
{{
|
||||
"relations": [
|
||||
{{"subject": "Entity A", "predicate": "related_to", "object": "Entity B", "confidence": 0.95}},
|
||||
{{"subject": "Subject Entity", "predicate": "action_verb", "object": "Object Entity", "confidence": 0.90}}
|
||||
{{
|
||||
"subject": "Apple", "predicate": "acquired", "object": "Beats",
|
||||
"confidence": 0.97,
|
||||
"valid_from": "2014-05-01", "valid_until": null,
|
||||
"temporal_confidence": 0.90, "temporal_source_text": "May 2014"
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
Instructions:
|
||||
1. Extract relations ONLY from the text provided below.
|
||||
2. Do not include any relations from the example above.
|
||||
2. Do not include any relations from the examples above.
|
||||
3. Use the provided entities list as a reference for subjects and objects.
|
||||
4. {relation_types_instruction}
|
||||
|
||||
@@ -1925,24 +1896,25 @@ Entities found in text: {entities_str}"""
|
||||
|
||||
try:
|
||||
# Use typed generation with Pydantic schema
|
||||
# Pass kwargs to allow max_tokens and other parameters to be used
|
||||
if verbose_mode:
|
||||
import sys
|
||||
print(f" [methods.extract_relations_llm] Calling llm.generate_typed ({provider}/{model})...", flush=True, file=sys.stdout)
|
||||
import sys
|
||||
print(f" [methods.extract_relations_llm] Calling llm.generate_typed ({provider}/{model})...", flush=True, file=sys.stdout)
|
||||
# Only forward minimal, safe parameters to provider calls
|
||||
call_kwargs = {}
|
||||
if "temperature" in kwargs:
|
||||
call_kwargs["temperature"] = kwargs["temperature"]
|
||||
if "verbose" in kwargs:
|
||||
call_kwargs["verbose"] = kwargs["verbose"]
|
||||
|
||||
|
||||
call_kwargs["max_retries"] = max_retries
|
||||
|
||||
result_obj = llm.generate_typed(prompt, schema=RelationsResponse, **call_kwargs)
|
||||
# Select schema based on whether temporal extraction is requested
|
||||
active_schema = RelationsWithTemporalResponse if extract_temporal_bounds else RelationsResponse
|
||||
result_obj = llm.generate_typed(prompt, schema=active_schema, **call_kwargs)
|
||||
if verbose_mode:
|
||||
import sys
|
||||
print(f" [methods.extract_relations_llm] Received response from {provider}.", flush=True, file=sys.stdout)
|
||||
|
||||
import sys
|
||||
print(f" [methods.extract_relations_llm] Received response from {provider}.", flush=True, file=sys.stdout)
|
||||
|
||||
# Convert back to internal Relation format (robust across providers)
|
||||
# Normalize typed result to a plain dict compatible with _parse_relation_result
|
||||
try:
|
||||
@@ -1959,12 +1931,16 @@ Entities found in text: {entities_str}"""
|
||||
elif isinstance(r, dict):
|
||||
rel_items.append(r)
|
||||
else:
|
||||
# Best-effort attribute access
|
||||
# Best-effort attribute access — include temporal fields when present
|
||||
rel_items.append({
|
||||
"subject": getattr(r, "subject", ""),
|
||||
"object": getattr(r, "object", ""),
|
||||
"predicate": getattr(r, "predicate", "related_to"),
|
||||
"confidence": getattr(r, "confidence", 0.9),
|
||||
"valid_from": getattr(r, "valid_from", None),
|
||||
"valid_until": getattr(r, "valid_until", None),
|
||||
"temporal_confidence": getattr(r, "temporal_confidence", 0.0),
|
||||
"temporal_source_text": getattr(r, "temporal_source_text", None),
|
||||
})
|
||||
parsed = {"relations": rel_items}
|
||||
else:
|
||||
@@ -1973,8 +1949,12 @@ Entities found in text: {entities_str}"""
|
||||
parsed = result_obj
|
||||
|
||||
# Use common parser to build internal Relation objects
|
||||
relations = _parse_relation_result(parsed, original_entities, text, provider, model)
|
||||
|
||||
relations = _parse_relation_result(
|
||||
parsed, original_entities, text, provider, model,
|
||||
extraction_method="llm_typed",
|
||||
extract_temporal_bounds=extract_temporal_bounds,
|
||||
)
|
||||
|
||||
# If typed path returned no relations, attempt a structured JSON fallback
|
||||
if not relations:
|
||||
try:
|
||||
@@ -1982,7 +1962,11 @@ Entities found in text: {entities_str}"""
|
||||
import sys
|
||||
print(" [methods.extract_relations_llm] Typed result empty, attempting structured JSON fallback...", flush=True, file=sys.stdout)
|
||||
raw_json = llm.generate_structured(prompt, **call_kwargs)
|
||||
relations = _parse_relation_result(raw_json, original_entities, text, provider, model)
|
||||
relations = _parse_relation_result(
|
||||
raw_json, original_entities, text, provider, model,
|
||||
extraction_method="llm_typed",
|
||||
extract_temporal_bounds=extract_temporal_bounds,
|
||||
)
|
||||
except Exception as _e:
|
||||
# Keep relations as empty if fallback fails
|
||||
pass
|
||||
@@ -1990,22 +1974,23 @@ Entities found in text: {entities_str}"""
|
||||
logger.info(f"Successfully extracted {len(relations)} relations using {provider}/{model} (typed)")
|
||||
_result_cache.set("relations", text, relations, **cache_params)
|
||||
return relations
|
||||
|
||||
|
||||
except Exception as e:
|
||||
# Check for length/token limit errors
|
||||
error_msg_str = str(e).lower()
|
||||
if "length" in error_msg_str or "max_tokens" in error_msg_str:
|
||||
logger.warning(f"LLM output truncated due to length limit. Reducing chunk size and retrying... ({e})")
|
||||
|
||||
|
||||
# Determine new chunk size (halve it)
|
||||
current_max = max_text_length or len(text)
|
||||
new_max = current_max // 2
|
||||
|
||||
if new_max > 100: # Minimum viable chunk size check
|
||||
|
||||
if new_max > 100: # Minimum viable chunk size check
|
||||
return _extract_relations_chunked(
|
||||
text, entities, provider=provider, model=model,
|
||||
silent_fail=silent_fail, max_text_length=new_max,
|
||||
structured_output_mode=structured_output_mode,
|
||||
extract_temporal_bounds=extract_temporal_bounds,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
@@ -2019,16 +2004,18 @@ Entities found in text: {entities_str}"""
|
||||
|
||||
|
||||
def _parse_relation_result(
|
||||
result: Any,
|
||||
entities: List[Entity],
|
||||
result: Any,
|
||||
entities: List[Entity],
|
||||
text: str,
|
||||
provider: str,
|
||||
model: Optional[str]
|
||||
provider: str,
|
||||
model: Optional[str],
|
||||
extraction_method: str = "llm",
|
||||
extract_temporal_bounds: bool = False,
|
||||
) -> List[Relation]:
|
||||
"""Helper to parse raw LLM result into Relation objects."""
|
||||
relations = []
|
||||
items = []
|
||||
|
||||
|
||||
if isinstance(result, list):
|
||||
items = result
|
||||
elif isinstance(result, dict):
|
||||
@@ -2042,13 +2029,13 @@ def _parse_relation_result(
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
|
||||
|
||||
subject_text = item.get("subject", "")
|
||||
object_text = item.get("object", "")
|
||||
|
||||
|
||||
if not subject_text or not object_text:
|
||||
continue
|
||||
|
||||
|
||||
# Ensure they are strings
|
||||
subject_text = str(subject_text)
|
||||
object_text = str(object_text)
|
||||
@@ -2071,6 +2058,33 @@ def _parse_relation_result(
|
||||
confidence=0.8, metadata={"synthetic": True},
|
||||
)
|
||||
|
||||
metadata: dict = {
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
"extraction_method": extraction_method,
|
||||
}
|
||||
|
||||
if extract_temporal_bounds:
|
||||
temporal_confidence = float(item.get("temporal_confidence") or 0.0)
|
||||
valid_from = item.get("valid_from")
|
||||
valid_until = item.get("valid_until")
|
||||
temporal_source_text = item.get("temporal_source_text")
|
||||
|
||||
metadata["valid_from"] = valid_from
|
||||
metadata["valid_until"] = valid_until
|
||||
metadata["temporal_confidence"] = temporal_confidence
|
||||
metadata["temporal_source_text"] = temporal_source_text
|
||||
|
||||
if temporal_confidence < 0.5 and (valid_from is not None or valid_until is not None):
|
||||
logger.warning(
|
||||
"Low temporal confidence (%.2f) for '%s' (%s → %s). Source: %r",
|
||||
temporal_confidence,
|
||||
item.get("predicate", ""),
|
||||
item.get("subject", ""),
|
||||
item.get("object", ""),
|
||||
temporal_source_text,
|
||||
)
|
||||
|
||||
relations.append(
|
||||
Relation(
|
||||
subject=subject_entity,
|
||||
@@ -2078,32 +2092,9 @@ def _parse_relation_result(
|
||||
object=object_entity,
|
||||
confidence=item.get("confidence", 0.9),
|
||||
context=text,
|
||||
metadata={
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
"extraction_method": "llm",
|
||||
},
|
||||
metadata=metadata,
|
||||
)
|
||||
)
|
||||
# Find matching entities using hybrid similarity
|
||||
subject_entity = match_entity(subject_text, entities)
|
||||
object_entity = match_entity(object_text, entities)
|
||||
|
||||
if subject_entity and object_entity:
|
||||
relations.append(
|
||||
Relation(
|
||||
subject=subject_entity,
|
||||
predicate=item.get("predicate", "related_to"),
|
||||
object=object_entity,
|
||||
confidence=item.get("confidence", 0.9),
|
||||
context=text,
|
||||
metadata={
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
"extraction_method": "llm",
|
||||
},
|
||||
)
|
||||
)
|
||||
return relations
|
||||
|
||||
|
||||
@@ -2116,6 +2107,7 @@ def _extract_relations_chunked(
|
||||
max_text_length: int,
|
||||
structured_output_mode: str = "typed",
|
||||
max_retries: int = 3,
|
||||
extract_temporal_bounds: bool = False,
|
||||
**kwargs
|
||||
) -> List[Relation]:
|
||||
"""Internal helper to extract relations from long text by chunking."""
|
||||
@@ -2159,6 +2151,7 @@ def _extract_relations_chunked(
|
||||
max_text_length=len(chunk.text) + 1,
|
||||
structured_output_mode=structured_output_mode,
|
||||
max_retries=max_retries,
|
||||
extract_temporal_bounds=extract_temporal_bounds,
|
||||
**limited_kwargs
|
||||
)
|
||||
future_to_chunk[future] = i
|
||||
|
||||
@@ -147,6 +147,7 @@ class NERExtractor:
|
||||
|
||||
# Initialize spaCy model if ML method is used
|
||||
self.nlp = None
|
||||
self._ml_runtime_usable = True
|
||||
if "ml" in self.method and SPACY_AVAILABLE:
|
||||
try:
|
||||
self.nlp = spacy.load(self.model_name)
|
||||
@@ -154,6 +155,13 @@ class NERExtractor:
|
||||
self.logger.warning(
|
||||
f"spaCy model {self.model_name} not found. ML method will fallback."
|
||||
)
|
||||
except Exception as exc:
|
||||
self._ml_runtime_usable = False
|
||||
self.logger.warning(
|
||||
"spaCy model %s failed to initialize and will be disabled for this extractor instance. ML method will fallback.",
|
||||
self.model_name,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def extract(self, text: Union[str, List[Dict[str, Any]], List[str]], pipeline_id: Optional[str] = None, **kwargs) -> Union[List[Entity], List[List[Entity]]]:
|
||||
"""
|
||||
@@ -342,6 +350,7 @@ class NERExtractor:
|
||||
methods = options.get("method", self.method)
|
||||
if isinstance(methods, str):
|
||||
methods = [methods]
|
||||
methods = self._filter_unusable_methods(methods)
|
||||
|
||||
min_confidence = options.get("min_confidence", self.min_confidence)
|
||||
entity_types = options.get("entity_types", self.entity_types)
|
||||
@@ -457,6 +466,24 @@ class NERExtractor:
|
||||
)
|
||||
raise
|
||||
|
||||
def _filter_unusable_methods(self, methods: List[str]) -> List[str]:
|
||||
"""Skip ML dispatch after a known spaCy runtime initialization failure."""
|
||||
filtered = []
|
||||
skipped_ml = False
|
||||
|
||||
for method_name in methods:
|
||||
if method_name in {"ml", "spacy"} and not self._ml_runtime_usable:
|
||||
skipped_ml = True
|
||||
continue
|
||||
filtered.append(method_name)
|
||||
|
||||
if skipped_ml:
|
||||
self.logger.debug(
|
||||
"Skipping ML entity extraction because spaCy runtime initialization previously failed for this extractor."
|
||||
)
|
||||
|
||||
return filtered
|
||||
|
||||
def _vote_entities(
|
||||
self, results: List[List[Entity]], threshold: float = 0.5
|
||||
) -> List[Entity]:
|
||||
|
||||
@@ -873,7 +873,7 @@ class OllamaProvider(BaseProvider):
|
||||
try:
|
||||
import ollama # type: ignore[import-untyped]
|
||||
|
||||
self.client = ollama
|
||||
self.client = ollama.Client(host=self.base_url)
|
||||
# Test connection
|
||||
try:
|
||||
self.client.list() # Test if Ollama is running
|
||||
@@ -988,6 +988,65 @@ class DeepSeekProvider(BaseProvider):
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to parse JSON from DeepSeek response: {e}")
|
||||
|
||||
|
||||
class NovitaProvider(BaseProvider):
|
||||
"""Novita AI provider implementation - OpenAI-compatible API."""
|
||||
|
||||
def __init__(self, api_key: Optional[str] = None, model: str = "deepseek/deepseek-v3.2", **kwargs):
|
||||
"""Initialize Novita provider."""
|
||||
super().__init__(**kwargs)
|
||||
self.api_key = api_key or config.get_api_key("novita")
|
||||
self.model = model
|
||||
self.base_url = "https://api.novita.ai/v1"
|
||||
self.client = None
|
||||
self._init_client()
|
||||
|
||||
def _init_client(self):
|
||||
try:
|
||||
from openai import OpenAI
|
||||
|
||||
if self.api_key:
|
||||
self.client = OpenAI(api_key=self.api_key, base_url=self.base_url)
|
||||
except (ImportError, OSError):
|
||||
self.client = None
|
||||
self.logger.warning(
|
||||
"openai library not installed. Install with: pip install semantica[llm-openai]"
|
||||
)
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return self.client is not None
|
||||
|
||||
def generate(self, prompt: str, **kwargs) -> str:
|
||||
if not self.client:
|
||||
raise ProcessingError("Novita client not initialized. Set NOVITA_API_KEY or pass api_key.")
|
||||
|
||||
create_kwargs = {
|
||||
"model": kwargs.get("model", self.model),
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
}
|
||||
self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens")
|
||||
|
||||
response = self.client.chat.completions.create(**create_kwargs)
|
||||
return response.choices[0].message.content
|
||||
|
||||
def generate_structured(self, prompt: str, **kwargs) -> Union[dict, list]:
|
||||
"""Generate structured output."""
|
||||
if not self.client:
|
||||
raise ProcessingError("Novita client not initialized.")
|
||||
|
||||
create_kwargs = {
|
||||
"model": kwargs.get("model", self.model),
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
self._add_if_set(create_kwargs, kwargs, "temperature", "max_tokens")
|
||||
|
||||
response = self.client.chat.completions.create(**create_kwargs)
|
||||
try:
|
||||
return self._parse_json(response.choices[0].message.content)
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to parse JSON from Novita response: {e}")
|
||||
|
||||
class HuggingFaceLLMProvider(BaseProvider):
|
||||
"""HuggingFace transformers for LLM tasks."""
|
||||
|
||||
@@ -1370,6 +1429,7 @@ class ProviderPool:
|
||||
"ollama": OllamaProvider,
|
||||
"huggingface_llm": HuggingFaceLLMProvider,
|
||||
"deepseek": DeepSeekProvider,
|
||||
"novita": NovitaProvider,
|
||||
}
|
||||
|
||||
provider_class = builtin.get(name.lower())
|
||||
|
||||
@@ -125,3 +125,60 @@ class RelationsResponse(BaseModel):
|
||||
class TripletsResponse(BaseModel):
|
||||
"""Wrapper for list of triplets."""
|
||||
triplets: List[TripletOut] = Field(default_factory=list)
|
||||
|
||||
|
||||
class RelationWithTemporalOut(BaseModel):
|
||||
"""Schema for relation extraction output with temporal validity bounds."""
|
||||
model_config = ConfigDict(populate_by_name=True, extra="ignore")
|
||||
|
||||
subject: str = Field(..., description="Source entity text")
|
||||
object: str = Field(..., description="Target entity text")
|
||||
predicate: str = Field(..., description="Relation type or predicate")
|
||||
confidence: float = Field(0.9, description="Confidence score between 0 and 1")
|
||||
metadata: dict = Field(default_factory=dict, description="Additional metadata including provenance")
|
||||
|
||||
valid_from: Optional[str] = Field(
|
||||
None,
|
||||
description="ISO 8601 date or natural-language phrase for when this relation became valid. Null if no temporal signal in text.",
|
||||
)
|
||||
valid_until: Optional[str] = Field(
|
||||
None,
|
||||
description="ISO 8601 date or phrase for when this relation ceased to be valid. Null if open-ended or not stated.",
|
||||
)
|
||||
temporal_confidence: float = Field(
|
||||
0.0,
|
||||
description="Confidence that temporal information was present and correctly extracted. 0.0 if no temporal signal.",
|
||||
)
|
||||
temporal_source_text: Optional[str] = Field(
|
||||
None,
|
||||
description="Exact verbatim substring from the source text containing the temporal signal. Null when temporal_confidence is 0.0.",
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def handle_aliases(cls, data):
|
||||
if isinstance(data, dict):
|
||||
if "subject" not in data and "source" in data:
|
||||
data["subject"] = data["source"]
|
||||
if "object" not in data and "target" in data:
|
||||
data["object"] = data["target"]
|
||||
if "predicate" not in data and "label" in data:
|
||||
data["predicate"] = data["label"]
|
||||
return data
|
||||
|
||||
@field_validator("confidence", "temporal_confidence", mode="before")
|
||||
@classmethod
|
||||
def normalize_confidence(cls, v):
|
||||
if isinstance(v, str):
|
||||
try:
|
||||
v = float(v)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
if isinstance(v, (int, float)):
|
||||
return max(0.0, min(1.0, float(v)))
|
||||
return 0.0
|
||||
|
||||
|
||||
class RelationsWithTemporalResponse(BaseModel):
|
||||
"""Wrapper for list of relations with temporal validity bounds."""
|
||||
relations: List[RelationWithTemporalOut] = Field(default_factory=list)
|
||||
|
||||
@@ -267,6 +267,88 @@ class QueryEngine:
|
||||
execution_steps=execution_steps,
|
||||
metadata={"optimization_enabled": self.enable_optimization},
|
||||
)
|
||||
|
||||
def expand_entity_uri(self, entity_uri: str, store_backend: Any, use_alignments: bool = False) -> List[str]:
|
||||
"""
|
||||
Expand an entity URI to include all aligned/equivalent entities.
|
||||
|
||||
Args:
|
||||
entity_uri: The original URI to expand
|
||||
store_backend: Triplet store backend to query
|
||||
use_alignments: If False, returns only the original URI
|
||||
|
||||
Returns:
|
||||
List of URIs including the original and any aligned entities
|
||||
"""
|
||||
if not use_alignments:
|
||||
return [entity_uri]
|
||||
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="triplet_store",
|
||||
submodule="QueryEngine",
|
||||
message=f"Expanding alignments for: {entity_uri}"
|
||||
)
|
||||
|
||||
# SPARQL query to find bidirectional alignments
|
||||
|
||||
safe_uri = self._sanitize_uri(entity_uri)
|
||||
query = f"""
|
||||
SELECT DISTINCT ?aligned WHERE {{
|
||||
{{ <{safe_uri}> ?p ?aligned }}
|
||||
UNION
|
||||
{{ ?aligned ?p <{safe_uri}> }}
|
||||
|
||||
FILTER (?p IN (
|
||||
<http://www.w3.org/2002/07/owl#equivalentClass>,
|
||||
<http://www.w3.org/2002/07/owl#equivalentProperty>,
|
||||
<http://www.w3.org/2002/07/owl#sameAs>,
|
||||
<http://www.w3.org/2004/02/skos/core#exactMatch>,
|
||||
<http://www.w3.org/2004/02/skos/core#closeMatch>,
|
||||
<http://www.w3.org/2004/02/skos/core#broadMatch>,
|
||||
<http://www.w3.org/2004/02/skos/core#narrowMatch>,
|
||||
<http://www.w3.org/2004/02/skos/core#relatedMatch>
|
||||
))
|
||||
}}
|
||||
"""
|
||||
|
||||
expanded_uris = set([entity_uri])
|
||||
try:
|
||||
if hasattr(store_backend, "execute_sparql"):
|
||||
result_data = store_backend.execute_sparql(query)
|
||||
for binding in result_data.get("bindings", []):
|
||||
val = binding.get("aligned", {})
|
||||
uri = val.get("value") if isinstance(val, dict) else val
|
||||
if uri:
|
||||
expanded_uris.add(uri)
|
||||
else:
|
||||
self.logger.warning(
|
||||
"store_backend does not support execute_sparql; returning original URI only"
|
||||
)
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Expanded to {len(expanded_uris)} URIs"
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to expand alignments for {entity_uri}: {e}")
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
|
||||
return list(expanded_uris)
|
||||
|
||||
def build_values_clause(self, variable_name: str, uris: List[str]) -> str:
|
||||
"""
|
||||
Helper to generate a SPARQL VALUES clause for a list of URIs.
|
||||
Allows higher-level components to build alignment-aware queries.
|
||||
|
||||
Example:
|
||||
uris = engine.expand_entity_uri("http://ex.org/Person", store, use_alignments=True)
|
||||
clause = engine.build_values_clause("subject", uris)
|
||||
# Returns: VALUES ?subject { <http://ex.org/Person> <http://other.org/Human> }
|
||||
"""
|
||||
if not uris:
|
||||
return ""
|
||||
formatted_uris = " ".join([f"<{self._sanitize_uri(uri)}>" for uri in uris])
|
||||
return f"VALUES ?{variable_name} {{ {formatted_uris} }}"
|
||||
|
||||
def _validate_query(self, query: str) -> bool:
|
||||
"""Validate SPARQL query syntax (basic)."""
|
||||
@@ -344,6 +426,12 @@ class QueryEngine:
|
||||
|
||||
cache_key = self._get_cache_key(query)
|
||||
self.query_cache[cache_key] = result
|
||||
|
||||
def _sanitize_uri(self, uri: str) -> str:
|
||||
"""Prevent SPARQL injection by percent-encoding dangerous characters."""
|
||||
if not isinstance(uri, str):
|
||||
return ""
|
||||
return uri.replace("<", "%3C").replace(">", "%3E")
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear query cache."""
|
||||
|
||||
@@ -63,6 +63,7 @@ from .exceptions import (
|
||||
ProcessingError,
|
||||
QualityError,
|
||||
SemanticaError,
|
||||
TemporalValidationError,
|
||||
ValidationError,
|
||||
format_exception,
|
||||
handle_exception,
|
||||
@@ -158,6 +159,7 @@ __all__ = [
|
||||
# Exceptions
|
||||
"SemanticaError",
|
||||
"ValidationError",
|
||||
"TemporalValidationError",
|
||||
"ProcessingError",
|
||||
"ConfigurationError",
|
||||
"QualityError",
|
||||
|
||||
@@ -152,6 +152,33 @@ class ValidationError(SemanticaError):
|
||||
self.constraint = details.get("constraint")
|
||||
|
||||
|
||||
class TemporalValidationError(ValidationError):
|
||||
"""
|
||||
Exception raised for invalid temporal values or inconsistent temporal state.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
temporal_context: Optional[Dict[str, Any]] = None,
|
||||
**details: Any,
|
||||
):
|
||||
super().__init__(message, validation_context=temporal_context, **details)
|
||||
self.error_code = "SEM001T"
|
||||
|
||||
|
||||
class TemporalAmbiguityWarning(UserWarning):
|
||||
"""
|
||||
Warning raised when a temporal expression is ambiguous and cannot be
|
||||
resolved without additional locale or context information.
|
||||
|
||||
Example: "03/04/2022" is ambiguous without knowing whether day-first or
|
||||
month-first ordering applies. Use ``warnings.catch_warnings()`` to handle.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ProcessingError(SemanticaError):
|
||||
"""
|
||||
Exception raised for data processing errors.
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import pytest
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.change_management.managers import TemporalVersionManager, ProcessingError
|
||||
from semantica.utils.exceptions import ValidationError
|
||||
|
||||
@pytest.fixture
|
||||
def setup_env():
|
||||
"""Fixture to provide a clean graph and manager for each test."""
|
||||
graph = ContextGraph()
|
||||
manager = TemporalVersionManager()
|
||||
manager.attach_to_graph(graph)
|
||||
return graph, manager
|
||||
|
||||
def test_audit_trail_records_mutations(setup_env):
|
||||
graph, manager = setup_env
|
||||
graph.add_node("user_1", "person", content="Alice")
|
||||
graph.add_node_attribute("user_1", {"age": 30})
|
||||
|
||||
|
||||
history = manager.get_node_history("user_1")
|
||||
assert len(history) == 2
|
||||
assert history[0]["operation"] == "ADD_NODE"
|
||||
assert history[1]["operation"] == "UPDATE_NODE"
|
||||
assert history[1]["payload"]["properties"]["age"] == 30
|
||||
|
||||
def test_version_tagging_and_retrieval(setup_env):
|
||||
graph, manager = setup_env
|
||||
graph.add_node("node_1", "concept", content="Test")
|
||||
|
||||
|
||||
manager.create_snapshot(
|
||||
graph.to_dict(),
|
||||
version_label="v1.0",
|
||||
author="test@example.com",
|
||||
description="Test Release"
|
||||
)
|
||||
|
||||
manager.tag_version("v1.0", "production-ready")
|
||||
|
||||
|
||||
tags = manager.list_tags()
|
||||
assert "production-ready" in tags
|
||||
assert tags["production-ready"] == "v1.0"
|
||||
history = manager.get_node_history("node_1")
|
||||
assert history[0]["version_label"] == "v1.0"
|
||||
|
||||
def test_tagging_nonexistent_version_fails(setup_env):
|
||||
_, manager = setup_env
|
||||
|
||||
with pytest.raises(Exception):
|
||||
manager.tag_version("v99.0", "invalid-tag")
|
||||
|
||||
def test_rollback_protection_enforcement(setup_env):
|
||||
graph, manager = setup_env
|
||||
graph.add_node("node_1", "concept")
|
||||
|
||||
manager.create_snapshot(
|
||||
graph.to_dict(),
|
||||
version_label="v1.0",
|
||||
author="test@example.com",
|
||||
description="Test"
|
||||
)
|
||||
|
||||
with pytest.raises(ProcessingError, match="Rollback protection active"):
|
||||
manager.restore_snapshot(graph, "v1.0")
|
||||
|
||||
success = manager.restore_snapshot(graph, "v1.0", require_confirmation=False)
|
||||
assert success is True
|
||||
assert len(graph.nodes) == 1
|
||||
|
||||
def test_restore_snapshot_does_not_record_replay_mutations(setup_env):
|
||||
graph, manager = setup_env
|
||||
graph.add_node("node_1", "concept")
|
||||
manager.create_snapshot(
|
||||
graph.to_dict(),
|
||||
version_label="v1.0",
|
||||
author="test@example.com",
|
||||
description="Initial snapshot",
|
||||
)
|
||||
graph.add_node_attribute("node_1", {"status": "changed"})
|
||||
|
||||
history_before_restore = manager.get_node_history("node_1")
|
||||
assert len(history_before_restore) == 2
|
||||
|
||||
manager.restore_snapshot(graph, "v1.0", require_confirmation=False)
|
||||
|
||||
history_after_restore = manager.get_node_history("node_1")
|
||||
assert len(history_after_restore) == 2
|
||||
@@ -9,10 +9,13 @@ import os
|
||||
import tempfile
|
||||
import pytest
|
||||
from semantica.change_management import (
|
||||
TemporalVersionManager,
|
||||
TemporalVersionManager,
|
||||
OntologyVersionManager,
|
||||
ChangeLogEntry
|
||||
)
|
||||
from semantica.change_management.change_log import generate_change_report, ChangeLogAnalyzer
|
||||
from semantica.change_management.ontology_version_manager import VersionManager
|
||||
from semantica.ontology.engine import OntologyEngine
|
||||
from semantica.utils.exceptions import ValidationError, ProcessingError
|
||||
|
||||
|
||||
@@ -68,6 +71,45 @@ class TestTemporalVersionManager:
|
||||
assert "timestamp" in snapshot
|
||||
assert len(snapshot["entities"]) == 2
|
||||
assert len(snapshot["relationships"]) == 1
|
||||
assert len(snapshot["nodes"]) == 2
|
||||
assert len(snapshot["edges"]) == 1
|
||||
|
||||
def test_create_snapshot_accepts_nodes_and_edges(self):
|
||||
"""Test ContextGraph-style node/edge payload support."""
|
||||
manager = TemporalVersionManager()
|
||||
graph = {
|
||||
"nodes": [
|
||||
{"id": "entity1", "name": "Entity 1", "type": "Person"},
|
||||
{"id": "entity2", "name": "Entity 2", "type": "Organization"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "entity1", "target": "entity2", "type": "works_for"}
|
||||
],
|
||||
}
|
||||
|
||||
snapshot = manager.create_snapshot(
|
||||
graph,
|
||||
"test_nodes_v1.0",
|
||||
"test@example.com",
|
||||
"Test node edge snapshot creation",
|
||||
)
|
||||
|
||||
assert len(snapshot["nodes"]) == 2
|
||||
assert len(snapshot["edges"]) == 1
|
||||
assert len(snapshot["entities"]) == 2
|
||||
assert len(snapshot["relationships"]) == 1
|
||||
|
||||
def test_create_snapshot_requires_supported_schema(self):
|
||||
"""Test schema validation for unsupported graph payloads."""
|
||||
manager = TemporalVersionManager()
|
||||
|
||||
with pytest.raises(ValidationError, match="Graph dictionary must contain"):
|
||||
manager.create_snapshot(
|
||||
{"foo": []},
|
||||
"invalid_v1.0",
|
||||
"test@example.com",
|
||||
"Invalid graph payload",
|
||||
)
|
||||
|
||||
def test_create_snapshot_with_invalid_author(self):
|
||||
"""Test snapshot creation with invalid author email."""
|
||||
@@ -114,6 +156,29 @@ class TestTemporalVersionManager:
|
||||
versions = manager.list_versions()
|
||||
assert len(versions) == 1
|
||||
assert versions[0]["label"] == "test_v1.0"
|
||||
|
||||
def test_list_versions_counts_nodes_and_edges(self):
|
||||
"""Test version metadata counts for node/edge snapshots."""
|
||||
manager = TemporalVersionManager()
|
||||
manager.create_snapshot(
|
||||
{
|
||||
"nodes": [
|
||||
{"id": "entity1", "name": "Entity 1", "type": "Person"},
|
||||
{"id": "entity2", "name": "Entity 2", "type": "Organization"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "entity1", "target": "entity2", "type": "works_for"}
|
||||
],
|
||||
},
|
||||
"test_nodes_v1.0",
|
||||
"test@example.com",
|
||||
"Test metadata counts",
|
||||
)
|
||||
|
||||
versions = manager.list_versions()
|
||||
assert len(versions) == 1
|
||||
assert versions[0]["entity_count"] == 2
|
||||
assert versions[0]["relationship_count"] == 1
|
||||
|
||||
def test_get_version(self):
|
||||
"""Test retrieving specific version."""
|
||||
@@ -203,6 +268,105 @@ class TestTemporalVersionManager:
|
||||
assert diff["entities_modified"][0]["id"] == "entity1"
|
||||
assert diff["entities_modified"][0]["changes"]["name"]["from"] == "Entity 1"
|
||||
assert diff["entities_modified"][0]["changes"]["name"]["to"] == "Entity 1 Updated"
|
||||
assert diff["summary"]["nodes_added"] == 1
|
||||
assert diff["summary"]["edges_added"] == 1
|
||||
|
||||
def test_create_snapshot_accepts_nodes_and_edges(self):
|
||||
"""Test ContextGraph-style node/edge payload support."""
|
||||
manager = TemporalVersionManager()
|
||||
graph = {
|
||||
"nodes": [
|
||||
{"id": "entity1", "name": "Entity 1", "type": "Person"},
|
||||
{"id": "entity2", "name": "Entity 2", "type": "Organization"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "entity1", "target": "entity2", "type": "works_for"}
|
||||
],
|
||||
}
|
||||
|
||||
snapshot = manager.create_snapshot(
|
||||
graph, "nodes_v1", "test@example.com", "Node/edge snapshot"
|
||||
)
|
||||
|
||||
assert len(snapshot["nodes"]) == 2
|
||||
assert len(snapshot["edges"]) == 1
|
||||
assert len(snapshot["entities"]) == 2
|
||||
assert len(snapshot["relationships"]) == 1
|
||||
|
||||
def test_compare_versions_supports_mixed_snapshot_schemas(self):
|
||||
"""Test diff compatibility between legacy and node/edge snapshots."""
|
||||
manager = TemporalVersionManager()
|
||||
manager.create_snapshot(
|
||||
{"entities": [{"id": "entity1"}], "relationships": []},
|
||||
"v1.0",
|
||||
"test@example.com",
|
||||
"Version 1",
|
||||
)
|
||||
manager.create_snapshot(
|
||||
{"nodes": [{"id": "entity1"}, {"id": "entity2"}], "edges": []},
|
||||
"v2.0",
|
||||
"test@example.com",
|
||||
"Version 2",
|
||||
)
|
||||
|
||||
diff = manager.compare_versions("v1.0", "v2.0")
|
||||
assert diff["summary"]["entities_added"] == 1
|
||||
assert diff["summary"]["nodes_added"] == 1
|
||||
assert diff["entities_added"][0]["id"] == "entity2"
|
||||
|
||||
def test_compare_versions_with_nodes_and_edges(self):
|
||||
"""Test detailed comparison for ContextGraph-style snapshots."""
|
||||
manager = TemporalVersionManager()
|
||||
graph_v1 = {
|
||||
"nodes": [
|
||||
{"id": "entity1", "name": "Entity 1", "type": "Person"},
|
||||
{"id": "entity2", "name": "Entity 2", "type": "Organization"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "entity1", "target": "entity2", "type": "works_for"}
|
||||
],
|
||||
}
|
||||
graph_v2 = {
|
||||
"nodes": [
|
||||
{"id": "entity1", "name": "Entity 1 Updated", "type": "Person"},
|
||||
{"id": "entity3", "name": "Entity 3", "type": "Project"},
|
||||
],
|
||||
"edges": [
|
||||
{"source": "entity1", "target": "entity3", "type": "manages"}
|
||||
],
|
||||
}
|
||||
|
||||
manager.create_snapshot(graph_v1, "v1.0", "test@example.com", "Version 1")
|
||||
manager.create_snapshot(graph_v2, "v2.0", "test@example.com", "Version 2")
|
||||
|
||||
diff = manager.compare_versions("v1.0", "v2.0")
|
||||
assert diff["summary"]["entities_added"] == 1
|
||||
assert diff["summary"]["entities_removed"] == 1
|
||||
assert diff["summary"]["entities_modified"] == 1
|
||||
assert diff["summary"]["relationships_added"] == 1
|
||||
assert diff["summary"]["relationships_removed"] == 1
|
||||
|
||||
def test_compare_versions_with_mixed_snapshot_schemas(self):
|
||||
"""Test diff compatibility between legacy and node/edge snapshots."""
|
||||
manager = TemporalVersionManager()
|
||||
graph_v1 = {
|
||||
"entities": [{"id": "entity1", "name": "Entity 1", "type": "Person"}],
|
||||
"relationships": [],
|
||||
}
|
||||
graph_v2 = {
|
||||
"nodes": [
|
||||
{"id": "entity1", "name": "Entity 1", "type": "Person"},
|
||||
{"id": "entity2", "name": "Entity 2", "type": "Organization"},
|
||||
],
|
||||
"edges": [],
|
||||
}
|
||||
|
||||
manager.create_snapshot(graph_v1, "v1.0", "test@example.com", "Version 1")
|
||||
manager.create_snapshot(graph_v2, "v2.0", "test@example.com", "Version 2")
|
||||
|
||||
diff = manager.compare_versions("v1.0", "v2.0")
|
||||
assert diff["summary"]["entities_added"] == 1
|
||||
assert diff["entities_added"][0]["id"] == "entity2"
|
||||
|
||||
|
||||
class TestOntologyVersionManager:
|
||||
@@ -330,3 +494,142 @@ class TestOntologyVersionManager:
|
||||
finally:
|
||||
if os.path.exists(db_path):
|
||||
os.remove(db_path)
|
||||
|
||||
|
||||
def test_diff_empty_ontologies(self):
|
||||
"""Test diffing entirely empty dictionaries."""
|
||||
manager = VersionManager()
|
||||
diff = manager.diff_ontologies({}, {})
|
||||
|
||||
assert len(diff["added_classes"]) == 0
|
||||
assert len(diff["removed_classes"]) == 0
|
||||
assert len(diff["changed_classes"]) == 0
|
||||
|
||||
def test_diff_unordered_list_equality(self):
|
||||
"""Test that list order doesn't trigger a false positive change."""
|
||||
manager = VersionManager()
|
||||
base = {"classes": [{"uri": "http://ex.org/C1", "domain": ["A", "B"]}]}
|
||||
target = {"classes": [{"uri": "http://ex.org/C1", "domain": ["B", "A"]}]}
|
||||
|
||||
diff = manager.diff_ontologies(base, target)
|
||||
assert len(diff["changed_classes"]) == 0
|
||||
|
||||
def test_diff_missing_uris(self):
|
||||
"""Tests whether missing URIs fallback to 'name' deterministically."""
|
||||
manager = VersionManager()
|
||||
base = {"classes": [{"name": "Person", "label": "Human"}]}
|
||||
target = {"classes": [{"name": "Person", "label": "Homo Sapiens"}]}
|
||||
|
||||
diff = manager.diff_ontologies(base, target)
|
||||
assert len(diff["changed_classes"]) == 1
|
||||
|
||||
change = diff["changed_classes"][0]
|
||||
assert change["name"] == "Person"
|
||||
assert change["changes"]["label"]["old"] == "Human"
|
||||
assert change["changes"]["label"]["new"] == "Homo Sapiens"
|
||||
|
||||
|
||||
class TestChangeLogAnalyzer:
|
||||
"""Test cases for Impact Analysis & Reporting."""
|
||||
|
||||
def test_breaking_change_removed_class(self):
|
||||
"""Test that removing a class is flagged as CRITICAL/BREAKING."""
|
||||
diff = {"removed_classes": [{"uri": "http://ex.org/Person"}]}
|
||||
report = generate_change_report(diff)
|
||||
|
||||
assert len(report["impact_classification"]["breaking"]) == 1
|
||||
assert report["impact_classification"]["breaking"][0]["severity"] == "critical"
|
||||
assert "removed" in report["impact_classification"]["breaking"][0]["description"]
|
||||
|
||||
def test_breaking_change_narrowed_domain(self):
|
||||
"""Test that narrowing a domain is flagged as HIGH/BREAKING."""
|
||||
diff = {
|
||||
"changed_properties": [{
|
||||
"uri": "http://ex.org/worksFor",
|
||||
"changes": {"domain": {"old": ["Person", "Organization"], "new": ["Person"]}}
|
||||
}]
|
||||
}
|
||||
report = generate_change_report(diff)
|
||||
|
||||
assert len(report["impact_classification"]["breaking"]) == 1
|
||||
assert report["impact_classification"]["breaking"][0]["severity"] == "high"
|
||||
assert "restricted" in report["impact_classification"]["breaking"][0]["description"]
|
||||
|
||||
def test_safe_change_added_class_and_label(self):
|
||||
"""Test that adding classes and changing annotations is SAFE."""
|
||||
diff = {
|
||||
"added_classes": [{"uri": "http://ex.org/NewClass"}],
|
||||
"changed_properties": [{
|
||||
"uri": "http://ex.org/name",
|
||||
"changes": {"label": {"old": "Name", "new": "Full Name"}}
|
||||
}]
|
||||
}
|
||||
report = generate_change_report(diff)
|
||||
|
||||
assert len(report["impact_classification"]["safe"]) == 2
|
||||
assert len(report["impact_classification"]["breaking"]) == 0
|
||||
assert len(report["impact_classification"]["potentially_breaking"]) == 0
|
||||
|
||||
|
||||
class TestOntologyEngineMigration:
|
||||
"""Test cases for Public API Orchestration."""
|
||||
|
||||
def test_compare_versions_with_dicts_override(self):
|
||||
"""Test for bypassing the DB fetch by passing dicts directly."""
|
||||
engine = OntologyEngine()
|
||||
|
||||
base_dict = {"classes": [{"uri": "http://ex.org/C1", "label": "Old"}]}
|
||||
target_dict = {"classes": [{"uri": "http://ex.org/C1", "label": "New"}]}
|
||||
|
||||
# We pass fake version IDs ("v1", "v2"), but the engine should use our dicts
|
||||
report = engine.compare_versions("v1", "v2", base_dict=base_dict, target_dict=target_dict)
|
||||
|
||||
assert report["summary"]["total_changes"] == 1
|
||||
assert len(report["impact_classification"]["safe"]) == 1
|
||||
assert report["impact_classification"]["safe"][0]["entity_uri"] == "http://ex.org/C1"
|
||||
|
||||
def test_compare_versions_version_not_found_raises(self):
|
||||
"""Test that compare_versions raises ProcessingError when version ID is not registered."""
|
||||
engine = OntologyEngine()
|
||||
|
||||
with pytest.raises(ProcessingError):
|
||||
engine.compare_versions("nonexistent_v1", "nonexistent_v2")
|
||||
|
||||
def test_compare_versions_diff_includes_individuals_and_axioms(self):
|
||||
"""Test that the diff covers individuals and axioms, not just classes/properties."""
|
||||
engine = OntologyEngine()
|
||||
|
||||
base_dict = {
|
||||
"classes": [],
|
||||
"properties": [],
|
||||
"individuals": [{"uri": "http://ex.org/john"}],
|
||||
"axioms": [{"uri": "http://ex.org/rule1", "expression": "Person hasName exactly 1 string"}],
|
||||
}
|
||||
target_dict = {
|
||||
"classes": [],
|
||||
"properties": [],
|
||||
"individuals": [
|
||||
{"uri": "http://ex.org/john"},
|
||||
{"uri": "http://ex.org/jane"},
|
||||
],
|
||||
"axioms": [],
|
||||
}
|
||||
|
||||
report = engine.compare_versions("v1", "v2", base_dict=base_dict, target_dict=target_dict)
|
||||
|
||||
diff = report["diff"]
|
||||
assert any(i.get("uri") == "http://ex.org/jane" for i in diff["added_individuals"])
|
||||
assert any(a.get("uri") == "http://ex.org/rule1" for a in diff["removed_axioms"])
|
||||
|
||||
def test_compare_versions_null_constraint_value_flagged_as_breaking(self):
|
||||
"""Test that a constraint field going from None to a value is flagged as breaking."""
|
||||
diff = {
|
||||
"changed_properties": [{
|
||||
"uri": "http://ex.org/worksFor",
|
||||
"changes": {"domain": {"old": None, "new": ["Person"]}}
|
||||
}]
|
||||
}
|
||||
report = generate_change_report(diff)
|
||||
|
||||
assert len(report["impact_classification"]["breaking"]) == 1
|
||||
assert report["impact_classification"]["breaking"][0]["severity"] == "high"
|
||||
|
||||
@@ -8,7 +8,10 @@ graphs, including persistent storage, detailed change tracking, and audit trails
|
||||
import os
|
||||
import tempfile
|
||||
import pytest
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import patch
|
||||
from semantica.kg.temporal_query import TemporalVersionManager
|
||||
from semantica.kg.temporal_model import TemporalBound
|
||||
from semantica.change_management import ChangeLogEntry, InMemoryVersionStorage, SQLiteVersionStorage
|
||||
from semantica.utils.exceptions import ValidationError, ProcessingError
|
||||
|
||||
@@ -234,6 +237,138 @@ class TestTemporalVersionManager:
|
||||
|
||||
with pytest.raises(ValidationError, match="Version not found: nonexistent"):
|
||||
manager.compare_versions("v1.0", "nonexistent")
|
||||
|
||||
def test_apply_revision_preserves_history_and_revises_valid_time(self):
|
||||
manager = TemporalVersionManager()
|
||||
snapshot = manager.create_snapshot(
|
||||
graph={
|
||||
"entities": [],
|
||||
"relationships": [
|
||||
{
|
||||
"id": "fact-1",
|
||||
"source": "drug_a",
|
||||
"target": "drug_b",
|
||||
"type": "interacts_with",
|
||||
"valid_from": "2021-01-01",
|
||||
"valid_until": TemporalBound.OPEN,
|
||||
}
|
||||
],
|
||||
},
|
||||
version_label="v1.0",
|
||||
author="alice@company.com",
|
||||
description="Initial version",
|
||||
)
|
||||
|
||||
revised = manager.apply_revision(
|
||||
snapshot,
|
||||
{
|
||||
"fact_ids": ["fact-1"],
|
||||
"new_valid_from": "2019-01-01",
|
||||
"new_valid_until": None,
|
||||
"revision_type": "retroactive",
|
||||
"author": "alice@company.com",
|
||||
"reason": "Backfilled evidence",
|
||||
},
|
||||
)
|
||||
|
||||
query_engine = __import__("semantica.kg.temporal_query", fromlist=["TemporalGraphQuery"]).TemporalGraphQuery()
|
||||
result = query_engine.query_at_time(revised, "", "2020-06-01")
|
||||
|
||||
assert len(result["relationships"]) == 1
|
||||
assert result["relationships"][0]["id"].startswith("fact-1__rev__")
|
||||
|
||||
original = manager.get_version("v1.0")
|
||||
assert original is not None
|
||||
assert manager.get_version(revised["label"]) is not None
|
||||
assert len(manager.list_versions()) == 2
|
||||
assert manager.verify_checksum(revised) is True
|
||||
|
||||
def test_apply_revision_recomputes_checksums_for_saved_snapshots(self):
|
||||
manager = TemporalVersionManager()
|
||||
snapshot = manager.create_snapshot(
|
||||
graph={
|
||||
"entities": [],
|
||||
"relationships": [
|
||||
{
|
||||
"id": "fact-1",
|
||||
"source": "drug_a",
|
||||
"target": "drug_b",
|
||||
"type": "interacts_with",
|
||||
"valid_from": "2021-01-01",
|
||||
"valid_until": TemporalBound.OPEN,
|
||||
}
|
||||
],
|
||||
},
|
||||
version_label="v1.0",
|
||||
author="alice@company.com",
|
||||
description="Initial version",
|
||||
)
|
||||
|
||||
revised = manager.apply_revision(
|
||||
snapshot,
|
||||
{
|
||||
"fact_ids": ["fact-1"],
|
||||
"new_valid_from": "2019-01-01",
|
||||
"new_valid_until": None,
|
||||
"revision_type": "retroactive",
|
||||
"author": "alice@company.com",
|
||||
"reason": "Backfilled evidence",
|
||||
},
|
||||
)
|
||||
|
||||
assert manager.verify_checksum(revised) is True
|
||||
persisted_original = manager.get_version("v1.0")
|
||||
assert persisted_original is not None
|
||||
assert manager.verify_checksum(persisted_original) is True
|
||||
|
||||
def test_generate_revision_suffix_is_collision_resistant(self):
|
||||
manager = TemporalVersionManager()
|
||||
revision_time = datetime(2026, 3, 23, 12, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
suffix_one = manager._generate_revision_suffix(revision_time)
|
||||
suffix_two = manager._generate_revision_suffix(revision_time)
|
||||
|
||||
assert suffix_one != suffix_two
|
||||
|
||||
def test_apply_revision_uses_collision_resistant_suffix_for_ids_and_labels(self):
|
||||
manager = TemporalVersionManager()
|
||||
snapshot = manager.create_snapshot(
|
||||
graph={
|
||||
"entities": [],
|
||||
"relationships": [
|
||||
{
|
||||
"id": "fact-1",
|
||||
"source": "drug_a",
|
||||
"target": "drug_b",
|
||||
"type": "interacts_with",
|
||||
"valid_from": "2021-01-01",
|
||||
"valid_until": TemporalBound.OPEN,
|
||||
}
|
||||
],
|
||||
},
|
||||
version_label="v1.0",
|
||||
author="alice@company.com",
|
||||
description="Initial version",
|
||||
)
|
||||
|
||||
with patch.object(manager, "_generate_revision_suffix", return_value="fixed_suffix"):
|
||||
revised = manager.apply_revision(
|
||||
snapshot,
|
||||
{
|
||||
"fact_ids": ["fact-1"],
|
||||
"new_valid_from": "2019-01-01",
|
||||
"new_valid_until": None,
|
||||
"revision_type": "retroactive",
|
||||
"author": "alice@company.com",
|
||||
"reason": "Backfilled evidence",
|
||||
},
|
||||
)
|
||||
|
||||
assert revised["label"].endswith("__revision__fixed_suffix")
|
||||
replacement = next(
|
||||
rel for rel in revised["relationships"] if rel["id"].startswith("fact-1__rev__")
|
||||
)
|
||||
assert replacement["id"] == "fact-1__rev__fixed_suffix"
|
||||
|
||||
def test_detailed_entity_diff(self):
|
||||
"""Test detailed entity-level differences."""
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import Dict, Any, List
|
||||
|
||||
from semantica.context.agent_context import AgentContext
|
||||
from semantica.context.decision_models import Decision, Policy
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
|
||||
|
||||
class TestAgentContextDecisions:
|
||||
@@ -288,6 +289,49 @@ class TestAgentContextDecisions:
|
||||
except Exception:
|
||||
# Should handle errors gracefully
|
||||
pass
|
||||
|
||||
def test_checkpoint_diff_and_flush(self, mock_vector_store):
|
||||
"""Checkpointing should capture and diff graph changes."""
|
||||
graph = ContextGraph()
|
||||
context = AgentContext(
|
||||
vector_store=mock_vector_store,
|
||||
knowledge_graph=graph,
|
||||
decision_tracking=True,
|
||||
)
|
||||
|
||||
context.checkpoint("before")
|
||||
decision_id = context.record_decision(
|
||||
category="policy",
|
||||
scenario="Apply updated policy",
|
||||
reasoning="New evidence available",
|
||||
outcome="approved",
|
||||
confidence=0.9,
|
||||
)
|
||||
graph.add_node("entity_001", "entity", content="Customer")
|
||||
graph.add_edge(decision_id, "entity_001", "involves")
|
||||
context.checkpoint("after")
|
||||
|
||||
diff = context.diff_checkpoints("before", "after")
|
||||
|
||||
assert any(item["id"] == decision_id for item in diff["decisions_added"])
|
||||
assert any(item["type"] == "involves" for item in diff["relationships_added"])
|
||||
|
||||
snapshot = context.flush_checkpoint("after")
|
||||
assert snapshot["label"] == "after"
|
||||
|
||||
def test_diff_checkpoints_unknown_label_raises_key_error(self, mock_vector_store):
|
||||
"""Unknown checkpoint labels should raise a clear KeyError."""
|
||||
graph = ContextGraph()
|
||||
context = AgentContext(
|
||||
vector_store=mock_vector_store,
|
||||
knowledge_graph=graph,
|
||||
decision_tracking=True,
|
||||
)
|
||||
|
||||
context.checkpoint("after")
|
||||
|
||||
with pytest.raises(KeyError, match="unknown"):
|
||||
context.diff_checkpoints("unknown", "after")
|
||||
|
||||
def test_decision_tracking_requires_knowledge_graph(self, mock_vector_store):
|
||||
"""Test that decision tracking requires knowledge graph."""
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import List, Dict, Any
|
||||
|
||||
from semantica.context.decision_models import Decision
|
||||
from semantica.context.causal_analyzer import CausalChainAnalyzer
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
|
||||
|
||||
class TestCausalChainAnalyzer:
|
||||
@@ -217,7 +218,13 @@ class TestCausalChainAnalyzer:
|
||||
mock_graph_store.execute_query.return_value = [
|
||||
{
|
||||
"decision_id": "decision_001",
|
||||
"loop_path": ["decision_001", "decision_002", "decision_003", "decision_001"],
|
||||
"decision_scenario": "Approve LATAM expansion",
|
||||
"loop_path": [
|
||||
{"decision_id": "decision_001", "scenario": "Approve LATAM expansion", "category": "strategy"},
|
||||
{"decision_id": "decision_002", "scenario": "Fund regional hiring", "category": "finance"},
|
||||
{"decision_id": "decision_003", "scenario": "Open Sao Paulo office", "category": "operations"},
|
||||
{"decision_id": "decision_001", "scenario": "Approve LATAM expansion", "category": "strategy"},
|
||||
],
|
||||
"loop_length": 3,
|
||||
"cycle_strength": 0.7
|
||||
}
|
||||
@@ -227,6 +234,8 @@ class TestCausalChainAnalyzer:
|
||||
|
||||
assert len(loops) == 1
|
||||
assert loops[0]["decision_id"] == "decision_001"
|
||||
assert loops[0]["decision_scenario"] == "Approve LATAM expansion"
|
||||
assert loops[0]["loop_path"][0]["scenario"] == "Approve LATAM expansion"
|
||||
assert len(loops[0]["loop_path"]) == 4 # Including return to start
|
||||
assert loops[0]["loop_length"] == 3
|
||||
|
||||
@@ -503,6 +512,84 @@ class TestCausalChainAnalyzer:
|
||||
assert chain[0].decision_id == "decision_001"
|
||||
assert chain[1].decision_id == "decision_002"
|
||||
|
||||
def test_trace_at_time_excludes_late_recorded_facts(self):
|
||||
"""trace_at_time should filter by relationship recorded_at, not valid time."""
|
||||
graph = ContextGraph()
|
||||
decision_1 = Decision(
|
||||
decision_id="decision_001",
|
||||
category="ops",
|
||||
scenario="Initial review",
|
||||
reasoning="Start",
|
||||
outcome="approved",
|
||||
confidence=0.9,
|
||||
timestamp=datetime.now(),
|
||||
decision_maker="agent",
|
||||
)
|
||||
decision_2 = Decision(
|
||||
decision_id="decision_002",
|
||||
category="ops",
|
||||
scenario="Follow-up",
|
||||
reasoning="Depends on initial review",
|
||||
outcome="approved",
|
||||
confidence=0.9,
|
||||
timestamp=datetime.now(),
|
||||
decision_maker="agent",
|
||||
)
|
||||
graph.add_decision(decision_1)
|
||||
graph.add_decision(decision_2)
|
||||
graph.add_edge(
|
||||
"decision_001",
|
||||
"decision_002",
|
||||
"CAUSED",
|
||||
recorded_at="2024-01-10T00:00:00",
|
||||
)
|
||||
|
||||
analyzer = CausalChainAnalyzer(graph_store=graph)
|
||||
|
||||
early_chain = analyzer.trace_at_time("decision_002", "2024-01-05T00:00:00")
|
||||
later_chain = analyzer.trace_at_time("decision_002", "2024-01-15T00:00:00")
|
||||
|
||||
assert early_chain == []
|
||||
assert [decision.decision_id for decision in later_chain] == ["decision_001"]
|
||||
|
||||
def test_trace_at_time_returns_empty_before_any_facts_recorded(self):
|
||||
"""When the cutoff predates all facts, trace_at_time should return an empty chain."""
|
||||
graph = ContextGraph()
|
||||
graph.add_decision(
|
||||
Decision(
|
||||
decision_id="decision_001",
|
||||
category="ops",
|
||||
scenario="Initial review",
|
||||
reasoning="Start",
|
||||
outcome="approved",
|
||||
confidence=0.9,
|
||||
timestamp=datetime.now(),
|
||||
decision_maker="agent",
|
||||
)
|
||||
)
|
||||
graph.add_decision(
|
||||
Decision(
|
||||
decision_id="decision_002",
|
||||
category="ops",
|
||||
scenario="Follow-up",
|
||||
reasoning="Depends on initial review",
|
||||
outcome="approved",
|
||||
confidence=0.9,
|
||||
timestamp=datetime.now(),
|
||||
decision_maker="agent",
|
||||
)
|
||||
)
|
||||
graph.add_edge(
|
||||
"decision_001",
|
||||
"decision_002",
|
||||
"CAUSED",
|
||||
recorded_at="2024-02-01T00:00:00",
|
||||
)
|
||||
|
||||
analyzer = CausalChainAnalyzer(graph_store=graph)
|
||||
|
||||
assert analyzer.trace_at_time("decision_002", "2024-01-01T00:00:00") == []
|
||||
|
||||
|
||||
class TestCausalAnalyzerEdgeCases:
|
||||
"""Test edge cases and boundary conditions."""
|
||||
|
||||
@@ -64,6 +64,23 @@ class TestContextModule(unittest.TestCase):
|
||||
# However, checking it runs without error is a good start.
|
||||
self.assertIsInstance(linked, list)
|
||||
|
||||
def test_entity_linker_find_similar_entities_returns_dicts(self):
|
||||
linker = EntityLinker(
|
||||
knowledge_graph={
|
||||
"entities": [
|
||||
{"id": "lang_python", "text": "Python programming language", "type": "Technology"}
|
||||
]
|
||||
}
|
||||
)
|
||||
linker.assign_uri("lang_python", "Python programming language", "Technology")
|
||||
|
||||
similar = linker.find_similar_entities("Python programming language", threshold=0.5)
|
||||
|
||||
self.assertEqual(len(similar), 1)
|
||||
self.assertEqual(similar[0]["entity_id"], "lang_python")
|
||||
self.assertEqual(similar[0]["text"], "Python programming language")
|
||||
self.assertIn("similarity", similar[0])
|
||||
|
||||
# --- ContextGraph Tests ---
|
||||
def test_context_graph_operations(self):
|
||||
graph = ContextGraph()
|
||||
@@ -92,6 +109,66 @@ class TestContextModule(unittest.TestCase):
|
||||
self.assertEqual(neighbors[0]["id"], "n2")
|
||||
self.assertEqual(neighbors[0]["relationship"], "knows")
|
||||
|
||||
def test_get_nodes_by_label_returns_metadata_copy(self):
|
||||
graph = ContextGraph()
|
||||
graph.add_node("n1", "person", "Alice", role="engineer")
|
||||
|
||||
nodes = graph.get_nodes_by_label("person")
|
||||
self.assertEqual(len(nodes), 1)
|
||||
|
||||
nodes[0]["metadata"]["role"] = "mutated"
|
||||
|
||||
self.assertEqual(graph.get_node_property("n1", "role"), "engineer")
|
||||
|
||||
def test_context_graph_preserves_full_decision_text(self):
|
||||
graph = ContextGraph()
|
||||
scenario = "Launch regional expansion plan " + ("X" * 140)
|
||||
root_id = graph.record_decision(
|
||||
category="strategy",
|
||||
scenario=scenario,
|
||||
reasoning="Growth opportunity with strong local demand",
|
||||
outcome="approved",
|
||||
confidence=0.91,
|
||||
)
|
||||
child_id = graph.record_decision(
|
||||
category="operations",
|
||||
scenario="Open Sao Paulo office",
|
||||
reasoning="Needed to support expansion",
|
||||
outcome="pending",
|
||||
confidence=0.74,
|
||||
)
|
||||
graph.add_causal_relationship(root_id, child_id, "CAUSED")
|
||||
|
||||
chain = graph.get_causal_chain(child_id)
|
||||
|
||||
self.assertEqual(graph.nodes[root_id].content, scenario)
|
||||
self.assertEqual(graph.nodes[root_id].properties["scenario"], scenario)
|
||||
self.assertEqual(chain[0].scenario, scenario)
|
||||
|
||||
def test_record_decision_metadata_cannot_override_core_fields(self):
|
||||
graph = ContextGraph()
|
||||
decision_id = graph.record_decision(
|
||||
category="strategy",
|
||||
scenario="Open LATAM expansion program",
|
||||
reasoning="High growth potential",
|
||||
outcome="approved",
|
||||
confidence=0.9,
|
||||
metadata={
|
||||
"scenario": "metadata override",
|
||||
"category": "metadata category",
|
||||
"outcome": "metadata outcome",
|
||||
"custom_note": "keep me",
|
||||
},
|
||||
)
|
||||
|
||||
node = graph.nodes[decision_id]
|
||||
|
||||
self.assertEqual(node.content, "Open LATAM expansion program")
|
||||
self.assertEqual(node.properties["scenario"], "Open LATAM expansion program")
|
||||
self.assertEqual(node.properties["category"], "strategy")
|
||||
self.assertEqual(node.properties["outcome"], "approved")
|
||||
self.assertEqual(node.properties["custom_note"], "keep me")
|
||||
|
||||
# --- AgentMemory Tests ---
|
||||
def test_agent_memory_store(self):
|
||||
memory = AgentMemory(vector_store=self.mock_vector_store)
|
||||
|
||||
@@ -352,6 +352,99 @@ class TestContextGraphDecisions:
|
||||
precedents = context_graph.find_precedents("decision_001", limit=10)
|
||||
|
||||
assert len(precedents) == 0
|
||||
|
||||
def test_find_precedents_by_scenario_filters_superseded_as_of(self, context_graph):
|
||||
"""Decisions outside the validity window should be excluded for as_of queries."""
|
||||
expired_id = context_graph.record_decision(
|
||||
category="policy",
|
||||
scenario="Use old underwriting threshold",
|
||||
reasoning="Legacy policy",
|
||||
outcome="approved",
|
||||
confidence=0.7,
|
||||
valid_until="2023-01-01T00:00:00",
|
||||
)
|
||||
active_id = context_graph.record_decision(
|
||||
category="policy",
|
||||
scenario="Use new underwriting threshold",
|
||||
reasoning="Replacement policy",
|
||||
outcome="approved",
|
||||
confidence=0.9,
|
||||
valid_from="2023-01-02T00:00:00",
|
||||
)
|
||||
|
||||
results = context_graph.find_precedents_by_scenario(
|
||||
scenario="underwriting threshold",
|
||||
category="policy",
|
||||
as_of="2024-01-01T00:00:00",
|
||||
similarity_threshold=0.0,
|
||||
)
|
||||
|
||||
returned_ids = {item["decision"]["id"] for item in results}
|
||||
assert expired_id not in returned_ids
|
||||
assert active_id in returned_ids
|
||||
|
||||
def test_find_precedents_by_scenario_can_include_superseded(self, context_graph):
|
||||
"""Superseded decisions remain queryable when explicitly requested."""
|
||||
expired_id = context_graph.record_decision(
|
||||
category="policy",
|
||||
scenario="Use old threshold",
|
||||
reasoning="Legacy rule",
|
||||
outcome="approved",
|
||||
confidence=0.7,
|
||||
valid_until="2023-01-01T00:00:00",
|
||||
)
|
||||
|
||||
results = context_graph.find_precedents_by_scenario(
|
||||
scenario="old threshold",
|
||||
category="policy",
|
||||
include_superseded=True,
|
||||
similarity_threshold=0.0,
|
||||
)
|
||||
|
||||
returned_ids = {item["decision"]["id"] for item in results}
|
||||
assert expired_id in returned_ids
|
||||
|
||||
def test_state_at_returns_only_valid_items_and_is_serializable(self, context_graph):
|
||||
"""state_at should filter by validity window without mutating the graph."""
|
||||
context_graph.add_node(
|
||||
"entity_active",
|
||||
"entity",
|
||||
content="Active entity",
|
||||
valid_from="2024-01-01T00:00:00",
|
||||
valid_until="2024-12-31T23:59:59",
|
||||
)
|
||||
context_graph.add_node(
|
||||
"entity_expired",
|
||||
"entity",
|
||||
content="Expired entity",
|
||||
valid_until="2023-12-31T23:59:59",
|
||||
)
|
||||
context_graph.add_edge(
|
||||
"entity_active",
|
||||
"entity_expired",
|
||||
"related_to",
|
||||
valid_until="2023-12-31T23:59:59",
|
||||
)
|
||||
decision_id = context_graph.record_decision(
|
||||
category="policy",
|
||||
scenario="Current policy",
|
||||
reasoning="Current reasoning",
|
||||
outcome="approved",
|
||||
confidence=0.95,
|
||||
valid_from="2024-01-01T00:00:00",
|
||||
)
|
||||
|
||||
snapshot = context_graph.state_at("2024-06-01T00:00:00")
|
||||
|
||||
assert decision_id in context_graph.nodes
|
||||
node_ids = {node["id"] for node in snapshot["nodes"]}
|
||||
decision_ids = {decision["id"] for decision in snapshot["decisions"]}
|
||||
assert "entity_active" in node_ids
|
||||
assert "entity_expired" not in node_ids
|
||||
assert decision_id in decision_ids
|
||||
|
||||
import json
|
||||
json.dumps(snapshot)
|
||||
|
||||
def test_complex_causal_network(self, context_graph):
|
||||
"""Test complex causal network with multiple relationship types."""
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
"""Tests for cross-graph linking, navigation, and save/load persistence."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _two_graphs():
|
||||
"""Return two graphs each with one node."""
|
||||
g1 = ContextGraph()
|
||||
g2 = ContextGraph()
|
||||
g1.add_node("src", "entity", content="source entity")
|
||||
g2.add_node("dst", "entity", content="destination entity")
|
||||
return g1, g2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# link_graph / navigate_to — basic contract
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLinkGraph:
|
||||
def test_returns_link_id(self):
|
||||
g1, g2 = _two_graphs()
|
||||
link_id = g1.link_graph(g2, "src", "dst")
|
||||
assert isinstance(link_id, str) and len(link_id) > 0
|
||||
|
||||
def test_navigate_to_returns_correct_graph_and_node(self):
|
||||
g1, g2 = _two_graphs()
|
||||
link_id = g1.link_graph(g2, "src", "dst")
|
||||
other, entry = g1.navigate_to(link_id)
|
||||
assert other is g2
|
||||
assert entry == "dst"
|
||||
|
||||
def test_navigate_to_unknown_link_raises(self):
|
||||
g1, _ = _two_graphs()
|
||||
with pytest.raises(KeyError):
|
||||
g1.navigate_to("nonexistent-link-id")
|
||||
|
||||
def test_source_not_in_graph_raises(self):
|
||||
g1, g2 = _two_graphs()
|
||||
with pytest.raises(KeyError):
|
||||
g1.link_graph(g2, "missing", "dst")
|
||||
|
||||
def test_target_not_in_other_graph_raises(self):
|
||||
g1, g2 = _two_graphs()
|
||||
with pytest.raises(KeyError):
|
||||
g1.link_graph(g2, "src", "missing")
|
||||
|
||||
def test_marker_node_has_cross_graph_link_type(self):
|
||||
"""link_graph() must NOT pollute graph with phantom 'entity' nodes."""
|
||||
g1, g2 = _two_graphs()
|
||||
link_id = g1.link_graph(g2, "src", "dst")
|
||||
marker_id = f"__cross_graph_{link_id}"
|
||||
assert marker_id in g1.nodes
|
||||
assert g1.nodes[marker_id].node_type == "cross_graph_link"
|
||||
|
||||
def test_no_phantom_entity_nodes(self):
|
||||
"""Only 'src' and the typed marker should exist in g1."""
|
||||
g1, g2 = _two_graphs()
|
||||
link_id = g1.link_graph(g2, "src", "dst")
|
||||
entity_nodes = [n for n in g1.nodes.values() if n.node_type == "entity"]
|
||||
assert len(entity_nodes) == 1 # only 'src'
|
||||
|
||||
def test_multiple_links_from_same_source(self):
|
||||
g1 = ContextGraph()
|
||||
g2 = ContextGraph()
|
||||
g3 = ContextGraph()
|
||||
g1.add_node("hub", "entity")
|
||||
g2.add_node("a", "entity")
|
||||
g3.add_node("b", "entity")
|
||||
lid1 = g1.link_graph(g2, "hub", "a")
|
||||
lid2 = g1.link_graph(g3, "hub", "b")
|
||||
other1, entry1 = g1.navigate_to(lid1)
|
||||
other2, entry2 = g1.navigate_to(lid2)
|
||||
assert other1 is g2 and entry1 == "a"
|
||||
assert other2 is g3 and entry2 == "b"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Persistence: save_to_file / load_from_file + resolve_links
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCrossGraphPersistence:
|
||||
def test_graph_id_preserved_after_save_load(self):
|
||||
g1, _ = _two_graphs()
|
||||
original_id = g1.graph_id
|
||||
with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f:
|
||||
path = f.name
|
||||
try:
|
||||
g1.save_to_file(path)
|
||||
g1b = ContextGraph()
|
||||
g1b.load_from_file(path)
|
||||
assert g1b.graph_id == original_id
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
def test_links_section_written_to_file(self):
|
||||
g1, g2 = _two_graphs()
|
||||
link_id = g1.link_graph(g2, "src", "dst")
|
||||
with tempfile.NamedTemporaryFile(suffix=".json", delete=False, mode="w") as f:
|
||||
path = f.name
|
||||
try:
|
||||
g1.save_to_file(path)
|
||||
with open(path) as fp:
|
||||
data = json.load(fp)
|
||||
assert "links" in data
|
||||
assert len(data["links"]) == 1
|
||||
lk = data["links"][0]
|
||||
assert lk["link_id"] == link_id
|
||||
assert lk["source_node_id"] == "src"
|
||||
assert lk["target_node_id"] == "dst"
|
||||
assert lk["other_graph_id"] == g2.graph_id
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
def test_navigate_to_raises_helpful_error_before_resolve(self):
|
||||
g1, g2 = _two_graphs()
|
||||
link_id = g1.link_graph(g2, "src", "dst")
|
||||
with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f:
|
||||
path = f.name
|
||||
try:
|
||||
g1.save_to_file(path)
|
||||
g1b = ContextGraph()
|
||||
g1b.load_from_file(path)
|
||||
with pytest.raises(KeyError, match="resolve_links"):
|
||||
g1b.navigate_to(link_id)
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
def test_resolve_links_restores_navigation(self):
|
||||
g1, g2 = _two_graphs()
|
||||
link_id = g1.link_graph(g2, "src", "dst")
|
||||
|
||||
with (
|
||||
tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f1,
|
||||
tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f2,
|
||||
):
|
||||
path1, path2 = f1.name, f2.name
|
||||
|
||||
try:
|
||||
g1.save_to_file(path1)
|
||||
g2.save_to_file(path2)
|
||||
|
||||
g1b = ContextGraph()
|
||||
g2b = ContextGraph()
|
||||
g1b.load_from_file(path1)
|
||||
g2b.load_from_file(path2)
|
||||
|
||||
resolved = g1b.resolve_links({g2b.graph_id: g2b})
|
||||
assert resolved == 1
|
||||
|
||||
other, entry = g1b.navigate_to(link_id)
|
||||
assert other is g2b
|
||||
assert entry == "dst"
|
||||
finally:
|
||||
os.unlink(path1)
|
||||
os.unlink(path2)
|
||||
|
||||
def test_resolve_links_returns_count(self):
|
||||
g1 = ContextGraph()
|
||||
g2 = ContextGraph()
|
||||
g3 = ContextGraph()
|
||||
g1.add_node("h", "entity")
|
||||
g2.add_node("a", "entity")
|
||||
g3.add_node("b", "entity")
|
||||
g1.link_graph(g2, "h", "a")
|
||||
g1.link_graph(g3, "h", "b")
|
||||
|
||||
with (
|
||||
tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f1,
|
||||
tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f2,
|
||||
tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f3,
|
||||
):
|
||||
p1, p2, p3 = f1.name, f2.name, f3.name
|
||||
try:
|
||||
g1.save_to_file(p1); g2.save_to_file(p2); g3.save_to_file(p3)
|
||||
g1b, g2b, g3b = ContextGraph(), ContextGraph(), ContextGraph()
|
||||
g1b.load_from_file(p1); g2b.load_from_file(p2); g3b.load_from_file(p3)
|
||||
resolved = g1b.resolve_links({g2b.graph_id: g2b, g3b.graph_id: g3b})
|
||||
assert resolved == 2
|
||||
finally:
|
||||
for p in (p1, p2, p3):
|
||||
os.unlink(p)
|
||||
|
||||
def test_resolve_links_partial_registry_leaves_unresolved(self):
|
||||
"""Passing only one graph to resolve_links should resolve only that link."""
|
||||
g1 = ContextGraph()
|
||||
g2 = ContextGraph()
|
||||
g3 = ContextGraph()
|
||||
g1.add_node("h", "entity")
|
||||
g2.add_node("a", "entity")
|
||||
g3.add_node("b", "entity")
|
||||
lid1 = g1.link_graph(g2, "h", "a")
|
||||
lid2 = g1.link_graph(g3, "h", "b")
|
||||
|
||||
with (
|
||||
tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f1,
|
||||
tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f2,
|
||||
tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f3,
|
||||
):
|
||||
p1, p2, p3 = f1.name, f2.name, f3.name
|
||||
try:
|
||||
g1.save_to_file(p1); g2.save_to_file(p2); g3.save_to_file(p3)
|
||||
g1b, g2b, g3b = ContextGraph(), ContextGraph(), ContextGraph()
|
||||
g1b.load_from_file(p1); g2b.load_from_file(p2); g3b.load_from_file(p3)
|
||||
|
||||
# Only resolve g2
|
||||
resolved = g1b.resolve_links({g2b.graph_id: g2b})
|
||||
assert resolved == 1
|
||||
|
||||
# lid1 navigable
|
||||
other, entry = g1b.navigate_to(lid1)
|
||||
assert other is g2b and entry == "a"
|
||||
|
||||
# lid2 still unresolved — must raise with hint
|
||||
with pytest.raises(KeyError, match="resolve_links"):
|
||||
g1b.navigate_to(lid2)
|
||||
finally:
|
||||
for p in (p1, p2, p3):
|
||||
os.unlink(p)
|
||||
@@ -302,20 +302,25 @@ class TestDecisionQuery:
|
||||
|
||||
mock_graph_store.execute_query.return_value = [
|
||||
{
|
||||
"path": "mock_path_1",
|
||||
"path_nodes": [{"decision_id": "d1", "scenario": "S1", "category": "C1"}],
|
||||
"path_rels": [{"from": "d1", "to": "d2", "type": "CAUSED"}],
|
||||
"path_length": 2
|
||||
},
|
||||
{
|
||||
"path": "mock_path_2",
|
||||
"path_nodes": [{"decision_id": "d2", "scenario": "S2", "category": "C2"}],
|
||||
"path_rels": [],
|
||||
"path_length": 3
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
paths = decision_query.trace_decision_path(decision_id, relationship_types)
|
||||
|
||||
|
||||
assert len(paths) == 2
|
||||
assert paths[0]["path"] == "mock_path_1"
|
||||
assert paths[0]["path_length"] == 2
|
||||
assert isinstance(paths[0]["nodes"], list)
|
||||
assert isinstance(paths[0]["relationships"], list)
|
||||
assert paths[0]["nodes"][0]["scenario"] == "S1"
|
||||
assert paths[0]["relationships"][0]["type"] == "CAUSED"
|
||||
|
||||
# Verify query was called with relationship types
|
||||
call_args = mock_graph_store.execute_query.call_args
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
"""
|
||||
Tests for ContextGraph-native fallback paths in DecisionQuery and DecisionRecorder.
|
||||
|
||||
Covers the full integration flow and individual method contracts.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.context.context_graph import ContextGraph
|
||||
from semantica.context.decision_models import Decision, PolicyException
|
||||
from semantica.context.decision_query import DecisionQuery
|
||||
from semantica.context.decision_recorder import DecisionRecorder
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.fixture
|
||||
def memory_components():
|
||||
cg = ContextGraph()
|
||||
recorder = DecisionRecorder(graph_store=cg)
|
||||
dq = DecisionQuery(graph_store=cg)
|
||||
return cg, recorder, dq
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def graph():
|
||||
return ContextGraph()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def recorder(graph):
|
||||
return DecisionRecorder(graph_store=graph)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def query(graph):
|
||||
return DecisionQuery(graph_store=graph, advanced_analytics=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_decision(category="approval", scenario="Test scenario", outcome="approved"):
|
||||
return Decision(
|
||||
decision_id=str(uuid.uuid4()),
|
||||
category=category,
|
||||
scenario=scenario,
|
||||
reasoning="Some reasoning",
|
||||
outcome=outcome,
|
||||
confidence=0.9,
|
||||
timestamp=datetime.now(),
|
||||
decision_maker="test_agent",
|
||||
)
|
||||
|
||||
|
||||
def _store(recorder, decision, entities=None):
|
||||
return recorder.record_decision(decision, entities or [], [])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration test (original from PR)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_decision_query_contextgraph_fallback(memory_components):
|
||||
"""Test all DecisionQuery paths with native ContextGraph fallback execution."""
|
||||
cg, recorder, dq = memory_components
|
||||
|
||||
entity_1 = "entity_user_1"
|
||||
entity_2 = "entity_company_1"
|
||||
now = datetime.now()
|
||||
|
||||
dec1_id = recorder.record_decision(
|
||||
Decision(
|
||||
decision_id="dec_1",
|
||||
category="loan_approval",
|
||||
scenario="User requested loan",
|
||||
reasoning="Good credit",
|
||||
outcome="approved",
|
||||
confidence=0.9,
|
||||
timestamp=now - timedelta(days=2),
|
||||
decision_maker="system",
|
||||
metadata={"amount": 5000}
|
||||
),
|
||||
entities=[entity_1],
|
||||
source_documents=[]
|
||||
)
|
||||
|
||||
dec2_id = recorder.record_decision(
|
||||
Decision(
|
||||
decision_id="dec_2",
|
||||
category="risk_assessment",
|
||||
scenario="Company requested credit line",
|
||||
reasoning="High debt ratio",
|
||||
outcome="rejected",
|
||||
confidence=0.95,
|
||||
timestamp=now - timedelta(days=1),
|
||||
decision_maker="analyst",
|
||||
metadata={"amount": 50000}
|
||||
),
|
||||
entities=[entity_2],
|
||||
source_documents=[]
|
||||
)
|
||||
|
||||
# Test find_by_category
|
||||
loans = dq.find_by_category("loan_approval")
|
||||
assert len(loans) == 1
|
||||
assert loans[0].decision_id == "dec_1"
|
||||
|
||||
# Test find_by_entity
|
||||
user_decisions = dq.find_by_entity(entity_1)
|
||||
assert len(user_decisions) == 1
|
||||
assert user_decisions[0].decision_id == "dec_1"
|
||||
|
||||
# Test find_by_time_range
|
||||
recent_decisions = dq.find_by_time_range(now - timedelta(days=3), now)
|
||||
assert len(recent_decisions) == 2
|
||||
|
||||
# Add a precedent link for tracing & multihop
|
||||
recorder.link_precedents(dec1_id, [dec2_id], ["SIMILAR_SCENARIO"])
|
||||
|
||||
# Test multi_hop_reasoning (Undirected Traversal)
|
||||
multi_hop = dq.multi_hop_reasoning(entity_1, "", max_hops=1)
|
||||
assert any(d.decision_id == "dec_1" for d in multi_hop), "Undirected traversal failed to find Dec 1 from Entity 1"
|
||||
|
||||
# Verify metadata preservation
|
||||
dec1 = [d for d in multi_hop if d.decision_id == "dec_1"][0]
|
||||
assert dec1.metadata.get("amount") == 5000, f"Custom metadata 'amount' lost: {dec1.metadata}"
|
||||
|
||||
# Test trace_decision_path
|
||||
paths = dq.trace_decision_path(dec1_id, ["SIMILAR_SCENARIO"])
|
||||
assert len(paths) == 1
|
||||
|
||||
# Test find_similar_exceptions
|
||||
recorder.record_exception(
|
||||
decision_id=dec2_id,
|
||||
policy_id="pol_1",
|
||||
reason="Market downturn special condition",
|
||||
approver="manager",
|
||||
approval_method="email",
|
||||
justification="Allowed due to macro factors"
|
||||
)
|
||||
exceptions = dq.find_similar_exceptions("Market downturn", limit=5)
|
||||
assert len(exceptions) == 1
|
||||
|
||||
print("\nALL FALLBACK VERIFICATIONS PASSED")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests: DecisionRecorder fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestDecisionRecorderFallback:
|
||||
|
||||
def test_store_and_retrieve_decision_node(self, graph, recorder):
|
||||
"""_store_decision_node must store all fields as flat node properties."""
|
||||
d = _make_decision()
|
||||
_store(recorder, d)
|
||||
|
||||
nodes = graph.find_nodes(node_type="Decision")
|
||||
assert any(n["id"] == d.decision_id for n in nodes), (
|
||||
"Decision node not found; likely stored under wrong key"
|
||||
)
|
||||
node = next(n for n in nodes if n["id"] == d.decision_id)
|
||||
meta = node.get("metadata", {})
|
||||
assert meta.get("category") == d.category
|
||||
assert meta.get("outcome") == d.outcome
|
||||
|
||||
def test_link_entities_creates_about_edges(self, graph, recorder):
|
||||
"""link_entities must create ABOUT edges to each entity node."""
|
||||
d = _make_decision()
|
||||
_store(recorder, d, entities=["entity_A", "entity_B"])
|
||||
|
||||
edges = graph.find_edges(edge_type="ABOUT")
|
||||
targets = {e["target"] for e in edges if e["source"] == d.decision_id}
|
||||
assert "entity_A" in targets
|
||||
assert "entity_B" in targets
|
||||
|
||||
def test_record_exception_creates_nodes_and_edges(self, graph, recorder):
|
||||
"""record_exception must persist exception node and GRANTED_EXCEPTION edge."""
|
||||
d = _make_decision()
|
||||
_store(recorder, d)
|
||||
|
||||
exc_id = recorder.record_exception(
|
||||
decision_id=d.decision_id,
|
||||
policy_id="pol_001",
|
||||
reason="Urgent override",
|
||||
approver="manager",
|
||||
approval_method="slack_dm",
|
||||
justification="Time-sensitive case",
|
||||
)
|
||||
|
||||
exc_nodes = graph.find_nodes(node_type="Exception")
|
||||
assert any(n["id"] == exc_id for n in exc_nodes)
|
||||
|
||||
granted_edges = graph.find_edges(edge_type="GRANTED_EXCEPTION")
|
||||
assert any(e["source"] == d.decision_id and e["target"] == exc_id
|
||||
for e in granted_edges)
|
||||
|
||||
def test_link_precedents_creates_edges(self, graph, recorder):
|
||||
"""link_precedents must create relationship edges between decisions."""
|
||||
d1 = _make_decision()
|
||||
d2 = _make_decision()
|
||||
_store(recorder, d1)
|
||||
_store(recorder, d2)
|
||||
|
||||
recorder.link_precedents(d1.decision_id, [d2.decision_id], ["INFLUENCED_BY"])
|
||||
edges = graph.find_edges(edge_type="INFLUENCED_BY")
|
||||
assert any(e["source"] == d1.decision_id and e["target"] == d2.decision_id
|
||||
for e in edges)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests: DecisionQuery fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestDecisionQueryFallback:
|
||||
|
||||
def test_find_precedents_basic_returns_decisions(self, graph, recorder, query):
|
||||
"""_find_precedents_basic must return stored decisions via ContextGraph."""
|
||||
for _ in range(3):
|
||||
_store(recorder, _make_decision())
|
||||
|
||||
results = query._find_precedents_basic("Test scenario", None, 10)
|
||||
assert len(results) == 3
|
||||
assert all(isinstance(r, Decision) for r in results)
|
||||
|
||||
def test_find_by_category_filters_correctly(self, graph, recorder, query):
|
||||
"""find_by_category must only return decisions matching the category."""
|
||||
_store(recorder, _make_decision(category="loan"))
|
||||
_store(recorder, _make_decision(category="loan"))
|
||||
_store(recorder, _make_decision(category="claim"))
|
||||
|
||||
loans = query.find_by_category("loan")
|
||||
assert len(loans) == 2
|
||||
assert all(d.category == "loan" for d in loans)
|
||||
|
||||
def test_find_by_entity_returns_linked_decisions(self, graph, recorder, query):
|
||||
"""find_by_entity must return decisions linked via ABOUT edges."""
|
||||
d = _make_decision()
|
||||
_store(recorder, d, entities=["customer_99"])
|
||||
|
||||
results = query.find_by_entity("customer_99")
|
||||
assert any(r.decision_id == d.decision_id for r in results)
|
||||
|
||||
def test_find_by_time_range_filters_correctly(self, graph, recorder, query):
|
||||
"""find_by_time_range must respect temporal bounds."""
|
||||
now = datetime.now()
|
||||
old = Decision(
|
||||
decision_id=str(uuid.uuid4()),
|
||||
category="x", scenario="old", reasoning="r", outcome="ok",
|
||||
confidence=0.5, timestamp=now - timedelta(days=10),
|
||||
decision_maker="agent",
|
||||
)
|
||||
recent = Decision(
|
||||
decision_id=str(uuid.uuid4()),
|
||||
category="x", scenario="recent", reasoning="r", outcome="ok",
|
||||
confidence=0.5, timestamp=now - timedelta(hours=1),
|
||||
decision_maker="agent",
|
||||
)
|
||||
recorder.record_decision(old, [], [])
|
||||
recorder.record_decision(recent, [], [])
|
||||
|
||||
results = query.find_by_time_range(now - timedelta(days=2), now + timedelta(hours=1))
|
||||
ids = {d.decision_id for d in results}
|
||||
assert recent.decision_id in ids
|
||||
assert old.decision_id not in ids
|
||||
|
||||
def test_find_by_time_range_tz_aware_naive_mix(self, graph, recorder, query):
|
||||
"""find_by_time_range must not crash when start is tz-aware and stored ts is naive."""
|
||||
d = _make_decision()
|
||||
recorder.record_decision(d, [], [])
|
||||
|
||||
start = datetime.now(tz=timezone.utc) - timedelta(hours=1)
|
||||
end = datetime.now(tz=timezone.utc) + timedelta(hours=1)
|
||||
results = query.find_by_time_range(start, end)
|
||||
assert isinstance(results, list)
|
||||
|
||||
def test_multi_hop_reasoning_finds_connected_decisions(self, graph, recorder, query):
|
||||
"""multi_hop_reasoning undirected BFS must find decisions via incoming edges."""
|
||||
d = _make_decision()
|
||||
_store(recorder, d, entities=["hub_entity"])
|
||||
|
||||
# ABOUT edge: d.decision_id → hub_entity (outgoing from decision)
|
||||
# Undirected BFS from hub_entity should walk the edge in reverse and find d
|
||||
results = query.multi_hop_reasoning("hub_entity", "context", max_hops=2)
|
||||
assert any(r.decision_id == d.decision_id for r in results)
|
||||
|
||||
def test_trace_decision_path_returns_paths(self, graph, recorder, query):
|
||||
"""trace_decision_path must return path dicts from a stored decision."""
|
||||
d1 = _make_decision()
|
||||
d2 = _make_decision()
|
||||
recorder.record_decision(d1, [], [])
|
||||
recorder.record_decision(d2, [], [])
|
||||
recorder.link_precedents(d1.decision_id, [d2.decision_id], ["INFLUENCED_BY"])
|
||||
|
||||
paths = query.trace_decision_path(d1.decision_id, ["INFLUENCED_BY"])
|
||||
assert len(paths) >= 1
|
||||
assert all("path_length" in p for p in paths)
|
||||
|
||||
def test_find_similar_exceptions_returns_exception_objects(self, graph, recorder, query):
|
||||
"""find_similar_exceptions must return PolicyException objects from ContextGraph."""
|
||||
d = _make_decision()
|
||||
_store(recorder, d)
|
||||
recorder.record_exception(
|
||||
decision_id=d.decision_id,
|
||||
policy_id="pol_002",
|
||||
reason="Budget exceeded",
|
||||
approver="director",
|
||||
approval_method="email",
|
||||
justification="Exceptional circumstances",
|
||||
)
|
||||
|
||||
results = query.find_similar_exceptions("budget issue", limit=10)
|
||||
assert len(results) >= 1
|
||||
assert all(isinstance(e, PolicyException) for e in results)
|
||||
|
||||
def test_isinstance_does_not_trigger_on_mock(self):
|
||||
"""type() is ContextGraph must not fire for Mock(spec=ContextGraph)."""
|
||||
from unittest.mock import Mock
|
||||
mock_store = Mock(spec=ContextGraph)
|
||||
assert type(mock_store) is not ContextGraph
|
||||
@@ -292,24 +292,27 @@ class TestEndToEndContextIntegration:
|
||||
{"graph_expansion": False, "max_results": 20},
|
||||
]
|
||||
|
||||
search_times = []
|
||||
for i, config in enumerate(search_configs):
|
||||
start_time = time.time()
|
||||
|
||||
|
||||
results = retriever.retrieve(
|
||||
query="Test document search",
|
||||
**config
|
||||
)
|
||||
|
||||
|
||||
search_time = time.time() - start_time
|
||||
search_times.append(search_time)
|
||||
print(f"[OK] Config {i+1}: {len(results)} results in {search_time:.3f}s")
|
||||
|
||||
|
||||
# Verify results
|
||||
assert len(results) <= config["max_results"], "Should respect max_results"
|
||||
assert all(isinstance(r, RetrievedContext) for r in results), "Should be RetrievedContext"
|
||||
|
||||
# Performance should be reasonable
|
||||
avg_time = sum(time.time() - start_time for _ in range(3)) / 3
|
||||
assert avg_time < 1.0, "Average search time should be under 1 second"
|
||||
|
||||
# Performance should be reasonable on development machines running real
|
||||
# sentence-transformers (384-dim); threshold is 5.0s per config on average
|
||||
avg_time = sum(search_times) / len(search_times)
|
||||
assert avg_time < 5.0, f"Average search time {avg_time:.3f}s should be under 5 seconds"
|
||||
|
||||
def test_multi_hop_reasoning(self):
|
||||
"""Test multi-hop reasoning capabilities."""
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user