diff --git a/.all-contributorsrc b/.all-contributorsrc deleted file mode 100644 index 77741d45..00000000 --- a/.all-contributorsrc +++ /dev/null @@ -1,17 +0,0 @@ -{ - "projectName": "Semantica", - "projectOwner": "Hawksight-AI", - "repoType": "github", - "repoHost": "https://github.com", - "files": [ - "CONTRIBUTORS.md" - ], - "imageSize": 100, - "commit": true, - "commitConvention": "conventional", - "contributors": [], - "contributorsPerLine": 7, - "badgeTemplate": "[![All Contributors](https://img.shields.io/badge/all_contributors-<%= contributors.length %>-orange.svg?style=flat-square)](#contributors)", - "skipCi": true -} - diff --git a/CHANGELOG.md b/CHANGELOG.md index 70668163..30590a19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,1122 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Fixed: Context Graphs decision tracking bugs and added comprehensive test coverage (PR #315 by @KaifAhmad1) + - Fixed empty/None decision ID handling in ContextGraph.add_decision() + - Fixed None metadata handling to prevent TypeError + - Fixed causal chain depth logic and node exclusion + - Fixed nonexistent node handling in add_causal_relationship() + - Added missing properties field in to_dict serialization + - Added missing from_dict method for graph deserialization + - Fixed precedent search direction in find_precedents() + - Fixed UUID generation logic in all decision models + - Added comprehensive test suite with 9 tests covering all features + - All 71 context tests now passing (100% success rate) + +- Fixed: PolicyEngine latest version selection on ContextGraph; AgentContext fallback robustness and secure logging (PR #TBD by @KaifAhmad1) +- Tests: Added ContextGraph fallback and AgentContext smoke tests; full suite passing + + - **Apache AGE Backend Security Fixes** (PR #311 by @Sameer6305, fixes by @KaifAhmad1): + - Added AgeStore class with GraphStore API compatibility + - Fixed SQL injection vulnerabilities with input validation + - Added psycopg2-binary dependency and migration guide + - Fixed parameter replacement and test mock leakage + - Enhanced error handling and Unicode display issues + +- **Context Engineering Enhancement** (PR #307 by @KaifAhmad1): + - Comprehensive decision tracking system with full lifecycle management (record → analyze → query → precedent → influence) + - Advanced KG algorithm integration: centrality analysis, community detection, node embeddings with ContextGraph + - Enhanced AgentContext with granular feature flags for decision tracking, KG algorithms, and vector store features + - PolicyException model replacing conflicting Exception name for meaningful business domain modeling + - GraphStore validation preventing runtime failures with explicit capability checking + - Hybrid search combining semantic, structural, and category similarity with configurable weights + - Decision influence analysis with centrality measures and causal chain tracking + - Policy management with versioning, compliance checking, and exception handling + - Production-ready architecture with audit trails, security, and scalability features + - 9 critical bug fixes: logging, security, audit trails, API compatibility, Cypher queries, centrality access, validation, naming + - Comprehensive documentation with usage guides, production examples, and API references + - 100% test coverage with all validation tests passing (9/9 tests) + - Enterprise-grade features for financial services, healthcare, legal, and business domains + - Complete backward compatibility with existing semantica components + - Performance optimizations: caching, indexing, and efficient graph operations + +- **Added PgVector Store Support** (PR #303 by @Sameer6305, @KaifAhmad1): + - Native PostgreSQL vector storage using pgvector extension with full integration + - Multiple distance metrics: cosine, L2/Euclidean, inner product with automatic score normalization + - Advanced indexing: HNSW and IVFFlat for approximate nearest neighbor search with tunable parameters + - JSONB metadata storage with flexible filtering capabilities and batch operations + - Connection pooling support with psycopg3/psycopg2 fallback and efficient resource management + - Comprehensive VectorStore integration with backend delegation and unified API + - Idempotent index creation and table management with safe migration support + - Production-ready security: SQL injection protection with psycopg_sql.SQL() and input validation + - Performance optimizations: UUID4-based IDs, batch executemany operations, connection pooling + - Full backward compatibility with existing vector store implementations + - 36+ comprehensive test cases with Docker integration and dependency skipping + - Complete documentation with setup guides, examples, and performance tuning + - CI/CD integration: resolved benchmark compatibility and fixed documentation links + +- **Improved Vector Store for Decision Tracking** (PR #293 by @KaifAhmad1): + - Comprehensive decision tracking capabilities with hybrid search combining semantic and structural embeddings + - New DecisionEmbeddingPipeline for generating semantic and structural embeddings with KG algorithm integration + - HybridSimilarityCalculator with configurable weights (semantic: 0.7, structural: 0.3) + - DecisionContext high-level interface for decision management with explainable AI features + - ContextRetriever with hybrid precedent search and multi-hop reasoning + - User-friendly convenience API: quick_decision(), find_precedents(), explain(), similar_to(), batch_decisions(), filter_decisions() + - Knowledge Graph algorithm integration: Node2Vec, PathFinder, CommunityDetector, CentralityCalculator, SimilarityCalculator, ConnectivityAnalyzer + - Explainable AI with path tracing, confidence scoring, and comprehensive decision explanations + - Performance optimizations: 0.028s per decision processing, 0.031s search performance, ~0.8KB per decision memory usage + - 100% backward compatibility maintained with existing VectorStore functionality + - 34+ comprehensive tests covering all functionality including end-to-end scenarios and performance benchmarks + - Real-world validation examples for banking and insurance domains + - Documentation with clear imports, examples, and API references + +- **Improved Graph Algorithms in KG Module** (PR #292 by @KaifAhmad1): + - Complete algorithm suite with 30+ graph algorithms across 7 categories + - Node Embeddings: Node2Vec, DeepWalk, Word2Vec for structural similarity analysis + - Similarity Analysis: Cosine, Euclidean, Manhattan, Correlation metrics with batch processing + - Path Finding: Dijkstra, A*, BFS, K-shortest paths for route and network analysis + - Link Prediction: Preferential attachment, Jaccard, Adamic-Adar for network completion + - Centrality Analysis: Degree, Betweenness, Closeness, PageRank for importance ranking + - Community Detection: Louvain, Leiden, Label propagation for clustering analysis + - Connectivity Analysis: Components, bridges, density for network robustness + - Unified provenance tracking system with GraphBuilderWithProvenance and AlgorithmTrackerWithProvenance + - Complete execution tracking with metadata, timestamps, and reproducibility IDs + - Comprehensive test coverage with 5 test suites and 40+ test methods + - Professional documentation overhaul for all modules and reference documentation + - Enterprise-ready functionality with error handling and NetworkX compatibility + - Performance optimizations with sparse matrix operations and batch processing + - Full backward compatibility maintained with gradual migration support + +- **Improved Security Configuration with Dependabot**: + - Configured bi-weekly security updates with manual review by @KaifAhmad1 + - Implemented automated security scans (Monday & Thursday at 7 AM IST) with Bandit, Safety, Semgrep + - Added security-critical package grouping (cryptography, requests, urllib3, certifi, pyopenssl) + - Enterprise-grade security with audit trail, compliance features, and zero auto-merge + - Optimized IST timezone scheduling (Security scans: 7 AM IST, PRs: 9 AM IST) + - Aligned with new Dependabot features: open-source proxy support, smart dependency grouping for Snowflake/Arrow/benchmark features, private registry support, semantic commit prefixes, and latest GitHub security best practices + +- **ResourceScheduler Deadlock Fix and Performance Improvements** (PR #299, #301 by @d4ndr4d3, @KaifAhmad1): + - Fixed critical deadlock in ResourceScheduler by replacing `threading.Lock()` with `threading.RLock()` + - Resolved nested lock acquisition issue in `allocate_resources()` → `allocate_cpu/memory/gpu()` calls + - Added allocation validation with `ValidationError` when no resources can be allocated + - Improved performance by moving progress tracking updates outside lock scope + - Implemented comprehensive resource cleanup on allocation failures to prevent leaks + - Added complete regression test suite (6 tests) for deadlock prevention and edge cases + - Improved error handling and documentation for better operator visibility + - Zero breaking changes, maintains thread safety and backward compatibility + +## [0.2.7] - 2026-02-09 + +### Added / Changed + +- **Snowflake Connector for Data Ingestion** (PR #276 by @Sameer6305): + - Native Snowflake connector with multi-authentication (password, OAuth, key-pair, SSO) + - Table and query ingestion with pagination, schema introspection, batch processing + - SQL injection prevention via identifier escaping, OAuth token validation + - Progress tracking integration, context manager support, document export + - 24 comprehensive unit tests with mocking, complete documentation and examples + - Added as optional dependency `db-snowflake` with snowflake-connector-python>=3.0.0 + +- **Apache Arrow Export Support** (PR #273 by @Sameer6305): + - Added Apache Arrow exporter with explicit schemas, entity/relationship export, compression support + - Integrated with export module and method registry, Pandas/DuckDB compatible + - 20 unit tests + 1 integration test, complete documentation with examples + +- **Comprehensive Benchmark Suite with Regression CLI** (PR #289 by @ZohaibHassan16, @KaifAhmad1): + - 137+ benchmarks across all 10 Semantica modules (Input, Core, Storage, Context, QA, Ontology, etc.) + - Environment-agnostic design with robust mocking system for CI/CD compatibility + - Statistical regression detection using Z-score analysis with configurable thresholds + - Automated performance auditing via GitHub Actions workflow + - Comprehensive documentation suite (benchmarks.md, architecture guides, usage examples) + - Zero breaking changes, production-ready with ultra-fast text processing (>10,000 ops/s) + - Added benchmark runner CLI: `python benchmarks/benchmark_runner.py` + +## [0.2.6] - 2026-02-03 + +### Added / Changed + +- **W3C PROV-O Compliant Provenance Tracking** (#254, #246): + - Comprehensive provenance tracking system with W3C PROV-O compliance across all 17 Semantica modules + - **Core Module**: `ProvenanceManager`, W3C PROV-O schemas, storage backends (InMemory, SQLite), SHA-256 integrity verification + - **Module Integrations**: Semantic Extract, LLMs (Groq, OpenAI, HuggingFace, LiteLLM), Pipeline, Context, Ingest, Embeddings, Graph/Vector/Triplet stores, Reasoning, Conflicts, Deduplication, Export, Parse, Normalize, Ontology, Visualization + - **Features**: Complete lineage tracking (Document → Chunk → Entity → Relationship → Graph), LLM tracking (tokens, costs, latency), source tracking, bridge axioms for domain transformations + - **Compliance Infrastructure**: W3C PROV-O, FDA 21 CFR Part 11, SOX, HIPAA, TNFD + - **Testing**: 237 tests covering core functionality, all 17 module integrations, edge cases, backward compatibility + - **Design**: Opt-in with `provenance=False` by default, zero breaking changes, no new dependencies + - Contributed by @KaifAhmad1 + +- **Enhanced Change Management Module** (#248, #243): + - Enterprise-grade version control for knowledge graphs and ontologies with persistent storage and audit trails + - **Core Classes**: `TemporalVersionManager` (KG versioning), `OntologyVersionManager` (ontology versioning), `ChangeLogEntry` (metadata) + - **Storage**: SQLite (persistent) and in-memory backends with thread-safe operations + - **Features**: SHA-256 checksums, detailed entity/relationship diffs, structural ontology comparison, email validation + - **Compliance Infrastructure**: HIPAA, SOX, FDA 21 CFR Part 11 with immutable audit trails + - **Testing**: 104 tests (100% pass) - unit, integration, compliance, performance, edge cases + - **Performance**: 17.6ms for 10k entities, 510+ ops/sec concurrent, handles 5k+ entity graphs + - **Migration**: Backward compatible, simplified class names, zero external dependencies + - Contributed by @KaifAhmad1 + +- CSV Ingestion Enhancements (PR #244 by @saloni0318) + - Auto-detect CSV encoding (chardet) and delimiter (csv.Sniffer) + - Tolerant decoding and malformed-row handling (`on_bad_lines='warn'`) + - Optional chunked reading for large files; metadata tracks detected values + - Expanded unit tests covering delimiters, quoted/multiline fields, header overrides, chunks, and NaN preservation + +- Tests: Comprehensive units for TextNormalizer (PR #242 by @ZohaibHassan16) + - Added focused test coverage for TextNormalizer behavior across inputs + +- Tests: Register integration mark and tidy ingest test warnings (PR #241 by @KaifAhmad1) + - Introduced integration test marker and reduced noisy warnings in ingest tests + +- **Ingest Unit Tests** (#239, #232): + - Comprehensive unit tests for ingestion modules (file, web, and feed ingestors) + - **Coverage**: File scanning (local/cloud S3/GCS/Azure), web ingestion (URL/sitemap/robots.txt), RSS/Atom feed parsing + - **Testing**: 998 lines of test code with mocked external dependencies for fast, isolated execution + - **Results**: file_ingestor (86%), web_ingestor (86%), feed_ingestor (80%) coverage + - Covers happy paths, edge cases, and error handling + - Contributed by @Mohammed2372 + +### Fixed + +- **Temperature Compatibility Fix** (#256, #252): + - Fixed hardcoded `temperature=0.3` that broke compatibility with models requiring specific temperature values (e.g., gpt-5-mini) + - Added `_add_if_set` helper method to `BaseProvider` that only passes parameters when explicitly set + - When `temperature=None`, parameter is omitted allowing APIs to use model defaults + - Updated all 5 providers: OpenAI, Groq, Gemini, Ollama, DeepSeek + - Reduced code by ~85 lines with cleaner parameter handling + - Comprehensive test coverage added (10 temperature tests, all passing) + - Backward compatible - no breaking changes + - Contributed by @F0rt1s and @IGES-Institut + +- **JenaStore Empty Graph Bug** (#257, #258): + - Fixed `ProcessingError: Graph not initialized` when operating on empty (but initialized) graphs + - Replaced implicit `if not self.graph:` checks with explicit `if self.graph is None:` validation in 5 methods (`add_triplets`, `get_triplets`, `delete_triplet`, `execute_sparql`, `serialize`) + - Properly distinguishes `None` (uninitialized) from empty graphs (initialized with 0 triplets) + - Unblocks benchmarking suite, fresh deployments, and testing workflows + - Contributed by @ZohaibHassan16 + +## [0.2.5] - 2026-01-27 + +### Added +- **Pinecone Vector Store Support**: + - Implemented native Pinecone support (`PineconeStore`) with full CRUD capabilities. + - Added support for serverless and pod-based indexes, namespaces, and metadata filtering. + - Integrated with `VectorStore` unified interface and registry. + - (Closes #219, Resolves #220) +- **Configurable LLM Retry Logic**: + - Exposed `max_retries` parameter in `NERExtractor`, `RelationExtractor`, `TripletExtractor` and low-level extraction methods (`extract_entities_llm`, `extract_relations_llm`, `extract_triplets_llm`). + - Defaults to 3 retries to prevent infinite loops during JSON validation failures or API timeouts. + - Propagated retry configuration through chunked processing helpers to ensure consistent behavior for long documents. + - Updated `03_Earnings_Call_Analysis.ipynb` to use `max_retries=3` by default. + +### Added +- **Bring Your Own Model (BYOM) Support**: + - Enabled full support for custom Hugging Face models in `NERExtractor`, `RelationExtractor`, and `TripletExtractor`. + - Added support for custom tokenizers in `HuggingFaceModelLoader` to handle models with non-standard tokenization requirements. + - Implemented robust fallback logic for model selection: runtime options (`extract(model=...)`) now correctly override configuration defaults. +- **Enhanced NER Implementation**: + - Added configurable aggregation strategies (`simple`, `first`, `average`, `max`) to `extract_entities_huggingface` for better sub-word token handling. + - Implemented robust IOB/BILOU parsing to reconstruct entities from raw model outputs when structured output is unavailable. + - Added confidence scoring for aggregated entities. +- **Relation Extraction Improvements**: + - Implemented standard entity marker technique (wrapping subject/object with ``, `` tags) in `extract_relations_huggingface` for compatibility with sequence classification models. + - Added structured output parsing to convert raw model predictions into validated `Relation` objects. +- **Triplet Extraction Completion**: + - Added specialized parsing for Seq2Seq models (e.g., REBEL) in `extract_triplets_huggingface` to generate structured triplets directly from text. + - Implemented post-processing logic to clean and validate generated triplets. + +### Fixed +- **LLM Extraction Stability**: + - Fixed infinite retry loops in `BaseProvider` by strictly enforcing `max_retries` limit during structured output generation. + - Resolved stuck execution in earnings call analysis notebooks when using smaller models (e.g., Llama 3 8B) that frequently produce invalid JSON. +- **Model Parameter Precedence**: + - Fixed issue where configuration defaults took precedence over runtime arguments in Hugging Face extractors. Runtime options now correctly override config values. +- **Import Handling**: + - Fixed circular import issues in test suites by implementing robust mocking strategies. + +## [0.2.4] - 2026-01-22 + +### Added +- **Ontology Ingestion Module**: + - Implemented `OntologyIngestor` in `semantica.ingest` for parsing RDF/OWL files (Turtle, RDF/XML, JSON-LD, N3) into standardized `OntologyData` objects. + - Added `ingest_ontology` convenience function and integrated it into the unified `ingest(source_type="ontology")` interface. + - Added recursive directory scanning support for batch ontology ingestion. + - Exposed ingestion tools in `semantica.ontology` for better discoverability. + - Added `OntologyData` dataclass for consistent metadata handling (source path, format, timestamps). +- **Documentation**: + - **Ontology Usage Guide**: Updated `ontology_usage.md` with comprehensive examples for single-file and directory ingestion. + - **API Reference**: Updated `ontology.md` with `OntologyIngestor` class documentation and method details. +- **Tests**: + - **Comprehensive Test Suite**: Added `tests/ingest/test_ontology_ingestor.py` covering all supported formats, error handling, and unified interface integration. + - **Demo Script**: Added `examples/demo_ontology_ingest.py` for end-to-end usage demonstration. + +## [0.2.3] - 2026-01-20 + +### Fixed +- **LLM Relation Extraction Parsing**: + - Fixed relation extraction returning zero relations despite successful API calls to Groq and other providers + - Normalized typed responses from instructor/OpenAI/Groq to consistent dict format before parsing + - Added structured JSON fallback when typed generation yields zero relations to avoid silent empty outputs + - Removed acceptance of extra kwargs (`max_tokens`, `max_entities_prompt`) from relation extraction internals + - Filtered kwargs passed to provider LLM calls to only `temperature` and `verbose` +- **API Parameter Handling**: + - Limited kwargs forwarded in chunked extraction helper to prevent parameter leakage + - Ensured minimal, safe parameters are passed to provider calls +- **Pipeline Circular Import (Issues #192, #193)**: + - Fixed circular import between `pipeline_builder` and `pipeline_validator` triggered during `semantica.pipeline` import + - Lazy-loaded `PipelineValidator` inside `PipelineBuilder.__init__` and guarded type hints with `TYPE_CHECKING` + - Ensured `from semantica.deduplication import DuplicateDetector` no longer fails even when pipeline module is imported +- **JupyterLab Progress Output (Issue #181)**: + - Added `SEMANTICA_DISABLE_JUPYTER_PROGRESS` environment variable to disable rich Jupyter/Colab progress tables + - When enabled, progress falls back to console-style output, preventing infinite scrolling and JupyterLab out-of-memory errors + +### Added +- **Comprehensive Test Suite**: +- - Added unit tests (`tests/test_relations_llm.py`) with mocked LLM provider covering both typed and structured response paths +- - Added integration tests (`tests/integration/test_relations_groq.py`) for real Groq API calls with environment variable API key +- - Tests validate relation extraction completion and result parsing across different response formats +- **Amazon Neptune Dev Environment**: +- - Added CloudFormation template (`cookbook/introduction/neptune-setup.yaml`) to provision a dev Neptune cluster with public endpoint and IAM auth enabled +- - Documented deployment, cost estimates, and IAM User vs IAM Role best practices in `cookbook/introduction/21_Amazon_Neptune_Store.ipynb` +- - Added `cfn-lint` to `.pre-commit-config.yaml` for validating CloudFormation templates while excluding `neptune-setup.yaml` from generic YAML linters +- **Vector Store High-Performance Ingestion**: +- - Added `VectorStore.add_documents` for high-throughput ingestion with automatic embedding generation, batching, and parallel processing +- - Added `VectorStore.embed_batch` helper for generating embeddings for lists of texts without immediately storing them +- - Enabled default parallel ingestion in `VectorStore` with `max_workers=6` for common workloads +- - Added dedicated documentation page `docs/vector_store_usage.md` describing high-performance vector store usage and configuration +- - Added `tests/vector_store/test_vector_store_parallel.py` covering parallel vs sequential performance, error handling, and edge cases for `add_documents` and `embed_batch` + +### Changed +- **Relation Extraction API**: +- - Simplified parameter interface by removing unused kwargs that were previously ignored +- - Improved error handling and verbose logging for debugging relation extraction issues +- - Enhanced robustness of post-response parsing across different LLM providers +- **Vector Store Defaults and Examples**: +- - Standardized `VectorStore` default concurrency to `max_workers=6` for parallel ingestion +- - Updated vector store reference documentation and usage guides to rely on implicit defaults instead of requiring manual `max_workers` configuration in examples + + +## [0.2.2] - 2026-01-15 + +### Added +- **Parallel Extraction Engine**: + - Implemented high-throughput parallel batch processing across all core extractors (`NERExtractor`, `RelationExtractor`, `TripletExtractor`, `EventDetector`, `SemanticNetworkExtractor`) using `concurrent.futures.ThreadPoolExecutor`. + - Added `max_workers` configuration parameter (default: 1) to all extractor `extract()` methods, allowing users to tune concurrency based on available CPU cores or API rate limits. + - **Parallel Chunking**: Implemented parallel processing for large document chunking in `_extract_entities_chunked` and `_extract_relations_chunked`, significantly reducing latency for long-form text analysis. + - **Thread-Safe Progress Tracking**: Enhanced `ProgressTracker` to handle concurrent updates from multiple threads without race conditions during batch processing. +- **Semantic Extract Performance & Regression**: + - Added edge-case regression suite covering max worker defaults, LLM prompt entity filtering, and extractor reuse. + - Added a runnable real-use-case benchmark script for batch latency across `NERExtractor`, `RelationExtractor`, `TripletExtractor`, `EventDetector`, `SemanticAnalyzer`, and `SemanticNetworkExtractor`. + - Added Groq LLM smoke tests that exercise LLM-based entities/relations/triplets when `GROQ_API_KEY` is available via environment configuration. + +### Security +- **Credential Sanitization**: + - Removed hardcoded API keys from 8 cookbook notebooks to prevent secret leakage. + - Enforced environment variable usage for `GROQ_API_KEY` across all examples. +- **Secure Caching**: + - Updated `ExtractionCache` to exclude sensitive parameters (e.g., `api_key`, `token`, `password`) from cache key generation, preventing secret leakage and enabling safe cache sharing. + - Upgraded cache key hashing algorithm from MD5 to **SHA-256** for enhanced collision resistance and security. + +### Changed +- **Gemini SDK Migration**: + - Migrated `GeminiProvider` to use the new `google-genai` SDK (v0.1.0+) to address deprecation warnings. + - Implemented graceful fallback to `google.generativeai` for backward compatibility. +- **Dependency Resolution**: + - Pinned `opentelemetry-api` and `opentelemetry-sdk` to `1.37.0` to resolve pip conflicts. + - Updated `protobuf` and `grpcio` constraints for better stability. +- **Entity Filtering Scope**: + - Removed entity filtering from non-LLM extraction flows to avoid accuracy regressions. + - Applied entity downselection only to LLM relation prompt construction, while matching returned entities against the full original entity list. +- **Batch Concurrency Defaults**: + - Standardized `max_workers` defaulting across `semantic_extract` and tuned for low-latency: ML-backed methods default to single-worker, while pattern/regex/rules/LLM/huggingface methods use a higher parallelism default capped by CPU. + - Raised the global `optimization.max_workers` default to 8 for better throughput on batch workloads. + +### Performance +- **Bottleneck Optimization (GitHub Issue #186)**: + - **Resolved Bottleneck #1 (Sequential Processing)**: Replaced sequential `for` loops with parallel execution for both document-level batches and intra-document chunks. + - **Performance Gains**: Achieved **~1.89x speedup** in real-world extraction scenarios (tested with Groq `llama-3.3-70b-versatile` on standard datasets). + - **Initialization Optimization**: Refactored test suite to use class-level `setUpClass` for LLM provider initialization, eliminating redundant API client creation overhead. +- **Low-Latency Entity Matching**: + - Avoided heavyweight embedding stack imports on common matches by improving fast matching heuristics and short-circuiting before embedding similarity. + - Optimized entity matching to prioritize exact/substring/word-boundary matches and only fall back to embedding similarity when needed, reducing CPU overhead in LLM relation/triplet mapping. + + +## [0.2.1] - 2026-01-12 + +### Fixed +- **LLM Output Stability (Bug #176)**: + - Fixed incomplete JSON output issues by correctly propagating `max_tokens` parameter in `extract_relations_llm`. + - Implemented automatic error handling that halves chunk sizes and retries when LLM context or output limits are exceeded. + - Fixed `AttributeError` in provider integration by ensuring consistent parameter passing via `**kwargs`. +- **Constraint Relaxations**: + - Removed hardcoded `max_length` constraints from `Entity`, `Relation`, and `Triplet` classes to support long-form semantic extraction (e.g., long descriptions or names). +- Fixed orchestrator lazy property initialization and configuration normalization logic in `Orchestrator`. +- Resolved `AssertionError` in orchestrator tests by aligning test mocks with production component usage. +- Fixed dependency compatibility issues by pinning `protobuf>=5.29.1,<7.0` and `grpcio>=1.71.2`. +- Added missing dependencies `GitPython` and `chardet` to `pyproject.toml`. +- Verified and aligned `FileObject.text` property usage in GraphRAG notebooks for consistent content decoding. + +### Changed +- **Chunking Defaults**: + - Increased default `max_text_length` for auto-chunking to **64,000 characters** (from 32k/16k) for OpenAI, Anthropic, Gemini, Groq, and DeepSeek providers. + - Unified chunking logic across `extract_entities_llm`, `extract_relations_llm`, and `extract_triplets_llm`. +- **Groq Support**: + - Standardized Groq provider defaults to use `llama-3.3-70b-versatile` with a 64k context window. + - Added native support for `max_tokens` and `max_completion_tokens` to prevent output truncation. + +### Added +- **Testing**: + - Added `tests/reproduce_issue_176.py` to validate `max_tokens` propagation and chunking behavior across all extractors. + + +## [0.2.0] - 2026-01-10 + +### Added +- **Amazon Neptune Support**: + - Added `AmazonNeptuneStore` providing Amazon Neptune graph database integration via Bolt protocol and OpenCypher. + - Implemented `NeptuneAuthTokenManager` extending Neo4j AuthManager for AWS IAM SigV4 signing with automatic token refresh. + - Added robust connection handling: retry logic with backoff for transient errors (signature expired, connection closed) and driver recreation. + - Added `graph-amazon-neptune` optional dependency group (boto3, neo4j). + - Comprehensive test suite covering all GraphStore interface methods. +- **Docling Integration**: + - Added `DoclingParser` in `semantica.parse` for high-fidelity document parsing using the Docling library. + - Supports multi-format parsing (PDF, DOCX, PPTX, XLSX, HTML, images) with superior table extraction and structure understanding. + - Implemented as a standalone parser supporting local execution, OCR, and multiple export formats (Markdown, HTML, JSON). +- **Robust Extraction Fallbacks**: + - Implemented comprehensive fallback chains ("ML/LLM" -> "Pattern" -> "Last Resort") across `NERExtractor`, `RelationExtractor`, and `TripletExtractor` to prevent empty result lists. + - Added "Last Resort" pattern matching in `NERExtractor` to identify capitalized words as generic entities when all other methods fail. + - Added "Last Resort" adjacency-based relation extraction in `RelationExtractor` to create weak connections between adjacent entities if no relations are found. + - Added fallback logic in `TripletExtractor` to convert relations to triplets or use rule-based extraction if standard methods fail. +- **Provenance & Tracking**: + - Added count tracking to batch processing logs in `NERExtractor`, `RelationExtractor`, and `TripletExtractor`. + - Added `batch_index` and `document_id` to the metadata of all extracted entities, relations, triplets, semantic roles, and clusters for better traceability. +- **Semantic Extract Improvements**: + - Introduced `auto-chunking` for long text processing in LLM extraction methods (`extract_entities_llm`, `extract_relations_llm`, `extract_triplets_llm`). + - Added `silent_fail` parameter to LLM extraction methods for configurable error handling. + - Implemented robust JSON parsing and automatic retry logic (3 attempts with exponential backoff) in `BaseProvider` for all LLM providers. + - Enhanced `GroqProvider` with better diagnostics and connectivity testing. + - Added comprehensive entity, relation, and triplet deduplication for chunked extraction. + - Added `semantica/semantic_extract/schemas.py` with canonical Pydantic models for consistent structured output. +- **Testing**: + - Added comprehensive robustness test suite `tests/semantic_extract/test_robustness_fallback.py` for validating extraction fallbacks and metadata propagation. + - Added comprehensive unit test suite `tests/embeddings/test_model_switching.py` for verifying dynamic model transitions and dimension updates. + - Added end-to-end integration test suite for Knowledge Graph pipeline validation (GraphBuilder -> EntityResolver -> GraphAnalyzer). +- **Other**: + - Added missing dependencies `GitPython` and `chardet` to `pyproject.toml`. + - Robustified ID extraction across `CentralityCalculator`, `CommunityDetector`, and `ConnectivityAnalyzer` to handle various entity formats. + - Improved `Entity` class hashability and equality logic in `utils/types.py`. + +### Changed +- **Deduplication & Conflict Logic**: + - Removed internal deduplication logic from `NERExtractor`, `RelationExtractor`, and `TripletExtractor`. + - Removed consistency/conflict checking from `ExtractionValidator` to defer to dedicated `semantica/conflicts` module. + - Removed `_deduplicate_*` methods from `semantica/semantic_extract/methods.py`. +- **Batch Processing & Consistency**: + - Standardized batch processing across all extractors (`NERExtractor`, `RelationExtractor`, `TripletExtractor`, `SemanticNetworkExtractor`, `EventDetector`, `SemanticAnalyzer`, `CoreferenceResolver`) using a unified `extract`/`analyze`/`resolve` method pattern with progress tracking. + - Added provenance metadata (`batch_index`, `document_id`) to `SemanticNetwork` nodes/edges, `Event` objects, `SemanticRole` results, `CoreferenceChain` mentions, and `SemanticCluster` (tracking source `document_ids`). + - Updated `SemanticClusterer.cluster` and `SemanticAnalyzer.cluster_semantically` to accept list of dictionaries (with `content` and `id` keys) for better document tracking during clustering. + - Removed legacy `check_triplet_consistency` from `TripletExtractor`. + - Removed `validate_consistency` and `_check_consistency` from `ExtractionValidator`. +- **Weighted Scoring**: + - Clarified weighted confidence scoring (50% Method Confidence + 50% Type Similarity) in comments. + - Explicitly labeled "Type Similarity" as "user-provided" in code comments to remove ambiguity. +- **Refactoring**: + - Fixed orchestrator lazy property initialization and configuration normalization logic in `Orchestrator`. + - Verified and aligned `FileObject.text` property usage in GraphRAG notebooks for consistent content decoding. + +### Fixed +- **Critical Fixes**: + - Resolved `NameError` in `extraction_validator.py` by adding missing `Union` import. + - Resolved issues where extractors would return empty lists for valid input text when primary extraction methods failed. + - Fixed metadata initialization issue in batch processing where `batch_index` and `document_id` were occasionally missing from extracted items. + - Ensured `LLMExtraction` methods (`enhance_entities`, `enhance_relations`) return original input instead of failing or returning empty results when LLM providers are unavailable. +- **Component Fixes**: + - Fixed model switching bug in `TextEmbedder` where internal state was not cleared, preventing dynamic updates between `fastembed` and `sentence_transformers` (#160). + - Implemented model-intrinsic embedding dimension detection in `TextEmbedder` to ensure consistency between models and vector databases. + - Updated `set_model` to properly refresh configuration and dimensions during model switches. + - Fixed `TypeError: unhashable type: 'Entity'` in `GraphAnalyzer` when processing graphs with raw `Entity` objects or dictionaries in relationships (#159). + - Resolved `AssertionError` in orchestrator tests by aligning test mocks with production component usage. + - Fixed dependency compatibility issues by pinning `protobuf==4.25.3` and `grpcio==1.67.1`. + - Fixed a bug in `TripletExtractor` where the `validate_triplets` method was shadowed by an internal attribute. + - Fixed incorrect `TextSplitter` import path in the `semantic_extract.methods` module. + +## [0.1.1] - 2026-01-05 + +### Added +- Exported `DoclingParser` and `DoclingMetadata` from `semantica.parse` for easier access. +- Added comprehensive `DoclingParser` usage examples to README and documentation. +- Added Windows-specific troubleshooting note for PyTorch DLL issues. + +### Fixed +- Fixed `DoclingParser` import/export issues across platforms (Windows, Linux, Google Colab). +- Improved error messaging when optional `docling` dependency is missing. +- Fixed versioning inconsistencies across the framework. + +## [0.1.0] - 2025-12-31 + +### Added +- New command-line interface (`semantica` CLI) with support for knowledge base building and info commands. +- Integrated FastAPI-based REST API server for remote access to framework functionality. +- Dedicated background worker component for scalable task processing and pipeline execution. +- Framework-level versioning configuration for PyPI distribution. +- Automated release workflow with Trusted Publishing support. + +### Changed +- Updated versioning across the framework to 0.1.0. +- Refined entry point configurations in `pyproject.toml`. +- Improved lazy module loading for core framework components. + +## [0.0.5] - 2025-11-26 + +### Changed +- Configured Trusted Publishing for secure automated PyPI deployments + +## [0.0.4] - 2025-11-26 + +### Changed +- Fixed PyPI deployment issues from v0.0.3 + +## [0.0.3] - 2025-11-25 + +### Changed +- Simplified CI/CD workflows - removed failing tests and strict linting +- Combined release and PyPI publishing into single workflow +- Simplified security scanning to weekly pip-audit only +- Streamlined GitHub Actions configuration + +### Added +- Comprehensive issue templates (Bug, Feature, Documentation, Support, Grant/Partnership) +- Updated pull request template with clear guidelines +- Community support documentation (SUPPORT.md) +- Funding and sponsorship configuration (FUNDING.yml) +- GitHub configuration README for maintainers +- 10+ new domain-specific cookbook examples (Finance, Healthcare, Cybersecurity, etc.) + +### Removed +- Redundant scripts folder (8 shell/PowerShell scripts) +- Unnecessary automation workflows (label-issues, mark-answered) +- Excessive issue templates + +## [0.0.2] - 2025-11-25 + +### Changed +- Updated README with streamlined content and better examples +- Added more notebooks to cookbook +- Improved documentation structure + +## [0.0.1] - 2024-01-XX + +### Added +- Core framework architecture +- Universal data ingestion (multiple file formats) +- Semantic intelligence engine (NER, relation extraction, event detection) +- Knowledge graph construction with entity resolution +- 6-stage ontology generation pipeline +- GraphRAG engine for hybrid retrieval +- Multi-agent system infrastructure +- Production-ready quality assurance modules +- Comprehensive documentation with MkDocs +- Cookbook with interactive tutorials +- Support for multiple vector stores (Weaviate, Qdrant, FAISS) +- Support for multiple graph databases (Neo4j, NetworkX, RDFLib) +- Temporal knowledge graph support +- Conflict detection and resolution +- Deduplication and entity merging +- Schema template enforcement +- Seed data management +- Multi-format export (RDF, JSON-LD, CSV, GraphML) +- Visualization tools +- Pipeline orchestration +- Streaming support (Kafka, RabbitMQ, Kinesis) +- Context engineering for AI agents +- Reasoning and inference engine + +### Documentation +- Getting started guide +- API reference for all modules +- Concepts and architecture documentation +- Use case examples +- Cookbook tutorials +- Community projects showcase + +--- + +## Types of Changes + +- **Added** for new features +- **Changed** for changes in existing functionality +- **Deprecated** for soon-to-be removed features +- **Removed** for now removed features +- **Fixed** for any bug fixes +- **Security** for vulnerability fixes + +## Migration Guides + +When breaking changes are introduced, migration guides will be provided in the release notes and documentation. + +--- + +For detailed release notes, see [GitHub Releases](https://github.com/Hawksight-AI/semantica/releases). + +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +- Fixed: PolicyEngine latest version selection on ContextGraph; AgentContext fallback robustness and secure logging (PR #TBD by @KaifAhmad1) +- Tests: Added ContextGraph fallback and AgentContext smoke tests; full suite passing + +- **Context Engineering Enhancement** (PR #307 by @KaifAhmad1): + - Comprehensive decision tracking system with full lifecycle management (record → analyze → query → precedent → influence) + - Advanced KG algorithm integration: centrality analysis, community detection, node embeddings with ContextGraph + - Enhanced AgentContext with granular feature flags for decision tracking, KG algorithms, and vector store features + - PolicyException model replacing conflicting Exception name for meaningful business domain modeling + - GraphStore validation preventing runtime failures with explicit capability checking + - Hybrid search combining semantic, structural, and category similarity with configurable weights + - Decision influence analysis with centrality measures and causal chain tracking + - Policy management with versioning, compliance checking, and exception handling + - Production-ready architecture with audit trails, security, and scalability features + - 9 critical bug fixes: logging, security, audit trails, API compatibility, Cypher queries, centrality access, validation, naming + - Comprehensive documentation with usage guides, production examples, and API references + - 100% test coverage with all validation tests passing (9/9 tests) + - Enterprise-grade features for financial services, healthcare, legal, and business domains + - Complete backward compatibility with existing semantica components + - Performance optimizations: caching, indexing, and efficient graph operations + +- **Added PgVector Store Support** (PR #303 by @Sameer6305, @KaifAhmad1): + - Native PostgreSQL vector storage using pgvector extension with full integration + - Multiple distance metrics: cosine, L2/Euclidean, inner product with automatic score normalization + - Advanced indexing: HNSW and IVFFlat for approximate nearest neighbor search with tunable parameters + - JSONB metadata storage with flexible filtering capabilities and batch operations + - Connection pooling support with psycopg3/psycopg2 fallback and efficient resource management + - Comprehensive VectorStore integration with backend delegation and unified API + - Idempotent index creation and table management with safe migration support + - Production-ready security: SQL injection protection with psycopg_sql.SQL() and input validation + - Performance optimizations: UUID4-based IDs, batch executemany operations, connection pooling + - Full backward compatibility with existing vector store implementations + - 36+ comprehensive test cases with Docker integration and dependency skipping + - Complete documentation with setup guides, examples, and performance tuning + - CI/CD integration: resolved benchmark compatibility and fixed documentation links + +- **Improved Vector Store for Decision Tracking** (PR #293 by @KaifAhmad1): + - Comprehensive decision tracking capabilities with hybrid search combining semantic and structural embeddings + - New DecisionEmbeddingPipeline for generating semantic and structural embeddings with KG algorithm integration + - HybridSimilarityCalculator with configurable weights (semantic: 0.7, structural: 0.3) + - DecisionContext high-level interface for decision management with explainable AI features + - ContextRetriever with hybrid precedent search and multi-hop reasoning + - User-friendly convenience API: quick_decision(), find_precedents(), explain(), similar_to(), batch_decisions(), filter_decisions() + - Knowledge Graph algorithm integration: Node2Vec, PathFinder, CommunityDetector, CentralityCalculator, SimilarityCalculator, ConnectivityAnalyzer + - Explainable AI with path tracing, confidence scoring, and comprehensive decision explanations + - Performance optimizations: 0.028s per decision processing, 0.031s search performance, ~0.8KB per decision memory usage + - 100% backward compatibility maintained with existing VectorStore functionality + - 34+ comprehensive tests covering all functionality including end-to-end scenarios and performance benchmarks + - Real-world validation examples for banking and insurance domains + - Documentation with clear imports, examples, and API references + +- **Improved Graph Algorithms in KG Module** (PR #292 by @KaifAhmad1): + - Complete algorithm suite with 30+ graph algorithms across 7 categories + - Node Embeddings: Node2Vec, DeepWalk, Word2Vec for structural similarity analysis + - Similarity Analysis: Cosine, Euclidean, Manhattan, Correlation metrics with batch processing + - Path Finding: Dijkstra, A*, BFS, K-shortest paths for route and network analysis + - Link Prediction: Preferential attachment, Jaccard, Adamic-Adar for network completion + - Centrality Analysis: Degree, Betweenness, Closeness, PageRank for importance ranking + - Community Detection: Louvain, Leiden, Label propagation for clustering analysis + - Connectivity Analysis: Components, bridges, density for network robustness + - Unified provenance tracking system with GraphBuilderWithProvenance and AlgorithmTrackerWithProvenance + - Complete execution tracking with metadata, timestamps, and reproducibility IDs + - Comprehensive test coverage with 5 test suites and 40+ test methods + - Professional documentation overhaul for all modules and reference documentation + - Enterprise-ready functionality with error handling and NetworkX compatibility + - Performance optimizations with sparse matrix operations and batch processing + - Full backward compatibility maintained with gradual migration support + +- **Improved Security Configuration with Dependabot**: + - Configured bi-weekly security updates with manual review by @KaifAhmad1 + - Implemented automated security scans (Monday & Thursday at 7 AM IST) with Bandit, Safety, Semgrep + - Added security-critical package grouping (cryptography, requests, urllib3, certifi, pyopenssl) + - Enterprise-grade security with audit trail, compliance features, and zero auto-merge + - Optimized IST timezone scheduling (Security scans: 7 AM IST, PRs: 9 AM IST) + - Aligned with new Dependabot features: open-source proxy support, smart dependency grouping for Snowflake/Arrow/benchmark features, private registry support, semantic commit prefixes, and latest GitHub security best practices + +- **ResourceScheduler Deadlock Fix and Performance Improvements** (PR #299, #301 by @d4ndr4d3, @KaifAhmad1): + - Fixed critical deadlock in ResourceScheduler by replacing `threading.Lock()` with `threading.RLock()` + - Resolved nested lock acquisition issue in `allocate_resources()` → `allocate_cpu/memory/gpu()` calls + - Added allocation validation with `ValidationError` when no resources can be allocated + - Improved performance by moving progress tracking updates outside lock scope + - Implemented comprehensive resource cleanup on allocation failures to prevent leaks + - Added complete regression test suite (6 tests) for deadlock prevention and edge cases + - Improved error handling and documentation for better operator visibility + - Zero breaking changes, maintains thread safety and backward compatibility + +## [0.2.7] - 2026-02-09 + +### Added / Changed + +- **Snowflake Connector for Data Ingestion** (PR #276 by @Sameer6305): + - Native Snowflake connector with multi-authentication (password, OAuth, key-pair, SSO) + - Table and query ingestion with pagination, schema introspection, batch processing + - SQL injection prevention via identifier escaping, OAuth token validation + - Progress tracking integration, context manager support, document export + - 24 comprehensive unit tests with mocking, complete documentation and examples + - Added as optional dependency `db-snowflake` with snowflake-connector-python>=3.0.0 + +- **Apache Arrow Export Support** (PR #273 by @Sameer6305): + - Added Apache Arrow exporter with explicit schemas, entity/relationship export, compression support + - Integrated with export module and method registry, Pandas/DuckDB compatible + - 20 unit tests + 1 integration test, complete documentation with examples + +- **Comprehensive Benchmark Suite with Regression CLI** (PR #289 by @ZohaibHassan16, @KaifAhmad1): + - 137+ benchmarks across all 10 Semantica modules (Input, Core, Storage, Context, QA, Ontology, etc.) + - Environment-agnostic design with robust mocking system for CI/CD compatibility + - Statistical regression detection using Z-score analysis with configurable thresholds + - Automated performance auditing via GitHub Actions workflow + - Comprehensive documentation suite (benchmarks.md, architecture guides, usage examples) + - Zero breaking changes, production-ready with ultra-fast text processing (>10,000 ops/s) + - Added benchmark runner CLI: `python benchmarks/benchmark_runner.py` + +## [0.2.6] - 2026-02-03 + +### Added / Changed + +- **W3C PROV-O Compliant Provenance Tracking** (#254, #246): + - Comprehensive provenance tracking system with W3C PROV-O compliance across all 17 Semantica modules + - **Core Module**: `ProvenanceManager`, W3C PROV-O schemas, storage backends (InMemory, SQLite), SHA-256 integrity verification + - **Module Integrations**: Semantic Extract, LLMs (Groq, OpenAI, HuggingFace, LiteLLM), Pipeline, Context, Ingest, Embeddings, Graph/Vector/Triplet stores, Reasoning, Conflicts, Deduplication, Export, Parse, Normalize, Ontology, Visualization + - **Features**: Complete lineage tracking (Document → Chunk → Entity → Relationship → Graph), LLM tracking (tokens, costs, latency), source tracking, bridge axioms for domain transformations + - **Compliance Infrastructure**: W3C PROV-O, FDA 21 CFR Part 11, SOX, HIPAA, TNFD + - **Testing**: 237 tests covering core functionality, all 17 module integrations, edge cases, backward compatibility + - **Design**: Opt-in with `provenance=False` by default, zero breaking changes, no new dependencies + - Contributed by @KaifAhmad1 + +- **Enhanced Change Management Module** (#248, #243): + - Enterprise-grade version control for knowledge graphs and ontologies with persistent storage and audit trails + - **Core Classes**: `TemporalVersionManager` (KG versioning), `OntologyVersionManager` (ontology versioning), `ChangeLogEntry` (metadata) + - **Storage**: SQLite (persistent) and in-memory backends with thread-safe operations + - **Features**: SHA-256 checksums, detailed entity/relationship diffs, structural ontology comparison, email validation + - **Compliance Infrastructure**: HIPAA, SOX, FDA 21 CFR Part 11 with immutable audit trails + - **Testing**: 104 tests (100% pass) - unit, integration, compliance, performance, edge cases + - **Performance**: 17.6ms for 10k entities, 510+ ops/sec concurrent, handles 5k+ entity graphs + - **Migration**: Backward compatible, simplified class names, zero external dependencies + - Contributed by @KaifAhmad1 + +- CSV Ingestion Enhancements (PR #244 by @saloni0318) + - Auto-detect CSV encoding (chardet) and delimiter (csv.Sniffer) + - Tolerant decoding and malformed-row handling (`on_bad_lines='warn'`) + - Optional chunked reading for large files; metadata tracks detected values + - Expanded unit tests covering delimiters, quoted/multiline fields, header overrides, chunks, and NaN preservation + +- Tests: Comprehensive units for TextNormalizer (PR #242 by @ZohaibHassan16) + - Added focused test coverage for TextNormalizer behavior across inputs + +- Tests: Register integration mark and tidy ingest test warnings (PR #241 by @KaifAhmad1) + - Introduced integration test marker and reduced noisy warnings in ingest tests + +- **Ingest Unit Tests** (#239, #232): + - Comprehensive unit tests for ingestion modules (file, web, and feed ingestors) + - **Coverage**: File scanning (local/cloud S3/GCS/Azure), web ingestion (URL/sitemap/robots.txt), RSS/Atom feed parsing + - **Testing**: 998 lines of test code with mocked external dependencies for fast, isolated execution + - **Results**: file_ingestor (86%), web_ingestor (86%), feed_ingestor (80%) coverage + - Covers happy paths, edge cases, and error handling + - Contributed by @Mohammed2372 + +### Fixed + +- **Temperature Compatibility Fix** (#256, #252): + - Fixed hardcoded `temperature=0.3` that broke compatibility with models requiring specific temperature values (e.g., gpt-5-mini) + - Added `_add_if_set` helper method to `BaseProvider` that only passes parameters when explicitly set + - When `temperature=None`, parameter is omitted allowing APIs to use model defaults + - Updated all 5 providers: OpenAI, Groq, Gemini, Ollama, DeepSeek + - Reduced code by ~85 lines with cleaner parameter handling + - Comprehensive test coverage added (10 temperature tests, all passing) + - Backward compatible - no breaking changes + - Contributed by @F0rt1s and @IGES-Institut + +- **JenaStore Empty Graph Bug** (#257, #258): + - Fixed `ProcessingError: Graph not initialized` when operating on empty (but initialized) graphs + - Replaced implicit `if not self.graph:` checks with explicit `if self.graph is None:` validation in 5 methods (`add_triplets`, `get_triplets`, `delete_triplet`, `execute_sparql`, `serialize`) + - Properly distinguishes `None` (uninitialized) from empty graphs (initialized with 0 triplets) + - Unblocks benchmarking suite, fresh deployments, and testing workflows + - Contributed by @ZohaibHassan16 + +## [0.2.5] - 2026-01-27 + +### Added +- **Pinecone Vector Store Support**: + - Implemented native Pinecone support (`PineconeStore`) with full CRUD capabilities. + - Added support for serverless and pod-based indexes, namespaces, and metadata filtering. + - Integrated with `VectorStore` unified interface and registry. + - (Closes #219, Resolves #220) +- **Configurable LLM Retry Logic**: + - Exposed `max_retries` parameter in `NERExtractor`, `RelationExtractor`, `TripletExtractor` and low-level extraction methods (`extract_entities_llm`, `extract_relations_llm`, `extract_triplets_llm`). + - Defaults to 3 retries to prevent infinite loops during JSON validation failures or API timeouts. + - Propagated retry configuration through chunked processing helpers to ensure consistent behavior for long documents. + - Updated `03_Earnings_Call_Analysis.ipynb` to use `max_retries=3` by default. + +### Added +- **Bring Your Own Model (BYOM) Support**: + - Enabled full support for custom Hugging Face models in `NERExtractor`, `RelationExtractor`, and `TripletExtractor`. + - Added support for custom tokenizers in `HuggingFaceModelLoader` to handle models with non-standard tokenization requirements. + - Implemented robust fallback logic for model selection: runtime options (`extract(model=...)`) now correctly override configuration defaults. +- **Enhanced NER Implementation**: + - Added configurable aggregation strategies (`simple`, `first`, `average`, `max`) to `extract_entities_huggingface` for better sub-word token handling. + - Implemented robust IOB/BILOU parsing to reconstruct entities from raw model outputs when structured output is unavailable. + - Added confidence scoring for aggregated entities. +- **Relation Extraction Improvements**: + - Implemented standard entity marker technique (wrapping subject/object with ``, `` tags) in `extract_relations_huggingface` for compatibility with sequence classification models. + - Added structured output parsing to convert raw model predictions into validated `Relation` objects. +- **Triplet Extraction Completion**: + - Added specialized parsing for Seq2Seq models (e.g., REBEL) in `extract_triplets_huggingface` to generate structured triplets directly from text. + - Implemented post-processing logic to clean and validate generated triplets. + +### Fixed +- **LLM Extraction Stability**: + - Fixed infinite retry loops in `BaseProvider` by strictly enforcing `max_retries` limit during structured output generation. + - Resolved stuck execution in earnings call analysis notebooks when using smaller models (e.g., Llama 3 8B) that frequently produce invalid JSON. +- **Model Parameter Precedence**: + - Fixed issue where configuration defaults took precedence over runtime arguments in Hugging Face extractors. Runtime options now correctly override config values. +- **Import Handling**: + - Fixed circular import issues in test suites by implementing robust mocking strategies. + +## [0.2.4] - 2026-01-22 + +### Added +- **Ontology Ingestion Module**: + - Implemented `OntologyIngestor` in `semantica.ingest` for parsing RDF/OWL files (Turtle, RDF/XML, JSON-LD, N3) into standardized `OntologyData` objects. + - Added `ingest_ontology` convenience function and integrated it into the unified `ingest(source_type="ontology")` interface. + - Added recursive directory scanning support for batch ontology ingestion. + - Exposed ingestion tools in `semantica.ontology` for better discoverability. + - Added `OntologyData` dataclass for consistent metadata handling (source path, format, timestamps). +- **Documentation**: + - **Ontology Usage Guide**: Updated `ontology_usage.md` with comprehensive examples for single-file and directory ingestion. + - **API Reference**: Updated `ontology.md` with `OntologyIngestor` class documentation and method details. +- **Tests**: + - **Comprehensive Test Suite**: Added `tests/ingest/test_ontology_ingestor.py` covering all supported formats, error handling, and unified interface integration. + - **Demo Script**: Added `examples/demo_ontology_ingest.py` for end-to-end usage demonstration. + +## [0.2.3] - 2026-01-20 + +### Fixed +- **LLM Relation Extraction Parsing**: + - Fixed relation extraction returning zero relations despite successful API calls to Groq and other providers + - Normalized typed responses from instructor/OpenAI/Groq to consistent dict format before parsing + - Added structured JSON fallback when typed generation yields zero relations to avoid silent empty outputs + - Removed acceptance of extra kwargs (`max_tokens`, `max_entities_prompt`) from relation extraction internals + - Filtered kwargs passed to provider LLM calls to only `temperature` and `verbose` +- **API Parameter Handling**: + - Limited kwargs forwarded in chunked extraction helper to prevent parameter leakage + - Ensured minimal, safe parameters are passed to provider calls +- **Pipeline Circular Import (Issues #192, #193)**: + - Fixed circular import between `pipeline_builder` and `pipeline_validator` triggered during `semantica.pipeline` import + - Lazy-loaded `PipelineValidator` inside `PipelineBuilder.__init__` and guarded type hints with `TYPE_CHECKING` + - Ensured `from semantica.deduplication import DuplicateDetector` no longer fails even when pipeline module is imported +- **JupyterLab Progress Output (Issue #181)**: + - Added `SEMANTICA_DISABLE_JUPYTER_PROGRESS` environment variable to disable rich Jupyter/Colab progress tables + - When enabled, progress falls back to console-style output, preventing infinite scrolling and JupyterLab out-of-memory errors + +### Added +- **Comprehensive Test Suite**: +- - Added unit tests (`tests/test_relations_llm.py`) with mocked LLM provider covering both typed and structured response paths +- - Added integration tests (`tests/integration/test_relations_groq.py`) for real Groq API calls with environment variable API key +- - Tests validate relation extraction completion and result parsing across different response formats +- **Amazon Neptune Dev Environment**: +- - Added CloudFormation template (`cookbook/introduction/neptune-setup.yaml`) to provision a dev Neptune cluster with public endpoint and IAM auth enabled +- - Documented deployment, cost estimates, and IAM User vs IAM Role best practices in `cookbook/introduction/21_Amazon_Neptune_Store.ipynb` +- - Added `cfn-lint` to `.pre-commit-config.yaml` for validating CloudFormation templates while excluding `neptune-setup.yaml` from generic YAML linters +- **Vector Store High-Performance Ingestion**: +- - Added `VectorStore.add_documents` for high-throughput ingestion with automatic embedding generation, batching, and parallel processing +- - Added `VectorStore.embed_batch` helper for generating embeddings for lists of texts without immediately storing them +- - Enabled default parallel ingestion in `VectorStore` with `max_workers=6` for common workloads +- - Added dedicated documentation page `docs/vector_store_usage.md` describing high-performance vector store usage and configuration +- - Added `tests/vector_store/test_vector_store_parallel.py` covering parallel vs sequential performance, error handling, and edge cases for `add_documents` and `embed_batch` + +### Changed +- **Relation Extraction API**: +- - Simplified parameter interface by removing unused kwargs that were previously ignored +- - Improved error handling and verbose logging for debugging relation extraction issues +- - Enhanced robustness of post-response parsing across different LLM providers +- **Vector Store Defaults and Examples**: +- - Standardized `VectorStore` default concurrency to `max_workers=6` for parallel ingestion +- - Updated vector store reference documentation and usage guides to rely on implicit defaults instead of requiring manual `max_workers` configuration in examples + + +## [0.2.2] - 2026-01-15 + +### Added +- **Parallel Extraction Engine**: + - Implemented high-throughput parallel batch processing across all core extractors (`NERExtractor`, `RelationExtractor`, `TripletExtractor`, `EventDetector`, `SemanticNetworkExtractor`) using `concurrent.futures.ThreadPoolExecutor`. + - Added `max_workers` configuration parameter (default: 1) to all extractor `extract()` methods, allowing users to tune concurrency based on available CPU cores or API rate limits. + - **Parallel Chunking**: Implemented parallel processing for large document chunking in `_extract_entities_chunked` and `_extract_relations_chunked`, significantly reducing latency for long-form text analysis. + - **Thread-Safe Progress Tracking**: Enhanced `ProgressTracker` to handle concurrent updates from multiple threads without race conditions during batch processing. +- **Semantic Extract Performance & Regression**: + - Added edge-case regression suite covering max worker defaults, LLM prompt entity filtering, and extractor reuse. + - Added a runnable real-use-case benchmark script for batch latency across `NERExtractor`, `RelationExtractor`, `TripletExtractor`, `EventDetector`, `SemanticAnalyzer`, and `SemanticNetworkExtractor`. + - Added Groq LLM smoke tests that exercise LLM-based entities/relations/triplets when `GROQ_API_KEY` is available via environment configuration. + +### Security +- **Credential Sanitization**: + - Removed hardcoded API keys from 8 cookbook notebooks to prevent secret leakage. + - Enforced environment variable usage for `GROQ_API_KEY` across all examples. +- **Secure Caching**: + - Updated `ExtractionCache` to exclude sensitive parameters (e.g., `api_key`, `token`, `password`) from cache key generation, preventing secret leakage and enabling safe cache sharing. + - Upgraded cache key hashing algorithm from MD5 to **SHA-256** for enhanced collision resistance and security. + +### Changed +- **Gemini SDK Migration**: + - Migrated `GeminiProvider` to use the new `google-genai` SDK (v0.1.0+) to address deprecation warnings. + - Implemented graceful fallback to `google.generativeai` for backward compatibility. +- **Dependency Resolution**: + - Pinned `opentelemetry-api` and `opentelemetry-sdk` to `1.37.0` to resolve pip conflicts. + - Updated `protobuf` and `grpcio` constraints for better stability. +- **Entity Filtering Scope**: + - Removed entity filtering from non-LLM extraction flows to avoid accuracy regressions. + - Applied entity downselection only to LLM relation prompt construction, while matching returned entities against the full original entity list. +- **Batch Concurrency Defaults**: + - Standardized `max_workers` defaulting across `semantic_extract` and tuned for low-latency: ML-backed methods default to single-worker, while pattern/regex/rules/LLM/huggingface methods use a higher parallelism default capped by CPU. + - Raised the global `optimization.max_workers` default to 8 for better throughput on batch workloads. + +### Performance +- **Bottleneck Optimization (GitHub Issue #186)**: + - **Resolved Bottleneck #1 (Sequential Processing)**: Replaced sequential `for` loops with parallel execution for both document-level batches and intra-document chunks. + - **Performance Gains**: Achieved **~1.89x speedup** in real-world extraction scenarios (tested with Groq `llama-3.3-70b-versatile` on standard datasets). + - **Initialization Optimization**: Refactored test suite to use class-level `setUpClass` for LLM provider initialization, eliminating redundant API client creation overhead. +- **Low-Latency Entity Matching**: + - Avoided heavyweight embedding stack imports on common matches by improving fast matching heuristics and short-circuiting before embedding similarity. + - Optimized entity matching to prioritize exact/substring/word-boundary matches and only fall back to embedding similarity when needed, reducing CPU overhead in LLM relation/triplet mapping. + + +## [0.2.1] - 2026-01-12 + +### Fixed +- **LLM Output Stability (Bug #176)**: + - Fixed incomplete JSON output issues by correctly propagating `max_tokens` parameter in `extract_relations_llm`. + - Implemented automatic error handling that halves chunk sizes and retries when LLM context or output limits are exceeded. + - Fixed `AttributeError` in provider integration by ensuring consistent parameter passing via `**kwargs`. +- **Constraint Relaxations**: + - Removed hardcoded `max_length` constraints from `Entity`, `Relation`, and `Triplet` classes to support long-form semantic extraction (e.g., long descriptions or names). +- Fixed orchestrator lazy property initialization and configuration normalization logic in `Orchestrator`. +- Resolved `AssertionError` in orchestrator tests by aligning test mocks with production component usage. +- Fixed dependency compatibility issues by pinning `protobuf>=5.29.1,<7.0` and `grpcio>=1.71.2`. +- Added missing dependencies `GitPython` and `chardet` to `pyproject.toml`. +- Verified and aligned `FileObject.text` property usage in GraphRAG notebooks for consistent content decoding. + +### Changed +- **Chunking Defaults**: + - Increased default `max_text_length` for auto-chunking to **64,000 characters** (from 32k/16k) for OpenAI, Anthropic, Gemini, Groq, and DeepSeek providers. + - Unified chunking logic across `extract_entities_llm`, `extract_relations_llm`, and `extract_triplets_llm`. +- **Groq Support**: + - Standardized Groq provider defaults to use `llama-3.3-70b-versatile` with a 64k context window. + - Added native support for `max_tokens` and `max_completion_tokens` to prevent output truncation. + +### Added +- **Testing**: + - Added `tests/reproduce_issue_176.py` to validate `max_tokens` propagation and chunking behavior across all extractors. + + +## [0.2.0] - 2026-01-10 + +### Added +- **Amazon Neptune Support**: + - Added `AmazonNeptuneStore` providing Amazon Neptune graph database integration via Bolt protocol and OpenCypher. + - Implemented `NeptuneAuthTokenManager` extending Neo4j AuthManager for AWS IAM SigV4 signing with automatic token refresh. + - Added robust connection handling: retry logic with backoff for transient errors (signature expired, connection closed) and driver recreation. + - Added `graph-amazon-neptune` optional dependency group (boto3, neo4j). + - Comprehensive test suite covering all GraphStore interface methods. +- **Docling Integration**: + - Added `DoclingParser` in `semantica.parse` for high-fidelity document parsing using the Docling library. + - Supports multi-format parsing (PDF, DOCX, PPTX, XLSX, HTML, images) with superior table extraction and structure understanding. + - Implemented as a standalone parser supporting local execution, OCR, and multiple export formats (Markdown, HTML, JSON). +- **Robust Extraction Fallbacks**: + - Implemented comprehensive fallback chains ("ML/LLM" -> "Pattern" -> "Last Resort") across `NERExtractor`, `RelationExtractor`, and `TripletExtractor` to prevent empty result lists. + - Added "Last Resort" pattern matching in `NERExtractor` to identify capitalized words as generic entities when all other methods fail. + - Added "Last Resort" adjacency-based relation extraction in `RelationExtractor` to create weak connections between adjacent entities if no relations are found. + - Added fallback logic in `TripletExtractor` to convert relations to triplets or use rule-based extraction if standard methods fail. +- **Provenance & Tracking**: + - Added count tracking to batch processing logs in `NERExtractor`, `RelationExtractor`, and `TripletExtractor`. + - Added `batch_index` and `document_id` to the metadata of all extracted entities, relations, triplets, semantic roles, and clusters for better traceability. +- **Semantic Extract Improvements**: + - Introduced `auto-chunking` for long text processing in LLM extraction methods (`extract_entities_llm`, `extract_relations_llm`, `extract_triplets_llm`). + - Added `silent_fail` parameter to LLM extraction methods for configurable error handling. + - Implemented robust JSON parsing and automatic retry logic (3 attempts with exponential backoff) in `BaseProvider` for all LLM providers. + - Enhanced `GroqProvider` with better diagnostics and connectivity testing. + - Added comprehensive entity, relation, and triplet deduplication for chunked extraction. + - Added `semantica/semantic_extract/schemas.py` with canonical Pydantic models for consistent structured output. +- **Testing**: + - Added comprehensive robustness test suite `tests/semantic_extract/test_robustness_fallback.py` for validating extraction fallbacks and metadata propagation. + - Added comprehensive unit test suite `tests/embeddings/test_model_switching.py` for verifying dynamic model transitions and dimension updates. + - Added end-to-end integration test suite for Knowledge Graph pipeline validation (GraphBuilder -> EntityResolver -> GraphAnalyzer). +- **Other**: + - Added missing dependencies `GitPython` and `chardet` to `pyproject.toml`. + - Robustified ID extraction across `CentralityCalculator`, `CommunityDetector`, and `ConnectivityAnalyzer` to handle various entity formats. + - Improved `Entity` class hashability and equality logic in `utils/types.py`. + +### Changed +- **Deduplication & Conflict Logic**: + - Removed internal deduplication logic from `NERExtractor`, `RelationExtractor`, and `TripletExtractor`. + - Removed consistency/conflict checking from `ExtractionValidator` to defer to dedicated `semantica/conflicts` module. + - Removed `_deduplicate_*` methods from `semantica/semantic_extract/methods.py`. +- **Batch Processing & Consistency**: + - Standardized batch processing across all extractors (`NERExtractor`, `RelationExtractor`, `TripletExtractor`, `SemanticNetworkExtractor`, `EventDetector`, `SemanticAnalyzer`, `CoreferenceResolver`) using a unified `extract`/`analyze`/`resolve` method pattern with progress tracking. + - Added provenance metadata (`batch_index`, `document_id`) to `SemanticNetwork` nodes/edges, `Event` objects, `SemanticRole` results, `CoreferenceChain` mentions, and `SemanticCluster` (tracking source `document_ids`). + - Updated `SemanticClusterer.cluster` and `SemanticAnalyzer.cluster_semantically` to accept list of dictionaries (with `content` and `id` keys) for better document tracking during clustering. + - Removed legacy `check_triplet_consistency` from `TripletExtractor`. + - Removed `validate_consistency` and `_check_consistency` from `ExtractionValidator`. +- **Weighted Scoring**: + - Clarified weighted confidence scoring (50% Method Confidence + 50% Type Similarity) in comments. + - Explicitly labeled "Type Similarity" as "user-provided" in code comments to remove ambiguity. +- **Refactoring**: + - Fixed orchestrator lazy property initialization and configuration normalization logic in `Orchestrator`. + - Verified and aligned `FileObject.text` property usage in GraphRAG notebooks for consistent content decoding. + +### Fixed +- **Critical Fixes**: + - Resolved `NameError` in `extraction_validator.py` by adding missing `Union` import. + - Resolved issues where extractors would return empty lists for valid input text when primary extraction methods failed. + - Fixed metadata initialization issue in batch processing where `batch_index` and `document_id` were occasionally missing from extracted items. + - Ensured `LLMExtraction` methods (`enhance_entities`, `enhance_relations`) return original input instead of failing or returning empty results when LLM providers are unavailable. +- **Component Fixes**: + - Fixed model switching bug in `TextEmbedder` where internal state was not cleared, preventing dynamic updates between `fastembed` and `sentence_transformers` (#160). + - Implemented model-intrinsic embedding dimension detection in `TextEmbedder` to ensure consistency between models and vector databases. + - Updated `set_model` to properly refresh configuration and dimensions during model switches. + - Fixed `TypeError: unhashable type: 'Entity'` in `GraphAnalyzer` when processing graphs with raw `Entity` objects or dictionaries in relationships (#159). + - Resolved `AssertionError` in orchestrator tests by aligning test mocks with production component usage. + - Fixed dependency compatibility issues by pinning `protobuf==4.25.3` and `grpcio==1.67.1`. + - Fixed a bug in `TripletExtractor` where the `validate_triplets` method was shadowed by an internal attribute. + - Fixed incorrect `TextSplitter` import path in the `semantic_extract.methods` module. + +## [0.1.1] - 2026-01-05 + +### Added +- Exported `DoclingParser` and `DoclingMetadata` from `semantica.parse` for easier access. +- Added comprehensive `DoclingParser` usage examples to README and documentation. +- Added Windows-specific troubleshooting note for PyTorch DLL issues. + +### Fixed +- Fixed `DoclingParser` import/export issues across platforms (Windows, Linux, Google Colab). +- Improved error messaging when optional `docling` dependency is missing. +- Fixed versioning inconsistencies across the framework. + +## [0.1.0] - 2025-12-31 + +### Added +- New command-line interface (`semantica` CLI) with support for knowledge base building and info commands. +- Integrated FastAPI-based REST API server for remote access to framework functionality. +- Dedicated background worker component for scalable task processing and pipeline execution. +- Framework-level versioning configuration for PyPI distribution. +- Automated release workflow with Trusted Publishing support. + +### Changed +- Updated versioning across the framework to 0.1.0. +- Refined entry point configurations in `pyproject.toml`. +- Improved lazy module loading for core framework components. + +## [0.0.5] - 2025-11-26 + +### Changed +- Configured Trusted Publishing for secure automated PyPI deployments + +## [0.0.4] - 2025-11-26 + +### Changed +- Fixed PyPI deployment issues from v0.0.3 + +## [0.0.3] - 2025-11-25 + +### Changed +- Simplified CI/CD workflows - removed failing tests and strict linting +- Combined release and PyPI publishing into single workflow +- Simplified security scanning to weekly pip-audit only +- Streamlined GitHub Actions configuration + +### Added +- Comprehensive issue templates (Bug, Feature, Documentation, Support, Grant/Partnership) +- Updated pull request template with clear guidelines +- Community support documentation (SUPPORT.md) +- Funding and sponsorship configuration (FUNDING.yml) +- GitHub configuration README for maintainers +- 10+ new domain-specific cookbook examples (Finance, Healthcare, Cybersecurity, etc.) + +### Removed +- Redundant scripts folder (8 shell/PowerShell scripts) +- Unnecessary automation workflows (label-issues, mark-answered) +- Excessive issue templates + +## [0.0.2] - 2025-11-25 + +### Changed +- Updated README with streamlined content and better examples +- Added more notebooks to cookbook +- Improved documentation structure + +## [0.0.1] - 2024-01-XX + +### Added +- Core framework architecture +- Universal data ingestion (multiple file formats) +- Semantic intelligence engine (NER, relation extraction, event detection) +- Knowledge graph construction with entity resolution +- 6-stage ontology generation pipeline +- GraphRAG engine for hybrid retrieval +- Multi-agent system infrastructure +- Production-ready quality assurance modules +- Comprehensive documentation with MkDocs +- Cookbook with interactive tutorials +- Support for multiple vector stores (Weaviate, Qdrant, FAISS) +- Support for multiple graph databases (Neo4j, NetworkX, RDFLib) +- Temporal knowledge graph support +- Conflict detection and resolution +- Deduplication and entity merging +- Schema template enforcement +- Seed data management +- Multi-format export (RDF, JSON-LD, CSV, GraphML) +- Visualization tools +- Pipeline orchestration +- Streaming support (Kafka, RabbitMQ, Kinesis) +- Context engineering for AI agents +- Reasoning and inference engine + +### Documentation +- Getting started guide +- API reference for all modules +- Concepts and architecture documentation +- Use case examples +- Cookbook tutorials +- Community projects showcase + +--- + +## Types of Changes + +- **Added** for new features +- **Changed** for changes in existing functionality +- **Deprecated** for soon-to-be removed features +- **Removed** for now removed features +- **Fixed** for any bug fixes +- **Security** for vulnerability fixes + +## Migration Guides + +When breaking changes are introduced, migration guides will be provided in the release notes and documentation. + +--- + +For detailed release notes, see [GitHub Releases](https://github.com/Hawksight-AI/semantica/releases). + +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +- Fixed: PolicyEngine latest version selection on ContextGraph; AgentContext fallback robustness and secure logging (PR #TBD by @KaifAhmad1) +- Tests: Added ContextGraph fallback and AgentContext smoke tests; full suite passing + - **Context Engineering Enhancement** (PR #307 by @KaifAhmad1): - Comprehensive decision tracking system with full lifecycle management (record → analyze → query → precedent → influence) - Advanced KG algorithm integration: centrality analysis, community detection, node embeddings with ContextGraph diff --git a/README.md b/README.md index 561697ca..40815b62 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ - --- ## 🚀 Why Semantica? @@ -35,30 +34,30 @@ pip install semantica ``` ```python -from semantica.context import AgentContext +from semantica.context import AgentContext, ContextGraph from semantica.vector_store import VectorStore -from semantica.kg import GraphBuilder -# Initialize context with advanced features +# Initialize with enhanced context features vs = VectorStore(backend="faiss", dimension=768) -kg = GraphBuilder().build({"entities": [], "relationships": []}) +kg = ContextGraph(advanced_analytics=True) context = AgentContext( vector_store=vs, knowledge_graph=kg, - enable_decision_tracking=True, - enable_advanced_analytics=True, - enable_kg_algorithms=True, - enable_vector_store_features=True + decision_tracking=True, + advanced_analytics=True, + kg_algorithms=True, + vector_store_features=True, + graph_expansion=True ) -# Store memory with context graphs +# Store memory with automatic context graph building memory_id = context.store( "User is working on a React project with FastAPI", conversation_id="session_1" ) -# Record decision with full context -decision_id = context.record_decision( +# Easy decision recording with convenience methods +decision_id = context.graph_builder.add_decision( category="technology_choice", scenario="Framework selection for web API", reasoning="React ecosystem with FastAPI provides best performance", @@ -66,15 +65,25 @@ decision_id = context.record_decision( confidence=0.92 ) -# Find similar decisions (precedents) -precedents = context.find_precedents_advanced( +# Find similar decisions with advanced analytics +similar_decisions = context.graph_builder.find_similar_decisions( scenario="Framework selection", - use_kg_features=True + max_results=5 ) +# Analyze decision impact and influence +impact = context.graph_builder.analyze_decision_impact(decision_id) + +# Check compliance with business rules +compliance = context.graph_builder.check_decision_rules({ + "category": "technology_choice", + "confidence": 0.92 +}) + print(f"Memory stored: {memory_id}") print(f"Decision recorded: {decision_id}") -print(f"Found {len(precedents)} precedents") +print(f"Found {len(similar_decisions)} similar decisions") +print(f"Compliance check: {compliance.get('compliant', False)}") ``` **[📖 Full Quick Start](#-quick-start)** • **[🍳 Cookbook Examples](#-semantica-cookbook)** • **[💬 Join Discord](https://discord.gg/N7WmAuDH)** • **[⭐ Star Us](https://github.com/Hawksight-AI/semantica)** @@ -136,71 +145,167 @@ print(f"Found {len(precedents)} precedents") - **Docling Support** — Document parsing with table extraction (PDF, DOCX, PPTX, XLSX) - **AWS Neptune** — Amazon Neptune graph database support with IAM authentication +- **Apache AGE** — PostgreSQL graph extension backend (openCypher via SQL) - **Custom Ontology Import** — Import existing ontologies (OWL, RDF, Turtle, JSON-LD) > **Built for environments where every answer must be explainable and governed.** --- -## 🧠 Context Module: Advanced Context Engineering +## 🧠 Context Module: Advanced Context Engineering & Decision Intelligence -The **Context Module** is Semantica's flagship component, providing sophisticated context management with **context graphs**, **decision tracking**, and **advanced knowledge engineering**. +The **Context Module** is Semantica's flagship component, providing sophisticated context management with **context graphs**, **advanced decision tracking**, **knowledge graph analytics**, and **easy-to-use interfaces**. ### 🎯 Core Capabilities | **Feature** | **Description** | **Use Case** | |------------|-------------|------------| | **Context Graphs** | Structured knowledge representation with entity relationships | Knowledge management, decision support | -| **Decision Tracking** | Complete decision lifecycle with precedent search | Banking approvals, healthcare decisions | -| **KG Algorithms** | Advanced graph analytics (centrality, community detection) | Influence analysis, similarity search | +| **Advanced Decision Tracking** | Complete decision lifecycle with precedent search, causal analysis, and policy enforcement | Banking approvals, healthcare decisions | +| **Easy-to-Use Methods** | 10 convenience methods for common operations without complexity | Rapid development, user-friendly API | +| **KG Algorithms** | Advanced graph analytics (centrality, community detection, Node2Vec) | Influence analysis, similarity search | +| **Policy Engine** | Automated compliance checking with business rules and exception handling | Regulatory compliance, business rules | | **Vector Store Integration** | Hybrid search with custom similarity weights | Advanced retrieval and filtering | | **Memory Management** | Hierarchical memory with short-term and long-term storage | Agent conversation history | -### 🚀 Advanced Features +### 🚀 Enhanced Features +- **Easy Decision Recording**: `add_decision()` with automatic entity linking +- **Smart Precedent Search**: `find_similar_decisions()` with hybrid similarity +- **Impact Analysis**: `analyze_decision_impact()` with influence scoring +- **Policy Compliance**: `check_decision_rules()` with automated validation +- **Causal Chains**: `trace_decision_chain()` for decision lineage +- **Graph Analytics**: `get_node_importance()`, `analyze_connections()` for insights - **Hybrid Retrieval**: Combines vector search, graph traversal, and keyword matching - **Multi-Hop Reasoning**: Trace relationships across multiple graph hops -- **Decision Influence Analysis**: Understand how decisions impact each other -- **Policy Engine**: Enforce business rules and compliance automatically -- **Causal Chain Analysis**: Trace decision causality and influence paths -- **Entity Linking**: Resolve ambiguities and maintain consistent entity references +- **Production Ready**: Comprehensive error handling and scalability -### Examples +### 🔧 Easy-to-Use API ```python -# Banking Decision System +# Simple usage with convenience methods +from semantica.context import ContextGraph + +graph = ContextGraph(advanced_analytics=True) + +# Add decision with ease +decision_id = graph.add_decision( + category="loan_approval", + scenario="Mortgage application", + reasoning="Good credit score", + outcome="approved", + confidence=0.95 +) + +# Find similar decisions +similar = graph.find_similar_decisions("mortgage", max_results=5) + +# Analyze impact +impact = graph.analyze_decision_impact(decision_id) + +# Check compliance +compliance = graph.check_decision_rules({ + "category": "loan_approval", + "confidence": 0.95 +}) +``` + +### 🏢 Enterprise Integration + +```python +# Full enterprise setup with AgentContext from semantica.context import AgentContext +from semantica.vector_store import VectorStore context = AgentContext( - vector_store=vs, - knowledge_graph=kg, - enable_decision_tracking=True, - enable_kg_algorithms=True + vector_store=VectorStore(backend="faiss"), + knowledge_graph=ContextGraph(advanced_analytics=True), + decision_tracking=True, + kg_algorithms=True, + vector_store_features=True ) -# Record loan decision +# Record decision with full context decision_id = context.record_decision( - category="mortgage_approval", - scenario="First-time homebuyer application", - reasoning="Strong credit score, stable employment", - outcome="approved", - confidence=0.94 + category="fraud_detection", + scenario="Suspicious transaction pattern", + reasoning="Multiple high-value transactions in short timeframe", + outcome="flagged_for_review", + confidence=0.87, + entities=["transaction_123", "customer_456"] ) -# Find similar decisions with KG features -precedents = context.find_precedents_advanced( - scenario="Mortgage application", - use_kg_features=True, - similarity_weights={"semantic": 0.5, "structural": 0.3, "category": 0.2} +# Advanced precedent search with KG features +precedents = context.find_precedents( + "suspicious transaction", + category="fraud_detection", + use_kg_features=True ) -# Analyze decision influence +# Comprehensive influence analysis influence = context.analyze_decision_influence(decision_id) ``` --- -## 🚨 The Problem: The Semantic Gap +## AgentContext - Your Agent's Brain + +The main interface that makes your agent intelligent. It handles memory, decisions, and knowledge organization automatically. + +### Quick Start +```python +from semantica.context import AgentContext +from semantica.vector_store import VectorStore + +# Create your intelligent agent +agent = AgentContext(vector_store=VectorStore(backend="inmemory", dimension=384)) + +# Your agent can now remember things +memory_id = agent.store("User asked about Python programming") +print(f"Agent remembered: {memory_id}") + +# And find information when needed +results = agent.retrieve("Python tutorials") +print(f"Agent found {len(results)} relevant memories") +``` + +### Easy Decision Learning +```python +# Your agent learns from its decisions +decision_id = agent.record_decision( + category="content_recommendation", + scenario="User wants Python tutorial", + reasoning="User mentioned being a beginner", + outcome="recommended_basics", + confidence=0.85 +) + +# Your agent can now find similar past decisions +similar_decisions = agent.find_precedents("Python tutorial", limit=3) +print(f"Agent found {len(similar_decisions)} similar past decisions") +``` + +### Getting Smarter Over Time +```python +# Enable all learning features +smart_agent = AgentContext( + vector_store=vector_store, + decision_tracking=True, # Learn from decisions + graph_expansion=True, # Find related information + advanced_analytics=True, # Understand patterns + kg_algorithms=True, # Advanced analysis + vector_store_features=True +) + +# Get insights about your agent's learning +insights = smart_agent.get_context_insights() +print(f"Total decisions learned: {insights.get('total_decisions', 0)}") +print(f"Decision categories: {list(insights.get('categories', {}).keys())}") +``` + +--- + +## The Problem: The Semantic Gap ### Most AI systems fail in high-stakes domains because they operate on **text similarity**, not **meaning**. @@ -246,44 +351,45 @@ The **semantic gap** is the fundamental disconnect between what AI systems can p --- -## 🆚 Semantica vs Traditional RAG +## Semantica vs Traditional RAG | Feature | Traditional RAG | Semantica | |:--------|:----------------|:----------| -| **Reasoning** | ❌ Black-box answers | ✅ Explainable reasoning paths | -| **Provenance** | ❌ No provenance | ✅ W3C PROV-O compliant lineage tracking | -| **Search** | ⚠️ Vector similarity only | ✅ Semantic + graph reasoning | -| **Quality** | ❌ No conflict handling | ✅ Explicit contradiction detection | -| **Safety** | ⚠️ Unsafe for high-stakes | ✅ Designed for governed environments | -| **Compliance** | ❌ No audit trails | ✅ Complete audit trails with integrity verification | +| **Reasoning** | Black-box answers | Explainable reasoning paths | +| **Provenance** | No provenance | W3C PROV-O compliant lineage tracking | +| **Search** | Vector similarity only | Semantic + graph reasoning | +| **Quality** | No conflict handling | Explicit contradiction detection | +| **Safety** | Unsafe for high-stakes | Designed for governed environments | +| **Compliance** | No audit trails | Complete audit trails with integrity verification | --- -## 🧩 Semantica Architecture +## Semantica Architecture -### 1️⃣ Input Layer — Governed Ingestion -- 📄 **Multiple Formats** — PDFs, DOCX, HTML, JSON, CSV, Excel, PPTX -- 🔧 **Docling Support** — Docling parser for table extraction -- 💾 **Data Sources** — Databases, APIs, streams, archives, web content -- 🎨 **Media Support** — Image parsing with OCR, audio/video metadata extraction -- 📊 **Single Pipeline** — Unified ingestion with metadata and source tracking +### Input Layer — Governed Ingestion +- **Multiple Formats** — PDFs, DOCX, HTML, JSON, CSV, Excel, PPTX +- **Docling Support** — Docling parser for table extraction +- **Data Sources** — Databases, APIs, streams, archives, web content +- **Media Support** — Image parsing with OCR, audio/video metadata extraction +- **Single Pipeline** — Unified ingestion with metadata and source tracking -### 2️⃣ Semantic Layer — Trust & Reasoning Engine -- 🔍 **Entity Extraction** — NER, normalization, classification -- 🔗 **Relationship Discovery** — Triplet generation, semantic links -- 📐 **Ontology Induction** — Automated domain rule generation -- 🔄 **Deduplication** — Jaro-Winkler similarity, conflict resolution -- ✅ **Quality Assurance** — Conflict detection, validation -- 📊 **Provenance Tracking** — W3C PROV-O compliant lineage tracking across all modules -- 🧠 **Reasoning Traces** — Explainable inference paths -- 🔐 **Change Management** — Version control with audit trails, checksums, compliance support +### Semantic Layer — Trust & Reasoning Engine +- **Entity Extraction** — NER, normalization, classification +- **Relationship Discovery** — Triplet generation, semantic links +- **Ontology Induction** — Automated domain rule generation +- **Deduplication** — Jaro-Winkler similarity, conflict resolution +- **Quality Assurance** — Conflict detection, validation +- **Provenance Tracking** — W3C PROV-O compliant lineage tracking across all modules +- **Reasoning Traces** — Explainable inference paths +- **Change Management** — Version control with audit trails, checksums, compliance support -### 3️⃣ Output Layer — Auditable Knowledge Assets -- 📊 **Knowledge Graphs** — Queryable, temporal, explainable -- 📐 **OWL Ontologies** — HermiT/Pellet validated, custom ontology import support -- 🔢 **Vector Embeddings** — FastEmbed by default -- ☁️ **AWS Neptune** — Amazon Neptune graph database support -- 🔍 **Provenance** — Every AI response links back to: +### Output Layer — Auditable Knowledge Assets +- **Knowledge Graphs** — Queryable, temporal, explainable +- **OWL Ontologies** — HermiT/Pellet validated, custom ontology import support +- **Vector Embeddings** — FastEmbed by default +- **AWS Neptune** — Amazon Neptune graph database support +- **Apache AGE** — PostgreSQL graph extension with openCypher support +- **Provenance** — Every AI response links back to: - 📄 Source documents - 🏷️ Extracted entities & relations - 📐 Ontology rules applied @@ -291,27 +397,27 @@ The **semantic gap** is the fundamental disconnect between what AI systems can p --- -## 🏥 Built for High-Stakes Domains +## Built for High-Stakes Domains Designed for domains where **mistakes have real consequences** and **every decision must be accountable**: -- **🏥 Healthcare & Life Sciences** — Clinical decision support, drug interaction analysis, medical literature reasoning, patient safety tracking -- **💰 Finance & Risk** — Fraud detection, regulatory support (SOX, GDPR, MiFID II), credit risk assessment, algorithmic trading validation -- **⚖️ Legal & Compliance** — Evidence-backed legal research, contract analysis, regulatory change tracking, case law reasoning -- **🔒 Cybersecurity & Intelligence** — Threat attribution, incident response, security audit trails, intelligence analysis -- **🏛️ Government & Defense** — Governed AI systems, policy decisions, classified information handling, defense intelligence -- **🏭 Critical Infrastructure** — Power grid management, transportation safety, water treatment, emergency response -- **🚗 Autonomous Systems** — Self-driving vehicles, drone navigation, robotics safety, industrial automation +- **Healthcare & Life Sciences** — Clinical decision support, drug interaction analysis, medical literature reasoning, patient safety tracking +- **Finance & Risk** — Fraud detection, regulatory support (SOX, GDPR, MiFID II), credit risk assessment, algorithmic trading validation +- **Legal & Compliance** — Evidence-backed legal research, contract analysis, regulatory change tracking, case law reasoning +- **Cybersecurity & Intelligence** — Threat attribution, incident response, security audit trails, intelligence analysis +- **Government & Defense** — Governed AI systems, policy decisions, classified information handling, defense intelligence +- **Critical Infrastructure** — Power grid management, transportation safety, water treatment, emergency response +- **Autonomous Systems** — Self-driving vehicles, drone navigation, robotics safety, industrial automation --- ## 👥 Who Uses Semantica? -- **🤖 AI / ML Engineers** — Building explainable GraphRAG & agents -- **⚙️ Data Engineers** — Creating governed semantic pipelines -- **📊 Knowledge Engineers** — Managing ontologies & KGs at scale -- **🏢 Enterprise Teams** — Requiring trustworthy AI infrastructure -- **🛡️ Risk & Compliance Teams** — Needing audit-ready systems +- **AI / ML Engineers** — Building explainable GraphRAG & agents +- **Data Engineers** — Creating governed semantic pipelines +- **Knowledge Engineers** — Managing ontologies & KGs at scale +- **Enterprise Teams** — Requiring trustworthy AI infrastructure +- **Risk & Compliance Teams** — Needing audit-ready systems --- @@ -509,13 +615,13 @@ results = vector_store.search(query="supply chain", top_k=5) ### Graph Store & Triplet Store -> **Neo4j, FalkorDB, Amazon Neptune** • **SPARQL queries** • **RDF triplets** +> **Neo4j, FalkorDB, Amazon Neptune, Apache AGE** • **SPARQL queries** • **RDF triplets** ```python from semantica.graph_store import GraphStore from semantica.triplet_store import TripletStore -# Graph Store (Neo4j, FalkorDB) +# Graph Store (Neo4j, FalkorDB, Apache AGE) graph_store = GraphStore(backend="neo4j", uri="bolt://localhost:7687", user="neo4j", password="password") graph_store.add_nodes([{"id": "n1", "labels": ["Person"], "properties": {"name": "Alice"}}]) @@ -537,6 +643,15 @@ neptune_store.add_nodes([ # Query Operations result = neptune_store.execute_query("MATCH (p:Person) RETURN p.name, p.age") +# Apache AGE Graph Store (PostgreSQL + openCypher) +age_store = GraphStore( + backend="age", + connection_string="host=localhost dbname=agedb user=postgres password=secret", + graph_name="semantica", +) +age_store.connect() +age_store.create_node(labels=["Person"], properties={"name": "Alice", "age": 30}) + # Triplet Store (Blazegraph, Jena, RDF4J) triplet_store = TripletStore(backend="blazegraph", endpoint="http://localhost:9999/blazegraph") triplet_store.add_triplet({"subject": "Alice", "predicate": "knows", "object": "Bob"}) @@ -592,12 +707,12 @@ is_valid = kg_manager.verify_checksum(snapshot) ``` **What We Provide:** -- 🔐 **Persistent Storage** — SQLite and in-memory backends implemented -- 📊 **Detailed Diffs** — Entity-level and relationship-level change tracking -- ✅ **Data Integrity** — SHA-256 checksums with tamper detection -- 📝 **Standardized Metadata** — ChangeLogEntry with author, timestamp, description -- ⚡ **Performance Tested** — Tested with large-scale entity datasets -- 🧪 **Test Coverage** — Comprehensive test coverage covering core functionality +- **Persistent Storage** — SQLite and in-memory backends implemented +- **Detailed Diffs** — Entity-level and relationship-level change tracking +- **Data Integrity** — SHA-256 checksums with tamper detection +- **Standardized Metadata** — ChangeLogEntry with author, timestamp, description +- **Performance Tested** — Tested with large-scale entity datasets +- **Test Coverage** — Comprehensive test coverage covering core functionality **Compliance Note:** Provides technical infrastructure (audit trails, checksums, temporal tracking) that supports compliance efforts for HIPAA, SOX, FDA 21 CFR Part 11. Organizations must implement additional policies and procedures for full regulatory compliance. @@ -713,11 +828,11 @@ retriever = context.retriever # Access underlying ContextRetriever results = retriever.retrieve( query="What is the user building?", max_results=10, - use_graph_expansion=True + graph_expansion=True ) # Retrieve with context expansion -results = context.retrieve("What is the user building?", use_graph_expansion=True) +results = context.retrieve("What is the user building?", graph_expansion=True) # Query with reasoning and LLM-generated responses llm_provider = Groq(model="llama-3.1-8b-instant", api_key=os.getenv("GROQ_API_KEY")) @@ -733,6 +848,106 @@ reasoned_result = context.query_with_reasoning( - **ContextRetriever**: Performs hybrid retrieval combining vector search, graph traversal, and memory for optimal context relevance - **AgentContext**: High-level interface integrating Context Graph and Context Retriever for GraphRAG applications +#### Context Graphs: Advanced Decision Tracking & Analytics + +```python +from semantica.context import AgentContext, ContextGraph +from semantica.vector_store import VectorStore + +# Initialize with advanced decision tracking +context = AgentContext( + vector_store=VectorStore(backend="inmemory", dimension=128), + knowledge_graph=ContextGraph(advanced_analytics=True), + decision_tracking=True, + kg_algorithms=True, # Enable advanced graph analytics +) + +# Easy decision recording with convenience methods +decision_id = context.graph_builder.add_decision( + category="credit_approval", + scenario="High-risk credit limit increase", + reasoning="Recent velocity-check failure and prior fraud flag", + outcome="rejected", + confidence=0.78, + entities=["customer:jessica_norris"], +) + +# Find similar decisions with advanced analytics +similar_decisions = context.graph_builder.find_similar_decisions( + scenario="credit increase", + category="credit_approval", + max_results=5, +) + +# Analyze decision impact and influence +impact_analysis = context.graph_builder.analyze_decision_impact(decision_id) +node_importance = context.graph_builder.get_node_importance("customer:jessica_norris") + +# Check compliance with business rules +compliance = context.graph_builder.check_decision_rules({ + "category": "credit_approval", + "scenario": "Credit limit increase", + "reasoning": "Risk assessment completed", + "outcome": "rejected", + "confidence": 0.78 +}) +``` + +**Enhanced Features:** +- **Easy-to-Use Methods**: 10 convenience methods for common operations +- **Decision Analytics**: Influence analysis, centrality measures, community detection +- **Policy Engine**: Automated compliance checking with business rules +- **Causal Analysis**: Trace decision causality and impact chains +- **Graph Analytics**: Advanced KG algorithms (Node2Vec, centrality, community detection) +- **Hybrid Search**: Semantic + structural + category similarity +- **Production Ready**: Scalable architecture with comprehensive error handling + +## Configuration Options + +### Simple Setup (Most Common) +```python +# Just memory and basic learning +agent = AgentContext(vector_store=vector_store) +``` + +### Smart Setup (Recommended) +```python +# Memory + decision learning +agent = AgentContext( + vector_store=vector_store, + decision_tracking=True, + graph_expansion=True +) +``` + +### Complete Setup (Maximum Power) +```python +# Everything enabled +agent = AgentContext( + vector_store=vector_store, + knowledge_graph=ContextGraph(advanced_analytics=True), + decision_tracking=True, + graph_expansion=True, + advanced_analytics=True, + kg_algorithms=True, + vector_store_features=True +) +``` + +### ContextGraph Options +```python +# Basic knowledge graph +graph = ContextGraph() + +# Advanced knowledge graph +graph = ContextGraph( + advanced_analytics=True, # Enable smart algorithms + centrality_analysis=True, # Find important concepts + community_detection=True, # Find groups of related concepts + node_embeddings=True # Understand concept similarity +) +``` + **Core Notebooks:** - [**Context Module Introduction**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/19_Context_Module.ipynb) - Basic memory and storage. - [**Advanced Context Engineering**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/advanced/11_Advanced_Context_Engineering.ipynb) - Hybrid retrieval, graph builders, and custom memory policies. diff --git a/docs/examples.md b/docs/examples.md index 5e52862a..48757e7d 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -332,7 +332,7 @@ from semantica.reasoning import Reasoner context = AgentContext( vector_store=vs, knowledge_graph=kg, - use_graph_expansion=True, + graph_expansion=True, hybrid_alpha=0.7 ) diff --git a/docs/graph_stores/apache_age.md b/docs/graph_stores/apache_age.md new file mode 100644 index 00000000..27c16a9d --- /dev/null +++ b/docs/graph_stores/apache_age.md @@ -0,0 +1,243 @@ +# Apache AGE Graph Store + +**Backend**: PostgreSQL + [Apache AGE](https://age.apache.org/) +**Driver**: `psycopg2` + +Apache AGE is a PostgreSQL extension that adds graph database functionality, enabling you to run openCypher queries alongside traditional SQL. This backend lets Semantica use AGE as a property graph store with the same interface as Neo4j and FalkorDB. + +--- + +## Prerequisites + +| Component | Version | +|-----------|---------| +| PostgreSQL | 12+ | +| Apache AGE | 1.4+ (compiled and installed) | +| psycopg2 | 2.9+ | + +```bash +pip install psycopg2-binary +``` + +> **Note**: Apache AGE must be compiled and installed into your PostgreSQL instance. See the [AGE installation guide](https://age.apache.org/age-manual/master/intro/setup.html). + +--- + +## Quick Start + +```python +from semantica.graph_store import GraphStore + +# Using the unified GraphStore facade +store = GraphStore( + backend="age", + connection_string="host=localhost dbname=agedb user=postgres password=secret", + graph_name="semantica", +) +store.connect() + +# Create nodes +alice = store.create_node(labels=["Person"], properties={"name": "Alice", "age": 30}) +bob = store.create_node(labels=["Person"], properties={"name": "Bob", "age": 25}) + +# Create relationship +rel = store.create_relationship(alice["id"], bob["id"], "KNOWS", {"since": 2023}) + +# Query +result = store.execute_query("MATCH (p:Person) RETURN p", cols="p agtype") +print(result["records"]) + +store.close() +``` + +### Direct Usage (without facade) + +```python +from semantica.graph_store.age_store import ApacheAgeStore + +store = ApacheAgeStore( + connection_string="host=localhost dbname=agedb user=postgres password=secret", + graph_name="my_graph", +) +store.connect() + +node = store.create_node(["Entity"], {"semantica_id": "ent-001", "value": "test"}) +print(node) +# {"id": 844424930131969, "labels": ["Entity"], "properties": {"semantica_id": "ent-001", "value": "test"}} + +store.close() +``` + +--- + +## Configuration + +### Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `GRAPH_STORE_AGE_CONNECTION_STRING` | PostgreSQL connection string | `host=localhost dbname=agedb user=postgres password=postgres` | +| `GRAPH_STORE_AGE_GRAPH_NAME` | AGE graph name | `semantica` | + +### Programmatic Configuration + +```python +from semantica.graph_store.config import graph_store_config + +graph_store_config.set("age_connection_string", "host=db.example.com dbname=prod_age user=app") +graph_store_config.set("age_graph_name", "production") +``` + +--- + +## Connection & Initialization + +On `connect()`, the store performs idempotent setup: + +1. `CREATE EXTENSION IF NOT EXISTS age;` +2. `LOAD 'age';` +3. `SET search_path = ag_catalog, "$user", public;` +4. Creates the named graph if it does not already exist. + +This is safe to call repeatedly. + +--- + +## ID Handling + +Apache AGE auto-generates internal vertex/edge IDs (large integers). These are **not** the same as any semantic or application-level ID you may want to assign. + +| Concept | Description | +|---------|-------------| +| **AGE internal ID** | Auto-generated by AGE. Exposed as `"id"` in all returned dicts. Used in `delete_node()`, `get_node()`, etc. | +| **Semantic ID** | Application-level identifier. Store it in the `semantica_id` property. | + +```python +node = store.create_node( + labels=["Document"], + properties={"semantica_id": "doc-abc-123", "title": "My Doc"}, +) +# node["id"] → AGE internal ID (e.g., 844424930131969) +# node["properties"]["semantica_id"] → "doc-abc-123" +``` + +> **Important**: Never mix AGE internal IDs with semantic IDs. Use `node["id"]` for graph operations (delete, update, traverse) and `node["properties"]["semantica_id"]` for application-level lookups. + +--- + +## Label Handling + +AGE supports exactly **one label per vertex**. Semantica handles this transparently: + +- `labels[0]` → used as the primary AGE vertex label. +- `labels[1:]` → stored in a `labels` property array on the vertex. + +When reading nodes, the store reconstructs the full label list automatically. + +```python +node = store.create_node( + labels=["Person", "Employee", "Admin"], + properties={"name": "Alice"}, +) +# In AGE: vertex with label "Person" and property labels=["Employee", "Admin"] +# Returned: {"id": ..., "labels": ["Person", "Employee", "Admin"], "properties": {"name": "Alice"}} +``` + +--- + +## Cypher Query Execution + +All Cypher queries are executed via AGE's SQL wrapper: + +```sql +SELECT * FROM cypher('graph_name', $$ $$) AS (col1 agtype, ...); +``` + +### Parameter Substitution + +AGE does not support `$param` style binding inside `cypher()` calls. The store safely converts parameters to Cypher literals with proper escaping: + +```python +result = store.execute_query( + "MATCH (p:Person) WHERE p.age > $min_age RETURN p", + parameters={"min_age": 25}, + cols="p agtype", +) +``` + +### Column Specification + +For custom queries, pass the `cols` option to specify the `AS` clause: + +```python +result = store.execute_query( + "MATCH (a)-[r]->(b) RETURN a, r, b", + cols="a agtype, r agtype, b agtype", +) +``` + +If omitted, the store attempts to infer columns from the `RETURN` clause. + +--- + +## Transactions + +The store uses explicit PostgreSQL transactions: + +- **Success** → `COMMIT` +- **Exception** → `ROLLBACK`, then re-raise as `ProcessingError` +- No silent failures + +--- + +## API Reference + +All methods match the standard Semantica graph store backend interface: + +| Method | Description | +|--------|-------------| +| `connect(**options)` | Connect and initialize AGE | +| `close()` | Close the connection | +| `create_node(labels, properties)` | Create a vertex | +| `create_nodes(nodes)` | Batch create vertices | +| `get_node(node_id)` | Get vertex by AGE ID | +| `get_nodes(labels, properties, limit)` | Query vertices | +| `update_node(node_id, properties, merge)` | Update vertex properties | +| `delete_node(node_id, detach)` | Delete a vertex | +| `create_relationship(start_id, end_id, type, properties)` | Create an edge | +| `get_relationships(node_id, rel_type, direction, limit)` | Query edges | +| `delete_relationship(rel_id)` | Delete an edge | +| `execute_query(query, parameters)` | Run arbitrary Cypher | +| `get_neighbors(node_id, rel_type, direction, depth)` | Graph traversal | +| `shortest_path(start_id, end_id, rel_type, max_depth)` | Path finding | +| `create_index(label, property_name, index_type)` | Create a PostgreSQL index | +| `get_stats()` | Graph statistics | + +--- + +## Docker Setup + +```yaml +services: + age: + image: apache/age:latest + ports: + - "5432:5432" + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: secret + POSTGRES_DB: agedb +``` + +```bash +docker compose up -d +``` + +Then connect: + +```python +store = GraphStore( + backend="age", + connection_string="host=localhost port=5432 dbname=agedb user=postgres password=secret", +) +``` diff --git a/docs/reference/context.md b/docs/reference/context.md index b5833f68..a66cc0c7 100644 --- a/docs/reference/context.md +++ b/docs/reference/context.md @@ -1,767 +1,499 @@ # Context Module Reference -> **The central nervous system for intelligent agents, managing memory, knowledge graphs, context graphs, decision tracking, and advanced context retrieval with KG algorithms and vector store integration.** +> **The intelligent brain for AI agents, providing memory, decision tracking, and knowledge organization with easy-to-use interfaces that make building smart agents simple and effective.** --- -## 🎯 System Overview +## 🎯 Overview -The **Context Module** provides agents with a persistent, searchable, and structured memory system with advanced decision tracking capabilities and **context graphs** for sophisticated knowledge representation, ensuring predictable state management and compatibility with modern vector stores and graph databases. +The **Context Module** gives your AI agents the ability to **remember**, **learn**, and **make smarter decisions** through intelligent memory management and knowledge organization. It's designed to be both powerful for production use and simple enough for rapid development. ### Key Capabilities
-- :material-brain:{ .lg .middle } **Hierarchical Memory** +- :material-brain:{ .lg .middle } **Smart Memory** --- - Mimics human memory with a fast, token-limited Short-Term buffer and infinite Long-Term vector storage. + Human-like memory that stores conversations, learns from experience, and retrieves relevant information when needed. -- :material-graph-outline:{ .lg .middle } **GraphRAG** +- :material-graph-outline:{ .lg .middle } **Decision Intelligence** --- - Combines unstructured vector search with structured knowledge graph traversal for deep contextual understanding. + Track decisions, learn from past choices, and make consistent, improving decisions over time. -- :material-scale-balance:{ .lg .middle } **Hybrid Retrieval** +- :material-lightbulb:{ .lg .middle } **Easy-to-Use API** --- - Intelligently blends Keyword (BM25), Vector (Dense), and Graph (Relational) scores for optimal relevance. + Simple methods that make complex features accessible without overwhelming complexity. -- :material-lightning-bolt:{ .lg .middle } **Token Management** +- :material-search:{ .lg .middle } **Smart Retrieval** --- - Automatic FIFO and importance-based pruning to keep context within LLM window limits. + Find relevant information quickly using hybrid search that understands context and relationships. -- :material-link-variant:{ .lg .middle } **Entity Linking** +- :material-account-tree:{ .lg .middle } **Knowledge Organization** --- - Resolves ambiguities by linking text mentions to unique entities in the knowledge graph. + Build intelligent knowledge graphs that understand relationships and context. -- :material-gavel:{ .lg .middle } **Decision Tracking** +- :material-trending-up:{ .lg .middle } **Learning & Analytics** --- - Complete decision lifecycle management with precedent search, causal analysis, and policy compliance. + Get insights about agent performance, decision patterns, and knowledge growth. -- :material-chart-line:{ .lg .middle } **KG Algorithms** +- :material-security:{ .lg .middle } **Production Ready** --- - Advanced graph analytics including centrality, community detection, embeddings, and link prediction. - -- :material-magnify:{ .lg .middle } **Vector Store Features** - - --- - - Hybrid search with custom similarity weights and advanced filtering capabilities. - -- :material-graph:{ .lg .middle } **Context Graphs** - - --- - - Structured knowledge representation with entity relationships, decision history, and semantic context for sophisticated reasoning. + Scalable, reliable, and tested for real-world applications.
-!!! tip "When to Use" - - **Memory Persistence**: Enabling agents to remember user preferences and history. - - **Complex Retrieval**: When simple vector search fails to capture relationships. - - **Knowledge Graph**: Building a structured world model from unstructured text. - - **Decision Management**: Tracking, analyzing, and learning from decisions. - - **Advanced Analytics**: Understanding influence, patterns, and relationships in decisions. +!!! tip "Perfect For" + - **AI Agents** that need to remember conversations and learn from decisions + - **Chatbots** that become smarter with every interaction + - **Decision Systems** that need to track choices and learn from patterns + - **Knowledge Management** that organizes information intelligently + - **Production Applications** that require reliable, scalable solutions --- -## 🏗️ Architecture Components +## 🤖 AgentContext - Your Agent's Brain -### AgentContext (The Orchestrator) -The high-level facade that unifies all context operations. It routes data to the appropriate subsystems (Memory, Graph, Vector Store, Decision Tracking) and manages the lifecycle of context. +The main interface that makes your agent intelligent. It handles memory, decisions, and knowledge organization automatically. -#### **Constructor Parameters** - -- `vector_store` (Required): The backing vector database instance (e.g., FAISS, Weaviate) -- `knowledge_graph` (Optional): The graph store instance for structured knowledge -- `token_limit` (Default: `2000`): The maximum number of tokens allowed in short-term memory before pruning occurs -- `short_term_limit` (Default: `10`): The maximum number of distinct memory items in short-term memory -- `hybrid_alpha` (Default: `0.5`): The weighting factor for retrieval (`0.0` = Pure Vector, `1.0` = Pure Graph) -- `use_graph_expansion` (Default: `True`): Whether to fetch neighbors of retrieved nodes from the graph -- `enable_decision_tracking` (Default: `False`): Enable advanced decision tracking features -- `enable_advanced_analytics` (Default: `False`): Enable KG algorithms and analytics -- `enable_kg_algorithms` (Default: `False`): Enable knowledge graph algorithm integration -- `enable_vector_store_features` (Default: `False`): Enable advanced vector store features - -#### **Core Methods** - -| Method | Description | -|--------|-------------| -| `store(content, ...)` | Writes information to memory. Handles auto-detection, write-through to vector store, and entity extraction. | -| `retrieve(query, ...)` | Fetches relevant context using hybrid search (Vector + Graph) and reranking. | -| `query_with_reasoning(query, llm_provider, ...)` | **GraphRAG with multi-hop reasoning**: Retrieves context, builds reasoning paths, and generates LLM-based natural language responses grounded in the knowledge graph. | -| `record_decision(category, scenario, reasoning, outcome, confidence, ...)` | Records decisions with full context and metadata for tracking and analysis. | -| `find_precedents(scenario, category, ...)` | Finds similar decisions using advanced search capabilities. | -| `find_precedents_advanced(scenario, similarity_weights, ...)` | Enhanced precedent search with KG features and custom similarity weights. | -| `analyze_decision_influence(decision_id)` | Analyzes decision influence using KG algorithms and centrality measures. | -| `predict_decision_relationships(decision_id)` | Predicts relationships between decisions using link prediction algorithms. | -| `get_context_insights()` | Returns comprehensive system analytics and feature status. | -| `get_causal_chain(decision_id, direction, max_depth)` | Traces decision causality and influence chains. | - -#### **Code Example** +### Quick Start ```python from semantica.context import AgentContext from semantica.vector_store import VectorStore -# 1. Initialize with Advanced Features -vs = VectorStore(backend="faiss", dimension=768) -context = AgentContext( - vector_store=vs, - knowledge_graph=kg, - enable_decision_tracking=True, - enable_advanced_analytics=True, - enable_kg_algorithms=True, - enable_vector_store_features=True +# Create your intelligent agent +agent = AgentContext(vector_store=VectorStore(backend="inmemory", dimension=384)) + +# Your agent can now remember things +memory_id = agent.store("User asked about Python programming") +print(f"Agent remembered: {memory_id}") + +# And find information when needed +results = agent.retrieve("Python tutorials") +print(f"Agent found {len(results)} relevant memories") +``` + +### Easy Decision Learning +```python +# Your agent learns from its decisions +decision_id = agent.record_decision( + category="content_recommendation", + scenario="User wants Python tutorial", + reasoning="User mentioned being a beginner", + outcome="recommended_basics", + confidence=0.85 ) -# 2. Store Memory -context.store( - "User is working on a React project.", - conversation_id="session_1", - user_id="user_123" +# Your agent can now find similar past decisions +similar_decisions = agent.find_precedents("Python tutorial", limit=3) +print(f"Agent found {len(similar_decisions)} similar past decisions") +``` + +### Getting Smarter Over Time +```python +# Enable all learning features +smart_agent = AgentContext( + vector_store=vector_store, + decision_tracking=True, # Learn from decisions + graph_expansion=True, # Find related information + advanced_analytics=True, # Understand patterns + kg_algorithms=True, # Advanced analysis + vector_store_features=True ) -# 3. Record Decision -decision_id = context.record_decision( - category="approval", - scenario="Loan application for first-time homebuyer", - reasoning="Strong credit score (750), stable employment, 20% down payment", - outcome="approved", - confidence=0.94, - decision_maker="loan_officer_001" +# Get insights about your agent's learning +insights = smart_agent.get_context_insights() +print(f"Total decisions learned: {insights.get('total_decisions', 0)}") +print(f"Decision categories: {list(insights.get('categories', {}).keys())}") +``` + +### Core Methods + +| Method | What It Does | When to Use | +|--------|-------------|------------| +| `store(content, ...)` | Remember information | Store conversations, facts, user preferences | +| `retrieve(query, ...)` | Find relevant memories | Search for information when needed | +| `record_decision(category, scenario, reasoning, outcome, confidence, ...)` | Learn from decisions | Track choices and improve over time | +| `find_precedents(scenario, category, ...)` | Find similar decisions | Make consistent choices based on experience | +| `get_context_insights()` | Understand performance | Get analytics about your agent | + +### Advanced Features +```python +# Enable all features for maximum intelligence +agent = AgentContext( + vector_store=vector_store, + knowledge_graph=ContextGraph(advanced_analytics=True), + decision_tracking=True, + graph_expansion=True, + advanced_analytics=True, + kg_algorithms=True, + vector_store_features=True ) -# 4. Retrieve Context -results = context.retrieve("What is the user building?") - -# 5. Find Similar Decisions -precedents = context.find_precedents_advanced( - scenario="High-value credit application", - category="approval", - use_kg_features=True, - similarity_weights={"semantic": 0.5, "structural": 0.3, "category": 0.2} -) - -# 6. Analyze Decision Influence -influence = context.analyze_decision_influence(decision_id) - -# 7. Get Context Insights -insights = context.get_context_insights() - -# 8. Query with Reasoning (GraphRAG) +# Query with multi-hop reasoning (GraphRAG) from semantica.llms import Groq import os -llm_provider = Groq( - model="llama-3.1-8b-instant", - api_key=os.getenv("GROQ_API_KEY") -) - -result = context.query_with_reasoning( - query="What IPs are associated with security alerts?", - llm_provider=llm_provider, - max_results=10, - max_hops=2 -) - -print(f"Response: {result['response']}") -print(f"Reasoning Path: {result['reasoning_path']}") -print(f"Confidence: {result['confidence']:.3f}") -``` - ---- - -### Decision Tracking System - -#### DecisionRecorder (The Decision Engine) -Records decisions with full context, policy applications, and provenance tracking. - -**Key Methods:** -| Method | Description | -|--------|-------------| -| `record_decision(category, scenario, reasoning, outcome, confidence, ...)` | Records decisions with full context and metadata | -| `apply_policy(decision_id, policy_id)` | Applies policies to decisions and checks compliance | -| `create_approval_chain(decision_id, approvers)` | Creates multi-level approval workflows | -| `track_provenance(decision_id, source_info)` | Tracks decision provenance and lineage | - -#### DecisionQuery (The Decision Search Engine) -Advanced decision querying with precedent search, filtering, and hybrid search operations. - -**Key Methods:** -| Method | Description | -|--------|-------------| -| `find_precedents_hybrid(scenario, category, limit)` | Hybrid search with KG and vector store integration | -| `find_precedents_advanced(scenario, similarity_weights, ...)` | Enhanced search with custom similarity weights | -| `analyze_decision_influence(decision_id)` | Analyze decision influence using KG algorithms | -| `predict_decision_relationships(decision_id)` | Predict relationships between decisions | -| `multi_hop_reasoning(decision_id, max_hops)` | Multi-hop reasoning for complex relationships | -| `get_decision_statistics()` | Get comprehensive decision analytics | - -#### CausalChainAnalyzer (The Influence Engine) -Analyzes decision causality, influence chains, and precedent relationships. - -**Key Methods:** -| Method | Description | -|--------|-------------| -| `get_causal_chain(decision_id, direction, max_depth)` | Trace causal chains from decisions | -| `find_influenced_decisions(decision_id)` | Find decisions influenced by a decision | -| `find_influencing_decisions(decision_id)` | Find decisions that influenced a decision | -| `analyze_causal_impact(decision_id, max_depth)` | Analyze causal impact and scope | -| `calculate_influence_score(decision_id)` | Calculate decision influence scores | - -#### PolicyEngine (The Governance Engine) -Policy management with versioning, compliance checking, and impact analysis. - -**Key Methods:** -| Method | Description | -|--------|-------------| -| `create_policy(name, rules, category)` | Create new policies with rules and constraints | -| `check_compliance(decision_id, policy_id)` | Check decision compliance with policies | -| `analyze_impact(policy_id, time_range)` | Analyze policy impact on decisions | -| `get_violations(decision_id)` | Get policy violations for decisions | - -#### **Decision Tracking Example** -```python -from semantica.context import DecisionRecorder, DecisionQuery, CausalChainAnalyzer, PolicyEngine - -# Initialize decision tracking components -recorder = DecisionRecorder(graph_store=kg, vector_store=vs) -query = DecisionQuery(graph_store=kg, vector_store=vs) -analyzer = CausalChainAnalyzer(graph_store=kg) -policy_engine = PolicyEngine(graph_store=kg) - -# Record a decision -decision_id = recorder.record_decision( - category="loan_approval", - scenario="Mortgage application for first-time homebuyer", - reasoning="Strong credit score (750), stable employment, 20% down payment", - outcome="approved", - confidence=0.94, - decision_maker="loan_officer_001" -) - -# Find similar decisions (precedents) -precedents = query.find_precedents_hybrid( - scenario="Mortgage application", - category="loan_approval", - limit=10 -) - -# Analyze decision influence -influence = analyzer.analyze_decision_influence(decision_id) - -# Check policy compliance -compliance = policy_engine.check_compliance(decision_id, "lending_policy_001") - -# Trace causal chain -causal_chain = analyzer.get_causal_chain(decision_id, "downstream", max_depth=3) -``` - ---- - -### Knowledge Graph Algorithm Integration - -#### Supported KG Algorithms -- **Centrality Analysis**: Degree, betweenness, closeness, eigenvector centrality -- **Community Detection**: Modularity-based community identification -- **Node Embeddings**: Node2Vec embeddings for similarity analysis -- **Path Finding**: Shortest path and advanced path algorithms -- **Link Prediction**: Relationship prediction between entities -- **Similarity Calculation**: Multi-type similarity measures - -#### Enhanced ContextGraph Features -```python -from semantica.context import ContextGraph - -# Initialize with KG Algorithms -graph = ContextGraph( - enable_advanced_analytics=True, - enable_centrality_analysis=True, - enable_community_detection=True, - enable_node_embeddings=True -) - -# Add nodes and edges -graph.add_node("Python", type="language", properties={"popularity": "high"}) -graph.add_edge("Python", "Programming", type="related_to") - -# Advanced analytics -centrality = graph.get_node_centrality("Python") -similar = graph.find_similar_nodes("Python", similarity_type="content") -analysis = graph.analyze_graph_with_kg() - -# Decision integration -graph.add_decision(decision_id, decision_data) -precedents = graph.find_precedents("loan_approval") -``` - ---- - -### Vector Store Integration - -#### Hybrid Search Features -- **Semantic + Structural Similarity**: Combined similarity scoring -- **Custom Similarity Weights**: Configurable similarity scoring -- **Advanced Precedent Search**: KG-enhanced similarity search -- **Multi-Embedding Support**: Multiple embedding types -- **Metadata Filtering**: Advanced filtering capabilities - -#### Code Example -```python -# Hybrid search with custom weights -precedents = query.find_precedents_hybrid( - scenario="Loan application", - category="approval", - limit=10, - similarity_weights={ - "semantic": 0.6, - "structural": 0.3, - "category": 0.1 - } -) -``` - ---- - -### AgentMemory (The Storage Engine) -Manages the storage and lifecycle of memory items. It implements the **Hierarchical Memory** pattern. - -#### **Features & Functions** -* **Short-Term Memory (Working Memory)** - * *Structure*: An in-memory list of recent `MemoryItem` objects. - * *Purpose*: Provides immediate context for the ongoing conversation. - * *Pruning Logic*: - * **FIFO**: Removes the oldest items first when limits are reached. - * **Token-Aware**: Calculates token counts to ensure the total buffer size stays under `token_limit`. -* **Long-Term Memory (Episodic Memory)** - * *Structure*: Vector embeddings stored in the `vector_store`. - * *Purpose*: Persists history indefinitely for semantic retrieval. - * *Synchronization*: Automatically syncs with Short-term memory during `store()` operations. -* **Retention Policy** - * *Time-Based*: Can automatically delete memories older than `retention_days`. - * *Count-Based*: Can limit the total number of memories to `max_memories`. - -#### **Key Methods** - -| Method | Description | -|--------|-------------| -| `store_vectors()` | Handles the low-level interaction with concrete Vector Store implementations. | -| `_prune_short_term_memory()` | Internal algorithm that enforces token and count limits. | -| `get_conversation_history()` | Retrieves a chronological list of interactions for a specific session. | - -#### **Code Example** -```python -# Accessing via AgentContext -memory = context.memory - -# Get conversation history -history = memory.get_conversation_history("session_1") -for item in history: - print(f"[{item.timestamp}] {item.content}") - -# Get statistics -stats = memory.get_statistics() -print(f"Stored Memories: {stats['total_memories']}") -``` - ---- - -### ContextGraph (The Knowledge Structure) -Manages the structured relationships between entities. It provides the "World Model" for the agent with advanced KG algorithm integration and serves as the foundation for **Context Graphs** that enable sophisticated reasoning and decision analysis. - -#### **What are Context Graphs?** -**Context Graphs** are structured representations of knowledge that capture: -- **Entity Relationships**: How concepts, people, and decisions are connected -- **Semantic Context**: The meaning and relevance of information within specific domains -- **Decision History**: How past decisions influence current and future choices -- **Knowledge Evolution**: How understanding grows and changes over time - -#### **Key Features of Context Graphs** -* **Dictionary-Based Interface** - * *Design*: Uses standard Python dictionaries for nodes and edges, removing dependencies on complex interface classes. - * *Benefit*: simpler serialization and easier integration with external APIs. -* **Advanced Graph Traversal** - * *Adjacency List*: optimized internal structure for fast neighbor lookups. - * *Multi-Hop Search*: Can traverse `k` hops from a starting node to find indirect connections. - * *Path Finding*: Shortest path and advanced path algorithms for relationship discovery. -* **Rich Node & Edge Types** - * *Typed Schema*: Supports distinct types for nodes (e.g., "Person", "Concept", "Decision") and edges (e.g., "KNOWS", "RELATED_TO", "INFLUENCES"). - * *Metadata Support*: Rich properties and attributes for detailed context capture. -* **Advanced Analytics Integration** - * *KG Algorithm Integration*: Centrality, community detection, embeddings, path finding - * *Decision Integration*: Store and analyze decisions in graph context - * *Similarity Analysis*: Advanced node similarity with multiple measures - * *Influence Analysis**: Track how decisions and entities influence each other - -#### **Context Graph Use Cases** -- **Knowledge Management**: Build and query structured knowledge bases -- **Decision Support**: Trace decision precedents and influence patterns -- **Recommendation Systems**: Find related concepts and entities -- **Social Network Analysis**: Understand relationships and influence -- **Research Networks**: Map collaborations and citation patterns - -#### **Key Methods** - -| Method | Description | -|--------|-------------| -| `add_nodes(nodes)` | Bulk adds nodes using a list of dictionaries. | -| `add_edges(edges)` | Bulk adds edges using a list of dictionaries. | -| `get_neighbors(node_id, hops)` | Returns connected nodes within a specified distance. | -| `query(query_str)` | Performs keyword-based search specifically on graph nodes. | -| `analyze_graph_with_kg()` | Comprehensive graph analysis with KG algorithms. | -| `get_node_centrality(node_id)` | Get centrality measures for specific nodes. | -| `find_similar_nodes(node_id, similarity_type)` | Find similar nodes using advanced similarity. | -| `add_decision(decision_id, decision_data)` | Add decisions with full context integration. | -| `find_precedents(scenario, category)` | Find decision precedents using graph traversal. | -| `trace_influence_paths(entity_id, max_depth)` | Trace how influence propagates through the graph. | -| `get_graph_metrics()` | Get comprehensive graph statistics and health metrics. | - -#### **Code Example** -```python -from semantica.context import ContextGraph - -# Initialize Context Graph with advanced features -graph = ContextGraph( - enable_advanced_analytics=True, - enable_centrality_analysis=True, - enable_community_detection=True, - enable_node_embeddings=True -) - -# Build Context Graph - Add entities and relationships -graph.add_nodes([ - { - "id": "Python", - "type": "Language", - "properties": { - "paradigm": "OO", - "popularity": "high", - "domain": "programming" - } - }, - { - "id": "FastAPI", - "type": "Framework", - "properties": { - "language": "Python", - "use_case": "web_api", - "performance": "high" - } - }, - { - "id": "DataScience", - "type": "Domain", - "properties": { - "description": "Data analysis and machine learning", - "tools": ["Python", "R", "SQL"] - } - } -]) - -# Create relationships in Context Graph -graph.add_edges([ - { - "source_id": "FastAPI", - "target_id": "Python", - "type": "WRITTEN_IN", - "properties": {"strength": 0.9} - }, - { - "source_id": "Python", - "target_id": "DataScience", - "type": "USED_IN", - "properties": {"popularity": 0.95} - }, - { - "source_id": "FastAPI", - "target_id": "DataScience", - "type": "SUPPORTS", - "properties": {"use_case": "api_for_ml"} - } -]) - -# Advanced Context Graph Analytics -centrality = graph.get_node_centrality("Python") -similar = graph.find_similar_nodes("Python", similarity_type="content") -analysis = graph.analyze_graph_with_kg() - -# Decision Integration in Context Graph -graph.add_decision("decision_001", { - "category": "technology_choice", - "scenario": "Framework selection for web API", - "reasoning": "Python ecosystem with FastAPI provides best performance", - "outcome": "selected_fastapi", - "confidence": 0.92 -}) - -# Find decision precedents in Context Graph -precedents = graph.find_precedents("technology_choice") - -# Trace influence through Context Graph -influence_paths = graph.trace_influence_paths("Python", max_depth=3) -``` - ---- - -### Production Graph Store Integration - -For production environments, you can replace the in-memory `ContextGraph` with a persistent `GraphStore` (Neo4j, FalkorDB) by passing it to the `knowledge_graph` parameter. - -```python -from semantica.context import AgentContext -from semantica.graph_store import GraphStore - -# 1. Initialize Persistent Graph Store (Neo4j) -gs = GraphStore( - backend="neo4j", - uri="bolt://localhost:7687", - user="neo4j", - password="password" -) - -# 2. Initialize Agent Context with Persistent Graph and Advanced Features -context = AgentContext( - vector_store=vs, # Your VectorStore instance - knowledge_graph=gs, # Your persistent GraphStore - enable_decision_tracking=True, - enable_advanced_analytics=True, - enable_kg_algorithms=True, - enable_vector_store_features=True, - use_graph_expansion=True -) - -# Now all graph operations (store, retrieve, build_graph) use Neo4j directly. -``` - ---- - -### ContextRetriever (The Search Engine) -The retrieval logic that powers the `retrieve()` command. It implements the **Hybrid Retrieval** algorithm with advanced KG and vector store integration. - -#### **Retrieval Strategy** -1. **Short-Term Check**: Scans the in-memory buffer for immediate, exact-match relevance. -2. **Vector Search**: Queries the `vector_store` for semantically similar long-term memories. -3. **Graph Expansion**: - * Identifies entities in the query. - * Finds those entities in the `ContextGraph`. - * Traverses edges to find related concepts that might not match keywords (e.g., finding "Python" when searching for "Coding"). -4. **Hybrid Scoring**: - * Formula: `Final_Score = (Vector_Score * (1 - α)) + (Graph_Score * α)` - * Allows tuning the balance between semantic similarity and structural relevance. -5. **KG Algorithm Enhancement**: Uses centrality, community detection, and similarity for advanced ranking. - -#### **Code Example** -```python -# The retriever is automatically used by AgentContext.retrieve() -# But can be accessed directly if needed: - -retriever = context.retriever - -# Perform a manual retrieval with advanced features -results = retriever.retrieve( - query="web frameworks", - max_results=5, - use_kg_features=True, - similarity_weights={"semantic": 0.7, "structural": 0.3} -) -``` - ---- - -### GraphRAG with Multi-Hop Reasoning - -The `query_with_reasoning()` method extends traditional retrieval by performing multi-hop graph traversal and generating natural language responses using LLMs. This enables deeper understanding of relationships and context-aware answer generation. - -#### **How It Works** - -1. **Context Retrieval**: Retrieves relevant context using hybrid search (vector + graph) -2. **Entity Extraction**: Extracts entities from query and retrieved context -3. **Multi-Hop Reasoning**: Traverses knowledge graph up to N hops to find related entities -4. **Reasoning Path Construction**: Builds reasoning chains showing entity relationships -5. **LLM Response Generation**: Generates natural language response grounded in graph context -6. **KG Algorithm Enhancement**: Uses centrality and community detection for enhanced reasoning - -#### **Key Features** - -- **Multi-Hop Reasoning**: Traverses graph up to configurable hops (default: 2) -- **Reasoning Trace**: Shows entity relationship paths used in reasoning -- **Grounded Responses**: LLM generates answers citing specific graph entities -- **Multiple LLM Providers**: Supports Groq, OpenAI, HuggingFace, and LiteLLM (100+ LLMs) -- **Fallback Handling**: Returns context with reasoning path if LLM unavailable -- **KG Algorithm Integration**: Uses centrality and community detection for enhanced reasoning - -#### **Method Signature** - -```python -def query_with_reasoning( - self, - query: str, - llm_provider: Any, # LLM provider from semantica.llms - max_results: int = 10, - max_hops: int = 2, - **kwargs -) -> Dict[str, Any]: -``` - -**Parameters:** -- `query` (str): User query -- `llm_provider`: LLM provider instance (from `semantica.llms`) -- `max_results` (int): Maximum context results to retrieve (default: 10) -- `max_hops` (int): Maximum graph traversal hops (default: 2) -- `**kwargs`: Additional retrieval options - -**Returns:** -- `response` (str): Generated natural language answer -- `reasoning_path` (str): Multi-hop reasoning trace -- `sources` (List[Dict]): Retrieved context items used -- `confidence` (float): Overall confidence score -- `num_sources` (int): Number of sources retrieved -- `num_reasoning_paths` (int): Number of reasoning paths found - -#### **Code Example** - -```python -from semantica.context import AgentContext -from semantica.llms import Groq -from semantica.vector_store import VectorStore -import os - -# Initialize context with advanced features -context = AgentContext( - vector_store=VectorStore(backend="faiss"), - knowledge_graph=kg, - enable_advanced_analytics=True, - enable_kg_algorithms=True -) - -# Configure LLM provider -llm_provider = Groq( - model="llama-3.1-8b-instant", - api_key=os.getenv("GROQ_API_KEY") -) - -# Query with reasoning -result = context.query_with_reasoning( - query="What IPs are associated with security alerts?", - llm_provider=llm_provider, - max_results=10, - max_hops=2 -) - -# Access results -print(f"Response: {result['response']}") -print(f"\nReasoning Path: {result['reasoning_path']}") -print(f"Confidence: {result['confidence']:.3f}") -``` - -#### **Using Different LLM Providers** - -```python -# Groq -from semantica.llms import Groq llm = Groq(model="llama-3.1-8b-instant", api_key=os.getenv("GROQ_API_KEY")) -# OpenAI -from semantica.llms import OpenAI -llm = OpenAI(model="gpt-4", api_key=os.getenv("OPENAI_API_KEY")) - -# LiteLLM (100+ providers) -from semantica.llms import LiteLLM -llm = LiteLLM(model="anthropic/claude-sonnet-4-20250514") - -# Use with query_with_reasoning -result = context.query_with_reasoning( - query="Your question here", +result = agent.query_with_reasoning( + query="What technologies work well together?", llm_provider=llm, - max_hops=3 + max_hops=2 ) -``` -!!! tip "When to Use" - - **Complex Queries**: When simple retrieval doesn't capture relationships - - **Explainable AI**: When you need to show reasoning paths - - **Multi-Hop Questions**: "What IPs are associated with alerts that affect users?" - - **Grounded Responses**: When you need answers citing specific graph entities - - **Decision Analysis**: When analyzing decision influence and relationships +print(f"Response: {result['response']}") +print(f"Reasoning: {result['reasoning_path']}") +``` --- -### EntityLinker (The Connector) -Resolves text mentions to unique entities and assigns URIs. +## 🏗️ ContextGraph - Knowledge Organization -#### **Key Methods** +When you need to organize complex information and understand relationships, ContextGraph helps you build intelligent knowledge networks. -| Method | Description | -|--------|-------------| -| `link_entities(source, target, type)` | Creates a link between two entities. | -| `assign_uri(entity_name, type)` | Generates a consistent URI for an entity. | - -#### **Code Example** +### Easy Knowledge Graph Building ```python -from semantica.context import EntityLinker +from semantica.context import ContextGraph -linker = EntityLinker(knowledge_graph=graph) +# Create a knowledge graph +knowledge = ContextGraph(advanced_analytics=True) -# Link two entities -linker.link_entities( - source_entity_id="Python", - target_entity_id="Programming", - link_type="IS_A", - confidence=0.95 +# Add things you want to remember (nodes) +knowledge.add_node("Python", "language", properties={"popularity": "high"}) +knowledge.add_node("Programming", "concept", properties={"type": "skill"}) +knowledge.add_node("FastAPI", "framework", properties={"language": "Python"}) + +# Connect related things (edges) +knowledge.add_edge("Python", "Programming", "related_to") +knowledge.add_edge("Python", "FastAPI", "supports") +knowledge.add_edge("FastAPI", "Programming", "used_for") +``` + +### Easy Decision Management +```python +# Record decisions in your knowledge graph +from semantica.context.decision_models import Decision +from datetime import datetime + +decision = Decision( + decision_id="tech_choice_001", + category="technology_choice", + scenario="Framework selection for web API", + reasoning="FastAPI provides better performance for Python APIs", + outcome="selected_fastapi", + confidence=0.92, + timestamp=datetime.now(), + decision_maker="system", + metadata={"entities": ["Python", "FastAPI", "web_project"]} +) +knowledge.add_decision(decision) + +# Or use the convenience method for quick decisions +decision_id = knowledge.add_decision_simple( + category="technology_choice", + scenario="Framework selection for web API", + reasoning="FastAPI provides better performance for Python APIs", + outcome="selected_fastapi", + confidence=0.92, + entities=["Python", "FastAPI", "web_project"] +) + +# Find similar decisions easily +similar = knowledge.find_precedents_by_scenario( + scenario="web framework", + category="technology_choice", + limit=3 +) + +print(f"Found {len(similar)} similar decisions") +``` + +### Smart Analytics +```python +# Understand decision impact +impact = knowledge.analyze_decision_impact(decision_id) +print(f"This decision influenced {impact.get('total_influenced', 0)} other decisions") + +# Get decision summary +summary = knowledge.get_decision_summary() +print(f"Total decisions: {summary.get('total_decisions', 0)}") +print(f"Categories: {list(summary.get('categories', {}).keys())}") + +# Trace decision chains +chains = knowledge.trace_decision_chain(decision_id) +print(f"Decision chain has {len(chains)} connections") + +# Check if decisions follow rules +compliance = knowledge.check_decision_rules({ + "category": "loan_approval", + "scenario": "Mortgage application", + "reasoning": "Good credit score, stable income", + "outcome": "approved", + "confidence": 0.95 +}) + +if compliance.get("compliant", False): + print("✅ Decision follows all rules") +else: + print(f"❌ Rule violations: {compliance.get('violations', [])}") +``` + +### Graph Analytics Made Simple +```python +# Get overview of your knowledge graph +summary = knowledge.get_graph_summary() +print(f"Knowledge graph has {summary.get('nodes', 0)} concepts") +print(f"And {summary.get('edges', 0)} relationships") + +# Find related concepts +related = knowledge.find_related_nodes("Python", how_many=5) +for concept_id, similarity in related: + print(f"Related to {concept_id}: {similarity:.2f}") + +# Understand which concepts are most important +importance = knowledge.get_node_importance("Python") +print(f"Python importance score: {importance.get('degree', 0)}") +``` + +### Core Methods + +| Method | What It Does | When to Use | +|--------|-------------|------------| +| `add_node(node_id, node_type, properties)` | Add concepts to remember | Build knowledge base | +| `add_edge(source, target, relation)` | Connect related concepts | Show relationships | +| `add_decision(category, scenario, reasoning, outcome, confidence, ...)` | Record decisions | Track choices and learn | +| `add_decision_simple(category, scenario, reasoning, outcome, confidence, ...)` | Easy decision recording | Quick decision tracking | +| `find_precedents(decision_id, limit)` | Find precedents by ID | Get connected decisions | +| `find_precedents_by_scenario(scenario, category, ...)` | Find similar decisions | Make consistent choices | +| `analyze_decision_impact(decision_id)` | Understand decision influence | See how decisions affect others | +| `get_decision_summary()` | Get decision statistics | Understand decision patterns | +| `trace_decision_chain(decision_id)` | Trace decision connections | Understand decision relationships | +| `check_decision_rules(decision_data)` | Validate decisions | Ensure compliance | +| `get_graph_summary()` | Get graph overview | Understand knowledge structure | +| `find_related_nodes(node_id, how_many)` | Find related concepts | Discover connections | +| `get_node_importance(node_id)` | Measure concept importance | Identify key concepts | + +--- + +## 🔄 Using Both Together - Complete Intelligence + +### Your Smart Agent System +```python +from semantica.context import AgentContext, ContextGraph +from semantica.vector_store import VectorStore + +# Create the components +vector_store = VectorStore(backend="inmemory", dimension=384) +knowledge = ContextGraph(advanced_analytics=True) + +# Create your intelligent agent +agent = AgentContext( + vector_store=vector_store, + knowledge_graph=knowledge, # Add knowledge graph + decision_tracking=True, + graph_expansion=True, + advanced_analytics=True +) + +# Your agent works like this: +# 1. Store information in memory +agent.store("User wants to learn web development with Python") +agent.store("User is a beginner programmer") +agent.store("User prefers hands-on tutorials") + +# 2. Find relevant information +results = agent.retrieve("Python web development tutorials") +print(f"Found {len(results)} relevant memories") + +# 3. Make smart decisions +decision_id = agent.record_decision( + category="content_recommendation", + scenario="Python web development learning path", + reasoning="Beginner needs hands-on Python web tutorial", + outcome="recommended_flask_tutorial", + confidence=0.89 +) + +# 4. Learn and improve over time +insights = agent.get_context_insights() +print(f"Agent insights: {insights}") + +# 5. Access advanced features when needed +graph_summary = agent.graph_builder.get_graph_summary() +node_importance = agent.graph_builder.get_node_importance("Python") +``` + +--- + +## 🎯 Real-World Applications + +### 🏦 Banking - Smart Loan Decisions +```python +# Track loan decisions and learn from patterns +bank_agent = AgentContext(vector_store=bank_vector_store, decision_tracking=True) + +# Store customer information +bank_agent.store("Customer has credit score 750, stable employment") +bank_agent.store("Customer is first-time homebuyer") + +# Make loan decision +loan_decision = bank_agent.record_decision( + category="loan_approval", + scenario="First-time homebuyer mortgage", + reasoning="Good credit score, stable income, 20% down payment", + outcome="approved", + confidence=0.94 +) + +# Find similar loan decisions for consistency +similar_loans = bank_agent.find_precedents("homebuyer", category="loan_approval") +print(f"Found {len(similar_loans)} similar loan decisions") +``` + +### 🏥 Healthcare - Patient Care Decisions +```python +# Track patient care decisions +health_agent = AgentContext(vector_store=medical_vector_store, decision_tracking=True) + +# Store patient information +health_agent.store("Patient has hypertension, type 2 diabetes") +health_agent.store("Patient allergic to penicillin") + +# Make treatment decision +treatment_decision = health_agent.record_decision( + category="treatment_plan", + scenario="Hypertension with diabetes", + reasoning="ACE inhibitors safe for diabetic patients", + outcome="prescribed_ace_inhibitor", + confidence=0.91 +) + +# Find similar treatment cases +similar_cases = health_agent.find_precedents("hypertension", category="treatment_plan") +``` + +### 🛒 E-commerce - Smart Recommendations +```python +# Track recommendation decisions +ecommerce_graph = ContextGraph() + +# Build user-product knowledge +ecommerce_graph.add_node("user_123", "user", {"segment": "premium"}) +ecommerce_graph.add_node("laptop_xyz", "product", {"category": "electronics"}) +ecommerce_graph.add_edge("user_123", "laptop_xyz", "viewed") + +# Make recommendation decision +from semantica.context.decision_models import Decision +from datetime import datetime + +rec_decision = Decision( + decision_id="rec_001", + category="product_recommendation", + scenario="Laptop recommendation for premium user", + reasoning="User prefers high-performance electronics", + outcome="recommended_gaming_laptop", + confidence=0.87, + timestamp=datetime.now(), + decision_maker="recommendation_system", + metadata={"entities": ["user_123", "laptop_xyz"]} +) +ecommerce_graph.add_decision(rec_decision) + +# Or use the convenience method +rec_decision_id = ecommerce_graph.add_decision_simple( + category="product_recommendation", + scenario="Laptop recommendation for premium user", + reasoning="User prefers high-performance electronics", + outcome="recommended_gaming_laptop", + confidence=0.87, + entities=["user_123", "laptop_xyz"] +) + +# Find similar recommendations +similar_recs = ecommerce_graph.find_precedents_by_scenario( + scenario="laptop recommendation", + limit=5 ) ``` --- -## ⚙️ Configuration +## ⚙️ Configuration Options -### Environment Variables - -```bash -# Global token limit -export CONTEXT_TOKEN_LIMIT=2000 +### Simple Setup (Most Common) +```python +# Just memory and basic learning +agent = AgentContext(vector_store=vector_store) ``` -### YAML Configuration +### Smart Setup (Recommended) +```python +# Memory + decision learning +agent = AgentContext( + vector_store=vector_store, + decision_tracking=True, + graph_expansion=True +) +``` -```yaml -context: - short_term_limit: 10 - retrieval: - hybrid_alpha: 0.5 # 0.0=Vector, 1.0=Graph - max_expansion_hops: 2 +### Complete Setup (Maximum Power) +```python +# Everything enabled +agent = AgentContext( + vector_store=vector_store, + knowledge_graph=ContextGraph(advanced_analytics=True), + decision_tracking=True, + graph_expansion=True, + advanced_analytics=True, + kg_algorithms=True, + vector_store_features=True +) +``` + +### ContextGraph Options +```python +# Basic knowledge graph +graph = ContextGraph() + +# Advanced knowledge graph +graph = ContextGraph( + advanced_analytics=True, # Enable smart algorithms + centrality_analysis=True, # Find important concepts + community_detection=True, # Find groups of related concepts + node_embeddings=True # Understand concept similarity +) ``` --- -## 📝 Data Structures +## 📊 Data Structures -### MemoryItem -The fundamental unit of storage. +### MemoryItem - The Basic Memory Unit ```python @dataclass class MemoryItem: content: str # The actual text content timestamp: datetime # When it was created - metadata: Dict # Arbitrary tags (user_id, source, etc.) - embedding: List[float] # The vector representation - entities: List[Dict] # Entities found in this content + metadata: Dict # Tags like user_id, conversation_id + embedding: List[float] # Vector representation + entities: List[Dict] # Entities found in content ``` -### Decision -The fundamental unit of decision tracking. +### Decision - The Decision Unit ```python @dataclass class Decision: @@ -775,10 +507,9 @@ class Decision: timestamp: datetime # When decision was made entities: List[str] # Related entities metadata: Dict # Additional decision metadata - embedding: List[float] # Decision embedding for similarity ``` -### Graph Node (Dict Format) +### Graph Node - Knowledge Concept ```python { "id": "node_unique_id", @@ -786,13 +517,12 @@ class Decision: "properties": { "content": "Description of the node", "weight": 1.0, - "centrality": 0.85, - "community": "cluster_1" + "importance": 0.85 } } ``` -### Graph Edge (Dict Format) +### Graph Edge - Knowledge Relationship ```python { "source_id": "origin_node", @@ -808,194 +538,93 @@ class Decision: --- -## 🧩 Advanced Usage +## 🚀 Advanced Features -### Context Graphs in Production - -#### Building Domain-Specific Context Graphs - -**Financial Services Context Graph** +### GraphRAG with Multi-Hop Reasoning ```python -from semantica.context import ContextGraph +# Query with reasoning and LLM integration +result = agent.query_with_reasoning( + query="What technologies work well together?", + llm_provider=llm_provider, + max_hops=2, + max_results=10 +) -# Create financial context graph -financial_graph = ContextGraph(enable_advanced_analytics=True) - -# Add financial entities -financial_graph.add_nodes([ - { - "id": "customer_001", - "type": "Customer", - "properties": { - "credit_score": 750, - "risk_profile": "low", - "account_type": "premium" - } - }, - { - "id": "mortgage_product", - "type": "Product", - "properties": { - "category": "loan", - "interest_rate": 3.5, - "max_amount": 500000 - } - }, - { - "id": "loan_officer_001", - "type": "Agent", - "properties": { - "department": "lending", - "experience_years": 5 - } - } -]) - -# Add relationships -financial_graph.add_edges([ - { - "source_id": "customer_001", - "target_id": "mortgage_product", - "type": "ELIGIBLE_FOR", - "properties": {"confidence": 0.92} - }, - { - "source_id": "loan_officer_001", - "target_id": "customer_001", - "type": "SERVES", - "properties": {"relationship_duration": "2_years"} - } -]) - -# Analyze financial context -centrality = financial_graph.get_node_centrality("customer_001") -similar_customers = financial_graph.find_similar_nodes("customer_001") +print(f"Response: {result['response']}") +print(f"Reasoning Path: {result['reasoning_path']}") +print(f"Confidence: {result['confidence']:.3f}") ``` -**Healthcare Context Graph** +### Production Integration ```python -# Create healthcare context graph -healthcare_graph = ContextGraph(enable_advanced_analytics=True) +# Use with persistent graph stores +from semantica.graph_store import GraphStore -# Add medical entities -healthcare_graph.add_nodes([ - { - "id": "patient_001", - "type": "Patient", - "properties": { - "condition": "diabetes_type_2", - "age": 45, - "risk_factors": ["obesity", "hypertension"] - } - }, - { - "id": "metformin", - "type": "Medication", - "properties": { - "class": "biguanide", - "uses": ["diabetes_treatment", "pcos"] - } - }, - { - "id": "dr_smith", - "type": "Physician", - "properties": { - "specialty": "endocrinology", - "hospital": "general_hospital" - } - } -]) +# Neo4j integration +neo4j_store = GraphStore( + backend="neo4j", + uri="bolt://localhost:7687", + user="neo4j", + password="password" +) -# Add medical relationships -healthcare_graph.add_edges([ - { - "source_id": "patient_001", - "target_id": "metformin", - "type": "PRESCRIBED", - "properties": {"dosage": "500mg", "frequency": "twice_daily"} - }, - { - "source_id": "dr_smith", - "target_id": "patient_001", - "type": "TREATS", - "properties": {"since": "2023-01-15"} - } -]) - -# Analyze healthcare context -treatment_patterns = healthcare_graph.analyze_graph_with_kg() -similar_patients = healthcare_graph.find_similar_nodes("patient_001") +# Production agent with persistent storage +production_agent = AgentContext( + vector_store=vector_store, + knowledge_graph=neo4j_store, + decision_tracking=True, + advanced_analytics=True +) ``` -#### Context Graph Analytics and Insights - +### Analytics and Insights ```python -# Get comprehensive graph insights -insights = graph.get_graph_metrics() -print(f"Graph Density: {insights['density']}") -print(f"Average Clustering: {insights['avg_clustering']}") -print(f"Number of Communities: {len(insights['communities'])}") +# Get comprehensive insights +insights = agent.get_context_insights() +print(f"Total decisions: {insights.get('total_decisions', 0)}") +print(f"Decision categories: {list(insights.get('categories', {}).keys())}") +print(f"Most common outcome: {insights.get('most_common_outcome', 'N/A')}") -# Find influential nodes -influential_nodes = [] -for node_id in graph.get_all_nodes(): - centrality = graph.get_node_centrality(node_id) - if centrality['betweenness'] > 0.8: - influential_nodes.append(node_id) - -# Trace decision influence -decision_influence = graph.trace_influence_paths("decision_001", max_depth=3) -for path in decision_influence: - print(f"Influence Path: {' -> '.join(path)}") +# Graph analytics +graph_insights = agent.graph_builder.get_graph_summary() +node_importance = agent.graph_builder.get_node_importance("key_concept") ``` -#### Context Graph Visualization +--- -```python -# Export context graph for visualization -graph_data = graph.export_graph(format="networkx") +## 📚 Need More Help? -# Create visualization (requires matplotlib/networkx) -import matplotlib.pyplot as plt -import networkx as nx +### For Beginners +- Start with **AgentContext** for most applications +- Use basic **store/retrieve** for memory management +- Add **decision tracking** to enable learning +- Enable features gradually as needed -G = nx.node_link_graph(graph_data) -pos = nx.spring_layout(G) +### For Advanced Users +- Add **ContextGraph** for knowledge organization +- Use **analytics** to understand patterns +- Implement **policies** for consistent decisions +- Use **persistence** for state management -# Draw the context graph -plt.figure(figsize=(12, 8)) -nx.draw(G, pos, with_labels=True, node_color='lightblue', - node_size=1000, font_size=8, edge_color='gray') -plt.title("Context Graph Visualization") -plt.show() -``` +### For Production +- Enable **all features** for maximum intelligence +- Use **save/load** for state persistence +- **Monitor performance** with insights and health checks +- **Test thoroughly** before deployment -### Method Registry (Extensibility) -Register custom implementations for graph building, memory management, or retrieval. +### Examples and Tutorials +- Look at the **real-world examples** above for your specific use case +- Check **configuration options** to customize your agent +- Start simple and add power as needed -#### **Code Example** -```python -from semantica.context import registry +--- -def custom_graph_builder(entities, relationships): - # Custom logic to build graph - return "my_graph_structure" +**Happy building intelligent agents!** 🎯 -# Register the new method -registry.register("graph", "custom_builder", custom_graph_builder) -``` +--- -### Configuration Manager -Programmatically manage configuration settings. +## 📚 See Also -#### **Code Example** -```python -from semantica.context.config import context_config - -# Update configuration at runtime -context_config.set("retention_days", 60) - -## See Also - [Vector Store](vector_store.md) - The long-term storage backend - [Graph Store](graph_store.md) - The knowledge graph backend - [KG Algorithms](kg.md) - Knowledge graph algorithms and analytics diff --git a/semantica/context/__init__.py b/semantica/context/__init__.py index c479cfeb..ff50605d 100644 --- a/semantica/context/__init__.py +++ b/semantica/context/__init__.py @@ -46,7 +46,7 @@ Enhanced Analytics: Main Classes: - AgentContext: High-level interface with KG integration - - ContextGraph: In-memory graph store with KG algorithm support + - ContextGraph: In-memory graph store with KG algorithm support and comprehensive decision management - ContextNode/ContextEdge: Graph data structures - AgentMemory: Persistent agent memory with RAG - MemoryItem: Memory item data structure @@ -63,12 +63,13 @@ Decision Tracking Classes: - Policy/Precedent/PolicyException: Decision tracking data structures Example Usage: - >>> from semantica.context import AgentContext + >>> from semantica.context import AgentContext, ContextGraph + >>> # Simple AgentContext with decision tracking >>> context = AgentContext(vector_store=vs, knowledge_graph=kg, - ... enable_decision_tracking=True, - ... enable_advanced_analytics=True, - ... enable_kg_algorithms=True, - ... enable_vector_store_features=True) + ... decision_tracking=True, + ... advanced_analytics=True, + ... kg_algorithms=True, + ... vector_store_features=True) >>> memory_id = context.store("User asked about Python", conversation_id="conv1") >>> results = context.retrieve("Python programming") >>> decision_id = context.record_decision(category="approval", @@ -81,6 +82,21 @@ Example Usage: ... use_kg_features=True) >>> influence = context.analyze_decision_influence(decision_id) >>> insights = context.get_context_insights() + + >>> # Comprehensive ContextGraph with all decision features + >>> graph = ContextGraph(advanced_analytics=True, enable_causality=True) + >>> decision_id = graph.record_decision( + ... category="loan_approval", + ... scenario="First-time homebuyer", + ... reasoning="Good credit score and stable income", + ... outcome="approved", + ... confidence=0.95, + ... entities=["customer_123", "property_456"] + ... ) + >>> precedents = graph.find_precedents("loan_approval", limit=5) + >>> influence = graph.analyze_decision_influence(decision_id) + >>> insights = graph.get_decision_insights() + >>> causality = graph.trace_decision_causality(decision_id) Production Examples: - Banking: Mortgage approvals, credit decisions, risk assessment diff --git a/semantica/context/agent_context.py b/semantica/context/agent_context.py index 6a2ecaaa..8ee178dd 100644 --- a/semantica/context/agent_context.py +++ b/semantica/context/agent_context.py @@ -46,10 +46,11 @@ Key Methods: Example Usage: >>> from semantica.context import AgentContext >>> context = AgentContext(vector_store=vs, knowledge_graph=kg, - ... enable_decision_tracking=True, - ... enable_advanced_analytics=True, - ... enable_kg_algorithms=True, - ... enable_vector_store_features=True) + ... decision_tracking=True, + ... advanced_analytics=True, + ... kg_algorithms=True, + ... vector_store_features=True, + ... graph_expansion=True) >>> memory_id = context.store("User asked about Python", conversation_id="conv1") >>> results = context.retrieve("Python programming") >>> decision_id = context.record_decision(category="approval", @@ -124,13 +125,13 @@ class AgentContext: knowledge_graph: Optional[Any] = None, retention_days: Optional[int] = 30, max_memories: int = 10000, - use_graph_expansion: bool = True, + graph_expansion: bool = True, max_expansion_hops: int = 2, hybrid_alpha: float = 0.5, - enable_decision_tracking: bool = False, - enable_advanced_analytics: bool = True, - enable_kg_algorithms: bool = True, - enable_vector_store_features: bool = True, + decision_tracking: bool = False, + advanced_analytics: bool = True, + kg_algorithms: bool = True, + vector_store_features: bool = True, **kwargs, ): """ @@ -141,14 +142,14 @@ class AgentContext: knowledge_graph: Knowledge graph instance (optional, enables GraphRAG) retention_days: Days to keep memories (default: 30, None=unlimited) max_memories: Maximum number of memories (default: 10000) - use_graph_expansion: Enable graph expansion for retrieval (default: True) + graph_expansion: Enable graph expansion for retrieval (default: True) max_expansion_hops: Maximum hops for graph expansion (default: 2) hybrid_alpha: Balance between vector (0) and graph (1) retrieval (default: 0.5) - enable_decision_tracking: Enable decision tracking features (default: False) - enable_advanced_analytics: Enable advanced analytics (default: True) - enable_kg_algorithms: Enable KG algorithms integration (default: True) - enable_vector_store_features: Enable vector store features (default: True) + decision_tracking: Enable decision tracking features (default: False) + advanced_analytics: Enable advanced analytics (default: True) + kg_algorithms: Enable KG algorithms integration (default: True) + vector_store_features: Enable vector store features (default: True) **kwargs: Additional options passed to underlying components Raises: @@ -168,11 +169,11 @@ class AgentContext: # Store advanced feature flags self.config = { - "enable_decision_tracking": enable_decision_tracking, - "enable_advanced_analytics": enable_advanced_analytics, - "enable_kg_algorithms": enable_kg_algorithms, - "enable_vector_store_features": enable_vector_store_features, - "use_graph_expansion": use_graph_expansion, + "decision_tracking": decision_tracking, + "advanced_analytics": advanced_analytics, + "kg_algorithms": kg_algorithms, + "vector_store_features": vector_store_features, + "graph_expansion": graph_expansion, "max_expansion_hops": max_expansion_hops, "hybrid_alpha": hybrid_alpha, **kwargs @@ -195,7 +196,7 @@ class AgentContext: "memory_store": self._memory, "knowledge_graph": knowledge_graph, "vector_store": vector_store, - "use_graph_expansion": use_graph_expansion, + "use_graph_expansion": graph_expansion, "max_expansion_hops": max_expansion_hops, "hybrid_alpha": hybrid_alpha, **kwargs, @@ -209,62 +210,81 @@ class AgentContext: if knowledge_graph and hasattr(knowledge_graph, "build_from_conversations"): self._graph_builder = knowledge_graph - # Store config - self.config = { + self.config.update({ "retention_days": retention_days, "max_memories": max_memories, - "use_graph_expansion": use_graph_expansion, - "max_expansion_hops": max_expansion_hops, - "hybrid_alpha": hybrid_alpha, - "enable_decision_tracking": enable_decision_tracking, - } + }) # Initialize decision tracking components if enabled + self._decision_backend = None self._decision_recorder = None self._decision_query = None self._causal_analyzer = None self._policy_engine = None - if enable_decision_tracking and knowledge_graph: - # Validate that knowledge_graph supports required GraphStore interface - if not hasattr(knowledge_graph, 'execute_query'): - self.logger.error( - "Decision tracking requires a GraphStore-compatible knowledge graph with execute_query() method. " - "Provided knowledge_graph type does not support Cypher queries. " - "Use GraphStore (Neo4j, FalkorDB) or disable decision tracking." - ) - raise ValueError( - "Decision tracking requires a GraphStore-compatible knowledge graph. " - "The provided knowledge_graph does not have an execute_query() method. " - "For decision tracking, use a GraphStore backend (Neo4j, FalkorDB) " - "or set enable_decision_tracking=False." - ) - - # Initialize enhanced decision tracking components - try: - self._decision_recorder = DecisionRecorder(knowledge_graph) - - # Enhanced DecisionQuery with KG and vector store integration - self._decision_query = DecisionQuery( - graph_store=knowledge_graph, - vector_store=vector_store if enable_vector_store_features else None, - enable_advanced_analytics=enable_advanced_analytics, - enable_centrality_analysis=enable_kg_algorithms, - enable_community_detection=enable_kg_algorithms, - enable_node_embeddings=enable_kg_algorithms - ) - - self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) + if decision_tracking and knowledge_graph: + if hasattr(knowledge_graph, "execute_query"): + self._decision_backend = "graph_store" + try: + self._decision_recorder = DecisionRecorder(knowledge_graph) + self._decision_query = DecisionQuery( + graph_store=knowledge_graph, + vector_store=vector_store if vector_store_features else None, + advanced_analytics=advanced_analytics, + centrality_analysis=kg_algorithms, + community_detection=kg_algorithms, + node_embeddings=kg_algorithms + ) + self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) + self._policy_engine = PolicyEngine(knowledge_graph) + self.logger.info("Enhanced decision tracking components initialized successfully") + except Exception as e: + self.logger.warning( + f"Failed to initialize enhanced decision tracking ({type(e).__name__})" + ) + self._decision_recorder = DecisionRecorder(knowledge_graph) + self._decision_query = DecisionQuery(knowledge_graph) + self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) + self._policy_engine = PolicyEngine(knowledge_graph) + else: + self._decision_backend = "context_graph" + # Initialize basic decision components for ContextGraph self._policy_engine = PolicyEngine(knowledge_graph) - - self.logger.info("Enhanced decision tracking components initialized successfully") - except Exception as e: - self.logger.warning(f"Failed to initialize enhanced decision tracking: {e}") - # Fallback to basic components - self._decision_recorder = DecisionRecorder(knowledge_graph) - self._decision_query = DecisionQuery(knowledge_graph) self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) - self._policy_engine = PolicyEngine(knowledge_graph) + + # Initialize DecisionQuery for ContextGraph + try: + self._decision_query = DecisionQuery( + graph_store=knowledge_graph, + vector_store=vector_store if vector_store_features else None, + advanced_analytics=advanced_analytics, + centrality_analysis=kg_algorithms, + community_detection=kg_algorithms, + node_embeddings=kg_algorithms + ) + self.logger.info("ContextGraph decision tracking initialized successfully") + except Exception as e: + self.logger.warning( + f"Failed to initialize DecisionQuery for ContextGraph ({type(e).__name__})" + ) + # Create a minimal DecisionQuery that delegates to ContextGraph + self._decision_query = type('MinimalDecisionQuery', (), { + 'analyze_decision_influence': lambda self, decision_id, max_depth=3: + knowledge_graph.analyze_decision_influence(decision_id, max_depth) if hasattr(knowledge_graph, 'analyze_decision_influence') else {}, + 'find_precedents': lambda self, query, category=None, limit=10: + knowledge_graph.find_precedents(query, category, limit) if hasattr(knowledge_graph, 'find_precedents') else [], + })() + + if vector_store_features and hasattr(self.vector_store, "initialize_decision_pipeline"): + try: + self.vector_store.initialize_decision_pipeline( + graph_store=knowledge_graph if kg_algorithms else None, + use_graph_features=kg_algorithms + ) + except Exception as e: + self.logger.warning( + f"Failed to initialize decision pipeline ({type(e).__name__})" + ) @property def memory(self) -> AgentMemory: @@ -766,7 +786,7 @@ class AgentContext: "edge_count": graph.get("statistics", {}).get("edge_count", 0), } except Exception as e: - self.logger.warning(f"Failed to build graph from documents: {e}") + self.logger.warning(f"Failed to build graph from documents ({type(e).__name__})") return {"node_count": 0, "edge_count": 0} def _context_to_dict( @@ -1467,7 +1487,7 @@ class AgentContext: if memory_id: imported += 1 except Exception as e: - self.logger.warning(f"Failed to import memory: {e}") + self.logger.warning(f"Failed to import memory ({type(e).__name__})") return imported @@ -1560,7 +1580,7 @@ class AgentContext: Raises: RuntimeError: If decision tracking is not enabled """ - if not self._decision_recorder: + if not self._decision_backend: raise RuntimeError("Decision tracking is not enabled") from .decision_models import Decision @@ -1579,16 +1599,33 @@ class AgentContext: entities = entities or [] source_documents = [] # Could be enhanced to capture source docs - - decision_id = self._decision_recorder.record_decision( - decision, entities, source_documents - ) - - # Capture cross-system context if provided - if cross_system_context: - self._decision_recorder.capture_cross_system_context( - decision_id, cross_system_context + + if self._decision_backend == "graph_store": + decision_id = self._decision_recorder.record_decision( + decision, entities, source_documents ) + + if cross_system_context: + self._decision_recorder.capture_cross_system_context( + decision_id, cross_system_context + ) + + return decision_id + + if not hasattr(self.knowledge_graph, "record_decision"): + raise RuntimeError("Decision tracking backend does not support decisions") + + # Delegate to ContextGraph + decision_id = self.knowledge_graph.record_decision( + category=category, + scenario=scenario, + reasoning=reasoning, + outcome=outcome, + confidence=confidence, + entities=entities, + decision_maker=decision_maker, + metadata={"cross_system_context": cross_system_context} if cross_system_context else None + ) return decision_id @@ -1618,24 +1655,130 @@ class AgentContext: Raises: RuntimeError: If decision tracking is not enabled """ - if not self._decision_query: + if not self._decision_backend: raise RuntimeError("Decision tracking is not enabled") - - if use_hybrid_search: + + # Delegate to ContextGraph if available + if self._decision_backend == "context_graph" and hasattr(self.knowledge_graph, "find_precedents_by_scenario"): try: - return self._decision_query.find_precedents_hybrid( - scenario, category, limit + precedents = self.knowledge_graph.find_precedents_by_scenario( + scenario=scenario, + category=category, + limit=limit, + use_semantic_search=use_hybrid_search ) - except Exception: - # Fallback to basic search if hybrid fails - return self._decision_query._find_precedents_basic(scenario, category, limit) - else: - # Simple category-based search + # Convert to Decision objects if needed + from .decision_models import Decision + decisions = [] + for precedent in precedents: + decision_data = precedent["decision"] + metadata = dict(decision_data.get("metadata", {}) or {}) + if "entities" in decision_data: + metadata["entities"] = decision_data.get("entities", []) + decision = Decision( + decision_id=decision_data["id"], + category=decision_data["category"], + scenario=decision_data["scenario"], + reasoning=decision_data["reasoning"], + outcome=decision_data["outcome"], + confidence=decision_data["confidence"], + timestamp=datetime.fromtimestamp(decision_data["timestamp"]), + decision_maker=decision_data.get("decision_maker"), + metadata=metadata, + ) + decisions.append(decision) + return decisions + except Exception as e: + self.logger.exception("ContextGraph find_precedents failed") + return [] + + # Fallback to DecisionQuery for graph_store backend + if self._decision_backend == "graph_store": + if use_hybrid_search: + try: + return self._decision_query.find_precedents_hybrid( + scenario, category, limit + ) + except Exception: + return self._decision_query._find_precedents_basic(scenario, category, limit) if category: return self._decision_query.find_by_category(category, limit) - else: - # Use basic search - return self._decision_query._find_precedents_basic(scenario, category, limit) + return self._decision_query._find_precedents_basic(scenario, category, limit) + + results: List[Decision] = [] + + def _safe_parse_timestamp(value: Any) -> datetime: + if isinstance(value, datetime): + return value + if not value: + return datetime.now() + try: + return datetime.fromisoformat(str(value)) + except Exception: + return datetime.now() + + if use_hybrid_search and hasattr(self.vector_store, "search_decisions"): + filters = {"category": category} if category else None + vector_results = self.vector_store.search_decisions( + query=scenario, + filters=filters, + limit=limit, + use_hybrid_search=True + ) + for r in vector_results: + meta = r.get("metadata") or {} + decision_id = meta.get("decision_id") or meta.get("id") + if decision_id and hasattr(self.knowledge_graph, "nodes") and decision_id in self.knowledge_graph.nodes: + node = self.knowledge_graph.nodes[decision_id] + if getattr(node, "node_type", None) == "Decision": + data = getattr(node, "properties", {}) or {} + decision = Decision( + decision_id=decision_id, + category=data.get("category", ""), + scenario=getattr(node, "content", ""), + reasoning=data.get("reasoning", ""), + outcome=data.get("outcome", ""), + confidence=float(data.get("confidence", 0.0) or 0.0), + timestamp=_safe_parse_timestamp(data.get("timestamp")), + decision_maker=data.get("decision_maker", "ai_agent"), + reasoning_embedding=data.get("reasoning_embedding"), + node2vec_embedding=data.get("node2vec_embedding"), + metadata={k: v for k, v in data.items() if k not in [ + "category", "reasoning", "outcome", "confidence", + "timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding" + ]} + ) + decision.metadata["score"] = r.get("score") + results.append(decision) + + if results: + return results[:limit] + + if hasattr(self.knowledge_graph, "find_nodes"): + for node in self.knowledge_graph.find_nodes(node_type="Decision"): + if category and node.get("metadata", {}).get("category") != category: + continue + data = node.get("metadata", {}) or {} + results.append( + Decision( + decision_id=node.get("id", ""), + category=data.get("category", ""), + scenario=node.get("content", ""), + reasoning=data.get("reasoning", ""), + outcome=data.get("outcome", ""), + confidence=float(data.get("confidence", 0.0) or 0.0), + timestamp=_safe_parse_timestamp(data.get("timestamp")), + decision_maker=data.get("decision_maker", "ai_agent"), + reasoning_embedding=data.get("reasoning_embedding"), + node2vec_embedding=data.get("node2vec_embedding"), + metadata={k: v for k, v in data.items() if k not in [ + "category", "reasoning", "outcome", "confidence", + "timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding" + ]} + ) + ) + + return results[:limit] def get_causal_chain( self, @@ -1657,12 +1800,35 @@ class AgentContext: Raises: RuntimeError: If decision tracking is not enabled """ - if not self._causal_analyzer: + if not self._decision_backend: raise RuntimeError("Decision tracking is not enabled") - - return self._causal_analyzer.get_causal_chain( - decision_id, direction, max_depth - ) + + if self._decision_backend == "graph_store": + return self._causal_analyzer.get_causal_chain( + decision_id, direction, max_depth + ) + + if self._decision_backend == "context_graph": + # Use ContextGraph's get_causal_chain method + if hasattr(self.knowledge_graph, "get_causal_chain"): + return self.knowledge_graph.get_causal_chain( + decision_id=decision_id, + direction=direction, + max_depth=max_depth + ) + # Fallback to causal analyzer + return self._causal_analyzer.get_causal_chain( + decision_id, direction, max_depth + ) + + if hasattr(self.knowledge_graph, "get_causal_chain"): + return self.knowledge_graph.get_causal_chain( + decision_id=decision_id, + direction=direction, + max_depth=max_depth + ) + + raise RuntimeError("Decision tracking backend does not support causal chains") def get_policy_engine(self) -> PolicyEngine: """ @@ -1791,17 +1957,50 @@ class AgentContext: Returns: Cross-system context """ - # This is a placeholder for cross-system context capture - # In practice, this would integrate with various systems context = {} - + for system in systems: - context[system] = { + captured_at = datetime.now().isoformat() + payload: Dict[str, Any] = { "entity_id": entity_id, "system_name": system, - "captured_at": datetime.now().isoformat(), - "status": "captured" + "captured_at": captured_at, } + + try: + # GraphStore-backed capture path + if self.knowledge_graph and hasattr(self.knowledge_graph, "execute_query"): + query = """ + MATCH (c:CrossSystemContext {system_name: $system_name}) + WHERE c.context_data IS NOT NULL + RETURN c + ORDER BY c.created_at DESC + LIMIT 5 + """ + result = self.knowledge_graph.execute_query( + query, {"system_name": system} + ) + records = result.get("records", []) if isinstance(result, dict) else result + payload["status"] = "captured" + payload["records_found"] = len(records) if isinstance(records, list) else 0 + payload["records"] = records if isinstance(records, list) else [] + else: + payload["status"] = "captured_without_backend" + payload["records_found"] = 0 + payload["records"] = [] + except Exception as e: + self.logger.warning( + "Cross-system input capture failed for system=%s entity_id=%s: %s", + system, + entity_id, + str(e), + ) + payload["status"] = "capture_failed" + payload["error"] = "internal_capture_error" + payload["records_found"] = 0 + payload["records"] = [] + + context[system] = payload return context @@ -1854,7 +2053,7 @@ class AgentContext: Returns: Comprehensive graph analysis results """ - if not self._graph_builder or not self.config.get("enable_advanced_analytics", True): + if not self._graph_builder or not self.config.get("advanced_analytics", True): return {"error": "Advanced analytics not available"} try: @@ -1869,7 +2068,7 @@ class AgentContext: "message": "Basic analysis only - KG features not available" } except Exception as e: - self.logger.error(f"Failed to analyze context graph: {e}") + self.logger.error(f"Failed to analyze context graph ({type(e).__name__})") return {"error": str(e)} def find_similar_entities( @@ -1896,7 +2095,7 @@ class AgentContext: # Fallback to basic content similarity return [] except Exception as e: - self.logger.error(f"Failed to find similar entities: {e}") + self.logger.error(f"Failed to find similar entities ({type(e).__name__})") return [] def get_entity_centrality(self, entity_id: str) -> Dict[str, float]: @@ -1918,7 +2117,7 @@ class AgentContext: else: return {"error": "Centrality analysis not available"} except Exception as e: - self.logger.error(f"Failed to get entity centrality: {e}") + self.logger.error(f"Failed to get entity centrality ({type(e).__name__})") return {"error": str(e)} def find_precedents_advanced( @@ -1958,7 +2157,7 @@ class AgentContext: # Fallback to basic method return self.find_precedents(scenario, category, limit) except Exception as e: - self.logger.error(f"Failed to find advanced precedents: {e}") + self.logger.error(f"Failed to find advanced precedents ({type(e).__name__})") return [] def analyze_decision_influence(self, decision_id: str, max_depth: int = 3) -> Dict[str, Any]: @@ -1975,11 +2174,21 @@ class AgentContext: if not self._decision_query: raise RuntimeError("Decision tracking is not enabled") + # Delegate to ContextGraph if available + if hasattr(self.knowledge_graph, "analyze_decision_influence"): + try: + return self.knowledge_graph.analyze_decision_influence(decision_id, max_depth) + except Exception as e: + self.logger.error(f"ContextGraph analyze_decision_influence failed: {e}") + # Fallback to DecisionQuery + pass + + # Fallback to DecisionQuery try: if hasattr(self._decision_query, 'analyze_decision_influence'): return self._decision_query.analyze_decision_influence(decision_id, max_depth) else: - # Fallback to basic causal chain + # Basic causal chain fallback return { "decision_id": decision_id, "downstream_decisions": self.get_causal_chain(decision_id, "downstream", max_depth), @@ -1987,7 +2196,7 @@ class AgentContext: "message": "Basic analysis only - KG features not available" } except Exception as e: - self.logger.error(f"Failed to analyze decision influence: {e}") + self.logger.error(f"Failed to analyze decision influence ({type(e).__name__})") return {"error": str(e)} def predict_decision_relationships(self, decision_id: str, top_k: int = 5) -> List[Dict]: @@ -2010,7 +2219,7 @@ class AgentContext: else: return [] except Exception as e: - self.logger.error(f"Failed to predict decision relationships: {e}") + self.logger.error(f"Failed to predict decision relationships ({type(e).__name__})") return [] def get_context_insights(self) -> Dict[str, Any]: @@ -2023,12 +2232,12 @@ class AgentContext: insights = { "timestamp": datetime.now().isoformat(), "memory_stats": self.stats(), - "decision_stats": self.get_decision_statistics() if self.config.get("enable_decision_tracking") and hasattr(self, 'get_decision_statistics') else {}, + "decision_stats": self.get_decision_statistics() if self.config.get("decision_tracking") and hasattr(self, 'get_decision_statistics') else {}, "graph_analysis": self.analyze_context_graph(), "advanced_features": { - "kg_algorithms_enabled": self.config.get("enable_kg_algorithms", False), - "vector_store_features_enabled": self.config.get("enable_vector_store_features", False), - "decision_tracking_enabled": self.config.get("enable_decision_tracking", False) + "kg_algorithms_enabled": self.config.get("kg_algorithms", False), + "vector_store_features_enabled": self.config.get("vector_store_features", False), + "decision_tracking_enabled": self.config.get("decision_tracking", False) } } diff --git a/semantica/context/causal_analyzer.py b/semantica/context/causal_analyzer.py index ab53699c..de4753ea 100644 --- a/semantica/context/causal_analyzer.py +++ b/semantica/context/causal_analyzer.py @@ -77,7 +77,7 @@ class CausalChainAnalyzer: using graph traversal. """ - def __init__(self, graph_store: GraphStore): + def __init__(self, graph_store: Any): """ Initialize CausalChainAnalyzer. @@ -105,6 +105,13 @@ class CausalChainAnalyzer: List of decisions in causal chain """ try: + if hasattr(self.graph_store, "get_causal_chain") and not hasattr(self.graph_store, "execute_query"): + return self.graph_store.get_causal_chain( + decision_id=decision_id, + direction=direction, + max_depth=max_depth + ) + if direction not in ["upstream", "downstream"]: raise ValueError("Direction must be 'upstream' or 'downstream'") @@ -124,10 +131,13 @@ class CausalChainAnalyzer: results = self.graph_store.execute_query(query, { "decision_id": decision_id }) + results = self._extract_records(results) decisions = [] for record in results: - decision_data = record.get("end", {}) + decision_data = record.get("end") if isinstance(record, dict) else None + if not isinstance(decision_data, dict): + decision_data = record if isinstance(record, dict) else {} decision = self._dict_to_decision(decision_data) decision.metadata["causal_distance"] = record.get("distance", 0) decisions.append(decision) @@ -165,10 +175,13 @@ class CausalChainAnalyzer: results = self.graph_store.execute_query(query, { "decision_id": decision_id }) + results = self._extract_records(results) decisions = [] for record in results: - decision_data = record.get("end", {}) + decision_data = record.get("end") if isinstance(record, dict) else None + if not isinstance(decision_data, dict): + decision_data = record if isinstance(record, dict) else {} decision = self._dict_to_decision(decision_data) decision.metadata["influence_depth"] = record.get("influence_depth", 0) decisions.append(decision) @@ -207,10 +220,13 @@ class CausalChainAnalyzer: results = self.graph_store.execute_query(query, { "decision_id": decision_id }) + results = self._extract_records(results) decisions = [] for record in results: - decision_data = record.get("end", {}) + decision_data = record.get("end") if isinstance(record, dict) else None + if not isinstance(decision_data, dict): + decision_data = record if isinstance(record, dict) else {} decision = self._dict_to_decision(decision_data) decision.metadata["precedent_depth"] = record.get("precedent_depth", 0) decision.metadata["relationship_types"] = record.get("relationship_types", []) @@ -243,7 +259,7 @@ class CausalChainAnalyzer: ORDER BY loop_length """ - results = self.graph_store.execute_query(query) + results = self._extract_records(self.graph_store.execute_query(query)) loops = [] for record in results: @@ -316,10 +332,13 @@ class CausalChainAnalyzer: results = self.graph_store.execute_query(query, { "decision_id": decision_id }) + results = self._extract_records(results) root_decisions = [] for record in results: - decision_data = record.get("root", {}) + decision_data = record.get("root") if isinstance(record, dict) else None + if not isinstance(decision_data, dict): + decision_data = record if isinstance(record, dict) else {} decision = self._dict_to_decision(decision_data) decision.metadata["root_distance"] = record.get("root_distance", 0) root_decisions.append(decision) @@ -411,9 +430,13 @@ class CausalChainAnalyzer: # Handle timestamp conversion if isinstance(data.get("timestamp"), str): data["timestamp"] = datetime.fromisoformat(data["timestamp"]) - + + decision_id = data.get("decision_id") or data.get("id") + if not decision_id: + raise KeyError("decision_id") + return Decision( - decision_id=data.get("decision_id", ""), + decision_id=decision_id, category=data.get("category", ""), scenario=data.get("scenario", ""), reasoning=data.get("reasoning", ""), @@ -423,5 +446,14 @@ class CausalChainAnalyzer: decision_maker=data.get("decision_maker", ""), reasoning_embedding=data.get("reasoning_embedding"), node2vec_embedding=data.get("node2vec_embedding"), - metadata=data.get("metadata", {}) + metadata=data.get("metadata", {}), ) + + def _extract_records(self, results: Any) -> List[Dict[str, Any]]: + """Normalize execute_query result shapes to a list of record maps.""" + if isinstance(results, dict): + records = results.get("records", []) + return records if isinstance(records, list) else [] + if isinstance(results, list): + return results + return [] diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 4084391c..b14bfbbc 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -12,6 +12,14 @@ Core Features: - Export to dictionary format - Decision tracking integration +Comprehensive Decision Management: + - Decision Recording: Store decisions with full context and metadata + - Precedent Search: Find similar decisions using hybrid search algorithms + - Influence Analysis: Analyze decision impact and relationships + - Causal Analysis: Trace decision causality chains + - Policy Enforcement: Built-in policy compliance checking + - Advanced Analytics: Comprehensive decision insights + KG Algorithm Integration: - Centrality Analysis: Degree, betweenness, closeness, eigenvector centrality - Community Detection: Modularity-based community identification @@ -47,23 +55,43 @@ Enhanced Methods: - analyze_graph_with_kg(): Comprehensive graph analysis - get_node_centrality(): Get centrality measures for nodes - find_similar_nodes(): Find similar nodes with advanced similarity - - add_decision(): Add decisions with context integration + - record_decision(): Add decisions with context integration - find_precedents(): Find decision precedents + - analyze_decision_influence(): Analyze decision influence + - get_decision_insights(): Get comprehensive decision analytics + - trace_decision_causality(): Trace decision causality + - enforce_decision_policy(): Enforce decision policies - get_graph_metrics(): Get comprehensive statistics - export_graph(): Export graph in various formats Example Usage: >>> from semantica.context import ContextGraph - >>> graph = ContextGraph(enable_advanced_analytics=True, - ... enable_centrality_analysis=True, - ... enable_community_detection=True, - ... enable_node_embeddings=True) + >>> graph = ContextGraph(advanced_analytics=True, + ... centrality_analysis=True, + ... community_detection=True, + ... node_embeddings=True) + >>> + >>> # Basic graph operations >>> graph.add_node("Python", type="language", properties={"popularity": "high"}) >>> graph.add_node("Programming", type="concept") >>> graph.add_edge("Python", "Programming", type="related_to") >>> centrality = graph.get_node_centrality("Python") >>> similar = graph.find_similar_nodes("Python", similarity_type="content") >>> analysis = graph.analyze_graph_with_kg() + >>> + >>> # Decision management + >>> decision_id = graph.record_decision( + ... category="loan_approval", + ... scenario="First-time homebuyer", + ... reasoning="Good credit score", + ... outcome="approved", + ... confidence=0.95, + ... entities=["customer_123", "property_456"] + ... ) + >>> precedents = graph.find_precedents("loan_approval", limit=5) + >>> influence = graph.analyze_decision_influence(decision_id) + >>> insights = graph.get_decision_insights() + >>> causality = graph.trace_decision_causality(decision_id) Production Use Cases: - Knowledge Management: Build and analyze knowledge graphs @@ -71,12 +99,17 @@ Production Use Cases: - Recommendation Systems: Graph-based recommendations - Social Networks: Analyze connections and influence - Research Networks: Map collaborations and citations + - Financial Services: Loan approvals, fraud detection, risk assessment + - Healthcare: Treatment decisions, policy compliance, clinical pathways + - Legal: Case precedent analysis, decision consistency + - Business: Workflow decisions, policy compliance, audit trails """ from collections import defaultdict, deque from dataclasses import dataclass, field from datetime import datetime from typing import Any, Dict, List, Optional, Set, Tuple, Union +import uuid from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -135,9 +168,16 @@ class ContextEdge: class ContextGraph: """ - In-memory implementation of context graph. - - Provides capabilities to build, store, and query a context graph. + Easy-to-Use Context Graph with All Advanced Features. + + This class provides simple methods for: + - Building knowledge graphs + - Recording and analyzing decisions + - Finding precedents and patterns + - Causal analysis and policy enforcement + - Advanced graph analytics + + Perfect for building intelligent AI agents that can learn from decisions! """ def __init__(self, config: Optional[Dict[str, Any]] = None, **kwargs): @@ -150,10 +190,10 @@ class ContextGraph: - extract_entities: Extract entities from content (default: True) - extract_relationships: Extract relationships (default: True) - entity_linker: Entity linker instance - - enable_advanced_analytics: Enable KG algorithms (default: True) - - enable_centrality_analysis: Enable centrality measures (default: True) - - enable_community_detection: Enable community detection (default: True) - - enable_node_embeddings: Enable Node2Vec embeddings (default: True) + - advanced_analytics: Enable KG algorithms (default: True) + - centrality_analysis: Enable centrality measures (default: True) + - community_detection: Enable community detection (default: True) + - node_embeddings: Enable Node2Vec embeddings (default: True) """ self.logger = get_logger("context_graph") self.config = config or {} @@ -185,15 +225,15 @@ class ContextGraph: self.kg_components = {} self._analytics_cache = {} - enable_advanced = self.config.get("enable_advanced_analytics", True) + enable_advanced = self.config.get("advanced_analytics", True) if KG_AVAILABLE and enable_advanced: try: - if self.config.get("enable_centrality_analysis", True): + if self.config.get("centrality_analysis", True): self.kg_components["centrality_calculator"] = CentralityCalculator() - if self.config.get("enable_community_detection", True): + if self.config.get("community_detection", True): self.kg_components["community_detector"] = CommunityDetector() - if self.config.get("enable_node_embeddings", True): + if self.config.get("node_embeddings", True): self.kg_components["node_embedder"] = NodeEmbedder() self.kg_components["path_finder"] = PathFinder() self.kg_components["similarity_calculator"] = SimilarityCalculator() @@ -260,7 +300,69 @@ class ContextGraph: count += 1 return count - def get_neighbors(self, node_id: str, hops: int = 1) -> List[Dict[str, Any]]: + def __contains__(self, node_id: object) -> bool: + if not isinstance(node_id, str): + return False + return node_id in self.nodes + + def has_node(self, node_id: str) -> bool: + return node_id in self.nodes + + def neighbors(self, node_id: str) -> List[str]: + return self.get_neighbor_ids(node_id) + + def get_neighbor_ids( + self, + node_id: str, + relationship_types: Optional[List[str]] = None, + ) -> List[str]: + if node_id not in self.nodes: + return [] + + rel_filter = set(relationship_types) if relationship_types else None + neighbor_ids: List[str] = [] + for edge in self._adjacency.get(node_id, []): + if rel_filter is None or edge.edge_type in rel_filter: + neighbor_ids.append(edge.target_id) + return neighbor_ids + + def get_nodes_by_label(self, label: str) -> List[str]: + return list(self.node_type_index.get(label, set())) + + def get_node_property(self, node_id: str, property_name: str) -> Any: + node = self.nodes.get(node_id) + if not node: + return None + return node.properties.get(property_name) + + def get_node_attributes(self, node_id: str) -> Dict[str, Any]: + node = self.nodes.get(node_id) + if not node: + return {} + return node.properties.copy() + + def add_node_attribute(self, node_id: str, attributes: Dict[str, Any]) -> None: + node = self.nodes.get(node_id) + if not node: + return + node.properties.update(attributes) + node.metadata.update(attributes) + + def get_edge_data(self, source_id: str, target_id: str) -> Dict[str, Any]: + for edge in self._adjacency.get(source_id, []): + if edge.target_id == target_id: + data = edge.metadata.copy() + data["type"] = edge.edge_type + data["weight"] = edge.weight + return data + return {} + + def get_neighbors( + self, + node_id: str, + hops: int = 1, + relationship_types: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: """ Get neighbors of a node. @@ -269,36 +371,39 @@ class ContextGraph: if node_id not in self.nodes: return [] - neighbors = [] + neighbors: List[Dict[str, Any]] = [] visited = {node_id} - queue = deque([(node_id, 0)]) # (current_id, current_hop) + queue = deque([(node_id, 0)]) + rel_filter = set(relationship_types) if relationship_types else None while queue: current_id, current_hop = queue.popleft() - if current_hop >= hops: continue - # Get outgoing edges outgoing_edges = self._adjacency.get(current_id, []) for edge in outgoing_edges: + if rel_filter is not None and edge.edge_type not in rel_filter: + continue neighbor_id = edge.target_id - if neighbor_id not in visited: - visited.add(neighbor_id) - queue.append((neighbor_id, current_hop + 1)) + if neighbor_id in visited: + continue + visited.add(neighbor_id) + queue.append((neighbor_id, current_hop + 1)) - if neighbor_id in self.nodes: - node = self.nodes[neighbor_id] - neighbors.append( - { - "id": node.node_id, - "type": node.node_type, - "content": node.content, - "relationship": edge.edge_type, - "weight": edge.weight, - "hop": current_hop + 1, - } - ) + node = self.nodes.get(neighbor_id) + if not node: + continue + neighbors.append( + { + "id": node.node_id, + "type": node.node_type, + "content": node.content, + "relationship": edge.edge_type, + "weight": edge.weight, + "hop": current_hop + 1, + } + ) return neighbors @@ -444,11 +549,14 @@ class ContextGraph: """Find a node by ID.""" node = self.nodes.get(node_id) if node: + merged_metadata = {} + merged_metadata.update(getattr(node, "metadata", {}) or {}) + merged_metadata.update(getattr(node, "properties", {}) or {}) return { "id": node.node_id, "type": node.node_type, "content": node.content, - "metadata": node.metadata, + "metadata": merged_metadata, } return None @@ -465,7 +573,7 @@ class ContextGraph: "id": n.node_id, "type": n.node_type, "content": n.content, - "metadata": n.metadata, + "metadata": {**(getattr(n, "metadata", {}) or {}), **(getattr(n, "properties", {}) or {})}, } for n in nodes ] @@ -508,10 +616,49 @@ class ContextGraph: # --- Internal Helpers --- + def _normalize_timestamp(self, timestamp_value) -> datetime: + """ + Normalize timestamp value to datetime object. + + Handles various timestamp formats: + - datetime: returns as-is + - int/float: converts from epoch seconds + - str: parses ISO format (with optional Z) + - None/invalid: returns current datetime + + Args: + timestamp_value: Timestamp value in various formats + + Returns: + datetime: Normalized datetime object + """ + from datetime import datetime + + if isinstance(timestamp_value, datetime): + return timestamp_value + elif isinstance(timestamp_value, (int, float)): + return datetime.fromtimestamp(timestamp_value) + elif isinstance(timestamp_value, str): + # Handle ISO format with optional Z suffix + timestamp_str = timestamp_value.rstrip('Z') # Remove Z if present + try: + return datetime.fromisoformat(timestamp_str) + except ValueError: + # Fallback to current datetime if parsing fails + return datetime.now() + else: + # Fallback for None or other types + return datetime.now() + def _add_internal_node(self, node: ContextNode) -> bool: """Internal method to add a node.""" self.nodes[node.node_id] = node - self.node_type_index[node.node_type].add(node.node_id) + # Handle edge case where node_type might be None or not a string + if hasattr(node, 'node_type') and isinstance(node.node_type, str): + self.node_type_index[node.node_type].add(node.node_id) + else: + # Use 'unknown' as fallback for invalid node_type + self.node_type_index['unknown'].add(node.node_id) return True def _add_internal_edge(self, edge: ContextEdge) -> bool: @@ -773,6 +920,7 @@ class ContextGraph: "id": n.node_id, "type": n.node_type, "content": n.content, + "properties": n.properties, "metadata": n.metadata, } for n in self.nodes.values() @@ -792,6 +940,34 @@ class ContextGraph: }, } + def from_dict(self, graph_dict: Dict[str, Any]) -> None: + """Load graph from dictionary format.""" + # Clear existing graph + self.nodes.clear() + self.edges.clear() + + # Add nodes + for node_data in graph_dict.get("nodes", []): + node = ContextNode( + node_id=node_data["id"], + node_type=node_data["type"], + content=node_data.get("content", ""), + properties=node_data.get("properties", {}), + metadata=node_data.get("metadata", {}) + ) + self._add_internal_node(node) + + # Add edges + for edge_data in graph_dict.get("edges", []): + edge = ContextEdge( + source_id=edge_data["source"], + target_id=edge_data["target"], + edge_type=edge_data["type"], + weight=edge_data.get("weight", 1.0), + metadata=edge_data.get("metadata", {}) + ) + self._add_internal_edge(edge) + # Decision Support Methods def add_decision(self, decision: "Decision") -> None: """ @@ -802,8 +978,18 @@ class ContextGraph: """ from .decision_models import Decision + # Handle empty decision ID by generating UUID for both None and empty string + # This ensures consistent behavior with Decision model's __post_init__ method + node_id = decision.decision_id if decision.decision_id else str(uuid.uuid4()) + + # Handle None metadata + metadata = decision.metadata or {} + + # Normalize timestamp to ensure consistent storage format + normalized_timestamp = self._normalize_timestamp(decision.timestamp) + node = ContextNode( - node_id=decision.decision_id, + node_id=node_id, node_type="Decision", content=decision.scenario, properties={ @@ -811,11 +997,11 @@ class ContextGraph: "reasoning": decision.reasoning, "outcome": decision.outcome, "confidence": decision.confidence, - "timestamp": decision.timestamp.isoformat(), + "timestamp": normalized_timestamp.isoformat(), "decision_maker": decision.decision_maker, "reasoning_embedding": decision.reasoning_embedding, "node2vec_embedding": decision.node2vec_embedding, - **decision.metadata + **metadata } ) self._add_internal_node(node) @@ -838,6 +1024,19 @@ class ContextGraph: if relationship_type not in valid_types: raise ValueError(f"Relationship type must be one of: {valid_types}") + # Check if decisions exist - if not, skip adding relationship + if source_decision_id not in self.nodes or target_decision_id not in self.nodes: + return + + # Check if nodes are decision nodes - if not, skip adding relationship + source_node = self.nodes[source_decision_id] + target_node = self.nodes[target_decision_id] + if (not hasattr(source_node, 'node_type') or not isinstance(source_node.node_type, str) or + not hasattr(target_node, 'node_type') or not isinstance(target_node.node_type, str) or + source_node.node_type.lower() != "decision" or + target_node.node_type.lower() != "decision"): + return + edge = ContextEdge( source_id=source_decision_id, target_id=target_decision_id, @@ -881,41 +1080,51 @@ class ContextGraph: visited.add(current_id) - # Get decision node - if current_id in self.nodes: - node = self.nodes[current_id] - if node.node_type == "Decision": - decision_data = node.properties - decision = Decision( - decision_id=current_id, - category=decision_data.get("category", ""), - scenario=node.content, - reasoning=decision_data.get("reasoning", ""), - outcome=decision_data.get("outcome", ""), - confidence=decision_data.get("confidence", 0.0), - timestamp=datetime.fromisoformat(decision_data.get("timestamp", datetime.now().isoformat())), - decision_maker=decision_data.get("decision_maker", ""), - reasoning_embedding=decision_data.get("reasoning_embedding"), - node2vec_embedding=decision_data.get("node2vec_embedding"), - metadata={k: v for k, v in decision_data.items() if k not in [ - "category", "reasoning", "outcome", "confidence", - "timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding" - ]} - ) - decision.metadata["causal_distance"] = depth - decisions.append(decision) + # Skip the starting decision - only add connected decisions + if current_id != decision_id: + # Get decision node + if current_id in self.nodes: + node = self.nodes[current_id] + if (hasattr(node, 'node_type') and isinstance(node.node_type, str) and + node.node_type.lower() == "decision"): + decision_data = node.properties + timestamp = self._normalize_timestamp(decision_data.get("timestamp")) + decision = Decision( + decision_id=current_id, + category=decision_data.get("category", ""), + scenario=node.content, + reasoning=decision_data.get("reasoning", ""), + outcome=decision_data.get("outcome", ""), + confidence=decision_data.get("confidence", 0.0), + timestamp=timestamp, + decision_maker=decision_data.get("decision_maker", ""), + reasoning_embedding=decision_data.get("reasoning_embedding"), + node2vec_embedding=decision_data.get("node2vec_embedding"), + metadata={k: v for k, v in decision_data.items() if k not in [ + "category", "reasoning", "outcome", "confidence", + "timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding" + ]} + ) + decision.metadata["causal_distance"] = depth + decisions.append(decision) # Find connected decisions for edge in self.edges: if direction == "upstream": if edge.target_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]: - if edge.source_id not in visited: + if edge.source_id not in visited and depth < max_depth: queue.append((edge.source_id, depth + 1)) else: # downstream if edge.source_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]: - if edge.target_id not in visited: + if edge.target_id not in visited and depth < max_depth: queue.append((edge.target_id, depth + 1)) + # Sort by depth for upstream (most distant first) and downstream (closest first) + if direction == "upstream": + decisions.sort(key=lambda d: d.metadata.get("causal_distance", 0), reverse=True) + else: + decisions.sort(key=lambda d: d.metadata.get("causal_distance", 0)) + return decisions def find_precedents(self, decision_id: str, limit: int = 10) -> List["Decision"]: @@ -932,17 +1141,19 @@ class ContextGraph: # Find decisions connected via PRECEDENT_FOR relationships precedent_ids = [] for edge in self.edges: - if edge.source_id == decision_id and edge.edge_type == "PRECEDENT_FOR": - precedent_ids.append(edge.target_id) + if edge.target_id == decision_id and edge.edge_type == "PRECEDENT_FOR": + precedent_ids.append(edge.source_id) # Convert to Decision objects decisions = [] for pid in precedent_ids[:limit]: if pid in self.nodes: node = self.nodes[pid] - if node.node_type == "Decision": + if (hasattr(node, 'node_type') and isinstance(node.node_type, str) and + node.node_type.lower() == "decision"): decision_data = node.properties from .decision_models import Decision + timestamp = self._normalize_timestamp(decision_data.get("timestamp")) decision = Decision( decision_id=pid, category=decision_data.get("category", ""), @@ -950,7 +1161,7 @@ class ContextGraph: reasoning=decision_data.get("reasoning", ""), outcome=decision_data.get("outcome", ""), confidence=decision_data.get("confidence", 0.0), - timestamp=datetime.fromisoformat(decision_data.get("timestamp", datetime.now().isoformat())), + timestamp=timestamp, decision_maker=decision_data.get("decision_maker", ""), reasoning_embedding=decision_data.get("reasoning_embedding"), node2vec_embedding=decision_data.get("node2vec_embedding"), @@ -1025,7 +1236,7 @@ class ContextGraph: except Exception as e: self.logger.error(f"Failed to analyze graph with KG: {e}") - return {"error": str(e)} + return {"error": "Graph analysis failed due to an internal error"} def get_node_centrality(self, node_id: str) -> Dict[str, float]: """ @@ -1062,7 +1273,7 @@ class ContextGraph: except Exception as e: self.logger.error(f"Failed to get node centrality: {e}") - return {"error": str(e)} + return {"error": "Node centrality calculation failed due to an internal error"} def find_similar_nodes( self, node_id: str, similarity_type: str = "content", top_k: int = 10 @@ -1208,6 +1419,834 @@ class ContextGraph: union = words1.union(words2) return len(intersection) / len(union) if union else 0.0 + + # --- Comprehensive Decision Management Features --- + + def record_decision( + self, + category: str, + scenario: str, + reasoning: str, + outcome: str, + confidence: float, + entities: Optional[List[str]] = None, + decision_maker: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + **kwargs + ) -> str: + """ + Record a decision with full context and analytics. + + Args: + category: Decision category (e.g., "loan_approval") + scenario: Decision scenario description + reasoning: Decision reasoning explanation + outcome: Decision outcome + confidence: Confidence score (0.0 to 1.0) + entities: Related entities + decision_maker: Who made the decision + metadata: Additional metadata + **kwargs: Additional decision data + + Returns: + Decision ID for reference + """ + import uuid + from datetime import datetime + + # Input validation + if not isinstance(category, str) or not category.strip(): + raise ValueError("Category must be a non-empty string") + if len(category.strip()) > 100: + raise ValueError("Category must be 100 characters or less") + + if not isinstance(scenario, str) or not scenario.strip(): + raise ValueError("Scenario must be a non-empty string") + if len(scenario.strip()) > 5000: + raise ValueError("Scenario must be 5000 characters or less") + + if not isinstance(reasoning, str) or not reasoning.strip(): + raise ValueError("Reasoning must be a non-empty string") + if len(reasoning.strip()) > 10000: + raise ValueError("Reasoning must be 10000 characters or less") + + if not isinstance(outcome, str) or not outcome.strip(): + raise ValueError("Outcome must be a non-empty string") + if len(outcome.strip()) > 1000: + raise ValueError("Outcome must be 1000 characters or less") + + if not isinstance(confidence, (int, float)): + raise ValueError("Confidence must be a number") + if not (0.0 <= confidence <= 1.0): + raise ValueError("Confidence must be between 0.0 and 1.0") + + if entities is not None: + if not isinstance(entities, list): + raise ValueError("Entities must be a list of strings") + for entity in entities: + if not isinstance(entity, str) or not entity.strip(): + raise ValueError("Each entity must be a non-empty string") + if len(entity.strip()) > 200: + raise ValueError("Each entity must be 200 characters or less") + + if decision_maker is not None: + if not isinstance(decision_maker, str) or not decision_maker.strip(): + raise ValueError("Decision maker must be a non-empty string") + if len(decision_maker.strip()) > 200: + raise ValueError("Decision maker must be 200 characters or less") + + if metadata is not None: + if not isinstance(metadata, dict): + raise ValueError("Metadata must be a dictionary") + for key, value in metadata.items(): + if not isinstance(key, str) or not key.strip(): + raise ValueError("Metadata keys must be non-empty strings") + if len(key.strip()) > 100: + raise ValueError("Metadata keys must be 100 characters or less") + if len(str(value)) > 1000: + raise ValueError("Metadata values must be 1000 characters or less") + + # Validate kwargs + for key, value in kwargs.items(): + if not isinstance(key, str) or not key.strip(): + raise ValueError("Additional field names must be non-empty strings") + if len(key.strip()) > 100: + raise ValueError("Additional field names must be 100 characters or less") + if len(str(value)) > 1000: + raise ValueError("Additional field values must be 1000 characters or less") + + decision_id = str(uuid.uuid4()) + timestamp = datetime.now().timestamp() + + # Sanitize inputs + category = category.strip() + scenario = scenario.strip() + reasoning = reasoning.strip() + outcome = outcome.strip() + confidence = float(confidence) + entities = [entity.strip() for entity in (entities or []) if entity.strip()] + decision_maker = decision_maker.strip() if decision_maker else None + + # Create decision record + decision = { + "id": decision_id, + "category": category, + "scenario": scenario, + "reasoning": reasoning, + "outcome": outcome, + "confidence": confidence, + "entities": entities, + "decision_maker": decision_maker, + "timestamp": timestamp, + "metadata": metadata or {}, + **kwargs + } + + # Store decision in graph + self._add_decision_to_graph(decision) + + # Store in internal decision storage + if not hasattr(self, '_decisions'): + self._decisions = {} + self._decision_index = defaultdict(set) + self._entity_index = defaultdict(set) + self._temporal_index = [] + + self._decisions[decision_id] = decision + self._decision_index[category].add(decision_id) + + for entity in entities or []: + self._entity_index[entity].add(decision_id) + + self._temporal_index.append((decision_id, timestamp)) + self._temporal_index.sort(key=lambda x: x[1], reverse=True) + + self.logger.info(f"Recorded decision {decision_id} in category {category}") + return decision_id + + def find_precedents_by_scenario( + self, + scenario: str, + category: Optional[str] = None, + limit: int = 10, + similarity_threshold: float = 0.5, + use_semantic_search: bool = True, + **filters + ) -> List[Dict[str, Any]]: + """ + Find similar decisions (precedents) using hybrid search. + + Args: + scenario: Scenario to find precedents for + category: Filter by decision category + limit: Maximum number of precedents + similarity_threshold: Minimum similarity score + use_semantic_search: Use vector embeddings for search + **filters: Additional filters + + Returns: + List of similar decisions with similarity scores + """ + if not hasattr(self, '_decisions') or not self._decisions: + return [] + + candidates = set() + + # Get candidates by category + if category: + candidates.update(self._decision_index.get(category, set())) + else: + candidates.update(self._decisions.keys()) + + # Filter by entities if provided + if "entities" in filters: + entity_candidates = set() + for entity in filters["entities"]: + entity_candidates.update(self._entity_index.get(entity, set())) + candidates = candidates.intersection(entity_candidates) + + # Calculate similarities + precedents = [] + for decision_id in candidates: + decision = self._decisions[decision_id] + + # Content similarity + content_sim = self._calculate_decision_content_similarity(scenario, decision) + + # Structural similarity (graph-based) + structural_sim = 0.0 + if self.config.get("advanced_analytics"): + structural_sim = self._calculate_structural_similarity_for_decision(decision_id, scenario) + + # Combined similarity + combined_sim = 0.7 * content_sim + 0.3 * structural_sim + + if combined_sim >= similarity_threshold: + precedents.append({ + "decision": decision, + "similarity": combined_sim, + "content_similarity": content_sim, + "structural_similarity": structural_sim + }) + + # Sort by similarity and limit + precedents.sort(key=lambda x: x["similarity"], reverse=True) + return precedents[:limit] + + def analyze_decision_influence( + self, + decision_id: str, + max_depth: int = 3, + include_indirect: bool = True + ) -> Dict[str, Any]: + """ + Analyze decision influence and impact. + + Args: + decision_id: Decision to analyze + max_depth: Maximum depth for influence analysis + include_indirect: Include indirect influences + + Returns: + Influence analysis results + """ + if not hasattr(self, '_decisions') or decision_id not in self._decisions: + raise ValueError(f"Decision {decision_id} not found") + + decision = self._decisions[decision_id] + + # Direct influence (same entities, category) + direct_influence = set() + for entity in decision["entities"]: + direct_influence.update(self._entity_index.get(entity, set())) + direct_influence.discard(decision_id) + direct_influence.update(self._decision_index.get(decision["category"], set())) + direct_influence.discard(decision_id) + + # Indirect influence (through graph relationships) + indirect_influence = set() + if include_indirect and self.config.get("advanced_analytics"): + indirect_influence = self._find_indirect_decision_influence(decision_id, max_depth) + + # Calculate influence scores + influence_scores = {} + for influenced_id in direct_influence | indirect_influence: + score = self._calculate_decision_influence_score(decision_id, influenced_id) + influence_scores[influenced_id] = score + + # Sort by influence score + sorted_influence = sorted( + influence_scores.items(), + key=lambda x: x[1], + reverse=True + ) + + return { + "decision_id": decision_id, + "direct_influence": list(direct_influence), + "indirect_influence": list(indirect_influence), + "influence_scores": sorted_influence, + "total_influenced": len(influence_scores), + "max_influence_score": max(influence_scores.values()) if influence_scores else 0.0 + } + + def get_decision_insights(self) -> Dict[str, Any]: + """ + Get comprehensive insights about all decisions. + + Returns: + Comprehensive analytics and insights + """ + if not hasattr(self, '_decisions') or not self._decisions: + return {"message": "No decisions recorded yet"} + + # Basic statistics + total_decisions = len(self._decisions) + categories = {} + outcomes = {} + confidence_scores = [] + + for decision in self._decisions.values(): + # Category distribution + categories[decision["category"]] = categories.get(decision["category"], 0) + 1 + + # Outcome distribution + outcomes[decision["outcome"]] = outcomes.get(decision["outcome"], 0) + 1 + + # Confidence scores + confidence_scores.append(decision["confidence"]) + + # Advanced analytics (if available) + advanced_insights = {} + if self.config.get("advanced_analytics"): + advanced_insights = self.analyze_graph_with_kg() + + # Temporal analysis + temporal_insights = self._get_decision_temporal_analysis() + + # Entity analysis + entity_insights = self._get_decision_entity_analysis() + + return { + "total_decisions": total_decisions, + "categories": categories, + "outcomes": outcomes, + "confidence_stats": { + "mean": sum(confidence_scores) / len(confidence_scores), + "min": min(confidence_scores), + "max": max(confidence_scores), + "median": sorted(confidence_scores)[len(confidence_scores) // 2] + }, + "advanced_analytics": advanced_insights, + "temporal_analysis": temporal_insights, + "entity_analysis": entity_insights, + "graph_metrics": self.get_graph_metrics() if hasattr(self, 'get_graph_metrics') else {} + } + + def trace_decision_causality( + self, + decision_id: str, + max_depth: int = 5 + ) -> List[Dict[str, Any]]: + """ + Trace causal chain for a decision. + + Args: + decision_id: Decision to trace + max_depth: Maximum depth for causal analysis + + Returns: + Causal chain as list of decision relationships + """ + if not hasattr(self, '_decisions') or decision_id not in self._decisions: + raise ValueError(f"Decision {decision_id} not found") + + try: + # Use graph traversal to find causal relationships + causal_chain = [] + visited = set() + + def trace_recursive(current_id, depth, path): + if depth >= max_depth or current_id in visited: + return + + visited.add(current_id) + current_decision = self._decisions[current_id] + + # Find potential causes (decisions that influenced this one) + potential_causes = [] + for entity in current_decision["entities"]: + for other_decision_id in self._entity_index.get(entity, set()): + if other_decision_id != current_id: + other_decision = self._decisions[other_decision_id] + if other_decision["timestamp"] < current_decision["timestamp"]: + potential_causes.append(other_decision_id) + + for cause_id in potential_causes: + cause_path = path + [{"from": cause_id, "to": current_id, "type": "influences"}] + causal_chain.append(cause_path) + trace_recursive(cause_id, depth + 1, cause_path) + + trace_recursive(decision_id, 0, []) + return causal_chain + + except Exception as e: + self.logger.error(f"Causal analysis failed: {e}") + return [{"error": "Causal analysis failed due to an internal error"}] + + def enforce_decision_policy( + self, + decision_data: Dict[str, Any], + policy_rules: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + """ + Enforce policies on decision data. + + Args: + decision_data: Decision data to check + policy_rules: Policy rules to enforce + + Returns: + Policy enforcement results + """ + # Simple policy enforcement implementation + violations = [] + warnings = [] + + # Default policy rules + default_rules = { + "min_confidence": 0.7, + "required_outcomes": ["approved", "rejected", "flagged"], + "required_metadata": ["decision_maker"], + "max_reasoning_length": 1000 + } + + rules = policy_rules or default_rules + + # Check confidence + if decision_data.get("confidence", 0) < rules.get("min_confidence", 0.7): + violations.append(f"Confidence too low: {decision_data.get('confidence', 0)}") + + # Check outcome + if decision_data.get("outcome") not in rules.get("required_outcomes", []): + violations.append(f"Invalid outcome: {decision_data.get('outcome')}") + + # Check required metadata + for required_field in rules.get("required_metadata", []): + if not decision_data.get(required_field): + violations.append(f"Missing required field: {required_field}") + + # Check reasoning length + reasoning = decision_data.get("reasoning", "") + if len(reasoning) > rules.get("max_reasoning_length", 1000): + warnings.append(f"Reasoning too long: {len(reasoning)} characters") + + return { + "compliant": len(violations) == 0, + "violations": violations, + "warnings": warnings, + "policy_rules": rules + } + + # --- Private helper methods for decision management --- + + def _add_decision_to_graph(self, decision: Dict[str, Any]) -> None: + """Add decision to context graph.""" + try: + # Add decision node + self.add_node( + decision["id"], + "decision", + category=decision["category"], + outcome=decision["outcome"], + confidence=decision["confidence"], + timestamp=decision["timestamp"], + scenario=decision["scenario"][:100] + "..." if len(decision["scenario"]) > 100 else decision["scenario"], + decision_maker=decision.get("decision_maker", ""), + reasoning=decision["reasoning"][:200] + "..." if len(decision["reasoning"]) > 200 else decision["reasoning"] + ) + + # Add entity nodes and relationships + for entity in decision["entities"]: + # Add entity node if not exists + if not self.find_node(entity): + self.add_node( + entity, + "entity", + name=entity + ) + + # Add relationship + self.add_edge( + decision["id"], + entity, + "involves", + confidence=decision["confidence"] + ) + + # Add category node and relationship + category_id = f"category_{decision['category']}" + if not self.find_node(category_id): + self.add_node( + category_id, + "category", + name=decision["category"] + ) + + self.add_edge( + decision["id"], + category_id, + "belongs_to" + ) + + # Add decision maker node if provided + if decision.get("decision_maker"): + maker_id = f"maker_{decision['decision_maker']}" + if not self.find_node(maker_id): + self.add_node( + maker_id, + "decision_maker", + name=decision["decision_maker"] + ) + + self.add_edge( + decision["id"], + maker_id, + "made_by" + ) + + except Exception as e: + self.logger.exception("Failed to add decision to graph") + + def _calculate_decision_content_similarity(self, scenario: str, decision: Dict[str, Any]) -> float: + """Calculate content similarity between scenario and decision.""" + try: + # Simple word-based similarity + scenario_words = set(scenario.lower().split()) + decision_text = f"{decision['scenario']} {decision['reasoning']} {' '.join(decision['entities'])}" + decision_words = set(decision_text.lower().split()) + + intersection = scenario_words.intersection(decision_words) + union = scenario_words.union(decision_words) + + return len(intersection) / len(union) if union else 0.0 + + except Exception as e: + self.logger.exception("Content similarity calculation failed") + return 0.0 + + def _calculate_structural_similarity_for_decision(self, decision_id: str, scenario: str) -> float: + """Calculate structural similarity using graph algorithms.""" + try: + if not self.config.get("advanced_analytics"): + return 0.0 + + # Use graph similarity algorithms + similar_nodes = self.find_similar_nodes( + decision_id, + similarity_type="structural", + top_k=5 + ) + + if similar_nodes: + # similar_nodes is List[Tuple[str, float]], extract similarity scores + return max(similarity for node_id, similarity in similar_nodes) + + except Exception as e: + self.logger.exception("Structural similarity calculation failed") + + return 0.0 + + def _find_indirect_decision_influence(self, decision_id: str, max_depth: int) -> Set[str]: + """Find indirect influences using graph traversal.""" + try: + influenced = set() + + # Get neighbors in graph + neighbors = self.get_neighbors(decision_id, hops=max_depth) + + for neighbor in neighbors: + if neighbor.get("type") == "decision": + influenced.add(neighbor["id"]) + + return influenced + + except Exception as e: + self.logger.warning(f"Indirect influence analysis failed: {e}") + return set() + + def _calculate_decision_influence_score(self, source_id: str, target_id: str) -> float: + """Calculate influence score between two decisions.""" + try: + if not hasattr(self, '_decisions'): + return 0.0 + + source_decision = self._decisions[source_id] + target_decision = self._decisions[target_id] + + # Base score from shared entities + shared_entities = set(source_decision["entities"]) & set(target_decision["entities"]) + entity_score = len(shared_entities) / max(len(source_decision["entities"]), 1) + + # Category similarity + category_score = 1.0 if source_decision["category"] == target_decision["category"] else 0.0 + + # Temporal proximity (more recent decisions have higher influence) + time_diff = abs(source_decision["timestamp"] - target_decision["timestamp"]) + time_score = max(0.0, 1.0 - time_diff / (30 * 24 * 3600)) # 30 days window + + # Combined score + combined_score = 0.5 * entity_score + 0.3 * category_score + 0.2 * time_score + + return combined_score + + except Exception as e: + self.logger.warning(f"Influence score calculation failed: {e}") + return 0.0 + + def _get_decision_temporal_analysis(self) -> Dict[str, Any]: + """Get temporal analysis of decisions.""" + try: + if not hasattr(self, '_temporal_index') or not self._temporal_index: + return {} + + # Group decisions by time periods + recent_decisions = [did for did, ts in self._temporal_index[:10]] + + return { + "recent_decisions": len(recent_decisions), + "oldest_decision": min(ts for _, ts in self._temporal_index), + "newest_decision": max(ts for _, ts in self._temporal_index), + "time_span": max(ts for _, ts in self._temporal_index) - min(ts for _, ts in self._temporal_index) + } + + except Exception as e: + self.logger.warning(f"Temporal analysis failed: {e}") + return {} + + def _get_decision_entity_analysis(self) -> Dict[str, Any]: + """Get entity analysis from decisions.""" + try: + if not hasattr(self, '_decisions'): + return {} + + entity_counts = {} + for decision in self._decisions.values(): + for entity in decision["entities"]: + entity_counts[entity] = entity_counts.get(entity, 0) + 1 + + # Get top entities + top_entities = sorted(entity_counts.items(), key=lambda x: x[1], reverse=True)[:10] + + return { + "total_entities": len(entity_counts), + "top_entities": top_entities, + "avg_entities_per_decision": sum(len(d["entities"]) for d in self._decisions.values()) / len(self._decisions) + } + + except Exception as e: + self.logger.warning(f"Entity analysis failed: {e}") + return {} + + # --- Easy-to-Use Convenience Methods --- + + def add_decision_simple( + self, + category: str, + scenario: str, + reasoning: str, + outcome: str, + confidence: float = 0.5, + entities: Optional[List[str]] = None, + decision_maker: Optional[str] = "system", + **kwargs + ) -> str: + """ + Easy way to record a decision. + + Args: + category: Decision category (e.g., "loan_approval") + scenario: What was the situation + reasoning: Why was this decision made + outcome: What was decided + confidence: How confident (0.0 to 1.0) + entities: Related entities (people, items, etc.) + decision_maker: Who made the decision + **kwargs: Additional information + + Returns: + Decision ID for reference + """ + return self.record_decision( + category=category, + scenario=scenario, + reasoning=reasoning, + outcome=outcome, + confidence=confidence, + entities=entities, + decision_maker=decision_maker, + metadata=kwargs + ) + + def find_similar_decisions( + self, + scenario: str, + category: Optional[str] = None, + max_results: int = 10, + min_similarity: float = 0.3 + ) -> List[Dict[str, Any]]: + """ + Easy way to find similar past decisions. + + Args: + scenario: What situation are you looking for + category: Filter by decision type + max_results: Maximum results to return + min_similarity: Minimum similarity score + + Returns: + List of similar decisions with similarity scores + """ + return self.find_precedents_by_scenario( + scenario=scenario, + category=category, + limit=max_results, + similarity_threshold=min_similarity + ) + + def analyze_decision_impact( + self, + decision_id: str, + include_indirect: bool = True + ) -> Dict[str, Any]: + """ + Easy way to analyze how a decision impacts others. + + Args: + decision_id: Decision to analyze + include_indirect: Include indirect impacts + + Returns: + Impact analysis results + """ + return self.analyze_decision_influence( + decision_id=decision_id, + max_depth=3, + include_indirect=include_indirect + ) + + def get_decision_summary(self) -> Dict[str, Any]: + """ + Easy way to get a summary of all decisions. + + Returns: + Summary statistics and insights + """ + return self.get_decision_insights() + + def trace_decision_chain( + self, + decision_id: str, + max_steps: int = 5 + ) -> List[Dict[str, Any]]: + """ + Easy way to trace how decisions are connected. + + Args: + decision_id: Starting decision + max_steps: Maximum steps to trace + + Returns: + Decision chain connections + """ + return self.trace_decision_causality( + decision_id=decision_id, + max_depth=max_steps + ) + + def check_decision_rules( + self, + decision_data: Dict[str, Any], + rules: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + """ + Easy way to check if a decision follows the rules. + + Args: + decision_data: Decision to check + rules: Custom rules (uses default if None) + + Returns: + Compliance check results + """ + return self.enforce_decision_policy( + decision_data=decision_data, + policy_rules=rules + ) + + def get_graph_summary(self) -> Dict[str, Any]: + """ + Easy way to get graph statistics. + + Returns: + Graph summary information + """ + if hasattr(self, 'get_graph_metrics'): + return self.get_graph_metrics() + else: + return { + "nodes": len(self.nodes), + "edges": len(self.edges), + "node_types": self._get_node_type_distribution(), + "edge_types": self._get_edge_type_distribution() + } + + def find_related_nodes( + self, + node_id: str, + how_many: int = 10, + similarity_type: str = "content" + ) -> List[Tuple[str, float]]: + """ + Easy way to find nodes similar to a given node. + + Args: + node_id: Reference node + how_many: How many similar nodes to find + similarity_type: Type of similarity ("content", "structural") + + Returns: + List of (node_id, similarity_score) tuples + """ + return self.find_similar_nodes( + node_id=node_id, + similarity_type=similarity_type, + top_k=how_many + ) + + def get_node_importance( + self, + node_id: str + ) -> Dict[str, float]: + """ + Easy way to get how important a node is in the graph. + + Args: + node_id: Node to analyze + + Returns: + Centrality measures (importance scores) + """ + return self.get_node_centrality(node_id) + + def analyze_connections(self) -> Dict[str, Any]: + """ + Easy way to analyze the entire graph structure. + + Returns: + Graph analysis results + """ + return self.analyze_graph_with_kg() # For backward compatibility diff --git a/semantica/context/context_retriever.py b/semantica/context/context_retriever.py index c1131f7d..d5faf8d1 100644 --- a/semantica/context/context_retriever.py +++ b/semantica/context/context_retriever.py @@ -2011,8 +2011,18 @@ Answer:""" try: # Basic neighbor expansion if hasattr(self.knowledge_graph, 'get_neighbors'): - neighbors = self.knowledge_graph.get_neighbors(entity_name) - for neighbor in neighbors[:5]: # Limit to prevent explosion + if hasattr(self.knowledge_graph, "neighbors"): + neighbor_ids = list(self.knowledge_graph.neighbors(entity_name)) + elif hasattr(self.knowledge_graph, "get_neighbor_ids"): + neighbor_ids = self.knowledge_graph.get_neighbor_ids(entity_name) + else: + neighbor_details = self.knowledge_graph.get_neighbors(entity_name, hops=1) + neighbor_ids = [ + n.get("id") for n in neighbor_details + if isinstance(n, dict) and n.get("id") + ] + + for neighbor in neighbor_ids[:5]: # Limit to prevent explosion expanded_entities.append({ "name": neighbor, "type": "related_entity", @@ -2082,8 +2092,17 @@ Answer:""" if hasattr(self.centrality_calculator, 'calculate_degree_centrality'): # Simplified centrality calculation if hasattr(self.knowledge_graph, 'get_neighbors'): - neighbors = self.knowledge_graph.get_neighbors(entity_name) - centrality_scores[entity_name] = len(neighbors) + if hasattr(self.knowledge_graph, "neighbors"): + neighbor_ids = list(self.knowledge_graph.neighbors(entity_name)) + elif hasattr(self.knowledge_graph, "get_neighbor_ids"): + neighbor_ids = self.knowledge_graph.get_neighbor_ids(entity_name) + else: + neighbor_details = self.knowledge_graph.get_neighbors(entity_name, hops=1) + neighbor_ids = [ + n.get("id") for n in neighbor_details + if isinstance(n, dict) and n.get("id") + ] + centrality_scores[entity_name] = len(neighbor_ids) else: centrality_scores[entity_name] = 1 @@ -2136,6 +2155,7 @@ Answer:""" safe_category = category[:20] if category else "unknown" self.logger.warning(f"Failed to find policies for {safe_category}: {type(e).__name__}") return policies + return policies # Decision Retrieval Methods def find_precedents_hybrid( diff --git a/semantica/context/context_usage.md b/semantica/context/context_usage.md index 38a0f084..4e7ac56e 100644 --- a/semantica/context/context_usage.md +++ b/semantica/context/context_usage.md @@ -1,1247 +1,477 @@ -# Context Module Usage Guide +# Context Module - Usage Guide -This guide demonstrates how to use the Semantica context module for building context graphs, managing agent memory, retrieving context, linking entities, and decision tracking with hybrid search capabilities and advanced KG algorithm integration. +## 🎯 What This Module Does -## Quick Imports +The context module gives your AI agents the ability to **remember**, **learn**, and **make smarter decisions** by organizing information in a way that's both powerful and easy to use. -```python -# Core context classes -from semantica.context import AgentContext, ContextGraph, ContextRetriever, DecisionContext +Think of it as giving your agent a brain that can: +- **Remember conversations** (like human memory) +- **Learn from past decisions** (become smarter over time) +- **Find relevant information** quickly (when it matters most) +- **Understand relationships** between concepts +- **Make consistent decisions** based on experience -# Memory management -from semantica.context import AgentMemory +--- -# Entity linking -from semantica.context import EntityLinker - -# Decision tracking with advanced features -from semantica.context import Decision, Policy, PolicyException, DecisionRecorder, DecisionQuery, CausalChainAnalyzer, PolicyEngine - -# For vector storage (often used with context) -from semantica.vector_store import VectorStore -``` - -## Quick Example - -```python -# Simple context setup -vector_store = VectorStore(backend="inmemory", dimension=384) -context = AgentContext(vector_store=vector_store) - -# Store a memory -memory_id = context.store("User likes Python programming", conversation_id="conv1") - -# Retrieve context -results = context.retrieve("Python programming", max_results=5) -print(f"Found {len(results)} results") -``` - -## Table of Contents - -1. [High-Level Interface (Quick Start)](#high-level-interface-quick-start) -2. [Basic Usage](#basic-usage) -3. [Enhanced AgentContext with Decision Tracking and KG Algorithms](#enhanced-agentcontext-with-decision-tracking-and-kg-algorithms) -4. [Context Graph Construction](#context-graph-construction) -5. [Enhanced ContextGraph with KG Algorithms](#enhanced-contextgraph-with-kg-algorithms) -6. [Agent Memory Management](#agent-memory-management) -7. [Context Retrieval](#context-retrieval) -8. [Entity Linking](#entity-linking) -9. [Decision Tracking](#decision-tracking) -10. [Policy Exception Management](#policy-exception-management) -11. [Hybrid Search for Decisions](#hybrid-search-for-decisions) -12. [Context Graphs with KG Algorithms](#context-graphs-with-kg-algorithms) -13. [Advanced Decision Analytics](#advanced-decision-analytics) -14. [Production Examples](#production-examples) -15. [Explainable AI](#explainable-ai) - -## High-Level Interface (Quick Start) - -The `AgentContext` class provides a simplified, generic interface for common use cases. It integrates vector storage, knowledge graphs, and memory management into a unified system. - -### Simple RAG (Vector Only) +## 🚀 Quick Start - 5 Minutes to Your First Smart Agent +### Step 1: Basic Setup ```python from semantica.context import AgentContext from semantica.vector_store import VectorStore -# Initialize vector store -vs = VectorStore(backend="faiss", dimension=768) - -# Initialize context -context = AgentContext(vector_store=vs) - -# Store a memory -memory_id = context.store("User likes Python programming", conversation_id="conv1") - -# Retrieve context -results = context.retrieve("Python programming", max_results=5) - -for result in results: - print(f"Content: {result['content']}") - print(f"Score: {result['score']:.2f}") -``` - -### GraphRAG (Vector + Graph) - -```python -from semantica.context import AgentContext, ContextGraph -from semantica.graph_store import GraphStore - -# Initialize persistent knowledge graph (Recommended for production) -try: - kg = GraphStore(backend="neo4j", uri="bolt://localhost:7687", user="neo4j", password="password") - kg.connect() -except: - print("Neo4j not available, falling back to in-memory graph") - kg = ContextGraph() - -# Initialize context with vector store and knowledge graph -context = AgentContext(vector_store=vs, knowledge_graph=kg) - -# Store documents (auto-builds graph) -documents = [ - "Python is a programming language used for machine learning", - "TensorFlow and PyTorch are popular ML frameworks", - "Machine learning involves training models on data" -] - -stats = context.store( - documents, - extract_entities=True, # Extract entities from documents - extract_relationships=True, # Extract relationships - link_entities=True # Link entities across documents -) - -print(f"Stored {stats['stored_count']} documents") -# Graph stats are available via the graph object directly or context stats -print(f"Graph nodes: {kg.stats()['node_count']}") - -# Retrieve with graph context (auto-detects GraphRAG) -results = context.retrieve( - "Python machine learning", - use_graph=True, # Explicitly use graph - include_entities=True, # Include related entities - expand_graph=True # Use graph expansion -) - -for result in results: - print(f"Content: {result['content']}") - print(f"Score: {result['score']:.2f}") - print(f"Related entities: {len(result.get('related_entities', []))}") -``` - -### Agent Memory Management (Hierarchical) - -The system uses a hierarchical memory structure with: -1. **Short-Term Memory**: Fast, in-memory buffer with token and item count limits. -2. **Long-Term Memory**: Persistent vector store. - -```python -context = AgentContext( - vector_store=vs, - retention_days=30, - short_term_limit=10, # Max items in short-term buffer - token_limit=2000 # Max tokens in short-term buffer -) - -# Store multiple memories in a conversation -context.store("Hello, I'm interested in Python", conversation_id="conv1", user_id="user123") -context.store("What can you tell me about machine learning?", conversation_id="conv1", user_id="user123") - -# Get conversation history -history = context.conversation( - "conv1", - reverse=True, # Most recent first - include_metadata=True # Include full metadata -) - -for item in history: - print(f"{item['timestamp']}: {item['content']}") - -# Delete old memories -deleted_count = context.forget(days_old=90) -print(f"Deleted {deleted_count} old memories") -``` - -### Persistence (Save/Load) - -You can save the entire state of the agent (Memory, Graph, and Vector Index) to disk and reload it later. - -```python -# Save state -context.save("./my_agent_state") - -# Load state -new_context = AgentContext(vector_store=VectorStore(), knowledge_graph=ContextGraph()) -new_context.load("./my_agent_state") -``` - -## Basic Usage - -### Initialization with Backends - -You can configure the `VectorStore` with different backends (`inmemory`, `faiss`, `chroma`, `qdrant`, `weaviate`, `milvus`) and embedding models (including FastEmbed). - -```python -from semantica.context import AgentContext, ContextGraph -from semantica.vector_store import VectorStore - -# Initialize Vector Store with FastEmbed -vs = VectorStore(backend="inmemory", dimension=384) -if hasattr(vs, "embedder") and vs.embedder: - vs.embedder.set_text_model(method="fastembed", model_name="BAAI/bge-small-en-v1.5") - -# Initialize Context Graph -kg = ContextGraph() - -# Initialize Agent Context -context = AgentContext(vector_store=vs, knowledge_graph=kg) -``` - -### Enhanced AgentContext with Decision Tracking and KG Algorithms - -The enhanced `AgentContext` supports advanced decision tracking, KG algorithm integration, and vector store features for production-grade context engineering. - -```python -from semantica.context import AgentContext, ContextGraph -from semantica.vector_store import VectorStore -from semantica.graph_store import GraphStore # For decision tracking - -# Initialize Vector Store -vs = VectorStore(backend="inmemory", dimension=768) - -# Initialize Graph Store (required for decision tracking) -# Note: Decision tracking requires a GraphStore with execute_query() support -gs = GraphStore(backend="neo4j", uri="bolt://localhost:7687") - -# Initialize Context Graph with KG algorithms -kg = ContextGraph( - enable_advanced_analytics=True, - enable_centrality_analysis=True, - enable_community_detection=True, - enable_node_embeddings=True -) - -# Initialize Enhanced Agent Context -context = AgentContext( - vector_store=vs, - knowledge_graph=kg, - enable_decision_tracking=True, # Enable decision lifecycle management - enable_advanced_analytics=True, # Enable KG algorithm integration - enable_kg_algorithms=True, # Enable centrality, community detection - enable_vector_store_features=True # Enable hybrid search capabilities -) - -# Record a decision with full context -decision_id = context.record_decision( - category="mortgage_approval", - scenario="First-time homebuyer application", - reasoning="Strong credit score, stable employment, low debt-to-income ratio", - outcome="approved", - confidence=0.94, - decision_maker="loan_officer_001" -) - -# Find similar decisions with KG-enhanced search -precedents = context.find_precedents_advanced( - scenario="Mortgage application", - use_kg_features=True, - similarity_weights={"semantic": 0.5, "structural": 0.3, "category": 0.2} -) - -# Analyze decision influence -influence = context.analyze_decision_influence(decision_id) -print(f"Influence score: {influence.get('influence_score', 0):.3f}") -print(f"Centrality measures: {influence.get('centrality_measures', {})}") - -# Get comprehensive context insights -insights = context.get_context_insights() -print(f"Advanced features: {insights.get('advanced_features', {})}") -``` - -## Context Graph Construction - -The `ContextGraph` class is an in-memory graph store. - -### Building from Entities and Relationships - -```python -from semantica.context import ContextGraph - -graph = ContextGraph() - -entities = [ - {"id": "e1", "text": "Python", "type": "PROGRAMMING_LANGUAGE"}, - {"id": "e2", "text": "Machine Learning", "type": "CONCEPT"}, - {"id": "e3", "text": "TensorFlow", "type": "FRAMEWORK"}, -] - -relationships = [ - {"source_id": "e1", "target_id": "e2", "type": "used_for", "confidence": 0.9}, - {"source_id": "e3", "target_id": "e2", "type": "implements", "confidence": 0.95}, -] - -graph_data = graph.build_from_entities_and_relationships(entities, relationships) - -print(f"Nodes: {graph.stats()['node_count']}") -print(f"Edges: {graph.stats()['edge_count']}") -``` - -### Building from Conversations - -```python -from semantica.context import ContextGraph - -graph = ContextGraph() - -conversations = [ - { - "id": "conv1", - "content": "User asked about Python programming", - "entities": [ - {"id": "e1", "text": "Python", "type": "PROGRAMMING_LANGUAGE"} - ], - "relationships": [] - } -] - -graph_data = graph.build_from_conversations( - conversations, - link_entities=True, - extract_intents=True -) -``` - -### Adding Nodes and Edges Manually - -```python -from semantica.context import ContextGraph - -graph = ContextGraph() - -# Add nodes -graph.add_node("node1", "entity", "Python programming", confidence=0.9) -graph.add_node("node2", "concept", "Machine Learning", confidence=0.95) - -# Add edges -graph.add_edge("node1", "node2", "related_to", weight=0.9) - -# Get neighbors -neighbors = graph.get_neighbors("node1", hops=2) -print(f"Neighbors: {neighbors}") - -# Query graph -results = graph.query("Python") # Keyword search on nodes -``` - -### Graph Statistics and Analysis - -```python -stats = graph.stats() -print(f"Node types: {stats['node_types']}") -print(f"Density: {stats['density']:.4f}") - -# Find specific nodes/edges -entities = graph.find_nodes(node_type="entity") -relations = graph.find_edges(edge_type="related_to") -node = graph.find_node("node1") -``` - -### Enhanced ContextGraph with KG Algorithms - -The enhanced `ContextGraph` supports advanced KG algorithms for centrality analysis, community detection, and node embeddings. - -```python -from semantica.context import ContextGraph - -# Initialize with KG algorithms enabled -graph = ContextGraph( - enable_advanced_analytics=True, - enable_centrality_analysis=True, - enable_community_detection=True, - enable_node_embeddings=True -) - -# Add nodes and edges -graph.add_node("Python", "Language", {"popularity": "high"}) -graph.add_node("FastAPI", "Framework", {"language": "Python"}) -graph.add_node("Django", "Framework", {"language": "Python"}) -graph.add_edge("FastAPI", "Python", "WRITTEN_IN") -graph.add_edge("Django", "Python", "WRITTEN_IN") - -# Centrality analysis -centrality = graph.get_node_centrality("Python") -print(f"Python centrality: {centrality}") - -# Find similar nodes using embeddings -similar_nodes = graph.find_similar_nodes("Python", similarity_type="content") -print(f"Similar nodes to Python: {[node['id'] for node in similar_nodes]}") - -# Community detection -analysis = graph.analyze_graph_with_kg() -communities = analysis.get('community_analysis', {}) -print(f"Communities found: {communities.get('num_communities', 0)}") - -# Node embeddings -embeddings = graph.get_node_embeddings("Python") -print(f"Python embedding dimension: {len(embeddings) if embeddings else 0}") -``` - -## Agent Memory Management - -The `AgentMemory` class handles short-term and long-term memory with hierarchical storage and token management. - -### Storing and Retrieving - -```python -from semantica.context import AgentMemory - -memory = AgentMemory( - vector_store=vs, - knowledge_graph=kg, - retention_policy="30_days", - max_memory_size=10000, - short_term_limit=20, # 20 items max in short-term - token_limit=4000 # 4000 tokens max in short-term -) - -# Store (automatically updates short-term and long-term) -memory_id = memory.store( - "User asked about Python programming", - metadata={"conversation_id": "conv_123"} -) - -# Store short-term only (fleeting thoughts) -temp_id = memory.store( - "Just checking status...", - skip_vector=True -) - -# Retrieve -results = memory.retrieve( - "Python programming", - max_results=5, - type="conversation" -) - -# Conversation History -history = memory.get_conversation_history("conv_123") -``` - -## Context Retrieval - -The `ContextRetriever` implements hybrid retrieval strategies. - -### Hybrid Retrieval - -```python -from semantica.context import ContextRetriever - -retriever = ContextRetriever( - memory_store=memory, - knowledge_graph=kg, - vector_store=vs, - use_graph_expansion=True, - max_expansion_hops=2, - hybrid_alpha=0.5 # Balance between vector (0.0) and graph (1.0) -) - -results = retriever.retrieve( - "Python programming", - max_results=5 -) - -for result in results: - print(f"Content: {result.content}") - print(f"Source: {result.source}") # 'vector', 'graph', or 'memory' -``` - -## Entity Linking - -The `EntityLinker` helps resolve entities to canonical forms or URIs. - -```python -from semantica.context import EntityLinker - -linker = EntityLinker() -uri = linker.generate_uri("Python Programming Language") -print(uri) # e.g., "python_programming_language" - -# Similarity matching -score = linker._calculate_text_similarity("Python", "Python Language") -``` - -## Decision Tracking - -The `DecisionContext` class provides decision tracking capabilities with hybrid search, explainable AI, and KG algorithm integration. - -### Basic Decision Recording - -```python -from semantica.context import DecisionContext -from semantica.vector_store import VectorStore - -# Initialize decision context +# Create your agent with memory vector_store = VectorStore(backend="inmemory", dimension=384) -decision_context = DecisionContext(vector_store=vector_store, graph_store=None) +agent = AgentContext(vector_store=vector_store) -# Record a decision -decision_id = decision_context.record_decision( - scenario="Credit limit increase for premium customer", - reasoning="Excellent payment history and high credit score", - outcome="approved", - confidence=0.92, - entities=["customer_123", "premium_segment", "credit_card"], - category="credit_approval", - amount=50000, - risk_level="low" +# Your agent can now remember things +memory_id = agent.store("User asked about Python programming") +print(f"Agent remembered: {memory_id}") + +# And find information when needed +results = agent.retrieve("Python tutorials") +print(f"Agent found {len(results)} relevant memories") +``` + +### Step 2: Add Decision Learning +```python +# Your agent learns from its decisions +decision_id = agent.record_decision( + category="content_recommendation", + scenario="User wants Python tutorial", + reasoning="User mentioned being a beginner", + outcome="recommended_basics", + confidence=0.85 ) -print(f"Recorded decision: {decision_id}") +# Your agent can now find similar past decisions +similar_decisions = agent.find_precedents("Python tutorial", limit=3) +print(f"Agent found {len(similar_decisions)} similar past decisions") ``` -### Batch Decision Processing - +### Step 3: Get Insights ```python -# Process multiple decisions -decisions = [ - { - "scenario": "Credit limit increase request", - "reasoning": "Good payment history", - "outcome": "approved", - "confidence": 0.85, - "entities": ["customer_456"], - "category": "credit_approval" - }, - { - "scenario": "Fraud detection alert", - "reasoning": "Suspicious transaction pattern", - "outcome": "blocked", - "confidence": 0.95, - "entities": ["transaction_789", "customer_456"], - "category": "fraud_detection" - } -] - -decision_ids = [] -for decision in decisions: - decision_id = decision_context.record_decision(**decision) - decision_ids.append(decision_id) - -print(f"Processed {len(decision_ids)} decisions") +# Understand how your agent is performing +insights = agent.get_context_insights() +print(f"Agent has made {insights.get('total_decisions', 0)} decisions") +print(f"Decision categories: {list(insights.get('categories', {}).keys())}") ``` -### Decision Context Retrieval +**That's it! Your agent now has memory and can learn from decisions.** 🎉 +--- + +## 🤖 AgentContext - Your Agent's Brain + +### Memory Management (Like Human Memory) ```python -# Get comprehensive decision context -context_info = decision_context.get_decision_context( - decision_id, - depth=2, - include_entities=True, - include_policies=True +# Store different types of memories +agent.store("User likes Python programming", conversation_id="chat_1") +agent.store("User is working on a web project", conversation_id="chat_2") +agent.store("User mentioned being a beginner", conversation_id="chat_3") + +# Find memories when needed +results = agent.retrieve("Python programming", conversation_id="chat_1") +for result in results: + print(f"Memory: {result['content']}") + +# Search across all conversations +all_results = agent.retrieve("beginner") +print(f"Found {len(all_results)} memories about beginners") +``` + +### Learning from Decisions +```python +# Record important decisions +decision_id = agent.record_decision( + category="content_recommendation", + scenario="User wants to learn web development", + reasoning="User is beginner, likes Python", + outcome="recommended_python_basics", + confidence=0.90 ) -print(f"Decision context: {len(context_info.related_entities)} entities") -print(f"Related relationships: {len(context_info.related_relationships)}") +# Find similar past decisions to make better choices +similar_decisions = agent.find_precedents("web development", limit=5) +for decision in similar_decisions: + print(f"Past decision: {decision.scenario}") + print(f"Result: {decision.outcome}") + print(f"Confidence: {decision.confidence}") + print("---") ``` -### Policy Exception Management - -The enhanced decision tracking system supports policy exceptions with proper audit trails. - +### Getting Smarter Over Time ```python -from semantica.context import PolicyException, PolicyEngine +# Enable all learning features +smart_agent = AgentContext( + vector_store=vector_store, + decision_tracking=True, # Learn from decisions + graph_expansion=True, # Find related information + advanced_analytics=True, # Understand patterns + kg_algorithms=True, # Advanced analysis + vector_store_features=True +) + +# Get insights about your agent's learning +insights = smart_agent.get_context_insights() +print(f"Total decisions learned: {insights.get('total_decisions', 0)}") +print(f"Decision categories: {list(insights.get('categories', {}).keys())}") +print(f"Most common outcome: {insights.get('most_common_outcome', 'N/A')}") +``` + +--- + +## 🏗️ ContextGraph - Knowledge Organization + +When you need to organize complex information, ContextGraph helps you build knowledge networks. + +### Build a Simple Knowledge Graph +```python +from semantica.context import ContextGraph + +# Create a knowledge graph +knowledge = ContextGraph(advanced_analytics=True) + +# Add things you want to remember (nodes) +knowledge.add_node("Python", "language", properties={"popularity": "high"}) +knowledge.add_node("Programming", "concept", properties={"type": "skill"}) +knowledge.add_node("FastAPI", "framework", properties={"language": "Python"}) +knowledge.add_node("Web Development", "field", properties={"complexity": "medium"}) + +# Connect related things (edges) +knowledge.add_edge("Python", "Programming", "related_to") +knowledge.add_edge("Python", "FastAPI", "supports") +knowledge.add_edge("FastAPI", "Web Development", "used_for") +knowledge.add_edge("Programming", "Web Development", "requires") +``` + +### Easy Decision Management +```python +# Record decisions in your knowledge graph +from semantica.context.decision_models import Decision from datetime import datetime -# Create a policy exception -exception = PolicyException( - exception_id="exc_001", - decision_id="decision_123", - policy_id="lending_policy_v2", - reason="Customer relationship exception - long-term premium client", - approver="branch_manager_001", - approval_timestamp=datetime.now(), - justification="Customer has 10-year history with excellent payment record" +decision = Decision( + decision_id="tech_choice_001", + category="technology_choice", + scenario="Framework selection for web API", + reasoning="FastAPI provides better performance for Python APIs", + outcome="selected_fastapi", + confidence=0.92, + timestamp=datetime.now(), + decision_maker="system", + metadata={"entities": ["Python", "FastAPI", "web_project"]} +) +knowledge.add_decision(decision) + +# Or use the convenience method for quick decisions +decision_id = knowledge.add_decision_simple( + category="technology_choice", + scenario="Framework selection for web API", + reasoning="FastAPI provides better performance for Python APIs", + outcome="selected_fastapi", + confidence=0.92, + entities=["Python", "FastAPI", "web_project"] ) -# Convert to dictionary for storage -exception_dict = exception.to_dict() -print(f"Exception recorded: {exception_dict['exception_id']}") - -# Create exception from dictionary (e.g., when loading from database) -recreated_exception = PolicyException.from_dict(exception_dict) -print(f"Recreated exception: {recreated_exception.reason}") - -# Policy engine can record exceptions in GraphStore -policy_engine = PolicyEngine(graph_store) -exception_id = policy_engine.record_exception( - decision_id="decision_123", - policy_id="lending_policy_v2", - reason="Long-term customer relationship exception" +# Find similar decisions easily +similar = knowledge.find_precedents_by_scenario( + scenario="web framework", + category="technology_choice", + max_results=3 ) -print(f"Policy exception recorded: {exception_id}") + +print(f"Found {len(similar)} similar decisions") +for decision in similar: + print(f" Similar scenario: {decision.get('scenario', 'N/A')}") + print(f" Outcome: {decision.get('outcome', 'N/A')}") ``` -## Hybrid Search for Decisions - -The context retriever supports hybrid search combining semantic and structural embeddings. - -### Finding Similar Decisions - +### Understand Decision Impact ```python -from semantica.context import ContextRetriever +# See how decisions affect other decisions +impact = knowledge.analyze_decision_impact(decision_id) +print(f"This decision influenced {impact.get('total_influenced', 0)} other decisions") -# Initialize retriever with decision context -retriever = ContextRetriever( - vector_store=vector_store, - knowledge_graph=None -) +# Get a summary of all decisions +summary = knowledge.get_decision_summary() +print(f"Total decisions: {summary.get('total_decisions', 0)}") +print(f"Categories: {list(summary.get('categories', {}).keys())}") -# Find similar decisions using hybrid search -precedents = decision_context.find_similar_decisions( - scenario="Credit limit increase for good customer", - limit=5, - use_hybrid_search=True, - semantic_weight=0.7, - structural_weight=0.3 -) - -for precedent in precedents: - print(f"Score: {precedent['score']:.3f}") - print(f"Content: {precedent['content'][:100]}...") - print(f"Entities: {precedent['related_entities']}") +# Trace decision chains (how decisions connect) +chains = knowledge.trace_decision_chain(decision_id) +print(f"Decision chain has {len(chains)} connections") ``` -### Decision Precedent Search - +### Smart Decision Checking ```python -# Search for decision precedents -precedents = retriever.retrieve_decision_precedents( - query="Credit approval for premium customers", - limit=10, - use_hybrid_search=True, - include_context=True -) - -print(f"Found {len(precedents)} precedents") - -for precedent in precedents: - print(f"Scenario: {precedent['scenario']}") - print(f"Outcome: {precedent['outcome']}") - print(f"Confidence: {precedent['confidence']}") -``` - -### Query Decisions with Context - -```python -# Query decisions with multi-hop context expansion -queried = retriever.query_decisions( - query="High-risk credit decisions", - max_hops=2, - include_context=True, - use_hybrid_search=True, - filters={"category": "credit_approval", "risk_level": "high"} -) - -print(f"Found {len(queried)} high-risk decisions") -``` - -## Explainable AI - -The decision tracking system provides comprehensive explanations with path tracing and confidence scoring. - -### Decision Explanations - -```python -# Generate comprehensive decision explanation -explanation = decision_context.explain_decision( - decision_id, - include_paths=True, - include_confidence=True, - include_weights=True -) - -print(f"Scenario: {explanation['scenario']}") -print(f"Reasoning: {explanation['reasoning']}") -print(f"Outcome: {explanation['outcome']}") -print(f"Confidence: {explanation['confidence']}") - -# Available explanation components -components = [ - "scenario", "reasoning", "outcome", "confidence", - "semantic_weight", "structural_weight", "embedding_info", - "related_entities", "similar_decisions", "path_tracing" -] - -for component in components: - if component in explanation: - print(f"{component}: {explanation[component]}") -``` - -### Path Tracing and Context - -```python -# Get decision with path tracing -explanation = decision_context.explain_decision( - decision_id, - include_paths=True, - max_depth=3 -) - -# Trace decision paths -if "path_tracing" in explanation: - paths = explanation["path_tracing"] - for path in paths: - print(f"Path: {' -> '.join(path['entities'])}") - print(f"Confidence: {path['confidence']}") - print(f"Relationships: {path['relationships']}") -``` - -### Confidence and Weight Analysis - -```python -# Analyze decision confidence and weights -explanation = decision_context.explain_decision( - decision_id, - include_confidence=True, - include_weights=True -) - -print(f"Decision confidence: {explanation['confidence']}") -print(f"Semantic weight: {explanation['semantic_weight']}") -print(f"Structural weight: {explanation['structural_weight']}") - -# Check if structural embedding was used -if "has_structural_embedding" in explanation: - has_structural = explanation["has_structural_embedding"] - print(f"Structural embedding used: {has_structural}") -``` - -### Real-World Examples - -```python -# Banking decision example -banking_decision = decision_context.record_decision( - scenario="Mortgage application approval", - reasoning="Strong credit score (750), stable employment, 20% down payment", - outcome="approved", - confidence=0.94, - entities=["applicant_001", "mortgage_30yr", "property_main"], - category="mortgage_approval", - loan_amount=350000, - credit_score=750 -) - -# Get banking decision explanation -banking_explanation = decision_context.explain_decision(banking_decision) -print(f"Banking decision: {banking_explanation['outcome']}") -print(f"Risk assessment: {banking_explanation['confidence']}") - -# Insurance decision example -insurance_decision = decision_context.record_decision( - scenario="Auto insurance claim approval", - reasoning="Clear liability, reasonable repair costs, no prior claims", - outcome="approved", - confidence=0.96, - entities=["claim_auto_001", "driver_safe", "policy_active"], - category="auto_insurance", - claim_amount=2500 -) - -# Find similar insurance decisions -insurance_precedents = decision_context.find_similar_decisions( - scenario="Auto claim with clear liability", - limit=5, - filters={"category": "auto_insurance"} -) - -print(f"Found {len(insurance_precedents)} similar insurance claims") -``` - -## Context Graphs with KG Algorithms - -The context module now integrates with `semantica.kg` algorithms to provide advanced graph analytics, centrality measures, community detection, and node embeddings for comprehensive context graph analysis. - -### Initializing Context Graph with KG Features - -```python -from semantica.context import ContextGraph -from semantica.graph_store import GraphStore - -# Context graph with KG algorithms -graph = ContextGraph( - enable_advanced_analytics=True, # Enable KG algorithms - enable_centrality_analysis=True, # Enable centrality measures - enable_community_detection=True, # Enable community detection - enable_node_embeddings=True # Enable Node2Vec embeddings -) - -print(f"KG components initialized: {len(graph.kg_components)}") -``` - -### Graph Analytics with KG Algorithms - -```python -# Comprehensive graph analysis -analysis = graph.analyze_graph_with_kg() - -print(f"Graph metrics:") -print(f" - Node count: {analysis['graph_metrics']['node_count']}") -print(f" - Edge count: {analysis['graph_metrics']['edge_count']}") -print(f" - Node types: {analysis['graph_metrics']['node_types']}") - -# Centrality analysis -if 'centrality_analysis' in analysis: - centrality = analysis['centrality_analysis'] - print(f" - Centrality measures available for {len(centrality)} nodes") - -# Community detection -if 'community_analysis' in analysis: - communities = analysis['community_analysis'] - print(f" - Found {communities['num_communities']} communities") - print(f" - Modularity: {communities['modularity']:.3f}") - -# Node embeddings -if 'node_embeddings' in analysis: - embeddings = analysis['node_embeddings'] - print(f" - Generated embeddings for {len(embeddings)} nodes") -``` - -### Node Centrality Analysis - -```python -# Get centrality measures for a specific node -node_id = "python_programming" -centrality_measures = graph.get_node_centrality(node_id) - -print(f"Centrality measures for {node_id}:") -print(f" - Degree centrality: {centrality_measures.get('degree_centrality', 0):.3f}") -print(f" - Betweenness centrality: {centrality_measures.get('betweenness_centrality', 0):.3f}") -print(f" - Closeness centrality: {centrality_measures.get('closeness_centrality', 0):.3f}") -print(f" - Eigenvector centrality: {centrality_measures.get('eigenvector_centrality', 0):.3f}") -``` - -### Finding Similar Nodes with Advanced Similarity - -```python -# Find similar nodes using different similarity measures -similar_nodes = graph.find_similar_nodes( - node_id="python_programming", - similarity_type="content", # "content", "structural", "embedding" - top_k=10 -) - -print(f"Similar nodes to 'python_programming':") -for node_id, similarity_score in similar_nodes: - print(f" - {node_id}: {similarity_score:.3f}") - -# Structural similarity -structural_similar = graph.find_similar_nodes( - node_id="python_programming", - similarity_type="structural", - top_k=5 -) -``` - -### AgentContext with KG Features - -```python -from semantica.context import AgentContext -from semantica.vector_store import VectorStore -from semantica.graph_store import GraphStore - -# Initialize AgentContext with all KG features -vector_store = VectorStore(backend="inmemory", dimension=384) -knowledge_graph = GraphStore(backend="neo4j", uri="bolt://localhost:7687") - -context = AgentContext( - vector_store=vector_store, - knowledge_graph=knowledge_graph, - enable_decision_tracking=True, - enable_advanced_analytics=True, # Enable KG algorithms - enable_kg_algorithms=True, # Enable KG integration - enable_vector_store_features=True # Enable vector store features -) - -print("AgentContext initialized with KG algorithms") -``` - -## Advanced Decision Analytics - -The decision tracking system provides advanced analytics using KG algorithms for decision influence analysis, relationship prediction, and comprehensive insights. - -### DecisionQuery with KG Integration - -```python -from semantica.context import DecisionQuery - -# Decision query with KG algorithms -query = DecisionQuery( - graph_store=knowledge_graph, - vector_store=vector_store, - enable_advanced_analytics=True, - enable_centrality_analysis=True, - enable_community_detection=True, - enable_node_embeddings=True -) - -print(f"DecisionQuery with {len(query.kg_components)} KG components") -``` - -### Advanced Precedent Search with Custom Weights - -```python -# Find precedents using advanced search with custom similarity weights -precedents = context.find_precedents_advanced( - scenario="Credit limit increase for premium customer", - category="credit_approval", - limit=10, - use_kg_features=True, - similarity_weights={ - "semantic": 0.4, # Vector similarity - "structural": 0.3, # Graph structure similarity - "text": 0.2, # Text overlap - "category": 0.1 # Category matching - } -) - -print(f"Found {len(precedents)} precedents with advanced search") -for precedent in precedents: - print(f" - Score: {precedent.metadata.get('similarity_score', 0):.3f}") - print(f" - Scenario: {precedent.scenario[:100]}...") -``` - -### Decision Influence Analysis - -```python -# Analyze decision influence using KG algorithms -decision_id = "decision_123" -influence_analysis = context.analyze_decision_influence( - decision_id=decision_id, - max_depth=3 -) - -print(f"Decision influence analysis for {decision_id}:") -print(f" - Influence score: {influence_analysis.get('influence_score', 0):.3f}") - -# Centrality measures -centrality = influence_analysis.get('centrality_measures', {}) -print(f" - Degree centrality: {centrality.get('degree_centrality', 0):.3f}") -print(f" - Betweenness centrality: {centrality.get('betweenness_centrality', 0):.3f}") - -# Community information -community = influence_analysis.get('community_info', {}) -if community: - print(f" - Community ID: {community.get('community_id')}") - print(f" - Community size: {community.get('community_size')}") - -# Related decisions -downstream = influence_analysis.get('downstream_decisions', []) -upstream = influence_analysis.get('upstream_decisions', []) -print(f" - Downstream decisions: {len(downstream)}") -print(f" - Upstream decisions: {len(upstream)}") -``` - -### Decision Relationship Prediction - -```python -# Predict potential relationships for decisions -predictions = context.predict_decision_relationships( - decision_id="decision_123", - top_k=5 -) - -print(f"Predicted relationships for decision_123:") -for prediction in predictions: - print(f" - Target: {prediction.get('target', 'unknown')}") - print(f" - Score: {prediction.get('score', 0):.3f}") - print(f" - Type: {prediction.get('type', 'unknown')}") -``` - -### Context Graph Analysis - -```python -# Analyze the entire context graph -graph_analysis = context.analyze_context_graph() - -print("Context graph analysis:") -if 'error' not in graph_analysis: - metrics = graph_analysis.get('graph_metrics', {}) - print(f" - Nodes: {metrics.get('node_count', 0)}") - print(f" - Edges: {metrics.get('edge_count', 0)}") - - centrality = graph_analysis.get('centrality_analysis', {}) - print(f" - Centrality analysis: {len(centrality)} nodes analyzed") - - communities = graph_analysis.get('community_analysis', {}) - print(f" - Communities: {communities.get('num_communities', 0)}") +# Check if decisions follow your rules +compliance = knowledge.check_decision_rules({ + "category": "loan_approval", + "scenario": "Mortgage application", + "reasoning": "Good credit score, stable income", + "outcome": "approved", + "confidence": 0.95 +}) + +if compliance.get("compliant", False): + print("✅ Decision follows all rules") else: - print(f" - Error: {graph_analysis['error']}") + print(f"❌ Rule violations: {compliance.get('violations', [])}") ``` -### Entity Similarity and Centrality - +### Graph Analytics Made Simple ```python -# Find similar entities in the context graph -similar_entities = context.find_similar_entities( - entity_id="python_programming", - similarity_type="content", - top_k=10 -) +# Get overview of your knowledge graph +summary = knowledge.get_graph_summary() +print(f"Knowledge graph has {summary.get('nodes', 0)} concepts") +print(f"And {summary.get('edges', 0)} relationships") -print(f"Similar entities to 'python_programming':") -for entity_id, similarity_score in similar_entities: - print(f" - {entity_id}: {similarity_score:.3f}") +# Find related concepts +related = knowledge.find_related_nodes("Python", how_many=5) +for concept_id, similarity in related: + print(f"Related to {concept_id}: {similarity:.2f}") -# Get entity centrality measures -entity_centrality = context.get_entity_centrality("python_programming") -print(f"Entity centrality: {entity_centrality}") +# Understand which concepts are most important +importance = knowledge.get_node_importance("Python") +print(f"Python importance score: {importance.get('degree', 0)}") ``` -### Comprehensive Context Insights +--- +## 🔄 Using Both Together - The Complete Setup + +### Your Smart Agent System ```python -# Get comprehensive insights about the context -insights = context.get_context_insights() +from semantica.context import AgentContext, ContextGraph +from semantica.vector_store import VectorStore -print("Context insights:") -print(f" - Timestamp: {insights.get('timestamp')}") +# Create the components +vector_store = VectorStore(backend="inmemory", dimension=384) +knowledge = ContextGraph(advanced_analytics=True) -# Memory statistics -memory_stats = insights.get('memory_stats', {}) -print(f" - Total memories: {memory_stats.get('total_items', 0)}") -print(f" - Memory usage: {memory_stats.get('memory_usage', {})}") +# Create your intelligent agent +agent = AgentContext( + vector_store=vector_store, + knowledge_graph=knowledge, # Add knowledge graph + decision_tracking=True, + graph_expansion=True, + advanced_analytics=True +) -# Decision statistics -decision_stats = insights.get('decision_stats', {}) -if decision_stats: - print(f" - Total decisions: {decision_stats.get('total_decisions', 0)}") - print(f" - Decision categories: {decision_stats.get('categories', [])}") +# Your agent works like this: +# 1. Store information in memory +agent.store("User wants to learn web development with Python") +agent.store("User is a beginner programmer") +agent.store("User prefers hands-on tutorials") -# Advanced features status -features = insights.get('advanced_features', {}) -print(f" - KG algorithms enabled: {features.get('kg_algorithms_enabled', False)}") -print(f" - Vector store features enabled: {features.get('vector_store_features_enabled', False)}") -print(f" - Decision tracking enabled: {features.get('decision_tracking_enabled', False)}") +# 2. Find relevant information +results = agent.retrieve("Python web development tutorials") +print(f"Found {len(results)} relevant memories") + +# 3. Make smart decisions +decision_id = agent.record_decision( + category="content_recommendation", + scenario="Python web development learning path", + reasoning="Beginner needs hands-on Python web tutorial", + outcome="recommended_flask_tutorial", + confidence=0.89 +) + +# 4. Learn and improve over time +insights = agent.get_context_insights() +print(f"Agent insights: {insights}") + +# 5. Access advanced features when needed +graph_summary = agent.graph_builder.get_graph_summary() +node_importance = agent.graph_builder.get_node_importance("Python") ``` -## Production Examples +--- -### Banking Decision System with KG Analytics +## 🎯 Real-World Examples +### 🏦 Banking - Smart Loan Decisions ```python -# Initialize banking decision system -banking_context = AgentContext( - vector_store=VectorStore(backend="faiss", dimension=768), - knowledge_graph=GraphStore(backend="neo4j", uri="bolt://localhost:7687"), - enable_decision_tracking=True, - enable_advanced_analytics=True, - enable_kg_algorithms=True, - enable_vector_store_features=True +# Track loan decisions and learn from patterns +bank_agent = AgentContext(vector_store=bank_vector_store, decision_tracking=True) + +# Store customer information +bank_agent.store("Customer has credit score 750, stable employment") +bank_agent.store("Customer is first-time homebuyer") + +# Make loan decision +loan_decision = bank_agent.record_decision( + category="loan_approval", + scenario="First-time homebuyer mortgage", + reasoning="Good credit score, stable income, 20% down payment", + outcome="approved", + confidence=0.94 ) -# Record banking decisions with full context -decisions = [ - { - "category": "mortgage_approval", - "scenario": "Mortgage application for first-time homebuyer", - "reasoning": "Strong credit score (750), stable employment, 20% down payment", - "outcome": "approved", - "confidence": 0.94, - "decision_maker": "loan_officer_001", - "amount": 350000, - "credit_score": 750, - "risk_level": "low" - }, - { - "category": "credit_card_approval", - "scenario": "Premium credit card application", - "reasoning": "Excellent credit history, high income, existing relationship", - "outcome": "approved", - "confidence": 0.96, - "decision_maker": "credit_analyst_002", - "credit_limit": 25000, - "credit_score": 820, - "risk_level": "very_low" - } -] - -# Process decisions -decision_ids = [] -for decision_data in decisions: - decision_id = banking_context.record_decision(**decision_data) - decision_ids.append(decision_id) - -# Analyze decision influence -for decision_id in decision_ids: - influence = banking_context.analyze_decision_influence(decision_id) - print(f"Decision {decision_id} influence: {influence.get('influence_score', 0):.3f}") - -# Find similar decisions with KG features -similar_decisions = banking_context.find_precedents_advanced( - scenario="High-value credit application", - category="credit_approval", - use_kg_features=True, - similarity_weights={"semantic": 0.5, "structural": 0.3, "category": 0.2} -) - -print(f"Found {len(similar_decisions)} similar decisions with KG analysis") - -# Get comprehensive insights -insights = banking_context.get_context_insights() -print(f"Banking system insights: {insights.get('memory_stats', {})}") +# Find similar loan decisions for consistency +similar_loans = bank_agent.find_precedents("homebuyer", category="loan_approval") +print(f"Found {len(similar_loans)} similar loan decisions") ``` -### Healthcare Decision Support System - +### 🏥 Healthcare - Patient Care Decisions ```python -# Healthcare decision system with advanced analytics -healthcare_context = AgentContext( - vector_store=VectorStore(backend="chroma", dimension=1536), - knowledge_graph=GraphStore(backend="neo4j"), - enable_decision_tracking=True, - enable_advanced_analytics=True, - enable_kg_algorithms=True +# Track patient care decisions +health_agent = AgentContext(vector_store=medical_vector_store, decision_tracking=True) + +# Store patient information +health_agent.store("Patient has hypertension, type 2 diabetes") +health_agent.store("Patient allergic to penicillin") + +# Make treatment decision +treatment_decision = health_agent.record_decision( + category="treatment_plan", + scenario="Hypertension with diabetes", + reasoning="ACE inhibitors safe for diabetic patients", + outcome="prescribed_ace_inhibitor", + confidence=0.91 ) -# Record medical decisions -medical_decisions = [ - { - "category": "treatment_approval", - "scenario": "Approval for experimental cancer treatment", - "reasoning": "Patient meets criteria, no alternative treatments available", - "outcome": "approved", - "confidence": 0.88, - "decision_maker": "dr_smith", - "patient_id": "patient_123", - "condition": "stage_4_lung_cancer", - "treatment_type": "immunotherapy" - }, - { - "category": "diagnostic_test", - "scenario": "MRI scan authorization", - "reasoning": "Symptoms indicate need for detailed imaging", - "outcome": "approved", - "confidence": 0.92, - "decision_maker": "dr_jones", - "patient_id": "patient_456", - "test_type": "brain_mri", - "urgency": "medium" - } -] - -# Process medical decisions -for decision in medical_decisions: - decision_id = healthcare_context.record_decision(**decision) - - # Analyze decision influence in medical context - influence = healthcare_context.analyze_decision_influence(decision_id) - print(f"Medical decision influence: {influence.get('influence_score', 0):.3f}") - -# Find similar treatment decisions -similar_treatments = healthcare_context.find_precedents_advanced( - scenario="Cancer treatment approval", - category="treatment_approval", - use_kg_features=True -) - -print(f"Found {len(similar_treatments)} similar treatment decisions") - -# Analyze healthcare context graph -graph_analysis = healthcare_context.analyze_context_graph() -if 'error' not in graph_analysis: - print(f"Healthcare graph: {graph_analysis.get('graph_metrics', {})}") +# Find similar treatment cases +similar_cases = health_agent.find_precedents("hypertension", category="treatment_plan") ``` -### E-commerce Personalization System - +### 🛒 E-commerce - Smart Recommendations ```python -# E-commerce system with KG recommendations -ecommerce_context = AgentContext( - vector_store=VectorStore(backend="qdrant", dimension=1024), - knowledge_graph=GraphStore(backend="neo4j"), - enable_decision_tracking=True, - enable_advanced_analytics=True, - enable_kg_algorithms=True +# Track recommendation decisions +ecommerce_graph = ContextGraph() + +# Build user-product knowledge +ecommerce_graph.add_node("user_123", "user", {"segment": "premium"}) +ecommerce_graph.add_node("laptop_xyz", "product", {"category": "electronics"}) +ecommerce_graph.add_edge("user_123", "laptop_xyz", "viewed") + +# Make recommendation decision +from semantica.context.decision_models import Decision +from datetime import datetime + +rec_decision = Decision( + decision_id="rec_001", + category="product_recommendation", + scenario="Laptop recommendation for premium user", + reasoning="User prefers high-performance electronics", + outcome="recommended_gaming_laptop", + confidence=0.87, + timestamp=datetime.now(), + decision_maker="recommendation_system", + metadata={"entities": ["user_123", "laptop_xyz"]} +) +ecommerce_graph.add_decision(rec_decision) + +# Or use the convenience method +rec_decision_id = ecommerce_graph.add_decision_simple( + category="product_recommendation", + scenario="Laptop recommendation for premium user", + reasoning="User prefers high-performance electronics", + outcome="recommended_gaming_laptop", + confidence=0.87, + entities=["user_123", "laptop_xyz"] ) -# Record personalization decisions -personalization_decisions = [ - { - "category": "product_recommendation", - "scenario": "Premium product recommendation for VIP customer", - "reasoning": "High purchase history, premium segment, similar preferences", - "outcome": "recommended", - "confidence": 0.91, - "decision_maker": "recommendation_engine", - "customer_id": "vip_customer_001", - "product_category": "luxury_goods", - "price_range": "high" - }, - { - "category": "pricing_decision", - "scenario": "Dynamic pricing adjustment", - "reasoning": "High demand, low inventory, competitor pricing", - "outcome": "price_increased", - "confidence": 0.87, - "decision_maker": "pricing_algorithm", - "product_id": "product_789", - "original_price": 99.99, - "new_price": 119.99 - } -] - -# Process e-commerce decisions -for decision in personalization_decisions: - decision_id = ecommerce_context.record_decision(**decision) - - # Find similar customers using KG features - similar_customers = ecommerce_context.find_similar_entities( - entity_id=decision.get('customer_id', ''), - similarity_type="structural", - top_k=5 - ) - print(f"Similar customers: {len(similar_customers)}") - -# Get comprehensive e-commerce insights -insights = ecommerce_context.get_context_insights() -print(f"E-commerce system status: {insights.get('advanced_features', {})}") +# Find similar recommendations +similar_recs = ecommerce_graph.find_precedents_by_scenario( + scenario="laptop recommendation", + limit=5 +) ``` -### Backward Compatibility Examples +--- -All existing code continues to work without changes: +## 💡 Pro Tips for Success +### 🌱 For Beginners +1. **Start with AgentContext** - It's simpler and handles most needs +2. **Use basic store/retrieve** - Like building human memory +3. **Add decision tracking** - Your agent gets smarter over time +4. **Enable features gradually** - Add complexity as you need it + +### 🚀 For Advanced Users +1. **Add ContextGraph** - When you need knowledge relationships +2. **Use analytics** - Understand patterns and get insights +3. **Implement policies** - Ensure consistent decisions +4. **Use persistence** - Save and load agent state + +### 🏭 For Production +1. **Enable all features** - Maximum intelligence and reliability +2. **Use save/load** - Persist agent state between sessions +3. **Monitor performance** - Use health checks and insights +4. **Test thoroughly** - Verify all functionality works + +--- + +## 🔧 Configuration Options + +### Simple Setup (Most Common) ```python -# Old API still works perfectly -context = AgentContext(vector_store=vector_store) -query = DecisionQuery(graph_store) +# Just memory and basic learning +agent = AgentContext(vector_store=vector_store) +``` + +### Smart Setup (Recommended) +```python +# Memory + decision learning +agent = AgentContext( + vector_store=vector_store, + decision_tracking=True, + graph_expansion=True +) +``` + +### Complete Setup (Maximum Power) +```python +# Everything enabled +agent = AgentContext( + vector_store=vector_store, + knowledge_graph=ContextGraph(advanced_analytics=True), + decision_tracking=True, + graph_expansion=True, + advanced_analytics=True, + kg_algorithms=True, + vector_store_features=True +) +``` + +### ContextGraph Options +```python +# Basic knowledge graph graph = ContextGraph() -# Store and retrieve as before -memory_id = context.store("User message", conversation_id="conv1") -results = context.retrieve("User query") - -# Decision tracking with old API -if hasattr(context, 'record_decision'): - decision_id = context.record_decision( - category="test", - scenario="Test scenario", - reasoning="Test reasoning", - outcome="approved", - confidence=0.8 - ) +# Advanced knowledge graph +graph = ContextGraph( + advanced_analytics=True, # Enable smart algorithms + centrality_analysis=True, # Find important concepts + community_detection=True, # Find groups of related concepts + node_embeddings=True # Understand concept similarity +) ``` -## Summary +--- -The context module provides: +## 🎉 You're Ready to Build Smart Agents! -- **Backward Compatibility**: All existing code works unchanged -- **KG Algorithm Integration**: Advanced graph analytics with centrality, community detection, embeddings -- **Vector Store Features**: Hybrid search combining semantic and structural similarity -- **Advanced Decision Analytics**: Influence analysis, relationship prediction, comprehensive insights -- **Production Ready**: Scalable architecture for real-world applications +With these examples, you can now: -Users can now build truly comprehensive context graphs with semantica, leveraging all advanced KG algorithms and vector store features while maintaining complete backward compatibility! +✅ **Build Smart Agents** - That remember and learn from experience +✅ **Track Decisions** - Make consistent, improving choices over time +✅ **Find Information** - Quick and relevant memory retrieval +✅ **Organize Knowledge** - Build intelligent knowledge graphs +✅ **Make Better Decisions** - Based on past experience and patterns +✅ **Build Real Applications** - Banking, healthcare, e-commerce, and more + +**Start simple, add power as needed! Your agents will get smarter with every decision.** 🚀 + +--- + +## 📚 Need More Help? + +- **Start with AgentContext** for most applications +- **Add ContextGraph** when you need knowledge organization +- **Look at the real-world examples** for your specific use case +- **Check configuration options** to customize your agent + +Happy building smart agents! 🎯 diff --git a/semantica/context/decision_methods.py b/semantica/context/decision_methods.py index 0eae2d40..6bc76aa3 100644 --- a/semantica/context/decision_methods.py +++ b/semantica/context/decision_methods.py @@ -6,6 +6,8 @@ offering simple interfaces for common use cases. """ from datetime import datetime +import hashlib +import json from typing import Any, Dict, List, Optional, Union from ..graph_store import GraphStore @@ -215,7 +217,15 @@ def multi_hop_query( def capture_decision_trace( decision: Decision, - cross_system_context: Dict[str, Any] + cross_system_context: Dict[str, Any], + graph_store: Optional[GraphStore] = None, + entities: Optional[List[str]] = None, + source_documents: Optional[List[str]] = None, + policy_ids: Optional[Union[str, Dict[str, str], List[Union[str, Dict[str, str]]]]] = None, + exceptions: Optional[List[Dict[str, Any]]] = None, + approvals: Optional[List[Dict[str, Any]]] = None, + precedents: Optional[List[Dict[str, str]]] = None, + immutable_audit_log: bool = True, ) -> str: """ Complete decision trace capture. @@ -223,6 +233,16 @@ def capture_decision_trace( Args: decision: Decision object to capture cross_system_context: Cross-system context + graph_store: Optional graph store used to persist full trace + entities: Optional list of linked entities + source_documents: Optional list of source documents + policy_ids: Optional list of policy refs. Supports: + - "policy_id" + - {"policy_id": "...", "version": "..."} + exceptions: Optional list of exception records + approvals: Optional list of approval records + precedents: Optional list of precedent links + immutable_audit_log: Whether to append immutable hash-chained trace events Returns: Decision ID @@ -230,16 +250,324 @@ def capture_decision_trace( logger = get_logger(__name__) try: - # This would typically use a global or context-specific graph store - # For now, return the decision ID as a placeholder - logger.info(f"Captured decision trace for: {decision.decision_id}") - return decision.decision_id + # Backward-compatible behavior: allow legacy call sites without graph_store. + if graph_store is None: + policy_refs = _normalize_policy_refs(policy_ids) + logger.warning( + "capture_decision_trace skipped persistence (no graph_store) | " + f"decision_id={decision.decision_id} " + f"decision_maker={decision.decision_maker} " + f"timestamp={decision.timestamp.isoformat() if hasattr(decision.timestamp, 'isoformat') else decision.timestamp} " + f"category={decision.category} " + f"outcome={decision.outcome} " + f"confidence={decision.confidence} " + f"cross_system_keys={list((cross_system_context or {}).keys())} " + f"policy_refs={policy_refs} " + f"exception_count={len(_normalize_record_list(exceptions))} " + f"approval_count={len(_normalize_record_list(approvals))} " + f"precedent_count={len(_normalize_precedents(precedents))} " + "mode=backward_compatible_non_persistent" + ) + return decision.decision_id + + recorder = DecisionRecorder(graph_store) + entities = _normalize_string_list(entities) + source_documents = _normalize_string_list(source_documents) + policy_refs = _normalize_policy_refs(policy_ids) + exceptions = _normalize_record_list(exceptions) + approvals = _normalize_record_list(approvals) + precedents = _normalize_precedents(precedents) + + decision_id = recorder.record_decision( + decision=decision, + entities=entities, + source_documents=source_documents, + ) + + trace_events: List[Dict[str, Any]] = [ + { + "event_type": "DECISION_RECORDED", + "payload": { + "decision_id": decision_id, + "category": decision.category, + "outcome": decision.outcome, + "confidence": decision.confidence, + "decision_maker": decision.decision_maker, + "entities": entities, + "source_documents": source_documents, + }, + } + ] + + if cross_system_context: + recorder.capture_cross_system_context(decision_id, cross_system_context) + trace_events.append( + { + "event_type": "CROSS_SYSTEM_CONTEXT_CAPTURED", + "payload": {"systems": list(cross_system_context.keys())}, + } + ) + + if policy_refs: + applied_policies = recorder.apply_policies(decision_id, policy_refs) + trace_events.append( + { + "event_type": "POLICIES_APPLIED", + "payload": { + "policy_ids": [p.get("policy_id") for p in policy_refs], + "applied_policies": applied_policies, + }, + } + ) + + if exceptions: + recorded_exception_ids: List[str] = [] + for exception_data in exceptions: + exception_id = recorder.record_exception( + decision_id=decision_id, + policy_id=exception_data.get("policy_id", ""), + reason=exception_data.get("reason", ""), + approver=exception_data.get("approver", "system"), + approval_method=exception_data.get("approval_method", "system"), + justification=exception_data.get("justification", ""), + ) + recorded_exception_ids.append(exception_id) + if recorded_exception_ids: + trace_events.append( + { + "event_type": "EXCEPTIONS_RECORDED", + "payload": {"exception_ids": recorded_exception_ids}, + } + ) + + if approvals: + approvers = [a.get("approver", "system") for a in approvals] + methods = [a.get("approval_method", "system") for a in approvals] + contexts = [a.get("approval_context", "") for a in approvals] + if approvers: + recorder.record_approval_chain( + decision_id=decision_id, + approvers=approvers, + methods=methods, + contexts=contexts, + ) + trace_events.append( + { + "event_type": "APPROVAL_CHAIN_RECORDED", + "payload": {"approvers": approvers, "methods": methods}, + } + ) + + if precedents: + precedent_ids = [p.get("precedent_id", "") for p in precedents if p.get("precedent_id")] + relationship_types = [ + p.get("relationship_type", "similar_scenario") + for p in precedents + if p.get("precedent_id") + ] + if precedent_ids: + recorder.link_precedents(decision_id, precedent_ids, relationship_types) + trace_events.append( + { + "event_type": "PRECEDENTS_LINKED", + "payload": {"precedent_ids": precedent_ids}, + } + ) + + if immutable_audit_log: + _append_immutable_trace_events(graph_store, decision_id, trace_events, logger) + + logger.info(f"Captured decision trace for: {decision_id}") + return decision_id except Exception as e: logger.error(f"Failed to capture decision trace: {e}") raise +def _append_immutable_trace_events( + graph_store: GraphStore, + decision_id: str, + events: List[Dict[str, Any]], + logger: Any, +) -> None: + """Append hash-chained trace events for immutable decision lineage.""" + if not events: + return + + previous_trace_id: Optional[str] = None + previous_hash = "" + next_index = 1 + + try: + previous_result = graph_store.execute_query( + """ + MATCH (d:Decision {decision_id: $decision_id})-[:HAS_TRACE_EVENT]->(t:DecisionTraceEvent) + RETURN t.trace_id as trace_id, t.event_index as event_index, t.event_hash as event_hash + ORDER BY t.event_index DESC + LIMIT 1 + """, + {"decision_id": decision_id}, + ) + records = previous_result.get("records", []) if isinstance(previous_result, dict) else previous_result + if records: + latest = records[0] + latest_map = latest.get("t", latest) if isinstance(latest, dict) else {} + previous_trace_id = latest_map.get("trace_id") + previous_hash = latest_map.get("event_hash", "") or "" + next_index = int(latest_map.get("event_index", 0) or 0) + 1 + except Exception as e: + logger.warning( + "Failed to lookup previous immutable trace event; starting new chain " + f"for decision_id={decision_id}: {e}" + ) + # Start a fresh chain if previous trace lookup fails. + previous_trace_id = None + previous_hash = "" + next_index = 1 + + for event in events: + event_type = event.get("event_type", "TRACE_EVENT") + payload = event.get("payload", {}) + payload_json = json.dumps(payload, sort_keys=True, default=str) + event_timestamp = datetime.now().isoformat() + trace_id = f"{decision_id}:{next_index}" + hash_input = ( + f"{decision_id}|{next_index}|{event_type}|{event_timestamp}|{payload_json}|{previous_hash}" + ) + event_hash = hashlib.sha256(hash_input.encode("utf-8")).hexdigest() + + graph_store.execute_query( + """ + MATCH (d:Decision {decision_id: $decision_id}) + CREATE (t:DecisionTraceEvent { + trace_id: $trace_id, + decision_id: $decision_id, + event_index: $event_index, + event_type: $event_type, + event_timestamp: $event_timestamp, + event_payload: $event_payload, + previous_hash: $previous_hash, + event_hash: $event_hash + }) + MERGE (d)-[:HAS_TRACE_EVENT]->(t) + """, + { + "decision_id": decision_id, + "trace_id": trace_id, + "event_index": next_index, + "event_type": event_type, + "event_timestamp": event_timestamp, + "event_payload": payload_json, + "previous_hash": previous_hash, + "event_hash": event_hash, + }, + ) + + if previous_trace_id: + graph_store.execute_query( + """ + MATCH (prev:DecisionTraceEvent {trace_id: $prev_trace_id}) + MATCH (curr:DecisionTraceEvent {trace_id: $curr_trace_id}) + MERGE (prev)-[:NEXT_TRACE_EVENT]->(curr) + """, + {"prev_trace_id": previous_trace_id, "curr_trace_id": trace_id}, + ) + + previous_trace_id = trace_id + previous_hash = event_hash + next_index += 1 + + logger.debug(f"Appended {len(events)} immutable trace events for {decision_id}") + + +def _normalize_string_list(value: Optional[Union[str, List[str]]]) -> List[str]: + """Normalize optional string/list payloads to a clean list of strings.""" + if value is None: + return [] + if isinstance(value, str): + return [value] if value else [] + if isinstance(value, list): + return [str(item) for item in value if item is not None and str(item)] + return [] + + +def _normalize_record_list( + value: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] +) -> List[Dict[str, Any]]: + """Normalize optional dict/list payloads to list[dict] for legacy callers.""" + if value is None: + return [] + if isinstance(value, dict): + return [value] + if isinstance(value, list): + return [item for item in value if isinstance(item, dict)] + return [] + + +def _normalize_policy_refs( + value: Optional[Union[str, Dict[str, str], List[Union[str, Dict[str, str]]]]] +) -> List[Dict[str, str]]: + """Normalize policy refs to [{policy_id, version?}] for version-safe matching.""" + if value is None: + return [] + + raw_items: List[Union[str, Dict[str, str]]] + if isinstance(value, (str, dict)): + raw_items = [value] + elif isinstance(value, list): + raw_items = value + else: + return [] + + normalized: List[Dict[str, str]] = [] + for item in raw_items: + if isinstance(item, str) and item: + normalized.append({"policy_id": item}) + elif isinstance(item, dict): + policy_id = item.get("policy_id") + if not policy_id: + continue + ref: Dict[str, str] = {"policy_id": str(policy_id)} + if item.get("version") is not None and str(item.get("version")): + ref["version"] = str(item.get("version")) + normalized.append(ref) + return normalized + + +def _normalize_precedents( + value: Optional[Union[str, Dict[str, str], List[Union[str, Dict[str, str]]]]] +) -> List[Dict[str, str]]: + """Normalize precedents payload from legacy forms to structured records.""" + if value is None: + return [] + + raw_items: List[Union[str, Dict[str, str]]] + if isinstance(value, (str, dict)): + raw_items = [value] + elif isinstance(value, list): + raw_items = value + else: + return [] + + normalized: List[Dict[str, str]] = [] + for item in raw_items: + if isinstance(item, str) and item: + normalized.append( + {"precedent_id": item, "relationship_type": "similar_scenario"} + ) + elif isinstance(item, dict) and item.get("precedent_id"): + normalized.append( + { + "precedent_id": str(item.get("precedent_id")), + "relationship_type": str( + item.get("relationship_type", "similar_scenario") + ), + } + ) + return normalized + + def find_exception_precedents( graph_store: GraphStore, exception_reason: str, @@ -549,7 +877,7 @@ def enhance_agent_context_with_decisions(agent_context: AgentContext) -> None: logger = get_logger(__name__) try: - if not agent_context.config.get("enable_decision_tracking"): + if not agent_context.config.get("decision_tracking"): logger.warning("Decision tracking not enabled in AgentContext") return diff --git a/semantica/context/decision_models.py b/semantica/context/decision_models.py index 59fe6069..d068b86d 100644 --- a/semantica/context/decision_models.py +++ b/semantica/context/decision_models.py @@ -99,10 +99,12 @@ class Decision: node2vec_embedding: Optional[List[float]] = None metadata: Dict[str, Any] = field(default_factory=dict) - def __post_init__(self): + def __post_init__(self, auto_generate_id: bool = True): """Validate decision data.""" - if not self.decision_id: + if auto_generate_id and not self.decision_id: # Handle both None and empty string self.decision_id = str(uuid.uuid4()) + elif not self.decision_id and not auto_generate_id: + raise ValueError("decision_id is required when auto_generate_id=False") if not 0 <= self.confidence <= 1: raise ValueError("Confidence must be between 0 and 1") @@ -141,10 +143,12 @@ class DecisionContext: cross_system_inputs: Dict[str, Any] = field(default_factory=dict) metadata: Dict[str, Any] = field(default_factory=dict) - def __post_init__(self): - """Validate context data.""" - if not self.context_id: + def __post_init__(self, auto_generate_id: bool = True): + """Validate decision context data.""" + if auto_generate_id and not self.context_id: # Handle both None and empty string self.context_id = str(uuid.uuid4()) + elif not self.context_id and not auto_generate_id: + raise ValueError("context_id is required when auto_generate_id=False") def to_dict(self) -> Dict[str, Any]: """Convert context to dictionary.""" @@ -177,10 +181,12 @@ class Policy: updated_at: datetime metadata: Dict[str, Any] = field(default_factory=dict) - def __post_init__(self): + def __post_init__(self, auto_generate_id: bool = True): """Validate policy data.""" - if not self.policy_id: + if auto_generate_id and not self.policy_id: # Handle both None and empty string self.policy_id = str(uuid.uuid4()) + elif not self.policy_id and not auto_generate_id: + raise ValueError("policy_id is required when auto_generate_id=False") def to_dict(self) -> Dict[str, Any]: """Convert policy to dictionary.""" @@ -218,10 +224,12 @@ class PolicyException: justification: str metadata: Dict[str, Any] = field(default_factory=dict) - def __post_init__(self): - """Validate exception data.""" - if not self.exception_id: + def __post_init__(self, auto_generate_id: bool = True): + """Validate policy exception data.""" + if auto_generate_id and not self.exception_id: # Handle both None and empty string self.exception_id = str(uuid.uuid4()) + elif not self.exception_id and not auto_generate_id: + raise ValueError("exception_id is required when auto_generate_id=False") def to_dict(self) -> Dict[str, Any]: """Convert exception to dictionary.""" @@ -254,10 +262,12 @@ class Precedent: relationship_type: str # "similar_scenario", "same_policy", "exception_precedent" metadata: Dict[str, Any] = field(default_factory=dict) - def __post_init__(self): + def __post_init__(self, auto_generate_id: bool = True): """Validate precedent data.""" - if not self.precedent_id: + if auto_generate_id and not self.precedent_id: # Handle both None and empty string self.precedent_id = str(uuid.uuid4()) + elif not self.precedent_id and not auto_generate_id: + raise ValueError("precedent_id is required when auto_generate_id=False") if not 0 <= self.similarity_score <= 1: raise ValueError("Similarity score must be between 0 and 1") valid_types = ["similar_scenario", "same_policy", "exception_precedent"] @@ -292,10 +302,12 @@ class ApprovalChain: timestamp: datetime metadata: Dict[str, Any] = field(default_factory=dict) - def __post_init__(self): - """Validate approval data.""" - if not self.approval_id: + def __post_init__(self, auto_generate_id: bool = True): + """Validate approval chain data.""" + if auto_generate_id and not self.approval_id: # Handle both None and empty string self.approval_id = str(uuid.uuid4()) + elif not self.approval_id and not auto_generate_id: + raise ValueError("approval_id is required when auto_generate_id=False") valid_methods = ["slack_dm", "zoom_call", "email", "system"] if self.approval_method not in valid_methods: raise ValueError(f"Approval method must be one of: {valid_methods}") diff --git a/semantica/context/decision_query.py b/semantica/context/decision_query.py index 1a17b941..75b922fd 100644 --- a/semantica/context/decision_query.py +++ b/semantica/context/decision_query.py @@ -55,10 +55,10 @@ Search Capabilities: Example Usage: >>> from semantica.context import DecisionQuery >>> query = DecisionQuery(graph_store=kg, vector_store=vs, - ... enable_advanced_analytics=True, - ... enable_centrality_analysis=True, - ... enable_community_detection=True, - ... enable_node_embeddings=True) + ... advanced_analytics=True, + ... centrality_analysis=True, + ... community_detection=True, + ... node_embeddings=True) >>> precedents = query.find_precedents_hybrid("Loan application", ... category="approval", ... limit=10) @@ -111,11 +111,11 @@ class DecisionQuery: graph_store: GraphStore, embedding_generator: Optional[EmbeddingGenerator] = None, vector_store: Optional[Any] = None, - enable_advanced_analytics: bool = True, - enable_node_embeddings: bool = True, - enable_centrality_analysis: bool = True, - enable_community_detection: bool = True, - enable_link_prediction: bool = True + advanced_analytics: bool = True, + node_embeddings: bool = True, + centrality_analysis: bool = True, + community_detection: bool = True, + link_prediction: bool = True ): """ Initialize DecisionQuery with optional advanced features. @@ -124,11 +124,11 @@ class DecisionQuery: graph_store: Graph database instance embedding_generator: Optional embedding generator for semantic search vector_store: Optional vector store for hybrid search - enable_advanced_analytics: Enable advanced graph analytics (requires semantica.kg) - enable_node_embeddings: Enable Node2Vec embeddings (requires semantica.kg) - enable_centrality_analysis: Enable centrality measures (requires semantica.kg) - enable_community_detection: Enable community detection (requires semantica.kg) - enable_link_prediction: Enable link prediction (requires semantica.kg) + advanced_analytics: Enable advanced graph analytics (requires semantica.kg) + node_embeddings: Enable Node2Vec embeddings (requires semantica.kg) + centrality_analysis: Enable centrality measures (requires semantica.kg) + community_detection: Enable community detection (requires semantica.kg) + link_prediction: Enable link prediction (requires semantica.kg) """ self.graph_store = graph_store self.embedding_generator = embedding_generator @@ -139,17 +139,17 @@ class DecisionQuery: self.kg_components = {} self.vector_components = {} - if KG_AVAILABLE and enable_advanced_analytics: + if KG_AVAILABLE and advanced_analytics: try: - if enable_centrality_analysis: + if centrality_analysis: self.kg_components["centrality_calculator"] = CentralityCalculator() - if enable_community_detection: + if community_detection: self.kg_components["community_detector"] = CommunityDetector() - if enable_node_embeddings: + if node_embeddings: self.kg_components["node_embedder"] = NodeEmbedder() self.kg_components["path_finder"] = PathFinder() self.kg_components["similarity_calculator"] = SimilarityCalculator() - if enable_link_prediction: + if link_prediction: self.kg_components["link_predictor"] = LinkPredictor() self.logger.info("Advanced KG components initialized successfully") @@ -344,11 +344,13 @@ class DecisionQuery: query_parts.append("LIMIT $limit") query = " ".join(query_parts) - results = self.graph_store.execute_query(query, params) - + results = self._extract_records(self.graph_store.execute_query(query, params)) + decisions = [] for record in results: - decision_data = record.get("d", {}) + decision_data = record.get("d") if isinstance(record, dict) else None + if not isinstance(decision_data, dict): + decision_data = record if isinstance(record, dict) else {} decision = self._dict_to_decision(decision_data) # Calculate similarity if embedding available @@ -393,10 +395,13 @@ class DecisionQuery: "category": category, "limit": limit }) + results = self._extract_records(results) decisions = [] for record in results: - decision_data = record.get("d", {}) + decision_data = record.get("d") if isinstance(record, dict) else None + if not isinstance(decision_data, dict): + decision_data = record if isinstance(record, dict) else {} decisions.append(self._dict_to_decision(decision_data)) self.logger.info(f"Found {len(decisions)} decisions in category {category}") @@ -429,10 +434,13 @@ class DecisionQuery: "entity_id": entity_id, "limit": limit }) + results = self._extract_records(results) decisions = [] for record in results: - decision_data = record.get("d", {}) + decision_data = record.get("d") if isinstance(record, dict) else None + if not isinstance(decision_data, dict): + decision_data = record if isinstance(record, dict) else {} decisions.append(self._dict_to_decision(decision_data)) self.logger.info(f"Found {len(decisions)} decisions about entity {entity_id}") @@ -472,10 +480,13 @@ class DecisionQuery: "end": end, "limit": limit }) + results = self._extract_records(results) decisions = [] for record in results: - decision_data = record.get("d", {}) + decision_data = record.get("d") if isinstance(record, dict) else None + if not isinstance(decision_data, dict): + decision_data = record if isinstance(record, dict) else {} decisions.append(self._dict_to_decision(decision_data)) self.logger.info(f"Found {len(decisions)} decisions in time range") @@ -518,10 +529,13 @@ class DecisionQuery: results = self.graph_store.execute_query(query, { "start_entity": start_entity }) + results = self._extract_records(results) decisions = [] for record in results: - decision_data = record.get("d", {}) + decision_data = record.get("d") if isinstance(record, dict) else None + if not isinstance(decision_data, dict): + decision_data = record if isinstance(record, dict) else {} decision = self._dict_to_decision(decision_data) decision.metadata["hop_count"] = record.get("hop_count", 0) decisions.append(decision) @@ -562,6 +576,7 @@ class DecisionQuery: results = self.graph_store.execute_query(query, { "decision_id": decision_id }) + results = self._extract_records(results) paths = [] for record in results: @@ -606,10 +621,13 @@ class DecisionQuery: LIMIT $limit """ results = self.graph_store.execute_query(query, {"limit": limit}) + results = self._extract_records(results) exceptions = [] for record in results: - exception_data = record.get("e", {}) + exception_data = record.get("e") if isinstance(record, dict) else None + if not isinstance(exception_data, dict): + exception_data = record if isinstance(record, dict) else {} exception = self._dict_to_exception(exception_data) # Calculate similarity if embedding available @@ -639,9 +657,13 @@ class DecisionQuery: # Handle timestamp conversion if isinstance(data.get("timestamp"), str): data["timestamp"] = datetime.fromisoformat(data["timestamp"]) - + + decision_id = data.get("decision_id") or data.get("id") + if not decision_id: + raise KeyError("decision_id") + return Decision( - decision_id=data.get("decision_id", ""), + decision_id=decision_id, category=data.get("category", ""), scenario=data.get("scenario", ""), reasoning=data.get("reasoning", ""), @@ -651,7 +673,7 @@ class DecisionQuery: decision_maker=data.get("decision_maker", ""), reasoning_embedding=data.get("reasoning_embedding"), node2vec_embedding=data.get("node2vec_embedding"), - metadata=data.get("metadata", {}) + metadata=data.get("metadata", {}), ) def _dict_to_exception(self, data: Dict[str, Any]) -> PolicyException: @@ -659,16 +681,22 @@ class DecisionQuery: # Handle timestamp conversion if isinstance(data.get("approval_timestamp"), str): data["approval_timestamp"] = datetime.fromisoformat(data["approval_timestamp"]) - + + exception_id = data.get("exception_id") or data.get("id") + decision_id = data.get("decision_id") + policy_id = data.get("policy_id") + if not exception_id or not decision_id or not policy_id: + raise KeyError("exception_id/decision_id/policy_id") + return PolicyException( - exception_id=data.get("exception_id", ""), - decision_id=data.get("decision_id", ""), - policy_id=data.get("policy_id", ""), + exception_id=exception_id, + decision_id=decision_id, + policy_id=policy_id, reason=data.get("reason", ""), approver=data.get("approver", ""), approval_timestamp=data.get("approval_timestamp", datetime.now()), justification=data.get("justification", ""), - metadata=data.get("metadata", {}) + metadata=data.get("metadata", {}), ) def _cosine_similarity(self, vec1: List[float], vec2: List[float]) -> float: @@ -805,6 +833,7 @@ class DecisionQuery: results = self.graph_store.execute_query(query, { "decision_id": decision_id }) + results = self._extract_records(results) return {"nodes": results, "max_depth": max_depth} except Exception: @@ -857,10 +886,12 @@ class DecisionQuery: downstream_results = self.graph_store.execute_query(downstream_query, { "decision_id": decision_id }) + downstream_results = self._extract_records(downstream_results) upstream_results = self.graph_store.execute_query(upstream_query, { "decision_id": decision_id }) + upstream_results = self._extract_records(upstream_results) # Process results for record in downstream_results: @@ -954,3 +985,12 @@ class DecisionQuery: except Exception as e: self.logger.error(f"Failed to predict relationships: {e}") return [] + + def _extract_records(self, results: Any) -> List[Dict[str, Any]]: + """Normalize execute_query result shapes to a list of record maps.""" + if isinstance(results, dict): + records = results.get("records", []) + return records if isinstance(records, list) else [] + if isinstance(results, list): + return results + return [] diff --git a/semantica/context/decision_recorder.py b/semantica/context/decision_recorder.py index 63b32381..872005aa 100644 --- a/semantica/context/decision_recorder.py +++ b/semantica/context/decision_recorder.py @@ -149,7 +149,7 @@ class DecisionRecorder: return decision.decision_id except Exception as e: - self.logger.error(f"Failed to record decision: {e}") + self.logger.exception("Failed to record decision") raise def link_entities(self, decision_id: str, entities: List[str]) -> None: @@ -176,35 +176,88 @@ class DecisionRecorder: self.logger.info(f"Linked decision {decision_id} to {len(entities)} entities") except Exception as e: - self.logger.error(f"Failed to link entities: {e}") + self.logger.exception("Failed to link entities") raise - def apply_policies(self, decision_id: str, policy_ids: List[str]) -> None: + def apply_policies( + self, + decision_id: str, + policy_ids: List[Union[str, Dict[str, str]]], + ) -> List[Dict[str, str]]: """ Track policy applications for a decision. Args: decision_id: Decision ID - policy_ids: List of policy IDs that were applied + policy_ids: List of policy IDs or policy refs with explicit version + + Returns: + Applied policy references with resolved versions """ try: - for policy_id in policy_ids: - # Create APPLIED_POLICY relationship + applied: List[Dict[str, str]] = [] + + for policy_ref in policy_ids: + if isinstance(policy_ref, dict): + policy_id = str(policy_ref.get("policy_id", "")) + policy_version = ( + str(policy_ref.get("version")) + if policy_ref.get("version") is not None + else None + ) + else: + policy_id = str(policy_ref) + policy_version = None + + if not policy_id: + continue + + # Resolve exactly one policy node: + # - explicit version when provided + # - latest available version for legacy callers query = """ MATCH (d:Decision {decision_id: $decision_id}) MATCH (p:Policy {policy_id: $policy_id}) - MERGE (d)-[:APPLIED_POLICY]->(p) - SET d.applied_at = timestamp() + WHERE $policy_version IS NULL OR p.version = $policy_version + WITH d, p + ORDER BY p.updated_at DESC, p.version DESC + LIMIT 1 + MERGE (d)-[r:APPLIED_POLICY]->(p) + SET r.policy_id = $policy_id, + r.policy_version = p.version, + d.applied_at = timestamp() + RETURN p.policy_id as policy_id, p.version as version """ - self.graph_store.execute_query(query, { + result = self.graph_store.execute_query(query, { "decision_id": decision_id, - "policy_id": policy_id + "policy_id": policy_id, + "policy_version": policy_version, }) + + records = ( + result.get("records", []) + if isinstance(result, dict) + else (result if isinstance(result, list) else []) + ) + if records: + record = records[0] + applied.append( + { + "policy_id": str(record.get("policy_id", policy_id)), + "version": str(record.get("version", policy_version or "")), + } + ) + else: + self.logger.warning( + f"No policy match found for {policy_id}" + + (f" version {policy_version}" if policy_version else "") + ) - self.logger.info(f"Applied {len(policy_ids)} policies to decision {decision_id}") + self.logger.info(f"Applied {len(applied)} policies to decision {decision_id}") + return applied except Exception as e: - self.logger.error(f"Failed to apply policies: {e}") + self.logger.exception("Failed to apply policies") raise def record_exception( @@ -231,7 +284,7 @@ class DecisionRecorder: Exception ID """ try: - exception = Exception( + exception = PolicyException( exception_id=str(uuid.uuid4()), decision_id=decision_id, policy_id=policy_id, @@ -262,7 +315,7 @@ class DecisionRecorder: return exception.exception_id except Exception as e: - self.logger.error(f"Failed to record exception: {e}") + self.logger.exception("Failed to record exception") raise def capture_cross_system_context( @@ -302,7 +355,7 @@ class DecisionRecorder: self.logger.info(f"Captured cross-system context for decision {decision_id}") except Exception as e: - self.logger.error(f"Failed to capture cross-system context: {e}") + self.logger.exception("Failed to capture cross-system context") raise def record_approval_chain( @@ -352,7 +405,7 @@ class DecisionRecorder: self.logger.info(f"Recorded approval chain with {len(approvers)} approvers") except Exception as e: - self.logger.error(f"Failed to record approval chain: {e}") + self.logger.exception("Failed to record approval chain") raise def link_precedents( @@ -389,7 +442,7 @@ class DecisionRecorder: self.logger.info(f"Linked {len(precedent_ids)} precedents to decision {decision_id}") except Exception as e: - self.logger.error(f"Failed to link precedents: {e}") + self.logger.exception("Failed to link precedents") raise def _store_decision_node(self, decision: Decision) -> None: @@ -423,7 +476,7 @@ class DecisionRecorder: "metadata": decision.metadata }) - def _store_exception_node(self, exception: Exception) -> None: + def _store_exception_node(self, exception: PolicyException) -> None: """Store exception node in graph database.""" query = """ CREATE (e:Exception { @@ -502,4 +555,4 @@ class DecisionRecorder: ) except Exception as e: - self.logger.warning(f"Failed to track provenance: {e}") + self.logger.exception("Failed to track provenance") diff --git a/semantica/context/graph_schema.py b/semantica/context/graph_schema.py index ec71c8b3..6b46e642 100644 --- a/semantica/context/graph_schema.py +++ b/semantica/context/graph_schema.py @@ -5,6 +5,7 @@ This module provides schema setup utilities for decision tracking, including node labels, relationship types, and indexes for graph databases. """ +import json from typing import Dict, Any, List from ..graph_store import GraphStore @@ -44,22 +45,51 @@ def create_decision_constraints(graph_store: GraphStore) -> None: constraints = [ # Decision nodes "CREATE CONSTRAINT decision_id_unique IF NOT EXISTS FOR (d:Decision) REQUIRE d.decision_id IS UNIQUE", - - # Policy nodes - "CREATE CONSTRAINT policy_id_unique IF NOT EXISTS FOR (p:Policy) REQUIRE p.policy_id IS UNIQUE", - + # Exception nodes "CREATE CONSTRAINT exception_id_unique IF NOT EXISTS FOR (e:Exception) REQUIRE e.exception_id IS UNIQUE", - + # ApprovalChain nodes "CREATE CONSTRAINT approval_id_unique IF NOT EXISTS FOR (a:ApprovalChain) REQUIRE a.approval_id IS UNIQUE", - + # DecisionContext nodes "CREATE CONSTRAINT context_id_unique IF NOT EXISTS FOR (c:DecisionContext) REQUIRE c.context_id IS UNIQUE", - + # Precedent nodes - "CREATE CONSTRAINT precedent_id_unique IF NOT EXISTS FOR (pr:Precedent) REQUIRE pr.precedent_id IS UNIQUE" + "CREATE CONSTRAINT precedent_id_unique IF NOT EXISTS FOR (pr:Precedent) REQUIRE pr.precedent_id IS UNIQUE", + + # Immutable trace nodes + "CREATE CONSTRAINT decision_trace_id_unique IF NOT EXISTS FOR (t:DecisionTraceEvent) REQUIRE t.trace_id IS UNIQUE", ] + + # Policy versioning needs (policy_id, version) identity. Keep legacy fallback for old backends. + try: + # Drop legacy constraint when possible so versioned policies can coexist. + graph_store.execute_query("DROP CONSTRAINT policy_id_unique IF EXISTS") + except Exception as e: + get_logger(__name__).warning( + "Failed to drop legacy policy_id_unique constraint before policy " + f"versioning migration: {e}" + ) + + try: + graph_store.execute_query( + "CREATE CONSTRAINT policy_identity_unique IF NOT EXISTS " + "FOR (p:Policy) REQUIRE (p.policy_id, p.version) IS UNIQUE" + ) + except Exception as e: + get_logger(__name__).warning( + "Composite policy constraint not supported; falling back to legacy " + "policy_id uniqueness (policy versioning may be limited)" + ) + try: + graph_store.execute_query( + "CREATE CONSTRAINT policy_id_unique IF NOT EXISTS FOR (p:Policy) REQUIRE p.policy_id IS UNIQUE" + ) + except Exception as fallback_error: + get_logger(__name__).debug( + f"Policy constraint creation failed (may already exist): {fallback_error}" + ) for constraint in constraints: try: @@ -77,6 +107,15 @@ def create_decision_indexes(graph_store: GraphStore) -> None: graph_store: Graph database instance """ indexes = [ + # Explicit identity indexes (helps verification and non-constraint lookups) + "CREATE INDEX decision_id_index IF NOT EXISTS FOR (d:Decision) ON (d.decision_id)", + "CREATE INDEX policy_id_index IF NOT EXISTS FOR (p:Policy) ON (p.policy_id)", + "CREATE INDEX exception_id_index IF NOT EXISTS FOR (e:Exception) ON (e.exception_id)", + "CREATE INDEX approval_id_index IF NOT EXISTS FOR (a:ApprovalChain) ON (a.approval_id)", + "CREATE INDEX context_id_index IF NOT EXISTS FOR (c:DecisionContext) ON (c.context_id)", + "CREATE INDEX precedent_id_index IF NOT EXISTS FOR (pr:Precedent) ON (pr.precedent_id)", + "CREATE INDEX decision_trace_id_index IF NOT EXISTS FOR (t:DecisionTraceEvent) ON (t.trace_id)", + # Decision indexes "CREATE INDEX decision_category_index IF NOT EXISTS FOR (d:Decision) ON (d.category)", "CREATE INDEX decision_timestamp_index IF NOT EXISTS FOR (d:Decision) ON (d.timestamp)", @@ -111,6 +150,11 @@ def create_decision_indexes(graph_store: GraphStore) -> None: # Cross-system context indexes "CREATE INDEX cross_system_name_index IF NOT EXISTS FOR (c:CrossSystemContext) ON (c.system_name)", "CREATE INDEX cross_system_created_at_index IF NOT EXISTS FOR (c:CrossSystemContext) ON (c.created_at)", + + # Decision trace indexes + "CREATE INDEX decision_trace_event_index IF NOT EXISTS FOR (t:DecisionTraceEvent) ON (t.event_index)", + "CREATE INDEX decision_trace_type_index IF NOT EXISTS FOR (t:DecisionTraceEvent) ON (t.event_type)", + "CREATE INDEX decision_trace_timestamp_index IF NOT EXISTS FOR (t:DecisionTraceEvent) ON (t.event_timestamp)", # Entity type indexes for general graph operations "CREATE INDEX entity_type_index IF NOT EXISTS FOR (n) ON (n.type)", @@ -153,7 +197,11 @@ def verify_schema(graph_store: GraphStore) -> bool: "policy_id_index", "policy_category_index", "exception_id_index", - "approval_id_index" + "approval_id_index", + "decision_trace_id_index", + "decision_trace_event_index", + "decision_trace_type_index", + "decision_trace_timestamp_index", ] for index_name in index_checks: @@ -178,6 +226,25 @@ def verify_schema(graph_store: GraphStore) -> bool: logger.warning(f"Schema verification failed for {index_name}: {e}") return False + # Verify policy constraint compatibility: + # prefer composite (policy_id, version), allow legacy policy_id uniqueness. + try: + constraints_result = graph_store.execute_query("SHOW CONSTRAINTS") + constraint_records = ( + constraints_result.get("records", []) + if isinstance(constraints_result, dict) + else constraints_result + ) + constraint_text = json.dumps(constraint_records, default=str).lower() + has_composite = "policy_identity_unique" in constraint_text + has_legacy = "policy_id_unique" in constraint_text + if not (has_composite or has_legacy): + logger.warning("Missing policy identity constraint (composite or legacy)") + return False + except Exception: + # Backend may not support SHOW CONSTRAINTS; skip hard failure here. + pass + # Check for node labels label_checks = [ "Decision", @@ -185,7 +252,8 @@ def verify_schema(graph_store: GraphStore) -> bool: "Exception", "ApprovalChain", "DecisionContext", - "Precedent" + "Precedent", + "DecisionTraceEvent", ] for label in label_checks: @@ -227,7 +295,7 @@ def get_schema_info() -> Dict[str, Any]: "policy_id", "name", "description", "rules", "category", "version", "created_at", "updated_at", "metadata" ], - "constraints": ["policy_id_unique"], + "constraints": ["policy_identity_unique"], "indexes": ["policy_id_index", "policy_category_index", "policy_version_index"] }, "Exception": { @@ -267,6 +335,19 @@ def get_schema_info() -> Dict[str, Any]: "context_id", "system_name", "context_data", "created_at" ], "indexes": ["cross_system_name_index", "cross_system_created_at_index"] + }, + "DecisionTraceEvent": { + "properties": [ + "trace_id", "decision_id", "event_index", "event_type", + "event_timestamp", "event_payload", "previous_hash", "event_hash" + ], + "constraints": ["decision_trace_id_unique"], + "indexes": [ + "decision_trace_id_index", + "decision_trace_event_index", + "decision_trace_type_index", + "decision_trace_timestamp_index" + ] } }, "relationship_types": { @@ -288,6 +369,9 @@ def get_schema_info() -> Dict[str, Any]: "Provenance relationships": [ "DERIVED_FROM", "INFLUENCED_BY", "BASED_ON" ], + "Decision trace relationships": [ + "HAS_TRACE_EVENT", "NEXT_TRACE_EVENT" + ], "Entity and context relationships": [ "REPORTED_BY", "RELATES_TO", "ESCALATED_TO", "SIMILAR_TO" ], @@ -302,13 +386,15 @@ def get_schema_info() -> Dict[str, Any]: "approval_id_index", "approval_method_index", "approval_approver_index", "context_id_index", "context_decision_id_index", "precedent_id_index", "precedent_source_index", "precedent_similarity_index", + "decision_trace_id_index", "decision_trace_event_index", "decision_trace_type_index", "decision_trace_timestamp_index", "cross_system_name_index", "cross_system_created_at_index", "entity_type_index", "entity_id_index", "relationship_strength_index", "temporal_before_index", "temporal_after_index", "temporal_during_index" ], "constraints": [ - "decision_id_unique", "policy_id_unique", "exception_id_unique", - "approval_id_unique", "context_id_unique", "precedent_id_unique" + "decision_id_unique", "policy_identity_unique", "exception_id_unique", + "approval_id_unique", "context_id_unique", "precedent_id_unique", + "decision_trace_id_unique" ] } @@ -383,11 +469,13 @@ def drop_decision_schema(graph_store: GraphStore) -> None: # Drop constraints constraints = [ "DROP CONSTRAINT decision_id_unique IF EXISTS", - "DROP CONSTRAINT policy_id_unique IF EXISTS", + "DROP CONSTRAINT policy_identity_unique IF EXISTS", + "DROP CONSTRAINT policy_id_unique IF EXISTS", "DROP CONSTRAINT exception_id_unique IF EXISTS", "DROP CONSTRAINT approval_id_unique IF EXISTS", "DROP CONSTRAINT context_id_unique IF EXISTS", - "DROP CONSTRAINT precedent_id_unique IF EXISTS" + "DROP CONSTRAINT precedent_id_unique IF EXISTS", + "DROP CONSTRAINT decision_trace_id_unique IF EXISTS", ] for constraint in constraints: @@ -398,12 +486,22 @@ def drop_decision_schema(graph_store: GraphStore) -> None: # Drop indexes indexes = [ + "DROP INDEX decision_id_index IF EXISTS", "DROP INDEX decision_category_index IF EXISTS", "DROP INDEX decision_timestamp_index IF EXISTS", + "DROP INDEX policy_id_index IF EXISTS", "DROP INDEX policy_category_index IF EXISTS", "DROP INDEX policy_version_index IF EXISTS", + "DROP INDEX exception_id_index IF EXISTS", "DROP INDEX exception_reason_index IF EXISTS", - "DROP INDEX approval_method_index IF EXISTS" + "DROP INDEX approval_id_index IF EXISTS", + "DROP INDEX approval_method_index IF EXISTS", + "DROP INDEX context_id_index IF EXISTS", + "DROP INDEX precedent_id_index IF EXISTS", + "DROP INDEX decision_trace_id_index IF EXISTS", + "DROP INDEX decision_trace_event_index IF EXISTS", + "DROP INDEX decision_trace_type_index IF EXISTS", + "DROP INDEX decision_trace_timestamp_index IF EXISTS" ] for index in indexes: @@ -416,6 +514,7 @@ def drop_decision_schema(graph_store: GraphStore) -> None: cleanup_query = """ MATCH (n) WHERE n:Decision OR n:Policy OR n:Exception OR n:ApprovalChain OR n:DecisionContext OR n:Precedent OR n:CrossSystemContext + OR n:DecisionTraceEvent DETACH DELETE n """ graph_store.execute_query(cleanup_query) diff --git a/semantica/context/policy_engine.py b/semantica/context/policy_engine.py index cb7eb669..b4d064ee 100644 --- a/semantica/context/policy_engine.py +++ b/semantica/context/policy_engine.py @@ -84,7 +84,7 @@ class PolicyEngine: records exceptions, and analyzes policy impact. """ - def __init__(self, graph_store: GraphStore): + def __init__(self, graph_store: Any): """ Initialize PolicyEngine. @@ -93,6 +93,7 @@ class PolicyEngine: """ self.graph_store = graph_store self.logger = get_logger(__name__) + self._supports_cypher = hasattr(graph_store, "execute_query") def add_policy(self, policy: Policy) -> str: """ @@ -105,37 +106,56 @@ class PolicyEngine: Policy ID """ try: - # Store policy node - query = """ - CREATE (p:Policy { - policy_id: $policy_id, - name: $name, - description: $description, - rules: $rules, - category: $category, - version: $version, - created_at: $created_at, - updated_at: $updated_at, - metadata: $metadata - }) - """ - self.graph_store.execute_query(query, { - "policy_id": policy.policy_id, - "name": policy.name, - "description": policy.description, - "rules": policy.rules, - "category": policy.category, - "version": policy.version, - "created_at": policy.created_at, - "updated_at": policy.updated_at, - "metadata": policy.metadata - }) - + if self._supports_cypher: + query = """ + CREATE (p:Policy { + policy_id: $policy_id, + name: $name, + description: $description, + rules: $rules, + category: $category, + version: $version, + created_at: $created_at, + updated_at: $updated_at, + metadata: $metadata + }) + """ + self.graph_store.execute_query(query, { + "policy_id": policy.policy_id, + "name": policy.name, + "description": policy.description, + "rules": policy.rules, + "category": policy.category, + "version": policy.version, + "created_at": policy.created_at, + "updated_at": policy.updated_at, + "metadata": policy.metadata + }) + self.logger.info(f"Added policy: {policy.policy_id} version {policy.version}") + return policy.policy_id + + if not hasattr(self.graph_store, "add_node"): + raise RuntimeError("Graph backend does not support policy storage") + + node_id = f"{policy.policy_id}:{policy.version}" + self.graph_store.add_node( + node_id=node_id, + node_type="Policy", + content=policy.name or policy.policy_id, + policy_id=policy.policy_id, + name=policy.name, + description=policy.description, + rules=policy.rules, + category=policy.category, + version=policy.version, + created_at=policy.created_at.isoformat() if hasattr(policy.created_at, "isoformat") else str(policy.created_at), + updated_at=policy.updated_at.isoformat() if hasattr(policy.updated_at, "isoformat") else str(policy.updated_at), + metadata=policy.metadata or {} + ) self.logger.info(f"Added policy: {policy.policy_id} version {policy.version}") return policy.policy_id - except Exception as e: - self.logger.error(f"Failed to add policy: {e}") + self.logger.exception("Failed to add policy") raise def update_policy( @@ -187,23 +207,32 @@ class PolicyEngine: # Store new version self.add_policy(updated_policy) - # Link versions - query = """ - MATCH (old:Policy {policy_id: $policy_id, version: $old_version}) - MATCH (new:Policy {policy_id: $policy_id, version: $new_version}) - MERGE (old)-[:VERSION_OF]->(new) - """ - self.graph_store.execute_query(query, { - "policy_id": policy_id, - "old_version": current_policy.version, - "new_version": new_version - }) + if self._supports_cypher: + query = """ + MATCH (old:Policy {policy_id: $policy_id, version: $old_version}) + MATCH (new:Policy {policy_id: $policy_id, version: $new_version}) + MERGE (old)-[:VERSION_OF]->(new) + """ + self.graph_store.execute_query(query, { + "policy_id": policy_id, + "old_version": current_policy.version, + "new_version": new_version + }) + else: + if hasattr(self.graph_store, "add_edge"): + self.graph_store.add_edge( + f"{policy_id}:{current_policy.version}", + f"{policy_id}:{new_version}", + edge_type="VERSION_OF", + changed_at=datetime.now().isoformat(), + change_reason=change_reason + ) self.logger.info(f"Updated policy {policy_id} to version {new_version}") return new_version except Exception as e: - self.logger.error(f"Failed to update policy: {e}") + self.logger.exception("Failed to update policy") raise def get_applicable_policies( @@ -222,32 +251,123 @@ class PolicyEngine: List of applicable policies (latest versions) """ try: + if self._supports_cypher: # Get latest policies for category - query = """ - MATCH (p:Policy {category: $category}) - WHERE NOT (p)-[:VERSION_OF]->(:Policy) - RETURN p - ORDER BY p.updated_at DESC - """ - results = self.graph_store.execute_query(query, {"category": category}) - - policies = [] - for record in results: - policy_data = record.get("p", {}) - policies.append(self._dict_to_policy(policy_data)) - - # Filter by entities if specified - if entities: - # This would require entity-specific policy relationships - # For now, return all category policies - pass - + query = """ + MATCH (p:Policy {category: $category}) + WHERE NOT (p)-[:VERSION_OF]->(:Policy) + RETURN p + ORDER BY p.updated_at DESC + """ + results = self.graph_store.execute_query(query, {"category": category}) + records = self._extract_records(results) + + policies = [] + for record in records: + policy_data = record.get("p") if isinstance(record, dict) else None + if not isinstance(policy_data, dict): + policy_data = record if isinstance(record, dict) else {} + if not isinstance(policy_data, dict) or not policy_data.get("policy_id"): + self.logger.debug( + "Skipping malformed policy record in get_applicable_policies: " + f"{record}" + ) + continue + policy = self._dict_to_policy(policy_data) + if self._policy_matches_entities(policy, entities): + policies.append(policy) + + self.logger.info(f"Found {len(policies)} applicable policies for category {category}") + return policies + + if not hasattr(self.graph_store, "find_nodes"): + return [] + + latest_by_policy_id: Dict[str, Dict[str, Any]] = {} + for node in self.graph_store.find_nodes(node_type="Policy"): + data = node.get("metadata", {}) or {} + if data.get("category") != category: + continue + pid = data.get("policy_id") + if not pid: + continue + updated_at = data.get("updated_at") or "" + prev = latest_by_policy_id.get(pid) + if not prev: + latest_by_policy_id[pid] = data + else: + if str(updated_at) > str(prev.get("updated_at") or ""): + latest_by_policy_id[pid] = data + + policies: List[Policy] = [] + for data in latest_by_policy_id.values(): + policy = self._dict_to_policy({ + "policy_id": data.get("policy_id"), + "name": data.get("name"), + "description": data.get("description"), + "rules": data.get("rules", {}), + "category": data.get("category"), + "version": data.get("version"), + "created_at": data.get("created_at"), + "updated_at": data.get("updated_at"), + "metadata": data.get("metadata", {}) + }) + if self._policy_matches_entities(policy, entities): + policies.append(policy) + self.logger.info(f"Found {len(policies)} applicable policies for category {category}") return policies except Exception as e: - self.logger.error(f"Failed to get applicable policies: {e}") + self.logger.exception("Failed to get applicable policies") raise + + def _extract_records(self, results: Any) -> List[Dict[str, Any]]: + """Normalize execute_query result shapes to record lists.""" + if isinstance(results, dict): + records = results.get("records", []) + if not isinstance(records, list): + return [] + + # FalkorDB shape: {"records": [[...], ...], "header": ["col1", ...]} + header = results.get("header") + if ( + isinstance(header, list) + and records + and all(isinstance(row, list) for row in records) + ): + normalized: List[Dict[str, Any]] = [] + for row in records: + row_map: Dict[str, Any] = dict(zip(header, row)) + normalized.append(row_map) + return normalized + + return records + if isinstance(results, list): + return results + return [] + + def _policy_matches_entities( + self, policy: Policy, entities: Optional[List[str]] + ) -> bool: + """ + Entity scoping for policies. + If no entity scope is defined on the policy, it is globally applicable. + """ + if not entities: + return True + + metadata = policy.metadata or {} + scoped_entities = ( + metadata.get("entities") + or metadata.get("entity_ids") + or metadata.get("applies_to_entities") + or [] + ) + if not scoped_entities: + return True + + return bool(set(str(e) for e in scoped_entities).intersection(set(entities))) def check_compliance(self, decision: Decision, policy_id: str) -> bool: """ @@ -285,7 +405,7 @@ class PolicyEngine: return True except Exception as e: - self.logger.error(f"Failed to check compliance: {e}") + self.logger.exception("Failed to check compliance") return False def record_policy_application( @@ -303,22 +423,35 @@ class PolicyEngine: version: Policy version that was applied """ try: - query = """ - MATCH (d:Decision {decision_id: $decision_id}) - MATCH (p:Policy {policy_id: $policy_id, version: $version}) - MERGE (d)-[:APPLIED_POLICY]->(p) - SET d.policy_applied_at = timestamp() - """ - self.graph_store.execute_query(query, { - "decision_id": decision_id, - "policy_id": policy_id, - "version": version - }) - + if self._supports_cypher: + query = """ + MATCH (d:Decision {decision_id: $decision_id}) + MATCH (p:Policy {policy_id: $policy_id, version: $version}) + MERGE (d)-[:APPLIED_POLICY]->(p) + SET d.policy_applied_at = timestamp() + """ + self.graph_store.execute_query(query, { + "decision_id": decision_id, + "policy_id": policy_id, + "version": version + }) + self.logger.info(f"Recorded policy application: {policy_id} v{version} to decision {decision_id}") + return + + if not hasattr(self.graph_store, "add_edge"): + raise RuntimeError("Graph backend does not support relationships") + policy_node_id = f"{policy_id}:{version}" + self.graph_store.add_edge( + decision_id, + policy_node_id, + edge_type="APPLIED_POLICY", + applied_at=datetime.now().isoformat(), + policy_id=policy_id, + version=version + ) self.logger.info(f"Recorded policy application: {policy_id} v{version} to decision {decision_id}") - except Exception as e: - self.logger.error(f"Failed to record policy application: {e}") + self.logger.exception("Failed to record policy application") raise def record_exception( @@ -340,42 +473,63 @@ class PolicyEngine: """ try: exception_id = str(uuid.uuid4()) - - query = """ - CREATE (e:Exception { - exception_id: $exception_id, - decision_id: $decision_id, - policy_id: $policy_id, - reason: $reason, - created_at: datetime() - }) - """ - self.graph_store.execute_query(query, { - "exception_id": exception_id, - "decision_id": decision_id, - "policy_id": policy_id, - "reason": reason - }) - - # Link to decision and policy - query = """ - MATCH (d:Decision {decision_id: $decision_id}) - MATCH (p:Policy {policy_id: $policy_id}) - MATCH (e:Exception {exception_id: $exception_id}) - MERGE (d)-[:GRANTED_EXCEPTION]->(e) - MERGE (e)-[:OVERRIDDEN_POLICY]->(p) - """ - self.graph_store.execute_query(query, { - "decision_id": decision_id, - "policy_id": policy_id, - "exception_id": exception_id - }) - + + if self._supports_cypher: + query = """ + CREATE (e:Exception { + exception_id: $exception_id, + decision_id: $decision_id, + policy_id: $policy_id, + reason: $reason, + created_at: datetime() + }) + """ + self.graph_store.execute_query(query, { + "exception_id": exception_id, + "decision_id": decision_id, + "policy_id": policy_id, + "reason": reason + }) + + query = """ + MATCH (d:Decision {decision_id: $decision_id}) + MATCH (p:Policy {policy_id: $policy_id}) + MATCH (e:Exception {exception_id: $exception_id}) + MERGE (d)-[:GRANTED_EXCEPTION]->(e) + MERGE (e)-[:OVERRIDDEN_POLICY]->(p) + """ + self.graph_store.execute_query(query, { + "decision_id": decision_id, + "policy_id": policy_id, + "exception_id": exception_id + }) + + self.logger.info(f"Recorded policy exception: {exception_id}") + return exception_id + + if not hasattr(self.graph_store, "add_node") or not hasattr(self.graph_store, "add_edge"): + raise RuntimeError("Graph backend does not support exceptions") + + self.graph_store.add_node( + node_id=exception_id, + node_type="Exception", + content=reason, + exception_id=exception_id, + decision_id=decision_id, + policy_id=policy_id, + reason=reason, + created_at=datetime.now().isoformat() + ) + self.graph_store.add_edge(decision_id, exception_id, edge_type="GRANTED_EXCEPTION") + + policy = self.get_policy(policy_id) + if policy: + self.graph_store.add_edge(exception_id, f"{policy_id}:{policy.version}", edge_type="OVERRIDDEN_POLICY") + self.logger.info(f"Recorded policy exception: {exception_id}") return exception_id - except Exception as e: - self.logger.error(f"Failed to record exception: {e}") + self.logger.exception("Failed to record exception") raise def get_policy_history(self, policy_id: str) -> List[Policy]: @@ -389,26 +543,48 @@ class PolicyEngine: List of policy versions """ try: - query = """ - MATCH (p:Policy {policy_id: $policy_id}) - OPTIONAL MATCH (p)-[:VERSION_OF*]->(future:Policy) - WITH collect(p) + collect(future) as all_versions - UNWIND all_versions as version - RETURN DISTINCT version - ORDER BY version.updated_at - """ - results = self.graph_store.execute_query(query, {"policy_id": policy_id}) + if self._supports_cypher: + query = """ + MATCH (p:Policy {policy_id: $policy_id}) + OPTIONAL MATCH (p)-[:VERSION_OF*]->(future:Policy) + WITH collect(p) + collect(future) as all_versions + UNWIND all_versions as version + RETURN DISTINCT version + ORDER BY version.updated_at + """ + results = self.graph_store.execute_query(query, {"policy_id": policy_id}) - policies = [] - for record in results: - policy_data = record.get("version", {}) - policies.append(self._dict_to_policy(policy_data)) + policies = [] + for record in results: + policy_data = record.get("version", {}) + policies.append(self._dict_to_policy(policy_data)) - self.logger.info(f"Found {len(policies)} versions for policy {policy_id}") - return policies + self.logger.info(f"Found {len(policies)} versions for policy {policy_id}") + return policies + + if not hasattr(self.graph_store, "find_nodes"): + return [] + versions: List[Policy] = [] + for node in self.graph_store.find_nodes(node_type="Policy"): + data = node.get("metadata", {}) or {} + if data.get("policy_id") != policy_id: + continue + versions.append(self._dict_to_policy({ + "policy_id": data.get("policy_id"), + "name": data.get("name"), + "description": data.get("description"), + "rules": data.get("rules", {}), + "category": data.get("category"), + "version": data.get("version"), + "created_at": data.get("created_at"), + "updated_at": data.get("updated_at"), + "metadata": data.get("metadata", {}) + })) + versions.sort(key=lambda p: str(p.updated_at)) + return versions except Exception as e: - self.logger.error(f"Failed to get policy history: {e}") + self.logger.exception("Failed to get policy history") raise def get_affected_decisions( @@ -429,27 +605,37 @@ class PolicyEngine: List of affected decision IDs """ try: - query = """ - MATCH (d:Decision)-[:APPLIED_POLICY]->(p:Policy { - policy_id: $policy_id, - version: $from_version - }) - RETURN d.decision_id as decision_id - """ - results = self.graph_store.execute_query(query, { - "policy_id": policy_id, - "from_version": from_version - }) + if self._supports_cypher: + query = """ + MATCH (d:Decision)-[:APPLIED_POLICY]->(p:Policy { + policy_id: $policy_id, + version: $from_version + }) + RETURN d.decision_id as decision_id + """ + results = self.graph_store.execute_query(query, { + "policy_id": policy_id, + "from_version": from_version + }) - decision_ids = [] - for record in results: - decision_ids.append(record.get("decision_id", "")) + decision_ids = [] + for record in results: + decision_ids.append(record.get("decision_id", "")) - self.logger.info(f"Found {len(decision_ids)} decisions affected by policy change") + self.logger.info(f"Found {len(decision_ids)} decisions affected by policy change") + return decision_ids + + if not hasattr(self.graph_store, "find_edges"): + return [] + policy_node_id = f"{policy_id}:{from_version}" + decision_ids: List[str] = [] + for edge in self.graph_store.find_edges(edge_type="APPLIED_POLICY"): + if edge.get("target") == policy_node_id: + decision_ids.append(edge.get("source")) return decision_ids except Exception as e: - self.logger.error(f"Failed to get affected decisions: {e}") + self.logger.exception("Failed to get affected decisions") raise def analyze_policy_impact( @@ -471,16 +657,38 @@ class PolicyEngine: current_policy = self.get_policy(policy_id) if not current_policy: raise ValueError(f"Policy {policy_id} not found") - - # Get decisions that used this policy - query = """ - MATCH (d:Decision)-[:APPLIED_POLICY]->(p:Policy {policy_id: $policy_id}) - RETURN d.decision_id as decision_id, d.confidence as confidence, - d.outcome as outcome, d.category as category - """ - results = self.graph_store.execute_query(query, {"policy_id": policy_id}) - - # Analyze impact + + results: List[Dict[str, Any]] = [] + if self._supports_cypher: + query = """ + MATCH (d:Decision)-[:APPLIED_POLICY]->(p:Policy {policy_id: $policy_id}) + RETURN d.decision_id as decision_id, d.confidence as confidence, + d.outcome as outcome, d.category as category + """ + results = self.graph_store.execute_query(query, {"policy_id": policy_id}) + else: + if hasattr(self.graph_store, "find_edges") and hasattr(self.graph_store, "nodes"): + for edge in self.graph_store.find_edges(edge_type="APPLIED_POLICY"): + target = edge.get("target") + if not target or not isinstance(target, str): + continue + policy_node = self.graph_store.nodes.get(target) + if not policy_node: + continue + props = getattr(policy_node, "properties", {}) or {} + if props.get("policy_id") != policy_id: + continue + decision_node = self.graph_store.nodes.get(edge.get("source")) + if not decision_node: + continue + dprops = getattr(decision_node, "properties", {}) or {} + results.append({ + "decision_id": edge.get("source"), + "confidence": dprops.get("confidence", 0.0), + "outcome": dprops.get("outcome", ""), + "category": dprops.get("category", "") + }) + impact_analysis = { "total_decisions": len(results), "affected_decisions": 0, @@ -488,19 +696,16 @@ class PolicyEngine: "risk_assessment": "low", "recommendations": [] } - + for record in results: decision_data = { "confidence": record.get("confidence", 0.0), "outcome": record.get("outcome", ""), "category": record.get("category", "") } - - # Check if decision would still comply with new rules would_comply = self._check_compliance_with_rules( decision_data, proposed_rules ) - if not would_comply: impact_analysis["affected_decisions"] += 1 @@ -531,7 +736,7 @@ class PolicyEngine: return impact_analysis except Exception as e: - self.logger.error(f"Failed to analyze policy impact: {e}") + self.logger.exception("Failed to analyze policy impact") raise def get_policy(self, policy_id: str, version: Optional[str] = None) -> Optional[Policy]: @@ -546,31 +751,78 @@ class PolicyEngine: Policy object or None """ try: + if self._supports_cypher: + if version: + query = """ + MATCH (p:Policy {policy_id: $policy_id, version: $version}) + RETURN p + """ + params = {"policy_id": policy_id, "version": version} + else: + query = """ + MATCH (p:Policy {policy_id: $policy_id}) + WHERE NOT (p)-[:VERSION_OF]->(:Policy) + RETURN p + """ + params = {"policy_id": policy_id} + + results = self.graph_store.execute_query(query, params) + + if results: + policy_data = results[0].get("p", {}) + return self._dict_to_policy(policy_data) + return None + + if not hasattr(self.graph_store, "find_nodes"): + return None + + candidates: List[Dict[str, Any]] = [] + for node in self.graph_store.find_nodes(node_type="Policy"): + data = node.get("metadata", {}) or {} + if data.get("policy_id") != policy_id: + continue + if version and data.get("version") != version: + continue + candidates.append(data) + + if not candidates: + return None + if version: - query = """ - MATCH (p:Policy {policy_id: $policy_id, version: $version}) - RETURN p - """ - params = {"policy_id": policy_id, "version": version} + data = candidates[0] else: - # Get latest version - query = """ - MATCH (p:Policy {policy_id: $policy_id}) - WHERE NOT (p)-[:VERSION_OF]->(:Policy) - RETURN p - """ - params = {"policy_id": policy_id} - - results = self.graph_store.execute_query(query, params) - - if results: - policy_data = results[0].get("p", {}) - return self._dict_to_policy(policy_data) - - return None - + # Prefer highest semantic version if available, fallback to updated_at + def _version_key(v: str) -> tuple: + try: + parts = [int(p) for p in str(v).split(".")] + # Normalize length for comparison + while len(parts) < 3: + parts.append(-1) + return tuple(parts[:3]) + except Exception: + return (-1, -1, -1) + + try: + data = max( + candidates, + key=lambda d: (_version_key(d.get("version")), str(d.get("updated_at") or "")), + ) + except Exception: + data = max(candidates, key=lambda d: str(d.get("updated_at") or "")) + + return self._dict_to_policy({ + "policy_id": data.get("policy_id"), + "name": data.get("name"), + "description": data.get("description"), + "rules": data.get("rules", {}), + "category": data.get("category"), + "version": data.get("version"), + "created_at": data.get("created_at"), + "updated_at": data.get("updated_at"), + "metadata": data.get("metadata", {}) + }) except Exception as e: - self.logger.error(f"Failed to get policy: {e}") + self.logger.exception("Failed to get policy") return None def _generate_next_version(self, current_version: str) -> str: @@ -619,7 +871,7 @@ class PolicyEngine: data[field] = datetime.fromisoformat(data[field]) return Policy( - policy_id=data.get("policy_id", ""), + policy_id=data["policy_id"], # Required field name=data.get("name", ""), description=data.get("description", ""), rules=data.get("rules", {}), diff --git a/semantica/graph_store/__init__.py b/semantica/graph_store/__init__.py index ea952b2e..51b02251 100644 --- a/semantica/graph_store/__init__.py +++ b/semantica/graph_store/__init__.py @@ -122,6 +122,7 @@ Author: Semantica Contributors License: MIT """ +from .age_store import ApacheAgeStore from .amazon_neptune import ( AmazonNeptuneStore, NeptuneAuthTokenManager, @@ -172,6 +173,8 @@ __all__ = [ "Neo4jStore", "Neo4jDriver", "Neo4jTransaction", + # Apache AGE + "ApacheAgeStore", # Amazon Neptune "AmazonNeptuneStore", "NeptuneAuthTokenManager", diff --git a/semantica/graph_store/age_store.py b/semantica/graph_store/age_store.py new file mode 100644 index 00000000..1f4ea8b6 --- /dev/null +++ b/semantica/graph_store/age_store.py @@ -0,0 +1,1312 @@ +""" +Apache AGE Store Module + +This module provides Apache AGE (PostgreSQL graph extension) integration for +property graph storage and Cypher querying in the Semantica framework, supporting +full CRUD operations, transactions, and graph analytics. + +Apache AGE extends PostgreSQL with graph database functionality, enabling +hybrid relational + graph workloads using openCypher queries executed via SQL. + +Key Features: + - OpenCypher query language support via SQL wrapper + - Node and relationship CRUD operations + - Transaction support with explicit commit/rollback + - Parameterized queries to prevent SQL injection + - AGE internal ID / semantic ID separation + - Multi-label emulation (one AGE label + property array) + - Batch operations with progress tracking + - Optional dependency handling (psycopg2) + +Main Classes: + - ApacheAgeStore: Main AGE store for graph operations + +Example Usage: + >>> from semantica.graph_store.age_store import ApacheAgeStore + >>> store = ApacheAgeStore( + ... connection_string="host=localhost dbname=agedb user=postgres password=secret", + ... graph_name="semantica" + ... ) + >>> store.connect() + >>> node = store.create_node(labels=["Person"], properties={"name": "Alice"}) + >>> results = store.execute_query("MATCH (p:Person) RETURN p") + >>> store.close() + +Note: + - AGE auto-generates internal vertex/edge IDs. + - The ``node_id`` parameter in CRUD methods refers to the AGE internal ID. + - Semantic IDs can be stored in the ``semantica_id`` property. + - AGE supports exactly one label per vertex; additional labels are stored + in a ``labels`` property array. + +Author: Semantica Contributors +License: MIT +""" + +import json +import re +from typing import Any, Dict, List, Optional, Union + +from ..utils.exceptions import ProcessingError, ValidationError +from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker + +# Optional psycopg2 import +try: + import psycopg2 + import psycopg2.extras + + PSYCOPG2_AVAILABLE = True +except (ImportError, OSError): + PSYCOPG2_AVAILABLE = False + psycopg2 = None + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _sanitize_label(label: str) -> str: + """ + Sanitize a Cypher label to prevent injection. + + Only allows alphanumeric characters and underscores. + + Args: + label: Raw label string. + + Returns: + Sanitized label string. + + Raises: + ValidationError: If the label contains invalid characters. + """ + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", label): + raise ValidationError( + f"Invalid label '{label}': must start with a letter or underscore " + "and contain only alphanumeric characters and underscores." + ) + return label + + +def _sanitize_rel_type(rel_type: str) -> str: + """ + Sanitize a relationship type string. + + Args: + rel_type: Raw relationship type. + + Returns: + Sanitized relationship type. + + Raises: + ValidationError: If the type contains invalid characters. + """ + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", rel_type): + raise ValidationError( + f"Invalid relationship type '{rel_type}': must start with a letter or " + "underscore and contain only alphanumeric characters and underscores." + ) + return rel_type + + +def _props_to_cypher_literal(properties: Dict[str, Any]) -> str: + """ + Convert a Python dict to an AGE-compatible Cypher map literal. + + AGE does not support ``$param`` style parameter binding inside + ``cypher()`` calls, so property values must be inlined as literals + with proper escaping. + + Args: + properties: Dictionary of property key-value pairs. + + Returns: + Cypher map literal string, e.g. ``{name: 'Alice', age: 30}``. + """ + if not properties: + return "{}" + parts = [] + for key, value in properties.items(): + # Validate key + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key): + raise ValidationError(f"Invalid property key: '{key}'") + parts.append(f"{key}: {_value_to_cypher_literal(value)}") + return "{" + ", ".join(parts) + "}" + + +def _value_to_cypher_literal(value: Any) -> str: + """ + Convert a single Python value to a Cypher literal string. + + Args: + value: Python value. + + Returns: + Cypher literal representation. + """ + if value is None: + return "null" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + return repr(value) + if isinstance(value, str): + # Escape single quotes for Cypher strings + escaped = value.replace("\\", "\\\\").replace("'", "\\'") + return f"'{escaped}'" + if isinstance(value, (list, tuple)): + inner = ", ".join(_value_to_cypher_literal(v) for v in value) + return f"[{inner}]" + if isinstance(value, dict): + return _props_to_cypher_literal(value) + # Fallback: convert to string + escaped = str(value).replace("\\", "\\\\").replace("'", "\\'") + return f"'{escaped}'" + + +def _parse_agtype(raw: Any) -> Any: + """ + Parse an agtype value returned by AGE into a Python object. + + AGE returns results as ``agtype`` which may be a JSON-like string + with an optional ``::vertex`` / ``::edge`` / ``::path`` suffix. + + Args: + raw: Raw value from the cursor. + + Returns: + Parsed Python object (dict, list, or scalar). + """ + if raw is None: + return None + if not isinstance(raw, str): + return raw + + text = raw.strip() + + # Strip AGE type suffixes + for suffix in ("::vertex", "::edge", "::path", "::numeric", + "::integer", "::float", "::boolean", "::text"): + if text.endswith(suffix): + text = text[: -len(suffix)].strip() + break + + # Try JSON parse + try: + return json.loads(text) + except (json.JSONDecodeError, ValueError): + pass + + # Boolean literals + if text.lower() == "true": + return True + if text.lower() == "false": + return False + + # Numeric + try: + if "." in text: + return float(text) + return int(text) + except ValueError: + pass + + return text + + +def _vertex_to_node_dict(vertex: Any) -> Dict[str, Any]: + """ + Convert a parsed AGE vertex dict to the standard node return format. + + Expected vertex dict shape from AGE:: + + {"id": , "label": "