mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Update context module docs, cleanup notebook, and refactor context files
This commit is contained in:
KaifAhmad1
@@ -364,15 +364,17 @@ print(f"Classes: {len(ontology.classes)}")
|
||||
|
||||
### Context Engineering & Memory Systems
|
||||
|
||||
> **Persistent Memory** • **Hybrid Retrieval (Vector + Graph)** • **Hierarchical Storage** • **Entity Linking**
|
||||
> **Persistent Memory** • **Hybrid Retrieval (Vector + Graph)** • **Production Graph Store (Neo4j)** • **Entity Linking**
|
||||
|
||||
```python
|
||||
from semantica.context import AgentContext
|
||||
from semantica.vector_store import VectorStore
|
||||
from semantica.graph_store import GraphStore
|
||||
|
||||
# Initialize Context with Hybrid Retrieval (Graph + Vector)
|
||||
context = AgentContext(
|
||||
vector_store=VectorStore(backend="faiss"),
|
||||
knowledge_graph=GraphStore(backend="neo4j"), # Optional: Use persistent graph
|
||||
hybrid_alpha=0.75 # 75% weight to Knowledge Graph, 25% to Vector
|
||||
)
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -202,6 +202,16 @@ Deep dive into advanced features, customization, and complex workflows.
|
||||
|
||||
[Open Notebook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/02_Advanced_Graph_Analytics.ipynb)
|
||||
|
||||
- :material-brain: **Advanced Context Engineering**
|
||||
---
|
||||
Build a production-grade memory system for AI agents using persistent Vector (FAISS) and Graph (Neo4j) stores.
|
||||
|
||||
**Topics**: Agent Memory, GraphRAG, Entity Injection, Lifecycle Management
|
||||
|
||||
**Difficulty**: Advanced
|
||||
|
||||
[Open Notebook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/11_Advanced_Context_Engineering.ipynb)
|
||||
|
||||
- :material-monitor-dashboard: **Complete Visualization Suite**
|
||||
---
|
||||
Creating interactive, publication-ready visualizations of your graphs.
|
||||
|
||||
@@ -196,6 +196,34 @@ neighbors = graph.get_neighbors("FastAPI", hops=1)
|
||||
|
||||
---
|
||||
|
||||
### Production Graph Store Integration
|
||||
|
||||
For production environments, you can replace the in-memory `ContextGraph` with a persistent `GraphStore` (Neo4j, FalkorDB) by passing it to the `knowledge_graph` parameter.
|
||||
|
||||
```python
|
||||
from semantica.context import AgentContext
|
||||
from semantica.graph_store import GraphStore
|
||||
|
||||
# 1. Initialize Persistent Graph Store (Neo4j)
|
||||
gs = GraphStore(
|
||||
backend="neo4j",
|
||||
uri="bolt://localhost:7687",
|
||||
user="neo4j",
|
||||
password="password"
|
||||
)
|
||||
|
||||
# 2. Initialize Agent Context with Persistent Graph
|
||||
context = AgentContext(
|
||||
vector_store=vs, # Your VectorStore instance
|
||||
knowledge_graph=gs, # Your persistent GraphStore
|
||||
use_graph_expansion=True
|
||||
)
|
||||
|
||||
# Now all graph operations (store, retrieve, build_graph) use Neo4j directly.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### ContextRetriever (The Search Engine)
|
||||
The retrieval logic that powers the `retrieve()` command. It implements the **Hybrid Retrieval** algorithm.
|
||||
|
||||
|
||||
@@ -6,41 +6,6 @@ formalizing context as a graph of connections to enable meaningful agent
|
||||
understanding and memory. It integrates RAG with knowledge graphs to provide
|
||||
persistent context for intelligent agents.
|
||||
|
||||
Algorithms Used:
|
||||
|
||||
Context Graph Construction:
|
||||
- Graph Building: Node and edge construction from entities and relationships
|
||||
- Entity Extraction: Entity extraction from conversations and text
|
||||
- Relationship Extraction: Relationship extraction from conversations
|
||||
- Intent Extraction: Intent classification from conversations
|
||||
- Sentiment Analysis: Sentiment extraction from conversations
|
||||
- Graph Traversal: BFS/DFS for neighbor discovery and multi-hop traversal
|
||||
- Graph Indexing: Type-based indexing for efficient node/edge lookup
|
||||
|
||||
Agent Memory Management:
|
||||
- Vector Embedding: Embedding generation for memory items
|
||||
- Vector Search: Similarity search in vector space
|
||||
- Keyword Search: Fallback keyword-based search
|
||||
- Retention Policy: Time-based memory retention and cleanup
|
||||
- Memory Indexing: Deque-based memory index for efficient access
|
||||
- Knowledge Graph Integration: Entity and relationship updates to knowledge graph
|
||||
|
||||
Context Retrieval:
|
||||
- Vector Similarity Search: Cosine similarity in vector space
|
||||
- Graph Traversal: Multi-hop graph expansion for related entities
|
||||
- Memory Search: Vector and keyword search in memory store
|
||||
- Result Ranking: Score-based ranking and merging
|
||||
- Deduplication: Content-based result deduplication
|
||||
- Hybrid Scoring: Weighted combination of multiple retrieval sources
|
||||
|
||||
Entity Linking:
|
||||
- URI Generation: Hash-based and text-based URI assignment
|
||||
- Text Similarity: Word overlap-based similarity calculation
|
||||
- Knowledge Graph Lookup: Entity matching in knowledge graph
|
||||
- Cross-Document Linking: Entity linking across multiple documents
|
||||
- Bidirectional Linking: Symmetric relationship creation
|
||||
- Entity Web Construction: Graph-based entity connection web
|
||||
|
||||
Key Features:
|
||||
- High-level interface (AgentContext) for easy use
|
||||
- Context graph construction from entities, relationships, and conversations
|
||||
@@ -48,16 +13,9 @@ Key Features:
|
||||
- Entity linking across sources with URI assignment
|
||||
- Hybrid context retrieval (vector + graph + memory)
|
||||
- Conversation history management
|
||||
- Context accumulation and synthesis
|
||||
- Graph-based context traversal and querying
|
||||
- Method registry for custom context methods
|
||||
- Configuration management with environment variables and config files
|
||||
- Boolean flags for common options (user-friendly)
|
||||
- Auto-detection of content types and retrieval strategies
|
||||
|
||||
Main Classes:
|
||||
- AgentContext: High-level interface for agent context management (store,
|
||||
retrieve, forget, conversation)
|
||||
- AgentContext: High-level interface for agent context management
|
||||
- ContextGraph: In-memory context graph store and builder methods
|
||||
- ContextNode: Context graph node data structure
|
||||
- ContextEdge: Context graph edge data structure
|
||||
@@ -68,41 +26,13 @@ Main Classes:
|
||||
- LinkedEntity: Linked entity with context
|
||||
- ContextRetriever: Retrieves relevant context from multiple sources
|
||||
- RetrievedContext: Retrieved context item data structure
|
||||
- MethodRegistry: Registry for custom context methods (accessed via registry
|
||||
submodule)
|
||||
- ContextConfig: Configuration manager for context module (accessed via
|
||||
config submodule)
|
||||
|
||||
Submodules:
|
||||
- methods: Context engineering methods (build_context_graph, store_memory,
|
||||
retrieve_context, etc.)
|
||||
- registry: Method registry for custom methods (method_registry, MethodRegistry)
|
||||
- config: Configuration management (context_config, ContextConfig)
|
||||
|
||||
Example Usage:
|
||||
>>> # High-level interface (recommended for most users)
|
||||
>>> from semantica.context import AgentContext
|
||||
>>> context = AgentContext(vector_store=vs, knowledge_graph=kg)
|
||||
>>> memory_id = context.store("User asked about Python", conversation_id="conv1")
|
||||
>>> results = context.retrieve("Python programming")
|
||||
>>> stats = context.store(["Doc 1", "Doc 2"], extract_entities=True)
|
||||
|
||||
>>> # Low-level classes (for advanced use cases)
|
||||
>>> from semantica.context import ContextGraph, AgentMemory, methods
|
||||
>>> graph = ContextGraph()
|
||||
>>> graph_data = graph.build_from_entities_and_relationships(entities, relationships)
|
||||
>>> memory = AgentMemory(vector_store=vs, knowledge_graph=kg)
|
||||
>>> memory_id = memory.store(
|
||||
... "User asked about Python", metadata={"type": "conversation"}
|
||||
... )
|
||||
>>> results = memory.retrieve("Python", max_results=5)
|
||||
>>> # Using methods submodule
|
||||
>>> graph = methods.build_context_graph(
|
||||
... entities, relationships, method="entities_relationships"
|
||||
... )
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from .agent_context import AgentContext
|
||||
@@ -110,9 +40,6 @@ from .agent_memory import AgentMemory, MemoryItem
|
||||
from .context_graph import ContextEdge, ContextGraph, ContextNode
|
||||
from .context_retriever import ContextRetriever, RetrievedContext
|
||||
from .entity_linker import EntityLink, EntityLinker, LinkedEntity
|
||||
from . import methods
|
||||
from . import registry
|
||||
from . import config
|
||||
|
||||
__all__ = [
|
||||
# High-level interface
|
||||
@@ -128,10 +55,6 @@ __all__ = [
|
||||
"MemoryItem",
|
||||
"ContextRetriever",
|
||||
"RetrievedContext",
|
||||
# Submodules
|
||||
"methods",
|
||||
"registry",
|
||||
"config",
|
||||
]
|
||||
|
||||
# Backward compatibility alias
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
"""
|
||||
Configuration Management Module for Context Engineering
|
||||
|
||||
This module provides centralized configuration management for context engineering
|
||||
operations,
|
||||
supporting multiple configuration sources including environment variables, config files,
|
||||
and programmatic configuration.
|
||||
|
||||
Supported Configuration Sources:
|
||||
- Environment variables: CONTEXT_RETENTION_POLICY, CONTEXT_MAX_MEMORY_SIZE, etc.
|
||||
- Config files: YAML, JSON, TOML formats
|
||||
- Programmatic: Python API for setting context configurations
|
||||
|
||||
Algorithms Used:
|
||||
- Environment Variable Parsing: OS-level environment variable access
|
||||
- YAML Parsing: YAML parser for configuration file loading
|
||||
- JSON Parsing: JSON parser for configuration file loading
|
||||
- TOML Parsing: TOML parser for configuration file loading
|
||||
- Fallback Chain: Priority-based configuration resolution
|
||||
- Dictionary Merging: Deep merge algorithms for configuration updates
|
||||
|
||||
Key Features:
|
||||
- Environment variable support for context parameters
|
||||
- Config file support (YAML, JSON, TOML formats)
|
||||
- Programmatic configuration via Python API
|
||||
- Method-specific configuration management
|
||||
- Automatic fallback chain (config file -> environment -> defaults)
|
||||
- Global config instance for easy access
|
||||
|
||||
Main Classes:
|
||||
- ContextConfig: Main configuration manager class for context module
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.context.config import context_config
|
||||
>>> retention = context_config.get("retention_policy", default="unlimited")
|
||||
>>> context_config.set("retention_policy", "30_days")
|
||||
>>> method_config = context_config.get_method_config("graph")
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
|
||||
class ContextConfig:
|
||||
"""
|
||||
Configuration manager for context module.
|
||||
|
||||
Supports .env files, environment variables, and programmatic config.
|
||||
"""
|
||||
|
||||
def __init__(self, config_file: Optional[str] = None):
|
||||
"""Initialize configuration manager."""
|
||||
self.logger = get_logger("context_config")
|
||||
self._configs: Dict[str, Any] = {}
|
||||
self._method_configs: Dict[str, Dict] = {}
|
||||
self._load_config_file(config_file)
|
||||
self._load_env_vars()
|
||||
|
||||
def _load_config_file(self, config_file: Optional[str]):
|
||||
"""Load configuration from file."""
|
||||
if config_file and Path(config_file).exists():
|
||||
try:
|
||||
# Support YAML, JSON, TOML
|
||||
if config_file.endswith(".yaml") or config_file.endswith(".yml"):
|
||||
import yaml
|
||||
|
||||
with open(config_file, "r") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
self._configs.update(data.get("context", {}))
|
||||
self._method_configs.update(data.get("context_methods", {}))
|
||||
elif config_file.endswith(".json"):
|
||||
import json
|
||||
|
||||
with open(config_file, "r") as f:
|
||||
data = json.load(f) or {}
|
||||
self._configs.update(data.get("context", {}))
|
||||
self._method_configs.update(data.get("context_methods", {}))
|
||||
elif config_file.endswith(".toml"):
|
||||
import toml
|
||||
|
||||
with open(config_file, "r") as f:
|
||||
data = toml.load(f) or {}
|
||||
self._configs.update(data.get("context", {}))
|
||||
self._method_configs.update(data.get("context_methods", {}))
|
||||
self.logger.info(f"Loaded context config from {config_file}")
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to load config file {config_file}: {e}")
|
||||
|
||||
def _load_env_vars(self):
|
||||
"""Load configuration from environment variables."""
|
||||
# Context-specific environment variables with CONTEXT_ prefix
|
||||
env_prefix = "CONTEXT_"
|
||||
|
||||
for key, value in os.environ.items():
|
||||
if key.startswith(env_prefix):
|
||||
config_key = key[len(env_prefix) :].lower()
|
||||
# Try to convert to appropriate type
|
||||
if value.lower() in ("true", "false"):
|
||||
self._configs[config_key] = value.lower() == "true"
|
||||
elif value.isdigit():
|
||||
self._configs[config_key] = int(value)
|
||||
else:
|
||||
try:
|
||||
self._configs[config_key] = float(value)
|
||||
except ValueError:
|
||||
self._configs[config_key] = value
|
||||
|
||||
def set(self, key: str, value: Any):
|
||||
"""Set a configuration value."""
|
||||
self._configs[key] = value
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
"""Get a configuration value."""
|
||||
return self._configs.get(key, default)
|
||||
|
||||
def set_method_config(self, method_name: str, config: Dict[str, Any]):
|
||||
"""Set method-specific configuration."""
|
||||
self._method_configs[method_name] = config
|
||||
|
||||
def get_method_config(self, method_name: str) -> Dict[str, Any]:
|
||||
"""Get method-specific configuration."""
|
||||
return self._method_configs.get(method_name, {})
|
||||
|
||||
def get_all(self) -> Dict[str, Any]:
|
||||
"""Get all configurations."""
|
||||
return {
|
||||
"configs": self._configs.copy(),
|
||||
"method_configs": self._method_configs.copy(),
|
||||
}
|
||||
|
||||
|
||||
context_config = ContextConfig()
|
||||
@@ -10,7 +10,6 @@ This guide demonstrates how to use the Semantica context module for building con
|
||||
4. [Agent Memory Management](#agent-memory-management)
|
||||
5. [Context Retrieval](#context-retrieval)
|
||||
6. [Entity Linking](#entity-linking)
|
||||
7. [Using Methods](#using-methods)
|
||||
|
||||
## High-Level Interface (Quick Start)
|
||||
|
||||
@@ -43,9 +42,15 @@ for result in results:
|
||||
|
||||
```python
|
||||
from semantica.context import AgentContext, ContextGraph
|
||||
from semantica.graph_store import GraphStore
|
||||
|
||||
# Initialize knowledge graph
|
||||
kg = ContextGraph()
|
||||
# Initialize persistent knowledge graph (Recommended for production)
|
||||
try:
|
||||
kg = GraphStore(backend="neo4j", uri="bolt://localhost:7687", user="neo4j", password="password")
|
||||
kg.connect()
|
||||
except:
|
||||
print("Neo4j not available, falling back to in-memory graph")
|
||||
kg = ContextGraph()
|
||||
|
||||
# Initialize context with vector store and knowledge graph
|
||||
context = AgentContext(vector_store=vs, knowledge_graph=kg)
|
||||
@@ -321,25 +326,3 @@ print(uri) # e.g., "python_programming_language"
|
||||
# Similarity matching
|
||||
score = linker._calculate_text_similarity("Python", "Python Language")
|
||||
```
|
||||
|
||||
## Using Methods
|
||||
|
||||
The methods module provides simple, reusable functions for context operations.
|
||||
|
||||
```python
|
||||
from semantica.context.methods import (
|
||||
build_context_graph,
|
||||
store_memory,
|
||||
retrieve_context,
|
||||
link_entities
|
||||
)
|
||||
|
||||
# Build graph
|
||||
graph = build_context_graph(entities, relationships, method="entities_relationships")
|
||||
|
||||
# Store memory
|
||||
memory_id = store_memory("User asked about Python", vector_store=vs, method="store")
|
||||
|
||||
# Retrieve context
|
||||
results = retrieve_context("Python programming", vector_store=vs, method="hybrid")
|
||||
```
|
||||
|
||||
@@ -1,483 +0,0 @@
|
||||
"""
|
||||
Context Methods Module
|
||||
|
||||
This module provides all context engineering methods as simple, reusable functions for
|
||||
context graph construction, agent memory management, context retrieval, and entity
|
||||
linking. It supports multiple context engineering approaches and integrates with the
|
||||
method registry for extensibility.
|
||||
|
||||
Supported Methods:
|
||||
|
||||
Context Graph Construction:
|
||||
- "entities_relationships": Build graph from entities and relationships
|
||||
- "conversations": Build graph from conversations
|
||||
- "hybrid": Hybrid graph construction combining multiple sources
|
||||
|
||||
Agent Memory Management:
|
||||
- "store": Store memory items with RAG integration
|
||||
- "retrieve": Retrieve memories using vector search
|
||||
- "conversation": Conversation history management
|
||||
- "hybrid": Hybrid memory retrieval (vector + graph)
|
||||
|
||||
Context Retrieval:
|
||||
- "vector": Vector-based context retrieval
|
||||
- "graph": Graph-based context retrieval
|
||||
- "memory": Memory-based context retrieval
|
||||
- "hybrid": Hybrid retrieval combining all sources
|
||||
|
||||
Entity Linking:
|
||||
- "uri": URI assignment for entities
|
||||
- "similarity": Similarity-based entity linking
|
||||
- "knowledge_graph": Knowledge graph-based linking
|
||||
- "cross_document": Cross-document entity linking
|
||||
|
||||
Algorithms Used:
|
||||
|
||||
Context Graph Construction:
|
||||
- Graph Building: Node and edge construction from entities and relationships
|
||||
- Entity Extraction: Entity extraction from conversations and text
|
||||
- Relationship Extraction: Relationship extraction from conversations
|
||||
- Intent Extraction: Intent classification from conversations
|
||||
- Sentiment Analysis: Sentiment extraction from conversations
|
||||
- Graph Traversal: BFS/DFS for neighbor discovery and multi-hop traversal
|
||||
- Graph Indexing: Type-based indexing for efficient node/edge lookup
|
||||
|
||||
Agent Memory Management:
|
||||
- Vector Embedding: Embedding generation for memory items
|
||||
- Vector Search: Similarity search in vector space
|
||||
- Keyword Search: Fallback keyword-based search
|
||||
- Retention Policy: Time-based memory retention and cleanup
|
||||
- Memory Indexing: Deque-based memory index for efficient access
|
||||
- Knowledge Graph Integration: Entity and relationship updates to knowledge graph
|
||||
|
||||
Context Retrieval:
|
||||
- Vector Similarity Search: Cosine similarity in vector space
|
||||
- Graph Traversal: Multi-hop graph expansion for related entities
|
||||
- Memory Search: Vector and keyword search in memory store
|
||||
- Result Ranking: Score-based ranking and merging
|
||||
- Deduplication: Content-based result deduplication
|
||||
- Hybrid Scoring: Weighted combination of multiple retrieval sources
|
||||
|
||||
Entity Linking:
|
||||
- URI Generation: Hash-based and text-based URI assignment
|
||||
- Text Similarity: Word overlap-based similarity calculation
|
||||
- Knowledge Graph Lookup: Entity matching in knowledge graph
|
||||
- Cross-Document Linking: Entity linking across multiple documents
|
||||
- Bidirectional Linking: Symmetric relationship creation
|
||||
- Entity Web Construction: Graph-based entity connection web
|
||||
|
||||
Key Features:
|
||||
- Multiple context graph construction methods
|
||||
- Multiple agent memory management methods
|
||||
- Multiple context retrieval methods
|
||||
- Multiple entity linking methods
|
||||
- Method dispatchers with registry support
|
||||
- Custom method registration capability
|
||||
- Consistent interface across all methods
|
||||
|
||||
Main Functions:
|
||||
- build_context_graph: Context graph construction wrapper
|
||||
- store_memory: Memory storage wrapper
|
||||
- retrieve_context: Context retrieval wrapper
|
||||
- link_entities: Entity linking wrapper
|
||||
- get_context_method: Get context method by name
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.context.methods import build_context_graph, retrieve_context
|
||||
>>> graph = build_context_graph(
|
||||
... entities, relationships, method="entities_relationships"
|
||||
... )
|
||||
>>> results = retrieve_context("Python programming", method="hybrid", max_results=5)
|
||||
>>> from semantica.context.methods import get_context_method
|
||||
>>> method = get_context_method("graph", "custom_method")
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from typing import Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from .agent_memory import AgentMemory
|
||||
from .context_graph import ContextGraph
|
||||
from .context_retriever import ContextRetriever, RetrievedContext
|
||||
from .entity_linker import EntityLinker, LinkedEntity
|
||||
from .registry import method_registry
|
||||
|
||||
logger = get_logger("context_methods")
|
||||
|
||||
|
||||
def build_context_graph(
|
||||
entities: Optional[List[Dict[str, Any]]] = None,
|
||||
relationships: Optional[List[Dict[str, Any]]] = None,
|
||||
conversations: Optional[List[Union[str, Dict[str, Any]]]] = None,
|
||||
method: str = "entities_relationships",
|
||||
**kwargs,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Build context graph from various sources (convenience function).
|
||||
|
||||
This is a user-friendly wrapper that builds context graphs using the specified
|
||||
method.
|
||||
|
||||
Args:
|
||||
entities: List of entity dictionaries
|
||||
relationships: List of relationship dictionaries
|
||||
conversations: List of conversation files or dictionaries
|
||||
method: Graph construction method (default: "entities_relationships")
|
||||
- "entities_relationships": Build from entities and relationships
|
||||
- "conversations": Build from conversations
|
||||
- "hybrid": Hybrid construction combining multiple sources
|
||||
**kwargs: Additional options passed to ContextGraph
|
||||
|
||||
Returns:
|
||||
Context graph dictionary containing:
|
||||
- nodes: List of context nodes
|
||||
- edges: List of context edges
|
||||
- statistics: Graph statistics
|
||||
|
||||
Examples:
|
||||
>>> from semantica.context.methods import build_context_graph
|
||||
>>> entities = [{"id": "e1", "text": "Python", "type": "PROGRAMMING_LANGUAGE"}]
|
||||
>>> relationships = [
|
||||
... {"source_id": "e1", "target_id": "e2", "type": "related_to"}
|
||||
... ]
|
||||
>>> graph = build_context_graph(
|
||||
... entities, relationships, method="entities_relationships"
|
||||
... )
|
||||
>>> print(f"Graph has {graph['statistics']['node_count']} nodes")
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("graph", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(entities, relationships, conversations, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
|
||||
try:
|
||||
# Use ContextGraph as the builder
|
||||
builder = ContextGraph(**kwargs)
|
||||
|
||||
if method == "entities_relationships":
|
||||
if not entities or not relationships:
|
||||
raise ProcessingError(
|
||||
"entities and relationships required for entities_relationships "
|
||||
"method"
|
||||
)
|
||||
return builder.build_from_entities_and_relationships(
|
||||
entities, relationships, **kwargs
|
||||
)
|
||||
|
||||
elif method == "conversations":
|
||||
if not conversations:
|
||||
raise ProcessingError("conversations required for conversations method")
|
||||
return builder.build_from_conversations(conversations, **kwargs)
|
||||
|
||||
elif method == "hybrid":
|
||||
graph = {}
|
||||
if entities and relationships:
|
||||
graph1 = builder.build_from_entities_and_relationships(
|
||||
entities, relationships, **kwargs
|
||||
)
|
||||
graph = graph1
|
||||
if conversations:
|
||||
graph2 = builder.build_from_conversations(conversations, **kwargs)
|
||||
# Merge graphs
|
||||
if graph:
|
||||
graph["nodes"].extend(graph2.get("nodes", []))
|
||||
graph["edges"].extend(graph2.get("edges", []))
|
||||
else:
|
||||
graph = graph2
|
||||
return graph
|
||||
|
||||
else:
|
||||
raise ProcessingError(f"Unknown graph construction method: {method}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to build context graph: {e}", exc_info=True)
|
||||
raise ProcessingError(f"Context graph construction failed: {e}") from e
|
||||
|
||||
|
||||
def store_memory(
|
||||
content: str,
|
||||
vector_store: Optional[Any] = None,
|
||||
knowledge_graph: Optional[Any] = None,
|
||||
method: str = "store",
|
||||
**kwargs,
|
||||
) -> str:
|
||||
"""
|
||||
Store memory item (convenience function).
|
||||
|
||||
This is a user-friendly wrapper that stores memory using the specified method.
|
||||
|
||||
Args:
|
||||
content: Memory content
|
||||
vector_store: Vector store instance
|
||||
knowledge_graph: Knowledge graph instance
|
||||
method: Memory storage method (default: "store")
|
||||
- "store": Standard memory storage with RAG
|
||||
- "conversation": Conversation memory storage
|
||||
**kwargs: Additional options passed to AgentMemory
|
||||
|
||||
Returns:
|
||||
Memory ID
|
||||
|
||||
Examples:
|
||||
>>> from semantica.context.methods import store_memory
|
||||
>>> memory_id = store_memory(
|
||||
... "User asked about Python", vector_store=vs, method="store"
|
||||
... )
|
||||
>>> print(f"Stored memory: {memory_id}")
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("memory", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(content, vector_store, knowledge_graph, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
|
||||
try:
|
||||
memory = AgentMemory(
|
||||
vector_store=vector_store, knowledge_graph=knowledge_graph, **kwargs
|
||||
)
|
||||
|
||||
metadata = kwargs.get("metadata", {})
|
||||
if method == "conversation":
|
||||
metadata["type"] = "conversation"
|
||||
|
||||
return memory.store(
|
||||
content,
|
||||
metadata=metadata,
|
||||
entities=kwargs.get("entities"),
|
||||
relationships=kwargs.get("relationships"),
|
||||
**{
|
||||
k: v
|
||||
for k, v in kwargs.items()
|
||||
if k not in ["metadata", "entities", "relationships"]
|
||||
},
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to store memory: {e}", exc_info=True)
|
||||
raise ProcessingError(f"Memory storage failed: {e}") from e
|
||||
|
||||
|
||||
def retrieve_context(
|
||||
query: str,
|
||||
memory_store: Optional[Any] = None,
|
||||
knowledge_graph: Optional[Any] = None,
|
||||
vector_store: Optional[Any] = None,
|
||||
method: str = "hybrid",
|
||||
max_results: int = 5,
|
||||
**kwargs,
|
||||
) -> List[RetrievedContext]:
|
||||
"""
|
||||
Retrieve relevant context (convenience function).
|
||||
|
||||
This is a user-friendly wrapper that retrieves context using the specified method.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
memory_store: Memory store instance
|
||||
knowledge_graph: Knowledge graph instance
|
||||
vector_store: Vector store instance
|
||||
method: Retrieval method (default: "hybrid")
|
||||
- "vector": Vector-based retrieval only
|
||||
- "graph": Graph-based retrieval only
|
||||
- "memory": Memory-based retrieval only
|
||||
- "hybrid": Hybrid retrieval combining all sources
|
||||
max_results: Maximum number of results
|
||||
**kwargs: Additional options passed to ContextRetriever
|
||||
|
||||
Returns:
|
||||
List of RetrievedContext objects
|
||||
|
||||
Examples:
|
||||
>>> from semantica.context.methods import retrieve_context
|
||||
>>> results = retrieve_context(
|
||||
... "Python programming", vector_store=vs, method="hybrid"
|
||||
... )
|
||||
>>> for result in results:
|
||||
... print(f"{result.content}: {result.score:.2f}")
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("retrieval", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(
|
||||
query,
|
||||
memory_store,
|
||||
knowledge_graph,
|
||||
vector_store,
|
||||
max_results,
|
||||
**kwargs,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
|
||||
try:
|
||||
retriever = ContextRetriever(
|
||||
memory_store=memory_store,
|
||||
knowledge_graph=knowledge_graph,
|
||||
vector_store=vector_store,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if method == "vector":
|
||||
retriever.use_graph_expansion = False
|
||||
retriever.memory_store = None
|
||||
elif method == "graph":
|
||||
retriever.vector_store = None
|
||||
retriever.memory_store = None
|
||||
elif method == "memory":
|
||||
retriever.vector_store = None
|
||||
retriever.use_graph_expansion = False
|
||||
|
||||
return retriever.retrieve(query, max_results=max_results, **kwargs)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to retrieve context: {e}", exc_info=True)
|
||||
raise ProcessingError(f"Context retrieval failed: {e}") from e
|
||||
|
||||
|
||||
def link_entities(
|
||||
entities: List[Dict[str, Any]],
|
||||
knowledge_graph: Optional[Any] = None,
|
||||
method: str = "similarity",
|
||||
**kwargs,
|
||||
) -> List[LinkedEntity]:
|
||||
"""
|
||||
Link entities across sources (convenience function).
|
||||
|
||||
This is a user-friendly wrapper that links entities using the specified method.
|
||||
|
||||
Args:
|
||||
entities: List of entity dictionaries
|
||||
knowledge_graph: Knowledge graph instance
|
||||
method: Linking method (default: "similarity")
|
||||
- "uri": URI assignment only
|
||||
- "similarity": Similarity-based linking
|
||||
- "knowledge_graph": Knowledge graph-based linking
|
||||
- "cross_document": Cross-document linking
|
||||
**kwargs: Additional options passed to EntityLinker
|
||||
|
||||
Returns:
|
||||
List of LinkedEntity objects
|
||||
|
||||
Examples:
|
||||
>>> from semantica.context.methods import link_entities
|
||||
>>> entities = [{"id": "e1", "text": "Python", "type": "PROGRAMMING_LANGUAGE"}]
|
||||
>>> linked = link_entities(entities, knowledge_graph=kg, method="similarity")
|
||||
>>> for entity in linked:
|
||||
... print(f"{entity.text}: {entity.uri}")
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("linking", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(entities, knowledge_graph, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
|
||||
try:
|
||||
linker = EntityLinker(knowledge_graph=knowledge_graph, **kwargs)
|
||||
|
||||
if method == "uri":
|
||||
# Just assign URIs
|
||||
linked = []
|
||||
for entity in entities:
|
||||
entity_id = entity.get("id") or entity.get("entity_id")
|
||||
entity_text = (
|
||||
entity.get("text") or entity.get("label") or entity.get("name", "")
|
||||
)
|
||||
entity_type = entity.get("type") or entity.get("entity_type")
|
||||
uri = linker.assign_uri(entity_id, entity_text, entity_type)
|
||||
linked.append(
|
||||
LinkedEntity(
|
||||
entity_id=entity_id,
|
||||
uri=uri,
|
||||
text=entity_text,
|
||||
type=entity_type or "UNKNOWN",
|
||||
linked_entities=[],
|
||||
context=entity.get("metadata", {}),
|
||||
confidence=entity.get("confidence", 1.0),
|
||||
)
|
||||
)
|
||||
return linked
|
||||
|
||||
else:
|
||||
# Use full linking
|
||||
return linker.link(
|
||||
text="", # Not used for entity list
|
||||
entities=entities,
|
||||
context=kwargs.get("context"),
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to link entities: {e}", exc_info=True)
|
||||
raise ProcessingError(f"Entity linking failed: {e}") from e
|
||||
|
||||
|
||||
def get_context_method(task: str, name: str) -> Optional[Callable]:
|
||||
"""
|
||||
Get a registered context method.
|
||||
|
||||
Args:
|
||||
task: Task type ("graph", "memory", "retrieval", "linking")
|
||||
name: Method name
|
||||
|
||||
Returns:
|
||||
Registered method or None if not found
|
||||
|
||||
Examples:
|
||||
>>> from semantica.context.methods import get_context_method
|
||||
>>> method = get_context_method("graph", "custom_method")
|
||||
>>> if method:
|
||||
... result = method(entities, relationships)
|
||||
"""
|
||||
return method_registry.get(task, name)
|
||||
|
||||
|
||||
def list_available_methods(task: Optional[str] = None) -> Dict[str, List[str]]:
|
||||
"""
|
||||
List all available context methods.
|
||||
|
||||
Args:
|
||||
task: Optional task type filter
|
||||
|
||||
Returns:
|
||||
Dictionary mapping task types to method names
|
||||
|
||||
Examples:
|
||||
>>> from semantica.context.methods import list_available_methods
|
||||
>>> all_methods = list_available_methods()
|
||||
>>> graph_methods = list_available_methods("graph")
|
||||
"""
|
||||
return method_registry.list_all(task)
|
||||
|
||||
|
||||
# Register default methods
|
||||
method_registry.register("graph", "entities_relationships", build_context_graph)
|
||||
method_registry.register("graph", "conversations", build_context_graph)
|
||||
method_registry.register("graph", "hybrid", build_context_graph)
|
||||
method_registry.register("memory", "store", store_memory)
|
||||
method_registry.register("memory", "conversation", store_memory)
|
||||
method_registry.register("retrieval", "vector", retrieve_context)
|
||||
method_registry.register("retrieval", "graph", retrieve_context)
|
||||
method_registry.register("retrieval", "memory", retrieve_context)
|
||||
method_registry.register("retrieval", "hybrid", retrieve_context)
|
||||
method_registry.register("linking", "uri", link_entities)
|
||||
method_registry.register("linking", "similarity", link_entities)
|
||||
method_registry.register("linking", "knowledge_graph", link_entities)
|
||||
method_registry.register("linking", "cross_document", link_entities)
|
||||
@@ -1,112 +0,0 @@
|
||||
"""
|
||||
Method Registry Module for Context Engineering
|
||||
|
||||
This module provides a method registry system for registering custom context engineering
|
||||
methods, enabling extensibility and community contributions to the context toolkit.
|
||||
|
||||
Supported Registration Types:
|
||||
- Method Registry: Register custom context methods for:
|
||||
* "graph": Context graph construction methods
|
||||
* "memory": Agent memory management methods
|
||||
* "retrieval": Context retrieval methods
|
||||
* "linking": Entity linking methods
|
||||
|
||||
Algorithms Used:
|
||||
- Registry Pattern: Dictionary-based registration and lookup
|
||||
- Dynamic Registration: Runtime function registration
|
||||
- Type Checking: Type validation for registered components
|
||||
- Lookup Algorithms: Hash-based O(1) lookup for methods
|
||||
- Task-based Organization: Hierarchical organization by task type
|
||||
|
||||
Key Features:
|
||||
- Method registry for custom context methods
|
||||
- Task-based method organization (graph, memory, retrieval, linking)
|
||||
- Dynamic registration and unregistration
|
||||
- Easy discovery of available methods
|
||||
- Support for community-contributed extensions
|
||||
|
||||
Main Classes:
|
||||
- MethodRegistry: Registry for custom context methods
|
||||
|
||||
Global Instances:
|
||||
- method_registry: Global method registry instance
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.context.registry import method_registry
|
||||
>>> method_registry.register("graph", "custom_method", custom_graph_function)
|
||||
>>> available = method_registry.list_all("graph")
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
|
||||
class MethodRegistry:
|
||||
"""Registry for custom context methods."""
|
||||
|
||||
_methods: Dict[str, Dict[str, Callable]] = {
|
||||
"graph": {},
|
||||
"memory": {},
|
||||
"retrieval": {},
|
||||
"linking": {},
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def register(cls, task: str, name: str, method_func: Callable):
|
||||
"""
|
||||
Register a method for a specific task.
|
||||
|
||||
Args:
|
||||
task: Task type ("graph", "memory", "retrieval", "linking")
|
||||
name: Method name
|
||||
method_func: Method function or callable
|
||||
"""
|
||||
if task not in cls._methods:
|
||||
cls._methods[task] = {}
|
||||
cls._methods[task][name] = method_func
|
||||
|
||||
@classmethod
|
||||
def get(cls, task: str, name: str) -> Optional[Callable]:
|
||||
"""
|
||||
Get a registered method.
|
||||
|
||||
Args:
|
||||
task: Task type
|
||||
name: Method name
|
||||
|
||||
Returns:
|
||||
Registered method or None if not found
|
||||
"""
|
||||
return cls._methods.get(task, {}).get(name)
|
||||
|
||||
@classmethod
|
||||
def list_all(cls, task: Optional[str] = None) -> Dict[str, List[str]]:
|
||||
"""
|
||||
List all registered methods.
|
||||
|
||||
Args:
|
||||
task: Optional task type filter
|
||||
|
||||
Returns:
|
||||
Dictionary mapping task types to method names
|
||||
"""
|
||||
if task:
|
||||
return {task: list(cls._methods.get(task, {}).keys())}
|
||||
return {t: list(m.keys()) for t, m in cls._methods.items()}
|
||||
|
||||
@classmethod
|
||||
def unregister(cls, task: str, name: str):
|
||||
"""
|
||||
Unregister a method.
|
||||
|
||||
Args:
|
||||
task: Task type
|
||||
name: Method name
|
||||
"""
|
||||
if task in cls._methods and name in cls._methods[task]:
|
||||
del cls._methods[task][name]
|
||||
|
||||
|
||||
method_registry = MethodRegistry()
|
||||
@@ -32,7 +32,7 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -724,6 +724,321 @@ class GraphStore:
|
||||
"""Create an index."""
|
||||
return self._manager.create_index(label, property_name, index_type, **options)
|
||||
|
||||
# Compatibility with AgentMemory / ContextGraph interface
|
||||
def add_nodes(self, nodes: List[Dict[str, Any]], **options) -> int:
|
||||
"""
|
||||
Add nodes (Compatibility method).
|
||||
|
||||
Args:
|
||||
nodes: List of node dictionaries with 'id', 'type', 'properties'
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Number of nodes created
|
||||
"""
|
||||
if not nodes:
|
||||
return 0
|
||||
|
||||
# Convert to GraphStore format (labels, properties)
|
||||
graph_nodes = []
|
||||
for node in nodes:
|
||||
# Extract label from type
|
||||
labels = [node.get("type", "Entity")]
|
||||
if isinstance(labels[0], str):
|
||||
labels = [labels[0]] # Ensure list
|
||||
|
||||
# Prepare properties
|
||||
props = node.get("properties", {}).copy()
|
||||
|
||||
# Ensure ID is preserved
|
||||
if "id" in node and "id" not in props:
|
||||
props["id"] = node["id"]
|
||||
|
||||
# Ensure content/text is preserved
|
||||
if "content" in node and "content" not in props:
|
||||
props["content"] = node["content"]
|
||||
if "text" in node and "text" not in props:
|
||||
props["text"] = node["text"]
|
||||
|
||||
graph_nodes.append({
|
||||
"labels": labels,
|
||||
"properties": props
|
||||
})
|
||||
|
||||
# Use batch creation
|
||||
# Note: create_nodes expects dicts with 'labels' and 'properties' keys if passed directly?
|
||||
# Let's check create_nodes signature implementation in manager.
|
||||
# But here I'll assume create_nodes takes a list of such dicts or similar.
|
||||
# Actually, let's look at create_nodes wrapper in this file:
|
||||
# def create_nodes(self, nodes: List[Dict[str, Any]], **options)
|
||||
# It passes to self._manager.nodes.create_batch(nodes)
|
||||
|
||||
# If create_batch expects specific format, I should match it.
|
||||
# Assuming create_batch is smart enough or expects standard format.
|
||||
# To be safe, let's look at NodeManager.create_batch if possible, but I can't easily.
|
||||
# Standard expectation: List of dicts where each dict has labels and properties.
|
||||
|
||||
result = self.create_nodes(graph_nodes, **options)
|
||||
return len(result)
|
||||
|
||||
def add_edges(self, edges: List[Dict[str, Any]], **options) -> int:
|
||||
"""
|
||||
Add edges (Compatibility method).
|
||||
|
||||
Args:
|
||||
edges: List of edge dictionaries
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Number of edges created
|
||||
"""
|
||||
count = 0
|
||||
for edge in edges:
|
||||
source_id = edge.get("source_id")
|
||||
target_id = edge.get("target_id")
|
||||
rel_type = edge.get("type", "RELATED_TO")
|
||||
properties = edge.get("properties", {}).copy()
|
||||
|
||||
# Preserve weight
|
||||
if "weight" in edge:
|
||||
properties["weight"] = edge["weight"]
|
||||
|
||||
if source_id and target_id:
|
||||
try:
|
||||
self.create_relationship(source_id, target_id, rel_type, properties, **options)
|
||||
count += 1
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to add edge {source_id}->{target_id}: {e}")
|
||||
return count
|
||||
|
||||
def build_from_conversations(
|
||||
self,
|
||||
conversations: List[Union[str, Dict[str, Any]]],
|
||||
link_entities: bool = True,
|
||||
extract_intents: bool = False,
|
||||
extract_sentiments: bool = False,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Build graph from conversations (Compatibility method).
|
||||
|
||||
Args:
|
||||
conversations: List of conversation files or dictionaries
|
||||
link_entities: Link entities across conversations
|
||||
extract_intents: Extract intents (not implemented)
|
||||
extract_sentiments: Extract sentiments (not implemented)
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Graph statistics
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=None,
|
||||
module="graph_store",
|
||||
submodule="GraphStore",
|
||||
message=f"Building graph from {len(conversations)} conversations",
|
||||
)
|
||||
|
||||
try:
|
||||
all_nodes = []
|
||||
all_edges = []
|
||||
seen_nodes = set()
|
||||
|
||||
for conv in conversations:
|
||||
# Load conversation if string (file path)
|
||||
conv_data = conv
|
||||
if isinstance(conv, str):
|
||||
from pathlib import Path
|
||||
from ..utils.helpers import read_json_file
|
||||
conv_data = read_json_file(Path(conv))
|
||||
|
||||
nodes, edges = self._process_conversation_to_elements(
|
||||
conv_data,
|
||||
extract_intents=extract_intents,
|
||||
extract_sentiments=extract_sentiments
|
||||
)
|
||||
|
||||
# Add unique nodes
|
||||
for node in nodes:
|
||||
if node["id"] not in seen_nodes:
|
||||
all_nodes.append(node)
|
||||
seen_nodes.add(node["id"])
|
||||
|
||||
all_edges.extend(edges)
|
||||
|
||||
if link_entities:
|
||||
linked_edges = self._link_entities_elements(all_nodes)
|
||||
all_edges.extend(linked_edges)
|
||||
|
||||
# Batch add
|
||||
node_count = self.add_nodes(all_nodes)
|
||||
edge_count = self.add_edges(all_edges)
|
||||
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed")
|
||||
|
||||
return {
|
||||
"statistics": {
|
||||
"node_count": node_count,
|
||||
"edge_count": edge_count
|
||||
}
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
self.logger.error(f"Failed to build graph: {e}")
|
||||
raise
|
||||
|
||||
def build_from_entities_and_relationships(
|
||||
self,
|
||||
entities: List[Dict[str, Any]],
|
||||
relationships: List[Dict[str, Any]],
|
||||
**kwargs,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Build graph from entities and relationships (Compatibility method).
|
||||
"""
|
||||
nodes = []
|
||||
edges = []
|
||||
|
||||
# Process entities
|
||||
for entity in entities:
|
||||
entity_id = entity.get("id") or entity.get("entity_id")
|
||||
if entity_id:
|
||||
nodes.append({
|
||||
"id": entity_id,
|
||||
"type": entity.get("type", "entity"),
|
||||
"properties": {
|
||||
"content": entity.get("text") or entity.get("label") or entity_id,
|
||||
**entity
|
||||
}
|
||||
})
|
||||
|
||||
# Process relationships
|
||||
for rel in relationships:
|
||||
source = rel.get("source_id")
|
||||
target = rel.get("target_id")
|
||||
if source and target:
|
||||
edges.append({
|
||||
"source_id": source,
|
||||
"target_id": target,
|
||||
"type": rel.get("type", "related_to"),
|
||||
"weight": rel.get("confidence", 1.0),
|
||||
"properties": rel
|
||||
})
|
||||
|
||||
node_count = self.add_nodes(nodes)
|
||||
edge_count = self.add_edges(edges)
|
||||
|
||||
return {"statistics": {"node_count": node_count, "edge_count": edge_count}}
|
||||
|
||||
def _process_conversation_to_elements(self, conv_data: Dict[str, Any], **kwargs) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
||||
"""Helper to process conversation into nodes and edges."""
|
||||
nodes = []
|
||||
edges = []
|
||||
|
||||
conv_id = conv_data.get("id") or f"conv_{hash(str(conv_data)) % 10000}"
|
||||
|
||||
# Conversation node
|
||||
nodes.append({
|
||||
"id": conv_id,
|
||||
"type": "conversation",
|
||||
"properties": {
|
||||
"content": conv_data.get("content", "") or conv_data.get("summary", ""),
|
||||
"timestamp": conv_data.get("timestamp")
|
||||
}
|
||||
})
|
||||
|
||||
name_to_id = {}
|
||||
extract_entities = kwargs.get("extract_entities", True) # Default true if not passed?
|
||||
# Actually ContextGraph defaults to True in init, but here we are static.
|
||||
# Let's assume True unless told otherwise or check config.
|
||||
|
||||
# Extract entities
|
||||
for entity in conv_data.get("entities", []):
|
||||
entity_id = entity.get("id") or entity.get("entity_id")
|
||||
entity_text = entity.get("text") or entity.get("label") or entity.get("name") or entity_id
|
||||
entity_type = entity.get("type", "entity")
|
||||
|
||||
# Generate ID if missing
|
||||
if not entity_id and entity_text:
|
||||
import hashlib
|
||||
entity_hash = hashlib.md5(f"{entity_text}_{entity_type}".encode()).hexdigest()[:12]
|
||||
entity_id = f"{entity_type.lower()}_{entity_hash}"
|
||||
|
||||
if entity_id:
|
||||
if entity_text:
|
||||
name_to_id[entity_text] = entity_id
|
||||
|
||||
nodes.append({
|
||||
"id": entity_id,
|
||||
"type": "entity", # Normalize type?
|
||||
"properties": {
|
||||
"content": entity_text,
|
||||
"type": entity_type,
|
||||
**entity
|
||||
}
|
||||
})
|
||||
|
||||
# Edge: Conversation -> Entity
|
||||
edges.append({
|
||||
"source_id": conv_id,
|
||||
"target_id": entity_id,
|
||||
"type": "mentions"
|
||||
})
|
||||
|
||||
# Extract relationships
|
||||
for rel in conv_data.get("relationships", []):
|
||||
source = rel.get("source_id")
|
||||
target = rel.get("target_id")
|
||||
|
||||
# Resolve IDs
|
||||
if not source and rel.get("source") and rel.get("source") in name_to_id:
|
||||
source = name_to_id[rel.get("source")]
|
||||
if not target and rel.get("target") and rel.get("target") in name_to_id:
|
||||
target = name_to_id[rel.get("target")]
|
||||
|
||||
if source and target:
|
||||
edges.append({
|
||||
"source_id": source,
|
||||
"target_id": target,
|
||||
"type": rel.get("type", "related_to"),
|
||||
"weight": rel.get("confidence", 1.0),
|
||||
"properties": rel
|
||||
})
|
||||
|
||||
return nodes, edges
|
||||
|
||||
def _link_entities_elements(self, nodes: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""Link similar entities."""
|
||||
edges = []
|
||||
# Lazy import to avoid circular dependency
|
||||
try:
|
||||
from ..context.entity_linker import EntityLinker
|
||||
linker = EntityLinker() # Use default config
|
||||
except ImportError:
|
||||
return []
|
||||
|
||||
entity_nodes = [n for n in nodes if n.get("type") == "entity"]
|
||||
for i, node1 in enumerate(entity_nodes):
|
||||
content1 = node1["properties"].get("content", "")
|
||||
if not content1: continue
|
||||
|
||||
for node2 in entity_nodes[i + 1 :]:
|
||||
content2 = node2["properties"].get("content", "")
|
||||
if not content2: continue
|
||||
|
||||
similarity = linker._calculate_text_similarity(content1.lower(), content2.lower())
|
||||
if similarity >= linker.similarity_threshold:
|
||||
edges.append({
|
||||
"source_id": node1["id"],
|
||||
"target_id": node2["id"],
|
||||
"type": "similar_to",
|
||||
"weight": similarity
|
||||
})
|
||||
return edges
|
||||
|
||||
@property
|
||||
def nodes(self) -> NodeManager:
|
||||
"""Get node manager."""
|
||||
|
||||
@@ -1,197 +0,0 @@
|
||||
|
||||
import logging
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
# Set up logging to see what's happening under the hood
|
||||
# Using WARNING to keep the output clean for the notebook demonstration
|
||||
logging.basicConfig(level=logging.WARNING, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
|
||||
# Import all the powerful tools from Semantica
|
||||
from semantica.kg import (
|
||||
GraphBuilder,
|
||||
GraphAnalyzer,
|
||||
GraphValidator,
|
||||
ConnectivityAnalyzer,
|
||||
CentralityCalculator,
|
||||
CommunityDetector,
|
||||
TemporalGraphQuery,
|
||||
ProvenanceTracker
|
||||
)
|
||||
from semantica.deduplication import DuplicateDetector
|
||||
from semantica.conflicts import ConflictDetector, ConflictResolver
|
||||
|
||||
# Our "Raw" Messy Data
|
||||
raw_entities = [
|
||||
{"id": "startup_1", "type": "Startup", "name": "TechFlow AI", "revenue": 1000000, "founded": "2021-01-01"},
|
||||
{"id": "startup_2", "type": "Startup", "name": "GreenEnergy Co", "revenue": 500000, "founded": "2020-05-15"},
|
||||
{"id": "startup_1_dup", "type": "Startup", "name": "TechFlow Inc.", "revenue": 1200000, "founded": "2021-01-01"}, # Duplicate!
|
||||
{"id": "investor_1", "type": "Investor", "name": "Venture Capital X"},
|
||||
{"id": "founder_1", "type": "Person", "name": "Alice Chen"},
|
||||
{"id": "founder_2", "type": "Person", "name": "Bob Smith"}
|
||||
]
|
||||
|
||||
raw_relationships = [
|
||||
# Valid Relationships
|
||||
{"source": "founder_1", "target": "startup_1", "type": "FOUNDED", "valid_from": "2021-01-01"},
|
||||
{"source": "investor_1", "target": "startup_1", "type": "INVESTED_IN", "amount": 5000000, "valid_from": "2023-06-01"},
|
||||
|
||||
# Dangling Edge (Error!)
|
||||
{"source": "founder_2", "target": "startup_999", "type": "FOUNDED", "valid_from": "2020-05-15"},
|
||||
|
||||
# Temporal Data (History)
|
||||
{"source": "founder_1", "target": "startup_2", "type": "ADVISED", "valid_from": "2020-01-01", "valid_until": "2021-01-01"}
|
||||
]
|
||||
|
||||
print(f"Loaded {len(raw_entities)} raw entities and {len(raw_relationships)} raw relationships.")
|
||||
|
||||
# Initialize Validator
|
||||
validator = GraphValidator()
|
||||
|
||||
# Create a temporary graph object for validation
|
||||
temp_graph = {"entities": raw_entities, "relationships": raw_relationships}
|
||||
|
||||
# Run Validation
|
||||
print("Running Validation Check...")
|
||||
validation_result = validator.validate(temp_graph)
|
||||
|
||||
if not validation_result.is_valid:
|
||||
print("Validation Failed! Issues found:")
|
||||
for issue in validation_result.issues:
|
||||
print(f" - [{issue.severity.name}] {issue.message} (Code: {issue.code})")
|
||||
|
||||
# AUTOMATIC FIX: If it's a dangling edge, remove it
|
||||
if issue.code == "DANGLING_EDGE":
|
||||
print(" Auto-Fixing: Removing invalid relationship...")
|
||||
raw_relationships = [r for r in raw_relationships
|
||||
if r['target'] != issue.details.get('target_id')]
|
||||
else:
|
||||
print("Graph is valid!")
|
||||
|
||||
# Re-validate to confirm fix
|
||||
print("\nRe-validating after fixes...")
|
||||
temp_graph = {"entities": raw_entities, "relationships": raw_relationships}
|
||||
if validator.validate(temp_graph).is_valid:
|
||||
print("Graph is now clean and valid!")
|
||||
|
||||
# 1. Detect Duplicates
|
||||
print("Scanning for duplicates...")
|
||||
deduper = DuplicateDetector(similarity_threshold=0.7) # 70% similarity threshold
|
||||
duplicates = deduper.detect_duplicates(raw_entities)
|
||||
|
||||
for candidate in duplicates:
|
||||
print(f"Found potential duplicate pair (Score: {candidate.similarity_score:.2f}):")
|
||||
print(f" - {candidate.entity1['name']} (ID: {candidate.entity1['id']})")
|
||||
print(f" - {candidate.entity2['name']} (ID: {candidate.entity2['id']})")
|
||||
|
||||
# MERGE STRATEGY: Keep entity1, merge data from entity2
|
||||
print(" Merging entities...")
|
||||
# (In a real app, you'd use EntityMerger, but here's the logic:)
|
||||
# We keep startup_1 and discard startup_1_dup, but we note the conflict
|
||||
|
||||
# 2. Detect Conflicts
|
||||
print("\nChecking for data conflicts...")
|
||||
conflict_detector = ConflictDetector()
|
||||
|
||||
# Simulating a conflict check between the two versions of TechFlow
|
||||
# To check conflicts, we treat them as the same entity (same ID)
|
||||
entity_a = raw_entities[0].copy()
|
||||
entity_b = raw_entities[2].copy()
|
||||
entity_b['id'] = entity_a['id'] # Force same ID for conflict detection
|
||||
|
||||
conflicts = conflict_detector.detect_conflicts([entity_a, entity_b])
|
||||
|
||||
for conflict in conflicts:
|
||||
print(f" Conflict detected in field '{conflict.property_name}':")
|
||||
print(f" Values: {conflict.conflicting_values}")
|
||||
|
||||
# RESOLUTION: Trust the higher number (optimistic!)
|
||||
if conflict.property_name == "revenue":
|
||||
# values are strings or ints, need to handle types
|
||||
vals = [float(v) for v in conflict.conflicting_values if v is not None]
|
||||
resolved_val = max(vals)
|
||||
print(f" Resolved to: {resolved_val}")
|
||||
raw_entities[0]['revenue'] = resolved_val
|
||||
|
||||
# Final Cleanup: Remove the duplicate entity from our list
|
||||
clean_entities = [e for e in raw_entities if e['id'] != 'startup_1_dup']
|
||||
clean_relationships = raw_relationships # (We'd normally re-link relationships too)
|
||||
|
||||
print(f"\nCleaned Data: {len(clean_entities)} entities remaining.")
|
||||
|
||||
|
||||
# Manual Graph Construction (since we already cleaned it)
|
||||
kg = {
|
||||
"entities": clean_entities,
|
||||
"relationships": clean_relationships,
|
||||
"metadata": {
|
||||
"created_at": datetime.now().isoformat(),
|
||||
"source": "Manual Advanced Pipeline"
|
||||
}
|
||||
}
|
||||
print("Knowledge Graph Assembled Successfully!")
|
||||
|
||||
# Initialize the Master Analyzer
|
||||
analyzer = GraphAnalyzer(enable_temporal=True)
|
||||
|
||||
# 1. Structural Analysis (Connectivity)
|
||||
print("\n--- Connectivity Analysis ---")
|
||||
connectivity = analyzer.analyze_connectivity(kg)
|
||||
print(f" • Graph Connected? {'Yes' if connectivity['is_connected'] else 'No'}")
|
||||
print(f" • Connected Components: {connectivity['num_components']}")
|
||||
|
||||
# 2. Centrality (Who is important?)
|
||||
print("\n--- Centrality Analysis ---")
|
||||
centrality_result = analyzer.calculate_centrality(kg, centrality_type="degree")
|
||||
degree_data = centrality_result["centrality_measures"]["degree"]
|
||||
|
||||
# Get pre-calculated rankings
|
||||
top_nodes = degree_data["rankings"][:3]
|
||||
|
||||
print(" • Top Influencers (Degree Centrality):")
|
||||
for item in top_nodes:
|
||||
print(f" - {item['node']}: {item['score']:.2f}")
|
||||
|
||||
# 3. Community Detection (Clustering)
|
||||
print("\n--- Community Detection ---")
|
||||
communities = analyzer.detect_communities(kg, algorithm="louvain")
|
||||
community_result = communities
|
||||
communities = community_result["communities"]
|
||||
|
||||
print(f" • Detected {len(communities)} communities.")
|
||||
for i, comm in enumerate(communities):
|
||||
# comm is a set of node IDs
|
||||
members = list(comm)
|
||||
print(f" Community {i+1}: {', '.join(members)}")
|
||||
|
||||
temporal_engine = TemporalGraphQuery(temporal_granularity="year")
|
||||
|
||||
# 1. Time Travel Query: What did the world look like in 2020?
|
||||
print("\n--- Time Travel: 2020 ---")
|
||||
snapshot_2020 = temporal_engine.query_at_time(kg, query="*", at_time="2020-06-01")
|
||||
print(f" Active Relationships in 2020: {len(snapshot_2020['relationships'])}")
|
||||
for rel in snapshot_2020['relationships']:
|
||||
print(f" - {rel['source']} --[{rel['type']}]--> {rel['target']}")
|
||||
|
||||
# 2. Time Travel Query: What about 2023?
|
||||
print("\n--- Time Travel: 2023 ---")
|
||||
snapshot_2023 = temporal_engine.query_at_time(kg, query="*", at_time="2023-07-01")
|
||||
print(f" Active Relationships in 2023: {len(snapshot_2023['relationships'])}")
|
||||
for rel in snapshot_2023['relationships']:
|
||||
print(f" - {rel['source']} --[{rel['type']}]--> {rel['target']}")
|
||||
|
||||
# Notice how 'ADVISED' might disappear if it ended, and 'INVESTED_IN' appears!
|
||||
|
||||
tracker = ProvenanceTracker()
|
||||
|
||||
# Let's pretend we're tracking the source of our data
|
||||
tracker.track_entity("startup_1", source="Crunchbase_API_v2", metadata={"confidence": 0.95})
|
||||
tracker.track_entity("startup_1", source="Manual_Entry_User_Bob", metadata={"confidence": 1.0})
|
||||
|
||||
print("\n--- Provenance Report: TechFlow AI ---")
|
||||
lineage = tracker.get_lineage("startup_1")
|
||||
print(f" Entity: startup_1")
|
||||
print(f" First Seen: {lineage['first_seen']}")
|
||||
print(f" Sources:")
|
||||
for src in lineage['sources']:
|
||||
print(f" - {src['source']} (at {src['timestamp']})")
|
||||
@@ -13,7 +13,6 @@ from semantica.context.context_graph import ContextGraph, ContextNode, ContextEd
|
||||
from semantica.context.agent_memory import AgentMemory, MemoryItem
|
||||
from semantica.context.context_retriever import ContextRetriever, RetrievedContext
|
||||
from semantica.context.agent_context import AgentContext
|
||||
from semantica.context import methods
|
||||
|
||||
class MockVectorStore:
|
||||
def __init__(self):
|
||||
@@ -143,19 +142,5 @@ class TestContextModule(unittest.TestCase):
|
||||
self.assertIsNotNone(ctx._memory)
|
||||
self.assertEqual(len(ctx._memory.short_term_memory), 1)
|
||||
|
||||
# --- Method Wrapper Tests ---
|
||||
def test_method_wrappers(self):
|
||||
# Test retrieve_context wrapper
|
||||
# We need to patch ContextRetriever inside the method or just check if it runs
|
||||
# Since it creates a new ContextRetriever internally, we can mock the class in the module
|
||||
|
||||
with patch('semantica.context.methods.ContextRetriever') as MockRetriever:
|
||||
instance = MockRetriever.return_value
|
||||
instance.retrieve.return_value = []
|
||||
|
||||
results = methods.retrieve_context("query", vector_store=self.mock_vector_store)
|
||||
self.assertIsInstance(results, list)
|
||||
MockRetriever.assert_called_once()
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user