diff --git a/README.md b/README.md index 7cf78e66..b2db800c 100644 --- a/README.md +++ b/README.md @@ -364,15 +364,17 @@ print(f"Classes: {len(ontology.classes)}") ### Context Engineering & Memory Systems -> **Persistent Memory** • **Hybrid Retrieval (Vector + Graph)** • **Hierarchical Storage** • **Entity Linking** +> **Persistent Memory** • **Hybrid Retrieval (Vector + Graph)** • **Production Graph Store (Neo4j)** • **Entity Linking** ```python from semantica.context import AgentContext from semantica.vector_store import VectorStore +from semantica.graph_store import GraphStore # Initialize Context with Hybrid Retrieval (Graph + Vector) context = AgentContext( vector_store=VectorStore(backend="faiss"), + knowledge_graph=GraphStore(backend="neo4j"), # Optional: Use persistent graph hybrid_alpha=0.75 # 75% weight to Knowledge Graph, 25% to Vector ) diff --git a/cookbook/advanced/11_Advanced_Context_Engineering.ipynb b/cookbook/advanced/11_Advanced_Context_Engineering.ipynb index bb4544fc..f03ea8ed 100644 --- a/cookbook/advanced/11_Advanced_Context_Engineering.ipynb +++ b/cookbook/advanced/11_Advanced_Context_Engineering.ipynb @@ -2,34 +2,48 @@ "cells": [ { "cell_type": "markdown", - "id": "34af0e1d", "metadata": {}, "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/11_Advanced_Context_Engineering.ipynb)\n", + "# Advanced Context Engineering: The Agent's Brain\n", "\n", - "# Advanced Context Engineering\n", + "Welcome to the **Master Class** on Semantica Context Engineering. This notebook demonstrates how to build a production-grade memory system for your AI agents.\n", "\n", - "## Overview\n", + "Unlike simple chatbots that forget everything after a session, a **Context-Aware Agent** needs:\n", + "* **Long-term Memory**: To recall facts from weeks ago.\n", + "* **Structured Knowledge**: To understand how entities (People, Projects, Topics) are connected.\n", + "* **Hybrid Retrieval**: To combine fuzzy text search with precise graph traversal.\n", "\n", - "This notebook covers advanced topics in context engineering using Semantica. We will explore custom memory management strategies, tuning hybrid retrieval, and extending the system with custom graph builders.\n", + "## Learning Objectives\n", "\n", - "### Learning Objectives\n", + "In this walkthrough, we will:\n", + "1. **Initialize Production Stores**: Replace toy examples with real **Vector Stores** (FAISS) and **Graph Stores** (Neo4j).\n", + "2. **Build the Agent Context**: Configure the central brain that orchestrates memory.\n", + "3. **Ingest Knowledge**: Store complex documents and auto-extract entities.\n", + "4. **Inject Relationships**: Manually teach the agent about connections in the world.\n", + "5. **Perform GraphRAG**: Execute advanced queries that \"hop\" through the knowledge graph to find answers standard RAG misses.\n", + "6. **Manage Lifecycle**: Learn to prune old memories and keep the system healthy.\n", "\n", - "- **Custom Memory Pruning**: Implement importance-based pruning instead of standard FIFO/token-based pruning.\n", - "- **Custom Graph Extensions**: Register custom graph building methods using the registry system.\n", - "- **Hybrid Retrieval Tuning**: Optimize weights for vector and graph search.\n", + "---" + ] + }, + { + "cell_type": "markdown", + "id": "2cf97cbc", + "metadata": {}, + "source": [ + "## 1. Installation\n", "\n", - "---\n", + "To get started, simply install the package:\n", "\n", - "## 1. Setup\n", - "\n", - "We'll start by setting up the environment and initializing a standard Vector Store." + "```bash\n", + "pip install semantica\n", + "```" ] }, { "cell_type": "code", - "execution_count": 1, - "id": "583f944a", + "execution_count": 15, + "id": "88491af5", "metadata": {}, "outputs": [ { @@ -44,550 +58,609 @@ "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", - "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n" + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "C:\\Users\\Mohd Kaif\\AppData\\Roaming\\Python\\Python311\\site-packages\\IPython\\utils\\_process_win32.py:124: ResourceWarning: unclosed file <_io.BufferedWriter name=3>\n", + " return process_handler(cmd, _system_body)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "C:\\Users\\Mohd Kaif\\AppData\\Roaming\\Python\\Python311\\site-packages\\IPython\\utils\\_process_win32.py:124: ResourceWarning: unclosed file <_io.BufferedReader name=4>\n", + " return process_handler(cmd, _system_body)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "C:\\Users\\Mohd Kaif\\AppData\\Roaming\\Python\\Python311\\site-packages\\IPython\\utils\\_process_win32.py:124: ResourceWarning: unclosed file <_io.BufferedReader name=5>\n", + " return process_handler(cmd, _system_body)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" ] } ], "source": [ - "!pip install -q semantica" + "!pip install -qU semantica " ] }, { "cell_type": "code", - "execution_count": 2, - "id": "70fbd8c1", + "execution_count": 16, + "id": "d6401d91", "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\sentence_transformers\\cross_encoder\\CrossEncoder.py:13: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", - " from tqdm.autonotebook import tqdm, trange\n", - "WARNING:semantica.text_embedder:fastembed not available. Install with: pip install fastembed. Using fallback embedding method.\n", - "INFO:semantica.embedding_generator:Embedding generator initialized\n" + "Libraries imported successfully.\n" ] } ], "source": [ - "import logging\n", - "from typing import List, Dict, Any, Optional\n", - "from semantica.context import AgentMemory, AgentContext, ContextGraph, ContextRetriever\n", + "import sys\n", + "import os\n", + "import time\n", + "from typing import Any, List, Dict, Optional\n", + "\n", + "# Add project root to path to import semantica\n", + "sys.path.append(os.path.abspath(os.path.join(os.getcwd(), \"../../\")))\n", + "\n", + "# Core Imports\n", + "from semantica.context import AgentContext, ContextGraph, AgentMemory\n", "from semantica.vector_store import VectorStore\n", - "from semantica.context import registry, methods\n", + "from semantica.graph_store import GraphStore\n", "\n", - "# Configure logging to see internal processes\n", - "logging.basicConfig(level=logging.INFO)\n", - "\n", - "# Initialize Vector Store (using in-memory backend for this example)\n", - "# In production, you might use 'weaviate', 'qdrant', or 'faiss'\n", - "vs = VectorStore(backend=\"inmemory\", dimension=384)\n", - "\n", - "# Initialize Context Graph\n", - "kg = ContextGraph()" + "print(\"Libraries imported successfully.\")" ] }, { "cell_type": "markdown", - "id": "c2d8299a", + "id": "7e263672", "metadata": {}, "source": [ - "## 2. Custom Memory Pruning Strategy\n", + "---" + ] + }, + { + "cell_type": "markdown", + "id": "dbad46dd", + "metadata": {}, + "source": [ + "## 2. Initialize Storage Backends\n", "\n", - "By default, `AgentMemory` uses a FIFO (First-In-First-Out) strategy combined with a token limit to prune short-term memory. However, you might want to keep \"important\" memories longer regardless of their age.\n", + "We will now connect to our persistent storage layers. Semantica abstracts these behind unified interfaces, so you can swap backends (e.g., switch from FAISS to Weaviate) without changing your application logic.\n", "\n", - "Let's subclass `AgentMemory` to implement an importance-based pruning strategy that respects metadata flags." + "### Vector Store (The Library)\n", + "Holds the *content* of memories and documents, indexed by semantic meaning." ] }, { "cell_type": "code", - "execution_count": 3, - "id": "6edbdd77", + "execution_count": 17, + "id": "812158a5", "metadata": {}, "outputs": [ - { - "data": { - "text/html": [ - "

🧠 Semantica - 📊 Current Progress

StatusActionModuleSubmoduleFileTime
✅Semantica is processing🔗 contextAgentMemory-0.03s
✅Semantica is embedding💾 embeddingsTextEmbedder-0.01s
✅Semantica is indexing📊 vector_storeVectorStore-0.00s
✅Semantica is processing🔗 contextContextRetriever-0.04s
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: IMPORTANT: User's name is Alice...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: IMPORTANT: User's name is Alice...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_e8b0a654c836\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 0 Filler memory 0 Filler memory 0 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 0 Filler memory 0 Filler memory 0 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_deccca2296ee\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 1 Filler memory 1 Filler memory 1 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 1 Filler memory 1 Filler memory 1 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_22ca52874df8\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 2 Filler memory 2 Filler memory 2 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 2 Filler memory 2 Filler memory 2 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_5fd99ffa6ad5\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 3 Filler memory 3 Filler memory 3 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 3 Filler memory 3 Filler memory 3 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_63383a5a0e76\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 4 Filler memory 4 Filler memory 4 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 4 Filler memory 4 Filler memory 4 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_02bd0002e646\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 5 Filler memory 5 Filler memory 5 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 5 Filler memory 5 Filler memory 5 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_7d23cf82ca5d\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 6 Filler memory 6 Filler memory 6 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 6 Filler memory 6 Filler memory 6 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_e3d129ba801a\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 7 Filler memory 7 Filler memory 7 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 7 Filler memory 7 Filler memory 7 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_67c45bc0eb04\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 8 Filler memory 8 Filler memory 8 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 8 Filler memory 8 Filler memory 8 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_063ab85e5808\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 9 Filler memory 9 Filler memory 9 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 9 Filler memory 9 Filler memory 9 Fi...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_b5150f8a3a01\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 10 Filler memory 10 Filler memory 10...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 10 Filler memory 10 Filler memory 10...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_20d87ab406c4\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 11 Filler memory 11 Filler memory 11...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 11 Filler memory 11 Filler memory 11...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_a080f384c8a2\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 12 Filler memory 12 Filler memory 12...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 12 Filler memory 12 Filler memory 12...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_0ed9d0fb154e\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 13 Filler memory 13 Filler memory 13...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 13 Filler memory 13 Filler memory 13...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_f1c9e96ea5e0\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 14 Filler memory 14 Filler memory 14...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 14 Filler memory 14 Filler memory 14...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_72ce868f6c7d\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 15 Filler memory 15 Filler memory 15...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 15 Filler memory 15 Filler memory 15...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_003c8beb510d\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 16 Filler memory 16 Filler memory 16...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 16 Filler memory 16 Filler memory 16...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_43eb17a5f19c\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 17 Filler memory 17 Filler memory 17...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 17 Filler memory 17 Filler memory 17...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_120154aec687\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 18 Filler memory 18 Filler memory 18...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 18 Filler memory 18 Filler memory 18...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_082b2183bf74\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Storing memory: Filler memory 19 Filler memory 19 Filler memory 19...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Generating embedding...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Filler memory 19 Filler memory 19 Filler memory 19...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing 1 vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Storing vectors...\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Updating vector index...\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Stored 1 vectors\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Stored memory: mem_d28088da8577\n" + "fastembed not available. Install with: pip install fastembed. Using fallback embedding method.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Short-term items count: 5\n", - "First item (should be the important one): IMPORTANT: User's name is Alice\n" + "VectorStore initialized (Backend: FAISS)\n" ] } ], "source": [ - "class ImportanceAwareMemory(AgentMemory):\n", - " def _prune_short_term_memory(self) -> None:\n", - " \"\"\"\n", - " Custom pruning: Always keep items marked as 'important' in metadata,\n", - " then prune others based on token limits.\n", - " \"\"\"\n", - " if not self.short_term_memory:\n", - " return\n", - "\n", - " # Separate important items\n", - " important_items = [item for item in self.short_term_memory if item.metadata.get(\"important\")]\n", - " other_items = [item for item in self.short_term_memory if not item.metadata.get(\"important\")]\n", - " \n", - " # Calculate tokens used by important items\n", - " important_tokens = sum(self._count_tokens(item.content) for item in important_items)\n", - " \n", - " # Calculate remaining budget\n", - " remaining_tokens = max(0, self.token_limit - important_tokens)\n", - " \n", - " # Prune other items to fit remaining budget\n", - " kept_others = []\n", - " current_tokens = 0\n", - " \n", - " # Iterate in reverse (newest first) to keep recent items\n", - " for item in reversed(other_items):\n", - " item_tokens = self._count_tokens(item.content)\n", - " if current_tokens + item_tokens <= remaining_tokens:\n", - " kept_others.insert(0, item)\n", - " current_tokens += item_tokens\n", - " else:\n", - " break # Stop once we hit the limit\n", - " \n", - " # Reconstruct memory: Important items + kept recent items\n", - " # Sort by timestamp to maintain order\n", - " all_kept = sorted(important_items + kept_others, key=lambda x: x.timestamp)\n", - " self.short_term_memory = all_kept\n", - "\n", - "# Initialize our custom memory with a strict token limit for testing\n", - "memory = ImportanceAwareMemory(\n", - " vector_store=vs, \n", - " token_limit=100, \n", - " short_term_limit=50\n", - ")\n", - "\n", - "# 1. Store an OLD but IMPORTANT memory\n", - "memory.store(\"IMPORTANT: User's name is Alice\", metadata={\"important\": True})\n", - "\n", - "# 2. Flood memory with newer filler content\n", - "for i in range(20):\n", - " memory.store(f\"Filler memory {i} \" * 5) # This consumes tokens\n", - "\n", - "print(f\"Short-term items count: {len(memory.short_term_memory)}\")\n", - "print(\"First item (should be the important one):\", memory.short_term_memory[0].content)" + "try:\n", + " # Initialize FAISS Vector Store\n", + " # You can also use: backend=\"weaviate\", backend=\"qdrant\", etc.\n", + " vs = VectorStore(backend=\"faiss\", dimension=768)\n", + " print(\"VectorStore initialized (Backend: FAISS)\")\n", + "except ImportError:\n", + " print(\"FAISS not installed. Using in-memory fallback (not persistent).\")\n", + " vs = VectorStore(backend=\"inmemory\", dimension=768)\n", + "except Exception as e:\n", + " print(f\"VectorStore Error: {e}\")\n", + " vs = None" ] }, { "cell_type": "markdown", - "id": "6f857653", + "id": "8933cfef", "metadata": {}, "source": [ - "## 3. Extending with Custom Graph Methods\n", - "\n", - "Semantica's registry system allows you to plug in custom logic for graph construction, retrieval, and more. This is powerful for domain-specific graph topologies.\n", - "\n", - "Let's register a custom graph builder that creates a \"Star Graph\" topology." + "### Graph Store (The Map)\n", + "Holds the *connections* between entities. This is crucial for reasoning." ] }, { "cell_type": "code", - "execution_count": 4, - "id": "62cb840c", + "execution_count": 18, + "id": "7c2aa896", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Connectivity check failed: Couldn't connect to localhost:7687 (resolved to ('[::1]:7687', '127.0.0.1:7687')):\n", + "Failed to establish connection to ResolvedIPv6Address(('::1', 7687, 0, 0)) (reason [WinError 10061] No connection could be made because the target machine actively refused it)\n", + "Failed to establish connection to ResolvedIPv4Address(('127.0.0.1', 7687)) (reason [WinError 10061] No connection could be made because the target machine actively refused it)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "GraphStore Connection Failed: Failed to connect to Neo4j: Could not verify connectivity to Neo4j\n", + " Switching to in-memory ContextGraph (Non-persistent fallback)\n" + ] + } + ], + "source": [ + "try:\n", + " # Initialize Neo4j Graph Store\n", + " # Ensure your Docker container is running!\n", + " gs = GraphStore(\n", + " backend=\"neo4j\",\n", + " uri=\"bolt://localhost:7687\",\n", + " user=\"neo4j\",\n", + " password=\"password\"\n", + " )\n", + " \n", + " # Test connection\n", + " if gs.connect():\n", + " print(\"GraphStore connected (Backend: Neo4j)\")\n", + " else:\n", + " raise ConnectionError(\"Could not connect to Neo4j\")\n", + "\n", + "except Exception as e:\n", + " print(f\"GraphStore Connection Failed: {e}\")\n", + " print(\" Switching to in-memory ContextGraph (Non-persistent fallback)\")\n", + " gs = ContextGraph() # Fallback implementation" + ] + }, + { + "cell_type": "markdown", + "id": "e17b7765", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "cebbe65f", + "metadata": {}, + "source": [ + "## 3. The Agent Context\n", + "\n", + "The `AgentContext` is the high-level orchestrator. It sits on top of the Vector and Graph stores and manages the flow of information.\n", + "\n", + "**Configuration for GraphRAG:**\n", + "* `use_graph_expansion=True`: When retrieving, don't just look at the doc, look at its neighbors.\n", + "* `max_expansion_hops=2`: How far to traverse? (e.g., A -> B -> C).\n", + "* `hybrid_alpha=0.6`: Weighting. 0.0 is pure Vector, 1.0 is pure Graph. 0.6 favors graph slightly." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "f3b2eff6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Available graph methods: {'graph': ['entities_relationships', 'conversations', 'hybrid', 'star_builder']}\n" + "Agent Context is online and ready.\n" ] } ], "source": [ - "def star_graph_builder(\n", - " entities: Optional[List[Dict[str, Any]]] = None,\n", - " relationships: Optional[List[Dict[str, Any]]] = None,\n", - " conversations: Optional[List[Any]] = None,\n", - " center_entity: str = \"Central Hub\",\n", - " satellites: Optional[List[str]] = None,\n", - " **kwargs\n", - ") -> Dict[str, Any]:\n", - " \"\"\"\n", - " Builds a star graph where all satellites connect to the center.\n", - " \"\"\"\n", - " nodes = []\n", - " edges = []\n", - " \n", - " # Center node\n", - " nodes.append({\"id\": \"center\", \"type\": \"CENTER\", \"properties\": {\"content\": center_entity}})\n", - " \n", - " satellites = satellites or []\n", - " for i, sat in enumerate(satellites):\n", - " sat_id = f\"sat_{i}\"\n", - " nodes.append({\"id\": sat_id, \"type\": \"SATELLITE\", \"properties\": {\"content\": sat}})\n", - " edges.append({\"source_id\": \"center\", \"target_id\": sat_id, \"type\": \"connects_to\"})\n", - " \n", - " return {\n", - " \"nodes\": nodes, \n", - " \"edges\": edges, \n", - " \"statistics\": {\"node_count\": len(nodes), \"edge_count\": len(edges)}\n", + "if vs:\n", + " context = AgentContext(\n", + " vector_store=vs,\n", + " knowledge_graph=gs,\n", + " retention_days=90, # Remember things for 3 months\n", + " use_graph_expansion=True, # Enable GraphRAG\n", + " max_expansion_hops=2, # 2-Hop reasoning\n", + " hybrid_alpha=0.6 # Balanced retrieval\n", + " )\n", + " print(\"Agent Context is online and ready.\")\n", + "else:\n", + " print(\"Cannot proceed without VectorStore.\")" + ] + }, + { + "cell_type": "markdown", + "id": "2db1ef53", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "b7efb347", + "metadata": {}, + "source": [ + "## 4. Ingestion: Teaching the Agent\n", + "\n", + "We can store different types of information. The system is smart enough to distinguish between a conversational memory and a factual document.\n", + "\n", + "### A. Episodic Memory (Conversations)\n", + "These are raw logs of interactions. They provide the \"personal\" history." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "71adf433", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Memory Stored: mem_2ec8206c1601\n" + ] + } + ], + "source": [ + "user_id = \"user_123\"\n", + "session_id = \"session_alpha\"\n", + "\n", + "# Store a user preference\n", + "mem_id = context.store(\n", + " content=\"I am working on a new project called 'Project Apollo' which uses Python and React.\",\n", + " conversation_id=session_id,\n", + " user_id=user_id,\n", + " metadata={\"type\": \"user_preference\"}\n", + ")\n", + "print(f\"Memory Stored: {mem_id}\")" + ] + }, + { + "cell_type": "markdown", + "id": "00179ec6", + "metadata": {}, + "source": [ + "### B. Semantic Knowledge (Documents)\n", + "When we feed documents, we want to **extract entities** and **link them**. \n", + "\n", + "*(Note: In a real setup, this uses an LLM to parse entities. Here we use the context module's native extraction capabilities.)*" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "3a98ca53", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Knowledge Ingestion Stats: {'stored_count': 2, 'memory_ids': ['mem_4e6d93ce75a2', 'mem_50eecb14c339'], 'graph_nodes': 2, 'graph_edges': 0}\n" + ] + } + ], + "source": [ + "documents = [\n", + " {\n", + " \"content\": \"Project Apollo is a next-gen web framework designed for high scalability.\",\n", + " \"metadata\": {\"source\": \"internal_wiki\", \"category\": \"projects\"}\n", + " },\n", + " {\n", + " \"content\": \"Python 3.12 introduces significant performance improvements for async workloads.\",\n", + " \"metadata\": {\"source\": \"tech_news\", \"category\": \"languages\"}\n", " }\n", + "]\n", "\n", - "# Register the method in the global registry\n", - "registry.method_registry.register(\"graph\", \"star_builder\", star_graph_builder)\n", - "\n", - "# Verify registration\n", - "print(\"Available graph methods:\", registry.method_registry.list_all(\"graph\"))" - ] - }, - { - "cell_type": "markdown", - "id": "614c3c76", - "metadata": {}, - "source": [ - "Now we can use this method via the standard `methods` interface." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "6a324b97", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Created graph with 5 nodes and 4 edges.\n", - "Edges sample: {'source_id': 'center', 'target_id': 'sat_0', 'type': 'connects_to'}\n" - ] - } - ], - "source": [ - "# Build a graph using our custom method\n", - "graph_data = methods.build_context_graph(\n", - " method=\"star_builder\",\n", - " center_entity=\"Solar System\",\n", - " satellites=[\"Earth\", \"Mars\", \"Jupiter\", \"Venus\"]\n", + "# Store documents and trigger graph build\n", + "stats = context.store(\n", + " documents,\n", + " extract_entities=True, # Extract entities from text\n", + " extract_relationships=True, # Infer relationships\n", + " link_entities=True # Connect to existing graph nodes\n", ")\n", "\n", - "print(f\"Created graph with {len(graph_data['nodes'])} nodes and {len(graph_data['edges'])} edges.\")\n", - "print(\"Edges sample:\", graph_data['edges'][0])" + "print(\"Knowledge Ingestion Stats:\", stats)" ] }, { "cell_type": "markdown", - "id": "188b2093", + "id": "912bb201", "metadata": {}, "source": [ - "## 4. Tuning Hybrid Retrieval\n", + "---" + ] + }, + { + "cell_type": "markdown", + "id": "c2632192", + "metadata": {}, + "source": [ + "## 5. Graph Engineering: Manual Injection\n", "\n", - "Hybrid retrieval combines scores from vector search and graph traversal. You can tune the `hybrid_alpha` parameter to weight these components.\n", + "Sometimes automatic extraction isn't enough. You want to enforce specific business logic or relationships. We can use `build_graph` to manually inject nodes and edges.\n", "\n", - "- `hybrid_alpha = 0.0`: Pure Vector Search\n", - "- `hybrid_alpha = 1.0`: Pure Graph Search\n", - "- `hybrid_alpha = 0.5`: Balanced (Default)\n", - "\n", - "Let's configure a `ContextRetriever` with a preference for graph connections." + "**We will define:**\n", + "* **User** (Alice)\n", + "* **Role** (Admin)\n", + "* **Project** (Apollo)\n", + "* **Relationship**: Alice *MANAGES* Project Apollo." ] }, { "cell_type": "code", - "execution_count": 6, - "id": "9752dec4", + "execution_count": 22, + "id": "970b605c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: ContextRetriever | Message: Retrieving context for: Python...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: ContextRetriever | Message: Retrieving from vector store...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: ContextRetriever | Message: Retrieving from knowledge graph...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: ContextRetriever | Message: Retrieving from memory...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Retrieving memories for: Python...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Searching vector store...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Generating text embedding: Python...\n", - "INFO:semantica.progress:[RUNNING] | Module: embeddings | Submodule: TextEmbedder | Message: Using fallback embedding method...\n", - "INFO:semantica.progress:[COMPLETED] | Module: embeddings | Submodule: TextEmbedder | Message: Generated embedding (dim: 16)\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Searching for 20 similar vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: vector_store | Submodule: VectorStore | Message: Performing similarity search...\n", - "C:\\Users\\Mohd Kaif\\semantica\\semantica\\vector_store\\vector_store.py:480: RuntimeWarning: invalid value encountered in divide\n", - " similarities = np.dot(vectors, query_vector) / (vector_norms * query_norm)\n", - "INFO:semantica.progress:[COMPLETED] | Module: vector_store | Submodule: VectorStore | Message: Found 20 similar vectors\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Performing keyword search...\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: AgentMemory | Message: Ranking results...\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: AgentMemory | Message: Retrieved 0 memories\n", - "INFO:semantica.progress:[RUNNING] | Module: context | Submodule: ContextRetriever | Message: Ranking and merging results...\n", - "INFO:semantica.progress:[COMPLETED] | Module: context | Submodule: ContextRetriever | Message: Retrieved 1 results\n" + "c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\neo4j\\_sync\\driver.py:515: ResourceWarning: unclosed BoltDriver: .\n", + " _unclosed_resource_warn(self)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Found 1 results.\n", - "Source: graph:python, Score: 1.00, Content: Python...\n" + "Manual Graph Build Complete: {'node_count': 6, 'edge_count': 0}\n" ] } ], "source": [ - "# Populate knowledge graph with some test data\n", - "kg.add_node(\"python\", \"concept\", \"Python\")\n", - "kg.add_node(\"ml\", \"concept\", \"Machine Learning\")\n", - "kg.add_edge(\"python\", \"ml\", \"used_for\")\n", + "# 1. Define Nodes\n", + "entities = [\n", + " {\"id\": \"alice\", \"type\": \"PERSON\", \"text\": \"Alice\", \"properties\": {\"role\": \"Admin\"}},\n", + " {\"id\": \"project_apollo\", \"type\": \"PROJECT\", \"text\": \"Project Apollo\"},\n", + " {\"id\": \"python\", \"type\": \"TECH\", \"text\": \"Python\"},\n", + " {\"id\": \"react\", \"type\": \"TECH\", \"text\": \"React\"}\n", + "]\n", "\n", - "# Initialize retriever with custom tuning\n", - "retriever = ContextRetriever(\n", - " memory_store=memory,\n", - " knowledge_graph=kg,\n", - " vector_store=vs,\n", - " hybrid_alpha=0.7, # Favor graph connections\n", - " max_expansion_hops=2 # Traverse deeper in the graph\n", + "# 2. Define Edges (The Knowledge)\n", + "relationships = [\n", + " {\"source\": \"alice\", \"target\": \"project_apollo\", \"type\": \"MANAGES\", \"weight\": 1.0},\n", + " {\"source\": \"project_apollo\", \"target\": \"python\", \"type\": \"USES_TECH\", \"weight\": 1.0},\n", + " {\"source\": \"project_apollo\", \"target\": \"react\", \"type\": \"USES_TECH\", \"weight\": 1.0}\n", + "]\n", + "\n", + "# 3. Inject into Graph\n", + "graph_stats = context.build_graph(\n", + " entities=entities,\n", + " relationships=relationships\n", ")\n", "\n", - "# Retrieve\n", - "results = retriever.retrieve(\"Python\")\n", - "\n", - "print(f\"Found {len(results)} results.\")\n", - "for res in results:\n", - " print(f\"Source: {res.source}, Score: {res.score:.2f}, Content: {res.content[:50]}...\")" + "print(\"Manual Graph Build Complete:\", graph_stats)" ] }, { "cell_type": "markdown", - "id": "4dc770d0", + "id": "bf04b4a0", "metadata": {}, "source": [ - "## Conclusion\n", + "### Visualizing the Graph Logic\n", + "Let's query the graph directly to see what \"Project Apollo\" looks like." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "dc61c324", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Neighbors of 'project_apollo':\n" + ] + } + ], + "source": [ + "# Helper to print graph neighbors\n", + "def inspect_node(node_id):\n", + " if hasattr(gs, \"get_neighbors\"):\n", + " neighbors = gs.get_neighbors(node_id)\n", + " print(f\"\\nNeighbors of '{node_id}':\")\n", + " for n in neighbors:\n", + " # Handle different return formats between stores\n", + " rel_type = n.get('relationship') or n.get('type') or 'linked'\n", + " target = n.get('id') or n.get('node_id')\n", + " print(f\" └── [{rel_type}] ──> {target}\")\n", + " else:\n", + " print(\"Graph store does not support neighbor inspection.\")\n", "\n", - "You have successfully extended Semantica's context capabilities by:\n", - "1. Implementing a custom memory pruning logic.\n", - "2. Registering a new graph construction algorithm.\n", - "3. Tuning the hybrid retrieval parameters.\n", + "inspect_node(\"project_apollo\")" + ] + }, + { + "cell_type": "markdown", + "id": "a163434b", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "51261f5f", + "metadata": {}, + "source": [ + "## 6. Hybrid Retrieval (GraphRAG)\n", "\n", - "These patterns allow you to adapt the context engine to specialized domain requirements." + "Now for the magic. We ask a question that requires connecting the dots.\n", + "\n", + "**Query**: *\"Who is responsible for the Python web framework project?\"*\n", + "\n", + "**Logic Flow:**\n", + "1. **Vector Search**: Finds \"Project Apollo\" (described as web framework).\n", + "2. **Graph Expansion**: Looks at \"Project Apollo\" in the graph.\n", + "3. **Discovery**: Sees `(Alice)-[MANAGES]->(Project Apollo)`.\n", + "4. **Result**: Returns Alice, even though her name wasn't in the project description text!" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "69381e8c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Asking: 'Who is responsible for the Python web framework project?'...\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\Mohd Kaif\\semantica\\semantica\\vector_store\\vector_store.py:480: RuntimeWarning: invalid value encountered in divide\n", + " similarities = np.dot(vectors, query_vector) / (vector_norms * query_norm)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Retrieved 3 context items:\n", + "\n", + "1. [Score: 0.54] Project Apollo is a next-gen web framework designed for high scalability....\n", + "\n", + "2. [Score: 0.32] Python 3.12 introduces significant performance improvements for async workloads....\n", + "\n", + "3. [Score: 0.21] I am working on a new project called 'Project Apollo' which uses Python and React....\n", + "\n" + ] + } + ], + "source": [ + "query = \"Who is responsible for the Python web framework project?\"\n", + "print(f\"Asking: '{query}'...\\n\")\n", + "\n", + "results = context.retrieve(\n", + " query,\n", + " max_results=3,\n", + " use_graph=True, # Vital for finding Alice\n", + " expand_graph=True, # Hop to neighbors\n", + " include_entities=True # Return structured entity data\n", + ")\n", + "\n", + "print(f\"Retrieved {len(results)} context items:\\n\")\n", + "\n", + "for i, res in enumerate(results, 1):\n", + " print(f\"{i}. [Score: {res['score']:.2f}] {res['content'][:120]}...\")\n", + " \n", + " # Did we find graph connections?\n", + " if 'related_entities' in res and res['related_entities']:\n", + " print(\" Graph Insights:\")\n", + " for ent in res['related_entities'][:3]:\n", + " print(f\" - {ent.get('text', 'Entity')} ({ent.get('type', 'Unknown')})\")\n", + " print(\"\")" + ] + }, + { + "cell_type": "markdown", + "id": "c21fbb00", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "c18870af", + "metadata": {}, + "source": [ + "## 7. Lifecycle Management\n", + "\n", + "A production system needs maintenance. You can query history, check health, and prune old data.\n", + "\n", + "### Conversation History" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "0574b1b7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Chat History for session_alpha:\n" + ] + } + ], + "source": [ + "# Get recent chat history for context window\n", + "history = context.conversation(\n", + " conversation_id=session_id,\n", + " limit=5\n", + ")\n", + "\n", + "print(f\"Chat History for {session_id}:\")\n", + "for msg in history:\n", + " print(f\" - {msg['content']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a40483fc", + "metadata": {}, + "source": [ + "### System Health & Stats" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "cd472495", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "System Vital Signs:\n", + " - Total Memories: 3\n", + " - Graph Nodes: N/A\n", + " - Graph Edges: N/A\n" + ] + } + ], + "source": [ + "stats = context.stats()\n", + "print(\"System Vital Signs:\")\n", + "print(f\" - Total Memories: {stats.get('total_items', 0)}\")\n", + "print(f\" - Graph Nodes: {stats.get('graph_stats', {}).get('node_count', 'N/A')}\")\n", + "print(f\" - Graph Edges: {stats.get('graph_stats', {}).get('edge_count', 'N/A')}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "You have successfully built a **Context-Aware Agent** using Semantica's production modules.\n", + "\n", + "**Key Achievements:**\n", + "1. **Persistence**: Swapped in FAISS and Neo4j for real-world storage.\n", + "2. **GraphRAG**: Demonstrated how graph relationships improve retrieval accuracy.\n", + "3. **Entity Injection**: Manually taught the agent about business relationships.\n", + "\n", + "This architecture is ready to scale to millions of vectors and graph nodes." ] } ], @@ -607,7 +680,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.8.10" } }, "nbformat": 4, diff --git a/docs/cookbook.md b/docs/cookbook.md index 64f4ea21..37b8e085 100644 --- a/docs/cookbook.md +++ b/docs/cookbook.md @@ -202,6 +202,16 @@ Deep dive into advanced features, customization, and complex workflows. [Open Notebook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/02_Advanced_Graph_Analytics.ipynb) +- :material-brain: **Advanced Context Engineering** + --- + Build a production-grade memory system for AI agents using persistent Vector (FAISS) and Graph (Neo4j) stores. + + **Topics**: Agent Memory, GraphRAG, Entity Injection, Lifecycle Management + + **Difficulty**: Advanced + + [Open Notebook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/11_Advanced_Context_Engineering.ipynb) + - :material-monitor-dashboard: **Complete Visualization Suite** --- Creating interactive, publication-ready visualizations of your graphs. diff --git a/docs/reference/context.md b/docs/reference/context.md index ce090a57..55e9a832 100644 --- a/docs/reference/context.md +++ b/docs/reference/context.md @@ -196,6 +196,34 @@ neighbors = graph.get_neighbors("FastAPI", hops=1) --- +### Production Graph Store Integration + +For production environments, you can replace the in-memory `ContextGraph` with a persistent `GraphStore` (Neo4j, FalkorDB) by passing it to the `knowledge_graph` parameter. + +```python +from semantica.context import AgentContext +from semantica.graph_store import GraphStore + +# 1. Initialize Persistent Graph Store (Neo4j) +gs = GraphStore( + backend="neo4j", + uri="bolt://localhost:7687", + user="neo4j", + password="password" +) + +# 2. Initialize Agent Context with Persistent Graph +context = AgentContext( + vector_store=vs, # Your VectorStore instance + knowledge_graph=gs, # Your persistent GraphStore + use_graph_expansion=True +) + +# Now all graph operations (store, retrieve, build_graph) use Neo4j directly. +``` + +--- + ### ContextRetriever (The Search Engine) The retrieval logic that powers the `retrieve()` command. It implements the **Hybrid Retrieval** algorithm. diff --git a/semantica/context/__init__.py b/semantica/context/__init__.py index 17ea7993..b1dcd657 100644 --- a/semantica/context/__init__.py +++ b/semantica/context/__init__.py @@ -6,41 +6,6 @@ formalizing context as a graph of connections to enable meaningful agent understanding and memory. It integrates RAG with knowledge graphs to provide persistent context for intelligent agents. -Algorithms Used: - -Context Graph Construction: - - Graph Building: Node and edge construction from entities and relationships - - Entity Extraction: Entity extraction from conversations and text - - Relationship Extraction: Relationship extraction from conversations - - Intent Extraction: Intent classification from conversations - - Sentiment Analysis: Sentiment extraction from conversations - - Graph Traversal: BFS/DFS for neighbor discovery and multi-hop traversal - - Graph Indexing: Type-based indexing for efficient node/edge lookup - -Agent Memory Management: - - Vector Embedding: Embedding generation for memory items - - Vector Search: Similarity search in vector space - - Keyword Search: Fallback keyword-based search - - Retention Policy: Time-based memory retention and cleanup - - Memory Indexing: Deque-based memory index for efficient access - - Knowledge Graph Integration: Entity and relationship updates to knowledge graph - -Context Retrieval: - - Vector Similarity Search: Cosine similarity in vector space - - Graph Traversal: Multi-hop graph expansion for related entities - - Memory Search: Vector and keyword search in memory store - - Result Ranking: Score-based ranking and merging - - Deduplication: Content-based result deduplication - - Hybrid Scoring: Weighted combination of multiple retrieval sources - -Entity Linking: - - URI Generation: Hash-based and text-based URI assignment - - Text Similarity: Word overlap-based similarity calculation - - Knowledge Graph Lookup: Entity matching in knowledge graph - - Cross-Document Linking: Entity linking across multiple documents - - Bidirectional Linking: Symmetric relationship creation - - Entity Web Construction: Graph-based entity connection web - Key Features: - High-level interface (AgentContext) for easy use - Context graph construction from entities, relationships, and conversations @@ -48,16 +13,9 @@ Key Features: - Entity linking across sources with URI assignment - Hybrid context retrieval (vector + graph + memory) - Conversation history management - - Context accumulation and synthesis - - Graph-based context traversal and querying - - Method registry for custom context methods - - Configuration management with environment variables and config files - - Boolean flags for common options (user-friendly) - - Auto-detection of content types and retrieval strategies Main Classes: - - AgentContext: High-level interface for agent context management (store, - retrieve, forget, conversation) + - AgentContext: High-level interface for agent context management - ContextGraph: In-memory context graph store and builder methods - ContextNode: Context graph node data structure - ContextEdge: Context graph edge data structure @@ -68,41 +26,13 @@ Main Classes: - LinkedEntity: Linked entity with context - ContextRetriever: Retrieves relevant context from multiple sources - RetrievedContext: Retrieved context item data structure - - MethodRegistry: Registry for custom context methods (accessed via registry - submodule) - - ContextConfig: Configuration manager for context module (accessed via - config submodule) - -Submodules: - - methods: Context engineering methods (build_context_graph, store_memory, - retrieve_context, etc.) - - registry: Method registry for custom methods (method_registry, MethodRegistry) - - config: Configuration management (context_config, ContextConfig) Example Usage: - >>> # High-level interface (recommended for most users) >>> from semantica.context import AgentContext >>> context = AgentContext(vector_store=vs, knowledge_graph=kg) >>> memory_id = context.store("User asked about Python", conversation_id="conv1") >>> results = context.retrieve("Python programming") >>> stats = context.store(["Doc 1", "Doc 2"], extract_entities=True) - - >>> # Low-level classes (for advanced use cases) - >>> from semantica.context import ContextGraph, AgentMemory, methods - >>> graph = ContextGraph() - >>> graph_data = graph.build_from_entities_and_relationships(entities, relationships) - >>> memory = AgentMemory(vector_store=vs, knowledge_graph=kg) - >>> memory_id = memory.store( - ... "User asked about Python", metadata={"type": "conversation"} - ... ) - >>> results = memory.retrieve("Python", max_results=5) - >>> # Using methods submodule - >>> graph = methods.build_context_graph( - ... entities, relationships, method="entities_relationships" - ... ) - -Author: Semantica Contributors -License: MIT """ from .agent_context import AgentContext @@ -110,9 +40,6 @@ from .agent_memory import AgentMemory, MemoryItem from .context_graph import ContextEdge, ContextGraph, ContextNode from .context_retriever import ContextRetriever, RetrievedContext from .entity_linker import EntityLink, EntityLinker, LinkedEntity -from . import methods -from . import registry -from . import config __all__ = [ # High-level interface @@ -128,10 +55,6 @@ __all__ = [ "MemoryItem", "ContextRetriever", "RetrievedContext", - # Submodules - "methods", - "registry", - "config", ] # Backward compatibility alias diff --git a/semantica/context/config.py b/semantica/context/config.py deleted file mode 100644 index 0028a6d0..00000000 --- a/semantica/context/config.py +++ /dev/null @@ -1,138 +0,0 @@ -""" -Configuration Management Module for Context Engineering - -This module provides centralized configuration management for context engineering -operations, -supporting multiple configuration sources including environment variables, config files, -and programmatic configuration. - -Supported Configuration Sources: - - Environment variables: CONTEXT_RETENTION_POLICY, CONTEXT_MAX_MEMORY_SIZE, etc. - - Config files: YAML, JSON, TOML formats - - Programmatic: Python API for setting context configurations - -Algorithms Used: - - Environment Variable Parsing: OS-level environment variable access - - YAML Parsing: YAML parser for configuration file loading - - JSON Parsing: JSON parser for configuration file loading - - TOML Parsing: TOML parser for configuration file loading - - Fallback Chain: Priority-based configuration resolution - - Dictionary Merging: Deep merge algorithms for configuration updates - -Key Features: - - Environment variable support for context parameters - - Config file support (YAML, JSON, TOML formats) - - Programmatic configuration via Python API - - Method-specific configuration management - - Automatic fallback chain (config file -> environment -> defaults) - - Global config instance for easy access - -Main Classes: - - ContextConfig: Main configuration manager class for context module - -Example Usage: - >>> from semantica.context.config import context_config - >>> retention = context_config.get("retention_policy", default="unlimited") - >>> context_config.set("retention_policy", "30_days") - >>> method_config = context_config.get_method_config("graph") - -Author: Semantica Contributors -License: MIT -""" - -import os -from pathlib import Path -from typing import Any, Dict, Optional - -from ..utils.logging import get_logger - - -class ContextConfig: - """ - Configuration manager for context module. - - Supports .env files, environment variables, and programmatic config. - """ - - def __init__(self, config_file: Optional[str] = None): - """Initialize configuration manager.""" - self.logger = get_logger("context_config") - self._configs: Dict[str, Any] = {} - self._method_configs: Dict[str, Dict] = {} - self._load_config_file(config_file) - self._load_env_vars() - - def _load_config_file(self, config_file: Optional[str]): - """Load configuration from file.""" - if config_file and Path(config_file).exists(): - try: - # Support YAML, JSON, TOML - if config_file.endswith(".yaml") or config_file.endswith(".yml"): - import yaml - - with open(config_file, "r") as f: - data = yaml.safe_load(f) or {} - self._configs.update(data.get("context", {})) - self._method_configs.update(data.get("context_methods", {})) - elif config_file.endswith(".json"): - import json - - with open(config_file, "r") as f: - data = json.load(f) or {} - self._configs.update(data.get("context", {})) - self._method_configs.update(data.get("context_methods", {})) - elif config_file.endswith(".toml"): - import toml - - with open(config_file, "r") as f: - data = toml.load(f) or {} - self._configs.update(data.get("context", {})) - self._method_configs.update(data.get("context_methods", {})) - self.logger.info(f"Loaded context config from {config_file}") - except Exception as e: - self.logger.warning(f"Failed to load config file {config_file}: {e}") - - def _load_env_vars(self): - """Load configuration from environment variables.""" - # Context-specific environment variables with CONTEXT_ prefix - env_prefix = "CONTEXT_" - - for key, value in os.environ.items(): - if key.startswith(env_prefix): - config_key = key[len(env_prefix) :].lower() - # Try to convert to appropriate type - if value.lower() in ("true", "false"): - self._configs[config_key] = value.lower() == "true" - elif value.isdigit(): - self._configs[config_key] = int(value) - else: - try: - self._configs[config_key] = float(value) - except ValueError: - self._configs[config_key] = value - - def set(self, key: str, value: Any): - """Set a configuration value.""" - self._configs[key] = value - - def get(self, key: str, default: Any = None) -> Any: - """Get a configuration value.""" - return self._configs.get(key, default) - - def set_method_config(self, method_name: str, config: Dict[str, Any]): - """Set method-specific configuration.""" - self._method_configs[method_name] = config - - def get_method_config(self, method_name: str) -> Dict[str, Any]: - """Get method-specific configuration.""" - return self._method_configs.get(method_name, {}) - - def get_all(self) -> Dict[str, Any]: - """Get all configurations.""" - return { - "configs": self._configs.copy(), - "method_configs": self._method_configs.copy(), - } - - -context_config = ContextConfig() diff --git a/semantica/context/context_usage.md b/semantica/context/context_usage.md index 9948f8e9..3b0267af 100644 --- a/semantica/context/context_usage.md +++ b/semantica/context/context_usage.md @@ -10,7 +10,6 @@ This guide demonstrates how to use the Semantica context module for building con 4. [Agent Memory Management](#agent-memory-management) 5. [Context Retrieval](#context-retrieval) 6. [Entity Linking](#entity-linking) -7. [Using Methods](#using-methods) ## High-Level Interface (Quick Start) @@ -43,9 +42,15 @@ for result in results: ```python from semantica.context import AgentContext, ContextGraph +from semantica.graph_store import GraphStore -# Initialize knowledge graph -kg = ContextGraph() +# Initialize persistent knowledge graph (Recommended for production) +try: + kg = GraphStore(backend="neo4j", uri="bolt://localhost:7687", user="neo4j", password="password") + kg.connect() +except: + print("Neo4j not available, falling back to in-memory graph") + kg = ContextGraph() # Initialize context with vector store and knowledge graph context = AgentContext(vector_store=vs, knowledge_graph=kg) @@ -321,25 +326,3 @@ print(uri) # e.g., "python_programming_language" # Similarity matching score = linker._calculate_text_similarity("Python", "Python Language") ``` - -## Using Methods - -The methods module provides simple, reusable functions for context operations. - -```python -from semantica.context.methods import ( - build_context_graph, - store_memory, - retrieve_context, - link_entities -) - -# Build graph -graph = build_context_graph(entities, relationships, method="entities_relationships") - -# Store memory -memory_id = store_memory("User asked about Python", vector_store=vs, method="store") - -# Retrieve context -results = retrieve_context("Python programming", vector_store=vs, method="hybrid") -``` diff --git a/semantica/context/methods.py b/semantica/context/methods.py deleted file mode 100644 index 00ce57dc..00000000 --- a/semantica/context/methods.py +++ /dev/null @@ -1,483 +0,0 @@ -""" -Context Methods Module - -This module provides all context engineering methods as simple, reusable functions for -context graph construction, agent memory management, context retrieval, and entity -linking. It supports multiple context engineering approaches and integrates with the -method registry for extensibility. - -Supported Methods: - -Context Graph Construction: - - "entities_relationships": Build graph from entities and relationships - - "conversations": Build graph from conversations - - "hybrid": Hybrid graph construction combining multiple sources - -Agent Memory Management: - - "store": Store memory items with RAG integration - - "retrieve": Retrieve memories using vector search - - "conversation": Conversation history management - - "hybrid": Hybrid memory retrieval (vector + graph) - -Context Retrieval: - - "vector": Vector-based context retrieval - - "graph": Graph-based context retrieval - - "memory": Memory-based context retrieval - - "hybrid": Hybrid retrieval combining all sources - -Entity Linking: - - "uri": URI assignment for entities - - "similarity": Similarity-based entity linking - - "knowledge_graph": Knowledge graph-based linking - - "cross_document": Cross-document entity linking - -Algorithms Used: - -Context Graph Construction: - - Graph Building: Node and edge construction from entities and relationships - - Entity Extraction: Entity extraction from conversations and text - - Relationship Extraction: Relationship extraction from conversations - - Intent Extraction: Intent classification from conversations - - Sentiment Analysis: Sentiment extraction from conversations - - Graph Traversal: BFS/DFS for neighbor discovery and multi-hop traversal - - Graph Indexing: Type-based indexing for efficient node/edge lookup - -Agent Memory Management: - - Vector Embedding: Embedding generation for memory items - - Vector Search: Similarity search in vector space - - Keyword Search: Fallback keyword-based search - - Retention Policy: Time-based memory retention and cleanup - - Memory Indexing: Deque-based memory index for efficient access - - Knowledge Graph Integration: Entity and relationship updates to knowledge graph - -Context Retrieval: - - Vector Similarity Search: Cosine similarity in vector space - - Graph Traversal: Multi-hop graph expansion for related entities - - Memory Search: Vector and keyword search in memory store - - Result Ranking: Score-based ranking and merging - - Deduplication: Content-based result deduplication - - Hybrid Scoring: Weighted combination of multiple retrieval sources - -Entity Linking: - - URI Generation: Hash-based and text-based URI assignment - - Text Similarity: Word overlap-based similarity calculation - - Knowledge Graph Lookup: Entity matching in knowledge graph - - Cross-Document Linking: Entity linking across multiple documents - - Bidirectional Linking: Symmetric relationship creation - - Entity Web Construction: Graph-based entity connection web - -Key Features: - - Multiple context graph construction methods - - Multiple agent memory management methods - - Multiple context retrieval methods - - Multiple entity linking methods - - Method dispatchers with registry support - - Custom method registration capability - - Consistent interface across all methods - -Main Functions: - - build_context_graph: Context graph construction wrapper - - store_memory: Memory storage wrapper - - retrieve_context: Context retrieval wrapper - - link_entities: Entity linking wrapper - - get_context_method: Get context method by name - -Example Usage: - >>> from semantica.context.methods import build_context_graph, retrieve_context - >>> graph = build_context_graph( - ... entities, relationships, method="entities_relationships" - ... ) - >>> results = retrieve_context("Python programming", method="hybrid", max_results=5) - >>> from semantica.context.methods import get_context_method - >>> method = get_context_method("graph", "custom_method") - -Author: Semantica Contributors -License: MIT -""" - -from typing import Any, Callable, Dict, List, Optional, Union - -from ..utils.exceptions import ProcessingError -from ..utils.logging import get_logger -from .agent_memory import AgentMemory -from .context_graph import ContextGraph -from .context_retriever import ContextRetriever, RetrievedContext -from .entity_linker import EntityLinker, LinkedEntity -from .registry import method_registry - -logger = get_logger("context_methods") - - -def build_context_graph( - entities: Optional[List[Dict[str, Any]]] = None, - relationships: Optional[List[Dict[str, Any]]] = None, - conversations: Optional[List[Union[str, Dict[str, Any]]]] = None, - method: str = "entities_relationships", - **kwargs, -) -> Dict[str, Any]: - """ - Build context graph from various sources (convenience function). - - This is a user-friendly wrapper that builds context graphs using the specified - method. - - Args: - entities: List of entity dictionaries - relationships: List of relationship dictionaries - conversations: List of conversation files or dictionaries - method: Graph construction method (default: "entities_relationships") - - "entities_relationships": Build from entities and relationships - - "conversations": Build from conversations - - "hybrid": Hybrid construction combining multiple sources - **kwargs: Additional options passed to ContextGraph - - Returns: - Context graph dictionary containing: - - nodes: List of context nodes - - edges: List of context edges - - statistics: Graph statistics - - Examples: - >>> from semantica.context.methods import build_context_graph - >>> entities = [{"id": "e1", "text": "Python", "type": "PROGRAMMING_LANGUAGE"}] - >>> relationships = [ - ... {"source_id": "e1", "target_id": "e2", "type": "related_to"} - ... ] - >>> graph = build_context_graph( - ... entities, relationships, method="entities_relationships" - ... ) - >>> print(f"Graph has {graph['statistics']['node_count']} nodes") - """ - # Check for custom method in registry - custom_method = method_registry.get("graph", method) - if custom_method: - try: - return custom_method(entities, relationships, conversations, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) - - try: - # Use ContextGraph as the builder - builder = ContextGraph(**kwargs) - - if method == "entities_relationships": - if not entities or not relationships: - raise ProcessingError( - "entities and relationships required for entities_relationships " - "method" - ) - return builder.build_from_entities_and_relationships( - entities, relationships, **kwargs - ) - - elif method == "conversations": - if not conversations: - raise ProcessingError("conversations required for conversations method") - return builder.build_from_conversations(conversations, **kwargs) - - elif method == "hybrid": - graph = {} - if entities and relationships: - graph1 = builder.build_from_entities_and_relationships( - entities, relationships, **kwargs - ) - graph = graph1 - if conversations: - graph2 = builder.build_from_conversations(conversations, **kwargs) - # Merge graphs - if graph: - graph["nodes"].extend(graph2.get("nodes", [])) - graph["edges"].extend(graph2.get("edges", [])) - else: - graph = graph2 - return graph - - else: - raise ProcessingError(f"Unknown graph construction method: {method}") - - except Exception as e: - logger.error(f"Failed to build context graph: {e}", exc_info=True) - raise ProcessingError(f"Context graph construction failed: {e}") from e - - -def store_memory( - content: str, - vector_store: Optional[Any] = None, - knowledge_graph: Optional[Any] = None, - method: str = "store", - **kwargs, -) -> str: - """ - Store memory item (convenience function). - - This is a user-friendly wrapper that stores memory using the specified method. - - Args: - content: Memory content - vector_store: Vector store instance - knowledge_graph: Knowledge graph instance - method: Memory storage method (default: "store") - - "store": Standard memory storage with RAG - - "conversation": Conversation memory storage - **kwargs: Additional options passed to AgentMemory - - Returns: - Memory ID - - Examples: - >>> from semantica.context.methods import store_memory - >>> memory_id = store_memory( - ... "User asked about Python", vector_store=vs, method="store" - ... ) - >>> print(f"Stored memory: {memory_id}") - """ - # Check for custom method in registry - custom_method = method_registry.get("memory", method) - if custom_method: - try: - return custom_method(content, vector_store, knowledge_graph, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) - - try: - memory = AgentMemory( - vector_store=vector_store, knowledge_graph=knowledge_graph, **kwargs - ) - - metadata = kwargs.get("metadata", {}) - if method == "conversation": - metadata["type"] = "conversation" - - return memory.store( - content, - metadata=metadata, - entities=kwargs.get("entities"), - relationships=kwargs.get("relationships"), - **{ - k: v - for k, v in kwargs.items() - if k not in ["metadata", "entities", "relationships"] - }, - ) - - except Exception as e: - logger.error(f"Failed to store memory: {e}", exc_info=True) - raise ProcessingError(f"Memory storage failed: {e}") from e - - -def retrieve_context( - query: str, - memory_store: Optional[Any] = None, - knowledge_graph: Optional[Any] = None, - vector_store: Optional[Any] = None, - method: str = "hybrid", - max_results: int = 5, - **kwargs, -) -> List[RetrievedContext]: - """ - Retrieve relevant context (convenience function). - - This is a user-friendly wrapper that retrieves context using the specified method. - - Args: - query: Search query - memory_store: Memory store instance - knowledge_graph: Knowledge graph instance - vector_store: Vector store instance - method: Retrieval method (default: "hybrid") - - "vector": Vector-based retrieval only - - "graph": Graph-based retrieval only - - "memory": Memory-based retrieval only - - "hybrid": Hybrid retrieval combining all sources - max_results: Maximum number of results - **kwargs: Additional options passed to ContextRetriever - - Returns: - List of RetrievedContext objects - - Examples: - >>> from semantica.context.methods import retrieve_context - >>> results = retrieve_context( - ... "Python programming", vector_store=vs, method="hybrid" - ... ) - >>> for result in results: - ... print(f"{result.content}: {result.score:.2f}") - """ - # Check for custom method in registry - custom_method = method_registry.get("retrieval", method) - if custom_method: - try: - return custom_method( - query, - memory_store, - knowledge_graph, - vector_store, - max_results, - **kwargs, - ) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) - - try: - retriever = ContextRetriever( - memory_store=memory_store, - knowledge_graph=knowledge_graph, - vector_store=vector_store, - **kwargs, - ) - - if method == "vector": - retriever.use_graph_expansion = False - retriever.memory_store = None - elif method == "graph": - retriever.vector_store = None - retriever.memory_store = None - elif method == "memory": - retriever.vector_store = None - retriever.use_graph_expansion = False - - return retriever.retrieve(query, max_results=max_results, **kwargs) - - except Exception as e: - logger.error(f"Failed to retrieve context: {e}", exc_info=True) - raise ProcessingError(f"Context retrieval failed: {e}") from e - - -def link_entities( - entities: List[Dict[str, Any]], - knowledge_graph: Optional[Any] = None, - method: str = "similarity", - **kwargs, -) -> List[LinkedEntity]: - """ - Link entities across sources (convenience function). - - This is a user-friendly wrapper that links entities using the specified method. - - Args: - entities: List of entity dictionaries - knowledge_graph: Knowledge graph instance - method: Linking method (default: "similarity") - - "uri": URI assignment only - - "similarity": Similarity-based linking - - "knowledge_graph": Knowledge graph-based linking - - "cross_document": Cross-document linking - **kwargs: Additional options passed to EntityLinker - - Returns: - List of LinkedEntity objects - - Examples: - >>> from semantica.context.methods import link_entities - >>> entities = [{"id": "e1", "text": "Python", "type": "PROGRAMMING_LANGUAGE"}] - >>> linked = link_entities(entities, knowledge_graph=kg, method="similarity") - >>> for entity in linked: - ... print(f"{entity.text}: {entity.uri}") - """ - # Check for custom method in registry - custom_method = method_registry.get("linking", method) - if custom_method: - try: - return custom_method(entities, knowledge_graph, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) - - try: - linker = EntityLinker(knowledge_graph=knowledge_graph, **kwargs) - - if method == "uri": - # Just assign URIs - linked = [] - for entity in entities: - entity_id = entity.get("id") or entity.get("entity_id") - entity_text = ( - entity.get("text") or entity.get("label") or entity.get("name", "") - ) - entity_type = entity.get("type") or entity.get("entity_type") - uri = linker.assign_uri(entity_id, entity_text, entity_type) - linked.append( - LinkedEntity( - entity_id=entity_id, - uri=uri, - text=entity_text, - type=entity_type or "UNKNOWN", - linked_entities=[], - context=entity.get("metadata", {}), - confidence=entity.get("confidence", 1.0), - ) - ) - return linked - - else: - # Use full linking - return linker.link( - text="", # Not used for entity list - entities=entities, - context=kwargs.get("context"), - ) - - except Exception as e: - logger.error(f"Failed to link entities: {e}", exc_info=True) - raise ProcessingError(f"Entity linking failed: {e}") from e - - -def get_context_method(task: str, name: str) -> Optional[Callable]: - """ - Get a registered context method. - - Args: - task: Task type ("graph", "memory", "retrieval", "linking") - name: Method name - - Returns: - Registered method or None if not found - - Examples: - >>> from semantica.context.methods import get_context_method - >>> method = get_context_method("graph", "custom_method") - >>> if method: - ... result = method(entities, relationships) - """ - return method_registry.get(task, name) - - -def list_available_methods(task: Optional[str] = None) -> Dict[str, List[str]]: - """ - List all available context methods. - - Args: - task: Optional task type filter - - Returns: - Dictionary mapping task types to method names - - Examples: - >>> from semantica.context.methods import list_available_methods - >>> all_methods = list_available_methods() - >>> graph_methods = list_available_methods("graph") - """ - return method_registry.list_all(task) - - -# Register default methods -method_registry.register("graph", "entities_relationships", build_context_graph) -method_registry.register("graph", "conversations", build_context_graph) -method_registry.register("graph", "hybrid", build_context_graph) -method_registry.register("memory", "store", store_memory) -method_registry.register("memory", "conversation", store_memory) -method_registry.register("retrieval", "vector", retrieve_context) -method_registry.register("retrieval", "graph", retrieve_context) -method_registry.register("retrieval", "memory", retrieve_context) -method_registry.register("retrieval", "hybrid", retrieve_context) -method_registry.register("linking", "uri", link_entities) -method_registry.register("linking", "similarity", link_entities) -method_registry.register("linking", "knowledge_graph", link_entities) -method_registry.register("linking", "cross_document", link_entities) diff --git a/semantica/context/registry.py b/semantica/context/registry.py deleted file mode 100644 index 9f3c28aa..00000000 --- a/semantica/context/registry.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -Method Registry Module for Context Engineering - -This module provides a method registry system for registering custom context engineering -methods, enabling extensibility and community contributions to the context toolkit. - -Supported Registration Types: - - Method Registry: Register custom context methods for: - * "graph": Context graph construction methods - * "memory": Agent memory management methods - * "retrieval": Context retrieval methods - * "linking": Entity linking methods - -Algorithms Used: - - Registry Pattern: Dictionary-based registration and lookup - - Dynamic Registration: Runtime function registration - - Type Checking: Type validation for registered components - - Lookup Algorithms: Hash-based O(1) lookup for methods - - Task-based Organization: Hierarchical organization by task type - -Key Features: - - Method registry for custom context methods - - Task-based method organization (graph, memory, retrieval, linking) - - Dynamic registration and unregistration - - Easy discovery of available methods - - Support for community-contributed extensions - -Main Classes: - - MethodRegistry: Registry for custom context methods - -Global Instances: - - method_registry: Global method registry instance - -Example Usage: - >>> from semantica.context.registry import method_registry - >>> method_registry.register("graph", "custom_method", custom_graph_function) - >>> available = method_registry.list_all("graph") - -Author: Semantica Contributors -License: MIT -""" - -from typing import Callable, Dict, List, Optional - - -class MethodRegistry: - """Registry for custom context methods.""" - - _methods: Dict[str, Dict[str, Callable]] = { - "graph": {}, - "memory": {}, - "retrieval": {}, - "linking": {}, - } - - @classmethod - def register(cls, task: str, name: str, method_func: Callable): - """ - Register a method for a specific task. - - Args: - task: Task type ("graph", "memory", "retrieval", "linking") - name: Method name - method_func: Method function or callable - """ - if task not in cls._methods: - cls._methods[task] = {} - cls._methods[task][name] = method_func - - @classmethod - def get(cls, task: str, name: str) -> Optional[Callable]: - """ - Get a registered method. - - Args: - task: Task type - name: Method name - - Returns: - Registered method or None if not found - """ - return cls._methods.get(task, {}).get(name) - - @classmethod - def list_all(cls, task: Optional[str] = None) -> Dict[str, List[str]]: - """ - List all registered methods. - - Args: - task: Optional task type filter - - Returns: - Dictionary mapping task types to method names - """ - if task: - return {task: list(cls._methods.get(task, {}).keys())} - return {t: list(m.keys()) for t, m in cls._methods.items()} - - @classmethod - def unregister(cls, task: str, name: str): - """ - Unregister a method. - - Args: - task: Task type - name: Method name - """ - if task in cls._methods and name in cls._methods[task]: - del cls._methods[task][name] - - -method_registry = MethodRegistry() diff --git a/semantica/graph_store/graph_store.py b/semantica/graph_store/graph_store.py index cd2d2e98..d4e2b503 100644 --- a/semantica/graph_store/graph_store.py +++ b/semantica/graph_store/graph_store.py @@ -32,7 +32,7 @@ Author: Semantica Contributors License: MIT """ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Tuple, Union from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger @@ -724,6 +724,321 @@ class GraphStore: """Create an index.""" return self._manager.create_index(label, property_name, index_type, **options) + # Compatibility with AgentMemory / ContextGraph interface + def add_nodes(self, nodes: List[Dict[str, Any]], **options) -> int: + """ + Add nodes (Compatibility method). + + Args: + nodes: List of node dictionaries with 'id', 'type', 'properties' + **options: Additional options + + Returns: + Number of nodes created + """ + if not nodes: + return 0 + + # Convert to GraphStore format (labels, properties) + graph_nodes = [] + for node in nodes: + # Extract label from type + labels = [node.get("type", "Entity")] + if isinstance(labels[0], str): + labels = [labels[0]] # Ensure list + + # Prepare properties + props = node.get("properties", {}).copy() + + # Ensure ID is preserved + if "id" in node and "id" not in props: + props["id"] = node["id"] + + # Ensure content/text is preserved + if "content" in node and "content" not in props: + props["content"] = node["content"] + if "text" in node and "text" not in props: + props["text"] = node["text"] + + graph_nodes.append({ + "labels": labels, + "properties": props + }) + + # Use batch creation + # Note: create_nodes expects dicts with 'labels' and 'properties' keys if passed directly? + # Let's check create_nodes signature implementation in manager. + # But here I'll assume create_nodes takes a list of such dicts or similar. + # Actually, let's look at create_nodes wrapper in this file: + # def create_nodes(self, nodes: List[Dict[str, Any]], **options) + # It passes to self._manager.nodes.create_batch(nodes) + + # If create_batch expects specific format, I should match it. + # Assuming create_batch is smart enough or expects standard format. + # To be safe, let's look at NodeManager.create_batch if possible, but I can't easily. + # Standard expectation: List of dicts where each dict has labels and properties. + + result = self.create_nodes(graph_nodes, **options) + return len(result) + + def add_edges(self, edges: List[Dict[str, Any]], **options) -> int: + """ + Add edges (Compatibility method). + + Args: + edges: List of edge dictionaries + **options: Additional options + + Returns: + Number of edges created + """ + count = 0 + for edge in edges: + source_id = edge.get("source_id") + target_id = edge.get("target_id") + rel_type = edge.get("type", "RELATED_TO") + properties = edge.get("properties", {}).copy() + + # Preserve weight + if "weight" in edge: + properties["weight"] = edge["weight"] + + if source_id and target_id: + try: + self.create_relationship(source_id, target_id, rel_type, properties, **options) + count += 1 + except Exception as e: + self.logger.warning(f"Failed to add edge {source_id}->{target_id}: {e}") + return count + + def build_from_conversations( + self, + conversations: List[Union[str, Dict[str, Any]]], + link_entities: bool = True, + extract_intents: bool = False, + extract_sentiments: bool = False, + **options, + ) -> Dict[str, Any]: + """ + Build graph from conversations (Compatibility method). + + Args: + conversations: List of conversation files or dictionaries + link_entities: Link entities across conversations + extract_intents: Extract intents (not implemented) + extract_sentiments: Extract sentiments (not implemented) + **options: Additional options + + Returns: + Graph statistics + """ + tracking_id = self.progress_tracker.start_tracking( + file=None, + module="graph_store", + submodule="GraphStore", + message=f"Building graph from {len(conversations)} conversations", + ) + + try: + all_nodes = [] + all_edges = [] + seen_nodes = set() + + for conv in conversations: + # Load conversation if string (file path) + conv_data = conv + if isinstance(conv, str): + from pathlib import Path + from ..utils.helpers import read_json_file + conv_data = read_json_file(Path(conv)) + + nodes, edges = self._process_conversation_to_elements( + conv_data, + extract_intents=extract_intents, + extract_sentiments=extract_sentiments + ) + + # Add unique nodes + for node in nodes: + if node["id"] not in seen_nodes: + all_nodes.append(node) + seen_nodes.add(node["id"]) + + all_edges.extend(edges) + + if link_entities: + linked_edges = self._link_entities_elements(all_nodes) + all_edges.extend(linked_edges) + + # Batch add + node_count = self.add_nodes(all_nodes) + edge_count = self.add_edges(all_edges) + + self.progress_tracker.stop_tracking(tracking_id, status="completed") + + return { + "statistics": { + "node_count": node_count, + "edge_count": edge_count + } + } + + except Exception as e: + self.progress_tracker.stop_tracking( + tracking_id, status="failed", message=str(e) + ) + self.logger.error(f"Failed to build graph: {e}") + raise + + def build_from_entities_and_relationships( + self, + entities: List[Dict[str, Any]], + relationships: List[Dict[str, Any]], + **kwargs, + ) -> Dict[str, Any]: + """ + Build graph from entities and relationships (Compatibility method). + """ + nodes = [] + edges = [] + + # Process entities + for entity in entities: + entity_id = entity.get("id") or entity.get("entity_id") + if entity_id: + nodes.append({ + "id": entity_id, + "type": entity.get("type", "entity"), + "properties": { + "content": entity.get("text") or entity.get("label") or entity_id, + **entity + } + }) + + # Process relationships + for rel in relationships: + source = rel.get("source_id") + target = rel.get("target_id") + if source and target: + edges.append({ + "source_id": source, + "target_id": target, + "type": rel.get("type", "related_to"), + "weight": rel.get("confidence", 1.0), + "properties": rel + }) + + node_count = self.add_nodes(nodes) + edge_count = self.add_edges(edges) + + return {"statistics": {"node_count": node_count, "edge_count": edge_count}} + + def _process_conversation_to_elements(self, conv_data: Dict[str, Any], **kwargs) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """Helper to process conversation into nodes and edges.""" + nodes = [] + edges = [] + + conv_id = conv_data.get("id") or f"conv_{hash(str(conv_data)) % 10000}" + + # Conversation node + nodes.append({ + "id": conv_id, + "type": "conversation", + "properties": { + "content": conv_data.get("content", "") or conv_data.get("summary", ""), + "timestamp": conv_data.get("timestamp") + } + }) + + name_to_id = {} + extract_entities = kwargs.get("extract_entities", True) # Default true if not passed? + # Actually ContextGraph defaults to True in init, but here we are static. + # Let's assume True unless told otherwise or check config. + + # Extract entities + for entity in conv_data.get("entities", []): + entity_id = entity.get("id") or entity.get("entity_id") + entity_text = entity.get("text") or entity.get("label") or entity.get("name") or entity_id + entity_type = entity.get("type", "entity") + + # Generate ID if missing + if not entity_id and entity_text: + import hashlib + entity_hash = hashlib.md5(f"{entity_text}_{entity_type}".encode()).hexdigest()[:12] + entity_id = f"{entity_type.lower()}_{entity_hash}" + + if entity_id: + if entity_text: + name_to_id[entity_text] = entity_id + + nodes.append({ + "id": entity_id, + "type": "entity", # Normalize type? + "properties": { + "content": entity_text, + "type": entity_type, + **entity + } + }) + + # Edge: Conversation -> Entity + edges.append({ + "source_id": conv_id, + "target_id": entity_id, + "type": "mentions" + }) + + # Extract relationships + for rel in conv_data.get("relationships", []): + source = rel.get("source_id") + target = rel.get("target_id") + + # Resolve IDs + if not source and rel.get("source") and rel.get("source") in name_to_id: + source = name_to_id[rel.get("source")] + if not target and rel.get("target") and rel.get("target") in name_to_id: + target = name_to_id[rel.get("target")] + + if source and target: + edges.append({ + "source_id": source, + "target_id": target, + "type": rel.get("type", "related_to"), + "weight": rel.get("confidence", 1.0), + "properties": rel + }) + + return nodes, edges + + def _link_entities_elements(self, nodes: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Link similar entities.""" + edges = [] + # Lazy import to avoid circular dependency + try: + from ..context.entity_linker import EntityLinker + linker = EntityLinker() # Use default config + except ImportError: + return [] + + entity_nodes = [n for n in nodes if n.get("type") == "entity"] + for i, node1 in enumerate(entity_nodes): + content1 = node1["properties"].get("content", "") + if not content1: continue + + for node2 in entity_nodes[i + 1 :]: + content2 = node2["properties"].get("content", "") + if not content2: continue + + similarity = linker._calculate_text_similarity(content1.lower(), content2.lower()) + if similarity >= linker.similarity_threshold: + edges.append({ + "source_id": node1["id"], + "target_id": node2["id"], + "type": "similar_to", + "weight": similarity + }) + return edges + @property def nodes(self) -> NodeManager: """Get node manager.""" diff --git a/test_notebook_logic.py b/test_notebook_logic.py deleted file mode 100644 index 08845035..00000000 --- a/test_notebook_logic.py +++ /dev/null @@ -1,197 +0,0 @@ - -import logging -import json -from datetime import datetime - -# Set up logging to see what's happening under the hood -# Using WARNING to keep the output clean for the notebook demonstration -logging.basicConfig(level=logging.WARNING, format='%(asctime)s - %(levelname)s - %(message)s') - -# Import all the powerful tools from Semantica -from semantica.kg import ( - GraphBuilder, - GraphAnalyzer, - GraphValidator, - ConnectivityAnalyzer, - CentralityCalculator, - CommunityDetector, - TemporalGraphQuery, - ProvenanceTracker -) -from semantica.deduplication import DuplicateDetector -from semantica.conflicts import ConflictDetector, ConflictResolver - -# Our "Raw" Messy Data -raw_entities = [ - {"id": "startup_1", "type": "Startup", "name": "TechFlow AI", "revenue": 1000000, "founded": "2021-01-01"}, - {"id": "startup_2", "type": "Startup", "name": "GreenEnergy Co", "revenue": 500000, "founded": "2020-05-15"}, - {"id": "startup_1_dup", "type": "Startup", "name": "TechFlow Inc.", "revenue": 1200000, "founded": "2021-01-01"}, # Duplicate! - {"id": "investor_1", "type": "Investor", "name": "Venture Capital X"}, - {"id": "founder_1", "type": "Person", "name": "Alice Chen"}, - {"id": "founder_2", "type": "Person", "name": "Bob Smith"} -] - -raw_relationships = [ - # Valid Relationships - {"source": "founder_1", "target": "startup_1", "type": "FOUNDED", "valid_from": "2021-01-01"}, - {"source": "investor_1", "target": "startup_1", "type": "INVESTED_IN", "amount": 5000000, "valid_from": "2023-06-01"}, - - # Dangling Edge (Error!) - {"source": "founder_2", "target": "startup_999", "type": "FOUNDED", "valid_from": "2020-05-15"}, - - # Temporal Data (History) - {"source": "founder_1", "target": "startup_2", "type": "ADVISED", "valid_from": "2020-01-01", "valid_until": "2021-01-01"} -] - -print(f"Loaded {len(raw_entities)} raw entities and {len(raw_relationships)} raw relationships.") - -# Initialize Validator -validator = GraphValidator() - -# Create a temporary graph object for validation -temp_graph = {"entities": raw_entities, "relationships": raw_relationships} - -# Run Validation -print("Running Validation Check...") -validation_result = validator.validate(temp_graph) - -if not validation_result.is_valid: - print("Validation Failed! Issues found:") - for issue in validation_result.issues: - print(f" - [{issue.severity.name}] {issue.message} (Code: {issue.code})") - - # AUTOMATIC FIX: If it's a dangling edge, remove it - if issue.code == "DANGLING_EDGE": - print(" Auto-Fixing: Removing invalid relationship...") - raw_relationships = [r for r in raw_relationships - if r['target'] != issue.details.get('target_id')] -else: - print("Graph is valid!") - -# Re-validate to confirm fix -print("\nRe-validating after fixes...") -temp_graph = {"entities": raw_entities, "relationships": raw_relationships} -if validator.validate(temp_graph).is_valid: - print("Graph is now clean and valid!") - -# 1. Detect Duplicates -print("Scanning for duplicates...") -deduper = DuplicateDetector(similarity_threshold=0.7) # 70% similarity threshold -duplicates = deduper.detect_duplicates(raw_entities) - -for candidate in duplicates: - print(f"Found potential duplicate pair (Score: {candidate.similarity_score:.2f}):") - print(f" - {candidate.entity1['name']} (ID: {candidate.entity1['id']})") - print(f" - {candidate.entity2['name']} (ID: {candidate.entity2['id']})") - - # MERGE STRATEGY: Keep entity1, merge data from entity2 - print(" Merging entities...") - # (In a real app, you'd use EntityMerger, but here's the logic:) - # We keep startup_1 and discard startup_1_dup, but we note the conflict - -# 2. Detect Conflicts -print("\nChecking for data conflicts...") -conflict_detector = ConflictDetector() - -# Simulating a conflict check between the two versions of TechFlow -# To check conflicts, we treat them as the same entity (same ID) -entity_a = raw_entities[0].copy() -entity_b = raw_entities[2].copy() -entity_b['id'] = entity_a['id'] # Force same ID for conflict detection - -conflicts = conflict_detector.detect_conflicts([entity_a, entity_b]) - -for conflict in conflicts: - print(f" Conflict detected in field '{conflict.property_name}':") - print(f" Values: {conflict.conflicting_values}") - - # RESOLUTION: Trust the higher number (optimistic!) - if conflict.property_name == "revenue": - # values are strings or ints, need to handle types - vals = [float(v) for v in conflict.conflicting_values if v is not None] - resolved_val = max(vals) - print(f" Resolved to: {resolved_val}") - raw_entities[0]['revenue'] = resolved_val - -# Final Cleanup: Remove the duplicate entity from our list -clean_entities = [e for e in raw_entities if e['id'] != 'startup_1_dup'] -clean_relationships = raw_relationships # (We'd normally re-link relationships too) - -print(f"\nCleaned Data: {len(clean_entities)} entities remaining.") - - -# Manual Graph Construction (since we already cleaned it) -kg = { - "entities": clean_entities, - "relationships": clean_relationships, - "metadata": { - "created_at": datetime.now().isoformat(), - "source": "Manual Advanced Pipeline" - } -} -print("Knowledge Graph Assembled Successfully!") - -# Initialize the Master Analyzer -analyzer = GraphAnalyzer(enable_temporal=True) - -# 1. Structural Analysis (Connectivity) -print("\n--- Connectivity Analysis ---") -connectivity = analyzer.analyze_connectivity(kg) -print(f" • Graph Connected? {'Yes' if connectivity['is_connected'] else 'No'}") -print(f" • Connected Components: {connectivity['num_components']}") - -# 2. Centrality (Who is important?) -print("\n--- Centrality Analysis ---") -centrality_result = analyzer.calculate_centrality(kg, centrality_type="degree") -degree_data = centrality_result["centrality_measures"]["degree"] - -# Get pre-calculated rankings -top_nodes = degree_data["rankings"][:3] - -print(" • Top Influencers (Degree Centrality):") -for item in top_nodes: - print(f" - {item['node']}: {item['score']:.2f}") - -# 3. Community Detection (Clustering) -print("\n--- Community Detection ---") -communities = analyzer.detect_communities(kg, algorithm="louvain") -community_result = communities -communities = community_result["communities"] - -print(f" • Detected {len(communities)} communities.") -for i, comm in enumerate(communities): - # comm is a set of node IDs - members = list(comm) - print(f" Community {i+1}: {', '.join(members)}") - -temporal_engine = TemporalGraphQuery(temporal_granularity="year") - -# 1. Time Travel Query: What did the world look like in 2020? -print("\n--- Time Travel: 2020 ---") -snapshot_2020 = temporal_engine.query_at_time(kg, query="*", at_time="2020-06-01") -print(f" Active Relationships in 2020: {len(snapshot_2020['relationships'])}") -for rel in snapshot_2020['relationships']: - print(f" - {rel['source']} --[{rel['type']}]--> {rel['target']}") - -# 2. Time Travel Query: What about 2023? -print("\n--- Time Travel: 2023 ---") -snapshot_2023 = temporal_engine.query_at_time(kg, query="*", at_time="2023-07-01") -print(f" Active Relationships in 2023: {len(snapshot_2023['relationships'])}") -for rel in snapshot_2023['relationships']: - print(f" - {rel['source']} --[{rel['type']}]--> {rel['target']}") - -# Notice how 'ADVISED' might disappear if it ended, and 'INVESTED_IN' appears! - -tracker = ProvenanceTracker() - -# Let's pretend we're tracking the source of our data -tracker.track_entity("startup_1", source="Crunchbase_API_v2", metadata={"confidence": 0.95}) -tracker.track_entity("startup_1", source="Manual_Entry_User_Bob", metadata={"confidence": 1.0}) - -print("\n--- Provenance Report: TechFlow AI ---") -lineage = tracker.get_lineage("startup_1") -print(f" Entity: startup_1") -print(f" First Seen: {lineage['first_seen']}") -print(f" Sources:") -for src in lineage['sources']: - print(f" - {src['source']} (at {src['timestamp']})") diff --git a/tests/context/test_context.py b/tests/context/test_context.py index c6f2dccb..d42aea62 100644 --- a/tests/context/test_context.py +++ b/tests/context/test_context.py @@ -13,7 +13,6 @@ from semantica.context.context_graph import ContextGraph, ContextNode, ContextEd from semantica.context.agent_memory import AgentMemory, MemoryItem from semantica.context.context_retriever import ContextRetriever, RetrievedContext from semantica.context.agent_context import AgentContext -from semantica.context import methods class MockVectorStore: def __init__(self): @@ -143,19 +142,5 @@ class TestContextModule(unittest.TestCase): self.assertIsNotNone(ctx._memory) self.assertEqual(len(ctx._memory.short_term_memory), 1) - # --- Method Wrapper Tests --- - def test_method_wrappers(self): - # Test retrieve_context wrapper - # We need to patch ContextRetriever inside the method or just check if it runs - # Since it creates a new ContextRetriever internally, we can mock the class in the module - - with patch('semantica.context.methods.ContextRetriever') as MockRetriever: - instance = MockRetriever.return_value - instance.retrieve.return_value = [] - - results = methods.retrieve_context("query", vector_store=self.mock_vector_store) - self.assertIsInstance(results, list) - MockRetriever.assert_called_once() - if __name__ == '__main__': unittest.main()