diff --git a/README.md b/README.md index 4a578f76..b027bace 100644 --- a/README.md +++ b/README.md @@ -340,23 +340,36 @@ print(f"Classes: {len(ontology.classes)}") [**Cookbook: Ontology**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/14_Ontology.ipynb) -### Context Engineering for AI Agents +### Context Engineering & Memory Systems -> **Persistent Memory** • RAG + Knowledge Graphs • MCP-Compatible Tools • FastEmbed Integrated +> **Persistent Memory** • **Hybrid Retrieval (Vector + Graph)** • **Hierarchical Storage** • **Entity Linking** ```python -from semantica.context import AgentMemory, ContextRetriever +from semantica.context import AgentContext from semantica.vector_store import VectorStore -# Uses FastEmbed by default for high-performance embedding generation -memory = AgentMemory(vector_store=VectorStore(backend="faiss"), retention_policy="unlimited") -memory.store("User prefers technical docs", metadata={"user_id": "user_123"}) +# Initialize Context with Hybrid Retrieval (Graph + Vector) +context = AgentContext( + vector_store=VectorStore(backend="faiss"), + hybrid_alpha=0.75 # 75% weight to Knowledge Graph, 25% to Vector +) -retriever = ContextRetriever(memory_store=memory) -context = retriever.retrieve("What are user preferences?", max_results=5) +# Store memory with automatic entity linking +context.store( + "User is building a RAG system with Semantica", + metadata={"priority": "high", "topic": "rag"} +) + +# Retrieve with context expansion +results = context.retrieve("What is the user building?", use_graph_expansion=True) ``` -[**Cookbook: Vector Store**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/13_Vector_Store.ipynb) • [**Embedding Generation**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/12_Embedding_Generation.ipynb) • [**Context Module**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/19_Context_Module.ipynb) • [**Advanced Vector Store**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb) +**Core Notebooks:** +- [**Context Module Introduction**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/19_Context_Module.ipynb) - Basic memory and storage. +- [**Advanced Context Engineering**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/advanced/11_Advanced_Context_Engineering.ipynb) - Hybrid retrieval, graph builders, and custom memory policies. + +**Related Components:** +[**Vector Store**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/13_Vector_Store.ipynb) • [**Embedding Generation**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/12_Embedding_Generation.ipynb) • [**Advanced Vector Store**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb) ### Knowledge Graph-Powered RAG (GraphRAG) diff --git a/context_tutorial_data/saved_agent/agent_memory.pkl b/context_tutorial_data/saved_agent/agent_memory.pkl new file mode 100644 index 00000000..77ab7b08 Binary files /dev/null and b/context_tutorial_data/saved_agent/agent_memory.pkl differ diff --git a/context_tutorial_data/saved_agent/knowledge_graph.json b/context_tutorial_data/saved_agent/knowledge_graph.json new file mode 100644 index 00000000..abf023ad --- /dev/null +++ b/context_tutorial_data/saved_agent/knowledge_graph.json @@ -0,0 +1,58 @@ +{ + "nodes": [ + { + "id": "semantica", + "type": "FRAMEWORK", + "properties": { + "content": "Semantica", + "id": "semantica", + "text": "Semantica", + "type": "FRAMEWORK" + } + }, + { + "id": "agentic_systems", + "type": "CONCEPT", + "properties": { + "content": "agentic systems", + "id": "agentic_systems", + "text": "agentic systems", + "type": "CONCEPT" + } + }, + { + "id": "context_eng", + "type": "TECHNIQUE", + "properties": { + "content": "Context Engineering", + "id": "context_eng", + "text": "Context Engineering", + "type": "TECHNIQUE" + } + } + ], + "edges": [ + { + "source_id": "semantica", + "target_id": "agentic_systems", + "type": "BUILDS", + "weight": 1.0, + "properties": { + "source_id": "semantica", + "target_id": "agentic_systems", + "type": "BUILDS" + } + }, + { + "source_id": "semantica", + "target_id": "context_eng", + "type": "USES", + "weight": 1.0, + "properties": { + "source_id": "semantica", + "target_id": "context_eng", + "type": "USES" + } + } + ] +} \ No newline at end of file diff --git a/context_tutorial_data/saved_agent/vector_store/store_data.pkl b/context_tutorial_data/saved_agent/vector_store/store_data.pkl new file mode 100644 index 00000000..88a84a3f Binary files /dev/null and b/context_tutorial_data/saved_agent/vector_store/store_data.pkl differ diff --git a/cookbook/advanced/11_Advanced_Context_Engineering.ipynb b/cookbook/advanced/11_Advanced_Context_Engineering.ipynb index fbe16366..d86b017f 100644 --- a/cookbook/advanced/11_Advanced_Context_Engineering.ipynb +++ b/cookbook/advanced/11_Advanced_Context_Engineering.ipynb @@ -2,6 +2,7 @@ "cells": [ { "cell_type": "markdown", + "id": "34af0e1d", "metadata": {}, "source": [ "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/11_Advanced_Context_Engineering.ipynb)\n", @@ -14,80 +15,72 @@ "\n", "### Learning Objectives\n", "\n", - "- **Custom Memory Pruning**: Implement importance-based pruning instead of FIFO.\n", - "- **Hybrid Retrieval Tuning**: Optimize weights for vector, graph, and keyword search.\n", - "- **Custom Extensions**: Register custom graph building methods.\n", - "- **Performance Optimization**: Balance token limits and retrieval latency.\n", + "- **Custom Memory Pruning**: Implement importance-based pruning instead of standard FIFO/token-based pruning.\n", + "- **Custom Graph Extensions**: Register custom graph building methods using the registry system.\n", + "- **Hybrid Retrieval Tuning**: Optimize weights for vector and graph search.\n", "\n", "---\n", "\n", "## 1. Setup\n", "\n", - "We'll start by setting up a mock vector store and importing necessary components." + "We'll start by setting up the environment and initializing a standard Vector Store." ] }, { "cell_type": "code", "execution_count": null, + "id": "583f944a", "metadata": {}, "outputs": [], "source": [ - "!pip install semantica\n" + "!pip install semantica" ] }, { "cell_type": "code", "execution_count": null, + "id": "70fbd8c1", "metadata": {}, "outputs": [], "source": [ + "import logging\n", "from typing import List, Dict, Any, Optional\n", - "from semantica.context import AgentMemory, AgentContext, ContextGraph, ContextRetriever, VectorStore\n", - "from semantica.context import registry\n", + "from semantica.context import AgentMemory, AgentContext, ContextGraph, ContextRetriever\n", + "from semantica.vector_store import VectorStore\n", + "from semantica.context import registry, methods\n", "\n", - "# Mock Vector Store (same as in introduction)\n", - "class MockVectorStore(VectorStore):\n", - " def __init__(self):\n", - " self.items = {}\n", - " self.counter = 0\n", - " def add(self, texts, metadata=None, **kwargs):\n", - " ids = []\n", - " for i, text in enumerate(texts):\n", - " id_ = f\"id_{self.counter}\"\n", - " self.items[id_] = {\"text\": text, \"metadata\": metadata[i] if metadata else {}}\n", - " ids.append(id_)\n", - " self.counter += 1\n", - " return ids\n", - " def search(self, query, limit=5, **kwargs):\n", - " return [{\n", - " \"id\": k, \"content\": v[\"text\"], \"score\": 0.85, \"metadata\": v[\"metadata\"]\n", - " } for k, v in list(self.items.items())[:limit]]\n", - " def delete(self, ids, **kwargs):\n", - " return True\n", + "# Configure logging to see internal processes\n", + "logging.basicConfig(level=logging.INFO)\n", "\n", - "vs = MockVectorStore()\n", + "# Initialize Vector Store (using in-memory backend for this example)\n", + "# In production, you might use 'weaviate', 'qdrant', or 'faiss'\n", + "vs = VectorStore(backend=\"inmemory\", dimension=384)\n", + "\n", + "# Initialize Context Graph\n", "kg = ContextGraph()" ] }, { "cell_type": "markdown", + "id": "c2d8299a", "metadata": {}, "source": [ "## 2. Custom Memory Pruning Strategy\n", "\n", "By default, `AgentMemory` uses a FIFO (First-In-First-Out) strategy combined with a token limit to prune short-term memory. However, you might want to keep \"important\" memories longer regardless of their age.\n", "\n", - "Let's subclass `AgentMemory` to implement an importance-based pruning strategy." + "Let's subclass `AgentMemory` to implement an importance-based pruning strategy that respects metadata flags." ] }, { "cell_type": "code", "execution_count": null, + "id": "6edbdd77", "metadata": {}, "outputs": [], "source": [ "class ImportanceAwareMemory(AgentMemory):\n", - " def _prune_short_term_memory(self):\n", + " def _prune_short_term_memory(self) -> None:\n", " \"\"\"\n", " Custom pruning: Always keep items marked as 'important' in metadata,\n", " then prune others based on token limits.\n", @@ -123,75 +116,51 @@ " all_kept = sorted(important_items + kept_others, key=lambda x: x.timestamp)\n", " self.short_term_memory = all_kept\n", "\n", - "# Test the custom memory\n", - "memory = ImportanceAwareMemory(vector_store=vs, token_limit=100)\n", + "# Initialize our custom memory with a strict token limit for testing\n", + "memory = ImportanceAwareMemory(\n", + " vector_store=vs, \n", + " token_limit=100, \n", + " short_term_limit=50\n", + ")\n", "\n", - "# Add an old important memory\n", + "# 1. Store an OLD but IMPORTANT memory\n", "memory.store(\"IMPORTANT: User's name is Alice\", metadata={\"important\": True})\n", "\n", - "# Fill with filler memories\n", + "# 2. Flood memory with newer filler content\n", "for i in range(20):\n", - " memory.store(f\"Filler memory {i} \" * 5) # Consumes tokens\n", + " memory.store(f\"Filler memory {i} \" * 5) # This consumes tokens\n", "\n", - "print(f\"Short-term items: {len(memory.short_term_memory)}\")\n", + "print(f\"Short-term items count: {len(memory.short_term_memory)}\")\n", "print(\"First item (should be the important one):\", memory.short_term_memory[0].content)" ] }, { "cell_type": "markdown", + "id": "6f857653", "metadata": {}, "source": [ - "## 3. Tuning Hybrid Retrieval\n", + "## 3. Extending with Custom Graph Methods\n", "\n", - "Hybrid retrieval combines scores from vector search and graph traversal. You can tune the `hybrid_alpha` parameter to weight these components.\n", + "Semantica's registry system allows you to plug in custom logic for graph construction, retrieval, and more. This is powerful for domain-specific graph topologies.\n", "\n", - "- `hybrid_alpha = 0.0`: Pure Vector Search\n", - "- `hybrid_alpha = 1.0`: Pure Graph Search\n", - "- `hybrid_alpha = 0.5`: Balanced (Default)\n", - "\n", - "Additionally, `max_expansion_hops` controls how far we traverse the graph from retrieved nodes." + "Let's register a custom graph builder that creates a \"Star Graph\" topology." ] }, { "cell_type": "code", "execution_count": null, + "id": "62cb840c", "metadata": {}, "outputs": [], "source": [ - "# Populate graph with some structure\n", - "kg.add_node(\"python\", \"concept\", \"Python\")\n", - "kg.add_node(\"ml\", \"concept\", \"Machine Learning\")\n", - "kg.add_edge(\"python\", \"ml\", \"used_for\")\n", - "\n", - "retriever = ContextRetriever(\n", - " memory_store=memory,\n", - " knowledge_graph=kg,\n", - " vector_store=vs,\n", - " hybrid_alpha=0.7, # Favor graph connections\n", - " max_expansion_hops=2 # Traverse deeper\n", - ")\n", - "\n", - "results = retriever.retrieve(\"Python\")\n", - "for res in results:\n", - " print(f\"Source: {res.source}, Score: {res.score:.2f}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Extending with Custom Methods\n", - "\n", - "Semantica's registry system allows you to plug in custom logic. Let's register a custom graph builder that creates a star graph topology." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def star_graph_builder(center_entity, satellites, **kwargs):\n", + "def star_graph_builder(\n", + " entities: Optional[List[Dict[str, Any]]] = None,\n", + " relationships: Optional[List[Dict[str, Any]]] = None,\n", + " conversations: Optional[List[Any]] = None,\n", + " center_entity: str = \"Central Hub\",\n", + " satellites: Optional[List[str]] = None,\n", + " **kwargs\n", + ") -> Dict[str, Any]:\n", " \"\"\"\n", " Builds a star graph where all satellites connect to the center.\n", " \"\"\"\n", @@ -199,36 +168,111 @@ " edges = []\n", " \n", " # Center node\n", - " nodes.append({\"id\": \"center\", \"label\": center_entity, \"type\": \"CENTER\"})\n", + " nodes.append({\"id\": \"center\", \"type\": \"CENTER\", \"properties\": {\"content\": center_entity}})\n", " \n", + " satellites = satellites or []\n", " for i, sat in enumerate(satellites):\n", " sat_id = f\"sat_{i}\"\n", - " nodes.append({\"id\": sat_id, \"label\": sat, \"type\": \"SATELLITE\"})\n", - " edges.append({\"source\": \"center\", \"target\": sat_id, \"relation\": \"connects_to\"})\n", + " nodes.append({\"id\": sat_id, \"type\": \"SATELLITE\", \"properties\": {\"content\": sat}})\n", + " edges.append({\"source_id\": \"center\", \"target_id\": sat_id, \"type\": \"connects_to\"})\n", " \n", - " return {\"nodes\": nodes, \"edges\": edges}\n", + " return {\n", + " \"nodes\": nodes, \n", + " \"edges\": edges, \n", + " \"statistics\": {\"node_count\": len(nodes), \"edge_count\": len(edges)}\n", + " }\n", "\n", - "# Register the method\n", + "# Register the method in the global registry\n", "registry.method_registry.register(\"graph\", \"star_builder\", star_graph_builder)\n", "\n", "# Verify registration\n", - "print(\"Available graph methods:\", registry.method_registry.list_all(\"graph\"))\n", - "\n", - "# Use it (conceptual - typically used via build_context_graph wrapper)\n", - "graph_data = star_graph_builder(\"Central Hub\", [\"Spoke 1\", \"Spoke 2\"])\n", - "print(f\"Created graph with {len(graph_data['nodes'])} nodes and {len(graph_data['edges'])} edges.\")" + "print(\"Available graph methods:\", registry.method_registry.list_all(\"graph\"))" ] }, { "cell_type": "markdown", + "id": "614c3c76", "metadata": {}, "source": [ - "## 5. Best Practices for Production\n", + "Now we can use this method via the standard `methods` interface." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a324b97", + "metadata": {}, + "outputs": [], + "source": [ + "# Build a graph using our custom method\n", + "graph_data = methods.build_context_graph(\n", + " method=\"star_builder\",\n", + " center_entity=\"Solar System\",\n", + " satellites=[\"Earth\", \"Mars\", \"Jupiter\", \"Venus\"]\n", + ")\n", "\n", - "1. **Token Limits**: Align `token_limit` with your LLM's context window minus the prompt template size.\n", - "2. **Vector Store**: Use a production-grade vector store (e.g., Weaviate, Qdrant) instead of the mock store.\n", - "3. **Asynchronous Operations**: For high-throughput systems, consider wrapping storage operations in async tasks (though the core logic is synchronous for simplicity).\n", - "4. **Entity Resolution**: Implement a robust `EntityLinker` strategy to prevent graph fragmentation (e.g., \"Alice\" vs \"Alice S.\")." + "print(f\"Created graph with {len(graph_data['nodes'])} nodes and {len(graph_data['edges'])} edges.\")\n", + "print(\"Edges sample:\", graph_data['edges'][0])" + ] + }, + { + "cell_type": "markdown", + "id": "188b2093", + "metadata": {}, + "source": [ + "## 4. Tuning Hybrid Retrieval\n", + "\n", + "Hybrid retrieval combines scores from vector search and graph traversal. You can tune the `hybrid_alpha` parameter to weight these components.\n", + "\n", + "- `hybrid_alpha = 0.0`: Pure Vector Search\n", + "- `hybrid_alpha = 1.0`: Pure Graph Search\n", + "- `hybrid_alpha = 0.5`: Balanced (Default)\n", + "\n", + "Let's configure a `ContextRetriever` with a preference for graph connections." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9752dec4", + "metadata": {}, + "outputs": [], + "source": [ + "# Populate knowledge graph with some test data\n", + "kg.add_node(\"python\", \"concept\", \"Python\")\n", + "kg.add_node(\"ml\", \"concept\", \"Machine Learning\")\n", + "kg.add_edge(\"python\", \"ml\", \"used_for\")\n", + "\n", + "# Initialize retriever with custom tuning\n", + "retriever = ContextRetriever(\n", + " memory_store=memory,\n", + " knowledge_graph=kg,\n", + " vector_store=vs,\n", + " hybrid_alpha=0.7, # Favor graph connections\n", + " max_expansion_hops=2 # Traverse deeper in the graph\n", + ")\n", + "\n", + "# Retrieve\n", + "results = retriever.retrieve(\"Python\")\n", + "\n", + "print(f\"Found {len(results)} results.\")\n", + "for res in results:\n", + " print(f\"Source: {res.source}, Score: {res.score:.2f}, Content: {res.content[:50]}...\")" + ] + }, + { + "cell_type": "markdown", + "id": "4dc770d0", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "You have successfully extended Semantica's context capabilities by:\n", + "1. Implementing a custom memory pruning logic.\n", + "2. Registering a new graph construction algorithm.\n", + "3. Tuning the hybrid retrieval parameters.\n", + "\n", + "These patterns allow you to adapt the context engine to specialized domain requirements." ] } ], @@ -253,4 +297,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} diff --git a/cookbook/introduction/19_Context_Module.ipynb b/cookbook/introduction/19_Context_Module.ipynb index 7548b8ce..bdbeffbe 100644 --- a/cookbook/introduction/19_Context_Module.ipynb +++ b/cookbook/introduction/19_Context_Module.ipynb @@ -2,602 +2,407 @@ "cells": [ { "cell_type": "markdown", + "id": "c21e9c8d", "metadata": {}, "source": [ "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/19_Context_Module.ipynb)\n", "\n", - "# Context Module \n", + "# Context Module — Practical Guide\n", "\n", - "## Overview\n", + "Semantica’s `context` module is the layer that makes an agent “stateful”. It combines:\n", "\n", - "The **Context Module** is the core state management system of Semantica. It allows agents to maintain coherent, persistent, and structured memory across long interactions. Unlike simple RAG systems that only use vector similarity, Semantica's Context Module combines:\n", + "- **Memory** (short-term + long-term) via `AgentMemory`\n", + "- **Graph context** via `ContextGraph`\n", + "- **Hybrid retrieval** (vector + memory + graph) via `ContextRetriever`\n", + "- **High-level UX** via `AgentContext` (recommended entry point)\n", + "- **Entity linking** via `EntityLinker`\n", + "- **Extensibility + config** via `registry` and `config`\n", "\n", - "1. **FastEmbed Integration**: High-performance, local embedding generation.\n", - "2. **Context Graph**: A structured knowledge graph for reasoning about relationships.\n", - "3. **Hierarchical Memory**: A tiered system with short-term (token-limited) and long-term (vector-backed) storage.\n", - "4. **Hybrid Retrieval**: Combining vector search, graph traversal (GraphRAG), and keyword matching.\n", - "5. **Persistence**: Full state serialization.\n", - "\n", - "This notebook provides a technical deep dive into these components.\n", - "\n", - "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/context/)\n", - "\n", - "---\n", - "\n", - "## 1. Setup and Configuration\n", - "\n", - "We need `semantica` and `fastembed` for local, high-speed embedding generation." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: semantica in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (0.0.5)\n", - "Requirement already satisfied: numpy>=1.21.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.26.4)\n", - "Requirement already satisfied: pandas>=1.3.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (2.1.4)\n", - "Requirement already satisfied: scikit-learn>=1.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.6.1)\n", - "Requirement already satisfied: spacy>=3.4.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (3.8.11)\n", - "Requirement already satisfied: transformers>=4.20.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (4.53.2)\n", - "Requirement already satisfied: torch>=1.12.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (2.2.1)\n", - "Requirement already satisfied: sentence-transformers>=2.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (3.2.1)\n", - "Requirement already satisfied: rdflib>=6.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (7.4.0)\n", - "Requirement already satisfied: networkx>=2.8.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (3.5)\n", - "Requirement already satisfied: matplotlib>=3.5.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (3.10.1)\n", - "Requirement already satisfied: seaborn>=0.11.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (0.13.2)\n", - "Requirement already satisfied: plotly>=5.10.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (5.17.0)\n", - "Requirement already satisfied: requests>=2.28.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (2.32.5)\n", - "Requirement already satisfied: beautifulsoup4>=4.11.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (4.12.3)\n", - "Requirement already satisfied: lxml>=4.9.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (6.0.2)\n", - "Requirement already satisfied: pypdf2>=2.10.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (3.0.1)\n", - "Requirement already satisfied: python-docx>=0.8.11 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.1.2)\n", - "Requirement already satisfied: openpyxl>=3.0.10 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (3.1.5)\n", - "Requirement already satisfied: pillow>=9.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (10.4.0)\n", - "Requirement already satisfied: librosa>=0.9.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (0.11.0)\n", - "Requirement already satisfied: opencv-python>=4.6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (4.9.0.80)\n", - "Requirement already satisfied: faiss-cpu>=1.7.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.9.0)\n", - "Requirement already satisfied: weaviate-client>=3.15.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (4.18.1)\n", - "Requirement already satisfied: qdrant-client>=1.3.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.12.2)\n", - "Requirement already satisfied: neo4j>=5.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (6.0.3)\n", - "Requirement already satisfied: falkordb>=1.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.2.2)\n", - "Requirement already satisfied: pymongo>=4.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (4.15.4)\n", - "Requirement already satisfied: sqlalchemy>=1.4.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (2.0.23)\n", - "Requirement already satisfied: psycopg2-binary>=2.9.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (2.9.9)\n", - "Requirement already satisfied: pymysql>=1.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.1.2)\n", - "Requirement already satisfied: redis>=4.3.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (6.4.0)\n", - "Requirement already satisfied: celery>=5.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (5.3.4)\n", - "Requirement already satisfied: kafka-python>=2.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (2.3.0)\n", - "Requirement already satisfied: pulsar-client>=3.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (3.3.0)\n", - "Requirement already satisfied: pika>=1.3.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.3.2)\n", - "Requirement already satisfied: boto3>=1.24.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.36.0)\n", - "Requirement already satisfied: azure-storage-blob>=12.12.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (12.27.1)\n", - "Requirement already satisfied: google-cloud-storage>=2.5.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (2.18.2)\n", - "Requirement already satisfied: pydantic>=1.10.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (2.12.3)\n", - "Requirement already satisfied: click>=8.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (8.2.1)\n", - "Requirement already satisfied: rich>=12.5.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (13.7.1)\n", - "Requirement already satisfied: tqdm>=4.64.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (4.67.1)\n", - "Requirement already satisfied: pyyaml>=6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (6.0.1)\n", - "Requirement already satisfied: toml>=0.10.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (0.10.2)\n", - "Requirement already satisfied: python-dotenv>=0.20.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.1.1)\n", - "Requirement already satisfied: loguru>=0.6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (0.7.3)\n", - "Requirement already satisfied: structlog>=22.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (24.4.0)\n", - "Requirement already satisfied: prometheus-client>=0.14.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (0.18.0)\n", - "Requirement already satisfied: opentelemetry-api>=1.12.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.38.0)\n", - "Requirement already satisfied: opentelemetry-sdk>=1.12.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.38.0)\n", - "Requirement already satisfied: opentelemetry-instrumentation in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (0.59b0)\n", - "Requirement already satisfied: fastapi>=0.78.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (0.120.4)\n", - "Requirement already satisfied: uvicorn>=0.18.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (0.38.0)\n", - "Requirement already satisfied: pytest>=7.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (7.4.3)\n", - "Requirement already satisfied: pytest-cov>=3.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (7.0.0)\n", - "Requirement already satisfied: pytest-asyncio>=0.19.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (0.21.1)\n", - "Requirement already satisfied: black>=22.6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (23.11.0)\n", - "Requirement already satisfied: isort>=5.10.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (5.12.0)\n", - "Requirement already satisfied: flake8>=4.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (7.3.0)\n", - "Requirement already satisfied: mypy>=0.971 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (1.17.1)\n", - "Requirement already satisfied: pre-commit>=2.19.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from semantica) (4.4.0)\n", - "Requirement already satisfied: azure-core>=1.30.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from azure-storage-blob>=12.12.0->semantica) (1.35.0)\n", - "Requirement already satisfied: cryptography>=2.1.4 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from azure-storage-blob>=12.12.0->semantica) (43.0.3)\n", - "Requirement already satisfied: typing-extensions>=4.6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from azure-storage-blob>=12.12.0->semantica) (4.14.1)\n", - "Requirement already satisfied: isodate>=0.6.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from azure-storage-blob>=12.12.0->semantica) (0.7.2)\n", - "Requirement already satisfied: six>=1.11.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from azure-core>=1.30.0->azure-storage-blob>=12.12.0->semantica) (1.16.0)\n", - "Requirement already satisfied: soupsieve>1.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from beautifulsoup4>=4.11.0->semantica) (2.5)\n", - "Requirement already satisfied: mypy-extensions>=0.4.3 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from black>=22.6.0->semantica) (1.0.0)\n", - "Requirement already satisfied: packaging>=22.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from black>=22.6.0->semantica) (24.2)\n", - "Requirement already satisfied: pathspec>=0.9.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from black>=22.6.0->semantica) (0.11.2)\n", - "Requirement already satisfied: platformdirs>=2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from black>=22.6.0->semantica) (3.11.0)\n", - "Requirement already satisfied: botocore<1.37.0,>=1.36.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from boto3>=1.24.0->semantica) (1.36.26)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from boto3>=1.24.0->semantica) (1.0.1)\n", - "Requirement already satisfied: s3transfer<0.12.0,>=0.11.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from boto3>=1.24.0->semantica) (0.11.3)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from botocore<1.37.0,>=1.36.0->boto3>=1.24.0->semantica) (2.9.0.post0)\n", - "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from botocore<1.37.0,>=1.36.0->boto3>=1.24.0->semantica) (1.26.20)\n", - "Requirement already satisfied: billiard<5.0,>=4.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from celery>=5.2.0->semantica) (4.2.1)\n", - "Requirement already satisfied: kombu<6.0,>=5.3.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from celery>=5.2.0->semantica) (5.5.3)\n", - "Requirement already satisfied: vine<6.0,>=5.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from celery>=5.2.0->semantica) (5.1.0)\n", - "Requirement already satisfied: click-didyoumean>=0.3.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from celery>=5.2.0->semantica) (0.3.1)\n", - "Requirement already satisfied: click-repl>=0.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from celery>=5.2.0->semantica) (0.3.0)\n", - "Requirement already satisfied: click-plugins>=1.1.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from celery>=5.2.0->semantica) (1.1.1)\n", - "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from celery>=5.2.0->semantica) (2025.2)\n", - "Requirement already satisfied: colorama in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from click>=8.1.0->semantica) (0.4.6)\n", - "Requirement already satisfied: amqp<6.0.0,>=5.1.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from kombu<6.0,>=5.3.2->celery>=5.2.0->semantica) (5.3.1)\n", - "Requirement already satisfied: prompt-toolkit>=3.0.36 in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from click-repl>=0.2.0->celery>=5.2.0->semantica) (3.0.40)\n", - "Requirement already satisfied: cffi>=1.12 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from cryptography>=2.1.4->azure-storage-blob>=12.12.0->semantica) (1.17.1)\n", - "Requirement already satisfied: pycparser in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from cffi>=1.12->cryptography>=2.1.4->azure-storage-blob>=12.12.0->semantica) (2.22)\n", - "Requirement already satisfied: starlette<0.50.0,>=0.40.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from fastapi>=0.78.0->semantica) (0.46.2)\n", - "Requirement already satisfied: annotated-doc>=0.0.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from fastapi>=0.78.0->semantica) (0.0.3)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pydantic>=1.10.0->semantica) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.41.4 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pydantic>=1.10.0->semantica) (2.41.4)\n", - "Requirement already satisfied: typing-inspection>=0.4.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pydantic>=1.10.0->semantica) (0.4.2)\n", - "Requirement already satisfied: anyio<5,>=3.6.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from starlette<0.50.0,>=0.40.0->fastapi>=0.78.0->semantica) (4.11.0)\n", - "Requirement already satisfied: idna>=2.8 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from anyio<5,>=3.6.2->starlette<0.50.0,>=0.40.0->fastapi>=0.78.0->semantica) (3.10)\n", - "Requirement already satisfied: sniffio>=1.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from anyio<5,>=3.6.2->starlette<0.50.0,>=0.40.0->fastapi>=0.78.0->semantica) (1.3.1)\n", - "Requirement already satisfied: mccabe<0.8.0,>=0.7.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from flake8>=4.0.0->semantica) (0.7.0)\n", - "Requirement already satisfied: pycodestyle<2.15.0,>=2.14.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from flake8>=4.0.0->semantica) (2.14.0)\n", - "Requirement already satisfied: pyflakes<3.5.0,>=3.4.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from flake8>=4.0.0->semantica) (3.4.0)\n", - "Requirement already satisfied: google-auth<3.0dev,>=2.26.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-cloud-storage>=2.5.0->semantica) (2.36.0)\n", - "Requirement already satisfied: google-api-core<3.0.0dev,>=2.15.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-cloud-storage>=2.5.0->semantica) (2.23.0)\n", - "Requirement already satisfied: google-cloud-core<3.0dev,>=2.3.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-cloud-storage>=2.5.0->semantica) (2.4.1)\n", - "Requirement already satisfied: google-resumable-media>=2.7.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-cloud-storage>=2.5.0->semantica) (2.7.2)\n", - "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-cloud-storage>=2.5.0->semantica) (1.6.0)\n", - "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-api-core<3.0.0dev,>=2.15.0->google-cloud-storage>=2.5.0->semantica) (1.66.0)\n", - "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0.dev0,>=3.19.5 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-api-core<3.0.0dev,>=2.15.0->google-cloud-storage>=2.5.0->semantica) (4.25.8)\n", - "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-api-core<3.0.0dev,>=2.15.0->google-cloud-storage>=2.5.0->semantica) (1.25.0)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-auth<3.0dev,>=2.26.1->google-cloud-storage>=2.5.0->semantica) (5.5.2)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-auth<3.0dev,>=2.26.1->google-cloud-storage>=2.5.0->semantica) (0.4.1)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from google-auth<3.0dev,>=2.26.1->google-cloud-storage>=2.5.0->semantica) (4.9)\n", - "Requirement already satisfied: charset_normalizer<4,>=2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests>=2.28.0->semantica) (3.4.0)\n", - "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests>=2.28.0->semantica) (2025.8.3)\n", - "Requirement already satisfied: pyasn1>=0.1.3 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from rsa<5,>=3.1.4->google-auth<3.0dev,>=2.26.1->google-cloud-storage>=2.5.0->semantica) (0.6.1)\n", - "Requirement already satisfied: audioread>=2.1.9 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica) (3.0.1)\n", - "Requirement already satisfied: numba>=0.51.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica) (0.61.2)\n", - "Requirement already satisfied: scipy>=1.6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica) (1.15.2)\n", - "Requirement already satisfied: joblib>=1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica) (1.3.2)\n", - "Requirement already satisfied: decorator>=4.3.0 in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.9.0->semantica) (5.1.1)\n", - "Requirement already satisfied: soundfile>=0.12.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica) (0.13.1)\n", - "Requirement already satisfied: pooch>=1.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica) (1.8.2)\n", - "Requirement already satisfied: soxr>=0.3.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica) (0.5.0.post1)\n", - "Requirement already satisfied: lazy_loader>=0.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica) (0.4)\n", - "Requirement already satisfied: msgpack>=1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from librosa>=0.9.0->semantica) (1.1.0)\n", - "Requirement already satisfied: win32-setctime>=1.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from loguru>=0.6.0->semantica) (1.1.0)\n", - "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib>=3.5.0->semantica) (1.3.2)\n", - "Requirement already satisfied: cycler>=0.10 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib>=3.5.0->semantica) (0.12.1)\n", - "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib>=3.5.0->semantica) (4.57.0)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib>=3.5.0->semantica) (1.4.8)\n", - "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib>=3.5.0->semantica) (3.2.0)\n", - "Requirement already satisfied: pytz in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from neo4j>=5.0.0->semantica) (2024.2)\n", - "Requirement already satisfied: llvmlite<0.45,>=0.44.0dev0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from numba>=0.51.0->librosa>=0.9.0->semantica) (0.44.0)\n", - "Requirement already satisfied: et-xmlfile in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from openpyxl>=3.0.10->semantica) (2.0.0)\n", - "Requirement already satisfied: importlib-metadata<8.8.0,>=6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from opentelemetry-api>=1.12.0->semantica) (6.8.0)\n", - "Requirement already satisfied: zipp>=0.5 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from importlib-metadata<8.8.0,>=6.0->opentelemetry-api>=1.12.0->semantica) (3.17.0)\n", - "Requirement already satisfied: opentelemetry-semantic-conventions==0.59b0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from opentelemetry-sdk>=1.12.0->semantica) (0.59b0)\n", - "Requirement already satisfied: tenacity>=6.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from plotly>=5.10.0->semantica) (8.5.0)\n", - "Requirement already satisfied: cfgv>=2.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pre-commit>=2.19.0->semantica) (3.5.0)\n", - "Requirement already satisfied: identify>=1.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pre-commit>=2.19.0->semantica) (2.6.15)\n", - "Requirement already satisfied: nodeenv>=0.11.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pre-commit>=2.19.0->semantica) (1.9.1)\n", - "Requirement already satisfied: virtualenv>=20.10.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pre-commit>=2.19.0->semantica) (20.24.6)\n", - "Requirement already satisfied: wcwidth in c:\\users\\mohd kaif\\appdata\\roaming\\python\\python311\\site-packages (from prompt-toolkit>=3.0.36->click-repl>=0.2.0->celery>=5.2.0->semantica) (0.2.9)\n", - "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pymongo>=4.2.0->semantica) (2.4.2)\n", - "Requirement already satisfied: iniconfig in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pytest>=7.1.0->semantica) (2.0.0)\n", - "Requirement already satisfied: pluggy<2.0,>=0.12 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pytest>=7.1.0->semantica) (1.5.0)\n", - "Requirement already satisfied: coverage>=7.10.6 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from coverage[toml]>=7.10.6->pytest-cov>=3.0.0->semantica) (7.12.0)\n", - "Requirement already satisfied: grpcio>=1.41.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from qdrant-client>=1.3.0->semantica) (1.68.0)\n", - "Requirement already satisfied: grpcio-tools>=1.41.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from qdrant-client>=1.3.0->semantica) (1.62.3)\n", - "Requirement already satisfied: httpx>=0.20.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from httpx[http2]>=0.20.0->qdrant-client>=1.3.0->semantica) (0.28.1)\n", - "Requirement already satisfied: portalocker<3.0.0,>=2.7.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from qdrant-client>=1.3.0->semantica) (2.10.1)\n", - "Requirement already satisfied: pywin32>=226 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from portalocker<3.0.0,>=2.7.0->qdrant-client>=1.3.0->semantica) (311)\n", - "Requirement already satisfied: setuptools in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from grpcio-tools>=1.41.0->qdrant-client>=1.3.0->semantica) (80.9.0)\n", - "Requirement already satisfied: httpcore==1.* in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client>=1.3.0->semantica) (1.0.9)\n", - "Requirement already satisfied: h11>=0.16 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from httpcore==1.*->httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client>=1.3.0->semantica) (0.16.0)\n", - "Requirement already satisfied: h2<5,>=3 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from httpx[http2]>=0.20.0->qdrant-client>=1.3.0->semantica) (4.1.0)\n", - "Requirement already satisfied: hyperframe<7,>=6.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client>=1.3.0->semantica) (6.0.1)\n", - "Requirement already satisfied: hpack<5,>=4.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client>=1.3.0->semantica) (4.0.0)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from rich>=12.5.0->semantica) (3.0.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from rich>=12.5.0->semantica) (2.19.2)\n", - "Requirement already satisfied: mdurl~=0.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from markdown-it-py>=2.2.0->rich>=12.5.0->semantica) (0.1.2)\n", - "Requirement already satisfied: threadpoolctl>=3.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from scikit-learn>=1.0.0->semantica) (3.2.0)\n", - "Requirement already satisfied: huggingface-hub>=0.20.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from sentence-transformers>=2.2.0->semantica) (0.30.2)\n", - "Requirement already satisfied: filelock in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from transformers>=4.20.0->semantica) (3.16.1)\n", - "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from transformers>=4.20.0->semantica) (2024.11.6)\n", - "Requirement already satisfied: tokenizers<0.22,>=0.21 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from transformers>=4.20.0->semantica) (0.21.4)\n", - "Requirement already satisfied: safetensors>=0.4.3 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from transformers>=4.20.0->semantica) (0.5.3)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from huggingface-hub>=0.20.0->sentence-transformers>=2.2.0->semantica) (2023.10.0)\n", - "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica) (3.0.12)\n", - "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica) (1.0.5)\n", - "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica) (1.0.15)\n", - "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica) (2.0.13)\n", - "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica) (3.0.12)\n", - "Requirement already satisfied: thinc<8.4.0,>=8.3.4 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica) (8.3.10)\n", - "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica) (1.1.3)\n", - "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica) (2.5.2)\n", - "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica) (2.0.10)\n", - "Requirement already satisfied: weasel<0.5.0,>=0.4.2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica) (0.4.3)\n", - "Requirement already satisfied: typer-slim<1.0.0,>=0.3.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica) (0.20.0)\n", - "Requirement already satisfied: jinja2 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from spacy>=3.4.0->semantica) (3.1.6)\n", - "Requirement already satisfied: blis<1.4.0,>=1.3.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from thinc<8.4.0,>=8.3.4->spacy>=3.4.0->semantica) (1.3.3)\n", - "Requirement already satisfied: confection<1.0.0,>=0.0.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from thinc<8.4.0,>=8.3.4->spacy>=3.4.0->semantica) (0.1.5)\n", - "Requirement already satisfied: cloudpathlib<1.0.0,>=0.7.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from weasel<0.5.0,>=0.4.2->spacy>=3.4.0->semantica) (0.23.0)\n", - "Requirement already satisfied: smart-open<8.0.0,>=5.2.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from weasel<0.5.0,>=0.4.2->spacy>=3.4.0->semantica) (7.1.0)\n", - "Requirement already satisfied: wrapt in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from smart-open<8.0.0,>=5.2.1->weasel<0.5.0,>=0.4.2->spacy>=3.4.0->semantica) (1.17.2)\n", - "Requirement already satisfied: greenlet!=0.4.17 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from sqlalchemy>=1.4.0->semantica) (3.2.3)\n", - "Requirement already satisfied: sympy in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from torch>=1.12.0->semantica) (1.13.3)\n", - "Requirement already satisfied: distlib<1,>=0.3.7 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from virtualenv>=20.10.0->pre-commit>=2.19.0->semantica) (0.3.7)\n", - "Requirement already satisfied: validators<1.0.0,>=0.34.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from weaviate-client>=3.15.0->semantica) (0.35.0)\n", - "Requirement already satisfied: authlib<2.0.0,>=1.2.1 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from weaviate-client>=3.15.0->semantica) (1.6.0)\n", - "Requirement already satisfied: deprecation<3.0.0,>=2.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from weaviate-client>=3.15.0->semantica) (2.1.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jinja2->spacy>=3.4.0->semantica) (2.1.3)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in c:\\users\\mohd kaif\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from sympy->torch>=1.12.0->semantica) (1.3.0)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", - "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", - "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", - "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", - "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", - "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", - "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", - "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", - "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n" - ] - } - ], - "source": [ - "!pip install semantica " - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\sentence_transformers\\cross_encoder\\CrossEncoder.py:13: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", - " from tqdm.autonotebook import tqdm, trange\n" - ] - } - ], - "source": [ - "import os\n", - "import shutil\n", - "import numpy as np\n", - "\n", - "from semantica.context import AgentContext, ContextGraph, AgentMemory, EntityLinker\n", - "from semantica.vector_store import VectorStore" + "This notebook focuses on small, runnable examples and keeps imports scoped to each cell." ] }, { "cell_type": "markdown", + "id": "a48e0f10", "metadata": {}, "source": [ - "## 2. Vector Store with FastEmbed\n", + "## 1) Vector store (for long-term memory)\n", "\n", - "The `VectorStore` manages long-term memory. We will configure it to use **FastEmbed**, which runs efficient, quantized embedding models locally on the CPU.\n", - "\n", - "We use the `inmemory` backend for this demo, but Semantica supports Qdrant, Weaviate, and FAISS for production." + "The `VectorStore` can generate embeddings via its internal embedder. If no embedder is available in your environment, it falls back to random vectors so the API stays usable for demos." ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, + "id": "8c845a94", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "fastembed not available. Install with: pip install fastembed. Using fallback embedding method.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Configuring VectorStore to use FastEmbed...\n" - ] - }, - { - "data": { - "text/html": [ - "

🧠 Semantica - 📊 Current Progress

StatusActionModuleSubmoduleFileTime
✅Semantica is embedding💾 embeddingsTextEmbedder-0.02s
✅Semantica is processing🔗 contextAgentMemory-0.05s
✅Semantica is indexing📊 vector_storeVectorStore-0.00s
✅Semantica is processing🔗 contextContextRetriever-0.05s
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generated embedding shape: (384,)\n" - ] - } - ], + "outputs": [], "source": [ - "# 1. Initialize Vector Store\n", + "from semantica.vector_store import VectorStore\n", + "\n", "vs = VectorStore(backend=\"inmemory\", dimension=384)\n", "\n", - "# 2. Configure FastEmbed\n", - "# We explicitly set the method to 'fastembed' and choose a lightweight, high-performance model.\n", - "if hasattr(vs, \"embedder\") and vs.embedder:\n", - " print(\"Configuring VectorStore to use FastEmbed...\")\n", - " vs.embedder.set_text_model(\n", - " method=\"fastembed\", \n", - " model_name=\"BAAI/bge-small-en-v1.5\"\n", - " )\n", + "if getattr(vs, \"embedder\", None) and hasattr(vs.embedder, \"set_text_model\"):\n", + " vs.embedder.set_text_model(method=\"fastembed\", model_name=\"BAAI/bge-small-en-v1.5\")\n", "\n", - "# 3. Verify Embedding Generation\n", - "text = \"Semantica enables complex agent behaviors.\"\n", - "vector = vs.embed(text)\n", - "print(f\"Generated embedding shape: {vector.shape}\")" + "vs.backend, vs.dimension" ] }, { "cell_type": "markdown", + "id": "c1b1ba34", "metadata": {}, "source": [ - "## 3. Context Graph Construction\n", + "## 2) Quick start with `AgentContext` (recommended)\n", "\n", - "The `ContextGraph` stores structured data. While vectors capture *similarity*, graphs capture *relationships*.\n", - "\n", - "We will manually build a small graph to understand the API:\n", - "- `add_node(node_id, node_type, content, **properties)`\n", - "- `add_edge(source_id, target_id, edge_type, **properties)`" + "`AgentContext` is the user-friendly interface that ties memory, vector store, and graph together. If you pass a `ContextGraph`, the system can do GraphRAG-style retrieval." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, + "id": "f4d788b5", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Neighbors of Python:\n", - " - [Relationship: POWERS] -> Semantica Framework (Project)\n" - ] - } - ], + "outputs": [], "source": [ + "from semantica.context import AgentContext, ContextGraph\n", + "\n", "kg = ContextGraph()\n", - "# Add Nodes\n", - "# Note: 'content' is what is indexed for keyword search.\n", - "kg.add_node(\n", - " node_id=\"user_alice\", \n", - " node_type=\"Person\", \n", - " content=\"Alice\", \n", - " role=\"Lead Engineer\"\n", - ")\n", - "kg.add_node(\n", - " node_id=\"tech_python\", \n", - " node_type=\"Technology\", \n", - " content=\"Python\", \n", - " version=\"3.11\"\n", - ")\n", - "kg.add_node(\n", - " node_id=\"project_semantica\", \n", - " node_type=\"Project\", \n", - " content=\"Semantica Framework\"\n", - ")\n", + "context = AgentContext(vector_store=vs, knowledge_graph=kg)\n", "\n", - "# Add Edges (Relationships)\n", - "kg.add_edge(source_id=\"user_alice\", target_id=\"tech_python\", edge_type=\"USES\")\n", - "kg.add_edge(source_id=\"tech_python\", target_id=\"project_semantica\", edge_type=\"POWERS\")\n", - "\n", - "# Traverse the Graph\n", - "print(\"Neighbors of Python:\")\n", - "neighbors = kg.get_neighbors(\"tech_python\")\n", - "for n in neighbors:\n", - " # The neighbor dict contains the connected node info and the relationship that led to it\n", - " print(f\" - [Relationship: {n['relationship']}] -> {n['content']} ({n['type']})\")" + "context.config" ] }, { "cell_type": "markdown", + "id": "638bdbc8", "metadata": {}, "source": [ - "## 4. AgentContext: The Unified Interface\n", + "## 3) Store and retrieve memory\n", "\n", - "`AgentContext` combines the `VectorStore` and `ContextGraph` into a single system. It handles:\n", - "1. **Memory Management**: Routing inputs to short-term or long-term memory.\n", - "2. **Hybrid Retrieval**: Querying both vectors and the graph simultaneously.\n", - "\n", - "We will initialize it with limits to demonstrate the hierarchy." + "A single string is treated as a memory item. You can attach `conversation_id` and `user_id` through metadata-friendly parameters." ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, + "id": "5d65eb00", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "--- Hybrid Retrieval Results ---\n", - "[0.50] Alice is optimizing the graph traversal algorithms in Semantica.\n", - "[0.20] Alice\n" - ] - } - ], + "outputs": [], "source": [ - "context = AgentContext(\n", - " vector_store=vs,\n", - " knowledge_graph=kg,\n", - " token_limit=500, # Max tokens in Short-Term Memory (STM)\n", - " short_term_limit=5 # Max items in STM\n", + "memory_id = context.store(\n", + " \"User prefers short answers about Python.\",\n", + " conversation_id=\"conv_1\",\n", + " user_id=\"user_1\",\n", + " metadata={\"type\": \"preference\"},\n", ")\n", "\n", - "# Store a new memory\n", - "# This is automatically embedded (FastEmbed) and indexed.\n", + "context.get_memory(memory_id)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c1be718", + "metadata": {}, + "outputs": [], + "source": [ "context.store(\n", - " content=\"Alice is optimizing the graph traversal algorithms in Semantica.\",\n", - " conversation_id=\"dev_sync_1\",\n", - " user_id=\"alice\"\n", + " \"User is working on Semantica context module examples.\",\n", + " conversation_id=\"conv_1\",\n", + " user_id=\"user_1\",\n", + " metadata={\"type\": \"note\"},\n", ")\n", "\n", - "# Retrieve Context\n", - "# 'use_graph=True' enables GraphRAG: it finds entities in the query ('Alice') \n", - "# and expands to their neighbors in the graph.\n", - "results = context.retrieve(\n", - " query=\"What is Alice working on?\",\n", - " use_graph=True,\n", - " expand_graph=True\n", - ")\n", - "\n", - "print(\"\\n--- Hybrid Retrieval Results ---\")\n", - "for res in results:\n", - " print(f\"[{res['score']:.2f}] {res['content']}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Hierarchical Memory Management\n", - "\n", - "Watch how the `AgentContext` manages memory pressure. We defined `short_term_limit=5`.\n", - "As we add more items, the oldest ones are flushed from the active buffer but remain safe in the Vector Store." + "context.retrieve(\"Python answers\", max_results=3)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, + "id": "485acf33", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "STM Count (Start): 1\n", - "STM Count (End): 5\n", - "\n", - "Current Short-Term Memory Items:\n", - " - Log entry 5: System status check.\n", - " - Log entry 6: System status check.\n", - " - Log entry 7: System status check.\n", - " - Log entry 8: System status check.\n", - " - Log entry 9: System status check.\n" - ] - } - ], + "outputs": [], "source": [ - "print(f\"STM Count (Start): {len(context.memory.short_term_memory)}\")\n", - "\n", - "# Fill up memory\n", - "for i in range(1, 10):\n", - " context.store(f\"Log entry {i}: System status check.\")\n", - "\n", - "print(f\"STM Count (End): {len(context.memory.short_term_memory)}\")\n", - "print(\"\\nCurrent Short-Term Memory Items:\")\n", - "for item in context.memory.short_term_memory:\n", - " print(f\" - {item.content}\")\n", - "\n", - "# Notice that earlier log entries are gone from this list, \n", - "# but they are still retrievable via search." + "context.conversation(\"conv_1\", max_items=10)" ] }, { "cell_type": "markdown", + "id": "1e43cddd", "metadata": {}, "source": [ - "## 6. Persistence\n", + "## 4) Export, save, load\n", "\n", - "To build stateful agents, we must save and load the context. \n", - "**Crucial Step**: When loading, we must ensure the new `VectorStore` instance is configured with the same embedding model (`FastEmbed`) so vectors match." + "`AgentContext` includes simple persistence helpers. This example uses a temporary directory." ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, + "id": "a264ef4d", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saving context to ./semantica_context_state...\n", - "Initializing fresh agent...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "fastembed not available. Install with: pip install fastembed. Using fallback embedding method.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Restored STM items: 5\n", - "Restored Graph nodes: 3\n", - "Cleanup complete.\n" - ] - } - ], + "outputs": [], "source": [ - "SAVE_PATH = \"./semantica_context_state\"\n", + "export_json = context.export(conversation_id=\"conv_1\", format=\"json\")\n", + "export_json[:300]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b62d1859", + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", "\n", - "# 1. Save state\n", - "print(f\"Saving context to {SAVE_PATH}...\")\n", - "context.save(SAVE_PATH)\n", + "with tempfile.TemporaryDirectory() as d:\n", + " context.save(d)\n", + " context.load(d)\n", "\n", - "# 2. Initialize a fresh AgentContext\n", - "print(\"Initializing fresh agent...\")\n", - "new_kg = ContextGraph()\n", - "new_vs = VectorStore(backend=\"inmemory\", dimension=384)\n", - "\n", - "# !!! IMPORTANT: Re-configure FastEmbed before loading !!!\n", - "if hasattr(new_vs, \"embedder\") and new_vs.embedder:\n", - " new_vs.embedder.set_text_model(\n", - " method=\"fastembed\", \n", - " model_name=\"BAAI/bge-small-en-v1.5\"\n", - " )\n", - "\n", - "restored_context = AgentContext(vector_store=new_vs, knowledge_graph=new_kg)\n", - "\n", - "# 3. Load state\n", - "restored_context.load(SAVE_PATH)\n", - "\n", - "# 4. Verify restoration\n", - "print(f\"Restored STM items: {len(restored_context.memory.short_term_memory)}\")\n", - "print(f\"Restored Graph nodes: {len(new_kg.nodes)}\")\n", - "\n", - "# Cleanup\n", - "if os.path.exists(SAVE_PATH):\n", - " shutil.rmtree(SAVE_PATH)\n", - " print(\"Cleanup complete.\")" + "context.conversation_summary(\"conv_1\")" ] }, { "cell_type": "markdown", + "id": "3b8bf553", "metadata": {}, "source": [ - "## Summary\n", + "## 5) Store documents and build a context graph\n", "\n", - "You have successfully built a persistent, graph-aware, memory-managed context system using Semantica 2.0 components.\n", + "If you store a list, `AgentContext.store(...)` treats it as documents. To keep this notebook lightweight and deterministic, we pass pre-extracted entities and relationships per document." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72930ae7", + "metadata": {}, + "outputs": [], + "source": [ + "documents = [\n", + " {\n", + " \"id\": \"doc_1\",\n", + " \"content\": \"Python is used for machine learning.\",\n", + " \"metadata\": {\"source\": \"docs\"},\n", + " \"entities\": [\n", + " {\"id\": \"e_python\", \"text\": \"Python\", \"type\": \"PROGRAMMING_LANGUAGE\"},\n", + " {\"id\": \"e_ml\", \"text\": \"Machine Learning\", \"type\": \"CONCEPT\"},\n", + " ],\n", + " \"relationships\": [\n", + " {\n", + " \"source_id\": \"e_python\",\n", + " \"target_id\": \"e_ml\",\n", + " \"type\": \"used_for\",\n", + " \"confidence\": 0.9,\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"id\": \"doc_2\",\n", + " \"content\": \"PyTorch is a machine learning framework.\",\n", + " \"metadata\": {\"source\": \"docs\"},\n", + " \"entities\": [\n", + " {\"id\": \"e_pytorch\", \"text\": \"PyTorch\", \"type\": \"FRAMEWORK\"},\n", + " {\"id\": \"e_ml\", \"text\": \"Machine Learning\", \"type\": \"CONCEPT\"},\n", + " ],\n", + " \"relationships\": [\n", + " {\n", + " \"source_id\": \"e_pytorch\",\n", + " \"target_id\": \"e_ml\",\n", + " \"type\": \"implements\",\n", + " \"confidence\": 0.95,\n", + " }\n", + " ],\n", + " },\n", + "]\n", "\n", - "**Key Takeaways:**\n", - "- Use `VectorStore` with `FastEmbed` for efficient local vectors.\n", - "- Use `ContextGraph` to map relationships (`add_node`, `add_edge`).\n", - "- Use `AgentContext` to manage the lifecycle of memories and retrieval.\n", - "- Always configure your embedder on the fresh instance before calling `load()`." + "stats = context.store(\n", + " documents,\n", + " extract_entities=False,\n", + " extract_relationships=False,\n", + " link_entities=True,\n", + ")\n", + "\n", + "stats" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24f8dba8", + "metadata": {}, + "outputs": [], + "source": [ + "kg.stats()" + ] + }, + { + "cell_type": "markdown", + "id": "f4671f2a", + "metadata": {}, + "source": [ + "## 6) Explore the graph with `ContextGraph`\n", + "\n", + "The graph supports keyword querying and neighbor expansion." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df2e5fcd", + "metadata": {}, + "outputs": [], + "source": [ + "kg.query(\"machine learning\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0836feeb", + "metadata": {}, + "outputs": [], + "source": [ + "kg.get_neighbors(\"e_python\", hops=2)" + ] + }, + { + "cell_type": "markdown", + "id": "1be1adf1", + "metadata": {}, + "source": [ + "## 7) Entity linking with `EntityLinker`\n", + "\n", + "`EntityLinker` assigns stable URIs and can link related or duplicate entities across sources." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24b011c0", + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.context import EntityLinker\n", + "\n", + "linker = EntityLinker(knowledge_graph={\"entities\": [{\"id\": \"e_py\", \"text\": \"Python\", \"type\": \"PROGRAMMING_LANGUAGE\"}]})\n", + "\n", + "entities = [\n", + " {\"id\": \"e1\", \"text\": \"Python\", \"type\": \"PROGRAMMING_LANGUAGE\"},\n", + " {\"id\": \"e2\", \"text\": \"PyTorch\", \"type\": \"FRAMEWORK\"},\n", + "]\n", + "\n", + "linked = linker.link(\"Python and PyTorch\", entities=entities)\n", + "[(e.entity_id, e.uri, len(e.linked_entities)) for e in linked]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a282de3a", + "metadata": {}, + "outputs": [], + "source": [ + "linker.link_entities(\"e1\", \"e2\", link_type=\"related_to\", confidence=0.8)\n", + "linker.get_entity_links(\"e1\")[:2]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2d11f87f", + "metadata": {}, + "outputs": [], + "source": [ + "linker.build_entity_web()[\"statistics\"]" + ] + }, + { + "cell_type": "markdown", + "id": "072efafd", + "metadata": {}, + "source": [ + "## 8) Low-level building blocks: `AgentMemory` and `ContextRetriever`\n", + "\n", + "If you want more control than `AgentContext`, you can wire the parts directly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3791d6c6", + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.context import AgentMemory, ContextRetriever\n", + "\n", + "memory = AgentMemory(vector_store=vs, knowledge_graph=kg, retention_policy=\"unlimited\")\n", + "memory.store(\"Python powers Semantica.\", metadata={\"type\": \"fact\", \"conversation_id\": \"conv_2\"})\n", + "\n", + "retriever = ContextRetriever(memory_store=memory, knowledge_graph=kg, vector_store=vs)\n", + "results = retriever.retrieve(\"Python Semantica\", max_results=5)\n", + "\n", + "[(r.content, r.source, round(r.score, 3)) for r in results]" + ] + }, + { + "cell_type": "markdown", + "id": "92060402", + "metadata": {}, + "source": [ + "## 9) Methods, registry, and configuration\n", + "\n", + "The `methods` layer exposes convenience functions, while `registry` lets you plug in your own implementations. `config` provides runtime configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "896e7001", + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.context.config import context_config\n", + "\n", + "context_config.set(\"retention_policy\", \"7_days\")\n", + "context_config.get(\"retention_policy\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e925a2e0", + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.context.methods import build_context_graph\n", + "from semantica.context.registry import method_registry\n", + "\n", + "def custom_graph_method(entities, relationships, conversations=None, **kwargs):\n", + " return {\n", + " \"nodes\": [],\n", + " \"edges\": [],\n", + " \"statistics\": {\"node_count\": 0, \"edge_count\": 0},\n", + " }\n", + "\n", + "method_registry.register(\"graph\", \"custom_demo\", custom_graph_method)\n", + "method_registry.list_all(\"graph\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21fa6cb3", + "metadata": {}, + "outputs": [], + "source": [ + "build_context_graph(\n", + " entities=[{\"id\": \"e1\", \"text\": \"Python\", \"type\": \"PROGRAMMING_LANGUAGE\"}],\n", + " relationships=[{\"source_id\": \"e1\", \"target_id\": \"e2\", \"type\": \"related_to\"}],\n", + " method=\"custom_demo\",\n", + ")" ] } ], @@ -608,16 +413,8 @@ "name": "python3" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.11" } }, "nbformat": 4, diff --git a/docs/modules.md b/docs/modules.md index f259de57..0315933b 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -798,12 +798,12 @@ These modules provide context engineering for agents and foundation data managem **Components:** -- `ContextGraphBuilder` — Builds context graphs from various sources +- `ContextGraph` — In-memory context graph store and builder methods - `ContextNode` — Context graph node data structure - `ContextEdge` — Context graph edge data structure - `AgentMemory` — Manages persistent agent memory with RAG - `MemoryItem` — Memory item data structure -- `EntityLinker` — Links entities across sources with URIs +- `EntityLinker` — Links entities across sources with URI assignment - `ContextRetriever` — Retrieves relevant context from multiple sources **Algorithms:** @@ -818,19 +818,19 @@ These modules provide context engineering for agents and foundation data managem **Quick Example:** ```python -from semantica.context import build_context, ContextGraphBuilder, AgentMemory +from semantica.context import ContextGraph, AgentMemory +from semantica.context.methods import build_context_graph # Using convenience function -result = build_context( +result = build_context_graph( entities=entities, relationships=relationships, - vector_store=vs, - knowledge_graph=kg + method="entities_relationships" ) # Using classes directly -builder = ContextGraphBuilder() -graph = builder.build_from_entities_and_relationships(entities, relationships) +graph = ContextGraph() +graph_data = graph.build_from_entities_and_relationships(entities, relationships) memory = AgentMemory(vector_store=vs, knowledge_graph=kg) memory_id = memory.store("User asked about Python", metadata={"type": "conversation"}) diff --git a/docs/reference/context.md b/docs/reference/context.md index e4dcd8c5..ce090a57 100644 --- a/docs/reference/context.md +++ b/docs/reference/context.md @@ -184,8 +184,8 @@ graph.add_nodes([ # Add Edges graph.add_edges([ { - "source": "FastAPI", - "target": "Python", + "source_id": "FastAPI", + "target_id": "Python", "type": "WRITTEN_IN" } ]) diff --git a/semantica/context/__init__.py b/semantica/context/__init__.py index 0f15d6bb..17ea7993 100644 --- a/semantica/context/__init__.py +++ b/semantica/context/__init__.py @@ -56,8 +56,9 @@ Key Features: - Auto-detection of content types and retrieval strategies Main Classes: - - AgentContext: High-level interface for agent context management (store, retrieve, forget, conversation) - - ContextGraphBuilder: Builds context graphs from various sources + - AgentContext: High-level interface for agent context management (store, + retrieve, forget, conversation) + - ContextGraph: In-memory context graph store and builder methods - ContextNode: Context graph node data structure - ContextEdge: Context graph edge data structure - AgentMemory: Manages persistent agent memory with RAG @@ -67,11 +68,14 @@ Main Classes: - LinkedEntity: Linked entity with context - ContextRetriever: Retrieves relevant context from multiple sources - RetrievedContext: Retrieved context item data structure - - MethodRegistry: Registry for custom context methods (accessed via registry submodule) - - ContextConfig: Configuration manager for context module (accessed via config submodule) + - MethodRegistry: Registry for custom context methods (accessed via registry + submodule) + - ContextConfig: Configuration manager for context module (accessed via + config submodule) Submodules: - - methods: Context engineering methods (build_context_graph, store_memory, retrieve_context, etc.) + - methods: Context engineering methods (build_context_graph, store_memory, + retrieve_context, etc.) - registry: Method registry for custom methods (method_registry, MethodRegistry) - config: Configuration management (context_config, ContextConfig) @@ -82,16 +86,20 @@ Example Usage: >>> memory_id = context.store("User asked about Python", conversation_id="conv1") >>> results = context.retrieve("Python programming") >>> stats = context.store(["Doc 1", "Doc 2"], extract_entities=True) - + >>> # Low-level classes (for advanced use cases) - >>> from semantica.context import ContextGraphBuilder, AgentMemory, methods - >>> builder = ContextGraphBuilder() - >>> graph = builder.build_from_entities_and_relationships(entities, relationships) + >>> from semantica.context import ContextGraph, AgentMemory, methods + >>> graph = ContextGraph() + >>> graph_data = graph.build_from_entities_and_relationships(entities, relationships) >>> memory = AgentMemory(vector_store=vs, knowledge_graph=kg) - >>> memory_id = memory.store("User asked about Python", metadata={"type": "conversation"}) + >>> memory_id = memory.store( + ... "User asked about Python", metadata={"type": "conversation"} + ... ) >>> results = memory.retrieve("Python", max_results=5) >>> # Using methods submodule - >>> graph = methods.build_context_graph(entities, relationships, method="entities_relationships") + >>> graph = methods.build_context_graph( + ... entities, relationships, method="entities_relationships" + ... ) Author: Semantica Contributors License: MIT diff --git a/semantica/context/agent_context.py b/semantica/context/agent_context.py index 42503052..cd6815a6 100644 --- a/semantica/context/agent_context.py +++ b/semantica/context/agent_context.py @@ -1,9 +1,10 @@ """ High-Level Agent Context Interface -This module provides a simplified, generic high-level interface for agent context management, -RAG, and GraphRAG. It uses generic method names (store, retrieve, forget, conversation) that -auto-detect content types and retrieval strategies, with boolean flags for common options. +This module provides a simplified, generic high-level interface for agent context +management, RAG, and GraphRAG. It uses generic method names (store, retrieve, +forget, conversation) that auto-detect content types and retrieval strategies, +with boolean flags for common options. Key Features: - Generic methods: store(), retrieve(), forget(), conversation() @@ -22,10 +23,8 @@ Example Usage: from datetime import datetime from typing import Any, Dict, List, Optional, Union -from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger from .agent_memory import AgentMemory -from .context_graph import ContextGraph from .context_retriever import ContextRetriever, RetrievedContext from .entity_linker import EntityLinker @@ -33,17 +32,19 @@ from .entity_linker import EntityLinker class AgentContext: """ High-level interface for agent context management, RAG, and GraphRAG. - + Provides generic methods (store, retrieve, forget, conversation) that auto-detect content types and retrieval strategies. Uses boolean flags for common options. - + Attributes: vector_store: Vector store instance knowledge_graph: Knowledge graph instance (optional) memory: AgentMemory instance (via property) - retriever: ContextRetriever instance (via property, if knowledge_graph available) - graph_builder: ContextGraph instance (via property, if knowledge_graph supports building) - + retriever: ContextRetriever instance (via property, if knowledge_graph + available) + graph_builder: ContextGraph instance (via property, if knowledge_graph + supports building) + Main Methods: - store(): Store memory or documents (auto-detects type) - retrieve(): Retrieve context (auto-detects RAG vs GraphRAG) @@ -53,7 +54,7 @@ class AgentContext: - stats(): Get memory statistics - link(): Link entities in text - build_graph(): Build context graph manually - + Example: >>> context = AgentContext(vector_store=vs, knowledge_graph=kg) >>> memory_id = context.store("User likes Python", conversation_id="conv1") @@ -71,11 +72,11 @@ class AgentContext: use_graph_expansion: bool = True, max_expansion_hops: int = 2, hybrid_alpha: float = 0.5, - **kwargs + **kwargs, ): """ Initialize AgentContext. - + Args: vector_store: Vector store instance (required) knowledge_graph: Knowledge graph instance (optional, enables GraphRAG) @@ -83,24 +84,25 @@ class AgentContext: max_memories: Maximum number of memories (default: 10000) use_graph_expansion: Enable graph expansion for retrieval (default: True) max_expansion_hops: Maximum hops for graph expansion (default: 2) - hybrid_alpha: Balance between vector (0) and graph (1) retrieval (default: 0.5) + hybrid_alpha: Balance between vector (0) and graph (1) retrieval + (default: 0.5) **kwargs: Additional options passed to underlying components - + Raises: ValueError: If vector_store is not provided """ self.logger = get_logger("agent_context") - + if vector_store is None: raise ValueError( "vector_store is required for AgentContext. " "Please provide a vector store instance. " "Example: AgentContext(vector_store=your_vector_store)" ) - + self.vector_store = vector_store self.knowledge_graph = knowledge_graph - + # Initialize AgentMemory retention_policy = f"{retention_days}_days" if retention_days else "unlimited" memory_config = { @@ -108,10 +110,10 @@ class AgentContext: "knowledge_graph": knowledge_graph, "retention_policy": retention_policy, "max_memory_size": max_memories, - **kwargs + **kwargs, } self._memory = AgentMemory(**memory_config) - + # Initialize ContextRetriever if knowledge_graph available if knowledge_graph: retriever_config = { @@ -121,17 +123,17 @@ class AgentContext: "use_graph_expansion": use_graph_expansion, "max_expansion_hops": max_expansion_hops, "hybrid_alpha": hybrid_alpha, - **kwargs + **kwargs, } self._retriever = ContextRetriever(**retriever_config) else: self._retriever = None - - # Initialize ContextGraphBuilder if knowledge_graph available and supports building + + # Initialize graph builder if knowledge graph supports building self._graph_builder = None if knowledge_graph and hasattr(knowledge_graph, "build_from_conversations"): - self._graph_builder = knowledge_graph - + self._graph_builder = knowledge_graph + # Store config self.config = { "retention_days": retention_days, @@ -159,22 +161,23 @@ class AgentContext: def save(self, path: str) -> None: """ Save context state (memory, vector store, graph) to disk. - + Args: path: Directory path to save to """ import os + os.makedirs(path, exist_ok=True) - + # 1. Save AgentMemory state if hasattr(self._memory, "save"): self._memory.save(path) - + # 2. Save VectorStore state if hasattr(self.vector_store, "save"): vs_path = os.path.join(path, "vector_store") self.vector_store.save(vs_path) - + # 3. Save KnowledgeGraph state if self.knowledge_graph: if hasattr(self.knowledge_graph, "save_to_file"): @@ -185,42 +188,44 @@ class AgentContext: # Generic save kg_path = os.path.join(path, "knowledge_graph") self.knowledge_graph.save(kg_path) - + self.logger.info(f"Saved agent context to {path}") def load(self, path: str) -> None: """ Load context state from disk. - + Args: path: Directory path to load from """ import os - + if not os.path.exists(path): self.logger.warning(f"Context path not found: {path}") return - + # 1. Load AgentMemory state if hasattr(self._memory, "load"): self._memory.load(path) - + # 2. Load VectorStore state if hasattr(self.vector_store, "load"): vs_path = os.path.join(path, "vector_store") if os.path.exists(vs_path): self.vector_store.load(vs_path) - + # 3. Load KnowledgeGraph state if self.knowledge_graph: kg_json_path = os.path.join(path, "knowledge_graph.json") kg_dir_path = os.path.join(path, "knowledge_graph") - - if hasattr(self.knowledge_graph, "load_from_file") and os.path.exists(kg_json_path): + + if hasattr(self.knowledge_graph, "load_from_file") and os.path.exists( + kg_json_path + ): self.knowledge_graph.load_from_file(kg_json_path) elif hasattr(self.knowledge_graph, "load") and os.path.exists(kg_dir_path): self.knowledge_graph.load(kg_dir_path) - + self.logger.info(f"Loaded agent context from {path}") def store( @@ -235,15 +240,15 @@ class AgentContext: extract_relationships: bool = True, link_entities: bool = True, auto_extract: bool = False, - **kwargs + **kwargs, ) -> Union[str, Dict[str, Any]]: """ Store content (memory or documents). - + Auto-detects content type: - Single string -> Memory item - List of strings/dicts -> Documents (builds graph if knowledge_graph available) - + Args: content: Content to store: - str: Single memory item @@ -257,17 +262,19 @@ class AgentContext: extract_entities: Extract entities from documents (default: True) extract_relationships: Extract relationships (default: True) link_entities: Link entities across documents (default: True) - auto_extract: Auto-extract entities/relationships if not provided (default: False) + auto_extract: Auto-extract entities/relationships if not provided + (default: False) **kwargs: Additional options passed to storage methods - + Returns: - str: Memory ID (for single memory) - - Dict: Statistics (for documents) with keys: stored_count, graph_nodes, graph_edges - + - Dict: Statistics (for documents) with stored_count, graph_nodes, + graph_edges + Example: >>> # Store memory >>> memory_id = context.store("User likes Python", conversation_id="conv1") - + >>> # Store documents >>> stats = context.store(["Doc 1", "Doc 2"], extract_entities=True) """ @@ -279,19 +286,19 @@ class AgentContext: memory_metadata["conversation_id"] = conversation_id if user_id: memory_metadata["user_id"] = user_id - + return self._memory.store( content, metadata=memory_metadata, entities=entities, relationships=relationships, - **kwargs + **kwargs, ) - + elif isinstance(content, list): # Documents - normalize to list of dicts documents = self._normalize_documents(content) - + # Add entities/relationships to documents if provided if entities or relationships: for doc in documents: @@ -299,40 +306,42 @@ class AgentContext: doc["entities"] = entities if relationships and "relationships" not in doc: doc["relationships"] = relationships - + # Store each document in vector store stored_ids = [] for doc in documents: doc_metadata = {**(metadata or {}), **doc.get("metadata", {})} doc_entities = doc.get("entities") doc_relationships = doc.get("relationships") - + doc_id = self._memory.store( doc["content"], metadata=doc_metadata, entities=doc_entities, relationships=doc_relationships, - **kwargs + **kwargs, ) stored_ids.append(doc_id) - + # Build graph if knowledge_graph available and flags set graph_stats = {} - if self.knowledge_graph and (extract_entities or extract_relationships or auto_extract): + if self.knowledge_graph and ( + extract_entities or extract_relationships or auto_extract + ): graph_stats = self._build_graph_from_documents( documents, extract_entities=extract_entities or auto_extract, extract_relationships=extract_relationships or auto_extract, - link_entities=link_entities + link_entities=link_entities, ) - + return { "stored_count": len(stored_ids), "memory_ids": stored_ids, "graph_nodes": graph_stats.get("node_count", 0), "graph_edges": graph_stats.get("edge_count", 0), } - + else: raise ValueError( f"Unsupported content type: {type(content)}. " @@ -352,19 +361,20 @@ class AgentContext: include_relationships: bool = False, expand_graph: bool = True, deduplicate: bool = True, - **kwargs + **kwargs, ) -> List[Dict[str, Any]]: """ Retrieve relevant context. - + Auto-detects best strategy: - If knowledge_graph available -> GraphRAG (hybrid retrieval) - Otherwise -> Simple RAG (vector only) - + Args: query: Search query max_results: Maximum results (default: 5) - use_graph: Force graph usage (None=auto-detect, True=force, False=vector only) + use_graph: Force graph usage (None=auto-detect, True=force, + False=vector only) min_score: Minimum relevance score (default: 0.0) conversation_id: Filter by conversation ID user_id: Filter by user ID @@ -373,27 +383,27 @@ class AgentContext: expand_graph: Use graph expansion (default: True) deduplicate: Deduplicate results (default: True) **kwargs: Additional filters (type, date_range, etc.) - + Returns: - List of context dicts with keys: content, score, source, metadata, related_entities - + List of context dicts with content, score, source, and metadata + Example: >>> # Auto-detects RAG vs GraphRAG >>> results = context.retrieve("Python programming") - + >>> # Force vector-only retrieval >>> results = context.retrieve("Python", use_graph=False) """ # Auto-detect strategy if use_graph is None: - use_graph = (self.knowledge_graph is not None and self._retriever is not None) - + use_graph = self.knowledge_graph is not None and self._retriever is not None + # Apply filters if provided if conversation_id: kwargs["conversation_id"] = conversation_id if user_id: kwargs["user_id"] = user_id - + if use_graph and self._retriever: # GraphRAG: Use ContextRetriever (hybrid retrieval) results = self._retriever.retrieve( @@ -401,17 +411,17 @@ class AgentContext: max_results=max_results, use_graph_expansion=expand_graph, min_relevance_score=min_score, - **kwargs + **kwargs, ) # Convert RetrievedContext to dicts - return [self._context_to_dict(r, include_entities, include_relationships) for r in results] + return [ + self._context_to_dict(r, include_entities, include_relationships) + for r in results + ] else: # Simple RAG: Use AgentMemory (vector + memory) results = self._memory.retrieve( - query, - max_results=max_results, - min_score=min_score, - **kwargs + query, max_results=max_results, min_score=min_score, **kwargs ) # Convert to dicts return [self._memory_to_dict(r) for r in results] @@ -422,21 +432,21 @@ class AgentContext: conversation_id: Optional[str] = None, user_id: Optional[str] = None, days_old: Optional[int] = None, - **filters + **filters, ) -> int: """ Delete memories. - + Args: memory_id: Delete specific memory by ID conversation_id: Delete all memories in conversation user_id: Delete all memories for user days_old: Delete memories older than N days **filters: Additional filters (type, date_range, etc.) - + Returns: Number of memories deleted - + Example: >>> context.forget(memory_id="mem123") >>> context.forget(conversation_id="conv1") @@ -445,7 +455,7 @@ class AgentContext: if memory_id: self._memory.delete_memory(memory_id) return 1 - + # Build filters dict filter_dict = {} if conversation_id: @@ -454,9 +464,12 @@ class AgentContext: filter_dict["user_id"] = user_id if days_old: from datetime import datetime, timedelta - filter_dict["start_date"] = (datetime.now() - timedelta(days=days_old)).isoformat() + + filter_dict["start_date"] = ( + datetime.now() - timedelta(days=days_old) + ).isoformat() filter_dict.update(filters) - + return self._memory.clear_memory(**filter_dict) def conversation( @@ -465,30 +478,31 @@ class AgentContext: max_items: int = 100, reverse: bool = False, include_metadata: bool = True, - **kwargs + **kwargs, ) -> List[Dict[str, Any]]: """ Get conversation history. - + Args: conversation_id: Conversation ID max_items: Maximum items to return (default: 100) reverse: Return in reverse chronological order (default: False) include_metadata: Include full metadata (default: True) **kwargs: Additional options - + Returns: List of memory dicts in conversation - + Example: >>> history = context.conversation("conv1") - >>> history = context.conversation("conv1", reverse=True, include_metadata=False) + >>> history = context.conversation( + ... "conv1", reverse=True, include_metadata=False + ... ) """ history = self._memory.get_conversation_history( - conversation_id=conversation_id, - max_items=max_items + conversation_id=conversation_id, max_items=max_items ) - + # Convert to dicts result = [] for item in history: @@ -500,38 +514,36 @@ class AgentContext: if include_metadata: item_dict["metadata"] = item.get("metadata", {}) result.append(item_dict) - + if reverse: result.reverse() - + return result - def _normalize_documents(self, content: List[Union[str, Dict[str, Any]]]) -> List[Dict[str, Any]]: + def _normalize_documents( + self, content: List[Union[str, Dict[str, Any]]] + ) -> List[Dict[str, Any]]: """Normalize documents to list of dicts.""" documents = [] for i, doc in enumerate(content): if isinstance(doc, str): - documents.append({ - "content": doc, - "id": f"doc_{i}", - "metadata": {} - }) + documents.append({"content": doc, "id": f"doc_{i}", "metadata": {}}) elif isinstance(doc, dict): if "content" not in doc: raise ValueError(f"Document dict must have 'content' key: {doc}") - + doc_dict = { "content": doc["content"], "id": doc.get("id", f"doc_{i}"), - "metadata": doc.get("metadata", {}) + "metadata": doc.get("metadata", {}), } - + # Preserve entities and relationships if present if "entities" in doc: doc_dict["entities"] = doc["entities"] if "relationships" in doc: doc_dict["relationships"] = doc["relationships"] - + documents.append(doc_dict) else: raise ValueError(f"Unsupported document type: {type(doc)}") @@ -542,12 +554,12 @@ class AgentContext: documents: List[Dict[str, Any]], extract_entities: bool = True, extract_relationships: bool = True, - link_entities: bool = True + link_entities: bool = True, ) -> Dict[str, Any]: """Build graph from documents.""" if not self._graph_builder: return {"node_count": 0, "edge_count": 0} - + try: # Convert documents to conversations format conversations = [] @@ -556,18 +568,18 @@ class AgentContext: "id": doc.get("id", "unknown"), "content": doc["content"], "entities": doc.get("entities", []), - "relationships": doc.get("relationships", []) + "relationships": doc.get("relationships", []), } conversations.append(conv) - + # Build graph from conversations graph = self._graph_builder.build_from_conversations( conversations, link_entities=link_entities, extract_intents=False, - extract_sentiments=False + extract_sentiments=False, ) - + return { "node_count": graph.get("statistics", {}).get("node_count", 0), "edge_count": graph.get("statistics", {}).get("edge_count", 0), @@ -580,7 +592,7 @@ class AgentContext: self, context: RetrievedContext, include_entities: bool = True, - include_relationships: bool = False + include_relationships: bool = False, ) -> Dict[str, Any]: """Convert RetrievedContext to dict.""" result = { @@ -589,13 +601,13 @@ class AgentContext: "source": context.source, "metadata": context.metadata, } - + if include_entities: result["related_entities"] = context.related_entities - + if include_relationships: result["related_relationships"] = context.related_relationships - + return result def _memory_to_dict(self, memory: Dict[str, Any]) -> Dict[str, Any]: @@ -612,13 +624,13 @@ class AgentContext: def get_memory(self, memory_id: str) -> Optional[Dict[str, Any]]: """ Get a specific memory by ID. - + Args: memory_id: Memory ID - + Returns: Memory dict or None if not found - + Example: >>> memory = context.get_memory("mem123") """ @@ -635,10 +647,10 @@ class AgentContext: def stats(self) -> Dict[str, Any]: """ Get statistics about stored memories. - + Returns: Dict with statistics: total_items, items_by_type, etc. - + Example: >>> stats = context.stats() >>> print(f"Total memories: {stats['total_items']}") @@ -650,23 +662,23 @@ class AgentContext: text: str, entities: Optional[List[Dict[str, Any]]] = None, similarity_threshold: float = 0.8, - **kwargs + **kwargs, ) -> List[Dict[str, Any]]: """ Link entities in text (if knowledge_graph available). - + Args: text: Text containing entities entities: List of entities to link similarity_threshold: Similarity threshold for linking (default: 0.8) **kwargs: Additional options - + Returns: List of linked entity dicts - + Raises: ValueError: If knowledge_graph not available - + Example: >>> linked = context.link("Python is used for ML", entities=[...]) """ @@ -676,17 +688,15 @@ class AgentContext: "Please initialize AgentContext with a knowledge_graph parameter. " "Example: AgentContext(vector_store=vs, knowledge_graph=kg)" ) - - from .entity_linker import EntityLinker - + linker = EntityLinker( knowledge_graph=self.knowledge_graph, similarity_threshold=similarity_threshold, - **kwargs + **kwargs, ) - + linked_entities = linker.link(text, entities=entities or []) - + return [ { "entity_id": e.entity_id, @@ -705,26 +715,28 @@ class AgentContext: relationships: Optional[List[Dict[str, Any]]] = None, conversations: Optional[List[Dict[str, Any]]] = None, link_entities: bool = True, - **kwargs + **kwargs, ) -> Dict[str, Any]: """ Build context graph from entities, relationships, or conversations. - + Args: entities: List of entities relationships: List of relationships conversations: List of conversations link_entities: Link entities (default: True) **kwargs: Additional options - + Returns: Graph statistics dict - + Raises: ValueError: If knowledge_graph not available - + Example: - >>> graph = context.build_graph(entities=entities, relationships=relationships) + >>> graph = context.build_graph( + ... entities=entities, relationships=relationships + ... ) """ if not self._graph_builder: raise ValueError( @@ -732,16 +744,14 @@ class AgentContext: "Please initialize AgentContext with a knowledge_graph parameter. " "Example: AgentContext(vector_store=vs, knowledge_graph=kg)" ) - + if entities and relationships: graph = self._graph_builder.build_from_entities_and_relationships( entities, relationships, **kwargs ) elif conversations: graph = self._graph_builder.build_from_conversations( - conversations, - link_entities=link_entities, - **kwargs + conversations, link_entities=link_entities, **kwargs ) else: raise ValueError( @@ -750,20 +760,20 @@ class AgentContext: f"relationships={relationships is not None}, " f"conversations={conversations is not None}" ) - + return graph.get("statistics", {}) # Memory Management Methods def exists(self, memory_id: str) -> bool: """ Check if memory exists. - + Args: memory_id: Memory ID to check - + Returns: True if memory exists, False otherwise - + Example: >>> if context.exists("mem123"): ... print("Memory exists") @@ -773,13 +783,13 @@ class AgentContext: def count(self, **filters) -> int: """ Get total memory count with optional filters. - + Args: **filters: Optional filters (conversation_id, user_id, type, etc.) - + Returns: Total count of memories matching filters - + Example: >>> total = context.count() >>> conv_count = context.count(conversation_id="conv1") @@ -787,7 +797,7 @@ class AgentContext: stats = self._memory.get_statistics() if not filters: return stats.get("total_items", 0) - + # Filter memories count = 0 for memory_id in self._memory.memory_items: @@ -799,13 +809,13 @@ class AgentContext: def get(self, memory_id: str) -> Optional[Dict[str, Any]]: """ Get memory by ID (alias for get_memory). - + Args: memory_id: Memory ID - + Returns: Memory dict or None if not found - + Example: >>> memory = context.get("mem123") """ @@ -816,20 +826,20 @@ class AgentContext: memory_id: str, content: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, - **kwargs + **kwargs, ) -> bool: """ Update existing memory. - + Args: memory_id: Memory ID to update content: New content (optional) metadata: New metadata (optional, merged with existing) **kwargs: Additional fields to update - + Returns: True if updated successfully, False if not found - + Example: >>> context.update("mem123", content="Updated content") >>> context.update("mem123", metadata={"new_key": "value"}) @@ -837,13 +847,15 @@ class AgentContext: memory_item = self._memory.get_memory(memory_id) if not memory_item: return False - + # Get current values - current_content = content if content is not None else memory_item.get("content", "") + current_content = ( + content if content is not None else memory_item.get("content", "") + ) current_metadata = memory_item.get("metadata", {}) if metadata: current_metadata.update(metadata) - + # Delete old and create new self._memory.delete_memory(memory_id) new_id = self._memory.store( @@ -851,21 +863,21 @@ class AgentContext: metadata=current_metadata, entities=memory_item.get("entities", []), relationships=memory_item.get("relationships", []), - **kwargs + **kwargs, ) - + return new_id is not None def delete(self, memory_id: str) -> bool: """ Delete memory by ID (alias for forget with memory_id). - + Args: memory_id: Memory ID to delete - + Returns: True if deleted, False if not found - + Example: >>> context.delete("mem123") """ @@ -877,13 +889,13 @@ class AgentContext: def clear(self, **filters) -> int: """ Clear memories with filters (alias for forget). - + Args: **filters: Filter criteria (conversation_id, user_id, days_old, etc.) - + Returns: Number of memories deleted - + Example: >>> deleted = context.clear(conversation_id="conv1") >>> deleted = context.clear(days_old=90) @@ -896,21 +908,21 @@ class AgentContext: user_id: Optional[str] = None, limit: int = 100, offset: int = 0, - **filters + **filters, ) -> List[Dict[str, Any]]: """ List memories with pagination. - + Args: conversation_id: Filter by conversation ID user_id: Filter by user ID limit: Maximum items to return (default: 100) offset: Number of items to skip (default: 0) **filters: Additional filters - + Returns: List of memory dicts - + Example: >>> memories = context.list(conversation_id="conv1", limit=50) >>> memories = context.list(user_id="user123", limit=20, offset=10) @@ -920,27 +932,31 @@ class AgentContext: all_filters["conversation_id"] = conversation_id if user_id: all_filters["user_id"] = user_id - + results = [] - for memory_id in list(self._memory.memory_items.keys())[offset:offset + limit]: + for memory_id in list(self._memory.memory_items.keys())[ + offset : offset + limit + ]: memory_item = self._memory.memory_items[memory_id] - if not all_filters or self._memory._matches_filters(memory_item, all_filters): + if not all_filters or self._memory._matches_filters( + memory_item, all_filters + ): mem_dict = self._memory.get_memory(memory_id) if mem_dict: results.append(mem_dict) - + return results def batch_store(self, items: List[Union[str, Dict[str, Any]]]) -> List[str]: """ Store multiple items at once. - + Args: items: List of items to store (strings or dicts with content) - + Returns: List of memory IDs - + Example: >>> ids = context.batch_store(["Item 1", "Item 2", "Item 3"]) """ @@ -952,12 +968,23 @@ class AgentContext: elif isinstance(item, dict): content = item.get("content", "") if content: + extra_fields = { + k: v + for k, v in item.items() + if k + not in [ + "content", + "metadata", + "conversation_id", + "user_id", + ] + } memory_id = self.store( content, metadata=item.get("metadata"), conversation_id=item.get("conversation_id"), user_id=item.get("user_id"), - **{k: v for k, v in item.items() if k not in ["content", "metadata", "conversation_id", "user_id"]} + **extra_fields, ) memory_ids.append(memory_id) return memory_ids @@ -965,13 +992,13 @@ class AgentContext: def batch_delete(self, memory_ids: List[str]) -> int: """ Delete multiple memories. - + Args: memory_ids: List of memory IDs to delete - + Returns: Number of memories deleted - + Example: >>> deleted = context.batch_delete(["mem1", "mem2", "mem3"]) """ @@ -984,13 +1011,13 @@ class AgentContext: def batch_update(self, updates: List[Dict[str, Any]]) -> int: """ Update multiple memories. - + Args: updates: List of update dicts with 'memory_id' and fields to update - + Returns: Number of memories updated - + Example: >>> updated = context.batch_update([ ... {"memory_id": "mem1", "content": "New content"}, @@ -1000,7 +1027,8 @@ class AgentContext: updated = 0 for update in updates: memory_id = update.get("memory_id") - if memory_id and self.update(memory_id, **{k: v for k, v in update.items() if k != "memory_id"}): + update_fields = {k: v for k, v in update.items() if k != "memory_id"} + if memory_id and self.update(memory_id, **update_fields): updated += 1 return updated @@ -1008,108 +1036,116 @@ class AgentContext: def search(self, query: str, **filters) -> List[Dict[str, Any]]: """ Simple search with filters (alias for retrieve). - + Args: query: Search query - **filters: Additional filters (max_results, min_score, conversation_id, etc.) - + **filters: Additional filters (max_results, min_score, + conversation_id, etc.) + Returns: List of context dicts - + Example: >>> results = context.search("Python", max_results=10) >>> results = context.search("Python", conversation_id="conv1") """ return self.retrieve(query, **filters) - def find_similar(self, content: str, limit: int = 5, **kwargs) -> List[Dict[str, Any]]: + def find_similar( + self, content: str, limit: int = 5, **kwargs + ) -> List[Dict[str, Any]]: """ Find similar content. - + Args: content: Content to find similar items for limit: Maximum results (default: 5) **kwargs: Additional options - + Returns: List of similar content dicts - + Example: >>> similar = context.find_similar("Python programming", limit=5) """ return self.retrieve(content, max_results=limit, **kwargs) - def get_context(self, query: str, max_results: int = 5, **kwargs) -> List[Dict[str, Any]]: + def get_context( + self, query: str, max_results: int = 5, **kwargs + ) -> List[Dict[str, Any]]: """ Get context for query. - + Args: query: Query string max_results: Maximum results (default: 5) **kwargs: Additional options - + Returns: List of context dicts - + Example: >>> context_data = context.get_context("Python", max_results=10) """ return self.retrieve(query, max_results=max_results, **kwargs) - def expand_query(self, query: str, max_hops: int = 2, **kwargs) -> List[Dict[str, Any]]: + def expand_query( + self, query: str, max_hops: int = 2, **kwargs + ) -> List[Dict[str, Any]]: """ Expand query with graph context. - + Args: query: Query string max_hops: Maximum graph expansion hops (default: 2) **kwargs: Additional options - + Returns: List of expanded context dicts - + Example: >>> expanded = context.expand_query("Python", max_hops=3) """ if not self._retriever: return self.retrieve(query, **kwargs) - + return self.retrieve( - query, - expand_graph=True, - max_expansion_hops=max_hops, - **kwargs + query, expand_graph=True, max_expansion_hops=max_hops, **kwargs ) # Conversation Methods - def get_conversation(self, conversation_id: str, limit: int = 100, **kwargs) -> List[Dict[str, Any]]: + def get_conversation( + self, conversation_id: str, limit: int = 100, **kwargs + ) -> List[Dict[str, Any]]: """ Get conversation (alias for conversation). - + Args: conversation_id: Conversation ID limit: Maximum items (default: 100) **kwargs: Additional options - + Returns: List of conversation memory dicts - + Example: >>> conv = context.get_conversation("conv1", limit=50) """ return self.conversation(conversation_id, max_items=limit, **kwargs) - def list_conversations(self, user_id: Optional[str] = None, limit: int = 50) -> List[str]: + def list_conversations( + self, user_id: Optional[str] = None, limit: int = 50 + ) -> List[str]: """ List all conversations. - + Args: user_id: Filter by user ID (optional) limit: Maximum conversations to return (default: 50) - + Returns: List of conversation IDs - + Example: >>> conversations = context.list_conversations() >>> user_convs = context.list_conversations(user_id="user123") @@ -1120,19 +1156,19 @@ class AgentContext: if conv_id: if user_id is None or memory_item.metadata.get("user_id") == user_id: conversation_ids.add(conv_id) - + return list(conversation_ids)[:limit] def delete_conversation(self, conversation_id: str) -> int: """ Delete entire conversation. - + Args: conversation_id: Conversation ID to delete - + Returns: Number of memories deleted - + Example: >>> deleted = context.delete_conversation("conv1") """ @@ -1141,18 +1177,18 @@ class AgentContext: def conversation_summary(self, conversation_id: str) -> Dict[str, Any]: """ Get conversation summary. - + Args: conversation_id: Conversation ID - + Returns: Summary dict with count, first_message, last_message, etc. - + Example: >>> summary = context.conversation_summary("conv1") """ history = self.conversation(conversation_id, max_items=1000) - + if not history: return { "conversation_id": conversation_id, @@ -1160,33 +1196,32 @@ class AgentContext: "first_message": None, "last_message": None, } - + return { "conversation_id": conversation_id, "message_count": len(history), "first_message": history[0] if history else None, "last_message": history[-1] if history else None, - "user_id": history[0].get("metadata", {}).get("user_id") if history else None, + "user_id": ( + history[0].get("metadata", {}).get("user_id") if history else None + ), } # Export/Import Methods def export( - self, - conversation_id: Optional[str] = None, - format: str = 'json', - **filters + self, conversation_id: Optional[str] = None, format: str = "json", **filters ) -> Union[str, Dict[str, Any]]: """ Export memories. - + Args: conversation_id: Export specific conversation (optional) format: Export format ('json' or 'dict', default: 'json') **filters: Additional filters - + Returns: Exported data (JSON string or dict) - + Example: >>> data = context.export(conversation_id="conv1") >>> data = context.export(format='dict') @@ -1194,47 +1229,51 @@ class AgentContext: all_filters = {**filters} if conversation_id: all_filters["conversation_id"] = conversation_id - + memories = self.list(**all_filters) - + export_data = { "exported_at": datetime.now().isoformat(), "count": len(memories), - "memories": memories + "memories": memories, } - - if format == 'json': + + if format == "json": import json + return json.dumps(export_data, indent=2, default=str) return export_data - def import_data(self, data: Union[str, Dict[str, Any]], format: str = 'json') -> int: + def import_data( + self, data: Union[str, Dict[str, Any]], format: str = "json" + ) -> int: """ Import memories. - + Args: data: Data to import (JSON string or dict) format: Data format ('json' or 'dict', default: 'json') - + Returns: Number of memories imported - + Example: >>> imported = context.import_data(json_string) >>> imported = context.import_data(data_dict, format='dict') """ - if format == 'json': + if format == "json": import json + if isinstance(data, str): data = json.loads(data) - + if not isinstance(data, dict): raise ValueError("Invalid data format. Expected dict or JSON string.") - + memories = data.get("memories", []) if not memories: return 0 - + imported = 0 for memory in memories: try: @@ -1248,36 +1287,36 @@ class AgentContext: imported += 1 except Exception as e: self.logger.warning(f"Failed to import memory: {e}") - + return imported def backup(self, **filters) -> str: """ Create backup (alias for export). - + Args: **filters: Filter criteria - + Returns: JSON string of backup data - + Example: >>> backup_data = context.backup() >>> backup_data = context.backup(conversation_id="conv1") """ - return self.export(format='json', **filters) + return self.export(format="json", **filters) - def restore(self, data: Union[str, Dict[str, Any]], format: str = 'json') -> int: + def restore(self, data: Union[str, Dict[str, Any]], format: str = "json") -> int: """ Restore from backup (alias for import_data). - + Args: data: Backup data (JSON string or dict) format: Data format ('json' or 'dict', default: 'json') - + Returns: Number of memories restored - + Example: >>> restored = context.restore(backup_data) """ @@ -1287,17 +1326,17 @@ class AgentContext: def health(self) -> Dict[str, Any]: """ Check system health. - + Returns: Health status dict - + Example: >>> health = context.health() >>> print(f"Status: {health['status']}") """ stats = self.stats() total = stats.get("total_items", 0) - + health_status = { "status": "healthy" if total >= 0 else "error", "total_memories": total, @@ -1306,47 +1345,46 @@ class AgentContext: "retriever_available": self._retriever is not None, "graph_builder_available": self._graph_builder is not None, } - + return health_status - def usage_stats(self, period: str = 'day') -> Dict[str, Any]: + def usage_stats(self, period: str = "day") -> Dict[str, Any]: """ Get usage statistics. - + Args: period: Time period ('day', 'week', 'month', default: 'day') - + Returns: Usage statistics dict - + Example: >>> usage = context.usage_stats(period='week') """ from datetime import datetime, timedelta - + now = datetime.now() - if period == 'day': + if period == "day": start = now - timedelta(days=1) - elif period == 'week': + elif period == "week": start = now - timedelta(weeks=1) - elif period == 'month': + elif period == "month": start = now - timedelta(days=30) else: start = now - timedelta(days=1) - + stats = self.stats() total = stats.get("total_items", 0) - + # Count recent memories recent_count = 0 for memory_id, memory_item in self._memory.memory_items.items(): if memory_item.timestamp >= start: recent_count += 1 - + return { "period": period, "total_memories": total, "recent_memories": recent_count, "conversations": len(self.list_conversations()), } - diff --git a/semantica/context/agent_memory.py b/semantica/context/agent_memory.py index 5dc16fe2..41acee35 100644 --- a/semantica/context/agent_memory.py +++ b/semantica/context/agent_memory.py @@ -40,13 +40,16 @@ Key Features: - Fallback keyword search when vector store unavailable Main Classes: - - MemoryItem: Memory item data structure with content, timestamp, metadata, entities, relationships + - MemoryItem: Memory item data structure with content, timestamp, metadata, + entities, relationships - AgentMemory: Agent memory manager with RAG integration Example Usage: >>> from semantica.context import AgentMemory >>> memory = AgentMemory(vector_store=vs, knowledge_graph=kg) - >>> memory_id = memory.store("User asked about Python", metadata={"type": "conversation"}) + >>> memory_id = memory.store( + ... "User asked about Python", metadata={"type": "conversation"} + ... ) >>> results = memory.retrieve("Python", max_results=5) >>> history = memory.get_conversation_history(conversation_id="conv_123") >>> stats = memory.get_statistics() @@ -62,7 +65,6 @@ from typing import Any, Dict, List, Optional, Union import numpy as np -from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker from ..utils.types import EntityDict, RelationshipDict @@ -113,14 +115,14 @@ class AgentMemory: self.retention_policy = self.config.get("retention_policy", "unlimited") self.max_memory_size = self.config.get("max_memory_size", 10000) self.short_term_limit = self.config.get("short_term_limit", 10) - self.token_limit = self.config.get("token_limit", 2000) # Default 2000 tokens for short-term + self.token_limit = self.config.get("token_limit", 2000) # In-memory storage self.memory_items: Dict[str, MemoryItem] = {} self.memory_index: deque = deque(maxlen=self.max_memory_size) - + # Hierarchical Memory: Short-term buffer - # Note: We use a list instead of deque for short-term to support flexible pruning (tokens & count) + # Note: We use a list for flexible pruning (tokens & count). self.short_term_memory: List[MemoryItem] = [] # Initialize progress tracker @@ -132,50 +134,53 @@ class AgentMemory: def save(self, path: str) -> None: """ Save memory state to disk. - + Args: path: Directory path to save to """ import os import pickle - + os.makedirs(path, exist_ok=True) - + data = { "memory_items": self.memory_items, "memory_index": self.memory_index, "short_term_memory": self.short_term_memory, - "stats": self.stats + "stats": self.stats, } - + with open(os.path.join(path, "agent_memory.pkl"), "wb") as f: pickle.dump(data, f) - + self.logger.info(f"Saved agent memory to {path}") def load(self, path: str) -> None: """ Load memory state from disk. - + Args: path: Directory path to load from """ import os import pickle - + file_path = os.path.join(path, "agent_memory.pkl") if not os.path.exists(file_path): self.logger.warning(f"Memory file not found: {file_path}") return - + with open(file_path, "rb") as f: data = pickle.load(f) - + self.memory_items = data.get("memory_items", {}) self.memory_index = data.get("memory_index", deque(maxlen=self.max_memory_size)) self.short_term_memory = data.get("short_term_memory", []) - self.stats = data.get("stats", {"total_items": 0, "items_by_type": {}, "last_accessed": None}) - + self.stats = data.get( + "stats", + {"total_items": 0, "items_by_type": {}, "last_accessed": None}, + ) + self.logger.info(f"Loaded agent memory from {path}") def store( @@ -241,7 +246,10 @@ class AgentMemory: # Store in vector store if hasattr(self.vector_store, "store_vectors"): # Use concrete VectorStore implementation - vectors = [np.array(memory_item.embedding)] if isinstance(memory_item.embedding, list) else [memory_item.embedding] + if isinstance(memory_item.embedding, list): + vectors = [np.array(memory_item.embedding)] + else: + vectors = [memory_item.embedding] meta = [memory_item.metadata] self.vector_store.store_vectors(vectors=vectors, metadata=meta) elif hasattr(self.vector_store, "add"): @@ -335,19 +343,22 @@ class AgentMemory: if hasattr(self.vector_store, "search_vectors"): # Use concrete VectorStore implementation query_vector = self._generate_embedding(query) - query_vector = np.array(query_vector) if isinstance(query_vector, list) else query_vector - + if isinstance(query_vector, list): + query_vector = np.array(query_vector) + raw_results = self.vector_store.search_vectors( query_vector=query_vector, k=max_results * 2 ) # Convert dict results to objects with .id attribute + class ResultObj: def __init__(self, d): self.id = d.get("id") self.score = d.get("score") self.metadata = d.get("metadata") + vector_results = [ResultObj(r) for r in raw_results] - + elif hasattr(self.vector_store, "search"): vector_results = self.vector_store.search( query=query, limit=max_results * 2 @@ -355,7 +366,7 @@ class AgentMemory: for result in vector_results: memory_id = result.id - + # Skip if already found in short-term if memory_id in seen_ids: continue @@ -535,32 +546,34 @@ class AgentMemory: """Search short-term memory (simple keyword match).""" results = [] query_terms = query.lower().split() - + # Iterate through short-term memory (most recent first) for item in reversed(self.short_term_memory): if not self._matches_filters(item, filters): continue - + content_lower = item.content.lower() - + # Simple scoring based on term overlap matches = sum(1 for term in query_terms if term in content_lower) if matches > 0: score = matches / len(query_terms) # Boost score for recent items (short-term) - score = min(1.0, score + 0.1) - - results.append({ - "memory_id": item.memory_id, - "content": item.content, - "score": score, - "timestamp": item.timestamp.isoformat(), - "metadata": item.metadata, - "entities": item.entities, - "relationships": item.relationships, - "source": "short_term" - }) - + score = min(1.0, score + 0.1) + + results.append( + { + "memory_id": item.memory_id, + "content": item.content, + "score": score, + "timestamp": item.timestamp.isoformat(), + "metadata": item.metadata, + "entities": item.entities, + "relationships": item.relationships, + "source": "short_term", + } + ) + return results def _generate_memory_id(self) -> str: @@ -577,16 +590,18 @@ class AgentMemory: def _prune_short_term_memory(self) -> None: """ Prune short-term memory based on count and token limits. - + Removes oldest items until constraints are met. """ # 1. Prune by count while len(self.short_term_memory) > self.short_term_limit: self.short_term_memory.pop(0) # Remove oldest - + # 2. Prune by tokens - current_tokens = sum(self._count_tokens(item.content) for item in self.short_term_memory) - + current_tokens = sum( + self._count_tokens(item.content) for item in self.short_term_memory + ) + while current_tokens > self.token_limit and self.short_term_memory: removed_item = self.short_term_memory.pop(0) # Remove oldest current_tokens -= self._count_tokens(removed_item.content) @@ -594,10 +609,10 @@ class AgentMemory: def _count_tokens(self, text: str) -> int: """ Estimate token count (approximation). - + Args: text: Input text - + Returns: Estimated token count """ @@ -628,14 +643,20 @@ class AgentMemory: for entity in entities: entity_id = entity.get("id") or entity.get("entity_id") if entity_id: - graph_nodes.append({ - "id": entity_id, - "type": entity.get("type", "entity"), - "properties": { - "content": entity.get("text") or entity.get("label") or entity_id, - **entity + graph_nodes.append( + { + "id": entity_id, + "type": entity.get("type", "entity"), + "properties": { + "content": ( + entity.get("text") + or entity.get("label") + or entity_id + ), + **entity, + }, } - }) + ) if graph_nodes: self.knowledge_graph.add_nodes(graph_nodes) @@ -645,16 +666,18 @@ class AgentMemory: source = rel.get("source_id") target = rel.get("target_id") if source and target: - graph_edges.append({ - "source_id": source, - "target_id": target, - "type": rel.get("type", "related_to"), - "weight": rel.get("confidence", 1.0), - "properties": rel - }) + graph_edges.append( + { + "source_id": source, + "target_id": target, + "type": rel.get("type", "related_to"), + "weight": rel.get("confidence", 1.0), + "properties": rel, + } + ) if graph_edges: self.knowledge_graph.add_edges(graph_edges) - + return # Legacy dict update @@ -788,13 +811,13 @@ class AgentMemory: def exists(self, memory_id: str) -> bool: """ Check if memory exists. - + Args: memory_id: Memory ID to check - + Returns: True if exists, False otherwise - + Example: >>> if memory.exists("mem123"): ... print("Memory exists") @@ -804,20 +827,20 @@ class AgentMemory: def count(self, **filters) -> int: """ Get count with filters. - + Args: **filters: Filter criteria - + Returns: Count of memories matching filters - + Example: >>> total = memory.count() >>> conv_count = memory.count(conversation_id="conv1") """ if not filters: return len(self.memory_items) - + count = 0 for memory_id, memory_item in self.memory_items.items(): if self._matches_filters(memory_item, filters): @@ -827,13 +850,13 @@ class AgentMemory: def get(self, memory_id: str) -> Optional[Dict[str, Any]]: """ Get memory by ID. - + Args: memory_id: Memory ID - + Returns: Memory dict or None if not found - + Example: >>> memory = memory.get("mem123") """ @@ -844,32 +867,32 @@ class AgentMemory: memory_id: str, content: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, - **kwargs + **kwargs, ) -> bool: """ Update memory. - + Args: memory_id: Memory ID to update content: New content (optional) metadata: New metadata (optional, merged with existing) **kwargs: Additional fields to update - + Returns: True if updated, False if not found - + Example: >>> memory.update("mem123", content="Updated content") """ if memory_id not in self.memory_items: return False - + memory_item = self.memory_items[memory_id] current_content = content if content is not None else memory_item.content current_metadata = memory_item.metadata.copy() if metadata: current_metadata.update(metadata) - + # Delete old and create new self.delete_memory(memory_id) new_id = self.store( @@ -877,21 +900,21 @@ class AgentMemory: metadata=current_metadata, entities=memory_item.entities, relationships=memory_item.relationships, - **kwargs + **kwargs, ) - + return new_id is not None def delete(self, memory_id: str) -> bool: """ Delete memory (alias for delete_memory). - + Args: memory_id: Memory ID to delete - + Returns: True if deleted, False if not found - + Example: >>> memory.delete("mem123") """ @@ -900,13 +923,13 @@ class AgentMemory: def clear(self, **filters) -> int: """ Clear with filters (alias for clear_memory). - + Args: **filters: Filter criteria - + Returns: Number of memories deleted - + Example: >>> deleted = memory.clear(conversation_id="conv1") """ @@ -916,31 +939,33 @@ class AgentMemory: def search(self, query: str, **filters) -> List[Dict[str, Any]]: """ Simple search (alias for retrieve). - + Args: query: Search query **filters: Additional filters - + Returns: List of memory dicts - + Example: >>> results = memory.search("Python", max_results=10) """ return self.retrieve(query, **filters) - def find_similar(self, content: str, limit: int = 5, **kwargs) -> List[Dict[str, Any]]: + def find_similar( + self, content: str, limit: int = 5, **kwargs + ) -> List[Dict[str, Any]]: """ Find similar content. - + Args: content: Content to find similar items for limit: Maximum results (default: 5) **kwargs: Additional options - + Returns: List of similar memory dicts - + Example: >>> similar = memory.find_similar("Python programming", limit=5) """ @@ -949,14 +974,14 @@ class AgentMemory: def find_by_entity(self, entity_id: str, limit: int = 10) -> List[Dict[str, Any]]: """ Find by entity. - + Args: entity_id: Entity ID to search for limit: Maximum results (default: 10) - + Returns: List of memory dicts containing the entity - + Example: >>> results = memory.find_by_entity("entity_123") """ @@ -972,17 +997,19 @@ class AgentMemory: break return results[:limit] - def find_by_relationship(self, relationship_type: str, limit: int = 10) -> List[Dict[str, Any]]: + def find_by_relationship( + self, relationship_type: str, limit: int = 10 + ) -> List[Dict[str, Any]]: """ Find by relationship. - + Args: relationship_type: Relationship type to search for limit: Maximum results (default: 10) - + Returns: List of memory dicts containing the relationship - + Example: >>> results = memory.find_by_relationship("related_to") """ @@ -1005,21 +1032,21 @@ class AgentMemory: user_id: Optional[str] = None, limit: int = 100, offset: int = 0, - **filters + **filters, ) -> List[Dict[str, Any]]: """ List memories. - + Args: conversation_id: Filter by conversation ID user_id: Filter by user ID limit: Maximum items (default: 100) offset: Number of items to skip (default: 0) **filters: Additional filters - + Returns: List of memory dicts - + Example: >>> memories = memory.list(conversation_id="conv1", limit=50) """ @@ -1028,50 +1055,57 @@ class AgentMemory: all_filters["conversation_id"] = conversation_id if user_id: all_filters["user_id"] = user_id - + results = [] - for memory_id in list(self.memory_items.keys())[offset:offset + limit]: + for memory_id in list(self.memory_items.keys())[offset : offset + limit]: memory_item = self.memory_items[memory_id] if not all_filters or self._matches_filters(memory_item, all_filters): # Also check user_id and conversation_id in metadata if user_id and memory_item.metadata.get("user_id") != user_id: continue - if conversation_id and memory_item.metadata.get("conversation_id") != conversation_id: + if ( + conversation_id + and memory_item.metadata.get("conversation_id") != conversation_id + ): continue - + mem_dict = self.get_memory(memory_id) if mem_dict: results.append(mem_dict) - + return results - def get_by_conversation(self, conversation_id: str, limit: int = 100) -> List[Dict[str, Any]]: + def get_by_conversation( + self, conversation_id: str, limit: int = 100 + ) -> List[Dict[str, Any]]: """ Get conversation memories. - + Args: conversation_id: Conversation ID limit: Maximum items (default: 100) - + Returns: List of memory dicts in conversation - + Example: >>> memories = memory.get_by_conversation("conv1") """ - return self.get_conversation_history(conversation_id=conversation_id, max_items=limit) + return self.get_conversation_history( + conversation_id=conversation_id, max_items=limit + ) def get_by_user(self, user_id: str, limit: int = 100) -> List[Dict[str, Any]]: """ Get user memories. - + Args: user_id: User ID limit: Maximum items (default: 100) - + Returns: List of memory dicts for user - + Example: >>> memories = memory.get_by_user("user123") """ @@ -1088,21 +1122,19 @@ class AgentMemory: def get_recent(self, limit: int = 10) -> List[Dict[str, Any]]: """ Get recent memories. - + Args: limit: Maximum items (default: 10) - + Returns: List of recent memory dicts - + Example: >>> recent = memory.get_recent(limit=20) """ results = [] sorted_items = sorted( - self.memory_items.items(), - key=lambda x: x[1].timestamp, - reverse=True + self.memory_items.items(), key=lambda x: x[1].timestamp, reverse=True ) for memory_id, _ in sorted_items[:limit]: mem_dict = self.get_memory(memory_id) @@ -1114,29 +1146,31 @@ class AgentMemory: self, start_date: Union[str, datetime], end_date: Union[str, datetime], - limit: int = 100 + limit: int = 100, ) -> List[Dict[str, Any]]: """ Get by date range. - + Args: start_date: Start date (ISO string or datetime) end_date: End date (ISO string or datetime) limit: Maximum items (default: 100) - + Returns: List of memory dicts in date range - + Example: >>> memories = memory.get_by_date("2024-01-01", "2024-12-31") """ if isinstance(start_date, str): from dateutil.parser import parse + start_date = parse(start_date) if isinstance(end_date, str): from dateutil.parser import parse + end_date = parse(end_date) - + results = [] for memory_id, memory_item in self.memory_items.items(): if start_date <= memory_item.timestamp <= end_date: @@ -1150,14 +1184,14 @@ class AgentMemory: def get_by_type(self, type: str, limit: int = 100) -> List[Dict[str, Any]]: """ Get by type. - + Args: type: Memory type limit: Maximum items (default: 100) - + Returns: List of memory dicts of specified type - + Example: >>> memories = memory.get_by_type("conversation") """ @@ -1175,13 +1209,13 @@ class AgentMemory: def batch_store(self, items: List[Union[str, Dict[str, Any]]]) -> List[str]: """ Batch store. - + Args: items: List of items to store - + Returns: List of memory IDs - + Example: >>> ids = memory.batch_store(["Item 1", "Item 2"]) """ @@ -1193,10 +1227,15 @@ class AgentMemory: elif isinstance(item, dict): content = item.get("content", "") if content: + extra_fields = { + k: v + for k, v in item.items() + if k not in ["content", "metadata"] + } memory_id = self.store( content, metadata=item.get("metadata"), - **{k: v for k, v in item.items() if k not in ["content", "metadata"]} + **extra_fields, ) memory_ids.append(memory_id) return memory_ids @@ -1204,13 +1243,13 @@ class AgentMemory: def batch_delete(self, memory_ids: List[str]) -> int: """ Batch delete. - + Args: memory_ids: List of memory IDs to delete - + Returns: Number of memories deleted - + Example: >>> deleted = memory.batch_delete(["mem1", "mem2"]) """ @@ -1223,92 +1262,94 @@ class AgentMemory: def batch_update(self, updates: List[Dict[str, Any]]) -> int: """ Batch update. - + Args: updates: List of update dicts with 'memory_id' and fields - + Returns: Number of memories updated - + Example: >>> updated = memory.batch_update([{"memory_id": "mem1", "content": "New"}]) """ updated = 0 for update in updates: memory_id = update.get("memory_id") - if memory_id and self.update(memory_id, **{k: v for k, v in update.items() if k != "memory_id"}): + update_fields = {k: v for k, v in update.items() if k != "memory_id"} + if memory_id and self.update(memory_id, **update_fields): updated += 1 return updated # Export/Import def export( - self, - conversation_id: Optional[str] = None, - format: str = 'json', - **filters + self, conversation_id: Optional[str] = None, format: str = "json", **filters ) -> Union[str, Dict[str, Any]]: """ Export memories. - + Args: conversation_id: Export specific conversation (optional) format: Export format ('json' or 'dict', default: 'json') **filters: Additional filters - + Returns: Exported data - + Example: >>> data = memory.export(conversation_id="conv1") """ all_filters = {**filters} if conversation_id: all_filters["conversation_id"] = conversation_id - + memories = [] for memory_id, memory_item in self.memory_items.items(): if not all_filters or self._matches_filters(memory_item, all_filters): mem_dict = self.get_memory(memory_id) if mem_dict: memories.append(mem_dict) - + export_data = { "exported_at": datetime.now().isoformat(), "count": len(memories), - "memories": memories + "memories": memories, } - - if format == 'json': + + if format == "json": import json + return json.dumps(export_data, indent=2, default=str) return export_data - def import_data(self, data: Union[str, Dict[str, Any]], format: str = 'json') -> int: + def import_data( + self, data: Union[str, Dict[str, Any]], format: str = "json" + ) -> int: """ Import memories. - + Args: data: Data to import format: Data format ('json' or 'dict', default: 'json') - + Returns: Number of memories imported - + Example: >>> imported = memory.import_data(json_string) """ - if format == 'json': + if format == "json": import json + if isinstance(data, str): data = json.loads(data) - + if not isinstance(data, dict): raise ValueError("Invalid data format") - + memories = data.get("memories", []) if not memories: return 0 - + imported = 0 for memory in memories: try: @@ -1320,20 +1361,20 @@ class AgentMemory: imported += 1 except Exception as e: self.logger.warning(f"Failed to import memory: {e}") - + return imported # Statistics def stats(self, **filters) -> Dict[str, Any]: """ Get statistics (enhance existing). - + Args: **filters: Optional filters - + Returns: Statistics dict - + Example: >>> stats = memory.stats() >>> conv_stats = memory.stats(conversation_id="conv1") @@ -1346,10 +1387,10 @@ class AgentMemory: def count_by_type(self) -> Dict[str, int]: """ Count by type. - + Returns: Dict mapping type to count - + Example: >>> counts = memory.count_by_type() """ @@ -1362,10 +1403,10 @@ class AgentMemory: def count_by_user(self) -> Dict[str, int]: """ Count by user. - + Returns: Dict mapping user_id to count - + Example: >>> counts = memory.count_by_user() """ @@ -1378,10 +1419,10 @@ class AgentMemory: def count_by_conversation(self) -> Dict[str, int]: """ Count by conversation. - + Returns: Dict mapping conversation_id to count - + Example: >>> counts = memory.count_by_conversation() """ @@ -1389,4 +1430,4 @@ class AgentMemory: for memory_item in self.memory_items.values(): conv_id = memory_item.metadata.get("conversation_id", "unknown") counts[conv_id] = counts.get(conv_id, 0) + 1 - return counts \ No newline at end of file + return counts diff --git a/semantica/context/config.py b/semantica/context/config.py index e5150d38..0028a6d0 100644 --- a/semantica/context/config.py +++ b/semantica/context/config.py @@ -1,7 +1,8 @@ """ Configuration Management Module for Context Engineering -This module provides centralized configuration management for context engineering operations, +This module provides centralized configuration management for context engineering +operations, supporting multiple configuration sources including environment variables, config files, and programmatic configuration. @@ -47,7 +48,11 @@ from ..utils.logging import get_logger class ContextConfig: - """Configuration manager for context module - supports .env files, environment variables, and programmatic config.""" + """ + Configuration manager for context module. + + Supports .env files, environment variables, and programmatic config. + """ def __init__(self, config_file: Optional[str] = None): """Initialize configuration manager.""" diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 1866fe68..0bd3d752 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -17,17 +17,17 @@ Key Features: from collections import defaultdict, deque from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Set, Union, Tuple +from typing import Any, Dict, List, Optional, Set, Union -from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker -from ..utils.types import EntityDict, RelationshipDict from .entity_linker import EntityLinker + @dataclass class ContextNode: """Context graph node (Internal implementation).""" + node_id: str node_type: str content: str @@ -39,15 +39,13 @@ class ContextNode: props = self.properties.copy() props.update(self.metadata) props["content"] = self.content - return { - "id": self.node_id, - "type": self.node_type, - "properties": props - } + return {"id": self.node_id, "type": self.node_type, "properties": props} + @dataclass class ContextEdge: """Context graph edge (Internal implementation).""" + source_id: str target_id: str edge_type: str @@ -61,13 +59,14 @@ class ContextEdge: "target_id": self.target_id, "type": self.edge_type, "weight": self.weight, - "properties": self.metadata + "properties": self.metadata, } + class ContextGraph: """ In-memory implementation of context graph. - + Provides capabilities to build, store, and query a context graph. """ @@ -88,20 +87,20 @@ class ContextGraph: self.extract_entities = self.config.get("extract_entities", True) self.extract_relationships = self.config.get("extract_relationships", True) - + self.entity_linker = self.config.get("entity_linker") or EntityLinker() # Graph structure self.nodes: Dict[str, ContextNode] = {} self.edges: List[ContextEdge] = [] - + # Adjacency list for efficient traversal: source_id -> list of edges self._adjacency: Dict[str, List[ContextEdge]] = defaultdict(list) # Indexes self.node_type_index: Dict[str, Set[str]] = defaultdict(set) self.edge_type_index: Dict[str, List[ContextEdge]] = defaultdict(list) - + # Progress tracker self.progress_tracker = get_progress_tracker() @@ -110,10 +109,10 @@ class ContextGraph: def add_nodes(self, nodes: List[Dict[str, Any]]) -> int: """ Add nodes to graph. - + Args: nodes: List of nodes to add (dicts with id, type, properties) - + Returns: Number of nodes added """ @@ -123,15 +122,15 @@ class ContextGraph: node_props = node.get("properties", {}) content = node_props.get("content", node.get("id")) metadata = {k: v for k, v in node_props.items() if k != "content"} - + internal_node = ContextNode( node_id=node.get("id"), node_type=node.get("type", "entity"), content=content, metadata=metadata, - properties=node_props + properties=node_props, ) - + if self._add_internal_node(internal_node): count += 1 return count @@ -139,10 +138,11 @@ class ContextGraph: def add_edges(self, edges: List[Dict[str, Any]]) -> int: """ Add edges to graph. - + Args: - edges: List of edges to add (dicts with source_id, target_id, type, weight, properties) - + edges: List of edges to add (dicts with source_id, target_id, type, + weight, properties) + Returns: Number of edges added """ @@ -153,9 +153,9 @@ class ContextGraph: target_id=edge.get("target_id"), edge_type=edge.get("type", "related_to"), weight=edge.get("weight", 1.0), - metadata=edge.get("properties", {}) + metadata=edge.get("properties", {}), ) - + if self._add_internal_edge(internal_edge): count += 1 return count @@ -163,7 +163,7 @@ class ContextGraph: def get_neighbors(self, node_id: str, hops: int = 1) -> List[Dict[str, Any]]: """ Get neighbors of a node. - + Returns list of dicts with neighbor info. """ if node_id not in self.nodes: @@ -175,7 +175,7 @@ class ContextGraph: while queue: current_id, current_hop = queue.popleft() - + if current_hop >= hops: continue @@ -186,52 +186,62 @@ class ContextGraph: if neighbor_id not in visited: visited.add(neighbor_id) queue.append((neighbor_id, current_hop + 1)) - + if neighbor_id in self.nodes: node = self.nodes[neighbor_id] - neighbors.append({ - "id": node.node_id, - "type": node.node_type, - "content": node.content, - "relationship": edge.edge_type, - "weight": edge.weight, - "hop": current_hop + 1 - }) + neighbors.append( + { + "id": node.node_id, + "type": node.node_type, + "content": node.content, + "relationship": edge.edge_type, + "weight": edge.weight, + "hop": current_hop + 1, + } + ) return neighbors def query(self, query: str) -> List[Dict[str, Any]]: """ Execute a simple keyword search query on the graph nodes. - + Args: query: Keyword query string - + Returns: List of matching node dicts """ results = [] query_lower = query.lower().split() - + for node in self.nodes.values(): content_lower = node.content.lower() if any(word in content_lower for word in query_lower): # Calculate simple score overlap = sum(1 for word in query_lower if word in content_lower) score = overlap / len(query_lower) if query_lower else 0.0 - - results.append({ - "node": node.to_dict(), - "score": score, - "content": node.content - }) - + + results.append( + { + "node": node.to_dict(), + "score": score, + "content": node.content, + } + ) + return sorted(results, key=lambda x: x["score"], reverse=True) - def add_node(self, node_id: str, node_type: str, content: Optional[str] = None, **properties) -> bool: + def add_node( + self, + node_id: str, + node_type: str, + content: Optional[str] = None, + **properties, + ) -> bool: """ Add a single node to the graph. - + Args: node_id: Unique identifier node_type: Node type (e.g., 'entity', 'concept') @@ -239,18 +249,27 @@ class ContextGraph: **properties: Additional properties """ content = content or node_id - return self._add_internal_node(ContextNode( - node_id=node_id, - node_type=node_type, - content=content, - metadata=properties, - properties=properties - )) + return self._add_internal_node( + ContextNode( + node_id=node_id, + node_type=node_type, + content=content, + metadata=properties, + properties=properties, + ) + ) - def add_edge(self, source_id: str, target_id: str, edge_type: str = "related_to", weight: float = 1.0, **properties) -> bool: + def add_edge( + self, + source_id: str, + target_id: str, + edge_type: str = "related_to", + weight: float = 1.0, + **properties, + ) -> bool: """ Add a single edge to the graph. - + Args: source_id: Source node ID target_id: Target node ID @@ -258,65 +277,67 @@ class ContextGraph: weight: Edge weight **properties: Additional properties """ - return self._add_internal_edge(ContextEdge( - source_id=source_id, - target_id=target_id, - edge_type=edge_type, - weight=weight, - metadata=properties - )) + return self._add_internal_edge( + ContextEdge( + source_id=source_id, + target_id=target_id, + edge_type=edge_type, + weight=weight, + metadata=properties, + ) + ) def save_to_file(self, path: str) -> None: """ Save context graph to file (JSON format). - + Args: path: File path to save to """ import json - + data = { "nodes": [node.to_dict() for node in self.nodes.values()], - "edges": [edge.to_dict() for edge in self.edges] + "edges": [edge.to_dict() for edge in self.edges], } - - with open(path, 'w', encoding='utf-8') as f: + + with open(path, "w", encoding="utf-8") as f: json.dump(data, f, indent=2, ensure_ascii=False) - + self.logger.info(f"Saved context graph to {path}") def load_from_file(self, path: str) -> None: """ Load context graph from file (JSON format). - + Args: path: File path to load from """ import json import os - + if not os.path.exists(path): self.logger.warning(f"File not found: {path}") return - - with open(path, 'r', encoding='utf-8') as f: + + with open(path, "r", encoding="utf-8") as f: data = json.load(f) - + # Clear existing self.nodes.clear() self.edges.clear() self._adjacency.clear() self.node_type_index.clear() self.edge_type_index.clear() - + # Load nodes nodes = data.get("nodes", []) self.add_nodes(nodes) - + # Load edges edges = data.get("edges", []) self.add_edges(edges) - + self.logger.info(f"Loaded context graph from {path}") def find_node(self, node_id: str) -> Optional[Dict[str, Any]]: @@ -327,11 +348,10 @@ class ContextGraph: "id": node.node_id, "type": node.node_type, "content": node.content, - "metadata": node.metadata + "metadata": node.metadata, } return None - def find_nodes(self, node_type: Optional[str] = None) -> List[Dict[str, Any]]: """Find nodes, optionally filtered by type.""" if node_type: @@ -339,13 +359,13 @@ class ContextGraph: nodes = [self.nodes[nid] for nid in node_ids] else: nodes = self.nodes.values() - + return [ { "id": n.node_id, "type": n.node_type, "content": n.content, - "metadata": n.metadata + "metadata": n.metadata, } for n in nodes ] @@ -356,14 +376,14 @@ class ContextGraph: edges = self.edge_type_index.get(edge_type, []) else: edges = self.edges - + return [ { "source": e.source_id, "target": e.target_id, "type": e.edge_type, "weight": e.weight, - "metadata": e.metadata + "metadata": e.metadata, } for e in edges ] @@ -375,7 +395,7 @@ class ContextGraph: "edge_count": len(self.edges), "node_types": {k: len(v) for k, v in self.node_type_index.items()}, "edge_types": {k: len(v) for k, v in self.edge_type_index.items()}, - "density": self.density() + "density": self.density(), } def density(self) -> float: @@ -398,10 +418,14 @@ class ContextGraph: """Internal method to add an edge.""" # Ensure nodes exist if edge.source_id not in self.nodes: - self._add_internal_node(ContextNode(edge.source_id, "entity", edge.source_id)) + self._add_internal_node( + ContextNode(edge.source_id, "entity", edge.source_id) + ) if edge.target_id not in self.nodes: - self._add_internal_node(ContextNode(edge.target_id, "entity", edge.target_id)) - + self._add_internal_node( + ContextNode(edge.target_id, "entity", edge.target_id) + ) + self.edges.append(edge) self.edge_type_index[edge.edge_type].append(edge) self._adjacency[edge.source_id].append(edge) @@ -419,11 +443,11 @@ class ContextGraph: ) -> Dict[str, Any]: """ Build context graph from conversations and return dict representation. - + Args: conversations: List of conversation files or dictionaries ... - + Returns: Graph dictionary (nodes, edges) """ @@ -436,11 +460,13 @@ class ContextGraph: try: for conv in conversations: - conv_data = conv if isinstance(conv, dict) else self._load_conversation(conv) + conv_data = ( + conv if isinstance(conv, dict) else self._load_conversation(conv) + ) self._process_conversation( - conv_data, + conv_data, extract_intents=extract_intents, - extract_sentiments=extract_sentiments + extract_sentiments=extract_sentiments, ) if link_entities: @@ -450,23 +476,25 @@ class ContextGraph: return self.to_dict() except Exception as e: - self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + self.progress_tracker.stop_tracking( + tracking_id, status="failed", message=str(e) + ) raise def build_from_entities_and_relationships( self, entities: List[Dict[str, Any]], relationships: List[Dict[str, Any]], - **kwargs + **kwargs, ) -> Dict[str, Any]: """ Build graph from entities and relationships. - + Args: entities: List of entity dictionaries relationships: List of relationship dictionaries **kwargs: Additional options - + Returns: Graph dictionary (nodes, edges) """ @@ -474,7 +502,10 @@ class ContextGraph: file=None, module="context", submodule="ContextGraph", - message=f"Building graph from {len(entities)} entities and {len(relationships)} relationships", + message=( + f"Building graph from {len(entities)} entities and " + f"{len(relationships)} relationships" + ), ) try: @@ -482,45 +513,55 @@ class ContextGraph: for entity in entities: entity_id = entity.get("id") or entity.get("entity_id") if entity_id: - self._add_internal_node(ContextNode( - node_id=entity_id, - node_type=entity.get("type", "entity"), - content=entity.get("text") or entity.get("label") or entity_id, - metadata=entity, - properties=entity - )) + self._add_internal_node( + ContextNode( + node_id=entity_id, + node_type=entity.get("type", "entity"), + content=entity.get("text") + or entity.get("label") + or entity_id, + metadata=entity, + properties=entity, + ) + ) # Add relationships for rel in relationships: source = rel.get("source_id") target = rel.get("target_id") if source and target: - self._add_internal_edge(ContextEdge( - source_id=source, - target_id=target, - edge_type=rel.get("type", "related_to"), - weight=rel.get("confidence", 1.0), - metadata=rel - )) - + self._add_internal_edge( + ContextEdge( + source_id=source, + target_id=target, + edge_type=rel.get("type", "related_to"), + weight=rel.get("confidence", 1.0), + metadata=rel, + ) + ) + self.progress_tracker.stop_tracking(tracking_id, status="completed") return self.to_dict() except Exception as e: - self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + self.progress_tracker.stop_tracking( + tracking_id, status="failed", message=str(e) + ) raise def _process_conversation(self, conv_data: Dict[str, Any], **kwargs) -> None: """Process a single conversation.""" conv_id = conv_data.get("id") or f"conv_{hash(str(conv_data)) % 10000}" - + # Add conversation node - self._add_internal_node(ContextNode( - node_id=conv_id, - node_type="conversation", - content=conv_data.get("content", "") or conv_data.get("summary", ""), - metadata={"timestamp": conv_data.get("timestamp")} - )) + self._add_internal_node( + ContextNode( + node_id=conv_id, + node_type="conversation", + content=conv_data.get("content", "") or conv_data.get("summary", ""), + metadata={"timestamp": conv_data.get("timestamp")}, + ) + ) # Track name to ID mapping for relationship resolution name_to_id = {} @@ -529,80 +570,99 @@ class ContextGraph: if self.extract_entities: for entity in conv_data.get("entities", []): entity_id = entity.get("id") or entity.get("entity_id") - entity_text = entity.get("text") or entity.get("label") or entity.get("name") or entity_id + entity_text = ( + entity.get("text") + or entity.get("label") + or entity.get("name") + or entity_id + ) entity_type = entity.get("type", "entity") - + # Generate ID if missing if not entity_id and entity_text and self.entity_linker: # Use EntityLinker to generate ID if hasattr(self.entity_linker, "_generate_entity_id"): - entity_id = self.entity_linker._generate_entity_id(entity_text, entity_type) + entity_id = self.entity_linker._generate_entity_id( + entity_text, entity_type + ) else: # Fallback ID generation import hashlib - entity_hash = hashlib.md5(f"{entity_text}_{entity_type}".encode()).hexdigest()[:12] + + entity_hash = hashlib.md5( + f"{entity_text}_{entity_type}".encode() + ).hexdigest()[:12] entity_id = f"{entity_type.lower()}_{entity_hash}" - + if entity_id: if entity_text: name_to_id[entity_text] = entity_id - - self._add_internal_node(ContextNode( - node_id=entity_id, - node_type="entity", - content=entity_text, - metadata={"type": entity_type, **entity} - )) - self._add_internal_edge(ContextEdge( - source_id=conv_id, - target_id=entity_id, - edge_type="mentions" - )) + + self._add_internal_node( + ContextNode( + node_id=entity_id, + node_type="entity", + content=entity_text, + metadata={"type": entity_type, **entity}, + ) + ) + self._add_internal_edge( + ContextEdge( + source_id=conv_id, + target_id=entity_id, + edge_type="mentions", + ) + ) # Extract relationships if self.extract_relationships: for rel in conv_data.get("relationships", []): source = rel.get("source_id") target = rel.get("target_id") - + # Resolve IDs from names if missing if not source and rel.get("source") and rel.get("source") in name_to_id: source = name_to_id[rel.get("source")] - + if not target and rel.get("target") and rel.get("target") in name_to_id: target = name_to_id[rel.get("target")] - + if source and target: - self._add_internal_edge(ContextEdge( - source_id=source, - target_id=target, - edge_type=rel.get("type", "related_to"), - weight=rel.get("confidence", 1.0) - )) + self._add_internal_edge( + ContextEdge( + source_id=source, + target_id=target, + edge_type=rel.get("type", "related_to"), + weight=rel.get("confidence", 1.0), + ) + ) def _link_entities(self) -> None: """Link similar entities using EntityLinker.""" if not self.entity_linker: return - + entity_nodes = [n for n in self.nodes.values() if n.node_type == "entity"] for i, node1 in enumerate(entity_nodes): - for node2 in entity_nodes[i+1:]: + for node2 in entity_nodes[i + 1 :]: similarity = self.entity_linker._calculate_text_similarity( node1.content.lower(), node2.content.lower() ) if similarity >= self.entity_linker.similarity_threshold: - self._add_internal_edge(ContextEdge( - source_id=node1.node_id, - target_id=node2.node_id, - edge_type="similar_to", - weight=similarity - )) + self._add_internal_edge( + ContextEdge( + source_id=node1.node_id, + target_id=node2.node_id, + edge_type="similar_to", + weight=similarity, + ) + ) def _load_conversation(self, file_path: str) -> Dict[str, Any]: """Load conversation from file.""" from ..utils.helpers import read_json_file from pathlib import Path + return read_json_file(Path(file_path)) def to_dict(self) -> Dict[str, Any]: @@ -613,7 +673,7 @@ class ContextGraph: "id": n.node_id, "type": n.node_type, "content": n.content, - "metadata": n.metadata + "metadata": n.metadata, } for n in self.nodes.values() ], @@ -622,15 +682,16 @@ class ContextGraph: "source": e.source_id, "target": e.target_id, "type": e.edge_type, - "weight": e.weight + "weight": e.weight, } for e in self.edges ], "statistics": { "node_count": len(self.nodes), - "edge_count": len(self.edges) - } + "edge_count": len(self.edges), + }, } + # For backward compatibility ContextGraphBuilder = ContextGraph diff --git a/semantica/context/context_retriever.py b/semantica/context/context_retriever.py index fcd63949..314d4867 100644 --- a/semantica/context/context_retriever.py +++ b/semantica/context/context_retriever.py @@ -43,12 +43,15 @@ Key Features: - Configurable retrieval strategies Main Classes: - - RetrievedContext: Retrieved context item data structure with content, score, source, metadata, related_entities, related_relationships + - RetrievedContext: Retrieved context item data structure with content, score, + source, metadata, related_entities, related_relationships - ContextRetriever: Context retriever for hybrid retrieval Example Usage: >>> from semantica.context import ContextRetriever - >>> retriever = ContextRetriever(memory_store=mem, knowledge_graph=kg, vector_store=vs) + >>> retriever = ContextRetriever( + ... memory_store=mem, knowledge_graph=kg, vector_store=vs + ... ) >>> results = retriever.retrieve("Python programming", max_results=5) >>> for result in results: ... print(f"{result.content}: {result.score:.2f}") @@ -62,7 +65,6 @@ from datetime import datetime from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Union -from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -103,7 +105,8 @@ class ContextRetriever: - vector_store: Vector store instance - use_graph_expansion: Use graph expansion (default: True) - max_expansion_hops: Maximum graph expansion hops (default: 2) - - hybrid_alpha: Weight for hybrid retrieval (0=vector only, 1=graph only, default: 0.5) + - hybrid_alpha: Weight for hybrid retrieval (0=vector only, 1=graph + only, default: 0.5) """ self.logger = get_logger("context_retriever") self.config = config or {} @@ -271,23 +274,27 @@ class ContextRetriever: # Check if knowledge_graph implements GraphStore protocol (has query method) if hasattr(self.knowledge_graph, "query"): graph_results = self.knowledge_graph.query(query) - + for res in graph_results: # Handle both interface dicts and raw dicts node = res.get("node") - if hasattr(node, "id"): # GraphNodeInterface + if hasattr(node, "id"): # GraphNodeInterface node_id = node.id node_type = node.type content = node.properties.get("content", "") metadata = node.properties - else: # Raw dict + else: # Raw dict node_id = res.get("id") or res.get("node", {}).get("id") node_type = res.get("type") or res.get("node", {}).get("type") - content = res.get("content") or res.get("node", {}).get("content") - metadata = res.get("metadata") or res.get("node", {}).get("metadata") + content = res.get("content") or res.get("node", {}).get( + "content" + ) + metadata = res.get("metadata") or res.get("node", {}).get( + "metadata" + ) score = res.get("score", 0.0) - + # Get related entities related_entities = self._get_related_entities( node_id, max_hops=max_hops @@ -306,7 +313,7 @@ class ContextRetriever: related_entities=related_entities, ) ) - + # Sort by score results.sort(key=lambda x: x.score, reverse=True) return results[:max_results] @@ -504,14 +511,14 @@ class ContextRetriever: def search(self, query: str, **options) -> List[RetrievedContext]: """ Simple search (alias for retrieve). - + Args: query: Search query **options: Additional options - + Returns: List of RetrievedContext objects - + Example: >>> results = retriever.search("Python", max_results=10) """ @@ -520,84 +527,84 @@ class ContextRetriever: def vector_search(self, query: str, **options) -> List[RetrievedContext]: """ Vector-only search. - + Args: query: Search query **options: Additional options - + Returns: List of RetrievedContext objects from vector store - + Example: >>> results = retriever.vector_search("Python") """ # Temporarily disable graph and memory original_graph = self.knowledge_graph original_memory = self.memory_store - + self.knowledge_graph = None self.memory_store = None - + try: results = self.retrieve(query, use_graph_expansion=False, **options) finally: self.knowledge_graph = original_graph self.memory_store = original_memory - + return results def graph_search(self, query: str, **options) -> List[RetrievedContext]: """ Graph-only search. - + Args: query: Search query **options: Additional options - + Returns: List of RetrievedContext objects from graph - + Example: >>> results = retriever.graph_search("Python") """ if not self.knowledge_graph: return [] - + # Temporarily disable vector and memory original_vector = self.vector_store original_memory = self.memory_store - + self.vector_store = None self.memory_store = None - + try: results = self.retrieve(query, use_graph_expansion=True, **options) finally: self.vector_store = original_vector self.memory_store = original_memory - + return results def memory_search(self, query: str, **options) -> List[RetrievedContext]: """ Memory-only search. - + Args: query: Search query **options: Additional options - + Returns: List of RetrievedContext objects from memory - + Example: >>> results = retriever.memory_search("Python") """ if not self.memory_store: return [] - + # Use memory store's retrieve method memory_results = self.memory_store.retrieve(query, **options) - + # Convert to RetrievedContext results = [] for mem in memory_results: @@ -609,72 +616,78 @@ class ContextRetriever: metadata=mem.get("metadata", {}), ) ) - + return results def hybrid_search(self, query: str, **options) -> List[RetrievedContext]: """ Hybrid search (all sources). - + Args: query: Search query **options: Additional options - + Returns: List of RetrievedContext objects from all sources - + Example: >>> results = retriever.hybrid_search("Python") """ return self.retrieve(query, **options) # Advanced Retrieval - def find_similar(self, content: str, limit: int = 5, **options) -> List[RetrievedContext]: + def find_similar( + self, content: str, limit: int = 5, **options + ) -> List[RetrievedContext]: """ Find similar content. - + Args: content: Content to find similar items for limit: Maximum results (default: 5) **options: Additional options - + Returns: List of similar RetrievedContext objects - + Example: >>> similar = retriever.find_similar("Python programming", limit=5) """ return self.retrieve(content, max_results=limit, **options) - def get_context(self, query: str, max_results: int = 5, **options) -> List[RetrievedContext]: + def get_context( + self, query: str, max_results: int = 5, **options + ) -> List[RetrievedContext]: """ Get context for query. - + Args: query: Query string max_results: Maximum results (default: 5) **options: Additional options - + Returns: List of RetrievedContext objects - + Example: >>> context_data = retriever.get_context("Python", max_results=10) """ return self.retrieve(query, max_results=max_results, **options) - def expand_query(self, query: str, max_hops: int = 2, **options) -> List[RetrievedContext]: + def expand_query( + self, query: str, max_hops: int = 2, **options + ) -> List[RetrievedContext]: """ Expand query with graph. - + Args: query: Query string max_hops: Maximum expansion hops (default: 2) **options: Additional options - + Returns: List of expanded RetrievedContext objects - + Example: >>> expanded = retriever.expand_query("Python", max_hops=3) """ @@ -682,58 +695,60 @@ class ContextRetriever: query, use_graph_expansion=True, max_hops=max_hops, - **options + **options, ) def get_related(self, entity_id: str, max_hops: int = 2) -> List[Dict[str, Any]]: """ Get related entities. - + Args: entity_id: Entity ID max_hops: Maximum hops (default: 2) - + Returns: List of related entity dicts - + Example: >>> related = retriever.get_related("entity_123", max_hops=2) """ if not self.knowledge_graph: return [] - + return self._get_related_entities(entity_id, max_hops=max_hops) - def get_path(self, source_id: str, target_id: str, max_hops: int = 5) -> List[Dict[str, Any]]: + def get_path( + self, source_id: str, target_id: str, max_hops: int = 5 + ) -> List[Dict[str, Any]]: """ Get path between entities. - + Args: source_id: Source entity ID target_id: Target entity ID max_hops: Maximum hops (default: 5) - + Returns: List of path nodes/edges - + Example: >>> path = retriever.get_path("entity_1", "entity_2", max_hops=5) """ if not self.knowledge_graph: return [] - + # Simple BFS path finding from collections import deque - + queue = deque([(source_id, [source_id])]) visited = {source_id} - + while queue: current_id, path = queue.popleft() - + if len(path) > max_hops: continue - + if current_id == target_id: # Return path with node info nodes = self.knowledge_graph.get("nodes", []) @@ -741,14 +756,16 @@ class ContextRetriever: for node_id in path: for node in nodes: if node.get("id") == node_id: - path_info.append({ - "id": node_id, - "content": node.get("content", ""), - "type": node.get("type", ""), - }) + path_info.append( + { + "id": node_id, + "content": node.get("content", ""), + "type": node.get("type", ""), + } + ) break return path_info - + # Get neighbors edges = self.knowledge_graph.get("edges", []) for edge in edges: @@ -757,26 +774,28 @@ class ContextRetriever: neighbor_id = edge.get("target") elif edge.get("target") == current_id: neighbor_id = edge.get("source") - + if neighbor_id and neighbor_id not in visited: visited.add(neighbor_id) queue.append((neighbor_id, path + [neighbor_id])) - + return [] # Filter Methods - def filter_by_entity(self, entity_id: str, query: str, **options) -> List[RetrievedContext]: + def filter_by_entity( + self, entity_id: str, query: str, **options + ) -> List[RetrievedContext]: """ Filter by entity. - + Args: entity_id: Entity ID to filter by query: Search query **options: Additional options - + Returns: Filtered RetrievedContext objects - + Example: >>> results = retriever.filter_by_entity("entity_123", "Python") """ @@ -790,18 +809,20 @@ class ContextRetriever: break return filtered - def filter_by_type(self, type: str, query: str, **options) -> List[RetrievedContext]: + def filter_by_type( + self, type: str, query: str, **options + ) -> List[RetrievedContext]: """ Filter by type. - + Args: type: Node/entity type to filter by query: Search query **options: Additional options - + Returns: Filtered RetrievedContext objects - + Example: >>> results = retriever.filter_by_type("PROGRAMMING_LANGUAGE", "Python") """ @@ -821,16 +842,16 @@ class ContextRetriever: ) -> List[RetrievedContext]: """ Filter by date. - + Args: start_date: Start date end_date: End date query: Search query **options: Additional options - + Returns: Filtered RetrievedContext objects - + Example: >>> results = retriever.filter_by_date("2024-01-01", "2024-12-31", "Python") """ @@ -840,7 +861,7 @@ class ContextRetriever: if isinstance(end_date, str): from dateutil.parser import parse end_date = parse(end_date) - + results = self.retrieve(query, **options) filtered = [] for result in results: @@ -861,15 +882,15 @@ class ContextRetriever: ) -> List[RetrievedContext]: """ Filter by score. - + Args: min_score: Minimum score threshold query: Search query **options: Additional options - + Returns: Filtered RetrievedContext objects - + Example: >>> results = retriever.filter_by_score(0.7, "Python") """ @@ -877,17 +898,19 @@ class ContextRetriever: return [r for r in results if r.score >= min_score] # Batch Operations - def batch_search(self, queries: List[str], **options) -> Dict[str, List[RetrievedContext]]: + def batch_search( + self, queries: List[str], **options + ) -> Dict[str, List[RetrievedContext]]: """ Search multiple queries. - + Args: queries: List of queries **options: Additional options - + Returns: Dict mapping query to results - + Example: >>> results = retriever.batch_search(["Python", "Java", "C++"]) """ @@ -904,19 +927,21 @@ class ContextRetriever: ) -> Dict[str, List[RetrievedContext]]: """ Get context for multiple queries. - + Args: queries: List of queries max_results: Maximum results per query (default: 5) **options: Additional options - + Returns: Dict mapping query to context results - + Example: - >>> contexts = retriever.batch_get_context(["Python", "Java"], max_results=5) + >>> contexts = retriever.batch_get_context( + ... ["Python", "Java"], max_results=5 + ... ) """ results = {} for query in queries: results[query] = self.get_context(query, max_results=max_results, **options) - return results \ No newline at end of file + return results diff --git a/semantica/context/entity_linker.py b/semantica/context/entity_linker.py index 369b8fa6..6cfb25d3 100644 --- a/semantica/context/entity_linker.py +++ b/semantica/context/entity_linker.py @@ -40,15 +40,19 @@ Key Features: - Configurable similarity thresholds Main Classes: - - EntityLink: Entity link data structure with source_entity_id, target_entity_id, link_type, confidence, source, metadata - - LinkedEntity: Linked entity with context including entity_id, uri, text, type, linked_entities, context, confidence + - EntityLink: Entity link data structure with source_entity_id, target_entity_id, + link_type, confidence, source, metadata + - LinkedEntity: Linked entity with context including entity_id, uri, text, type, + linked_entities, context, confidence - EntityLinker: Entity linker for context engineering Example Usage: >>> from semantica.context import EntityLinker >>> linker = EntityLinker(knowledge_graph=kg) >>> uri = linker.assign_uri("entity_1", "Python", "PROGRAMMING_LANGUAGE") - >>> linked_entities = linker.link("Python is a programming language", entities=entities) + >>> linked_entities = linker.link( + ... "Python is a programming language", entities=entities + ... ) >>> linker.link_entities("entity_1", "entity_2", "related_to", confidence=0.9) >>> web = linker.build_entity_web() @@ -58,10 +62,9 @@ License: MIT import hashlib from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from urllib.parse import quote -from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker from ..utils.types import EntityDict @@ -114,7 +117,8 @@ class EntityLinker: - knowledge_graph: Knowledge graph for entity lookup - similarity_threshold: Similarity threshold for linking (default: 0.8) - base_uri: Base URI for entity URIs - - enable_cross_document_linking: Enable cross-document linking (default: True) + - enable_cross_document_linking: Enable cross-document linking + (default: True) """ self.logger = get_logger("entity_linker") self.config = config or {} @@ -197,7 +201,7 @@ class EntityLinker: file=None, module="context", submodule="EntityLinker", - message=f"Linking entities in text", + message="Linking entities in text", ) try: @@ -524,17 +528,19 @@ class EntityLinker: return web # Linking Methods - def link_text(self, text: str, entities: Optional[List[EntityDict]] = None) -> List[LinkedEntity]: + def link_text( + self, text: str, entities: Optional[List[EntityDict]] = None + ) -> List[LinkedEntity]: """ Link entities in text. - + Args: text: Text containing entities entities: List of entities to link (optional) - + Returns: List of LinkedEntity objects - + Example: >>> linked = linker.link_text("Python is used for ML", entities=[...]) """ @@ -543,13 +549,13 @@ class EntityLinker: def link_batch(self, entities: List[EntityDict]) -> List[LinkedEntity]: """ Link multiple entities. - + Args: entities: List of entities to link - + Returns: List of LinkedEntity objects - + Example: >>> linked = linker.link_batch(entities) """ @@ -558,16 +564,16 @@ class EntityLinker: uri = self.assign_uri( entity.get("id", ""), entity.get("text"), - entity.get("type") + entity.get("type"), ) - + # Find similar entities similar = self.find_similar_entities( entity.get("text", ""), entity.get("type", ""), - threshold=self.similarity_threshold + threshold=self.similarity_threshold, ) - + linked_entities = [] for similar_id, similarity in similar: linked_entities.append( @@ -575,10 +581,10 @@ class EntityLinker: source_entity_id=entity.get("id", ""), target_entity_id=similar_id, link_type="similar_to", - confidence=similarity + confidence=similarity, ) ) - + linked_results.append( LinkedEntity( entity_id=entity.get("id", ""), @@ -586,24 +592,26 @@ class EntityLinker: text=entity.get("text", ""), type=entity.get("type", ""), linked_entities=linked_entities, - confidence=1.0 + confidence=1.0, ) ) - + return linked_results # Search Methods - def find_similar(self, entity: Union[str, EntityDict], threshold: float = 0.8) -> List[Tuple[str, float]]: + def find_similar( + self, entity: Union[str, EntityDict], threshold: float = 0.8 + ) -> List[Tuple[str, float]]: """ Find similar entities. - + Args: entity: Entity text or EntityDict threshold: Similarity threshold (default: 0.8) - + Returns: List of (entity_id, similarity) tuples - + Example: >>> similar = linker.find_similar("Python", threshold=0.8) """ @@ -613,20 +621,20 @@ class EntityLinker: else: text = entity entity_type = "" - + return self.find_similar_entities(text, entity_type, threshold=threshold) def find_by_text(self, text: str, threshold: float = 0.8) -> List[Dict[str, Any]]: """ Find by text. - + Args: text: Text to search for threshold: Similarity threshold (default: 0.8) - + Returns: List of matching entity dicts - + Example: >>> entities = linker.find_by_text("Python", threshold=0.8) """ @@ -637,29 +645,32 @@ class EntityLinker: nodes = self.knowledge_graph.get("nodes", []) for node in nodes: if node.get("id") == entity_id: - similarity = self._calculate_similarity(text, node.get("content", "")) + node_content = node.get("content", "") + similarity = self._calculate_similarity(text, node_content) if similarity >= threshold: - results.append({ - "entity_id": entity_id, - "uri": uri, - "text": node.get("content", ""), - "type": node.get("type", ""), - "similarity": similarity, - }) + results.append( + { + "entity_id": entity_id, + "uri": uri, + "text": node_content, + "type": node.get("type", ""), + "similarity": similarity, + } + ) break return sorted(results, key=lambda x: x.get("similarity", 0), reverse=True) def find_by_type(self, type: str, limit: int = 10) -> List[Dict[str, Any]]: """ Find by type. - + Args: type: Entity type limit: Maximum results (default: 10) - + Returns: List of entity dicts of specified type - + Example: >>> entities = linker.find_by_type("PROGRAMMING_LANGUAGE", limit=10) """ @@ -670,12 +681,14 @@ class EntityLinker: if node.get("type") == type: entity_id = node.get("id", "") uri = self.entity_registry.get(entity_id, "") - results.append({ - "entity_id": entity_id, - "uri": uri, - "text": node.get("content", ""), - "type": type, - }) + results.append( + { + "entity_id": entity_id, + "uri": uri, + "text": node.get("content", ""), + "type": type, + } + ) if len(results) >= limit: break return results @@ -683,21 +696,21 @@ class EntityLinker: def find_related(self, entity_id: str, max_hops: int = 2) -> List[Dict[str, Any]]: """ Find related entities. - + Args: entity_id: Entity ID max_hops: Maximum hops (default: 2) - + Returns: List of related entity dicts - + Example: >>> related = linker.find_related("entity_123", max_hops=2) """ related = [] visited = {entity_id} current_level = {entity_id} - + for hop in range(max_hops): next_level = set() for current_id in current_level: @@ -707,39 +720,41 @@ class EntityLinker: if target_id not in visited: visited.add(target_id) next_level.add(target_id) - + # Get entity info uri = self.entity_registry.get(target_id, "") if self.knowledge_graph: nodes = self.knowledge_graph.get("nodes", []) for node in nodes: if node.get("id") == target_id: - related.append({ - "entity_id": target_id, - "uri": uri, - "text": node.get("content", ""), - "type": node.get("type", ""), - "link_type": link.link_type, - "confidence": link.confidence, - "hop": hop + 1, - }) + related.append( + { + "entity_id": target_id, + "uri": uri, + "text": node.get("content", ""), + "type": node.get("type", ""), + "link_type": link.link_type, + "confidence": link.confidence, + "hop": hop + 1, + } + ) break - + current_level = next_level - + return related # URI Methods def get_uri(self, entity_id: str) -> Optional[str]: """ Get URI for entity. - + Args: entity_id: Entity ID - + Returns: URI or None if not found - + Example: >>> uri = linker.get_uri("entity_123") """ @@ -748,10 +763,10 @@ class EntityLinker: def get_all_uris(self) -> Dict[str, str]: """ Get all URIs. - + Returns: Dict mapping entity_id to URI - + Example: >>> uris = linker.get_all_uris() """ @@ -760,13 +775,13 @@ class EntityLinker: def resolve_uri(self, uri: str) -> Optional[Dict[str, Any]]: """ Resolve URI to entity. - + Args: uri: URI to resolve - + Returns: Entity dict or None if not found - + Example: >>> entity = linker.resolve_uri("https://semantica.dev/entity/python") """ @@ -794,13 +809,13 @@ class EntityLinker: def link_count(self, entity_id: str) -> int: """ Get link count for entity. - + Args: entity_id: Entity ID - + Returns: Number of links for entity - + Example: >>> count = linker.link_count("entity_123") """ @@ -809,10 +824,10 @@ class EntityLinker: def stats(self) -> Dict[str, Any]: """ Get statistics. - + Returns: Statistics dict - + Example: >>> stats = linker.stats() """ @@ -829,28 +844,30 @@ class EntityLinker: def most_linked(self, limit: int = 10) -> List[Dict[str, Any]]: """ Get most linked entities. - + Args: limit: Maximum results (default: 10) - + Returns: List of entity dicts sorted by link count - + Example: >>> top = linker.most_linked(limit=10) """ entity_link_counts = [] for entity_id, links in self.entity_links.items(): uri = self.entity_registry.get(entity_id, "") - entity_link_counts.append({ - "entity_id": entity_id, - "uri": uri, - "link_count": len(links), - }) - + entity_link_counts.append( + { + "entity_id": entity_id, + "uri": uri, + "link_count": len(links), + } + ) + # Sort by link count entity_link_counts.sort(key=lambda x: x["link_count"], reverse=True) - + # Add entity text if available if self.knowledge_graph: nodes = self.knowledge_graph.get("nodes", []) @@ -860,5 +877,5 @@ class EntityLinker: if node: item["text"] = node.get("content", "") item["type"] = node.get("type", "") - - return entity_link_counts[:limit] \ No newline at end of file + + return entity_link_counts[:limit] diff --git a/semantica/context/methods.py b/semantica/context/methods.py index 6f864c42..00ce57dc 100644 --- a/semantica/context/methods.py +++ b/semantica/context/methods.py @@ -2,9 +2,9 @@ Context Methods Module This module provides all context engineering methods as simple, reusable functions for -context graph construction, agent memory management, context retrieval, and entity linking. -It supports multiple context engineering approaches and integrates with the method registry -for extensibility. +context graph construction, agent memory management, context retrieval, and entity +linking. It supports multiple context engineering approaches and integrates with the +method registry for extensibility. Supported Methods: @@ -84,7 +84,9 @@ Main Functions: Example Usage: >>> from semantica.context.methods import build_context_graph, retrieve_context - >>> graph = build_context_graph(entities, relationships, method="entities_relationships") + >>> graph = build_context_graph( + ... entities, relationships, method="entities_relationships" + ... ) >>> results = retrieve_context("Python programming", method="hybrid", max_results=5) >>> from semantica.context.methods import get_context_method >>> method = get_context_method("graph", "custom_method") @@ -93,15 +95,14 @@ Author: Semantica Contributors License: MIT """ -from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Union -from ..utils.exceptions import ConfigurationError, ProcessingError +from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger -from .agent_memory import AgentMemory, MemoryItem -from .context_graph import ContextEdge, ContextGraph, ContextNode +from .agent_memory import AgentMemory +from .context_graph import ContextGraph from .context_retriever import ContextRetriever, RetrievedContext -from .entity_linker import EntityLink, EntityLinker, LinkedEntity +from .entity_linker import EntityLinker, LinkedEntity from .registry import method_registry logger = get_logger("context_methods") @@ -117,7 +118,8 @@ def build_context_graph( """ Build context graph from various sources (convenience function). - This is a user-friendly wrapper that builds context graphs using the specified method. + This is a user-friendly wrapper that builds context graphs using the specified + method. Args: entities: List of entity dictionaries @@ -138,8 +140,12 @@ def build_context_graph( Examples: >>> from semantica.context.methods import build_context_graph >>> entities = [{"id": "e1", "text": "Python", "type": "PROGRAMMING_LANGUAGE"}] - >>> relationships = [{"source_id": "e1", "target_id": "e2", "type": "related_to"}] - >>> graph = build_context_graph(entities, relationships, method="entities_relationships") + >>> relationships = [ + ... {"source_id": "e1", "target_id": "e2", "type": "related_to"} + ... ] + >>> graph = build_context_graph( + ... entities, relationships, method="entities_relationships" + ... ) >>> print(f"Graph has {graph['statistics']['node_count']} nodes") """ # Check for custom method in registry @@ -159,7 +165,8 @@ def build_context_graph( if method == "entities_relationships": if not entities or not relationships: raise ProcessingError( - "entities and relationships required for entities_relationships method" + "entities and relationships required for entities_relationships " + "method" ) return builder.build_from_entities_and_relationships( entities, relationships, **kwargs @@ -221,7 +228,9 @@ def store_memory( Examples: >>> from semantica.context.methods import store_memory - >>> memory_id = store_memory("User asked about Python", vector_store=vs, method="store") + >>> memory_id = store_memory( + ... "User asked about Python", vector_store=vs, method="store" + ... ) >>> print(f"Stored memory: {memory_id}") """ # Check for custom method in registry @@ -292,7 +301,9 @@ def retrieve_context( Examples: >>> from semantica.context.methods import retrieve_context - >>> results = retrieve_context("Python programming", vector_store=vs, method="hybrid") + >>> results = retrieve_context( + ... "Python programming", vector_store=vs, method="hybrid" + ... ) >>> for result in results: ... print(f"{result.content}: {result.score:.2f}") """ diff --git a/semantica/context/registry.py b/semantica/context/registry.py index 004062bf..9f3c28aa 100644 --- a/semantica/context/registry.py +++ b/semantica/context/registry.py @@ -1,8 +1,8 @@ """ Method Registry Module for Context Engineering -This module provides a method registry system for registering custom context engineering methods, -enabling extensibility and community contributions to the context toolkit. +This module provides a method registry system for registering custom context engineering +methods, enabling extensibility and community contributions to the context toolkit. Supported Registration Types: - Method Registry: Register custom context methods for: @@ -40,7 +40,7 @@ Author: Semantica Contributors License: MIT """ -from typing import Any, Callable, Dict, List, Optional +from typing import Callable, Dict, List, Optional class MethodRegistry: