From f3dd7a05bdf22f68fff0ddc98b2396f479b173ce Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Fri, 12 Dec 2025 20:19:17 +0530 Subject: [PATCH] Refactor: Remove Pinecone and enhance vector store backend support - Removed all Pinecone references, adapters, and documentation to align with open-source, self-hosted focus. - Removed PineconeAdapter and related dependencies. - Updated VectorStore to enforce supported backends (FAISS, Weaviate, Qdrant, Milvus, InMemory). - Updated cookbooks (e.g., 13_Vector_Store.ipynb) to use Weaviate/FAISS examples instead of Pinecone. - Updated core documentation (modules.md, rchitecture.md, etc.) to reflect backend changes. - Added new tests ( est_pinecone_removal.py, est_vector_store_deepdive.py) to verify removal and validate remaining backends. - Verified all vector store tests pass. --- CHANGELOG.md | 2 +- PR_DESCRIPTION.md | 61 +- .../advanced/05_Multi_Format_Export.ipynb | 2 +- .../11_Advanced_Context_Engineering.ipynb | 2 +- cookbook/introduction/13_Vector_Store.ipynb | 1147 ++++++++--------- .../04_Healthcare_GraphRAG_Hybrid.ipynb | 2 +- .../05_Medical_Database_Integration.ipynb | 1 - docs/CodeExamples.md | 4 +- docs/architecture.md | 2 +- docs/community-projects.md | 1 - docs/modules.md | 3 +- docs/reference/context.md | 2 +- docs/reference/embeddings.md | 6 +- docs/reference/vector_store.md | 54 +- pyproject.toml | 1 - semantica/embeddings/embeddings_usage.md | 12 +- .../embeddings/vector_embedding_manager.py | 27 +- semantica/export/__init__.py | 2 +- semantica/export/methods.py | 2 +- semantica/export/vector_exporter.py | 35 +- semantica/pipeline/pipeline_templates.py | 2 +- semantica/pipeline/pipeline_usage.md | 5 +- semantica/utils/constants.py | 1 - semantica/vector_store/__init__.py | 49 +- semantica/vector_store/config.py | 2 - semantica/vector_store/pinecone_adapter.py | 510 -------- semantica/vector_store/vector_store.py | 8 + semantica/vector_store/vector_store_usage.md | 52 +- tests/vector_store/test_pinecone_removal.py | 62 + .../test_vector_store_deepdive.py | 372 ++++++ 30 files changed, 1113 insertions(+), 1318 deletions(-) delete mode 100644 semantica/vector_store/pinecone_adapter.py create mode 100644 tests/vector_store/test_pinecone_removal.py create mode 100644 tests/vector_store/test_vector_store_deepdive.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c5cc51a3..d1868930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Production-ready quality assurance modules - Comprehensive documentation with MkDocs - Cookbook with interactive tutorials -- Support for multiple vector stores (Pinecone, Weaviate, Qdrant, FAISS) +- Support for multiple vector stores (Weaviate, Qdrant, FAISS) - Support for multiple graph databases (Neo4j, NetworkX, RDFLib) - Temporal knowledge graph support - Conflict detection and resolution diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md index 45b55867..ba68c6fc 100644 --- a/PR_DESCRIPTION.md +++ b/PR_DESCRIPTION.md @@ -1,39 +1,32 @@ -# Fix & Align Split Module with Documentation +# Refactor: Rename `triple_store` to `triplet_store` -## ๐Ÿ“ Summary -This PR aligns the `semantica.split` module with its documentation, ensuring that all documented chunking strategies are fully implemented, registered, and accessible via the unified `TextSplitter` interface. It specifically enables direct usage of "structural" and "sliding_window" methods and fixes kwargs handling in hierarchical chunking. +## Summary +This Pull Request renames the `semantica/triple_store` module to `semantica/triplet_store` and updates all references across the codebase to ensure consistent naming conventions. -## ๐Ÿš€ Motivation -Previously, the `docs/reference/split.md` documentation listed "structural" and "sliding_window" as available methods for `TextSplitter`, but they were not registered in the `_SPLIT_METHODS` dictionary in `methods.py`. This caused `TextSplitter(method="structural")` to fail or fallback unexpectedly. Additionally, there were minor discrepancies in method signatures and argument handling (specifically `chunk_size` collisions) that needed resolution. +## Motivation +The term "triplet" is the standard terminology used within the Semantica framework. This refactor aligns the module name, class names, and documentation with this convention, eliminating ambiguity and "triple"/"triplet" inconsistencies. -## ๐Ÿ” Key Changes +## Changes +- **Module Rename**: Renamed directory `semantica/triple_store` -> `semantica/triplet_store`. +- **Core Updates**: + - Updated `__init__.py`, `triplet_manager.py`, `query_engine.py`, `bulk_loader.py`, and all adapters (`blazegraph`, `jena`, `rdf4j`, `virtuoso`) to use `triplet_store` imports. + - Renamed classes: `TripleManager` -> `TripletManager`, `TripleStore` -> `TripletStore`. +- **Notebook Refactoring**: + - Updated imports and usage in `cookbook/introduction/20_Triplet_Store.ipynb`. + - Updated `cookbook/advanced/09_Semantic_Layer_Construction.ipynb`. + - Updated healthcare use cases: `01_Clinical_Reports_Processing.ipynb`, `04_Healthcare_GraphRAG_Hybrid.ipynb`, `05_Medical_Database_Integration.ipynb`, `06_Patient_Records_Temporal.ipynb`. +- **Documentation**: + - Renamed `docs/reference/triple_store.md` -> `triplet_store.md`. + - Updated `README.md`, `docs/modules.md`, `docs/glossary.md`, `docs/CodeExamples.md`, `docs/reference/graph_store.md`, `docs/reference/reasoning.md`. + - Updated `mkdocs.yml` navigation. +- **Tests**: + - Renamed `tests/triple_store` -> `tests/triplet_store`. + - Updated `test_triplet_store.py` to test the renamed module. -### 1. Method Registration & Implementation -- **New Wrappers**: Added `split_structural` and `split_sliding_window` wrapper functions in `semantica/split/methods.py`. -- **Registry Update**: Registered these methods in `_SPLIT_METHODS`, enabling: - ```python - # Now works out-of-the-box - splitter = TextSplitter(method="structural") - splitter = TextSplitter(method="sliding_window") - ``` -- **Conditional Imports**: Added robust import handling for specialized chunkers to ensure the module remains usable even if optional dependencies are missing. +## Verification +- **Tests**: Ran `pytest tests/triplet_store/test_triplet_store.py`. All tests passed. +- **Static Analysis**: Verified no lingering `semantica.triple_store` imports remain in the codebase (grep check). -### 2. Documentation Alignment (`docs/reference/split.md`) -- **Signature Updates**: Updated method signatures in the documentation to exactly match the code implementation (e.g., `StructuralChunker`, `TableChunker`). -- **TableChunker**: Clarified `TableChunker` usage, documenting its specialized methods (`chunk_table`, `chunk_to_text_chunks`) since it handles structured data differently from standard text splitters. -- **NER Aliases**: Confirmed and documented support for `ner_method="ml"` (mapping to spaCy) in Entity/Relation aware chunking. - -### 3. Bug Fixes -- **Hierarchical Splitting**: Fixed a `TypeError: multiple values for keyword argument 'chunk_size'` bug in `split_hierarchical` by properly managing `kwargs` when delegating to sub-splitters (paragraph/sentence). -- **Import Handling**: Resolved potential circular imports and improved error messages for missing dependencies. - -## ๐Ÿงช Verification -- [x] **Registry Check**: Verified that `list_available_methods()` now returns `structural` and `sliding_window`. -- [x] **Runtime Verification**: Confirmed `TextSplitter` successfully delegates to the new wrappers. -- [x] **Documentation**: Verified that documentation tables match the actual code capabilities. -- [x] **Kwargs Handling**: Verified hierarchical splitting no longer throws duplicate argument errors. - -## โœ… Checklist -- [x] Code follows the project's coding standards. -- [x] Documentation has been updated to reflect the changes. -- [x] All chunking strategies listed in docs are now functionally accessible. +## Breaking Changes +- `semantica.triple_store` is no longer available. Users must update imports to `semantica.triplet_store`. +- `TripleManager` and `TripleStore` classes are renamed to `TripletManager` and `TripletStore`. diff --git a/cookbook/advanced/05_Multi_Format_Export.ipynb b/cookbook/advanced/05_Multi_Format_Export.ipynb index 5f24841d..0de32a99 100644 --- a/cookbook/advanced/05_Multi_Format_Export.ipynb +++ b/cookbook/advanced/05_Multi_Format_Export.ipynb @@ -312,7 +312,7 @@ "- NumPy format\n", "- Binary format\n", "- FAISS format\n", - "- Vector store integration (Pinecone, Weaviate, Qdrant)\n" + "- Vector store integration (Weaviate, Qdrant)\n" ] }, { diff --git a/cookbook/advanced/11_Advanced_Context_Engineering.ipynb b/cookbook/advanced/11_Advanced_Context_Engineering.ipynb index b8100959..c8abb8c7 100644 --- a/cookbook/advanced/11_Advanced_Context_Engineering.ipynb +++ b/cookbook/advanced/11_Advanced_Context_Engineering.ipynb @@ -217,7 +217,7 @@ "## 5. Best Practices for Production\n", "\n", "1. **Token Limits**: Align `token_limit` with your LLM's context window minus the prompt template size.\n", - "2. **Vector Store**: Use a production-grade vector store (e.g., Pinecone, Weaviate, Qdrant) instead of the mock store.\n", + "2. **Vector Store**: Use a production-grade vector store (e.g., Weaviate, Qdrant) instead of the mock store.\n", "3. **Asynchronous Operations**: For high-throughput systems, consider wrapping storage operations in async tasks (though the core logic is synchronous for simplicity).\n", "4. **Entity Resolution**: Implement a robust `EntityLinker` strategy to prevent graph fragmentation (e.g., \"Alice\" vs \"Alice S.\")." ] diff --git a/cookbook/introduction/13_Vector_Store.ipynb b/cookbook/introduction/13_Vector_Store.ipynb index a7a8932a..2f7e0861 100644 --- a/cookbook/introduction/13_Vector_Store.ipynb +++ b/cookbook/introduction/13_Vector_Store.ipynb @@ -1,575 +1,574 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb)\n", - "\n", - "# Vector Store - Comprehensive Guide\n", - "\n", - "## Overview\n", - "\n", - "This notebook provides a **comprehensive walkthrough** of Semantica's vector_store module, demonstrating vector storage, similarity search, hybrid search, and multi-backend support for semantic retrieval.\n", - "\n", - "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/vector_store/)\n", - "\n", - "### Learning Objectives\n", - "\n", - "By the end of this notebook, you will be able to:\n", - "\n", - "- Store and manage vectors with metadata\n", - "- Perform similarity search with different metrics\n", - "- Use hybrid search combining vectors and metadata\n", - "- Work with multiple vector store backends (FAISS, Pinecone, etc.)\n", - "- Create and manage vector indices\n", - "- Filter and rank search results\n", - "- Implement namespace isolation for multi-tenancy\n", - "\n", - "### What You'll Learn\n", - "\n", - "| Component | Purpose | When to Use |\n", - "|-----------|---------|-------------|\n", - "| `VectorStore` | Main vector storage | All vector operations |\n", - "| `VectorIndexer` | Index creation | Performance optimization |\n", - "| `VectorRetriever` | Similarity search | Finding similar vectors |\n", - "| `HybridSearch` | Combined search | Vector + metadata filtering |\n", - "| `MetadataFilter` | Metadata filtering | Filtering by attributes |\n", - "| `MetadataStore` | Metadata management | Storing vector metadata |\n", - "| `NamespaceManager` | Multi-tenancy | Isolating vector collections |\n", - "\n", - "---\n", - "\n", - "## Installation\n", - "\n", - "Install Semantica from PyPI:\n", - "\n", - "```bash\n", - "pip install semantica\n", - "# Or with all optional dependencies:\n", - "pip install semantica[all]\n", - "```\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Basic Vector Storage\n", - "\n", - "Let's start with the `VectorStore` for basic vector storage and retrieval.\n", - "\n", - "### What is VectorStore?\n", - "\n", - "`VectorStore` is the main interface for vector operations:\n", - "- **Storage**: Store vectors with metadata\n", - "- **Search**: Find similar vectors\n", - "- **CRUD**: Create, Read, Update, Delete operations\n", - "- **Multi-backend**: Support for FAISS, Pinecone, Weaviate, Qdrant, Milvus" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import VectorStore\n", - "from semantica.embeddings import TextEmbedder\n", - "import numpy as np\n", - "\n", - "# 1. Initialize Embedder (Select Provider & Model)\n", - "# You can choose 'sentence_transformers' or 'fastembed'\n", - "embedder = TextEmbedder(method=\"sentence_transformers\", model_name=\"all-MiniLM-L6-v2\")\n", - "dimension = embedder.get_embedding_dimension()\n", - "\n", - "# 2. Create vector store\n", - "store = VectorStore(backend=\"faiss\", dimension=dimension)\n", - "\n", - "# 3. Generate Real Embeddings\n", - "texts = [f\"Document {i}\" for i in range(100)]\n", - "vectors = embedder.embed_batch(texts)\n", - "\n", - "metadata = [\n", - " {\"text\": txt, \"category\": \"science\" if i % 2 == 0 else \"technology\", \"year\": 2020 + (i % 4)}\n", - " for i, txt in enumerate(texts)\n", - "]\n", - "\n", - "# 4. Store vectors\n", - "vector_ids = store.store_vectors(vectors, metadata=metadata)\n", - "\n", - "print(f\"Stored {len(vector_ids)} vectors\")\n", - "print(f\"First 3 IDs: {vector_ids[:3]}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Similarity Search\n", - "\n", - "Search for similar vectors using different similarity metrics.\n", - "\n", - "### Similarity Metrics\n", - "\n", - "- **Cosine Similarity**: Best for semantic similarity\n", - "- **L2 Distance**: Euclidean distance\n", - "- **Dot Product**: Fast, requires normalized vectors" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Create query vector\n", - "query_vector = np.random.rand(768)\n", - "\n", - "# Search for similar vectors\n", - "results = store.search_vectors(query_vector, k=10)\n", - "\n", - "print(f\"Found {len(results)} similar vectors\")\n", - "print(\"\\nTop 5 results:\")\n", - "for i, result in enumerate(results[:5], 1):\n", - " print(f\"{i}. ID: {result['id']}, Score: {result['score']:.3f}\")\n", - " print(f\" Metadata: {result.get('metadata', {})}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Vector Indexing\n", - "\n", - "Create indices for faster search on large datasets.\n", - "\n", - "### Index Types (FAISS)\n", - "\n", - "- **Flat**: Exact search (brute force)\n", - "- **IVF**: Inverted file index (approximate)\n", - "- **HNSW**: Hierarchical graph (best balance)\n", - "- **PQ**: Product quantization (compressed)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import VectorIndexer, FAISSAdapter\n", - "\n", - "# Create indexer\n", - "indexer = VectorIndexer(backend=\"faiss\", dimension=768)\n", - "\n", - "# Create HNSW index for fast approximate search\n", - "adapter = FAISSAdapter(dimension=768)\n", - "index = adapter.create_index(index_type=\"hnsw\", metric=\"L2\", m=16)\n", - "\n", - "# Add vectors to index\n", - "vectors_array = np.array(vectors).astype('float32')\n", - "adapter.add_vectors(index, vectors_array, ids=vector_ids)\n", - "\n", - "# Search using index\n", - "query_array = query_vector.astype('float32')\n", - "distances, indices = adapter.search(index, query_array, k=10)\n", - "\n", - "print(f\"Index search found {len(indices)} results\")\n", - "print(f\"Distances: {distances[:5]}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Hybrid Search\n", - "\n", - "Combine vector similarity with metadata filtering.\n", - "\n", - "### Hybrid Search Benefits\n", - "\n", - "- Filter by metadata before vector search\n", - "- Combine multiple search criteria\n", - "- More precise results" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import HybridSearch, MetadataFilter\n", - "\n", - "# Create hybrid search\n", - "hybrid_search = HybridSearch()\n", - "\n", - "# Create metadata filter\n", - "filter = MetadataFilter() \\\n", - " .eq(\"category\", \"science\") \\\n", - " .gt(\"year\", 2021)\n", - "\n", - "# Perform hybrid search\n", - "hybrid_results = hybrid_search.search(\n", - " query_vector,\n", - " vectors,\n", - " metadata,\n", - " vector_ids,\n", - " filter=filter,\n", - " k=10\n", - ")\n", - "\n", - "print(f\"Hybrid search found {len(hybrid_results)} results\")\n", - "print(\"\\nFiltered results (science, year > 2021):\")\n", - "for i, result in enumerate(hybrid_results[:5], 1):\n", - " meta = result.get('metadata', {})\n", - " print(f\"{i}. Category: {meta.get('category')}, Year: {meta.get('year')}, Score: {result['score']:.3f}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Metadata Management\n", - "\n", - "Store and query metadata separately from vectors.\n", - "\n", - "### Metadata Operations\n", - "\n", - "- Store metadata for vectors\n", - "- Query by metadata conditions\n", - "- Update metadata\n", - "- Schema validation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import MetadataStore, MetadataSchema\n", - "\n", - "# Create metadata store\n", - "meta_store = MetadataStore()\n", - "\n", - "# Store metadata\n", - "for i, vec_id in enumerate(vector_ids[:10]):\n", - " meta_store.store_metadata(vec_id, metadata[i])\n", - "\n", - "# Query metadata\n", - "matching_ids = meta_store.query_metadata(\n", - " {\"category\": \"science\"},\n", - " operator=\"AND\"\n", - ")\n", - "\n", - "print(f\"Found {len(matching_ids)} vectors with category='science'\")\n", - "\n", - "# Define schema for validation\n", - "schema = MetadataSchema({\n", - " \"text\": {\"type\": str, \"required\": True},\n", - " \"category\": {\"type\": str, \"required\": True},\n", - " \"year\": {\"type\": int, \"required\": True}\n", - "})\n", - "\n", - "# Validate metadata\n", - "is_valid = schema.validate(metadata[0])\n", - "print(f\"\\nMetadata validation: {is_valid}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 6: Result Ranking and Fusion\n", - "\n", - "Combine and rank results from multiple searches.\n", - "\n", - "### Ranking Strategies\n", - "\n", - "- **Reciprocal Rank Fusion (RRF)**: Combine ranked lists\n", - "- **Weighted Average**: Weight scores from different sources" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import SearchRanker\n", - "\n", - "# Create ranker with RRF strategy\n", - "ranker = SearchRanker(strategy=\"reciprocal_rank_fusion\")\n", - "\n", - "# Simulate multiple search results\n", - "results1 = [\n", - " {\"id\": \"vec_1\", \"score\": 0.9},\n", - " {\"id\": \"vec_2\", \"score\": 0.8},\n", - " {\"id\": \"vec_3\", \"score\": 0.7}\n", - "]\n", - "\n", - "results2 = [\n", - " {\"id\": \"vec_2\", \"score\": 0.85},\n", - " {\"id\": \"vec_4\", \"score\": 0.75},\n", - " {\"id\": \"vec_1\", \"score\": 0.7}\n", - "]\n", - "\n", - "# Fuse results using RRF\n", - "fused_results = ranker.rank([results1, results2], k=60)\n", - "\n", - "print(\"Fused results using RRF:\")\n", - "for i, result in enumerate(fused_results, 1):\n", - " print(f\"{i}. ID: {result['id']}, Fused Score: {result['score']:.3f}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 7: Namespace Management\n", - "\n", - "Isolate vectors for multi-tenant applications.\n", - "\n", - "### Namespace Features\n", - "\n", - "- Tenant isolation\n", - "- Access control\n", - "- Per-namespace operations" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import NamespaceManager\n", - "\n", - "# Create namespace manager\n", - "ns_manager = NamespaceManager()\n", - "\n", - "# Create namespaces for different tenants\n", - "ns1 = ns_manager.create_namespace(\"tenant1\", \"Tenant 1 vectors\")\n", - "ns2 = ns_manager.create_namespace(\"tenant2\", \"Tenant 2 vectors\")\n", - "\n", - "# Add vectors to namespaces\n", - "for i in range(5):\n", - " ns_manager.add_vector_to_namespace(f\"t1_vec_{i}\", \"tenant1\")\n", - " ns_manager.add_vector_to_namespace(f\"t2_vec_{i}\", \"tenant2\")\n", - "\n", - "# Get namespace vectors\n", - "tenant1_vectors = ns_manager.get_namespace_vectors(\"tenant1\")\n", - "tenant2_vectors = ns_manager.get_namespace_vectors(\"tenant2\")\n", - "\n", - "print(f\"Tenant 1: {len(tenant1_vectors)} vectors\")\n", - "print(f\"Tenant 2: {len(tenant2_vectors)} vectors\")\n", - "\n", - "# Set access control\n", - "ns1.set_access_control(\"user1\", [\"read\", \"write\"])\n", - "ns1.set_access_control(\"user2\", [\"read\"])\n", - "\n", - "print(f\"\\nUser1 can write: {ns1.has_permission('user1', 'write')}\")\n", - "print(f\"User2 can write: {ns1.has_permission('user2', 'write')}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 8: Convenience Functions\n", - "\n", - "Use convenience functions for quick operations.\n", - "\n", - "### Available Functions\n", - "\n", - "- `store_vectors()`: Store vectors\n", - "- `search_vectors()`: Search vectors\n", - "- `hybrid_search()`: Hybrid search\n", - "- `update_vectors()`: Update vectors\n", - "- `delete_vectors()`: Delete vectors" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import (\n", - " store_vectors,\n", - " search_vectors,\n", - " hybrid_search as hybrid_search_func,\n", - " update_vectors,\n", - " delete_vectors\n", - ")\n", - "\n", - "# Store vectors using convenience function\n", - "new_vectors = [np.random.rand(768) for _ in range(10)]\n", - "new_metadata = [{\"text\": f\"New doc {i}\"} for i in range(10)]\n", - "new_ids = store_vectors(new_vectors, metadata=new_metadata, method=\"default\")\n", - "\n", - "print(f\"Stored {len(new_ids)} new vectors\")\n", - "\n", - "# Search using convenience function\n", - "search_results = search_vectors(\n", - " query_vector,\n", - " new_vectors,\n", - " new_ids,\n", - " k=5,\n", - " method=\"default\"\n", - ")\n", - "\n", - "print(f\"Search found {len(search_results)} results\")\n", - "\n", - "# Update vectors\n", - "updated_vectors = [np.random.rand(768) for _ in range(2)]\n", - "success = update_vectors(new_ids[:2], updated_vectors, method=\"default\")\n", - "print(f\"\\nUpdated vectors: {success}\")\n", - "\n", - "# Delete vectors\n", - "success = delete_vectors(new_ids[-2:], method=\"default\")\n", - "print(f\"Deleted vectors: {success}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 9: Multi-Backend Support\n", - "\n", - "Work with different vector store backends.\n", - "\n", - "### Supported Backends\n", - "\n", - "| Backend | Type | Best For |\n", - "|---------|------|----------|\n", - "| FAISS | Local | Development, small datasets |\n", - "| Pinecone | Cloud | Production, managed service |\n", - "| Weaviate | Self-hosted | Schema-aware storage |\n", - "| Qdrant | Self-hosted | High performance |\n", - "| Milvus | Cloud/Self-hosted | Large scale |" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import FAISSAdapter, VectorManager\n", - "\n", - "# FAISS (local)\n", - "faiss_adapter = FAISSAdapter(dimension=768)\n", - "faiss_index = faiss_adapter.create_index(index_type=\"flat\", metric=\"L2\")\n", - "print(\"Created FAISS index\")\n", - "\n", - "# Vector Manager for multi-store management\n", - "manager = VectorManager()\n", - "faiss_store = manager.create_store(\"faiss\", {\"dimension\": 768})\n", - "print(f\"\\nCreated store via manager\")\n", - "\n", - "# List all stores\n", - "stores = manager.list_stores()\n", - "print(f\"Active stores: {stores}\")\n", - "\n", - "# Note: For cloud backends (Pinecone, Weaviate, etc.),\n", - "# you would need API keys and endpoints\n", - "# Example:\n", - "# from semantica.vector_store import PineconeAdapter\n", - "# pinecone = PineconeAdapter(api_key=\"your-key\", environment=\"us-west1-gcp\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 10: Best Practices\n", - "\n", - "### Performance Tips\n", - "\n", - "1. **Normalize Vectors**: Always normalize for cosine similarity\n", - "2. **Use HNSW**: Best balance for speed/accuracy\n", - "3. **Batch Operations**: Process in batches (100-1000)\n", - "4. **Filter First**: Apply metadata filters before vector search\n", - "\n", - "### Backend Selection\n", - "\n", - "- **Development**: FAISS (local, fast)\n", - "- **Production**: Pinecone (managed, scalable)\n", - "- **Self-hosted**: Qdrant or Milvus (control, performance)\n", - "- **Schema-aware**: Weaviate (rich metadata)\n", - "\n", - "### Index Configuration\n", - "\n", - "- **Small datasets (<10K)**: Flat index\n", - "- **Medium datasets (10K-1M)**: HNSW\n", - "- **Large datasets (>1M)**: IVF + PQ" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "### What You've Learned\n", - "\n", - "In this notebook, you've learned how to:\n", - "\n", - "- Store and search vectors with VectorStore\n", - "- Create indices for performance optimization\n", - "- Use hybrid search with metadata filtering\n", - "- Manage metadata separately from vectors\n", - "- Rank and fuse search results\n", - "- Implement namespace isolation\n", - "- Use convenience functions for quick operations\n", - "- Work with multiple backend adapters\n", - "- Apply best practices for production use\n", - "\n", - "### Key Takeaways\n", - "\n", - "1. **Multi-Backend**: Choose the right backend for your needs\n", - "2. **Hybrid Search**: Combine vectors with metadata for precision\n", - "3. **Indexing**: Use appropriate index types for performance\n", - "4. **Metadata**: Separate metadata management for flexibility\n", - "5. **Namespaces**: Isolate vectors for multi-tenancy\n", - "\n", - "### Next Steps\n", - "\n", - "**Further Reading**:\n", - "- [Vector Store API Reference](https://semantica.readthedocs.io/reference/vector_store/)\n", - "- [Advanced Vector Store Notebook](../advanced/Advanced_Vector_Store_and_Search.ipynb)\n", - "- [Embedding Generation](12_Embedding_Generation.ipynb)\n", - "\n", - "---\n", - "\n", - "**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb)\n", + "\n", + "# Vector Store - Comprehensive Guide\n", + "\n", + "## Overview\n", + "\n", + "This notebook provides a **comprehensive walkthrough** of Semantica's vector_store module, demonstrating vector storage, similarity search, hybrid search, and multi-backend support for semantic retrieval.\n", + "\n", + "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/vector_store/)\n", + "\n", + "### Learning Objectives\n", + "\n", + "By the end of this notebook, you will be able to:\n", + "\n", + "- Store and manage vectors with metadata\n", + "- Perform similarity search with different metrics\n", + "- Use hybrid search combining vectors and metadata\n", + "- Work with multiple vector store backends (FAISS, Weaviate, etc.)\n", + "- Create and manage vector indices\n", + "- Filter and rank search results\n", + "- Implement namespace isolation for multi-tenancy\n", + "\n", + "### What You'll Learn\n", + "\n", + "| Component | Purpose | When to Use |\n", + "|-----------|---------|-------------|\n", + "| `VectorStore` | Main vector storage | All vector operations |\n", + "| `VectorIndexer` | Index creation | Performance optimization |\n", + "| `VectorRetriever` | Similarity search | Finding similar vectors |\n", + "| `HybridSearch` | Combined search | Vector + metadata filtering |\n", + "| `MetadataFilter` | Metadata filtering | Filtering by attributes |\n", + "| `MetadataStore` | Metadata management | Storing vector metadata |\n", + "| `NamespaceManager` | Multi-tenancy | Isolating vector collections |\n", + "\n", + "---\n", + "\n", + "## Installation\n", + "\n", + "Install Semantica from PyPI:\n", + "\n", + "```bash\n", + "pip install semantica\n", + "# Or with all optional dependencies:\n", + "pip install semantica[all]\n", + "```\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Basic Vector Storage\n", + "\n", + "Let's start with the `VectorStore` for basic vector storage and retrieval.\n", + "\n", + "### What is VectorStore?\n", + "\n", + "`VectorStore` is the main interface for vector operations:\n", + "- **Storage**: Store vectors with metadata\n", + "- **Search**: Find similar vectors\n", + "- **CRUD**: Create, Read, Update, Delete operations\n", + "- **Multi-backend**: Support for FAISS, Weaviate, Qdrant, Milvus" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import VectorStore\n", + "from semantica.embeddings import TextEmbedder\n", + "import numpy as np\n", + "\n", + "# 1. Initialize Embedder (Select Provider & Model)\n", + "# You can choose 'sentence_transformers' or 'fastembed'\n", + "embedder = TextEmbedder(method=\"sentence_transformers\", model_name=\"all-MiniLM-L6-v2\")\n", + "dimension = embedder.get_embedding_dimension()\n", + "\n", + "# 2. Create vector store\n", + "store = VectorStore(backend=\"faiss\", dimension=dimension)\n", + "\n", + "# 3. Generate Real Embeddings\n", + "texts = [f\"Document {i}\" for i in range(100)]\n", + "vectors = embedder.embed_batch(texts)\n", + "\n", + "metadata = [\n", + " {\"text\": txt, \"category\": \"science\" if i % 2 == 0 else \"technology\", \"year\": 2020 + (i % 4)}\n", + " for i, txt in enumerate(texts)\n", + "]\n", + "\n", + "# 4. Store vectors\n", + "vector_ids = store.store_vectors(vectors, metadata=metadata)\n", + "\n", + "print(f\"Stored {len(vector_ids)} vectors\")\n", + "print(f\"First 3 IDs: {vector_ids[:3]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Similarity Search\n", + "\n", + "Search for similar vectors using different similarity metrics.\n", + "\n", + "### Similarity Metrics\n", + "\n", + "- **Cosine Similarity**: Best for semantic similarity\n", + "- **L2 Distance**: Euclidean distance\n", + "- **Dot Product**: Fast, requires normalized vectors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create query vector\n", + "query_vector = np.random.rand(768)\n", + "\n", + "# Search for similar vectors\n", + "results = store.search_vectors(query_vector, k=10)\n", + "\n", + "print(f\"Found {len(results)} similar vectors\")\n", + "print(\"\\nTop 5 results:\")\n", + "for i, result in enumerate(results[:5], 1):\n", + " print(f\"{i}. ID: {result['id']}, Score: {result['score']:.3f}\")\n", + " print(f\" Metadata: {result.get('metadata', {})}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Vector Indexing\n", + "\n", + "Create indices for faster search on large datasets.\n", + "\n", + "### Index Types (FAISS)\n", + "\n", + "- **Flat**: Exact search (brute force)\n", + "- **IVF**: Inverted file index (approximate)\n", + "- **HNSW**: Hierarchical graph (best balance)\n", + "- **PQ**: Product quantization (compressed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import VectorIndexer, FAISSAdapter\n", + "\n", + "# Create indexer\n", + "indexer = VectorIndexer(backend=\"faiss\", dimension=768)\n", + "\n", + "# Create HNSW index for fast approximate search\n", + "adapter = FAISSAdapter(dimension=768)\n", + "index = adapter.create_index(index_type=\"hnsw\", metric=\"L2\", m=16)\n", + "\n", + "# Add vectors to index\n", + "vectors_array = np.array(vectors).astype('float32')\n", + "adapter.add_vectors(index, vectors_array, ids=vector_ids)\n", + "\n", + "# Search using index\n", + "query_array = query_vector.astype('float32')\n", + "distances, indices = adapter.search(index, query_array, k=10)\n", + "\n", + "print(f\"Index search found {len(indices)} results\")\n", + "print(f\"Distances: {distances[:5]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Hybrid Search\n", + "\n", + "Combine vector similarity with metadata filtering.\n", + "\n", + "### Hybrid Search Benefits\n", + "\n", + "- Filter by metadata before vector search\n", + "- Combine multiple search criteria\n", + "- More precise results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import HybridSearch, MetadataFilter\n", + "\n", + "# Create hybrid search\n", + "hybrid_search = HybridSearch()\n", + "\n", + "# Create metadata filter\n", + "filter = MetadataFilter() \\\n", + " .eq(\"category\", \"science\") \\\n", + " .gt(\"year\", 2021)\n", + "\n", + "# Perform hybrid search\n", + "hybrid_results = hybrid_search.search(\n", + " query_vector,\n", + " vectors,\n", + " metadata,\n", + " vector_ids,\n", + " filter=filter,\n", + " k=10\n", + ")\n", + "\n", + "print(f\"Hybrid search found {len(hybrid_results)} results\")\n", + "print(\"\\nFiltered results (science, year > 2021):\")\n", + "for i, result in enumerate(hybrid_results[:5], 1):\n", + " meta = result.get('metadata', {})\n", + " print(f\"{i}. Category: {meta.get('category')}, Year: {meta.get('year')}, Score: {result['score']:.3f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Metadata Management\n", + "\n", + "Store and query metadata separately from vectors.\n", + "\n", + "### Metadata Operations\n", + "\n", + "- Store metadata for vectors\n", + "- Query by metadata conditions\n", + "- Update metadata\n", + "- Schema validation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import MetadataStore, MetadataSchema\n", + "\n", + "# Create metadata store\n", + "meta_store = MetadataStore()\n", + "\n", + "# Store metadata\n", + "for i, vec_id in enumerate(vector_ids[:10]):\n", + " meta_store.store_metadata(vec_id, metadata[i])\n", + "\n", + "# Query metadata\n", + "matching_ids = meta_store.query_metadata(\n", + " {\"category\": \"science\"},\n", + " operator=\"AND\"\n", + ")\n", + "\n", + "print(f\"Found {len(matching_ids)} vectors with category='science'\")\n", + "\n", + "# Define schema for validation\n", + "schema = MetadataSchema({\n", + " \"text\": {\"type\": str, \"required\": True},\n", + " \"category\": {\"type\": str, \"required\": True},\n", + " \"year\": {\"type\": int, \"required\": True}\n", + "})\n", + "\n", + "# Validate metadata\n", + "is_valid = schema.validate(metadata[0])\n", + "print(f\"\\nMetadata validation: {is_valid}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Result Ranking and Fusion\n", + "\n", + "Combine and rank results from multiple searches.\n", + "\n", + "### Ranking Strategies\n", + "\n", + "- **Reciprocal Rank Fusion (RRF)**: Combine ranked lists\n", + "- **Weighted Average**: Weight scores from different sources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import SearchRanker\n", + "\n", + "# Create ranker with RRF strategy\n", + "ranker = SearchRanker(strategy=\"reciprocal_rank_fusion\")\n", + "\n", + "# Simulate multiple search results\n", + "results1 = [\n", + " {\"id\": \"vec_1\", \"score\": 0.9},\n", + " {\"id\": \"vec_2\", \"score\": 0.8},\n", + " {\"id\": \"vec_3\", \"score\": 0.7}\n", + "]\n", + "\n", + "results2 = [\n", + " {\"id\": \"vec_2\", \"score\": 0.85},\n", + " {\"id\": \"vec_4\", \"score\": 0.75},\n", + " {\"id\": \"vec_1\", \"score\": 0.7}\n", + "]\n", + "\n", + "# Fuse results using RRF\n", + "fused_results = ranker.rank([results1, results2], k=60)\n", + "\n", + "print(\"Fused results using RRF:\")\n", + "for i, result in enumerate(fused_results, 1):\n", + " print(f\"{i}. ID: {result['id']}, Fused Score: {result['score']:.3f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Namespace Management\n", + "\n", + "Isolate vectors for multi-tenant applications.\n", + "\n", + "### Namespace Features\n", + "\n", + "- Tenant isolation\n", + "- Access control\n", + "- Per-namespace operations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import NamespaceManager\n", + "\n", + "# Create namespace manager\n", + "ns_manager = NamespaceManager()\n", + "\n", + "# Create namespaces for different tenants\n", + "ns1 = ns_manager.create_namespace(\"tenant1\", \"Tenant 1 vectors\")\n", + "ns2 = ns_manager.create_namespace(\"tenant2\", \"Tenant 2 vectors\")\n", + "\n", + "# Add vectors to namespaces\n", + "for i in range(5):\n", + " ns_manager.add_vector_to_namespace(f\"t1_vec_{i}\", \"tenant1\")\n", + " ns_manager.add_vector_to_namespace(f\"t2_vec_{i}\", \"tenant2\")\n", + "\n", + "# Get namespace vectors\n", + "tenant1_vectors = ns_manager.get_namespace_vectors(\"tenant1\")\n", + "tenant2_vectors = ns_manager.get_namespace_vectors(\"tenant2\")\n", + "\n", + "print(f\"Tenant 1: {len(tenant1_vectors)} vectors\")\n", + "print(f\"Tenant 2: {len(tenant2_vectors)} vectors\")\n", + "\n", + "# Set access control\n", + "ns1.set_access_control(\"user1\", [\"read\", \"write\"])\n", + "ns1.set_access_control(\"user2\", [\"read\"])\n", + "\n", + "print(f\"\\nUser1 can write: {ns1.has_permission('user1', 'write')}\")\n", + "print(f\"User2 can write: {ns1.has_permission('user2', 'write')}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 8: Convenience Functions\n", + "\n", + "Use convenience functions for quick operations.\n", + "\n", + "### Available Functions\n", + "\n", + "- `store_vectors()`: Store vectors\n", + "- `search_vectors()`: Search vectors\n", + "- `hybrid_search()`: Hybrid search\n", + "- `update_vectors()`: Update vectors\n", + "- `delete_vectors()`: Delete vectors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import (\n", + " store_vectors,\n", + " search_vectors,\n", + " hybrid_search as hybrid_search_func,\n", + " update_vectors,\n", + " delete_vectors\n", + ")\n", + "\n", + "# Store vectors using convenience function\n", + "new_vectors = [np.random.rand(768) for _ in range(10)]\n", + "new_metadata = [{\"text\": f\"New doc {i}\"} for i in range(10)]\n", + "new_ids = store_vectors(new_vectors, metadata=new_metadata, method=\"default\")\n", + "\n", + "print(f\"Stored {len(new_ids)} new vectors\")\n", + "\n", + "# Search using convenience function\n", + "search_results = search_vectors(\n", + " query_vector,\n", + " new_vectors,\n", + " new_ids,\n", + " k=5,\n", + " method=\"default\"\n", + ")\n", + "\n", + "print(f\"Search found {len(search_results)} results\")\n", + "\n", + "# Update vectors\n", + "updated_vectors = [np.random.rand(768) for _ in range(2)]\n", + "success = update_vectors(new_ids[:2], updated_vectors, method=\"default\")\n", + "print(f\"\\nUpdated vectors: {success}\")\n", + "\n", + "# Delete vectors\n", + "success = delete_vectors(new_ids[-2:], method=\"default\")\n", + "print(f\"Deleted vectors: {success}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 9: Multi-Backend Support\n", + "\n", + "Work with different vector store backends.\n", + "\n", + "### Supported Backends\n", + "\n", + "| Backend | Type | Best For |\n", + "|---------|------|----------|\n", + "| FAISS | Local | Development, small datasets |\n", + "| Weaviate | Self-hosted | Schema-aware storage |\n", + "| Qdrant | Self-hosted | High performance |\n", + "| Milvus | Cloud/Self-hosted | Large scale |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.vector_store import FAISSAdapter, VectorManager\n", + "\n", + "# FAISS (local)\n", + "faiss_adapter = FAISSAdapter(dimension=768)\n", + "faiss_index = faiss_adapter.create_index(index_type=\"flat\", metric=\"L2\")\n", + "print(\"Created FAISS index\")\n", + "\n", + "# Vector Manager for multi-store management\n", + "manager = VectorManager()\n", + "faiss_store = manager.create_store(\"faiss\", {\"dimension\": 768})\n", + "print(f\"\\nCreated store via manager\")\n", + "\n", + "# List all stores\n", + "stores = manager.list_stores()\n", + "print(f\"Active stores: {stores}\")\n", + "\n", + "# Note: For cloud/remote backends (Weaviate, Qdrant, etc.),\n", + "# you would need API keys and endpoints\n", + "# Example:\n", + "# from semantica.vector_store import WeaviateAdapter\n", + "# weaviate = WeaviateAdapter(url=\"http://localhost:8080\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 10: Best Practices\n", + "\n", + "### Performance Tips\n", + "\n", + "1. **Normalize Vectors**: Always normalize for cosine similarity\n", + "2. **Use HNSW**: Best balance for speed/accuracy\n", + "3. **Batch Operations**: Process in batches (100-1000)\n", + "4. **Filter First**: Apply metadata filters before vector search\n", + "\n", + "### Backend Selection\n", + "\n", + "- **Development**: FAISS (local, fast)\n", + "- **Production**: Weaviate/Qdrant (scalable, self-hosted)\n", + "- **Self-hosted**: Qdrant or Milvus (control, performance)\n", + "- **Schema-aware**: Weaviate (rich metadata)\n", + "\n", + "### Index Configuration\n", + "\n", + "- **Small datasets (<10K)**: Flat index\n", + "- **Medium datasets (10K-1M)**: HNSW\n", + "- **Large datasets (>1M)**: IVF + PQ" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### What You've Learned\n", + "\n", + "In this notebook, you've learned how to:\n", + "\n", + "- Store and search vectors with VectorStore\n", + "- Create indices for performance optimization\n", + "- Use hybrid search with metadata filtering\n", + "- Manage metadata separately from vectors\n", + "- Rank and fuse search results\n", + "- Implement namespace isolation\n", + "- Use convenience functions for quick operations\n", + "- Work with multiple backend adapters\n", + "- Apply best practices for production use\n", + "\n", + "### Key Takeaways\n", + "\n", + "1. **Multi-Backend**: Choose the right backend for your needs\n", + "2. **Hybrid Search**: Combine vectors with metadata for precision\n", + "3. **Indexing**: Use appropriate index types for performance\n", + "4. **Metadata**: Separate metadata management for flexibility\n", + "5. **Namespaces**: Isolate vectors for multi-tenancy\n", + "\n", + "### Next Steps\n", + "\n", + "**Further Reading**:\n", + "- [Vector Store API Reference](https://semantica.readthedocs.io/reference/vector_store/)\n", + "- [Advanced Vector Store Notebook](../advanced/Advanced_Vector_Store_and_Search.ipynb)\n", + "- [Embedding Generation](12_Embedding_Generation.ipynb)\n", + "\n", + "---\n", + "\n", + "**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/cookbook/use_cases/healthcare/04_Healthcare_GraphRAG_Hybrid.ipynb b/cookbook/use_cases/healthcare/04_Healthcare_GraphRAG_Hybrid.ipynb index 59ed4289..4a790949 100644 --- a/cookbook/use_cases/healthcare/04_Healthcare_GraphRAG_Hybrid.ipynb +++ b/cookbook/use_cases/healthcare/04_Healthcare_GraphRAG_Hybrid.ipynb @@ -774,7 +774,7 @@ "\n", "### Semantica-Specific Performance Considerations\n", "\n", - "- **Vector Store**: Use Semantica's VectorStore with appropriate backend (FAISS for local, Pinecone/Weaviate for cloud)\n", + "- **Vector Store**: Use Semantica's VectorStore with appropriate backend (FAISS for local, Weaviate for cloud)\n", "- **Graph Analytics**: Leverage Semantica's GraphAnalyzer for efficient centrality and community detection\n", "- **Pipeline Execution**: Use Semantica's ExecutionEngine for parallel execution of pipeline steps\n", "- **Caching**: Utilize Semantica's ContextRetriever caching for frequently accessed contexts\n", diff --git a/cookbook/use_cases/healthcare/05_Medical_Database_Integration.ipynb b/cookbook/use_cases/healthcare/05_Medical_Database_Integration.ipynb index b42492c8..ef3f9548 100644 --- a/cookbook/use_cases/healthcare/05_Medical_Database_Integration.ipynb +++ b/cookbook/use_cases/healthcare/05_Medical_Database_Integration.ipynb @@ -69,7 +69,6 @@ "from semantica.kg import GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n", "from semantica.triplet_store import TripletStore, TripletManager, QueryEngine\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, OntologyVisualizer, TemporalVisualizer\n", "import json\n", diff --git a/docs/CodeExamples.md b/docs/CodeExamples.md index 87766e0b..1ffda8f2 100644 --- a/docs/CodeExamples.md +++ b/docs/CodeExamples.md @@ -32,7 +32,7 @@ from semantica import Semantica core = Semantica( llm_provider="openai", embedding_model="text-embedding-3-large", - vector_store="pinecone", + vector_store="weaviate", graph_db="neo4j" ) @@ -357,7 +357,7 @@ semantic_chunks = embedder.semantic_chunk(documents) embeddings = embedder.generate_embeddings(semantic_chunks) # Store in vector database -vector_store = core.get_vector_store("pinecone") +vector_store = core.get_vector_store("weaviate") vector_store.store_embeddings(semantic_chunks, embeddings) # Semantic search diff --git a/docs/architecture.md b/docs/architecture.md index 96b32136..24fc2941 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -66,7 +66,7 @@ graph TB ### Knowledge Graphs - **`semantica.kg`** - Knowledge graph construction -- **`semantica.vector_store`** - Vector storage (Pinecone, Weaviate, FAISS) +- **`semantica.vector_store`** - Vector storage (Weaviate, FAISS) - **`semantica.triplet_store`** - RDF triplet storage (Jena, Blazegraph) - **`semantica.graph_store`** - Property graphs (Neo4j, FalkorDB) diff --git a/docs/community-projects.md b/docs/community-projects.md index 4a806efa..ba5cd6ed 100644 --- a/docs/community-projects.md +++ b/docs/community-projects.md @@ -25,7 +25,6 @@ Projects and integrations from the Semantica community. ## ๐Ÿ”Œ Integrations ### Vector Databases -- Pinecone - Weaviate - Qdrant - FAISS diff --git a/docs/modules.md b/docs/modules.md index 7069ac33..b121f66b 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -468,7 +468,7 @@ print(f"Similarity: {similarity:.3f}") **Key Features:** -- Multiple backend support (FAISS, Pinecone, Weaviate, Qdrant, Milvus) +- Multiple backend support (FAISS, Weaviate, Qdrant, Milvus) - Hybrid search (vector + keyword) - Metadata filtering - Batch operations @@ -480,7 +480,6 @@ print(f"Similarity: {similarity:.3f}") - `VectorStore` โ€” Main vector store interface - `FAISSAdapter` โ€” FAISS integration -- `PineconeAdapter` โ€” Pinecone integration - `WeaviateAdapter` โ€” Weaviate integration - `HybridSearch` โ€” Combine vector and keyword search - `VectorRetriever` โ€” Retrieve relevant vectors diff --git a/docs/reference/context.md b/docs/reference/context.md index 1396e0a2..e4dcd8c5 100644 --- a/docs/reference/context.md +++ b/docs/reference/context.md @@ -57,7 +57,7 @@ The **Context Module** provides agents with a persistent, searchable, and struct The high-level facade that unifies all context operations. It routes data to the appropriate subsystems (Memory, Graph, Vector Store) and manages the lifecycle of context. #### **Constructor Parameters** -* `vector_store` (Required): The backing vector database instance (e.g., FAISS, Pinecone). +* `vector_store` (Required): The backing vector database instance (e.g., FAISS, Weaviate). * `knowledge_graph` (Optional): The graph store instance for structured knowledge. * `token_limit` (Default: `2000`): The maximum number of tokens allowed in short-term memory before pruning occurs. * `short_term_limit` (Default: `10`): The maximum number of distinct memory items in short-term memory. diff --git a/docs/reference/embeddings.md b/docs/reference/embeddings.md index e980c58e..d9445de1 100644 --- a/docs/reference/embeddings.md +++ b/docs/reference/embeddings.md @@ -34,7 +34,7 @@ The **Embeddings Module** provides a unified interface for generating vector rep --- - Automatic formatting and validation for FAISS, Pinecone, Qdrant, and Weaviate. + Automatic formatting and validation for FAISS, Qdrant, and Weaviate. @@ -122,13 +122,13 @@ print(f"Dimension: {embedder.get_embedding_dimension()}") --- ### VectorEmbeddingManager (The Bridge) -A utility class that prepares raw embeddings for insertion into specific vector databases. It handles formatting differences between backends like FAISS and Pinecone. +A utility class that prepares raw embeddings for insertion into specific vector databases. It handles formatting differences between backends like FAISS and Weaviate. #### **Core Methods** | Method | Description | |--------|-------------| -| `prepare_for_vector_db(embeddings, backend, ...)` | Formats data for the target DB. | +| `prepare_for_vector_db(embeddings, metadata, backend)` | Formats data for the target DB. | | `validate_dimensions(embeddings, expected_dim)` | Ensures vectors match the index configuration. | | `batch_prepare(embeddings_list)` | Prepares a batch of embeddings for storage. | diff --git a/docs/reference/vector_store.md b/docs/reference/vector_store.md index 71b99a53..1309e99b 100644 --- a/docs/reference/vector_store.md +++ b/docs/reference/vector_store.md @@ -1,6 +1,6 @@ # Vector Store -> **Unified vector database interface supporting FAISS, Pinecone, Weaviate, Qdrant, and Milvus with Hybrid Search.** +> **Unified vector database interface supporting FAISS, Weaviate, Qdrant, and Milvus with Hybrid Search.** --- @@ -12,7 +12,7 @@ --- - Seamlessly switch between FAISS (Local), Pinecone, Weaviate, Qdrant, and Milvus + Seamlessly switch between FAISS (Local), Weaviate, Qdrant, and Milvus - :material-magnify-plus:{ .lg .middle } **Hybrid Search** @@ -230,7 +230,6 @@ results = searcher.search( Backend-specific implementations: - `FAISSAdapter`: Local, in-memory/disk. -- `PineconeAdapter`: Managed cloud service. - `WeaviateAdapter`: Schema-aware vector DB. - `QdrantAdapter`: Rust-based high-performance DB. - `MilvusAdapter`: Scalable cloud-native DB. @@ -265,41 +264,6 @@ query = np.random.rand(768).astype('float32') distances, indices = adapter.search(index, query, k=10) ``` -#### PineconeAdapter - -Managed cloud vector database. - -**Helper Classes:** -- `PineconeIndex`: Index management -- `PineconeQuery`: Query operations -- `PineconeMetadata`: Metadata handling - -**Example:** - -```python -from semantica.vector_store import PineconeAdapter - -adapter = PineconeAdapter(api_key="your-key", environment="us-west1-gcp") -adapter.connect() - -# Create index -index = adapter.create_index("my-index", dimension=768, metric="cosine") - -# Upsert with metadata -adapter.upsert_vectors( - vectors=[[0.1, 0.2, ...], ...], - ids=["vec_1", "vec_2"], - metadata=[{"category": "news"}, ...] -) - -# Query with filter -results = adapter.query_vectors( - query_vector=[0.1, 0.2, ...], - top_k=10, - filter={"category": {"$eq": "news"}} -) -``` - #### WeaviateAdapter Schema-aware vector database with GraphQL. @@ -716,25 +680,23 @@ print(f"Available methods: {methods}") ### Environment Variables ```bash -export VECTOR_STORE_BACKEND=pinecone -export PINECONE_API_KEY=sk-... -export PINECONE_ENV=us-west1-gcp +export VECTOR_STORE_BACKEND=weaviate +export WEAVIATE_URL=http://localhost:8080 ``` ### YAML Configuration ```yaml vector_store: - backend: faiss # or pinecone, weaviate, etc. + backend: faiss # or weaviate, qdrant, milvus dimension: 1536 metric: cosine faiss: index_type: HNSW - pinecone: - environment: us-west1-gcp - index_name: my-index + weaviate: + url: http://localhost:8080 ``` --- @@ -777,7 +739,7 @@ print(f"Context: {context}") **Solution**: Ensure your embedding model dimension (e.g., 1536 for OpenAI) matches the VectorStore dimension. **Issue**: FAISS index not saved. -**Solution**: Call `store.save("index.faiss")` explicitly for local FAISS indices, or use a persistent backend like Pinecone/Qdrant. +**Solution**: Call `store.save("index.faiss")` explicitly for local FAISS indices, or use a persistent backend like Weaviate/Qdrant. --- diff --git a/pyproject.toml b/pyproject.toml index 6af89586..14eff6e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,6 @@ dependencies = [ "librosa>=0.9.0", "opencv-python>=4.6.0", "faiss-cpu>=1.7.0", - "pinecone-client>=2.2.0", "weaviate-client>=3.15.0", "qdrant-client>=1.3.0", "neo4j>=5.0.0", diff --git a/semantica/embeddings/embeddings_usage.md b/semantica/embeddings/embeddings_usage.md index 41e2edee..ff03cb06 100644 --- a/semantica/embeddings/embeddings_usage.md +++ b/semantica/embeddings/embeddings_usage.md @@ -447,14 +447,6 @@ from semantica.embeddings import VectorEmbeddingManager manager = VectorEmbeddingManager() -# Prepare for Pinecone -pinecone_data = manager.prepare_for_vector_db( - embeddings, - metadata=metadata, - backend="pinecone", - namespace="my_namespace" -) - # Prepare for Weaviate weaviate_data = manager.prepare_for_vector_db( embeddings, @@ -486,9 +478,9 @@ from semantica.embeddings import VectorEmbeddingManager manager = VectorEmbeddingManager() # Validate dimensions for specific backend -is_valid = manager.validate_dimensions(embeddings, backend="pinecone") +is_valid = manager.validate_dimensions(embeddings, backend="weaviate") if is_valid: - print("Embeddings meet Pinecone requirements") + print("Embeddings meet Weaviate requirements") else: print("Embeddings do not meet requirements") ``` diff --git a/semantica/embeddings/vector_embedding_manager.py b/semantica/embeddings/vector_embedding_manager.py index 8daa5bc3..33fcabb4 100644 --- a/semantica/embeddings/vector_embedding_manager.py +++ b/semantica/embeddings/vector_embedding_manager.py @@ -9,7 +9,7 @@ Key Features: - Validate embedding dimensions for different backends - Normalize embeddings for vector DB requirements - Create metadata compatible with vector DBs - - Integration helpers for FAISS, Pinecone, Weaviate, Qdrant, Milvus + - Integration helpers for FAISS, Weaviate, Qdrant, Milvus Example Usage: >>> from semantica.embeddings import VectorEmbeddingManager @@ -36,7 +36,6 @@ class VectorEmbeddingManager: Supported Backends: - FAISS: Local vector storage - - Pinecone: Cloud vector database - Weaviate: GraphQL-based vector database - Qdrant: Vector similarity search engine - Milvus: Open-source vector database @@ -50,7 +49,7 @@ class VectorEmbeddingManager: ... backend="faiss" ... ) >>> # Validate dimensions - >>> is_valid = manager.validate_dimensions(embeddings, backend="pinecone") + >>> is_valid = manager.validate_dimensions(embeddings, backend="weaviate") """ def __init__(self, embedding_generator: Optional[EmbeddingGenerator] = None): @@ -67,7 +66,6 @@ class VectorEmbeddingManager: # Backend-specific dimension requirements self.backend_requirements = { "faiss": {"min_dim": 1, "max_dim": None, "dtype": np.float32}, - "pinecone": {"min_dim": 1, "max_dim": 20000, "dtype": np.float32}, "weaviate": {"min_dim": 1, "max_dim": None, "dtype": np.float32}, "qdrant": {"min_dim": 1, "max_dim": None, "dtype": np.float32}, "milvus": {"min_dim": 1, "max_dim": 32768, "dtype": np.float32}, @@ -90,7 +88,7 @@ class VectorEmbeddingManager: Args: embeddings: Embeddings array (n_samples, embedding_dim) or (embedding_dim,) metadata: Optional list of metadata dictionaries (one per embedding) - backend: Vector DB backend ("faiss", "pinecone", "weaviate", "qdrant", "milvus") + backend: Vector DB backend ("faiss", "weaviate", "qdrant", "milvus") normalize: Whether to normalize embeddings (default: True) **options: Additional backend-specific options @@ -108,7 +106,7 @@ class VectorEmbeddingManager: >>> embeddings = np.random.rand(10, 384).astype(np.float32) >>> metadata = [{"text": f"doc_{i}"} for i in range(10)] >>> result = manager.prepare_for_vector_db( - ... embeddings, metadata, backend="pinecone" + ... embeddings, metadata, backend="weaviate" ... ) """ if backend.lower() not in self.backend_requirements: @@ -228,7 +226,7 @@ class VectorEmbeddingManager: bool: True if dimensions are valid, False otherwise Example: - >>> is_valid = manager.validate_dimensions(embeddings, backend="pinecone") + >>> is_valid = manager.validate_dimensions(embeddings, backend="weaviate") """ if backend.lower() not in self.backend_requirements: self.logger.warning(f"Unknown backend: {backend}, skipping validation") @@ -312,7 +310,7 @@ class VectorEmbeddingManager: Example: >>> metadata = [{"text": "doc1", "category": "science"}] - >>> formatted = manager.create_metadata(metadata, backend="pinecone") + >>> formatted = manager.create_metadata(metadata, backend="weaviate") """ formatted = [] @@ -321,16 +319,7 @@ class VectorEmbeddingManager: formatted_meta = meta.copy() # Backend-specific formatting - if backend.lower() == "pinecone": - # Pinecone has specific metadata requirements - # Remove None values and ensure types are compatible - formatted_meta = { - k: v - for k, v in formatted_meta.items() - if v is not None - and isinstance(v, (str, int, float, bool, list)) - } - elif backend.lower() == "weaviate": + if backend.lower() == "weaviate": # Weaviate uses specific property types # Ensure values are compatible formatted_meta = { @@ -374,8 +363,6 @@ class VectorEmbeddingManager: # Add backend-specific details if backend.lower() == "faiss": info["index_type"] = options.get("index_type", "flat") - elif backend.lower() == "pinecone": - info["namespace"] = options.get("namespace", "default") elif backend.lower() == "weaviate": info["class_name"] = options.get("class_name", "Document") diff --git a/semantica/export/__init__.py b/semantica/export/__init__.py index 16736e81..79a4f99f 100644 --- a/semantica/export/__init__.py +++ b/semantica/export/__init__.py @@ -62,7 +62,7 @@ OWL Export: Vector Export: - Vector Serialization: Multiple format support (JSON, NumPy, Binary, FAISS) - - Vector Store Integration: Format conversion for Pinecone, Weaviate, Qdrant, FAISS + - Vector Store Integration: Format conversion for Weaviate, Qdrant, FAISS - Metadata Association: Vector-to-metadata mapping and serialization - Batch Export: Efficient batch vector export processing - Multi-dimensional Support: Variable dimension vector handling diff --git a/semantica/export/methods.py b/semantica/export/methods.py index 62126b14..e06d1de9 100644 --- a/semantica/export/methods.py +++ b/semantica/export/methods.py @@ -110,7 +110,7 @@ OWL Export: Vector Export: - Vector Serialization: Multiple format support (JSON, NumPy, Binary, FAISS) - - Vector Store Integration: Format conversion for Pinecone, Weaviate, Qdrant, FAISS + - Vector Store Integration: Format conversion for Weaviate, Qdrant, FAISS - Metadata Association: Vector-to-metadata mapping and serialization - Batch Export: Efficient batch vector export processing - Multi-dimensional Support: Variable dimension vector handling diff --git a/semantica/export/vector_exporter.py b/semantica/export/vector_exporter.py index cf56dfc8..c839ef8e 100644 --- a/semantica/export/vector_exporter.py +++ b/semantica/export/vector_exporter.py @@ -7,7 +7,7 @@ embedding systems. Key Features: - Multiple vector format export (JSON, NumPy, Binary, FAISS) - - Vector store integration (Pinecone, Weaviate, Qdrant, FAISS) + - Vector store integration (Weaviate, Qdrant, FAISS) - Metadata and document association - Batch vector export - Multi-dimensional vector support @@ -16,7 +16,7 @@ Example Usage: >>> from semantica.export import VectorExporter >>> exporter = VectorExporter(format="json", include_metadata=True) >>> exporter.export(vectors, "vectors.json") - >>> exporter.export_for_vector_store(vectors, "pinecone.json", vector_store_type="pinecone") + >>> exporter.export_for_vector_store(vectors, "weaviate.json", vector_store_type="weaviate") Author: Semantica Contributors License: MIT @@ -43,7 +43,7 @@ class VectorExporter: Features: - Multiple vector format export (JSON, NumPy, Binary, FAISS) - - Vector store integration (Pinecone, Weaviate, Qdrant, FAISS) + - Vector store integration (Weaviate, Qdrant, FAISS) - Metadata and document association - Batch vector export - Multi-dimensional vector support @@ -471,7 +471,7 @@ class VectorExporter: self, vectors: List[Dict[str, Any]], file_path: Union[str, Path], - vector_store_type: str = "pinecone", + vector_store_type: str = "weaviate", **options, ) -> None: """ @@ -480,12 +480,10 @@ class VectorExporter: Args: vectors: List of vector dictionaries file_path: Output file path - vector_store_type: Vector store type ('pinecone', 'weaviate', 'qdrant', 'faiss') + vector_store_type: Vector store type ('weaviate', 'qdrant', 'faiss') **options: Additional options """ - if vector_store_type == "pinecone": - self._export_pinecone_format(vectors, file_path, **options) - elif vector_store_type == "weaviate": + if vector_store_type == "weaviate": self._export_weaviate_format(vectors, file_path, **options) elif vector_store_type == "qdrant": self._export_qdrant_format(vectors, file_path, **options) @@ -495,27 +493,6 @@ class VectorExporter: # Default to JSON self._export_json(vectors, Path(file_path), {}, **options) - def _export_pinecone_format( - self, vectors: List[Dict[str, Any]], file_path: Path, **options - ) -> None: - """Export in Pinecone format.""" - pinecone_data = [] - - for vec_data in vectors: - vector_id = vec_data.get("id") or vec_data.get("vector_id", "") - vector = vec_data.get("vector") or vec_data.get("embedding", []) - metadata = vec_data.get("metadata", {}) - - if "text" in vec_data and self.include_text: - metadata["text"] = vec_data["text"] - - pinecone_data.append( - {"id": vector_id, "values": vector, "metadata": metadata} - ) - - export_data = {"vectors": pinecone_data} - write_json_file(export_data, file_path, indent=2) - def _export_weaviate_format( self, vectors: List[Dict[str, Any]], file_path: Path, **options ) -> None: diff --git a/semantica/pipeline/pipeline_templates.py b/semantica/pipeline/pipeline_templates.py index 953cc582..2170e330 100644 --- a/semantica/pipeline/pipeline_templates.py +++ b/semantica/pipeline/pipeline_templates.py @@ -148,7 +148,7 @@ class PipelineTemplateManager: { "name": "store_vectors", "type": "store_vectors", - "config": {"store": "pinecone"}, + "config": {"store": "weaviate"}, "dependencies": ["embed"], }, ], diff --git a/semantica/pipeline/pipeline_usage.md b/semantica/pipeline/pipeline_usage.md index 37cc3651..0322656c 100644 --- a/semantica/pipeline/pipeline_usage.md +++ b/semantica/pipeline/pipeline_usage.md @@ -632,7 +632,7 @@ builder = template_manager.create_pipeline_from_template( "rag_pipeline", chunk={"chunk_size": 512}, embed={"model": "text-embedding-3-large"}, - store_vectors={"store": "pinecone"} + store_vectors={"store": "weaviate"} ) pipeline = builder.build() @@ -1124,7 +1124,8 @@ builder = template_manager.create_pipeline_from_template( ingest={"source": "./documents"}, chunk={"chunk_size": 512, "overlap": 50}, embed={"model": "text-embedding-3-large", "batch_size": 32}, - store_vectors={"store": "pinecone", "index_name": "documents"} + # Step-specific overrides + store_vectors={"store": "weaviate", "index_name": "documents"} ) pipeline = builder.build() diff --git a/semantica/utils/constants.py b/semantica/utils/constants.py index 453b4612..3fcf9476 100644 --- a/semantica/utils/constants.py +++ b/semantica/utils/constants.py @@ -71,7 +71,6 @@ SUPPORTED_RDF_FORMATS = ["turtle", "rdfxml", "jsonld", "n3", "ntriples"] # Supported Vector Store Backends SUPPORTED_VECTOR_STORES = [ "faiss", - "pinecone", "weaviate", "qdrant", "milvus", diff --git a/semantica/vector_store/__init__.py b/semantica/vector_store/__init__.py index 9ded3b91..0a039933 100644 --- a/semantica/vector_store/__init__.py +++ b/semantica/vector_store/__init__.py @@ -3,7 +3,7 @@ Vector Store Management Module This module provides comprehensive vector storage and retrieval capabilities for the Semantica framework, including support for multiple vector store backends (FAISS, -Pinecone, Weaviate, Qdrant, Milvus), hybrid search combining vector similarity and +Weaviate, Qdrant, Milvus), hybrid search combining vector similarity and metadata filtering, metadata management, and namespace isolation. Algorithms Used: @@ -50,25 +50,30 @@ Namespace Management: Adapter Pattern: - FAISS Adapter: Local vector storage, FAISS index management, index persistence (save/load), batch operations, multiple index types support - - Pinecone Adapter: Cloud vector database integration, HTTP API communication, index management, upsert operations, query operations, metadata filtering - - Weaviate Adapter: GraphQL-based queries, schema management, object-oriented storage, rich metadata support, batch operations - - Qdrant Adapter: REST API communication, collection management, vector operations, payload (metadata) filtering, batch operations - - Milvus Adapter: gRPC communication, collection management, vector operations, metadata filtering, batch operations - - Unified Interface: Common interface for all adapters, backend-specific operation delegation, adapter factory pattern, connection management + - Weaviate Adapter: Schema-aware storage, GraphQL query support, object-oriented data model, batch operations, schema management + - Qdrant Adapter: Point-based storage, payload filtering, collection management, optimized search, batch operations + - Milvus Adapter: Scalable vector database, collection management, partitioning, complex querying, index building -Batch Operations: - - Batch Vector Operations: Chunking algorithm (fixed-size batch creation), batch processing, progress tracking, error handling per batch, retry mechanism - - Batch Indexing: Batch vector addition to index, incremental index updates, batch index training, batch index optimization - - Batch Search: Batch query processing, parallel search execution (when supported), result aggregation, batch result formatting +Supported Backends: + - FAISS: In-memory/local disk (Facebook AI Similarity Search) + - Weaviate: Cloud/Self-hosted (Schema-aware vector database) + - Qdrant: Cloud/Self-hosted (Vector database for the next generation of AI) + - Milvus: Cloud/Self-hosted (Highly scalable vector database) + - InMemory: Simple list-based storage for testing/small datasets -Performance Optimization: - - Vector Normalization: L2 normalization for cosine similarity, normalization caching, batch normalization - - Index Optimization: Index parameter tuning, index rebuilding for better performance, memory optimization, search speed optimization - - Caching: Query result caching, vector caching, metadata caching, cache invalidation strategies - - Parallel Processing: Batch-level parallelization, multi-threaded search (when supported), concurrent index operations +Configuration: + - Environment variables (SEMANTICA_VECTOR_STORE_*) + - Configuration files (yaml/json) + - Runtime configuration via VectorStoreConfig + +Dependencies: + - faiss-cpu (or faiss-gpu) + - weaviate-client + - qdrant-client + - pymilvus Key Features: - - Multi-backend vector store support (FAISS, Pinecone, Weaviate, Qdrant, Milvus) + - Multi-backend vector store support (FAISS, Weaviate, Qdrant, Milvus) - Vector indexing and similarity search - Metadata indexing and filtering - Hybrid search combining vector and metadata queries @@ -84,7 +89,6 @@ Main Classes: - VectorRetriever: Vector retrieval and similarity search - VectorManager: Vector store management and operations - FAISSAdapter: FAISS integration for local vector storage - - PineconeAdapter: Pinecone cloud vector database integration - WeaviateAdapter: Weaviate vector database integration - QdrantAdapter: Qdrant vector database integration - MilvusAdapter: Milvus vector database integration @@ -141,12 +145,6 @@ from .methods import ( ) from .milvus_adapter import MilvusAdapter, MilvusClient, MilvusCollection, MilvusSearch from .namespace_manager import Namespace, NamespaceManager -from .pinecone_adapter import ( - PineconeAdapter, - PineconeIndex, - PineconeMetadata, - PineconeQuery, -) from .qdrant_adapter import QdrantAdapter, QdrantClient, QdrantCollection, QdrantSearch from .registry import MethodRegistry, method_registry from .vector_store import VectorIndexer, VectorManager, VectorRetriever, VectorStore @@ -168,11 +166,6 @@ __all__ = [ "FAISSIndex", "FAISSSearch", "FAISSIndexBuilder", - # Pinecone - "PineconeAdapter", - "PineconeIndex", - "PineconeQuery", - "PineconeMetadata", # Weaviate "WeaviateAdapter", "WeaviateClient", diff --git a/semantica/vector_store/config.py b/semantica/vector_store/config.py index 9cf3e8f7..39064cfd 100644 --- a/semantica/vector_store/config.py +++ b/semantica/vector_store/config.py @@ -116,8 +116,6 @@ class VectorStoreConfig: "VECTOR_STORE_ENABLE_HYBRID_SEARCH": "enable_hybrid_search", "VECTOR_STORE_NAMESPACE": "default_namespace", "VECTOR_STORE_FAISS_INDEX_TYPE": "faiss_index_type", - "VECTOR_STORE_PINECONE_API_KEY": "pinecone_api_key", - "VECTOR_STORE_PINECONE_ENVIRONMENT": "pinecone_environment", "VECTOR_STORE_WEAVIATE_URL": "weaviate_url", "VECTOR_STORE_QDRANT_URL": "qdrant_url", "VECTOR_STORE_MILVUS_HOST": "milvus_host", diff --git a/semantica/vector_store/pinecone_adapter.py b/semantica/vector_store/pinecone_adapter.py deleted file mode 100644 index b8fb6e01..00000000 --- a/semantica/vector_store/pinecone_adapter.py +++ /dev/null @@ -1,510 +0,0 @@ -""" -Pinecone Adapter Module - -This module provides Pinecone cloud vector database integration for vector storage -and similarity search in the Semantica framework, supporting serverless and pod-based -deployments with namespace isolation and metadata filtering. - -Key Features: - - Cloud-based vector storage and retrieval - - Serverless and pod-based index specifications - - Namespace isolation for multi-tenant support - - Metadata filtering and querying - - Batch upsert and query operations - - Index statistics and monitoring - - Optional dependency handling - -Main Classes: - - PineconeAdapter: Main Pinecone adapter for cloud vector operations - - PineconeIndex: Pinecone index wrapper with operations - - PineconeQuery: Pinecone query builder and executor - - PineconeMetadata: Metadata validation and sanitization - -Example Usage: - >>> from semantica.vector_store import PineconeAdapter - >>> adapter = PineconeAdapter(api_key="your-api-key") - >>> adapter.connect() - >>> index = adapter.create_index("my-index", dimension=768, metric="cosine") - >>> adapter.upsert_vectors(vectors, ids, metadata, namespace="docs") - >>> results = adapter.query_vectors(query_vector, top_k=10, namespace="docs") - >>> stats = adapter.get_stats() - -Author: Semantica Contributors -License: MIT -""" - -from typing import Any, Dict, List, Optional, Union - -import numpy as np - -from ..utils.exceptions import ProcessingError, ValidationError -from ..utils.logging import get_logger -from ..utils.progress_tracker import get_progress_tracker - -# Optional Pinecone import -try: - import pinecone - from pinecone import Pinecone, PodSpec, ServerlessSpec - - PINECONE_AVAILABLE = True -except ImportError: - PINECONE_AVAILABLE = False - pinecone = None - Pinecone = None - ServerlessSpec = None - PodSpec = None - - -class PineconeIndex: - """Pinecone index wrapper.""" - - def __init__(self, index: Any, index_name: str): - """Initialize Pinecone index wrapper.""" - self.index = index - self.index_name = index_name - self.logger = get_logger("pinecone_index") - - def upsert_vectors( - self, vectors: List[Dict[str, Any]], namespace: Optional[str] = None, **options - ) -> Dict[str, Any]: - """Upsert vectors to index.""" - if not PINECONE_AVAILABLE: - raise ProcessingError("Pinecone not available") - - try: - response = self.index.upsert( - vectors=vectors, namespace=namespace, **options - ) - return response - except Exception as e: - raise ProcessingError(f"Failed to upsert vectors: {str(e)}") - - def query_vectors( - self, - query_vector: np.ndarray, - top_k: int = 10, - namespace: Optional[str] = None, - filter: Optional[Dict[str, Any]] = None, - **options, - ) -> Dict[str, Any]: - """Query similar vectors.""" - if not PINECONE_AVAILABLE: - raise ProcessingError("Pinecone not available") - - try: - response = self.index.query( - vector=query_vector.tolist(), - top_k=top_k, - namespace=namespace, - filter=filter, - include_metadata=True, - **options, - ) - return response - except Exception as e: - raise ProcessingError(f"Failed to query vectors: {str(e)}") - - def delete_vectors( - self, ids: List[str], namespace: Optional[str] = None, **options - ) -> Dict[str, Any]: - """Delete vectors from index.""" - if not PINECONE_AVAILABLE: - raise ProcessingError("Pinecone not available") - - try: - response = self.index.delete(ids=ids, namespace=namespace, **options) - return response - except Exception as e: - raise ProcessingError(f"Failed to delete vectors: {str(e)}") - - def fetch_vectors( - self, ids: List[str], namespace: Optional[str] = None, **options - ) -> Dict[str, Any]: - """Fetch vectors by IDs.""" - if not PINECONE_AVAILABLE: - raise ProcessingError("Pinecone not available") - - try: - response = self.index.fetch(ids=ids, namespace=namespace, **options) - return response - except Exception as e: - raise ProcessingError(f"Failed to fetch vectors: {str(e)}") - - def describe_index_stats(self, namespace: Optional[str] = None) -> Dict[str, Any]: - """Get index statistics.""" - if not PINECONE_AVAILABLE: - raise ProcessingError("Pinecone not available") - - try: - stats = self.index.describe_index_stats(namespace=namespace) - return stats - except Exception as e: - raise ProcessingError(f"Failed to get index stats: {str(e)}") - - -class PineconeQuery: - """Pinecone query builder.""" - - def __init__(self, index: PineconeIndex): - """Initialize Pinecone query builder.""" - self.index = index - self.logger = get_logger("pinecone_query") - - def build_query( - self, - query_vector: np.ndarray, - top_k: int = 10, - namespace: Optional[str] = None, - filter: Optional[Dict[str, Any]] = None, - **options, - ) -> Dict[str, Any]: - """Build query parameters.""" - return { - "vector": query_vector.tolist(), - "top_k": top_k, - "namespace": namespace, - "filter": filter, - **options, - } - - def execute(self, query_params: Dict[str, Any]) -> List[Dict[str, Any]]: - """Execute query and format results.""" - response = self.index.query_vectors(**query_params) - - results = [] - for match in response.get("matches", []): - results.append( - { - "id": match.get("id"), - "score": match.get("score", 0.0), - "metadata": match.get("metadata", {}), - } - ) - - return results - - -class PineconeMetadata: - """Pinecone metadata handler.""" - - @staticmethod - def validate_metadata(metadata: Dict[str, Any]) -> Dict[str, Any]: - """Validate and sanitize metadata.""" - # Pinecone metadata restrictions - validated = {} - - for key, value in metadata.items(): - # Convert to allowed types - if isinstance(value, (str, int, float, bool, list)): - validated[key] = value - elif isinstance(value, dict): - # Nested dicts not directly supported - validated[key] = str(value) - else: - validated[key] = str(value) - - return validated - - -class PineconeAdapter: - """ - Pinecone adapter for vector storage and similarity search. - - โ€ข Pinecone connection and authentication - โ€ข Vector storage and retrieval - โ€ข Similarity search and filtering - โ€ข Namespace and index management - โ€ข Performance optimization - โ€ข Error handling and recovery - """ - - def __init__( - self, api_key: Optional[str] = None, environment: Optional[str] = None, **config - ): - """Initialize Pinecone adapter.""" - self.logger = get_logger("pinecone_adapter") - self.config = config - self.progress_tracker = get_progress_tracker() - self.api_key = api_key or config.get("api_key") - self.environment = environment or config.get("environment") - - self.client: Optional[Any] = None - self.index: Optional[PineconeIndex] = None - self.query_builder: Optional[PineconeQuery] = None - - # Check Pinecone availability - if not PINECONE_AVAILABLE: - self.logger.warning( - "Pinecone not available. Install with: pip install pinecone-client" - ) - - def connect(self, api_key: Optional[str] = None, **options) -> bool: - """ - Connect to Pinecone service. - - Args: - api_key: Pinecone API key - **options: Connection options - - Returns: - True if connected successfully - """ - if not PINECONE_AVAILABLE: - raise ProcessingError( - "Pinecone is not available. Install it with: pip install pinecone-client" - ) - - api_key = api_key or self.api_key - if not api_key: - raise ValidationError("Pinecone API key is required") - - try: - self.client = Pinecone(api_key=api_key) - self.logger.info("Connected to Pinecone") - return True - except Exception as e: - raise ProcessingError(f"Failed to connect to Pinecone: {str(e)}") - - def create_index( - self, - index_name: str, - dimension: int, - metric: str = "cosine", - spec: Optional[Dict[str, Any]] = None, - **options, - ) -> PineconeIndex: - """ - Create new vector index. - - Args: - index_name: Name of the index - dimension: Vector dimension - metric: Distance metric ("cosine", "euclidean", "dotproduct") - spec: Index specification (serverless or pod) - **options: Additional options - - Returns: - PineconeIndex instance - """ - if self.client is None: - self.connect() - - if not PINECONE_AVAILABLE: - raise ProcessingError("Pinecone not available") - - try: - # Check if index exists - existing_indexes = [idx.name for idx in self.client.list_indexes()] - if index_name in existing_indexes: - self.logger.info(f"Index {index_name} already exists") - return self.get_index(index_name) - - # Create index specification - if spec is None: - spec = ServerlessSpec(cloud="aws", region="us-east-1") - - # Create index - self.client.create_index( - name=index_name, - dimension=dimension, - metric=metric, - spec=spec, - **options, - ) - - self.logger.info(f"Created Pinecone index: {index_name}") - return self.get_index(index_name) - - except Exception as e: - raise ProcessingError(f"Failed to create index: {str(e)}") - - def get_index(self, index_name: str) -> PineconeIndex: - """ - Get existing index. - - Args: - index_name: Name of the index - - Returns: - PineconeIndex instance - """ - if self.client is None: - self.connect() - - if not PINECONE_AVAILABLE: - raise ProcessingError("Pinecone not available") - - try: - index = self.client.Index(index_name) - self.index = PineconeIndex(index, index_name) - self.query_builder = PineconeQuery(self.index) - return self.index - except Exception as e: - raise ProcessingError(f"Failed to get index: {str(e)}") - - def upsert_vectors( - self, - vectors: List[Union[np.ndarray, List[float]]], - ids: List[str], - metadata: Optional[List[Dict[str, Any]]] = None, - namespace: Optional[str] = None, - **options, - ) -> Dict[str, Any]: - """ - Insert or update vectors. - - Args: - vectors: List of vectors - ids: Vector IDs - metadata: Vector metadata - namespace: Namespace name - **options: Additional options - - Returns: - Upsert response - """ - tracking_id = self.progress_tracker.start_tracking( - module="vector_store", - submodule="PineconeAdapter", - message=f"Upserting {len(vectors)} vectors to Pinecone", - ) - - try: - if self.index is None: - self.progress_tracker.stop_tracking( - tracking_id, status="failed", message="Index not initialized" - ) - raise ProcessingError( - "Index not initialized. Call create_index() or get_index() first." - ) - - # Format vectors - self.progress_tracker.update_tracking( - tracking_id, message="Formatting vectors..." - ) - formatted_vectors = [] - for i, vector in enumerate(vectors): - if isinstance(vector, np.ndarray): - vector = vector.tolist() - - vector_data = {"id": ids[i], "values": vector} - - if metadata and i < len(metadata): - vector_data["metadata"] = PineconeMetadata.validate_metadata( - metadata[i] - ) - - formatted_vectors.append(vector_data) - - self.progress_tracker.update_tracking( - tracking_id, message="Upserting vectors to Pinecone..." - ) - result = self.index.upsert_vectors(formatted_vectors, namespace, **options) - - self.progress_tracker.stop_tracking( - tracking_id, - status="completed", - message=f"Upserted {len(vectors)} vectors", - ) - return result - except Exception as e: - self.progress_tracker.stop_tracking( - tracking_id, status="failed", message=str(e) - ) - raise - - def query_vectors( - self, - query_vector: np.ndarray, - top_k: int = 10, - namespace: Optional[str] = None, - filter: Optional[Dict[str, Any]] = None, - **options, - ) -> List[Dict[str, Any]]: - """ - Query similar vectors. - - Args: - query_vector: Query vector - top_k: Number of results - namespace: Namespace name - filter: Metadata filter - **options: Additional options - - Returns: - List of search results - """ - tracking_id = self.progress_tracker.start_tracking( - module="vector_store", - submodule="PineconeAdapter", - message=f"Querying {top_k} similar vectors from Pinecone", - ) - - try: - if self.query_builder is None: - self.progress_tracker.stop_tracking( - tracking_id, status="failed", message="Index not initialized" - ) - raise ProcessingError( - "Index not initialized. Call create_index() or get_index() first." - ) - - self.progress_tracker.update_tracking( - tracking_id, message="Building query..." - ) - query_params = self.query_builder.build_query( - query_vector, top_k, namespace, filter, **options - ) - - self.progress_tracker.update_tracking( - tracking_id, message="Executing query..." - ) - results = self.query_builder.execute(query_params) - - self.progress_tracker.stop_tracking( - tracking_id, - status="completed", - message=f"Query completed: {len(results) if isinstance(results, list) else 'N/A'} results", - ) - return results - except Exception as e: - self.progress_tracker.stop_tracking( - tracking_id, status="failed", message=str(e) - ) - raise - - def delete_vectors( - self, ids: List[str], namespace: Optional[str] = None, **options - ) -> Dict[str, Any]: - """ - Delete vectors from index. - - Args: - ids: Vector IDs to delete - namespace: Namespace name - **options: Additional options - - Returns: - Delete response - """ - if self.index is None: - raise ProcessingError( - "Index not initialized. Call create_index() or get_index() first." - ) - - return self.index.delete_vectors(ids, namespace, **options) - - def get_stats(self, namespace: Optional[str] = None) -> Dict[str, Any]: - """Get index statistics.""" - if self.index is None: - raise ProcessingError( - "Index not initialized. Call create_index() or get_index() first." - ) - - stats = self.index.describe_index_stats(namespace) - return { - "total_vector_count": stats.get("total_vector_count", 0), - "dimension": stats.get("dimension", 0), - "index_fullness": stats.get("index_fullness", 0.0), - "namespaces": stats.get("namespaces", {}), - } diff --git a/semantica/vector_store/vector_store.py b/semantica/vector_store/vector_store.py index 48435795..da572626 100644 --- a/semantica/vector_store/vector_store.py +++ b/semantica/vector_store/vector_store.py @@ -58,8 +58,16 @@ class VectorStore: โ€ข Provides vector store operations """ + SUPPORTED_BACKENDS = {"faiss", "weaviate", "qdrant", "milvus", "inmemory"} + def __init__(self, backend="faiss", config=None, **kwargs): """Initialize vector store.""" + if backend.lower() not in self.SUPPORTED_BACKENDS: + raise ValueError( + f"Unsupported backend: {backend}. " + f"Supported backends are: {', '.join(sorted(self.SUPPORTED_BACKENDS))}" + ) + self.logger = get_logger("vector_store") self.config = config or {} self.config.update(kwargs) diff --git a/semantica/vector_store/vector_store_usage.md b/semantica/vector_store/vector_store_usage.md index 65ec2db0..d9248dc1 100644 --- a/semantica/vector_store/vector_store_usage.md +++ b/semantica/vector_store/vector_store_usage.md @@ -1,6 +1,6 @@ # Vector Store Module Usage Guide -This comprehensive guide demonstrates how to use the vector store module for vector storage and retrieval, supporting multiple vector store backends (FAISS, Pinecone, Weaviate, Qdrant, Milvus), hybrid search combining vector similarity and metadata filtering, metadata management, and namespace isolation. +This comprehensive guide demonstrates how to use the vector store module for vector storage and retrieval, supporting multiple vector store backends (FAISS, Weaviate, Qdrant, Milvus), hybrid search combining vector similarity and metadata filtering, metadata management, and namespace isolation. ## Table of Contents @@ -687,34 +687,6 @@ distances, indices = adapter.search(index, query_vector, k=10) print(f"Found {len(indices)} similar vectors") ``` -### Pinecone Adapter - -```python -from semantica.vector_store import PineconeAdapter -import numpy as np - -# Create Pinecone adapter -adapter = PineconeAdapter(api_key="your-api-key", environment="us-west1-gcp") - -# Connect -adapter.connect() - -# Create index -index = adapter.create_index("my-index", dimension=768, metric="cosine") - -# Upsert vectors -vectors = [np.random.rand(768).tolist() for _ in range(100)] -ids = [f"vec_{i}" for i in range(100)] -metadata = [{"category": "science"} for _ in range(100)] -adapter.upsert_vectors(vectors, ids, metadata) - -# Query -query_vector = np.random.rand(768).tolist() -results = adapter.query_vectors(query_vector, top_k=10, include_metadata=True) - -print(f"Found {len(results)} results") -``` - ### Weaviate Adapter ```python @@ -1104,10 +1076,6 @@ export VECTOR_STORE_NAMESPACE=default # FAISS configuration export VECTOR_STORE_FAISS_INDEX_TYPE=flat -# Pinecone configuration -export VECTOR_STORE_PINECONE_API_KEY=your-api-key -export VECTOR_STORE_PINECONE_ENVIRONMENT=us-west1-gcp - # Weaviate configuration export VECTOR_STORE_WEAVIATE_URL=http://localhost:8080 @@ -1153,8 +1121,6 @@ vector_store: enable_hybrid_search: true default_namespace: default faiss_index_type: flat - pinecone_api_key: your-api-key - pinecone_environment: us-west1-gcp weaviate_url: http://localhost:8080 qdrant_url: http://localhost:6333 milvus_host: localhost @@ -1204,7 +1170,7 @@ print(f"Found {len(results)} hybrid search results") ### Multi-Backend Vector Store ```python -from semantica.vector_store import FAISSAdapter, PineconeAdapter +from semantica.vector_store import FAISSAdapter, WeaviateAdapter import numpy as np # Local FAISS store @@ -1213,17 +1179,17 @@ faiss_index = faiss_adapter.create_index(index_type="flat", metric="L2") faiss_vectors = np.random.rand(1000, 768).astype('float32') faiss_adapter.add_vectors(faiss_index, faiss_vectors, ids=[f"faiss_{i}" for i in range(1000)]) -# Cloud Pinecone store -pinecone_adapter = PineconeAdapter(api_key="your-key") -pinecone_adapter.connect() -pinecone_index = pinecone_adapter.create_index("my-index", dimension=768) -pinecone_vectors = [np.random.rand(768).tolist() for _ in range(1000)] -pinecone_adapter.upsert_vectors(pinecone_vectors, [f"pinecone_{i}" for i in range(1000)]) +# Self-hosted Weaviate store +weaviate_adapter = WeaviateAdapter(url="http://localhost:8080") +weaviate_adapter.connect() +weaviate_index = weaviate_adapter.create_index("my-index", dimension=768) +weaviate_vectors = [np.random.rand(768).tolist() for _ in range(1000)] +weaviate_adapter.upsert_vectors(weaviate_vectors, [f"weaviate_{i}" for i in range(1000)]) # Search both query_vector = np.random.rand(768) faiss_results = faiss_adapter.search(faiss_index, query_vector, k=10) -pinecone_results = pinecone_adapter.query_vectors(query_vector, top_k=10) +weaviate_results = weaviate_adapter.query_vectors(query_vector, top_k=10) ``` ### Hybrid Search with Custom Ranking diff --git a/tests/vector_store/test_pinecone_removal.py b/tests/vector_store/test_pinecone_removal.py new file mode 100644 index 00000000..0f869334 --- /dev/null +++ b/tests/vector_store/test_pinecone_removal.py @@ -0,0 +1,62 @@ +import unittest +from unittest.mock import MagicMock, patch +import os +import sys + +# Ensure semantica is in path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))) + +from semantica.vector_store.vector_store import VectorStore +from semantica.vector_store.registry import method_registry +from semantica.vector_store.config import vector_store_config + +class TestPineconeRemoval(unittest.TestCase): + """Verify that Pinecone has been completely removed from the system.""" + + def test_pinecone_backend_rejected(self): + """Test that initializing VectorStore with backend='pinecone' raises an error.""" + with self.assertRaises(ValueError) as context: + VectorStore(backend="pinecone") + + # The error message might be generic "Unknown backend" or specific. + # We just want to ensure it fails. + self.assertTrue("pinecone" in str(context.exception).lower() or "unknown" in str(context.exception).lower()) + + def test_registry_clean(self): + """Test that no Pinecone methods are registered.""" + # Check all task types + task_types = ["store", "search", "index", "hybrid_search", "metadata", "namespace"] + + for task in task_types: + methods = method_registry.list_all(task) + # Flatten if it's a dict + if isinstance(methods, dict): + method_names = methods.get(task, []) + else: + method_names = methods + + for name in method_names: + self.assertNotIn("pinecone", name.lower(), f"Found pinecone reference in registry task {task}: {name}") + + def test_config_clean(self): + """Test that configuration does not contain Pinecone keys.""" + config = vector_store_config.get_all() + + for key in config.keys(): + self.assertNotIn("pinecone", key.lower(), f"Found pinecone key in config: {key}") + + def test_adapters_existence(self): + """Verify that other adapters exist but PineconeAdapter does not.""" + try: + from semantica.vector_store import faiss_adapter + from semantica.vector_store import weaviate_adapter + from semantica.vector_store import qdrant_adapter + from semantica.vector_store import milvus_adapter + except ImportError as e: + self.fail(f"Failed to import a required adapter: {e}") + + with self.assertRaises(ImportError): + from semantica.vector_store import pinecone_adapter + +if __name__ == '__main__': + unittest.main() diff --git a/tests/vector_store/test_vector_store_deepdive.py b/tests/vector_store/test_vector_store_deepdive.py new file mode 100644 index 00000000..c12dfc7c --- /dev/null +++ b/tests/vector_store/test_vector_store_deepdive.py @@ -0,0 +1,372 @@ +import unittest +from unittest.mock import MagicMock, patch, ANY +import numpy as np +import sys +from pathlib import Path + +# Add project root to path +sys.path.append(str(Path(__file__).parent.parent.parent)) + +from semantica.vector_store.vector_store import VectorStore, VectorIndexer, VectorRetriever, VectorManager +from semantica.vector_store.registry import MethodRegistry, method_registry +from semantica.vector_store.faiss_adapter import FAISSAdapter, FAISSIndex, FAISSIndexBuilder, FAISSSearch +from semantica.vector_store.milvus_adapter import MilvusAdapter, MilvusClient, MilvusCollection, MilvusSearch +from semantica.vector_store.qdrant_adapter import QdrantAdapter +from semantica.vector_store.weaviate_adapter import WeaviateAdapter +from semantica.vector_store.hybrid_search import HybridSearch, MetadataFilter, SearchRanker + +class TestVectorStoreDeepDive(unittest.TestCase): + + def setUp(self): + self.vectors = [np.array([1.0, 0.0]), np.array([0.0, 1.0])] + self.ids = ["vec_1", "vec_2"] + self.metadata = [{"type": "a"}, {"type": "b"}] + + def test_vector_store_in_memory(self): + """Test the default in-memory VectorStore implementation.""" + store = VectorStore(backend="inmemory", dimension=2) + + # Test storing vectors + ids = store.store_vectors(self.vectors, self.metadata) + self.assertEqual(len(ids), 2) + self.assertEqual(store.vectors[ids[0]].tolist(), self.vectors[0].tolist()) + + # Test searching vectors (exact match) + results = store.search_vectors(np.array([1.0, 0.0]), k=1) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["id"], ids[0]) + # Score should be close to 1.0 (cosine similarity of identical vectors) + self.assertAlmostEqual(results[0]["score"], 1.0) + + # Test searching vectors (orthogonal) + results = store.search_vectors(np.array([0.0, 1.0]), k=1) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["id"], ids[1]) + + # Test updating vectors + new_vec = np.array([0.5, 0.5]) + store.update_vectors([ids[0]], [new_vec]) + self.assertTrue(np.array_equal(store.get_vector(ids[0]), new_vec)) + + # Test deleting vectors + store.delete_vectors([ids[0]]) + self.assertIsNone(store.get_vector(ids[0])) + self.assertEqual(len(store.vectors), 1) + + def test_vector_indexer_retriever(self): + """Test VectorIndexer and VectorRetriever directly.""" + indexer = VectorIndexer(backend="inmemory", dimension=2) + index = indexer.create_index(self.vectors, self.ids) + self.assertIsNotNone(index) + self.assertEqual(len(index["vectors"]), 2) + + retriever = VectorRetriever(backend="inmemory") + results = retriever.search_similar( + np.array([1.0, 0.0]), + self.vectors, + self.ids, + k=1 + ) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["id"], "vec_1") + + # Test hybrid search (metadata filter) + results = retriever.search_hybrid( + np.array([1.0, 0.0]), + {"type": "b"}, # Filter for vec_2 + self.vectors, + self.metadata, + k=1 + ) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["vector"].tolist(), self.vectors[1].tolist()) + + def test_method_registry(self): + """Test the MethodRegistry.""" + registry = MethodRegistry() + + def custom_store(): return "stored" + + # Register + registry.register("store", "custom", custom_store, version="1.0") + self.assertTrue(registry.has("store", "custom")) + + # Get + func = registry.get("store", "custom") + self.assertEqual(func(), "stored") + + # Metadata + meta = registry.get_metadata("store", "custom") + self.assertEqual(meta["version"], "1.0") + + # List + all_methods = registry.list_all("store") + self.assertEqual(all_methods["store"], ["custom"]) + + # Unregister + registry.unregister("store", "custom") + self.assertFalse(registry.has("store", "custom")) + + @patch('semantica.vector_store.faiss_adapter.faiss') + @patch('semantica.vector_store.faiss_adapter.FAISS_AVAILABLE', True) + def test_faiss_adapter(self, mock_faiss): + """Test FAISSAdapter with mocked faiss.""" + # Setup mock + mock_index = MagicMock() + mock_faiss.IndexFlatL2.return_value = mock_index + mock_faiss.read_index.return_value = mock_index + + # Mock search return + # distances, indices + mock_index.search.return_value = (np.array([[0.0, 0.1]]), np.array([[0, 1]])) + mock_index.ntotal = 2 + + # Test Init + adapter = FAISSAdapter(dimension=2) + + # Test Create Index + adapter.create_index(index_type="flat") + mock_faiss.IndexFlatL2.assert_called_with(2) + + # Test Add Vectors + adapter.add_vectors(self.vectors, self.ids, self.metadata) + mock_index.add.assert_called() + self.assertEqual(len(adapter.index.vector_ids), 2) + + # Test Search + results = adapter.search_similar(np.array([1.0, 0.0]), k=2) + self.assertEqual(len(results), 2) + self.assertEqual(results[0]["id"], "vec_1") + + # Test Save + adapter.save_index("test.index") + mock_faiss.write_index.assert_called() + + # Test Load + adapter.load_index("test.index") + mock_faiss.read_index.assert_called() + + @patch('semantica.vector_store.milvus_adapter.connections') + @patch('semantica.vector_store.milvus_adapter.Collection') + @patch('semantica.vector_store.milvus_adapter.utility') + @patch('semantica.vector_store.milvus_adapter.DataType') + @patch('semantica.vector_store.milvus_adapter.FieldSchema') + @patch('semantica.vector_store.milvus_adapter.CollectionSchema') + @patch('semantica.vector_store.milvus_adapter.MILVUS_AVAILABLE', True) + def test_milvus_adapter(self, mock_collection_schema, mock_field_schema, mock_data_type, mock_utility, mock_collection_cls, mock_connections): + """Test MilvusAdapter with mocked pymilvus.""" + # Setup mocks + mock_data_type.INT64 = 1 + mock_data_type.FLOAT_VECTOR = 2 + # Setup mocks + mock_utility.has_collection.return_value = False + mock_collection_instance = MagicMock() + mock_collection_cls.return_value = mock_collection_instance + + # Mock search results + mock_hit = MagicMock() + mock_hit.id = 1 + mock_hit.distance = 0.1 + mock_collection_instance.search.return_value = [[mock_hit]] + + # Test Init + adapter = MilvusAdapter(host="localhost") + + # Test Connect + adapter.connect() + mock_connections.connect.assert_called_with( + alias="default", host="localhost", port=19530, user=None, password=None + ) + + # Test Create Collection + adapter.create_collection("test_coll", dimension=2) + mock_collection_cls.assert_called() + mock_collection_instance.create_index.assert_called() + + # Test Insert + adapter.insert_vectors(self.vectors) + mock_collection_instance.insert.assert_called() + + # Test Search + results = adapter.search_vectors(np.array([1.0, 0.0]), limit=1) + mock_collection_instance.search.assert_called() + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["id"], 1) + + @patch('semantica.vector_store.qdrant_adapter.QdrantClientLib') + @patch('semantica.vector_store.qdrant_adapter.VectorParams') + @patch('semantica.vector_store.qdrant_adapter.Distance') + @patch('semantica.vector_store.qdrant_adapter.PointStruct') + @patch('semantica.vector_store.qdrant_adapter.QDRANT_AVAILABLE', True) + def test_qdrant_adapter(self, mock_point_struct, mock_distance, mock_vector_params, mock_qdrant_cls): + """Test QdrantAdapter with mocked qdrant_client.""" + mock_client = MagicMock() + mock_qdrant_cls.return_value = mock_client + + # Mock search response + mock_hit = MagicMock() + mock_hit.id = "vec_1" + mock_hit.score = 0.9 + mock_hit.payload = {"type": "a"} + mock_client.search.return_value = [mock_hit] + + adapter = QdrantAdapter(url="http://localhost:6333") + + # Connect + adapter.connect() + mock_qdrant_cls.assert_called() + + # Create Collection + adapter.create_collection("test-collection", vector_size=2) + mock_client.create_collection.assert_called() + + # Insert + adapter.insert_vectors(self.vectors, self.ids, payloads=self.metadata) + mock_client.upsert.assert_called() + + # Search + results = adapter.search_vectors(np.array([1.0, 0.0]), limit=1) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["id"], "vec_1") + + @patch('semantica.vector_store.weaviate_adapter.weaviate') + @patch('semantica.vector_store.weaviate_adapter.MetadataQuery') + @patch('semantica.vector_store.weaviate_adapter.WEAVIATE_AVAILABLE', True) + def test_weaviate_adapter(self, mock_metadata_query, mock_weaviate): + """Test WeaviateAdapter with mocked weaviate.""" + mock_client = MagicMock() + mock_weaviate.connect_to_local.return_value = mock_client + + mock_collection = MagicMock() + mock_client.collections.get.return_value = mock_collection + + # Mock search response + mock_obj = MagicMock() + mock_obj.uuid = "uuid-1" + mock_obj.properties = {"text": "hello"} + mock_obj.metadata.distance = 0.1 + + mock_query_response = MagicMock() + mock_query_response.objects = [mock_obj] + + mock_collection.query.near_vector.return_value = mock_query_response + + adapter = WeaviateAdapter(url="http://localhost:8080") + + # Connect + adapter.connect() + mock_weaviate.connect_to_local.assert_called() + + # Create Schema + adapter.create_schema("TestClass", properties=[]) + mock_client.collections.create.assert_called() + + # Add Objects + # Need to mock batch context manager + mock_batch = MagicMock() + mock_collection.batch.dynamic.return_value.__enter__.return_value = mock_batch + + adapter.get_collection("TestClass") + adapter.add_objects([{"text": "hello"}], vectors=self.vectors) + mock_batch.add_object.assert_called() + + # Query + results = adapter.query_vectors(np.array([1.0, 0.0]), limit=1) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["id"], "uuid-1") + + def test_hybrid_search(self): + """Test HybridSearch, MetadataFilter and SearchRanker.""" + search = HybridSearch() + + # Test MetadataFilter + meta_filter = MetadataFilter().eq("type", "a") + self.assertTrue(meta_filter.matches({"type": "a"})) + self.assertFalse(meta_filter.matches({"type": "b"})) + + meta_filter = MetadataFilter().gt("val", 10) + self.assertTrue(meta_filter.matches({"val": 20})) + self.assertFalse(meta_filter.matches({"val": 5})) + + # Test Search + results = search.search( + query_vector=np.array([1.0, 0.0]), + vectors=self.vectors, + metadata=self.metadata, + vector_ids=self.ids, + k=2 + ) + self.assertEqual(len(results), 2) + self.assertEqual(results[0]["id"], "vec_1") + + # Test Filtered Search + results = search.search( + query_vector=np.array([1.0, 0.0]), + vectors=self.vectors, + metadata=self.metadata, + vector_ids=self.ids, + k=2, + metadata_filter=MetadataFilter().eq("type", "b") + ) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["id"], "vec_2") + + # Test Ranker + ranker = SearchRanker(strategy="reciprocal_rank_fusion") + res1 = [{"id": "1", "score": 0.9}, {"id": "2", "score": 0.8}] + res2 = [{"id": "2", "score": 0.85}, {"id": "1", "score": 0.7}] + + fused = ranker.rank([res1, res2]) + self.assertEqual(len(fused), 2) + # ID 2 should be top because it's high in both? Or ID 1? + # RRF: 1/(k+1) + 1/(k+2) vs 1/(k+2) + 1/(k+1). They are equal rank-wise (1st and 2nd). + + # Multi-source search + sources = [ + {"vectors": [self.vectors[0]], "metadata": [self.metadata[0]], "ids": ["vec_1"]}, + {"vectors": [self.vectors[1]], "metadata": [self.metadata[1]], "ids": ["vec_2"]} + ] + multi_res = search.multi_source_search(np.array([1.0, 0.0]), sources, k=2) + self.assertEqual(len(multi_res), 2) + + def test_vector_manager(self): + """Test VectorManager.""" + manager = VectorManager() + store = VectorStore(backend="inmemory") + store.store_vectors(self.vectors, self.metadata) + + # Test statistics + stats = manager.collect_statistics(store) + self.assertEqual(stats["total_vectors"], 2) + self.assertEqual(stats["backend"], "inmemory") + + # Test maintenance + health = manager.maintain_store(store) + self.assertTrue(health["healthy"]) + + # Test manage_store wrapper + results = manager.manage_store(store, statistics=True, optimize=True) + self.assertIn("statistics", results) + self.assertIn("optimize", results) + + def test_config(self): + """Test VectorStoreConfig.""" + from semantica.vector_store.config import vector_store_config + + # Test get default + self.assertEqual(vector_store_config.get("default_backend"), "faiss") + + # Test set + vector_store_config.set("test_key", "test_value") + self.assertEqual(vector_store_config.get("test_key"), "test_value") + + # Test update + vector_store_config.update({"test_key_2": "val2"}) + self.assertEqual(vector_store_config.get("test_key_2"), "val2") + + # Test method config + vector_store_config.set_method_config("test_method", {"param": 1}) + self.assertEqual(vector_store_config.get_method_config("test_method")["param"], 1) + +if __name__ == '__main__': + unittest.main()