diff --git a/pyproject.toml b/pyproject.toml index 69eaf7cc..9aa57a57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,16 @@ graph-all = [ "semantica[graph-neo4j,graph-falkordb,graph-amazon-neptune]" ] +# ---- Vector Store Backends ---- +vectorstore-qdrant = ["qdrant-client>=1.0.0"] +vectorstore-weaviate = ["weaviate-client>=4.0.0"] +vectorstore-pinecone = ["pinecone-client>=3.0.0"] +vectorstore-milvus = ["pymilvus>=2.0.0"] + +vectorstore-all = [ + "semantica[vectorstore-qdrant,vectorstore-weaviate,vectorstore-pinecone,vectorstore-milvus]" +] + # ---- Infra / Queues / Workers ---- infra = [ "redis>=4.3.0", @@ -177,7 +187,7 @@ dev = [ # ---- Everything ---- all = [ - "semantica[dev,viz,gpu,infra,cloud,monitoring,llm-all,models-huggingface,split-all,graph-all,parse-docling]" + "semantica[dev,viz,gpu,infra,cloud,monitoring,llm-all,models-huggingface,split-all,graph-all,vectorstore-all,parse-docling]" ] # ---------------- ENTRYPOINTS ---------------- diff --git a/semantica/vector_store/__init__.py b/semantica/vector_store/__init__.py index 825aec35..6cd40f0d 100644 --- a/semantica/vector_store/__init__.py +++ b/semantica/vector_store/__init__.py @@ -3,7 +3,7 @@ Vector Store Management Module This module provides comprehensive vector storage and retrieval capabilities for the Semantica framework, including support for multiple vector store backends (FAISS, -Weaviate, Qdrant, Milvus), hybrid search combining vector similarity and +Weaviate, Qdrant, Pinecone, Milvus), hybrid search combining vector similarity and metadata filtering, metadata management, and namespace isolation. Algorithms Used: @@ -73,7 +73,7 @@ Dependencies: - pymilvus Key Features: - - Multi-backend vector store support (FAISS, Weaviate, Qdrant, Milvus) + - Multi-backend vector store support (FAISS, Weaviate, Qdrant, Pinecone, Milvus) - Vector indexing and similarity search - Metadata indexing and filtering - Hybrid search combining vector and metadata queries @@ -91,6 +91,7 @@ Main Classes: - FAISSStore: FAISS integration for local vector storage - WeaviateStore: Weaviate vector database integration - QdrantStore: Qdrant vector database integration + - PineconeStore: Pinecone vector database integration - MilvusStore: Milvus vector database integration - HybridSearch: Hybrid vector and metadata search - MetadataStore: Metadata indexing and management @@ -145,6 +146,7 @@ from .methods import ( ) from .milvus_store import MilvusStore, MilvusClient, MilvusCollection, MilvusSearch from .namespace_manager import Namespace, NamespaceManager +from .pinecone_store import PineconeStore, PineconeClient, PineconeIndex, PineconeSearch from .qdrant_store import QdrantStore, QdrantClient, QdrantCollection, QdrantSearch from .registry import MethodRegistry, method_registry from .vector_store import VectorIndexer, VectorManager, VectorRetriever, VectorStore @@ -181,6 +183,11 @@ __all__ = [ "MilvusClient", "MilvusCollection", "MilvusSearch", + # Pinecone + "PineconeStore", + "PineconeClient", + "PineconeIndex", + "PineconeSearch", # Hybrid search "HybridSearch", "MetadataFilter", diff --git a/semantica/vector_store/pinecone_store.py b/semantica/vector_store/pinecone_store.py new file mode 100644 index 00000000..f1cc5135 --- /dev/null +++ b/semantica/vector_store/pinecone_store.py @@ -0,0 +1,639 @@ +""" +Pinecone Store Module + +This module provides Pinecone vector database integration for vector storage and +similarity search in the Semantica framework, supporting managed vector database +service with serverless and pod-based indexes, namespace isolation, and efficient +vector operations with metadata filtering. + +Key Features: + - Serverless and Pod-based index management + - Namespace isolation for multi-tenant support + - Metadata filtering during search + - Batch operations for efficient data loading + - Index creation, deletion, and listing + - Optional dependency handling + +Main Classes: + - PineconeStore: Main Pinecone store for vector operations + - PineconeClient: Pinecone client wrapper + - PineconeIndex: Index wrapper with operations + - PineconeSearch: Search operations and filtering + +Example Usage: + >>> from semantica.vector_store import PineconeStore + >>> store = PineconeStore(api_key="your-api-key") + >>> store.connect() + >>> store.create_index("my-index", dimension=768) + >>> store.upsert_vectors(vectors, ids, metadata=metadata) + >>> results = store.search_vectors(query_vector, k=10, filter={"category": "science"}) + >>> stats = store.get_stats() + +Author: Semantica Contributors +License: MIT +""" + +from typing import Any, Dict, List, Optional, Union + +import numpy as np + +from ..utils.exceptions import ProcessingError, ValidationError +from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker + +# Optional Pinecone import +try: + from pinecone import Pinecone as PineconeClientLib, ServerlessSpec, PodSpec + + PINECONE_AVAILABLE = True +except (ImportError, OSError): + PINECONE_AVAILABLE = False + PineconeClientLib = None + ServerlessSpec = None + PodSpec = None + + +class PineconeClient: + """Pinecone client wrapper.""" + + def __init__(self, client: Any): + """Initialize Pinecone client wrapper.""" + self.client = client + self.logger = get_logger("pinecone_client") + + def create_index( + self, + index_name: str, + dimension: int, + metric: str = "cosine", + spec: Optional[Dict[str, Any]] = None, + **options, + ) -> bool: + """Create an index in Pinecone.""" + if not PINECONE_AVAILABLE: + raise ProcessingError("Pinecone not available") + + try: + # Default to serverless spec if not provided + if spec is None: + spec = ServerlessSpec(cloud="aws", region="us-east-1") + + # Map metric names + metric_map = { + "cosine": "cosine", + "euclidean": "euclidean_distance", + "dot": "dotproduct", + } + pinecone_metric = metric_map.get(metric.lower(), "cosine") + + self.client.create_index( + name=index_name, + dimension=dimension, + metric=pinecone_metric, + spec=spec, + **options, + ) + return True + except Exception as e: + raise ProcessingError(f"Failed to create index: {str(e)}") + + def delete_index(self, index_name: str) -> bool: + """Delete an index from Pinecone.""" + if not PINECONE_AVAILABLE: + raise ProcessingError("Pinecone not available") + + try: + self.client.delete_index(index_name) + return True + except Exception as e: + raise ProcessingError(f"Failed to delete index: {str(e)}") + + def list_indexes(self) -> List[str]: + """List available indexes.""" + if not PINECONE_AVAILABLE: + raise ProcessingError("Pinecone not available") + + try: + indexes = self.client.list_indexes() + return [index.name for index in indexes] + except Exception as e: + raise ProcessingError(f"Failed to list indexes: {str(e)}") + + def get_index(self, index_name: str) -> Any: + """Get index object.""" + if not PINECONE_AVAILABLE: + raise ProcessingError("Pinecone not available") + + try: + return self.client.Index(index_name) + except Exception as e: + raise ProcessingError(f"Failed to get index: {str(e)}") + + +class PineconeIndex: + """Pinecone index wrapper.""" + + def __init__(self, index: Any): + """Initialize Pinecone index wrapper.""" + self.index = index + self.logger = get_logger("pinecone_index") + + def upsert_vectors( + self, + vectors: List[List[float]], + ids: List[str], + metadata: Optional[List[Dict[str, Any]]] = None, + namespace: str = "", + **options, + ) -> Dict[str, Any]: + """Upsert vectors to index.""" + if not PINECONE_AVAILABLE: + raise ProcessingError("Pinecone not available") + + try: + # Prepare vectors for upsert + upsert_data = [] + for i, (vector, vector_id) in enumerate(zip(vectors, ids)): + vector_dict = {"id": vector_id, "values": vector} + if metadata and i < len(metadata): + vector_dict["metadata"] = metadata[i] + upsert_data.append(vector_dict) + + response = self.index.upsert( + vectors=upsert_data, namespace=namespace, **options + ) + return {"upserted_count": response.upserted_count} + except Exception as e: + raise ProcessingError(f"Failed to upsert vectors: {str(e)}") + + def search_vectors( + self, + query_vector: List[float], + k: int = 10, + filter: Optional[Dict[str, Any]] = None, + namespace: str = "", + **options, + ) -> List[Dict[str, Any]]: + """Search for similar vectors.""" + if not PINECONE_AVAILABLE: + raise ProcessingError("Pinecone not available") + + try: + response = self.index.query( + vector=query_vector, + top_k=k, + filter=filter, + namespace=namespace, + include_metadata=True, + include_values=False, + **options, + ) + + results = [] + for match in response.matches: + results.append( + { + "id": match.id, + "score": match.score, + "metadata": match.metadata or {}, + } + ) + + return results + except Exception as e: + raise ProcessingError(f"Failed to search vectors: {str(e)}") + + def delete_vectors( + self, vector_ids: List[str], namespace: str = "", **options + ) -> Dict[str, Any]: + """Delete vectors from index.""" + if not PINECONE_AVAILABLE: + raise ProcessingError("Pinecone not available") + + try: + response = self.index.delete(ids=vector_ids, namespace=namespace, **options) + return {"deleted": True} + except Exception as e: + raise ProcessingError(f"Failed to delete vectors: {str(e)}") + + def fetch_vectors( + self, vector_ids: List[str], namespace: str = "", **options + ) -> Dict[str, Any]: + """Fetch vectors by ID.""" + if not PINECONE_AVAILABLE: + raise ProcessingError("Pinecone not available") + + try: + response = self.index.fetch(ids=vector_ids, namespace=namespace, **options) + return { + "vectors": { + vector_id: { + "values": vector.values, + "metadata": vector.metadata or {}, + } + for vector_id, vector in response.vectors.items() + } + } + except Exception as e: + raise ProcessingError(f"Failed to fetch vectors: {str(e)}") + + def describe_index_stats(self, **options) -> Dict[str, Any]: + """Get index statistics.""" + if not PINECONE_AVAILABLE: + raise ProcessingError("Pinecone not available") + + try: + stats = self.index.describe_index_stats(**options) + return { + "dimension": stats.dimension, + "index_fullness": stats.index_fullness, + "total_vector_count": stats.total_vector_count, + "namespaces": stats.namespaces, + } + except Exception as e: + raise ProcessingError(f"Failed to get index stats: {str(e)}") + + +class PineconeSearch: + """Pinecone search operations.""" + + def __init__(self, index: PineconeIndex): + """Initialize Pinecone search.""" + self.index = index + self.logger = get_logger("pinecone_search") + + def similarity_search( + self, + query_vector: np.ndarray, + limit: int = 10, + filter: Optional[Dict[str, Any]] = None, + namespace: str = "", + **options, + ) -> List[Dict[str, Any]]: + """ + Perform similarity search. + + Args: + query_vector: Query vector + limit: Number of results + filter: Metadata filter + namespace: Namespace to search in + **options: Additional options + + Returns: + List of search results + """ + return self.index.search_vectors( + query_vector.tolist(), limit, filter, namespace, **options + ) + + +class PineconeStore: + """ + Pinecone store for vector storage and similarity search. + + • Pinecone connection and authentication + • Index and namespace management + • Vector storage and retrieval + • Similarity search and filtering + • Performance optimization + • Error handling and recovery + """ + + def __init__( + self, + api_key: Optional[str] = None, + environment: Optional[str] = None, + **config, + ): + """Initialize Pinecone store.""" + self.logger = get_logger("pinecone_store") + self.config = config + self.progress_tracker = get_progress_tracker() + # Ensure progress tracker is enabled + if not self.progress_tracker.enabled: + self.progress_tracker.enabled = True + + self.api_key = api_key or config.get("api_key") + self.environment = environment or config.get("environment") + + self.client: Optional[PineconeClient] = None + self.index: Optional[PineconeIndex] = None + self.search_engine: Optional[PineconeSearch] = None + + # Check Pinecone availability + if not PINECONE_AVAILABLE: + self.logger.warning( + "Pinecone not available. Install with: pip install pinecone-client" + ) + + def connect(self, **kwargs) -> bool: + """ + Connect to Pinecone service. + + Args: + **kwargs: Connection options + + Returns: + True if connected successfully + """ + if not PINECONE_AVAILABLE: + raise ProcessingError( + "Pinecone is not available. Install it with: pip install pinecone-client" + ) + + api_key = kwargs.get("api_key") or self.api_key + if not api_key: + raise ValidationError("Pinecone API key is required") + + try: + pinecone_client = PineconeClientLib(api_key=api_key, **kwargs) + self.client = PineconeClient(pinecone_client) + + self.logger.info("Connected to Pinecone") + return True + except Exception as e: + raise ProcessingError(f"Failed to connect to Pinecone: {str(e)}") + + def create_index( + self, + index_name: str, + dimension: int, + metric: str = "cosine", + spec: Optional[Dict[str, Any]] = None, + **kwargs, + ): + """ + Create a Pinecone index. + + Args: + index_name: Name of the index + dimension: Vector dimension + metric: Distance metric ("cosine", "euclidean", "dot") + spec: Index specification (ServerlessSpec or PodSpec) + **kwargs: Additional options + + Returns: + PineconeIndex instance + """ + if self.client is None: + self.connect() + + if not PINECONE_AVAILABLE: + raise ProcessingError("Pinecone not available") + + try: + # Create index spec if not provided + if spec is None: + spec = ServerlessSpec(cloud="aws", region="us-east-1") + + self.client.create_index(index_name, dimension, metric, spec, **kwargs) + + # Get the index + pinecone_index = self.client.get_index(index_name) + self.index = PineconeIndex(pinecone_index) + self.search_engine = PineconeSearch(self.index) + + self.logger.info(f"Created Pinecone index: {index_name}") + return self.index + + except Exception as e: + raise ProcessingError(f"Failed to create index: {str(e)}") + + def get_index(self, index_name: str) -> PineconeIndex: + """ + Get existing index. + + Args: + index_name: Name of the index + + Returns: + PineconeIndex instance + """ + if self.client is None: + self.connect() + + if not PINECONE_AVAILABLE: + raise ProcessingError("Pinecone not available") + + try: + pinecone_index = self.client.get_index(index_name) + self.index = PineconeIndex(pinecone_index) + self.search_engine = PineconeSearch(self.index) + return self.index + except Exception as e: + raise ProcessingError(f"Failed to get index: {str(e)}") + + def delete_index(self, index_name: str) -> bool: + """ + Delete an index. + + Args: + index_name: Name of the index to delete + + Returns: + True if deleted successfully + """ + if self.client is None: + self.connect() + + return self.client.delete_index(index_name) + + def list_indexes(self) -> List[str]: + """ + List available indexes. + + Returns: + List of index names + """ + if self.client is None: + self.connect() + + return self.client.list_indexes() + + def upsert_vectors( + self, + vectors: List[Any], + ids: List[str], + metadata: Optional[List[Dict[str, Any]]] = None, + namespace: str = "", + **options, + ) -> Dict[str, Any]: + """ + Upsert vectors to index. + + Args: + vectors: List of vectors + ids: Vector IDs + metadata: Optional metadata for each vector + namespace: Namespace to upsert into + **options: Additional options + + Returns: + Upsert response + """ + tracking_id = self.progress_tracker.start_tracking( + module="vector_store", + submodule="PineconeStore", + message=f"Upserting {len(vectors)} vectors to Pinecone index", + ) + + try: + if self.index is None: + self.progress_tracker.stop_tracking( + tracking_id, status="failed", message="Index not initialized" + ) + raise ProcessingError( + "Index not initialized. Call create_index() or get_index() first." + ) + + if not PINECONE_AVAILABLE: + self.progress_tracker.stop_tracking( + tracking_id, status="failed", message="Pinecone not available" + ) + raise ProcessingError("Pinecone not available") + + self.progress_tracker.update_tracking( + tracking_id, message="Preparing vectors..." + ) + + # Convert vectors to list format + vector_list = [] + for vector in vectors: + if isinstance(vector, np.ndarray): + vector_list.append(vector.tolist()) + else: + vector_list.append(list(vector)) + + self.progress_tracker.update_tracking( + tracking_id, message="Upserting vectors to index..." + ) + result = self.index.upsert_vectors( + vector_list, ids, metadata, namespace, **options + ) + + self.progress_tracker.stop_tracking( + tracking_id, + status="completed", + message=f"Upserted {len(vectors)} vectors", + ) + return result + + except Exception as e: + self.progress_tracker.stop_tracking( + tracking_id, status="failed", message=str(e) + ) + raise ProcessingError(f"Failed to upsert vectors: {str(e)}") + + def search_vectors( + self, + query_vector: Any, + k: int = 10, + filter: Optional[Dict[str, Any]] = None, + namespace: str = "", + **options, + ) -> List[Dict[str, Any]]: + """ + Search vectors in index. + + Args: + query_vector: Query vector + k: Number of results + filter: Metadata filter + namespace: Namespace to search in + **options: Additional options + + Returns: + List of search results + """ + tracking_id = self.progress_tracker.start_tracking( + module="vector_store", + submodule="PineconeStore", + message=f"Searching for {k} similar vectors in Pinecone", + ) + + try: + if self.search_engine is None: + self.progress_tracker.stop_tracking( + tracking_id, status="failed", message="Index not initialized" + ) + raise ProcessingError( + "Index not initialized. Call create_index() or get_index() first." + ) + + self.progress_tracker.update_tracking( + tracking_id, message="Performing similarity search..." + ) + + # Convert query vector to list + if isinstance(query_vector, np.ndarray): + query_vector = query_vector.tolist() + else: + query_vector = list(query_vector) + + results = self.search_engine.similarity_search( + np.array(query_vector), k, filter, namespace, **options + ) + + self.progress_tracker.stop_tracking( + tracking_id, + status="completed", + message=f"Found {len(results)} similar vectors", + ) + return results + except Exception as e: + self.progress_tracker.stop_tracking( + tracking_id, status="failed", message=str(e) + ) + raise + + def delete_vectors( + self, vector_ids: List[str], namespace: str = "", **options + ) -> Dict[str, Any]: + """ + Delete vectors from index. + + Args: + vector_ids: Vector IDs to delete + namespace: Namespace to delete from + **options: Additional options + + Returns: + Delete response + """ + if self.index is None: + raise ProcessingError( + "Index not initialized. Call create_index() or get_index() first." + ) + + return self.index.delete_vectors(vector_ids, namespace, **options) + + def fetch_vectors( + self, vector_ids: List[str], namespace: str = "", **options + ) -> Dict[str, Any]: + """ + Fetch vectors by ID. + + Args: + vector_ids: Vector IDs to fetch + namespace: Namespace to fetch from + **options: Additional options + + Returns: + Fetch response + """ + if self.index is None: + raise ProcessingError( + "Index not initialized. Call create_index() or get_index() first." + ) + + return self.index.fetch_vectors(vector_ids, namespace, **options) + + def get_stats(self, **options) -> Dict[str, Any]: + """Get index statistics.""" + if self.index is None: + raise ProcessingError( + "Index not initialized. Call create_index() or get_index() first." + ) + + return self.index.describe_index_stats(**options) diff --git a/semantica/vector_store/vector_store.py b/semantica/vector_store/vector_store.py index 2bff50ee..e6735660 100644 --- a/semantica/vector_store/vector_store.py +++ b/semantica/vector_store/vector_store.py @@ -60,7 +60,7 @@ class VectorStore: • Provides vector store operations """ - SUPPORTED_BACKENDS = {"faiss", "weaviate", "qdrant", "milvus", "inmemory"} + SUPPORTED_BACKENDS = {"faiss", "weaviate", "qdrant", "milvus", "pinecone", "inmemory"} def __init__(self, backend="faiss", config=None, max_workers: int = 6, **kwargs): """Initialize vector store.""" diff --git a/tests/vector_store/test_pinecone_removal.py b/tests/vector_store/test_pinecone_removal.py index 3677d1fa..af296689 100644 --- a/tests/vector_store/test_pinecone_removal.py +++ b/tests/vector_store/test_pinecone_removal.py @@ -1,62 +1,3 @@ -import unittest -from unittest.mock import MagicMock, patch -import os -import sys - -# Ensure semantica is in path -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))) - -from semantica.vector_store.vector_store import VectorStore -from semantica.vector_store.registry import method_registry -from semantica.vector_store.config import vector_store_config - -class TestPineconeRemoval(unittest.TestCase): - """Verify that Pinecone has been completely removed from the system.""" - - def test_pinecone_backend_rejected(self): - """Test that initializing VectorStore with backend='pinecone' raises an error.""" - with self.assertRaises(ValueError) as context: - VectorStore(backend="pinecone") - - # The error message might be generic "Unknown backend" or specific. - # We just want to ensure it fails. - self.assertTrue("pinecone" in str(context.exception).lower() or "unknown" in str(context.exception).lower()) - - def test_registry_clean(self): - """Test that no Pinecone methods are registered.""" - # Check all task types - task_types = ["store", "search", "index", "hybrid_search", "metadata", "namespace"] - - for task in task_types: - methods = method_registry.list_all(task) - # Flatten if it's a dict - if isinstance(methods, dict): - method_names = methods.get(task, []) - else: - method_names = methods - - for name in method_names: - self.assertNotIn("pinecone", name.lower(), f"Found pinecone reference in registry task {task}: {name}") - - def test_config_clean(self): - """Test that configuration does not contain Pinecone keys.""" - config = vector_store_config.get_all() - - for key in config.keys(): - self.assertNotIn("pinecone", key.lower(), f"Found pinecone key in config: {key}") - - def test_stores_existence(self): - """Verify that other stores exist but PineconeStore does not.""" - try: - from semantica.vector_store import faiss_store - from semantica.vector_store import weaviate_store - from semantica.vector_store import qdrant_store - from semantica.vector_store import milvus_store - except ImportError as e: - self.fail(f"Failed to import a required store: {e}") - - with self.assertRaises(ImportError): - from semantica.vector_store import pinecone_store - -if __name__ == '__main__': - unittest.main() +# This test file has been updated as Pinecone support has been re-added to Semantica. +# Pinecone is now a supported vector store backend (PR #220). +# See test_pinecone_store.py for Pinecone-specific tests. diff --git a/tests/vector_store/test_pinecone_store.py b/tests/vector_store/test_pinecone_store.py new file mode 100644 index 00000000..9d03cc9d --- /dev/null +++ b/tests/vector_store/test_pinecone_store.py @@ -0,0 +1,289 @@ +import unittest +from unittest.mock import MagicMock, patch +import numpy as np +import sys +import os + +# Ensure semantica is in path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))) + +from semantica.vector_store.pinecone_store import ( + PineconeStore, + PineconeClient, + PineconeIndex, + PineconeSearch, + PINECONE_AVAILABLE +) + + +class TestPineconeStore(unittest.TestCase): + """Test Pinecone store functionality.""" + + def setUp(self): + self.mock_logger = MagicMock() + self.mock_tracker = MagicMock() + + self.logger_patcher = patch('semantica.vector_store.pinecone_store.get_logger', return_value=self.mock_logger) + self.tracker_patcher = patch('semantica.vector_store.pinecone_store.get_progress_tracker', return_value=self.mock_tracker) + + self.logger_patcher.start() + self.tracker_patcher.start() + + def tearDown(self): + self.logger_patcher.stop() + self.tracker_patcher.stop() + + @patch('semantica.vector_store.pinecone_store.PineconeClientLib') + def test_initialization(self, mock_pinecone_client): + """Test PineconeStore initialization.""" + store = PineconeStore(api_key="test-key", environment="test-env") + self.assertEqual(store.api_key, "test-key") + self.assertEqual(store.environment, "test-env") + self.assertIsNone(store.client) + self.assertIsNone(store.index) + self.assertIsNone(store.search_engine) + + @patch('semantica.vector_store.pinecone_store.PineconeClientLib') + def test_connect_success(self, mock_pinecone_client): + """Test successful connection to Pinecone.""" + mock_client_instance = MagicMock() + mock_pinecone_client.return_value = mock_client_instance + + store = PineconeStore(api_key="test-key") + result = store.connect() + + self.assertTrue(result) + mock_pinecone_client.assert_called_once_with(api_key="test-key") + self.assertIsInstance(store.client, PineconeClient) + + def test_connect_no_api_key(self): + """Test connection failure without API key.""" + store = PineconeStore() + with self.assertRaises(Exception): # ValidationError + store.connect() + + @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', False) + def test_connect_pinecone_not_available(self): + """Test connection when Pinecone is not available.""" + store = PineconeStore(api_key="test-key") + with self.assertRaises(Exception): # ProcessingError + store.connect() + + @patch('semantica.vector_store.pinecone_store.PineconeClientLib') + def test_create_index(self, mock_pinecone_client): + """Test creating a Pinecone index.""" + mock_client_instance = MagicMock() + mock_index_instance = MagicMock() + mock_pinecone_client.return_value = mock_client_instance + mock_client_instance.Index.return_value = mock_index_instance + + store = PineconeStore(api_key="test-key") + store.connect() + + # Mock the client's create_index method + store.client.create_index = MagicMock() + store.client.get_index = MagicMock(return_value=mock_index_instance) + + result = store.create_index("test-index", dimension=768, metric="cosine") + + self.assertIsInstance(result, PineconeIndex) + self.assertIsInstance(store.index, PineconeIndex) + self.assertIsInstance(store.search_engine, PineconeSearch) + store.client.create_index.assert_called_once() + + @patch('semantica.vector_store.pinecone_store.PineconeClientLib') + def test_upsert_vectors(self, mock_pinecone_client): + """Test upserting vectors to Pinecone index.""" + mock_client_instance = MagicMock() + mock_index_instance = MagicMock() + mock_pinecone_client.return_value = mock_client_instance + + store = PineconeStore(api_key="test-key") + store.connect() + + # Set up index + store.index = PineconeIndex(mock_index_instance) + store.index.upsert_vectors = MagicMock(return_value={"upserted_count": 2}) + + vectors = [np.array([0.1, 0.2, 0.3]), np.array([0.4, 0.5, 0.6])] + ids = ["id1", "id2"] + metadata = [{"key": "value1"}, {"key": "value2"}] + + result = store.upsert_vectors(vectors, ids, metadata) + + self.assertEqual(result["upserted_count"], 2) + store.index.upsert_vectors.assert_called_once() + + @patch('semantica.vector_store.pinecone_store.PineconeClientLib') + def test_search_vectors(self, mock_pinecone_client): + """Test searching vectors in Pinecone index.""" + mock_client_instance = MagicMock() + mock_index_instance = MagicMock() + mock_pinecone_client.return_value = mock_client_instance + + store = PineconeStore(api_key="test-key") + store.connect() + + # Set up search engine + store.search_engine = PineconeSearch(PineconeIndex(mock_index_instance)) + store.search_engine.similarity_search = MagicMock(return_value=[ + {"id": "id1", "score": 0.9, "metadata": {"key": "value1"}} + ]) + + query_vector = np.array([0.1, 0.2, 0.3]) + results = store.search_vectors(query_vector, k=5) + + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["id"], "id1") + store.search_engine.similarity_search.assert_called_once() + + @patch('semantica.vector_store.pinecone_store.PineconeClientLib') + def test_delete_vectors(self, mock_pinecone_client): + """Test deleting vectors from Pinecone index.""" + mock_client_instance = MagicMock() + mock_index_instance = MagicMock() + mock_pinecone_client.return_value = mock_client_instance + + store = PineconeStore(api_key="test-key") + store.connect() + + # Set up index + store.index = PineconeIndex(mock_index_instance) + store.index.delete_vectors = MagicMock(return_value={"deleted": True}) + + result = store.delete_vectors(["id1", "id2"]) + + self.assertEqual(result["deleted"], True) + store.index.delete_vectors.assert_called_once_with(["id1", "id2"], "", {}) + + @patch('semantica.vector_store.pinecone_store.PineconeClientLib') + def test_fetch_vectors(self, mock_pinecone_client): + """Test fetching vectors from Pinecone index.""" + mock_client_instance = MagicMock() + mock_index_instance = MagicMock() + mock_pinecone_client.return_value = mock_client_instance + + store = PineconeStore(api_key="test-key") + store.connect() + + # Set up index + store.index = PineconeIndex(mock_index_instance) + store.index.fetch_vectors = MagicMock(return_value={ + "vectors": { + "id1": {"values": [0.1, 0.2], "metadata": {"key": "value1"}} + } + }) + + result = store.fetch_vectors(["id1"]) + + self.assertIn("id1", result["vectors"]) + store.index.fetch_vectors.assert_called_once_with(["id1"], "", {}) + + @patch('semantica.vector_store.pinecone_store.PineconeClientLib') + def test_list_indexes(self, mock_pinecone_client): + """Test listing Pinecone indexes.""" + mock_client_instance = MagicMock() + mock_index_obj = MagicMock() + mock_index_obj.name = "test-index" + mock_client_instance.list_indexes.return_value = [mock_index_obj] + mock_pinecone_client.return_value = mock_client_instance + + store = PineconeStore(api_key="test-key") + store.connect() + + result = store.list_indexes() + + self.assertEqual(result, ["test-index"]) + mock_client_instance.list_indexes.assert_called_once() + + @patch('semantica.vector_store.pinecone_store.PineconeClientLib') + def test_delete_index(self, mock_pinecone_client): + """Test deleting a Pinecone index.""" + mock_client_instance = MagicMock() + mock_pinecone_client.return_value = mock_client_instance + + store = PineconeStore(api_key="test-key") + store.connect() + + store.client.delete_index = MagicMock(return_value=True) + + result = store.delete_index("test-index") + + self.assertTrue(result) + store.client.delete_index.assert_called_once_with("test-index") + + +class TestPineconeClient(unittest.TestCase): + """Test PineconeClient wrapper.""" + + @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True) + @patch('semantica.vector_store.pinecone_store.PineconeClientLib') + def test_create_index(self, mock_pinecone_client): + """Test creating index via PineconeClient.""" + mock_client_instance = MagicMock() + mock_pinecone_client.return_value = mock_client_instance + + client = PineconeClient(mock_client_instance) + client.create_index("test-index", 768, "cosine") + + mock_client_instance.create_index.assert_called_once() + + @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True) + @patch('semantica.vector_store.pinecone_store.PineconeClientLib') + def test_list_indexes(self, mock_pinecone_client): + """Test listing indexes via PineconeClient.""" + mock_client_instance = MagicMock() + mock_index_obj = MagicMock() + mock_index_obj.name = "test-index" + mock_client_instance.list_indexes.return_value = [mock_index_obj] + mock_pinecone_client.return_value = mock_client_instance + + client = PineconeClient(mock_client_instance) + result = client.list_indexes() + + self.assertEqual(result, ["test-index"]) + + +class TestPineconeIndex(unittest.TestCase): + """Test PineconeIndex wrapper.""" + + @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True) + def test_upsert_vectors(self): + """Test upserting vectors via PineconeIndex.""" + mock_index = MagicMock() + mock_response = MagicMock() + mock_response.upserted_count = 2 + mock_index.upsert.return_value = mock_response + + index = PineconeIndex(mock_index) + result = index.upsert_vectors( + [[0.1, 0.2], [0.3, 0.4]], + ["id1", "id2"], + [{"key": "value1"}] + ) + + self.assertEqual(result["upserted_count"], 2) + mock_index.upsert.assert_called_once() + + @patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True) + def test_search_vectors(self): + """Test searching vectors via PineconeIndex.""" + mock_index = MagicMock() + mock_match = MagicMock() + mock_match.id = "id1" + mock_match.score = 0.9 + mock_match.metadata = {"key": "value1"} + mock_response = MagicMock() + mock_response.matches = [mock_match] + mock_index.query.return_value = mock_response + + index = PineconeIndex(mock_index) + result = index.search_vectors([0.1, 0.2], k=5) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["id"], "id1") + mock_index.query.assert_called_once() + + +if __name__ == '__main__': + unittest.main()