mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
fix: Apply code review fixes for Pinecone integration (PR #220)
- Fix variable shadowing in fetch_vectors (use vector_id instead of id) - Remove redundant PINECONE_AVAILABLE check in create_index - Add Pinecone imports and exports to __init__.py - Add 'pinecone' to SUPPORTED_BACKENDS in vector_store.py - Add vectorstore-pinecone dependency group to pyproject.toml - Create vectorstore-all optional dependency group - Fix duplicate MagicMock import in test_pinecone_store.py - Update test_pinecone_removal.py with explanatory comment - Update all docstrings to include Pinecone in supported backends All fixes address code review feedback and ensure proper integration.
This commit is contained in:
+11
-1
@@ -114,6 +114,16 @@ graph-all = [
|
||||
"semantica[graph-neo4j,graph-falkordb,graph-amazon-neptune]"
|
||||
]
|
||||
|
||||
# ---- Vector Store Backends ----
|
||||
vectorstore-qdrant = ["qdrant-client>=1.0.0"]
|
||||
vectorstore-weaviate = ["weaviate-client>=4.0.0"]
|
||||
vectorstore-pinecone = ["pinecone-client>=3.0.0"]
|
||||
vectorstore-milvus = ["pymilvus>=2.0.0"]
|
||||
|
||||
vectorstore-all = [
|
||||
"semantica[vectorstore-qdrant,vectorstore-weaviate,vectorstore-pinecone,vectorstore-milvus]"
|
||||
]
|
||||
|
||||
# ---- Infra / Queues / Workers ----
|
||||
infra = [
|
||||
"redis>=4.3.0",
|
||||
@@ -177,7 +187,7 @@ dev = [
|
||||
|
||||
# ---- Everything ----
|
||||
all = [
|
||||
"semantica[dev,viz,gpu,infra,cloud,monitoring,llm-all,models-huggingface,split-all,graph-all,parse-docling]"
|
||||
"semantica[dev,viz,gpu,infra,cloud,monitoring,llm-all,models-huggingface,split-all,graph-all,vectorstore-all,parse-docling]"
|
||||
]
|
||||
|
||||
# ---------------- ENTRYPOINTS ----------------
|
||||
|
||||
@@ -3,7 +3,7 @@ Vector Store Management Module
|
||||
|
||||
This module provides comprehensive vector storage and retrieval capabilities for the
|
||||
Semantica framework, including support for multiple vector store backends (FAISS,
|
||||
Weaviate, Qdrant, Milvus), hybrid search combining vector similarity and
|
||||
Weaviate, Qdrant, Pinecone, Milvus), hybrid search combining vector similarity and
|
||||
metadata filtering, metadata management, and namespace isolation.
|
||||
|
||||
Algorithms Used:
|
||||
@@ -73,7 +73,7 @@ Dependencies:
|
||||
- pymilvus
|
||||
|
||||
Key Features:
|
||||
- Multi-backend vector store support (FAISS, Weaviate, Qdrant, Milvus)
|
||||
- Multi-backend vector store support (FAISS, Weaviate, Qdrant, Pinecone, Milvus)
|
||||
- Vector indexing and similarity search
|
||||
- Metadata indexing and filtering
|
||||
- Hybrid search combining vector and metadata queries
|
||||
@@ -91,6 +91,7 @@ Main Classes:
|
||||
- FAISSStore: FAISS integration for local vector storage
|
||||
- WeaviateStore: Weaviate vector database integration
|
||||
- QdrantStore: Qdrant vector database integration
|
||||
- PineconeStore: Pinecone vector database integration
|
||||
- MilvusStore: Milvus vector database integration
|
||||
- HybridSearch: Hybrid vector and metadata search
|
||||
- MetadataStore: Metadata indexing and management
|
||||
@@ -145,6 +146,7 @@ from .methods import (
|
||||
)
|
||||
from .milvus_store import MilvusStore, MilvusClient, MilvusCollection, MilvusSearch
|
||||
from .namespace_manager import Namespace, NamespaceManager
|
||||
from .pinecone_store import PineconeStore, PineconeClient, PineconeIndex, PineconeSearch
|
||||
from .qdrant_store import QdrantStore, QdrantClient, QdrantCollection, QdrantSearch
|
||||
from .registry import MethodRegistry, method_registry
|
||||
from .vector_store import VectorIndexer, VectorManager, VectorRetriever, VectorStore
|
||||
@@ -181,6 +183,11 @@ __all__ = [
|
||||
"MilvusClient",
|
||||
"MilvusCollection",
|
||||
"MilvusSearch",
|
||||
# Pinecone
|
||||
"PineconeStore",
|
||||
"PineconeClient",
|
||||
"PineconeIndex",
|
||||
"PineconeSearch",
|
||||
# Hybrid search
|
||||
"HybridSearch",
|
||||
"MetadataFilter",
|
||||
|
||||
@@ -0,0 +1,639 @@
|
||||
"""
|
||||
Pinecone Store Module
|
||||
|
||||
This module provides Pinecone vector database integration for vector storage and
|
||||
similarity search in the Semantica framework, supporting managed vector database
|
||||
service with serverless and pod-based indexes, namespace isolation, and efficient
|
||||
vector operations with metadata filtering.
|
||||
|
||||
Key Features:
|
||||
- Serverless and Pod-based index management
|
||||
- Namespace isolation for multi-tenant support
|
||||
- Metadata filtering during search
|
||||
- Batch operations for efficient data loading
|
||||
- Index creation, deletion, and listing
|
||||
- Optional dependency handling
|
||||
|
||||
Main Classes:
|
||||
- PineconeStore: Main Pinecone store for vector operations
|
||||
- PineconeClient: Pinecone client wrapper
|
||||
- PineconeIndex: Index wrapper with operations
|
||||
- PineconeSearch: Search operations and filtering
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.vector_store import PineconeStore
|
||||
>>> store = PineconeStore(api_key="your-api-key")
|
||||
>>> store.connect()
|
||||
>>> store.create_index("my-index", dimension=768)
|
||||
>>> store.upsert_vectors(vectors, ids, metadata=metadata)
|
||||
>>> results = store.search_vectors(query_vector, k=10, filter={"category": "science"})
|
||||
>>> stats = store.get_stats()
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
# Optional Pinecone import
|
||||
try:
|
||||
from pinecone import Pinecone as PineconeClientLib, ServerlessSpec, PodSpec
|
||||
|
||||
PINECONE_AVAILABLE = True
|
||||
except (ImportError, OSError):
|
||||
PINECONE_AVAILABLE = False
|
||||
PineconeClientLib = None
|
||||
ServerlessSpec = None
|
||||
PodSpec = None
|
||||
|
||||
|
||||
class PineconeClient:
|
||||
"""Pinecone client wrapper."""
|
||||
|
||||
def __init__(self, client: Any):
|
||||
"""Initialize Pinecone client wrapper."""
|
||||
self.client = client
|
||||
self.logger = get_logger("pinecone_client")
|
||||
|
||||
def create_index(
|
||||
self,
|
||||
index_name: str,
|
||||
dimension: int,
|
||||
metric: str = "cosine",
|
||||
spec: Optional[Dict[str, Any]] = None,
|
||||
**options,
|
||||
) -> bool:
|
||||
"""Create an index in Pinecone."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
# Default to serverless spec if not provided
|
||||
if spec is None:
|
||||
spec = ServerlessSpec(cloud="aws", region="us-east-1")
|
||||
|
||||
# Map metric names
|
||||
metric_map = {
|
||||
"cosine": "cosine",
|
||||
"euclidean": "euclidean_distance",
|
||||
"dot": "dotproduct",
|
||||
}
|
||||
pinecone_metric = metric_map.get(metric.lower(), "cosine")
|
||||
|
||||
self.client.create_index(
|
||||
name=index_name,
|
||||
dimension=dimension,
|
||||
metric=pinecone_metric,
|
||||
spec=spec,
|
||||
**options,
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to create index: {str(e)}")
|
||||
|
||||
def delete_index(self, index_name: str) -> bool:
|
||||
"""Delete an index from Pinecone."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
self.client.delete_index(index_name)
|
||||
return True
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to delete index: {str(e)}")
|
||||
|
||||
def list_indexes(self) -> List[str]:
|
||||
"""List available indexes."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
indexes = self.client.list_indexes()
|
||||
return [index.name for index in indexes]
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to list indexes: {str(e)}")
|
||||
|
||||
def get_index(self, index_name: str) -> Any:
|
||||
"""Get index object."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
return self.client.Index(index_name)
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to get index: {str(e)}")
|
||||
|
||||
|
||||
class PineconeIndex:
|
||||
"""Pinecone index wrapper."""
|
||||
|
||||
def __init__(self, index: Any):
|
||||
"""Initialize Pinecone index wrapper."""
|
||||
self.index = index
|
||||
self.logger = get_logger("pinecone_index")
|
||||
|
||||
def upsert_vectors(
|
||||
self,
|
||||
vectors: List[List[float]],
|
||||
ids: List[str],
|
||||
metadata: Optional[List[Dict[str, Any]]] = None,
|
||||
namespace: str = "",
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""Upsert vectors to index."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
# Prepare vectors for upsert
|
||||
upsert_data = []
|
||||
for i, (vector, vector_id) in enumerate(zip(vectors, ids)):
|
||||
vector_dict = {"id": vector_id, "values": vector}
|
||||
if metadata and i < len(metadata):
|
||||
vector_dict["metadata"] = metadata[i]
|
||||
upsert_data.append(vector_dict)
|
||||
|
||||
response = self.index.upsert(
|
||||
vectors=upsert_data, namespace=namespace, **options
|
||||
)
|
||||
return {"upserted_count": response.upserted_count}
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to upsert vectors: {str(e)}")
|
||||
|
||||
def search_vectors(
|
||||
self,
|
||||
query_vector: List[float],
|
||||
k: int = 10,
|
||||
filter: Optional[Dict[str, Any]] = None,
|
||||
namespace: str = "",
|
||||
**options,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search for similar vectors."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
response = self.index.query(
|
||||
vector=query_vector,
|
||||
top_k=k,
|
||||
filter=filter,
|
||||
namespace=namespace,
|
||||
include_metadata=True,
|
||||
include_values=False,
|
||||
**options,
|
||||
)
|
||||
|
||||
results = []
|
||||
for match in response.matches:
|
||||
results.append(
|
||||
{
|
||||
"id": match.id,
|
||||
"score": match.score,
|
||||
"metadata": match.metadata or {},
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to search vectors: {str(e)}")
|
||||
|
||||
def delete_vectors(
|
||||
self, vector_ids: List[str], namespace: str = "", **options
|
||||
) -> Dict[str, Any]:
|
||||
"""Delete vectors from index."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
response = self.index.delete(ids=vector_ids, namespace=namespace, **options)
|
||||
return {"deleted": True}
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to delete vectors: {str(e)}")
|
||||
|
||||
def fetch_vectors(
|
||||
self, vector_ids: List[str], namespace: str = "", **options
|
||||
) -> Dict[str, Any]:
|
||||
"""Fetch vectors by ID."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
response = self.index.fetch(ids=vector_ids, namespace=namespace, **options)
|
||||
return {
|
||||
"vectors": {
|
||||
vector_id: {
|
||||
"values": vector.values,
|
||||
"metadata": vector.metadata or {},
|
||||
}
|
||||
for vector_id, vector in response.vectors.items()
|
||||
}
|
||||
}
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to fetch vectors: {str(e)}")
|
||||
|
||||
def describe_index_stats(self, **options) -> Dict[str, Any]:
|
||||
"""Get index statistics."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
stats = self.index.describe_index_stats(**options)
|
||||
return {
|
||||
"dimension": stats.dimension,
|
||||
"index_fullness": stats.index_fullness,
|
||||
"total_vector_count": stats.total_vector_count,
|
||||
"namespaces": stats.namespaces,
|
||||
}
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to get index stats: {str(e)}")
|
||||
|
||||
|
||||
class PineconeSearch:
|
||||
"""Pinecone search operations."""
|
||||
|
||||
def __init__(self, index: PineconeIndex):
|
||||
"""Initialize Pinecone search."""
|
||||
self.index = index
|
||||
self.logger = get_logger("pinecone_search")
|
||||
|
||||
def similarity_search(
|
||||
self,
|
||||
query_vector: np.ndarray,
|
||||
limit: int = 10,
|
||||
filter: Optional[Dict[str, Any]] = None,
|
||||
namespace: str = "",
|
||||
**options,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Perform similarity search.
|
||||
|
||||
Args:
|
||||
query_vector: Query vector
|
||||
limit: Number of results
|
||||
filter: Metadata filter
|
||||
namespace: Namespace to search in
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
List of search results
|
||||
"""
|
||||
return self.index.search_vectors(
|
||||
query_vector.tolist(), limit, filter, namespace, **options
|
||||
)
|
||||
|
||||
|
||||
class PineconeStore:
|
||||
"""
|
||||
Pinecone store for vector storage and similarity search.
|
||||
|
||||
• Pinecone connection and authentication
|
||||
• Index and namespace management
|
||||
• Vector storage and retrieval
|
||||
• Similarity search and filtering
|
||||
• Performance optimization
|
||||
• Error handling and recovery
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: Optional[str] = None,
|
||||
environment: Optional[str] = None,
|
||||
**config,
|
||||
):
|
||||
"""Initialize Pinecone store."""
|
||||
self.logger = get_logger("pinecone_store")
|
||||
self.config = config
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
# Ensure progress tracker is enabled
|
||||
if not self.progress_tracker.enabled:
|
||||
self.progress_tracker.enabled = True
|
||||
|
||||
self.api_key = api_key or config.get("api_key")
|
||||
self.environment = environment or config.get("environment")
|
||||
|
||||
self.client: Optional[PineconeClient] = None
|
||||
self.index: Optional[PineconeIndex] = None
|
||||
self.search_engine: Optional[PineconeSearch] = None
|
||||
|
||||
# Check Pinecone availability
|
||||
if not PINECONE_AVAILABLE:
|
||||
self.logger.warning(
|
||||
"Pinecone not available. Install with: pip install pinecone-client"
|
||||
)
|
||||
|
||||
def connect(self, **kwargs) -> bool:
|
||||
"""
|
||||
Connect to Pinecone service.
|
||||
|
||||
Args:
|
||||
**kwargs: Connection options
|
||||
|
||||
Returns:
|
||||
True if connected successfully
|
||||
"""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError(
|
||||
"Pinecone is not available. Install it with: pip install pinecone-client"
|
||||
)
|
||||
|
||||
api_key = kwargs.get("api_key") or self.api_key
|
||||
if not api_key:
|
||||
raise ValidationError("Pinecone API key is required")
|
||||
|
||||
try:
|
||||
pinecone_client = PineconeClientLib(api_key=api_key, **kwargs)
|
||||
self.client = PineconeClient(pinecone_client)
|
||||
|
||||
self.logger.info("Connected to Pinecone")
|
||||
return True
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to connect to Pinecone: {str(e)}")
|
||||
|
||||
def create_index(
|
||||
self,
|
||||
index_name: str,
|
||||
dimension: int,
|
||||
metric: str = "cosine",
|
||||
spec: Optional[Dict[str, Any]] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Create a Pinecone index.
|
||||
|
||||
Args:
|
||||
index_name: Name of the index
|
||||
dimension: Vector dimension
|
||||
metric: Distance metric ("cosine", "euclidean", "dot")
|
||||
spec: Index specification (ServerlessSpec or PodSpec)
|
||||
**kwargs: Additional options
|
||||
|
||||
Returns:
|
||||
PineconeIndex instance
|
||||
"""
|
||||
if self.client is None:
|
||||
self.connect()
|
||||
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
# Create index spec if not provided
|
||||
if spec is None:
|
||||
spec = ServerlessSpec(cloud="aws", region="us-east-1")
|
||||
|
||||
self.client.create_index(index_name, dimension, metric, spec, **kwargs)
|
||||
|
||||
# Get the index
|
||||
pinecone_index = self.client.get_index(index_name)
|
||||
self.index = PineconeIndex(pinecone_index)
|
||||
self.search_engine = PineconeSearch(self.index)
|
||||
|
||||
self.logger.info(f"Created Pinecone index: {index_name}")
|
||||
return self.index
|
||||
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to create index: {str(e)}")
|
||||
|
||||
def get_index(self, index_name: str) -> PineconeIndex:
|
||||
"""
|
||||
Get existing index.
|
||||
|
||||
Args:
|
||||
index_name: Name of the index
|
||||
|
||||
Returns:
|
||||
PineconeIndex instance
|
||||
"""
|
||||
if self.client is None:
|
||||
self.connect()
|
||||
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
pinecone_index = self.client.get_index(index_name)
|
||||
self.index = PineconeIndex(pinecone_index)
|
||||
self.search_engine = PineconeSearch(self.index)
|
||||
return self.index
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to get index: {str(e)}")
|
||||
|
||||
def delete_index(self, index_name: str) -> bool:
|
||||
"""
|
||||
Delete an index.
|
||||
|
||||
Args:
|
||||
index_name: Name of the index to delete
|
||||
|
||||
Returns:
|
||||
True if deleted successfully
|
||||
"""
|
||||
if self.client is None:
|
||||
self.connect()
|
||||
|
||||
return self.client.delete_index(index_name)
|
||||
|
||||
def list_indexes(self) -> List[str]:
|
||||
"""
|
||||
List available indexes.
|
||||
|
||||
Returns:
|
||||
List of index names
|
||||
"""
|
||||
if self.client is None:
|
||||
self.connect()
|
||||
|
||||
return self.client.list_indexes()
|
||||
|
||||
def upsert_vectors(
|
||||
self,
|
||||
vectors: List[Any],
|
||||
ids: List[str],
|
||||
metadata: Optional[List[Dict[str, Any]]] = None,
|
||||
namespace: str = "",
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Upsert vectors to index.
|
||||
|
||||
Args:
|
||||
vectors: List of vectors
|
||||
ids: Vector IDs
|
||||
metadata: Optional metadata for each vector
|
||||
namespace: Namespace to upsert into
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Upsert response
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="vector_store",
|
||||
submodule="PineconeStore",
|
||||
message=f"Upserting {len(vectors)} vectors to Pinecone index",
|
||||
)
|
||||
|
||||
try:
|
||||
if self.index is None:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message="Index not initialized"
|
||||
)
|
||||
raise ProcessingError(
|
||||
"Index not initialized. Call create_index() or get_index() first."
|
||||
)
|
||||
|
||||
if not PINECONE_AVAILABLE:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message="Pinecone not available"
|
||||
)
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Preparing vectors..."
|
||||
)
|
||||
|
||||
# Convert vectors to list format
|
||||
vector_list = []
|
||||
for vector in vectors:
|
||||
if isinstance(vector, np.ndarray):
|
||||
vector_list.append(vector.tolist())
|
||||
else:
|
||||
vector_list.append(list(vector))
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Upserting vectors to index..."
|
||||
)
|
||||
result = self.index.upsert_vectors(
|
||||
vector_list, ids, metadata, namespace, **options
|
||||
)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Upserted {len(vectors)} vectors",
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise ProcessingError(f"Failed to upsert vectors: {str(e)}")
|
||||
|
||||
def search_vectors(
|
||||
self,
|
||||
query_vector: Any,
|
||||
k: int = 10,
|
||||
filter: Optional[Dict[str, Any]] = None,
|
||||
namespace: str = "",
|
||||
**options,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search vectors in index.
|
||||
|
||||
Args:
|
||||
query_vector: Query vector
|
||||
k: Number of results
|
||||
filter: Metadata filter
|
||||
namespace: Namespace to search in
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
List of search results
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="vector_store",
|
||||
submodule="PineconeStore",
|
||||
message=f"Searching for {k} similar vectors in Pinecone",
|
||||
)
|
||||
|
||||
try:
|
||||
if self.search_engine is None:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message="Index not initialized"
|
||||
)
|
||||
raise ProcessingError(
|
||||
"Index not initialized. Call create_index() or get_index() first."
|
||||
)
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Performing similarity search..."
|
||||
)
|
||||
|
||||
# Convert query vector to list
|
||||
if isinstance(query_vector, np.ndarray):
|
||||
query_vector = query_vector.tolist()
|
||||
else:
|
||||
query_vector = list(query_vector)
|
||||
|
||||
results = self.search_engine.similarity_search(
|
||||
np.array(query_vector), k, filter, namespace, **options
|
||||
)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Found {len(results)} similar vectors",
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise
|
||||
|
||||
def delete_vectors(
|
||||
self, vector_ids: List[str], namespace: str = "", **options
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Delete vectors from index.
|
||||
|
||||
Args:
|
||||
vector_ids: Vector IDs to delete
|
||||
namespace: Namespace to delete from
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Delete response
|
||||
"""
|
||||
if self.index is None:
|
||||
raise ProcessingError(
|
||||
"Index not initialized. Call create_index() or get_index() first."
|
||||
)
|
||||
|
||||
return self.index.delete_vectors(vector_ids, namespace, **options)
|
||||
|
||||
def fetch_vectors(
|
||||
self, vector_ids: List[str], namespace: str = "", **options
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch vectors by ID.
|
||||
|
||||
Args:
|
||||
vector_ids: Vector IDs to fetch
|
||||
namespace: Namespace to fetch from
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Fetch response
|
||||
"""
|
||||
if self.index is None:
|
||||
raise ProcessingError(
|
||||
"Index not initialized. Call create_index() or get_index() first."
|
||||
)
|
||||
|
||||
return self.index.fetch_vectors(vector_ids, namespace, **options)
|
||||
|
||||
def get_stats(self, **options) -> Dict[str, Any]:
|
||||
"""Get index statistics."""
|
||||
if self.index is None:
|
||||
raise ProcessingError(
|
||||
"Index not initialized. Call create_index() or get_index() first."
|
||||
)
|
||||
|
||||
return self.index.describe_index_stats(**options)
|
||||
@@ -60,7 +60,7 @@ class VectorStore:
|
||||
• Provides vector store operations
|
||||
"""
|
||||
|
||||
SUPPORTED_BACKENDS = {"faiss", "weaviate", "qdrant", "milvus", "inmemory"}
|
||||
SUPPORTED_BACKENDS = {"faiss", "weaviate", "qdrant", "milvus", "pinecone", "inmemory"}
|
||||
|
||||
def __init__(self, backend="faiss", config=None, max_workers: int = 6, **kwargs):
|
||||
"""Initialize vector store."""
|
||||
|
||||
@@ -1,62 +1,3 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Ensure semantica is in path
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')))
|
||||
|
||||
from semantica.vector_store.vector_store import VectorStore
|
||||
from semantica.vector_store.registry import method_registry
|
||||
from semantica.vector_store.config import vector_store_config
|
||||
|
||||
class TestPineconeRemoval(unittest.TestCase):
|
||||
"""Verify that Pinecone has been completely removed from the system."""
|
||||
|
||||
def test_pinecone_backend_rejected(self):
|
||||
"""Test that initializing VectorStore with backend='pinecone' raises an error."""
|
||||
with self.assertRaises(ValueError) as context:
|
||||
VectorStore(backend="pinecone")
|
||||
|
||||
# The error message might be generic "Unknown backend" or specific.
|
||||
# We just want to ensure it fails.
|
||||
self.assertTrue("pinecone" in str(context.exception).lower() or "unknown" in str(context.exception).lower())
|
||||
|
||||
def test_registry_clean(self):
|
||||
"""Test that no Pinecone methods are registered."""
|
||||
# Check all task types
|
||||
task_types = ["store", "search", "index", "hybrid_search", "metadata", "namespace"]
|
||||
|
||||
for task in task_types:
|
||||
methods = method_registry.list_all(task)
|
||||
# Flatten if it's a dict
|
||||
if isinstance(methods, dict):
|
||||
method_names = methods.get(task, [])
|
||||
else:
|
||||
method_names = methods
|
||||
|
||||
for name in method_names:
|
||||
self.assertNotIn("pinecone", name.lower(), f"Found pinecone reference in registry task {task}: {name}")
|
||||
|
||||
def test_config_clean(self):
|
||||
"""Test that configuration does not contain Pinecone keys."""
|
||||
config = vector_store_config.get_all()
|
||||
|
||||
for key in config.keys():
|
||||
self.assertNotIn("pinecone", key.lower(), f"Found pinecone key in config: {key}")
|
||||
|
||||
def test_stores_existence(self):
|
||||
"""Verify that other stores exist but PineconeStore does not."""
|
||||
try:
|
||||
from semantica.vector_store import faiss_store
|
||||
from semantica.vector_store import weaviate_store
|
||||
from semantica.vector_store import qdrant_store
|
||||
from semantica.vector_store import milvus_store
|
||||
except ImportError as e:
|
||||
self.fail(f"Failed to import a required store: {e}")
|
||||
|
||||
with self.assertRaises(ImportError):
|
||||
from semantica.vector_store import pinecone_store
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
# This test file has been updated as Pinecone support has been re-added to Semantica.
|
||||
# Pinecone is now a supported vector store backend (PR #220).
|
||||
# See test_pinecone_store.py for Pinecone-specific tests.
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
import numpy as np
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Ensure semantica is in path
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')))
|
||||
|
||||
from semantica.vector_store.pinecone_store import (
|
||||
PineconeStore,
|
||||
PineconeClient,
|
||||
PineconeIndex,
|
||||
PineconeSearch,
|
||||
PINECONE_AVAILABLE
|
||||
)
|
||||
|
||||
|
||||
class TestPineconeStore(unittest.TestCase):
|
||||
"""Test Pinecone store functionality."""
|
||||
|
||||
def setUp(self):
|
||||
self.mock_logger = MagicMock()
|
||||
self.mock_tracker = MagicMock()
|
||||
|
||||
self.logger_patcher = patch('semantica.vector_store.pinecone_store.get_logger', return_value=self.mock_logger)
|
||||
self.tracker_patcher = patch('semantica.vector_store.pinecone_store.get_progress_tracker', return_value=self.mock_tracker)
|
||||
|
||||
self.logger_patcher.start()
|
||||
self.tracker_patcher.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.logger_patcher.stop()
|
||||
self.tracker_patcher.stop()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_initialization(self, mock_pinecone_client):
|
||||
"""Test PineconeStore initialization."""
|
||||
store = PineconeStore(api_key="test-key", environment="test-env")
|
||||
self.assertEqual(store.api_key, "test-key")
|
||||
self.assertEqual(store.environment, "test-env")
|
||||
self.assertIsNone(store.client)
|
||||
self.assertIsNone(store.index)
|
||||
self.assertIsNone(store.search_engine)
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_connect_success(self, mock_pinecone_client):
|
||||
"""Test successful connection to Pinecone."""
|
||||
mock_client_instance = MagicMock()
|
||||
mock_pinecone_client.return_value = mock_client_instance
|
||||
|
||||
store = PineconeStore(api_key="test-key")
|
||||
result = store.connect()
|
||||
|
||||
self.assertTrue(result)
|
||||
mock_pinecone_client.assert_called_once_with(api_key="test-key")
|
||||
self.assertIsInstance(store.client, PineconeClient)
|
||||
|
||||
def test_connect_no_api_key(self):
|
||||
"""Test connection failure without API key."""
|
||||
store = PineconeStore()
|
||||
with self.assertRaises(Exception): # ValidationError
|
||||
store.connect()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', False)
|
||||
def test_connect_pinecone_not_available(self):
|
||||
"""Test connection when Pinecone is not available."""
|
||||
store = PineconeStore(api_key="test-key")
|
||||
with self.assertRaises(Exception): # ProcessingError
|
||||
store.connect()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_create_index(self, mock_pinecone_client):
|
||||
"""Test creating a Pinecone index."""
|
||||
mock_client_instance = MagicMock()
|
||||
mock_index_instance = MagicMock()
|
||||
mock_pinecone_client.return_value = mock_client_instance
|
||||
mock_client_instance.Index.return_value = mock_index_instance
|
||||
|
||||
store = PineconeStore(api_key="test-key")
|
||||
store.connect()
|
||||
|
||||
# Mock the client's create_index method
|
||||
store.client.create_index = MagicMock()
|
||||
store.client.get_index = MagicMock(return_value=mock_index_instance)
|
||||
|
||||
result = store.create_index("test-index", dimension=768, metric="cosine")
|
||||
|
||||
self.assertIsInstance(result, PineconeIndex)
|
||||
self.assertIsInstance(store.index, PineconeIndex)
|
||||
self.assertIsInstance(store.search_engine, PineconeSearch)
|
||||
store.client.create_index.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_upsert_vectors(self, mock_pinecone_client):
|
||||
"""Test upserting vectors to Pinecone index."""
|
||||
mock_client_instance = MagicMock()
|
||||
mock_index_instance = MagicMock()
|
||||
mock_pinecone_client.return_value = mock_client_instance
|
||||
|
||||
store = PineconeStore(api_key="test-key")
|
||||
store.connect()
|
||||
|
||||
# Set up index
|
||||
store.index = PineconeIndex(mock_index_instance)
|
||||
store.index.upsert_vectors = MagicMock(return_value={"upserted_count": 2})
|
||||
|
||||
vectors = [np.array([0.1, 0.2, 0.3]), np.array([0.4, 0.5, 0.6])]
|
||||
ids = ["id1", "id2"]
|
||||
metadata = [{"key": "value1"}, {"key": "value2"}]
|
||||
|
||||
result = store.upsert_vectors(vectors, ids, metadata)
|
||||
|
||||
self.assertEqual(result["upserted_count"], 2)
|
||||
store.index.upsert_vectors.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_search_vectors(self, mock_pinecone_client):
|
||||
"""Test searching vectors in Pinecone index."""
|
||||
mock_client_instance = MagicMock()
|
||||
mock_index_instance = MagicMock()
|
||||
mock_pinecone_client.return_value = mock_client_instance
|
||||
|
||||
store = PineconeStore(api_key="test-key")
|
||||
store.connect()
|
||||
|
||||
# Set up search engine
|
||||
store.search_engine = PineconeSearch(PineconeIndex(mock_index_instance))
|
||||
store.search_engine.similarity_search = MagicMock(return_value=[
|
||||
{"id": "id1", "score": 0.9, "metadata": {"key": "value1"}}
|
||||
])
|
||||
|
||||
query_vector = np.array([0.1, 0.2, 0.3])
|
||||
results = store.search_vectors(query_vector, k=5)
|
||||
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "id1")
|
||||
store.search_engine.similarity_search.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_delete_vectors(self, mock_pinecone_client):
|
||||
"""Test deleting vectors from Pinecone index."""
|
||||
mock_client_instance = MagicMock()
|
||||
mock_index_instance = MagicMock()
|
||||
mock_pinecone_client.return_value = mock_client_instance
|
||||
|
||||
store = PineconeStore(api_key="test-key")
|
||||
store.connect()
|
||||
|
||||
# Set up index
|
||||
store.index = PineconeIndex(mock_index_instance)
|
||||
store.index.delete_vectors = MagicMock(return_value={"deleted": True})
|
||||
|
||||
result = store.delete_vectors(["id1", "id2"])
|
||||
|
||||
self.assertEqual(result["deleted"], True)
|
||||
store.index.delete_vectors.assert_called_once_with(["id1", "id2"], "", {})
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_fetch_vectors(self, mock_pinecone_client):
|
||||
"""Test fetching vectors from Pinecone index."""
|
||||
mock_client_instance = MagicMock()
|
||||
mock_index_instance = MagicMock()
|
||||
mock_pinecone_client.return_value = mock_client_instance
|
||||
|
||||
store = PineconeStore(api_key="test-key")
|
||||
store.connect()
|
||||
|
||||
# Set up index
|
||||
store.index = PineconeIndex(mock_index_instance)
|
||||
store.index.fetch_vectors = MagicMock(return_value={
|
||||
"vectors": {
|
||||
"id1": {"values": [0.1, 0.2], "metadata": {"key": "value1"}}
|
||||
}
|
||||
})
|
||||
|
||||
result = store.fetch_vectors(["id1"])
|
||||
|
||||
self.assertIn("id1", result["vectors"])
|
||||
store.index.fetch_vectors.assert_called_once_with(["id1"], "", {})
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_list_indexes(self, mock_pinecone_client):
|
||||
"""Test listing Pinecone indexes."""
|
||||
mock_client_instance = MagicMock()
|
||||
mock_index_obj = MagicMock()
|
||||
mock_index_obj.name = "test-index"
|
||||
mock_client_instance.list_indexes.return_value = [mock_index_obj]
|
||||
mock_pinecone_client.return_value = mock_client_instance
|
||||
|
||||
store = PineconeStore(api_key="test-key")
|
||||
store.connect()
|
||||
|
||||
result = store.list_indexes()
|
||||
|
||||
self.assertEqual(result, ["test-index"])
|
||||
mock_client_instance.list_indexes.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_delete_index(self, mock_pinecone_client):
|
||||
"""Test deleting a Pinecone index."""
|
||||
mock_client_instance = MagicMock()
|
||||
mock_pinecone_client.return_value = mock_client_instance
|
||||
|
||||
store = PineconeStore(api_key="test-key")
|
||||
store.connect()
|
||||
|
||||
store.client.delete_index = MagicMock(return_value=True)
|
||||
|
||||
result = store.delete_index("test-index")
|
||||
|
||||
self.assertTrue(result)
|
||||
store.client.delete_index.assert_called_once_with("test-index")
|
||||
|
||||
|
||||
class TestPineconeClient(unittest.TestCase):
|
||||
"""Test PineconeClient wrapper."""
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_create_index(self, mock_pinecone_client):
|
||||
"""Test creating index via PineconeClient."""
|
||||
mock_client_instance = MagicMock()
|
||||
mock_pinecone_client.return_value = mock_client_instance
|
||||
|
||||
client = PineconeClient(mock_client_instance)
|
||||
client.create_index("test-index", 768, "cosine")
|
||||
|
||||
mock_client_instance.create_index.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_list_indexes(self, mock_pinecone_client):
|
||||
"""Test listing indexes via PineconeClient."""
|
||||
mock_client_instance = MagicMock()
|
||||
mock_index_obj = MagicMock()
|
||||
mock_index_obj.name = "test-index"
|
||||
mock_client_instance.list_indexes.return_value = [mock_index_obj]
|
||||
mock_pinecone_client.return_value = mock_client_instance
|
||||
|
||||
client = PineconeClient(mock_client_instance)
|
||||
result = client.list_indexes()
|
||||
|
||||
self.assertEqual(result, ["test-index"])
|
||||
|
||||
|
||||
class TestPineconeIndex(unittest.TestCase):
|
||||
"""Test PineconeIndex wrapper."""
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
def test_upsert_vectors(self):
|
||||
"""Test upserting vectors via PineconeIndex."""
|
||||
mock_index = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.upserted_count = 2
|
||||
mock_index.upsert.return_value = mock_response
|
||||
|
||||
index = PineconeIndex(mock_index)
|
||||
result = index.upsert_vectors(
|
||||
[[0.1, 0.2], [0.3, 0.4]],
|
||||
["id1", "id2"],
|
||||
[{"key": "value1"}]
|
||||
)
|
||||
|
||||
self.assertEqual(result["upserted_count"], 2)
|
||||
mock_index.upsert.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
def test_search_vectors(self):
|
||||
"""Test searching vectors via PineconeIndex."""
|
||||
mock_index = MagicMock()
|
||||
mock_match = MagicMock()
|
||||
mock_match.id = "id1"
|
||||
mock_match.score = 0.9
|
||||
mock_match.metadata = {"key": "value1"}
|
||||
mock_response = MagicMock()
|
||||
mock_response.matches = [mock_match]
|
||||
mock_index.query.return_value = mock_response
|
||||
|
||||
index = PineconeIndex(mock_index)
|
||||
result = index.search_vectors([0.1, 0.2], k=5)
|
||||
|
||||
self.assertEqual(len(result), 1)
|
||||
self.assertEqual(result[0]["id"], "id1")
|
||||
mock_index.query.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user