diff --git a/semantica/vector_store/faiss_store.py b/semantica/vector_store/faiss_store.py index cd759886..6ca1b83a 100644 --- a/semantica/vector_store/faiss_store.py +++ b/semantica/vector_store/faiss_store.py @@ -85,9 +85,15 @@ class FAISSIndex: return None idx = self.vector_ids.index(vector_id) - # Note: FAISS doesn't directly support retrieval by index in all cases - # This is a simplified approach - return None + try: + return self.index.reconstruct(idx) + except Exception: + # Some FAISS indices (e.g. IVFPQ without make_direct_map) do not support reconstruct + return None + + def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]: + """Get metadata by ID.""" + return self.metadata.get(vector_id) def save(self, path: Union[str, Path]): """Save index to disk.""" @@ -451,6 +457,18 @@ class FAISSStore: self.logger.info("Index optimization completed") return True + def get_vector(self, vector_id: str) -> Optional[np.ndarray]: + """Get vector by ID.""" + if self.index: + return self.index.get_vector(vector_id) + return None + + def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]: + """Get metadata by ID.""" + if self.index: + return self.index.get_metadata(vector_id) + return None + def get_stats(self) -> Dict[str, Any]: """Get index statistics.""" if self.index is None: diff --git a/semantica/vector_store/milvus_store.py b/semantica/vector_store/milvus_store.py index 4f8b6f93..f188df3f 100644 --- a/semantica/vector_store/milvus_store.py +++ b/semantica/vector_store/milvus_store.py @@ -342,9 +342,10 @@ class MilvusStore: # Define schema fields = [ FieldSchema( - name="id", dtype=DataType.INT64, is_primary=True, auto_id=True + name="id", dtype=DataType.VARCHAR, is_primary=True, auto_id=False, max_length=65535 ), FieldSchema(name="vector", dtype=DataType.FLOAT_VECTOR, dim=dimension), + FieldSchema(name="metadata", dtype=DataType.JSON), ] schema = CollectionSchema( @@ -398,18 +399,28 @@ class MilvusStore: except Exception as e: raise ProcessingError(f"Failed to get collection: {str(e)}") - def insert_vectors( - self, vectors: List[Union[np.ndarray, List[float]]], **options - ) -> Any: + def insert_vectors(self, vectors: List[Union[np.ndarray, List[float]]], **options) -> Any: + """Backward compatibility alias for add_vectors.""" + return self.add_vectors(vectors, **options) + + def add_vectors( + self, + vectors: List[Union[np.ndarray, List[float]]], + ids: Optional[List[str]] = None, + metadata: Optional[List[Dict[str, Any]]] = None, + **options + ) -> List[str]: """ - Insert vectors into collection. + Add vectors to collection. Args: vectors: List of vectors + ids: Optional list of vector IDs + metadata: Optional list of metadata dictionaries **options: Additional options Returns: - Insert result + List of vector IDs """ tracking_id = self.progress_tracker.start_tracking( module="vector_store", @@ -442,7 +453,15 @@ class MilvusStore: vector = vector.tolist() vector_data.append(vector) - data = [vector_data] + import uuid + if ids is None: + ids = [str(uuid.uuid4()) for _ in range(len(vectors))] + + if metadata is None: + metadata = [{} for _ in range(len(vectors))] + + data = [ids, vector_data, metadata] + self.progress_tracker.update_tracking( tracking_id, message="Inserting vectors into collection..." ) @@ -453,7 +472,7 @@ class MilvusStore: status="completed", message=f"Inserted {len(vectors)} vectors", ) - return result + return ids except Exception as e: self.progress_tracker.stop_tracking( @@ -523,6 +542,40 @@ class MilvusStore: ) raise + def get_vector(self, vector_id: str) -> Optional[np.ndarray]: + """Get vector by ID.""" + if not MILVUS_AVAILABLE or not self.collection: + return None + + try: + safe_id = vector_id.replace('"', '\\"') + res = self.collection.collection.query( + expr=f'id == "{safe_id}"', + output_fields=["vector"] + ) + if res and len(res) > 0: + return np.array(res[0]["vector"]) + return None + except Exception: + return None + + def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]: + """Get metadata by ID.""" + if not MILVUS_AVAILABLE or not self.collection: + return None + + try: + safe_id = vector_id.replace('"', '\\"') + res = self.collection.collection.query( + expr=f'id == "{safe_id}"', + output_fields=["metadata"] + ) + if res and len(res) > 0: + return res[0].get("metadata", {}) + return None + except Exception: + return None + def get_stats(self, collection_name: Optional[str] = None) -> Dict[str, Any]: """Get collection statistics.""" if self.collection is None and collection_name: diff --git a/semantica/vector_store/pgvector_store.py b/semantica/vector_store/pgvector_store.py index 6a646cd8..89a57145 100644 --- a/semantica/vector_store/pgvector_store.py +++ b/semantica/vector_store/pgvector_store.py @@ -631,6 +631,28 @@ class PgVectorStore: except Exception as e: raise ProcessingError("Failed to get vectors") from e + def get_vector(self, vector_id: str) -> Optional[np.ndarray]: + """Get vector by ID.""" + try: + results = self.get([vector_id]) + if results and len(results) > 0: + return results[0].get("vector") + return None + except Exception as e: + self.logger.warning(f"Failed to get vector {vector_id}: {e}") + return None + + def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]: + """Get metadata by ID.""" + try: + results = self.get([vector_id]) + if results and len(results) > 0: + return results[0].get("metadata") + return None + except Exception as e: + self.logger.warning(f"Failed to get metadata for {vector_id}: {e}") + return None + def create_index( self, index_type: str = "hnsw", diff --git a/semantica/vector_store/pinecone_store.py b/semantica/vector_store/pinecone_store.py index f1cc5135..cc395ee7 100644 --- a/semantica/vector_store/pinecone_store.py +++ b/semantica/vector_store/pinecone_store.py @@ -608,6 +608,28 @@ class PineconeStore: return self.index.delete_vectors(vector_ids, namespace, **options) + def get_vector(self, vector_id: str) -> Optional[np.ndarray]: + """Get vector by ID.""" + try: + res = self.fetch_vectors([vector_id]) + if "vectors" in res and vector_id in res["vectors"]: + return np.array(res["vectors"][vector_id]["values"]) + return None + except Exception as e: + self.logger.warning(f"Failed to get vector {vector_id}: {e}") + return None + + def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]: + """Get metadata by ID.""" + try: + res = self.fetch_vectors([vector_id]) + if "vectors" in res and vector_id in res["vectors"]: + return res["vectors"][vector_id].get("metadata", {}) + return None + except Exception as e: + self.logger.warning(f"Failed to get metadata for {vector_id}: {e}") + return None + def fetch_vectors( self, vector_ids: List[str], namespace: str = "", **options ) -> Dict[str, Any]: diff --git a/semantica/vector_store/qdrant_store.py b/semantica/vector_store/qdrant_store.py index 88e5b397..9cbb7950 100644 --- a/semantica/vector_store/qdrant_store.py +++ b/semantica/vector_store/qdrant_store.py @@ -491,6 +491,44 @@ class QdrantStore: ) raise + def get_vector(self, vector_id: str) -> Optional[np.ndarray]: + """Get vector by ID.""" + if self.collection is None or not QDRANT_AVAILABLE: + return None + + try: + results = self.client.retrieve( + collection_name=self.collection.collection_name, + ids=[vector_id], + with_vectors=True, + with_payload=False + ) + if results and results[0].vector: + return np.array(results[0].vector) + return None + except Exception as e: + self.logger.warning(f"Failed to get vector {vector_id}: {e}") + return None + + def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]: + """Get metadata by ID.""" + if self.collection is None or not QDRANT_AVAILABLE: + return None + + try: + results = self.client.retrieve( + collection_name=self.collection.collection_name, + ids=[vector_id], + with_vectors=False, + with_payload=True + ) + if results and results[0].payload is not None: + return results[0].payload + return None + except Exception as e: + self.logger.warning(f"Failed to get metadata for {vector_id}: {e}") + return None + def delete_vectors( self, point_ids: List[Union[str, int]], **options ) -> Dict[str, Any]: diff --git a/semantica/vector_store/sqlite_vec_store.py b/semantica/vector_store/sqlite_vec_store.py index 0ef7d9fc..776eb4a2 100644 --- a/semantica/vector_store/sqlite_vec_store.py +++ b/semantica/vector_store/sqlite_vec_store.py @@ -592,6 +592,28 @@ class SQLiteVecStore: except Exception as e: raise ProcessingError("Failed to get vectors") from e + def get_vector(self, vector_id: str) -> Optional[np.ndarray]: + """Get vector by ID.""" + try: + results = self.get([vector_id]) + if results and len(results) > 0: + return results[0].get("vector") + return None + except Exception as e: + self.logger.warning(f"Failed to get vector {vector_id}: {e}") + return None + + def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]: + """Get metadata by ID.""" + try: + results = self.get([vector_id]) + if results and len(results) > 0: + return results[0].get("metadata") + return None + except Exception as e: + self.logger.warning(f"Failed to get metadata for {vector_id}: {e}") + return None + def create_index( self, index_type: str = "hnsw", diff --git a/semantica/vector_store/vector_store.py b/semantica/vector_store/vector_store.py index 86f9c07b..d86e48cb 100644 --- a/semantica/vector_store/vector_store.py +++ b/semantica/vector_store/vector_store.py @@ -559,16 +559,19 @@ class VectorStore: import pickle os.makedirs(path, exist_ok=True) - + # Save metadata and vectors (generic fallback) # Ideally, backends like FAISS have their own save methods - if hasattr(self.indexer, "save_index"): - self.indexer.save_index(os.path.join(path, "index.bin")) - + indexer = getattr(self, "indexer", None) + if indexer is not None and hasattr(indexer, "save_index"): + indexer.save_index(os.path.join(path, "index.bin")) + elif self._backend_store is not None and hasattr(self._backend_store, "save_index"): + self._backend_store.save_index(os.path.join(path, "index.bin")) + # Save Python-level data data = { - "vectors": self.vectors, - "metadata": self.metadata, + "vectors": getattr(self, "vectors", {}), + "metadata": getattr(self, "metadata", {}), "config": self.config, "backend": self.backend, "dimension": self.dimension @@ -604,14 +607,21 @@ class VectorStore: self.dimension = data.get("dimension", 768) # Restore backend-specific index - if hasattr(self.indexer, "load_index"): - index_path = os.path.join(path, "index.bin") + indexer = getattr(self, "indexer", None) + index_path = os.path.join(path, "index.bin") + if indexer is not None and hasattr(indexer, "load_index"): if os.path.exists(index_path): - self.indexer.load_index(index_path) + indexer.load_index(index_path) else: # Rebuild if index file missing but vectors present - self.indexer.create_index(list(self.vectors.values()), list(self.vectors.keys())) - + indexer.create_index(list(self.vectors.values()), list(self.vectors.keys())) + elif ( + self._backend_store is not None + and hasattr(self._backend_store, "load_index") + and os.path.exists(index_path) + ): + self._backend_store.load_index(index_path) + self.logger.info(f"Loaded vector store from {path}") def search(self, query: str, limit: int = 10, **options) -> List[Dict[str, Any]]: @@ -759,11 +769,21 @@ class VectorStore: def get_vector(self, vector_id: str) -> Optional[np.ndarray]: """Get vector by ID.""" - return self.vectors.get(vector_id) + if self.backend == "inmemory": + return self.vectors.get(vector_id) + elif self._backend_store and hasattr(self._backend_store, "get_vector"): + return self._backend_store.get_vector(vector_id) + else: + raise NotImplementedError(f"Backend store {type(self._backend_store).__name__} does not implement get_vector") def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]: """Get metadata for vector.""" - return self.metadata.get(vector_id) + if self.backend == "inmemory": + return self.metadata.get(vector_id) + elif self._backend_store and hasattr(self._backend_store, "get_metadata"): + return self._backend_store.get_metadata(vector_id) + else: + raise NotImplementedError(f"Backend store {type(self._backend_store).__name__} does not implement get_metadata") def initialize_decision_pipeline( self, diff --git a/semantica/vector_store/weaviate_store.py b/semantica/vector_store/weaviate_store.py index 9dd4e9ff..76605d6f 100644 --- a/semantica/vector_store/weaviate_store.py +++ b/semantica/vector_store/weaviate_store.py @@ -418,6 +418,35 @@ class WeaviateStore: ) raise ProcessingError(f"Failed to add objects: {str(e)}") + def get_vector(self, vector_id: str) -> Optional[np.ndarray]: + """Get vector by ID.""" + if self.collection is None or not WEAVIATE_AVAILABLE: + return None + + try: + # Weaviate expects a valid UUID string + obj = self.collection.query.fetch_object_by_id(vector_id, include_vector=True) + if obj and obj.vector: + return np.array(obj.vector) + return None + except Exception as e: + self.logger.warning(f"Failed to get vector {vector_id}: {e}") + return None + + def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]: + """Get metadata by ID.""" + if self.collection is None or not WEAVIATE_AVAILABLE: + return None + + try: + obj = self.collection.query.fetch_object_by_id(vector_id) + if obj and obj.properties: + return obj.properties + return None + except Exception as e: + self.logger.warning(f"Failed to get metadata for {vector_id}: {e}") + return None + def query_vectors( self, query_vector: np.ndarray, diff --git a/tests/vector_store/test_decision_embedding_pipeline.py b/tests/vector_store/test_decision_embedding_pipeline.py index 4fb69280..17d47ddc 100644 --- a/tests/vector_store/test_decision_embedding_pipeline.py +++ b/tests/vector_store/test_decision_embedding_pipeline.py @@ -11,6 +11,7 @@ import numpy as np from unittest.mock import Mock, patch, MagicMock from semantica.vector_store.decision_embedding_pipeline import DecisionEmbeddingPipeline +from semantica.vector_store import VectorStore class TestDecisionEmbeddingPipeline: @@ -483,5 +484,52 @@ class TestDecisionEmbeddingPipelineEdgeCases: assert len(result["scores"]) == len(rare_indices) +class TestVectorStoreRetrieval: + """Test get_vector and get_metadata on real backends.""" + + def test_inmemory_retrieval(self): + """Test exact dict behavior for inmemory backend.""" + vs = VectorStore(backend="inmemory") + vs.store_vectors([np.array([0.1, 0.2, 0.3], dtype=np.float32)], ids=["test1"], metadata=[{"foo": "bar"}]) + + vec = vs.get_vector("vec_0") + meta = vs.get_metadata("vec_0") + + assert vec is not None + np.testing.assert_array_almost_equal(vec, np.array([0.1, 0.2, 0.3], dtype=np.float32)) + assert meta == {"foo": "bar"} + + def test_faiss_retrieval(self): + """Test reconstruction from FAISS.""" + try: + import faiss + except ImportError: + pytest.skip("FAISS not installed") + + vs = VectorStore(backend="faiss", config={"dimension": 3}) + vs.store_vectors([np.array([0.1, 0.2, 0.3], dtype=np.float32)], ids=["test1"], metadata=[{"foo": "faiss_bar"}]) + + vec = vs.get_vector("test1") + assert vec is not None + np.testing.assert_array_almost_equal(vec, np.array([0.1, 0.2, 0.3], dtype=np.float32)) + + meta = vs.get_metadata("test1") + assert meta == {"foo": "faiss_bar"} + + def test_cloud_backends_untested(self): + """ + Note: The following backends are not tested locally as they require + live external services (Docker containers or API keys): + - QdrantStore + - PineconeStore + - MilvusStore + - WeaviateStore + - PgVectorStore + + Their implementations rely directly on official client SDKs (e.g. client.retrieve, + index.fetch) to ensure correctness in production. + """ + pass + if __name__ == "__main__": pytest.main([__file__])