Merge branch 'main' into fix/845-standardize-search-vectors-output-schema

README.md

@@ -2,6 +2,8 @@
 
 <img src="Semantica Logo.png" alt="Semantica" width="420"/>
 
+<a href="https://trendshift.io/repositories/18986?utm_source=repository-badge&amp;utm_medium=badge&amp;utm_campaign=badge-repository-18986" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/18986" alt="semantica-agi%2Fsemantica | Trendshift" width="250" height="55"/></a>
+
 ### Graph-Native Infrastructure for Context and Accountable AI Systems
 
 #### *The Open Source Palantir for AI Agents*

semantica/vector_store/faiss_store.py

@@ -85,9 +85,35 @@ class FAISSIndex:
             return None
 
         idx = self.vector_ids.index(vector_id)
-        # Note: FAISS doesn't directly support retrieval by index in all cases
-        # This is a simplified approach
-        return None
+        reconstruct = getattr(self.index, "reconstruct", None)
+        if not callable(reconstruct):
+            return None
+
+        try:
+            return np.asarray(reconstruct(idx), dtype=np.float32)
+        except NotImplementedError:
+            return None
+        except RuntimeError as exc:
+            message = str(exc).casefold()
+            unsupported_errors = (
+                "reconstruct not implemented",
+                "reconstruct_from_offset not implemented",
+            )
+            if any(error in message for error in unsupported_errors):
+                return None
+            if "direct map not initialized" in message:
+                # IVF-family indices (e.g. IndexIVFFlat) support exact reconstruction
+                # but need their DirectMap built once before reconstruct() works.
+                make_direct_map = getattr(self.index, "make_direct_map", None)
+                if not callable(make_direct_map):
+                    return None
+                make_direct_map()
+                return np.asarray(reconstruct(idx), dtype=np.float32)
+            raise
+
+    def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+        """Get metadata by ID."""
+        return self.metadata.get(vector_id)
 
     def save(self, path: Union[str, Path]):
         """Save index to disk."""
@@ -452,6 +478,18 @@ class FAISSStore:
         self.logger.info("Index optimization completed")
         return True
 
+    def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+        """Get vector by ID."""
+        if self.index:
+            return self.index.get_vector(vector_id)
+        return None
+
+    def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+        """Get metadata by ID."""
+        if self.index:
+            return self.index.get_metadata(vector_id)
+        return None
+
     def get_stats(self) -> Dict[str, Any]:
         """Get index statistics."""
         if self.index is None:

semantica/vector_store/milvus_store.py

@@ -346,9 +346,10 @@ class MilvusStore:
             # Define schema
             fields = [
                 FieldSchema(
-                    name="id", dtype=DataType.INT64, is_primary=True, auto_id=True
+                    name="id", dtype=DataType.VARCHAR, is_primary=True, auto_id=False, max_length=65535
                 ),
                 FieldSchema(name="vector", dtype=DataType.FLOAT_VECTOR, dim=dimension),
+                FieldSchema(name="metadata", dtype=DataType.JSON),
             ]
 
             schema = CollectionSchema(
@@ -402,18 +403,28 @@ class MilvusStore:
         except Exception as e:
             raise ProcessingError(f"Failed to get collection: {str(e)}")
 
-    def insert_vectors(
-        self, vectors: List[Union[np.ndarray, List[float]]], **options
-    ) -> Any:
+    def insert_vectors(self, vectors: List[Union[np.ndarray, List[float]]], **options) -> Any:
+        """Backward compatibility alias for add_vectors."""
+        return self.add_vectors(vectors, **options)
+
+    def add_vectors(
+        self, 
+        vectors: List[Union[np.ndarray, List[float]]], 
+        ids: Optional[List[str]] = None,
+        metadata: Optional[List[Dict[str, Any]]] = None,
+        **options
+    ) -> List[str]:
         """
-        Insert vectors into collection.
+        Add vectors to collection.
 
         Args:
             vectors: List of vectors
+            ids: Optional list of vector IDs
+            metadata: Optional list of metadata dictionaries
             **options: Additional options
 
         Returns:
-            Insert result
+            List of vector IDs
         """
         tracking_id = self.progress_tracker.start_tracking(
             module="vector_store",
@@ -446,7 +457,15 @@ class MilvusStore:
                     vector = vector.tolist()
                 vector_data.append(vector)
 
-            data = [vector_data]
+            import uuid
+            if ids is None:
+                ids = [str(uuid.uuid4()) for _ in range(len(vectors))]
+                
+            if metadata is None:
+                metadata = [{} for _ in range(len(vectors))]
+
+            data = [ids, vector_data, metadata]
+            
             self.progress_tracker.update_tracking(
                 tracking_id, message="Inserting vectors into collection..."
             )
@@ -457,7 +476,7 @@ class MilvusStore:
                 status="completed",
                 message=f"Inserted {len(vectors)} vectors",
             )
-            return result
+            return ids
 
         except Exception as e:
             self.progress_tracker.stop_tracking(
@@ -527,6 +546,40 @@ class MilvusStore:
             )
             raise
 
+    def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+        """Get vector by ID."""
+        if not MILVUS_AVAILABLE or not self.collection:
+            return None
+            
+        try:
+            safe_id = vector_id.replace('"', '\\"')
+            res = self.collection.collection.query(
+                expr=f'id == "{safe_id}"', 
+                output_fields=["vector"]
+            )
+            if res and len(res) > 0:
+                return np.array(res[0]["vector"])
+            return None
+        except Exception:
+            return None
+
+    def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+        """Get metadata by ID."""
+        if not MILVUS_AVAILABLE or not self.collection:
+            return None
+            
+        try:
+            safe_id = vector_id.replace('"', '\\"')
+            res = self.collection.collection.query(
+                expr=f'id == "{safe_id}"', 
+                output_fields=["metadata"]
+            )
+            if res and len(res) > 0:
+                return res[0].get("metadata", {})
+            return None
+        except Exception:
+            return None
+
     def get_stats(self, collection_name: Optional[str] = None) -> Dict[str, Any]:
         """Get collection statistics."""
         if self.collection is None and collection_name:

semantica/vector_store/pgvector_store.py

@@ -633,6 +633,28 @@ class PgVectorStore:
             except Exception as e:
                 raise ProcessingError("Failed to get vectors") from e
 
+    def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+        """Get vector by ID."""
+        try:
+            results = self.get([vector_id])
+            if results and len(results) > 0:
+                return results[0].get("vector")
+            return None
+        except Exception as e:
+            self.logger.warning(f"Failed to get vector {vector_id}: {e}")
+            return None
+
+    def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+        """Get metadata by ID."""
+        try:
+            results = self.get([vector_id])
+            if results and len(results) > 0:
+                return results[0].get("metadata")
+            return None
+        except Exception as e:
+            self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
+            return None
+
     def create_index(
         self,
         index_type: str = "hnsw",

semantica/vector_store/pinecone_store.py

@@ -619,6 +619,28 @@ class PineconeStore:
 
         return self.index.delete_vectors(vector_ids, namespace, **options)
 
+    def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+        """Get vector by ID."""
+        try:
+            res = self.fetch_vectors([vector_id])
+            if "vectors" in res and vector_id in res["vectors"]:
+                return np.array(res["vectors"][vector_id]["values"])
+            return None
+        except Exception as e:
+            self.logger.warning(f"Failed to get vector {vector_id}: {e}")
+            return None
+
+    def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+        """Get metadata by ID."""
+        try:
+            res = self.fetch_vectors([vector_id])
+            if "vectors" in res and vector_id in res["vectors"]:
+                return res["vectors"][vector_id].get("metadata", {})
+            return None
+        except Exception as e:
+            self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
+            return None
+
     def fetch_vectors(
         self, vector_ids: List[str], namespace: str = "", **options
     ) -> Dict[str, Any]:

semantica/vector_store/qdrant_store.py

@@ -500,6 +500,44 @@ class QdrantStore:
             )
             raise
 
+    def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+        """Get vector by ID."""
+        if self.collection is None or not QDRANT_AVAILABLE:
+            return None
+        
+        try:
+            results = self.client.retrieve(
+                collection_name=self.collection.collection_name,
+                ids=[vector_id],
+                with_vectors=True,
+                with_payload=False
+            )
+            if results and results[0].vector:
+                return np.array(results[0].vector)
+            return None
+        except Exception as e:
+            self.logger.warning(f"Failed to get vector {vector_id}: {e}")
+            return None
+
+    def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+        """Get metadata by ID."""
+        if self.collection is None or not QDRANT_AVAILABLE:
+            return None
+            
+        try:
+            results = self.client.retrieve(
+                collection_name=self.collection.collection_name,
+                ids=[vector_id],
+                with_vectors=False,
+                with_payload=True
+            )
+            if results and results[0].payload is not None:
+                return results[0].payload
+            return None
+        except Exception as e:
+            self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
+            return None
+
     def delete_vectors(
         self, point_ids: List[Union[str, int]], **options
     ) -> Dict[str, Any]:

semantica/vector_store/sqlite_vec_store.py

@@ -594,6 +594,28 @@ class SQLiteVecStore:
             except Exception as e:
                 raise ProcessingError("Failed to get vectors") from e
 
+    def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+        """Get vector by ID."""
+        try:
+            results = self.get([vector_id])
+            if results and len(results) > 0:
+                return results[0].get("vector")
+            return None
+        except Exception as e:
+            self.logger.warning(f"Failed to get vector {vector_id}: {e}")
+            return None
+
+    def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+        """Get metadata by ID."""
+        try:
+            results = self.get([vector_id])
+            if results and len(results) > 0:
+                return results[0].get("metadata")
+            return None
+        except Exception as e:
+            self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
+            return None
+
     def create_index(
         self,
         index_type: str = "hnsw",

semantica/vector_store/vector_store.py

@@ -577,16 +577,19 @@ class VectorStore:
         import pickle
         
         os.makedirs(path, exist_ok=True)
-        
+
         # Save metadata and vectors (generic fallback)
         # Ideally, backends like FAISS have their own save methods
-        if hasattr(self.indexer, "save_index"):
-             self.indexer.save_index(os.path.join(path, "index.bin"))
-        
+        indexer = getattr(self, "indexer", None)
+        if indexer is not None and hasattr(indexer, "save_index"):
+            indexer.save_index(os.path.join(path, "index.bin"))
+        elif self._backend_store is not None and hasattr(self._backend_store, "save_index"):
+            self._backend_store.save_index(os.path.join(path, "index.bin"))
+
         # Save Python-level data
         data = {
-            "vectors": self.vectors,
-            "metadata": self.metadata,
+            "vectors": getattr(self, "vectors", {}),
+            "metadata": getattr(self, "metadata", {}),
             "config": self.config,
             "backend": self.backend,
             "dimension": self.dimension
@@ -622,14 +625,21 @@ class VectorStore:
         self.dimension = data.get("dimension", 768)
         
         # Restore backend-specific index
-        if hasattr(self.indexer, "load_index"):
-            index_path = os.path.join(path, "index.bin")
+        indexer = getattr(self, "indexer", None)
+        index_path = os.path.join(path, "index.bin")
+        if indexer is not None and hasattr(indexer, "load_index"):
             if os.path.exists(index_path):
-                self.indexer.load_index(index_path)
+                indexer.load_index(index_path)
             else:
                 # Rebuild if index file missing but vectors present
-                self.indexer.create_index(list(self.vectors.values()), list(self.vectors.keys()))
-        
+                indexer.create_index(list(self.vectors.values()), list(self.vectors.keys()))
+        elif (
+            self._backend_store is not None
+            and hasattr(self._backend_store, "load_index")
+            and os.path.exists(index_path)
+        ):
+            self._backend_store.load_index(index_path)
+
         self.logger.info(f"Loaded vector store from {path}")
 
     def search(self, query: str, limit: int = 10, **options) -> List[Dict[str, Any]]:
@@ -779,11 +789,21 @@ class VectorStore:
 
     def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
         """Get vector by ID."""
-        return self.vectors.get(vector_id)
+        if self.backend == "inmemory":
+            return self.vectors.get(vector_id)
+        elif self._backend_store and hasattr(self._backend_store, "get_vector"):
+            return self._backend_store.get_vector(vector_id)
+        else:
+            raise NotImplementedError(f"Backend store {type(self._backend_store).__name__} does not implement get_vector")
 
     def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
         """Get metadata for vector."""
-        return self.metadata.get(vector_id)
+        if self.backend == "inmemory":
+            return self.metadata.get(vector_id)
+        elif self._backend_store and hasattr(self._backend_store, "get_metadata"):
+            return self._backend_store.get_metadata(vector_id)
+        else:
+            raise NotImplementedError(f"Backend store {type(self._backend_store).__name__} does not implement get_metadata")
 
     def initialize_decision_pipeline(
         self,

semantica/vector_store/weaviate_store.py

@@ -416,6 +416,35 @@ class WeaviateStore:
             )
             raise ProcessingError(f"Failed to add objects: {str(e)}")
 
+    def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+        """Get vector by ID."""
+        if self.collection is None or not WEAVIATE_AVAILABLE:
+            return None
+        
+        try:
+            # Weaviate expects a valid UUID string
+            obj = self.collection.query.fetch_object_by_id(vector_id, include_vector=True)
+            if obj and obj.vector:
+                return np.array(obj.vector)
+            return None
+        except Exception as e:
+            self.logger.warning(f"Failed to get vector {vector_id}: {e}")
+            return None
+
+    def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+        """Get metadata by ID."""
+        if self.collection is None or not WEAVIATE_AVAILABLE:
+            return None
+        
+        try:
+            obj = self.collection.query.fetch_object_by_id(vector_id)
+            if obj and obj.properties:
+                return obj.properties
+            return None
+        except Exception as e:
+            self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
+            return None
+
     def query_vectors(
         self,
         query_vector: np.ndarray,

tests/vector_store/test_decision_embedding_pipeline.py

@@ -11,6 +11,7 @@ import numpy as np
 from unittest.mock import Mock, patch, MagicMock
 
 from semantica.vector_store.decision_embedding_pipeline import DecisionEmbeddingPipeline
+from semantica.vector_store import VectorStore
 
 
 class TestDecisionEmbeddingPipeline:
@@ -483,5 +484,52 @@ class TestDecisionEmbeddingPipelineEdgeCases:
         assert len(result["scores"]) == len(rare_indices)
 
 
+class TestVectorStoreRetrieval:
+    """Test get_vector and get_metadata on real backends."""
+    
+    def test_inmemory_retrieval(self):
+        """Test exact dict behavior for inmemory backend."""
+        vs = VectorStore(backend="inmemory")
+        vs.store_vectors([np.array([0.1, 0.2, 0.3], dtype=np.float32)], ids=["test1"], metadata=[{"foo": "bar"}])
+        
+        vec = vs.get_vector("vec_0")
+        meta = vs.get_metadata("vec_0")
+        
+        assert vec is not None
+        np.testing.assert_array_almost_equal(vec, np.array([0.1, 0.2, 0.3], dtype=np.float32))
+        assert meta == {"foo": "bar"}
+        
+    def test_faiss_retrieval(self):
+        """Test reconstruction from FAISS."""
+        try:
+            import faiss
+        except ImportError:
+            pytest.skip("FAISS not installed")
+            
+        vs = VectorStore(backend="faiss", config={"dimension": 3})
+        vs.store_vectors([np.array([0.1, 0.2, 0.3], dtype=np.float32)], ids=["test1"], metadata=[{"foo": "faiss_bar"}])
+        
+        vec = vs.get_vector("test1")
+        assert vec is not None
+        np.testing.assert_array_almost_equal(vec, np.array([0.1, 0.2, 0.3], dtype=np.float32))
+        
+        meta = vs.get_metadata("test1")
+        assert meta == {"foo": "faiss_bar"}
+            
+    def test_cloud_backends_untested(self):
+        """
+        Note: The following backends are not tested locally as they require 
+        live external services (Docker containers or API keys):
+        - QdrantStore
+        - PineconeStore
+        - MilvusStore
+        - WeaviateStore
+        - PgVectorStore
+        
+        Their implementations rely directly on official client SDKs (e.g. client.retrieve, 
+        index.fetch) to ensure correctness in production.
+        """
+        pass
+
 if __name__ == "__main__":
     pytest.main([__file__])

tests/vector_store/test_faiss_index.py

@@ -0,0 +1,122 @@
+from unittest.mock import MagicMock
+
+import numpy as np
+import pytest
+
+from semantica.vector_store.faiss_store import FAISSIndex
+
+
+def test_get_vector_reconstructs_from_flat_l2_index():
+    faiss = pytest.importorskip("faiss")
+    vectors = np.array(
+        [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]],
+        dtype=np.float32,
+    )
+    index = FAISSIndex(faiss.IndexFlatL2(3), dimension=3)
+    index.add_vectors(vectors, ids=["vec_first", "vec_target"])
+
+    result = index.get_vector("vec_target")
+
+    np.testing.assert_array_equal(result, vectors[1])
+
+
+def test_get_vector_reconstructs_vector_at_matching_id_position():
+    backend_index = MagicMock()
+    backend_index.reconstruct.return_value = [0.25, 0.5, 0.75]
+    index = FAISSIndex(backend_index, dimension=3)
+    index.vector_ids = ["vec_first", "vec_target"]
+
+    result = index.get_vector("vec_target")
+
+    backend_index.reconstruct.assert_called_once_with(1)
+    np.testing.assert_array_equal(result, np.array([0.25, 0.5, 0.75], dtype=np.float32))
+
+
+def test_get_vector_returns_none_for_unknown_id_without_reconstructing():
+    backend_index = MagicMock()
+    index = FAISSIndex(backend_index, dimension=3)
+    index.vector_ids = ["vec_known"]
+
+    assert index.get_vector("vec_missing") is None
+    backend_index.reconstruct.assert_not_called()
+
+
+def test_get_vector_returns_none_when_index_has_no_reconstruct_method():
+    index = FAISSIndex(object(), dimension=3)
+    index.vector_ids = ["vec_known"]
+
+    assert index.get_vector("vec_known") is None
+
+
+@pytest.mark.parametrize(
+    "error",
+    [
+        NotImplementedError(),
+        RuntimeError("reconstruct not implemented for this type of index"),
+        RuntimeError("reconstruct_from_offset not implemented"),
+    ],
+)
+def test_get_vector_returns_none_when_reconstruction_is_unsupported(error):
+    backend_index = MagicMock()
+    backend_index.reconstruct.side_effect = error
+    index = FAISSIndex(backend_index, dimension=3)
+    index.vector_ids = ["vec_known"]
+
+    assert index.get_vector("vec_known") is None
+
+
+def test_get_vector_propagates_unexpected_runtime_errors():
+    backend_index = MagicMock()
+    runtime_error = RuntimeError("index is not trained")
+    backend_index.reconstruct.side_effect = runtime_error
+    index = FAISSIndex(backend_index, dimension=3)
+    index.vector_ids = ["vec_known"]
+
+    with pytest.raises(RuntimeError) as exc_info:
+        index.get_vector("vec_known")
+
+    assert exc_info.value is runtime_error
+
+
+def test_get_vector_builds_direct_map_and_retries_when_not_initialized():
+    backend_index = MagicMock()
+    backend_index.reconstruct.side_effect = [
+        RuntimeError("direct map not initialized"),
+        [0.25, 0.5, 0.75],
+    ]
+    index = FAISSIndex(backend_index, dimension=3)
+    index.vector_ids = ["vec_known"]
+
+    result = index.get_vector("vec_known")
+
+    backend_index.make_direct_map.assert_called_once_with()
+    assert backend_index.reconstruct.call_count == 2
+    np.testing.assert_array_equal(result, np.array([0.25, 0.5, 0.75], dtype=np.float32))
+
+
+def test_get_vector_returns_none_when_direct_map_unavailable_and_not_initialized():
+    backend_index = MagicMock(spec=["reconstruct"])
+    backend_index.reconstruct.side_effect = RuntimeError("direct map not initialized")
+    index = FAISSIndex(backend_index, dimension=3)
+    index.vector_ids = ["vec_known"]
+
+    assert index.get_vector("vec_known") is None
+
+
+def test_get_vector_reconstructs_from_real_ivfflat_index_without_prior_direct_map():
+    faiss = pytest.importorskip("faiss")
+    dimension = 3
+    vectors = np.array(
+        [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9], [1.0, 1.1, 1.2]],
+        dtype=np.float32,
+    )
+    quantizer = faiss.IndexFlatL2(dimension)
+    backend_index = faiss.IndexIVFFlat(quantizer, dimension, 2)
+    backend_index.train(vectors)
+
+    index = FAISSIndex(backend_index, dimension=dimension)
+    index.add_vectors(vectors, ids=["vec_0", "vec_1", "vec_2", "vec_target"])
+
+    result = index.get_vector("vec_target")
+
+    np.testing.assert_allclose(result, vectors[3], atol=1e-6)