diff --git a/README.md b/README.md
index 985b6baf..745c54e4 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,8 @@
+
+
### Graph-Native Infrastructure for Context and Accountable AI Systems
#### *The Open Source Palantir for AI Agents*
diff --git a/semantica/vector_store/faiss_store.py b/semantica/vector_store/faiss_store.py
index 3368e43f..3fe871c6 100644
--- a/semantica/vector_store/faiss_store.py
+++ b/semantica/vector_store/faiss_store.py
@@ -85,9 +85,35 @@ class FAISSIndex:
return None
idx = self.vector_ids.index(vector_id)
- # Note: FAISS doesn't directly support retrieval by index in all cases
- # This is a simplified approach
- return None
+ reconstruct = getattr(self.index, "reconstruct", None)
+ if not callable(reconstruct):
+ return None
+
+ try:
+ return np.asarray(reconstruct(idx), dtype=np.float32)
+ except NotImplementedError:
+ return None
+ except RuntimeError as exc:
+ message = str(exc).casefold()
+ unsupported_errors = (
+ "reconstruct not implemented",
+ "reconstruct_from_offset not implemented",
+ )
+ if any(error in message for error in unsupported_errors):
+ return None
+ if "direct map not initialized" in message:
+ # IVF-family indices (e.g. IndexIVFFlat) support exact reconstruction
+ # but need their DirectMap built once before reconstruct() works.
+ make_direct_map = getattr(self.index, "make_direct_map", None)
+ if not callable(make_direct_map):
+ return None
+ make_direct_map()
+ return np.asarray(reconstruct(idx), dtype=np.float32)
+ raise
+
+ def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+ """Get metadata by ID."""
+ return self.metadata.get(vector_id)
def save(self, path: Union[str, Path]):
"""Save index to disk."""
@@ -452,6 +478,18 @@ class FAISSStore:
self.logger.info("Index optimization completed")
return True
+ def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+ """Get vector by ID."""
+ if self.index:
+ return self.index.get_vector(vector_id)
+ return None
+
+ def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+ """Get metadata by ID."""
+ if self.index:
+ return self.index.get_metadata(vector_id)
+ return None
+
def get_stats(self) -> Dict[str, Any]:
"""Get index statistics."""
if self.index is None:
diff --git a/semantica/vector_store/milvus_store.py b/semantica/vector_store/milvus_store.py
index d9a99885..d6b7fb2a 100644
--- a/semantica/vector_store/milvus_store.py
+++ b/semantica/vector_store/milvus_store.py
@@ -346,9 +346,10 @@ class MilvusStore:
# Define schema
fields = [
FieldSchema(
- name="id", dtype=DataType.INT64, is_primary=True, auto_id=True
+ name="id", dtype=DataType.VARCHAR, is_primary=True, auto_id=False, max_length=65535
),
FieldSchema(name="vector", dtype=DataType.FLOAT_VECTOR, dim=dimension),
+ FieldSchema(name="metadata", dtype=DataType.JSON),
]
schema = CollectionSchema(
@@ -402,18 +403,28 @@ class MilvusStore:
except Exception as e:
raise ProcessingError(f"Failed to get collection: {str(e)}")
- def insert_vectors(
- self, vectors: List[Union[np.ndarray, List[float]]], **options
- ) -> Any:
+ def insert_vectors(self, vectors: List[Union[np.ndarray, List[float]]], **options) -> Any:
+ """Backward compatibility alias for add_vectors."""
+ return self.add_vectors(vectors, **options)
+
+ def add_vectors(
+ self,
+ vectors: List[Union[np.ndarray, List[float]]],
+ ids: Optional[List[str]] = None,
+ metadata: Optional[List[Dict[str, Any]]] = None,
+ **options
+ ) -> List[str]:
"""
- Insert vectors into collection.
+ Add vectors to collection.
Args:
vectors: List of vectors
+ ids: Optional list of vector IDs
+ metadata: Optional list of metadata dictionaries
**options: Additional options
Returns:
- Insert result
+ List of vector IDs
"""
tracking_id = self.progress_tracker.start_tracking(
module="vector_store",
@@ -446,7 +457,15 @@ class MilvusStore:
vector = vector.tolist()
vector_data.append(vector)
- data = [vector_data]
+ import uuid
+ if ids is None:
+ ids = [str(uuid.uuid4()) for _ in range(len(vectors))]
+
+ if metadata is None:
+ metadata = [{} for _ in range(len(vectors))]
+
+ data = [ids, vector_data, metadata]
+
self.progress_tracker.update_tracking(
tracking_id, message="Inserting vectors into collection..."
)
@@ -457,7 +476,7 @@ class MilvusStore:
status="completed",
message=f"Inserted {len(vectors)} vectors",
)
- return result
+ return ids
except Exception as e:
self.progress_tracker.stop_tracking(
@@ -527,6 +546,40 @@ class MilvusStore:
)
raise
+ def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+ """Get vector by ID."""
+ if not MILVUS_AVAILABLE or not self.collection:
+ return None
+
+ try:
+ safe_id = vector_id.replace('"', '\\"')
+ res = self.collection.collection.query(
+ expr=f'id == "{safe_id}"',
+ output_fields=["vector"]
+ )
+ if res and len(res) > 0:
+ return np.array(res[0]["vector"])
+ return None
+ except Exception:
+ return None
+
+ def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+ """Get metadata by ID."""
+ if not MILVUS_AVAILABLE or not self.collection:
+ return None
+
+ try:
+ safe_id = vector_id.replace('"', '\\"')
+ res = self.collection.collection.query(
+ expr=f'id == "{safe_id}"',
+ output_fields=["metadata"]
+ )
+ if res and len(res) > 0:
+ return res[0].get("metadata", {})
+ return None
+ except Exception:
+ return None
+
def get_stats(self, collection_name: Optional[str] = None) -> Dict[str, Any]:
"""Get collection statistics."""
if self.collection is None and collection_name:
diff --git a/semantica/vector_store/pgvector_store.py b/semantica/vector_store/pgvector_store.py
index 393261a3..3b75ed1f 100644
--- a/semantica/vector_store/pgvector_store.py
+++ b/semantica/vector_store/pgvector_store.py
@@ -633,6 +633,28 @@ class PgVectorStore:
except Exception as e:
raise ProcessingError("Failed to get vectors") from e
+ def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+ """Get vector by ID."""
+ try:
+ results = self.get([vector_id])
+ if results and len(results) > 0:
+ return results[0].get("vector")
+ return None
+ except Exception as e:
+ self.logger.warning(f"Failed to get vector {vector_id}: {e}")
+ return None
+
+ def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+ """Get metadata by ID."""
+ try:
+ results = self.get([vector_id])
+ if results and len(results) > 0:
+ return results[0].get("metadata")
+ return None
+ except Exception as e:
+ self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
+ return None
+
def create_index(
self,
index_type: str = "hnsw",
diff --git a/semantica/vector_store/pinecone_store.py b/semantica/vector_store/pinecone_store.py
index 69b60fe9..a3204c28 100644
--- a/semantica/vector_store/pinecone_store.py
+++ b/semantica/vector_store/pinecone_store.py
@@ -619,6 +619,28 @@ class PineconeStore:
return self.index.delete_vectors(vector_ids, namespace, **options)
+ def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+ """Get vector by ID."""
+ try:
+ res = self.fetch_vectors([vector_id])
+ if "vectors" in res and vector_id in res["vectors"]:
+ return np.array(res["vectors"][vector_id]["values"])
+ return None
+ except Exception as e:
+ self.logger.warning(f"Failed to get vector {vector_id}: {e}")
+ return None
+
+ def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+ """Get metadata by ID."""
+ try:
+ res = self.fetch_vectors([vector_id])
+ if "vectors" in res and vector_id in res["vectors"]:
+ return res["vectors"][vector_id].get("metadata", {})
+ return None
+ except Exception as e:
+ self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
+ return None
+
def fetch_vectors(
self, vector_ids: List[str], namespace: str = "", **options
) -> Dict[str, Any]:
diff --git a/semantica/vector_store/qdrant_store.py b/semantica/vector_store/qdrant_store.py
index 49c71bfc..67c950e0 100644
--- a/semantica/vector_store/qdrant_store.py
+++ b/semantica/vector_store/qdrant_store.py
@@ -500,6 +500,44 @@ class QdrantStore:
)
raise
+ def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+ """Get vector by ID."""
+ if self.collection is None or not QDRANT_AVAILABLE:
+ return None
+
+ try:
+ results = self.client.retrieve(
+ collection_name=self.collection.collection_name,
+ ids=[vector_id],
+ with_vectors=True,
+ with_payload=False
+ )
+ if results and results[0].vector:
+ return np.array(results[0].vector)
+ return None
+ except Exception as e:
+ self.logger.warning(f"Failed to get vector {vector_id}: {e}")
+ return None
+
+ def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+ """Get metadata by ID."""
+ if self.collection is None or not QDRANT_AVAILABLE:
+ return None
+
+ try:
+ results = self.client.retrieve(
+ collection_name=self.collection.collection_name,
+ ids=[vector_id],
+ with_vectors=False,
+ with_payload=True
+ )
+ if results and results[0].payload is not None:
+ return results[0].payload
+ return None
+ except Exception as e:
+ self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
+ return None
+
def delete_vectors(
self, point_ids: List[Union[str, int]], **options
) -> Dict[str, Any]:
diff --git a/semantica/vector_store/sqlite_vec_store.py b/semantica/vector_store/sqlite_vec_store.py
index edf2625a..b8695ad9 100644
--- a/semantica/vector_store/sqlite_vec_store.py
+++ b/semantica/vector_store/sqlite_vec_store.py
@@ -594,6 +594,28 @@ class SQLiteVecStore:
except Exception as e:
raise ProcessingError("Failed to get vectors") from e
+ def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+ """Get vector by ID."""
+ try:
+ results = self.get([vector_id])
+ if results and len(results) > 0:
+ return results[0].get("vector")
+ return None
+ except Exception as e:
+ self.logger.warning(f"Failed to get vector {vector_id}: {e}")
+ return None
+
+ def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+ """Get metadata by ID."""
+ try:
+ results = self.get([vector_id])
+ if results and len(results) > 0:
+ return results[0].get("metadata")
+ return None
+ except Exception as e:
+ self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
+ return None
+
def create_index(
self,
index_type: str = "hnsw",
diff --git a/semantica/vector_store/vector_store.py b/semantica/vector_store/vector_store.py
index 0c7e7c5f..aa42f620 100644
--- a/semantica/vector_store/vector_store.py
+++ b/semantica/vector_store/vector_store.py
@@ -577,16 +577,19 @@ class VectorStore:
import pickle
os.makedirs(path, exist_ok=True)
-
+
# Save metadata and vectors (generic fallback)
# Ideally, backends like FAISS have their own save methods
- if hasattr(self.indexer, "save_index"):
- self.indexer.save_index(os.path.join(path, "index.bin"))
-
+ indexer = getattr(self, "indexer", None)
+ if indexer is not None and hasattr(indexer, "save_index"):
+ indexer.save_index(os.path.join(path, "index.bin"))
+ elif self._backend_store is not None and hasattr(self._backend_store, "save_index"):
+ self._backend_store.save_index(os.path.join(path, "index.bin"))
+
# Save Python-level data
data = {
- "vectors": self.vectors,
- "metadata": self.metadata,
+ "vectors": getattr(self, "vectors", {}),
+ "metadata": getattr(self, "metadata", {}),
"config": self.config,
"backend": self.backend,
"dimension": self.dimension
@@ -622,14 +625,21 @@ class VectorStore:
self.dimension = data.get("dimension", 768)
# Restore backend-specific index
- if hasattr(self.indexer, "load_index"):
- index_path = os.path.join(path, "index.bin")
+ indexer = getattr(self, "indexer", None)
+ index_path = os.path.join(path, "index.bin")
+ if indexer is not None and hasattr(indexer, "load_index"):
if os.path.exists(index_path):
- self.indexer.load_index(index_path)
+ indexer.load_index(index_path)
else:
# Rebuild if index file missing but vectors present
- self.indexer.create_index(list(self.vectors.values()), list(self.vectors.keys()))
-
+ indexer.create_index(list(self.vectors.values()), list(self.vectors.keys()))
+ elif (
+ self._backend_store is not None
+ and hasattr(self._backend_store, "load_index")
+ and os.path.exists(index_path)
+ ):
+ self._backend_store.load_index(index_path)
+
self.logger.info(f"Loaded vector store from {path}")
def search(self, query: str, limit: int = 10, **options) -> List[Dict[str, Any]]:
@@ -779,11 +789,21 @@ class VectorStore:
def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
"""Get vector by ID."""
- return self.vectors.get(vector_id)
+ if self.backend == "inmemory":
+ return self.vectors.get(vector_id)
+ elif self._backend_store and hasattr(self._backend_store, "get_vector"):
+ return self._backend_store.get_vector(vector_id)
+ else:
+ raise NotImplementedError(f"Backend store {type(self._backend_store).__name__} does not implement get_vector")
def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
"""Get metadata for vector."""
- return self.metadata.get(vector_id)
+ if self.backend == "inmemory":
+ return self.metadata.get(vector_id)
+ elif self._backend_store and hasattr(self._backend_store, "get_metadata"):
+ return self._backend_store.get_metadata(vector_id)
+ else:
+ raise NotImplementedError(f"Backend store {type(self._backend_store).__name__} does not implement get_metadata")
def initialize_decision_pipeline(
self,
diff --git a/semantica/vector_store/weaviate_store.py b/semantica/vector_store/weaviate_store.py
index a1906153..c58b8f80 100644
--- a/semantica/vector_store/weaviate_store.py
+++ b/semantica/vector_store/weaviate_store.py
@@ -416,6 +416,35 @@ class WeaviateStore:
)
raise ProcessingError(f"Failed to add objects: {str(e)}")
+ def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
+ """Get vector by ID."""
+ if self.collection is None or not WEAVIATE_AVAILABLE:
+ return None
+
+ try:
+ # Weaviate expects a valid UUID string
+ obj = self.collection.query.fetch_object_by_id(vector_id, include_vector=True)
+ if obj and obj.vector:
+ return np.array(obj.vector)
+ return None
+ except Exception as e:
+ self.logger.warning(f"Failed to get vector {vector_id}: {e}")
+ return None
+
+ def get_metadata(self, vector_id: str) -> Optional[Dict[str, Any]]:
+ """Get metadata by ID."""
+ if self.collection is None or not WEAVIATE_AVAILABLE:
+ return None
+
+ try:
+ obj = self.collection.query.fetch_object_by_id(vector_id)
+ if obj and obj.properties:
+ return obj.properties
+ return None
+ except Exception as e:
+ self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
+ return None
+
def query_vectors(
self,
query_vector: np.ndarray,
diff --git a/tests/vector_store/test_decision_embedding_pipeline.py b/tests/vector_store/test_decision_embedding_pipeline.py
index 4fb69280..17d47ddc 100644
--- a/tests/vector_store/test_decision_embedding_pipeline.py
+++ b/tests/vector_store/test_decision_embedding_pipeline.py
@@ -11,6 +11,7 @@ import numpy as np
from unittest.mock import Mock, patch, MagicMock
from semantica.vector_store.decision_embedding_pipeline import DecisionEmbeddingPipeline
+from semantica.vector_store import VectorStore
class TestDecisionEmbeddingPipeline:
@@ -483,5 +484,52 @@ class TestDecisionEmbeddingPipelineEdgeCases:
assert len(result["scores"]) == len(rare_indices)
+class TestVectorStoreRetrieval:
+ """Test get_vector and get_metadata on real backends."""
+
+ def test_inmemory_retrieval(self):
+ """Test exact dict behavior for inmemory backend."""
+ vs = VectorStore(backend="inmemory")
+ vs.store_vectors([np.array([0.1, 0.2, 0.3], dtype=np.float32)], ids=["test1"], metadata=[{"foo": "bar"}])
+
+ vec = vs.get_vector("vec_0")
+ meta = vs.get_metadata("vec_0")
+
+ assert vec is not None
+ np.testing.assert_array_almost_equal(vec, np.array([0.1, 0.2, 0.3], dtype=np.float32))
+ assert meta == {"foo": "bar"}
+
+ def test_faiss_retrieval(self):
+ """Test reconstruction from FAISS."""
+ try:
+ import faiss
+ except ImportError:
+ pytest.skip("FAISS not installed")
+
+ vs = VectorStore(backend="faiss", config={"dimension": 3})
+ vs.store_vectors([np.array([0.1, 0.2, 0.3], dtype=np.float32)], ids=["test1"], metadata=[{"foo": "faiss_bar"}])
+
+ vec = vs.get_vector("test1")
+ assert vec is not None
+ np.testing.assert_array_almost_equal(vec, np.array([0.1, 0.2, 0.3], dtype=np.float32))
+
+ meta = vs.get_metadata("test1")
+ assert meta == {"foo": "faiss_bar"}
+
+ def test_cloud_backends_untested(self):
+ """
+ Note: The following backends are not tested locally as they require
+ live external services (Docker containers or API keys):
+ - QdrantStore
+ - PineconeStore
+ - MilvusStore
+ - WeaviateStore
+ - PgVectorStore
+
+ Their implementations rely directly on official client SDKs (e.g. client.retrieve,
+ index.fetch) to ensure correctness in production.
+ """
+ pass
+
if __name__ == "__main__":
pytest.main([__file__])
diff --git a/tests/vector_store/test_faiss_index.py b/tests/vector_store/test_faiss_index.py
new file mode 100644
index 00000000..b00052d2
--- /dev/null
+++ b/tests/vector_store/test_faiss_index.py
@@ -0,0 +1,122 @@
+from unittest.mock import MagicMock
+
+import numpy as np
+import pytest
+
+from semantica.vector_store.faiss_store import FAISSIndex
+
+
+def test_get_vector_reconstructs_from_flat_l2_index():
+ faiss = pytest.importorskip("faiss")
+ vectors = np.array(
+ [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]],
+ dtype=np.float32,
+ )
+ index = FAISSIndex(faiss.IndexFlatL2(3), dimension=3)
+ index.add_vectors(vectors, ids=["vec_first", "vec_target"])
+
+ result = index.get_vector("vec_target")
+
+ np.testing.assert_array_equal(result, vectors[1])
+
+
+def test_get_vector_reconstructs_vector_at_matching_id_position():
+ backend_index = MagicMock()
+ backend_index.reconstruct.return_value = [0.25, 0.5, 0.75]
+ index = FAISSIndex(backend_index, dimension=3)
+ index.vector_ids = ["vec_first", "vec_target"]
+
+ result = index.get_vector("vec_target")
+
+ backend_index.reconstruct.assert_called_once_with(1)
+ np.testing.assert_array_equal(result, np.array([0.25, 0.5, 0.75], dtype=np.float32))
+
+
+def test_get_vector_returns_none_for_unknown_id_without_reconstructing():
+ backend_index = MagicMock()
+ index = FAISSIndex(backend_index, dimension=3)
+ index.vector_ids = ["vec_known"]
+
+ assert index.get_vector("vec_missing") is None
+ backend_index.reconstruct.assert_not_called()
+
+
+def test_get_vector_returns_none_when_index_has_no_reconstruct_method():
+ index = FAISSIndex(object(), dimension=3)
+ index.vector_ids = ["vec_known"]
+
+ assert index.get_vector("vec_known") is None
+
+
+@pytest.mark.parametrize(
+ "error",
+ [
+ NotImplementedError(),
+ RuntimeError("reconstruct not implemented for this type of index"),
+ RuntimeError("reconstruct_from_offset not implemented"),
+ ],
+)
+def test_get_vector_returns_none_when_reconstruction_is_unsupported(error):
+ backend_index = MagicMock()
+ backend_index.reconstruct.side_effect = error
+ index = FAISSIndex(backend_index, dimension=3)
+ index.vector_ids = ["vec_known"]
+
+ assert index.get_vector("vec_known") is None
+
+
+def test_get_vector_propagates_unexpected_runtime_errors():
+ backend_index = MagicMock()
+ runtime_error = RuntimeError("index is not trained")
+ backend_index.reconstruct.side_effect = runtime_error
+ index = FAISSIndex(backend_index, dimension=3)
+ index.vector_ids = ["vec_known"]
+
+ with pytest.raises(RuntimeError) as exc_info:
+ index.get_vector("vec_known")
+
+ assert exc_info.value is runtime_error
+
+
+def test_get_vector_builds_direct_map_and_retries_when_not_initialized():
+ backend_index = MagicMock()
+ backend_index.reconstruct.side_effect = [
+ RuntimeError("direct map not initialized"),
+ [0.25, 0.5, 0.75],
+ ]
+ index = FAISSIndex(backend_index, dimension=3)
+ index.vector_ids = ["vec_known"]
+
+ result = index.get_vector("vec_known")
+
+ backend_index.make_direct_map.assert_called_once_with()
+ assert backend_index.reconstruct.call_count == 2
+ np.testing.assert_array_equal(result, np.array([0.25, 0.5, 0.75], dtype=np.float32))
+
+
+def test_get_vector_returns_none_when_direct_map_unavailable_and_not_initialized():
+ backend_index = MagicMock(spec=["reconstruct"])
+ backend_index.reconstruct.side_effect = RuntimeError("direct map not initialized")
+ index = FAISSIndex(backend_index, dimension=3)
+ index.vector_ids = ["vec_known"]
+
+ assert index.get_vector("vec_known") is None
+
+
+def test_get_vector_reconstructs_from_real_ivfflat_index_without_prior_direct_map():
+ faiss = pytest.importorskip("faiss")
+ dimension = 3
+ vectors = np.array(
+ [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9], [1.0, 1.1, 1.2]],
+ dtype=np.float32,
+ )
+ quantizer = faiss.IndexFlatL2(dimension)
+ backend_index = faiss.IndexIVFFlat(quantizer, dimension, 2)
+ backend_index.train(vectors)
+
+ index = FAISSIndex(backend_index, dimension=dimension)
+ index.add_vectors(vectors, ids=["vec_0", "vec_1", "vec_2", "vec_target"])
+
+ result = index.get_vector("vec_target")
+
+ np.testing.assert_allclose(result, vectors[3], atol=1e-6)