mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-09 04:00:52 +00:00
feat(milvus): add delete_vectors to MilvusStore (#1391)
* feat(milvus): add delete_vectors to MilvusStore Adds delete_vectors(ids) so the ErasureCoordinator can erase embeddings on a Milvus backend. Ids are escaped with _format_milvus_value before building the delete expression, and the backend delete count is returned so a delete that removed nothing is distinguishable from a failure. * test(milvus): cover delete_vectors incl. erasure integration Unit tests assert the single-id equality expression, multi-id in expression, id escaping, empty-id noop, missing-collection and backend error paths. Integration tests bind MilvusStore as a VectorStore backend and assert ErasureCoordinator reports the vector leg erased. * test(milvus): cover vector store delete facade --------- Co-authored-by: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Co-authored-by: Sameer Kadam <sskadam6305@gmail.com> Co-authored-by: Sameer Kadam <sameerkadam@Mac.lan>
This commit is contained in:
co-authored by
Mohd Kaif
Sameer Kadam
Sameer Kadam
parent
008b9e0b64
commit
28f4fc7bc4
@@ -14,10 +14,12 @@ not. It *composes* the existing public APIs; nothing in ``context_graph.py`` or
|
||||
``agent_memory.py`` changes, and ``ContextGraph`` keeps its graph-scope
|
||||
contract.
|
||||
|
||||
The property that matters is honest partial reporting. Three vector backends
|
||||
(FAISS, Milvus, Weaviate) expose no delete at all, so erasure is genuinely not
|
||||
completable on them today. The receipt says ``unsupported`` for those rather
|
||||
than reporting a success it did not achieve -- a receipt that reads
|
||||
The property that matters is honest partial reporting. FAISS exposes no delete
|
||||
at all -- a flat FAISS index cannot remove individual vectors without a full
|
||||
rebuild -- so erasure is genuinely not completable on it today. Milvus and
|
||||
Weaviate now expose ``delete_vectors`` and are fully supported. The receipt
|
||||
says ``unsupported`` for FAISS rather than reporting a success it did not
|
||||
achieve -- a receipt that reads
|
||||
"graph: erased, memory: 14 erased, vectors: unsupported on faiss" is
|
||||
actionable; a bare ``True`` is a compliance liability.
|
||||
|
||||
@@ -29,9 +31,9 @@ Example:
|
||||
... "customer-4471", reason="GDPR Art. 17 request #882"
|
||||
... )
|
||||
>>> receipt.complete
|
||||
False
|
||||
True
|
||||
>>> receipt.stores["vectors"]["status"]
|
||||
'unsupported'
|
||||
'not_configured'
|
||||
"""
|
||||
|
||||
import copy
|
||||
@@ -377,8 +379,8 @@ class ErasureCoordinator:
|
||||
|
||||
method_name, target = _vector_delete_capability(self.vector_store)
|
||||
if method_name is None:
|
||||
# FAISS, Milvus and Weaviate expose no delete at all; FAISS in
|
||||
# particular cannot remove from a flat index without a rebuild.
|
||||
# FAISS exposes no delete at all; it cannot remove vectors from a
|
||||
# flat index without a full rebuild.
|
||||
self.logger.warning(
|
||||
"Vector backend %r exposes no delete; %d vector id(s) for %r "
|
||||
"were not erased",
|
||||
|
||||
@@ -627,6 +627,46 @@ class MilvusStore:
|
||||
)
|
||||
raise
|
||||
|
||||
def delete_vectors(self, vector_ids: List[str], **options) -> Dict[str, Any]:
|
||||
"""Delete vectors from collection by their ids.
|
||||
|
||||
Args:
|
||||
vector_ids: Vector ids to delete
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
A dict with the number of matching entities that were deleted
|
||||
(``delete_count``).
|
||||
"""
|
||||
if self.collection is None:
|
||||
raise ProcessingError(
|
||||
"Collection not initialized. Call create_collection() or get_collection() first."
|
||||
)
|
||||
|
||||
if not vector_ids:
|
||||
return {"delete_count": 0}
|
||||
|
||||
try:
|
||||
# Milvus DELETE deletes by expression. Escape each id so a quote or
|
||||
# backslash in an id cannot break out of the string literal.
|
||||
if len(vector_ids) == 1:
|
||||
expr = f"id == {_format_milvus_value(vector_ids[0])}"
|
||||
else:
|
||||
formatted = ", ".join(_format_milvus_value(i) for i in vector_ids)
|
||||
expr = f"id in [{formatted}]"
|
||||
result = self.collection.collection.delete(expr=expr, **options)
|
||||
delete_count = getattr(result, "delete_count", 0)
|
||||
if delete_count is None:
|
||||
delete_count = 0
|
||||
elif isinstance(delete_count, (str, bytes)):
|
||||
try:
|
||||
delete_count = int(delete_count)
|
||||
except (TypeError, ValueError):
|
||||
delete_count = 0
|
||||
return {"delete_count": delete_count}
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to delete vectors: {str(e)}")
|
||||
|
||||
def get_vector(self, vector_id: str) -> Optional[np.ndarray]:
|
||||
"""Get vector by ID."""
|
||||
if not MILVUS_AVAILABLE or not self.collection:
|
||||
|
||||
@@ -89,7 +89,7 @@ class _DeleteStore:
|
||||
|
||||
|
||||
class _NoDeleteStore:
|
||||
"""Backend shaped like FAISS/Milvus/Weaviate: no delete surface at all."""
|
||||
"""Backend shaped like FAISS: no delete surface at all."""
|
||||
|
||||
backend = "faiss"
|
||||
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Tests for MilvusStore.delete_vectors (#1374)."""
|
||||
|
||||
from unittest import TestCase
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from semantica.context.erasure import STATUS_ERASED, ErasureCoordinator
|
||||
from semantica.utils.exceptions import ProcessingError
|
||||
from semantica.vector_store import VectorStore
|
||||
from semantica.vector_store.milvus_store import MilvusStore
|
||||
|
||||
|
||||
class MilvusStoreDeleteVectorsTest(TestCase):
|
||||
def setUp(self):
|
||||
self.patches = [
|
||||
patch("semantica.vector_store.milvus_store.MILVUS_AVAILABLE", True)
|
||||
]
|
||||
for p in self.patches:
|
||||
p.start()
|
||||
|
||||
def tearDown(self):
|
||||
for p in reversed(self.patches):
|
||||
p.stop()
|
||||
|
||||
def _store(self, result=None, error=None):
|
||||
"""Return (store, exprs) where delete() records exprs, returns result."""
|
||||
exprs = []
|
||||
coll = MagicMock()
|
||||
|
||||
def _delete(expr, **kwargs):
|
||||
exprs.append(expr)
|
||||
if error is not None:
|
||||
raise error
|
||||
if result is None:
|
||||
return MagicMock(delete_count=0)
|
||||
return result
|
||||
|
||||
coll.collection.delete.side_effect = _delete
|
||||
store = MilvusStore()
|
||||
store.collection = coll
|
||||
return store, exprs
|
||||
|
||||
def test_delete_single_id_uses_equality_expr(self):
|
||||
store, exprs = self._store(result=MagicMock(delete_count=1))
|
||||
ret = store.delete_vectors(["abc"])
|
||||
self.assertEqual(exprs, ['id == "abc"'])
|
||||
self.assertEqual(ret, {"delete_count": 1})
|
||||
|
||||
def test_delete_many_ids_uses_in_expr(self):
|
||||
store, exprs = self._store(result=MagicMock(delete_count=2))
|
||||
ret = store.delete_vectors(["a", "b"])
|
||||
self.assertEqual(exprs, ['id in ["a", "b"]'])
|
||||
self.assertEqual(ret, {"delete_count": 2})
|
||||
|
||||
def test_delete_escapes_quote_and_backslash_in_id(self):
|
||||
store, exprs = self._store()
|
||||
store.delete_vectors(['he said "hi"', "a\\b"])
|
||||
self.assertEqual(exprs, ['id in ["he said \\"hi\\"", "a\\\\b"]'])
|
||||
|
||||
def test_delete_empty_ids_is_noop(self):
|
||||
store, _ = self._store()
|
||||
ret = store.delete_vectors([])
|
||||
self.assertEqual(ret, {"delete_count": 0})
|
||||
store.collection.collection.delete.assert_not_called()
|
||||
|
||||
def test_delete_without_collection_raises(self):
|
||||
store = MilvusStore()
|
||||
with self.assertRaises(ProcessingError):
|
||||
store.delete_vectors(["a"])
|
||||
|
||||
def test_delete_backend_error_raises_processing_error(self):
|
||||
store, _ = self._store(error=RuntimeError("connection reset"))
|
||||
with self.assertRaises(ProcessingError):
|
||||
store.delete_vectors(["a"])
|
||||
|
||||
def test_delete_string_delete_count_is_parsed(self):
|
||||
store, _ = self._store(result=MagicMock(delete_count="3"))
|
||||
ret = store.delete_vectors(["a", "b", "c"])
|
||||
self.assertEqual(ret, {"delete_count": 3})
|
||||
|
||||
def test_delete_none_delete_count_defaults_zero(self):
|
||||
store, _ = self._store(result=MagicMock(delete_count=None))
|
||||
ret = store.delete_vectors(["a"])
|
||||
self.assertEqual(ret, {"delete_count": 0})
|
||||
|
||||
|
||||
class MilvusErasureIntegrationTest(TestCase):
|
||||
"""ErasureCoordinator reaches the real MilvusStore.delete_vectors path."""
|
||||
|
||||
def _bind_milvus_as_vector_store(self):
|
||||
vs = VectorStore(backend="milvus", config={"dimension": 3})
|
||||
milvus = MilvusStore()
|
||||
coll = MagicMock()
|
||||
coll.collection.delete.return_value = MagicMock(delete_count=0)
|
||||
milvus.collection = coll
|
||||
vs._backend_store = milvus
|
||||
return vs, coll
|
||||
|
||||
def test_erasure_reports_erased_when_delete_runs(self):
|
||||
vs, coll = self._bind_milvus_as_vector_store()
|
||||
coord = ErasureCoordinator(vector_store=vs)
|
||||
receipt = coord.erase_entity("customer-4471")
|
||||
coll.collection.delete.assert_called()
|
||||
self.assertEqual(receipt.stores["vectors"]["status"], STATUS_ERASED)
|
||||
|
||||
def test_erasure_backend_name_is_milvus(self):
|
||||
vs, _ = self._bind_milvus_as_vector_store()
|
||||
coord = ErasureCoordinator(vector_store=vs)
|
||||
receipt = coord.erase_entity("customer-4471")
|
||||
self.assertEqual(receipt.stores["vectors"]["backend"], "milvus")
|
||||
|
||||
def test_facade_delete_vectors_forwards_to_milvus(self):
|
||||
"""VectorStore.delete_vectors() delegates to MilvusStore and returns its dict.
|
||||
|
||||
ErasureCoordinator probes _backend_store directly, so this test
|
||||
exercises the public VectorStore facade path that other callers use.
|
||||
"""
|
||||
vs, coll = self._bind_milvus_as_vector_store()
|
||||
coll.collection.delete.return_value = MagicMock(delete_count=2)
|
||||
|
||||
ret = vs.delete_vectors(["id-1", "id-2"])
|
||||
|
||||
coll.collection.delete.assert_called_once()
|
||||
self.assertEqual(ret, {"delete_count": 2})
|
||||
Reference in New Issue
Block a user