Merge branch 'main' into fix/mcp-server-version

This commit is contained in:
Sameer Kadam
2026-08-12 13:01:29 +05:30
committed by GitHub
14 changed files with 1365 additions and 89 deletions
+12
View File
@@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Fixed
- **`VectorStore._filter_by_metadata()` `AttributeError` on all persistent backends** (#857, closes #849) by @TaherTadpatri
- `_filter_by_metadata()` iterated `self.metadata` directly, which only exists on the `inmemory` backend — any persistent backend (`faiss`, `qdrant`, `pinecone`, `milvus`, `pgvector`, `sqlite`, `weaviate`) crashed with `AttributeError` on `filter_decisions(query=None, ...)` / metadata-only filtering. Filtering is now delegated to a native `filter_by_metadata()` implemented on each backend store, using backend-native payload/SQL/JSON filtering (Qdrant `scroll()`, Pinecone `query()`, Milvus expression filters, PostgreSQL JSONB, SQLite `json_extract()`, Weaviate collection filters)
- **Fixed along the way**: `PineconeStore.get_index()` and `filter_by_metadata()` called a nonexistent `self.describe_index_stats()` on the store itself (the method only exists on the `PineconeIndex` wrapper returned by `self.index`); the resulting `AttributeError` was silently swallowed, so dimension auto-detection always failed quietly. Now correctly calls `self.index.describe_index_stats()`
- **Fixed along the way**: `PineconeStore.filter_by_metadata()` probed for filter-only matches using an all-zero dummy query vector, which Pinecone rejects for cosine-metric indexes — the library's own default — making metadata-only filtering silently non-functional out of the box. Now uses a unit vector instead
- **Fixed along the way**: `PgVectorStore.filter_by_metadata()`'s list-filter branch formatted boolean values with `str(v)` (`'True'`/`'False'`), never matching PostgreSQL JSONB's lowercase `'true'`/`'false'` text rendering, even though the equivalent scalar-filter branch already handled this correctly
- **Fixed along the way**: list-valued metadata fields (e.g. `{"tags": ["python", "js"]}`) could never match a list filter on the SQLite or PostgreSQL backends, because both extracted the whole array as its JSON/text representation instead of matching individual elements — silently diverging from the in-memory backend's set-intersection semantics. SQLite now uses `json_each()` over a `json_type`-guarded array/scalar wrapper; PostgreSQL now uses the `?|` "any array element" operator alongside the existing scalar `= ANY(...)` path
- **Fixed along the way**: `FAISSStore.filter_by_metadata(limit=0)` returned one result instead of zero, because the limit check ran after appending the current match
- **Fixed along the way**: `MilvusStore`'s metadata expression builder rendered `NaN`/`Infinity` filter values as bare unquoted tokens, producing an invalid Milvus expression whose server-side rejection was then swallowed by a broad `except`, indistinguishable from "no matches"; these values are now rejected up front with a clear `ValidationError`
- New/expanded test coverage in `tests/vector_store/test_backend_metadata_filtering.py` (all 7 backends, including the Pinecone dimension/zero-vector, PgVector boolean-list, FAISS `limit=0`, and Milvus `NaN` regressions) and `tests/vector_store/test_sqlite_vec_store.py` (new `TestSQLiteVecStoreFilterByMetadata`, run against the real `sqlite-vec` extension, including the array-vs-scalar intersection case)
## [0.6.5] - 2026-08-11
### Added
+36
View File
@@ -490,6 +490,42 @@ class FAISSStore:
return self.index.get_metadata(vector_id)
return None
def filter_by_metadata(
self, filters: Dict[str, Any], limit: int = 10
) -> List[Dict[str, Any]]:
"""
Filter stored vectors by metadata.
Args:
filters: Metadata filter criteria
limit: Maximum number of results
Returns:
List of matching result dicts with 'id', 'metadata', and 'vector'
"""
if self.index is None or not hasattr(self.index, "metadata"):
return []
from .vector_store import _matches_filter
if limit <= 0:
return []
results = []
for vector_id, metadata in self.index.metadata.items():
if _matches_filter(metadata, filters):
results.append(
{
"id": vector_id,
"metadata": metadata,
"vector": self.get_vector(vector_id),
}
)
if len(results) >= limit:
break
return results
def get_stats(self) -> Dict[str, Any]:
"""Get index statistics."""
if self.index is None:
+106 -6
View File
@@ -35,6 +35,8 @@ Author: Semantica Contributors
License: MIT
"""
import math
import re
from typing import Any, Dict, List, Optional, Union
import numpy as np
@@ -43,6 +45,45 @@ from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
def _validate_milvus_key(key: str) -> str:
"""Validate and escape a metadata filter key for Milvus queries."""
if not key or not isinstance(key, str) or not re.match(r"^[a-zA-Z0-9_.-]+$", key):
raise ValidationError(f"Invalid metadata filter key: '{key}'")
return key.replace("\\", "\\\\").replace('"', '\\"')
def _format_milvus_value(val: Any) -> str:
"""Format and escape a filter value for Milvus expression syntax."""
if isinstance(val, bool):
return "true" if val else "false"
elif isinstance(val, (int, float)):
if isinstance(val, float) and not math.isfinite(val):
raise ValidationError(
f"Invalid metadata filter value: {val!r}. NaN/Infinity are not "
"valid Milvus expression literals."
)
return str(val)
elif isinstance(val, str):
escaped = (
val.replace("\\", "\\\\")
.replace('"', '\\"')
.replace("\n", "\\n")
.replace("\r", "\\r")
)
return f'"{escaped}"'
elif val is None:
return "null"
else:
escaped = (
str(val)
.replace("\\", "\\\\")
.replace('"', '\\"')
.replace("\n", "\\n")
.replace("\r", "\\r")
)
return f'"{escaped}"'
# Optional Milvus import
try:
from pymilvus import (
@@ -550,11 +591,11 @@ class MilvusStore:
"""Get vector by ID."""
if not MILVUS_AVAILABLE or not self.collection:
return None
try:
safe_id = vector_id.replace('"', '\\"')
safe_id = vector_id.replace("\\", "\\\\").replace('"', '\\"')
res = self.collection.collection.query(
expr=f'id == "{safe_id}"',
expr=f'id == "{safe_id}"',
output_fields=["vector"]
)
if res and len(res) > 0:
@@ -567,11 +608,11 @@ class MilvusStore:
"""Get metadata by ID."""
if not MILVUS_AVAILABLE or not self.collection:
return None
try:
safe_id = vector_id.replace('"', '\\"')
safe_id = vector_id.replace("\\", "\\\\").replace('"', '\\"')
res = self.collection.collection.query(
expr=f'id == "{safe_id}"',
expr=f'id == "{safe_id}"',
output_fields=["metadata"]
)
if res and len(res) > 0:
@@ -580,6 +621,65 @@ class MilvusStore:
except Exception:
return None
def filter_by_metadata(
self, filters: Dict[str, Any], limit: int = 10
) -> List[Dict[str, Any]]:
"""
Filter vectors by metadata using Milvus expression filtering.
Args:
filters: Metadata filter criteria
limit: Maximum number of results
Returns:
List of matching result dicts with 'id', 'metadata', and 'vector'
"""
if self.collection is None or not MILVUS_AVAILABLE:
return []
expr_parts = []
if filters:
for key, value in filters.items():
safe_key = _validate_milvus_key(key)
if isinstance(value, dict):
if "min" in value and value["min"] is not None:
min_val = _format_milvus_value(value["min"])
expr_parts.append(f'metadata["{safe_key}"] >= {min_val}')
if "max" in value and value["max"] is not None:
max_val = _format_milvus_value(value["max"])
expr_parts.append(f'metadata["{safe_key}"] <= {max_val}')
elif isinstance(value, list):
formatted_vals = [_format_milvus_value(v) for v in value]
expr_parts.append(
f'metadata["{safe_key}"] in [{", ".join(formatted_vals)}]'
)
else:
formatted_val = _format_milvus_value(value)
expr_parts.append(f'metadata["{safe_key}"] == {formatted_val}')
expr = " and ".join(expr_parts) if expr_parts else "id != ''"
try:
query_results = self.collection.collection.query(
expr=expr,
limit=limit,
output_fields=["id", "vector", "metadata"],
)
results = []
for item in query_results:
vec = item.get("vector")
results.append(
{
"id": str(item.get("id")),
"metadata": item.get("metadata") or {},
"vector": np.array(vec) if vec is not None else None,
}
)
return results
except Exception as e:
self.logger.warning(f"Failed to query Milvus vectors by metadata expression: {e}")
return []
def get_stats(self, collection_name: Optional[str] = None) -> Dict[str, Any]:
"""Get collection statistics."""
if self.collection is None and collection_name:
+111
View File
@@ -63,6 +63,7 @@ except (ImportError, OSError):
except (ImportError, OSError):
PSYCOPG2_AVAILABLE = False
psycopg2 = None
psycopg_sql = None
# Optional pgvector import
try:
@@ -655,6 +656,116 @@ class PgVectorStore:
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
return None
def filter_by_metadata(
self, filters: Dict[str, Any], limit: int = 10
) -> List[Dict[str, Any]]:
"""
Filter stored vectors by metadata using PostgreSQL JSONB queries.
Args:
filters: Dictionary of metadata filter conditions
limit: Maximum number of results
Returns:
List of results containing id, metadata, and vector
"""
if not PSYCOPG3_AVAILABLE and not PSYCOPG2_AVAILABLE:
raise ProcessingError(
"Neither psycopg3 nor psycopg2 is available. "
"Install with: pip install psycopg[binary] or psycopg2-binary"
)
filter_conditions = []
filter_values = []
if filters:
for key, value in filters.items():
if not self._is_safe_identifier(key):
raise ValidationError(
f"Invalid filter key: {key!r}. "
"Keys must be alphanumeric with underscores/hyphens only."
)
if isinstance(value, dict):
if "min" in value and value["min"] is not None:
filter_conditions.append(psycopg_sql.SQL("(metadata->>{})::numeric >= %s").format(
psycopg_sql.Literal(key)
))
filter_values.append(value["min"])
if "max" in value and value["max"] is not None:
filter_conditions.append(psycopg_sql.SQL("(metadata->>{})::numeric <= %s").format(
psycopg_sql.Literal(key)
))
filter_values.append(value["max"])
elif isinstance(value, list):
# Same lowercase-bool rule as the scalar branch below: ->> renders
# JSON booleans as 'true'/'false', not str()'s 'True'/'False'.
str_values = [
('true' if v else 'false') if isinstance(v, bool) else str(v)
for v in value
]
# If the metadata value at this key is itself a JSON array, match on
# intersection (mirrors the in-memory backend's set-intersection
# semantics) via the jsonb `?|` "any array element matches" operator;
# otherwise fall back to plain scalar membership. `->>` renders an
# array as its whole text representation, so it cannot be reused for
# the array case.
filter_conditions.append(psycopg_sql.SQL(
"(CASE WHEN jsonb_typeof(metadata->{0}) = 'array' "
"THEN metadata->{0} ?| %s "
"ELSE metadata->>{0} = ANY(%s) END)"
).format(psycopg_sql.Literal(key)))
filter_values.append(str_values)
filter_values.append(str_values)
elif isinstance(value, bool):
# PostgreSQL JSONB ->> returns lowercase 'true'/'false' for JSON booleans.
# str(True)='True' and str(False)='False' would never match; use the
# correct lowercase text that ->> actually produces.
filter_conditions.append(psycopg_sql.SQL("metadata->>{} = %s").format(
psycopg_sql.Literal(key)
))
filter_values.append('true' if value else 'false')
else:
filter_conditions.append(psycopg_sql.SQL("metadata->>{} = %s").format(
psycopg_sql.Literal(key)
))
filter_values.append(str(value))
if filter_conditions:
where_clause = psycopg_sql.SQL(" WHERE ") + psycopg_sql.SQL(" AND ").join(filter_conditions)
else:
where_clause = psycopg_sql.SQL("")
query_sql = psycopg_sql.SQL("""
SELECT id, vector, metadata
FROM {table}
{where}
LIMIT %s
""").format(
table=psycopg_sql.Identifier(self.table_name),
where=where_clause
)
params = filter_values + [limit]
with self._get_connection() as conn:
try:
cur = conn.cursor()
cur.execute(query_sql, params)
rows = cur.fetchall()
cur.close()
results = []
for row in rows:
vec_id, vector_data, meta = row
vec = np.array(vector_data) if vector_data is not None else None
results.append({
"id": vec_id,
"metadata": meta if isinstance(meta, dict) else json.loads(meta) if meta else {},
"vector": vec
})
return results
except Exception as e:
raise ProcessingError(f"Failed to filter vectors by metadata: {str(e)}") from e
def create_index(
self,
index_type: str = "hnsw",
+96 -2
View File
@@ -75,7 +75,7 @@ class PineconeClient:
try:
# Default to serverless spec if not provided
if spec is None:
if spec is None and ServerlessSpec is not None:
spec = ServerlessSpec(cloud="aws", region="us-east-1")
# Map metric names
@@ -327,6 +327,7 @@ class PineconeStore:
self.api_key = api_key or config.get("api_key")
self.environment = environment or config.get("environment")
self.dimension: Optional[int] = config.get("dimension")
self.client: Optional[PineconeClient] = None
self.index: Optional[PineconeIndex] = None
@@ -395,7 +396,7 @@ class PineconeStore:
try:
# Create index spec if not provided
if spec is None:
if spec is None and ServerlessSpec is not None:
spec = ServerlessSpec(cloud="aws", region="us-east-1")
self.client.create_index(index_name, dimension, metric, spec, **kwargs)
@@ -404,6 +405,7 @@ class PineconeStore:
pinecone_index = self.client.get_index(index_name)
self.index = PineconeIndex(pinecone_index)
self.search_engine = PineconeSearch(self.index)
self.dimension = dimension
self.logger.info(f"Created Pinecone index: {index_name}")
return self.index
@@ -431,6 +433,13 @@ class PineconeStore:
pinecone_index = self.client.get_index(index_name)
self.index = PineconeIndex(pinecone_index)
self.search_engine = PineconeSearch(self.index)
if self.dimension is None:
try:
stats = self.index.describe_index_stats()
if stats and isinstance(stats, dict) and stats.get("dimension"):
self.dimension = int(stats["dimension"])
except Exception as e:
self.logger.warning(f"Could not determine index dimension for '{index_name}': {e}")
return self.index
except Exception as e:
raise ProcessingError(f"Failed to get index: {str(e)}")
@@ -516,6 +525,9 @@ class PineconeStore:
else:
vector_list.append(list(vector))
if self.dimension is None and vector_list:
self.dimension = len(vector_list[0])
self.progress_tracker.update_tracking(
tracking_id, message="Upserting vectors to index..."
)
@@ -582,6 +594,9 @@ class PineconeStore:
else:
query_vector = list(query_vector)
if self.dimension is None and query_vector:
self.dimension = len(query_vector)
results = self.search_engine.similarity_search(
np.array(query_vector), k, filter, namespace, **options
)
@@ -641,6 +656,85 @@ class PineconeStore:
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
return None
def filter_by_metadata(
self, filters: Dict[str, Any], limit: int = 10, namespace: str = ""
) -> List[Dict[str, Any]]:
"""
Filter vectors by metadata using Pinecone metadata filters.
Args:
filters: Metadata filter criteria
limit: Maximum number of results
namespace: Namespace to search in
Returns:
List of matching result dicts with 'id', 'metadata', and 'vector'
"""
if self.index is None or not PINECONE_AVAILABLE:
return []
dimension = self.dimension
if dimension is None:
try:
stats = self.index.describe_index_stats()
if stats and isinstance(stats, dict) and stats.get("dimension"):
dimension = int(stats["dimension"])
self.dimension = dimension
except Exception:
pass
if not dimension:
raise ProcessingError(
"Index dimension is unknown. Please specify 'dimension' when initializing PineconeStore "
"or call create_index()/get_index() first."
)
pinecone_filter = {}
if filters:
for key, value in filters.items():
if isinstance(value, dict):
cond = {}
if "min" in value and value["min"] is not None:
cond["$gte"] = value["min"]
if "max" in value and value["max"] is not None:
cond["$lte"] = value["max"]
if cond:
pinecone_filter[key] = cond
elif isinstance(value, list):
pinecone_filter[key] = {"$in": value}
else:
pinecone_filter[key] = value
# A literal zero vector is rejected by Pinecone for cosine-metric indexes
# ("Query vector must not be the zero vector"). Use a unit vector instead so
# this works regardless of the index's distance metric; since this call only
# cares about which vectors match `filter`, not similarity ranking, any
# fixed non-zero query vector is an equally valid probe.
dummy_vector = [1.0 / (dimension ** 0.5)] * dimension
try:
response = self.index.index.query(
vector=dummy_vector,
top_k=limit,
filter=pinecone_filter if pinecone_filter else None,
namespace=namespace,
include_metadata=True,
include_values=True,
)
results = []
for match in response.matches:
results.append(
{
"id": match.id,
"metadata": match.metadata or {},
"vector": np.array(match.values) if match.values else None,
}
)
return results
except Exception as e:
self.logger.warning(f"Failed to filter Pinecone vectors by metadata: {e}")
return []
def fetch_vectors(
self, vector_ids: List[str], namespace: str = "", **options
) -> Dict[str, Any]:
+66
View File
@@ -49,8 +49,10 @@ try:
Distance,
FieldCondition,
Filter,
MatchAny,
MatchValue,
PointStruct,
Range,
VectorParams,
)
@@ -63,7 +65,9 @@ except (ImportError, OSError):
PointStruct = None
Filter = None
FieldCondition = None
MatchAny = None
MatchValue = None
Range = None
CollectionStatus = None
@@ -538,6 +542,68 @@ class QdrantStore:
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
return None
def filter_by_metadata(
self, filters: Dict[str, Any], limit: int = 10
) -> List[Dict[str, Any]]:
"""
Filter vectors by metadata using Qdrant payload filtering.
Args:
filters: Metadata filter criteria
limit: Maximum number of results
Returns:
List of matching result dicts with 'id', 'metadata', and 'vector'
"""
if self.collection is None or self.client is None or not QDRANT_AVAILABLE:
return []
conditions = []
if filters:
for key, value in filters.items():
if isinstance(value, dict):
cond_kwargs = {}
if "min" in value and value["min"] is not None:
cond_kwargs["gte"] = value["min"]
if "max" in value and value["max"] is not None:
cond_kwargs["lte"] = value["max"]
if cond_kwargs:
conditions.append(
FieldCondition(key=key, range=Range(**cond_kwargs))
)
elif isinstance(value, list):
conditions.append(
FieldCondition(key=key, match=MatchAny(any=value))
)
else:
conditions.append(
FieldCondition(key=key, match=MatchValue(value=value))
)
query_filter = Filter(must=conditions) if conditions else None
try:
records, _ = self.client.scroll(
collection_name=self.collection.collection_name,
scroll_filter=query_filter,
limit=limit,
with_payload=True,
with_vectors=True,
)
results = []
for rec in records:
results.append(
{
"id": str(rec.id),
"metadata": rec.payload or {},
"vector": np.array(rec.vector) if rec.vector is not None else None,
}
)
return results
except Exception as e:
self.logger.warning(f"Failed to scroll Qdrant points by metadata filter: {e}")
return []
def delete_vectors(
self, point_ids: List[Union[str, int]], **options
) -> Dict[str, Any]:
@@ -616,6 +616,96 @@ class SQLiteVecStore:
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
return None
def filter_by_metadata(
self, filters: Dict[str, Any], limit: int = 10
) -> List[Dict[str, Any]]:
"""
Filter stored vectors by metadata using SQLite JSON functions.
Args:
filters: Dictionary of metadata filter conditions
limit: Maximum number of results
Returns:
List of results containing id, metadata, and vector
"""
filter_conditions = []
filter_params = []
if filters:
for key, value in filters.items():
if not self._is_safe_identifier(key):
raise ValidationError(
f"Invalid filter key: {key!r}. "
"Keys must start with a letter or underscore and contain "
"only alphanumeric characters and underscores."
)
if isinstance(value, dict):
if "min" in value and value["min"] is not None:
filter_conditions.append(f"CAST(json_extract(metadata, '$.{key}') AS NUMERIC) >= ?")
filter_params.append(value["min"])
if "max" in value and value["max"] is not None:
filter_conditions.append(f"CAST(json_extract(metadata, '$.{key}') AS NUMERIC) <= ?")
filter_params.append(value["max"])
elif isinstance(value, list):
# If the metadata value at this key is itself a JSON array, match on
# intersection (mirrors the in-memory backend's set-intersection
# semantics); otherwise fall back to plain scalar membership. Both
# cases are handled uniformly via json_each: a non-array value is
# wrapped in a one-element array first so json_each always sees a
# valid JSON array to iterate.
placeholders = ", ".join(["?"] * len(value))
filter_conditions.append(
f"EXISTS (SELECT 1 FROM json_each("
f" CASE WHEN json_type(metadata, '$.{key}') = 'array'"
f" THEN json_extract(metadata, '$.{key}')"
f" ELSE json_array(json_extract(metadata, '$.{key}'))"
f" END"
f") je WHERE je.value IN ({placeholders}))"
)
filter_params.extend([str(v) if not isinstance(v, (int, float, bool)) else v for v in value])
else:
filter_conditions.append(f"json_extract(metadata, '$.{key}') = ?")
if isinstance(value, bool):
filter_params.append(1 if value else 0)
else:
filter_params.append(value)
where_clause = ""
if filter_conditions:
where_clause = " WHERE " + " AND ".join(filter_conditions)
query_sql = f"""
SELECT id, embedding, metadata
FROM {self.table_name}
{where_clause}
LIMIT ?
"""
params = filter_params + [limit]
with self._lock, self._get_connection() as conn:
try:
cur = conn.cursor()
cur.execute(query_sql, params)
rows = cur.fetchall()
cur.close()
results = []
for row in rows:
vec_id, embedding_blob, meta_json = row
vec = None
if embedding_blob:
vec = np.frombuffer(embedding_blob, dtype=np.float32).copy()
results.append({
"id": vec_id,
"metadata": json.loads(meta_json) if meta_json else {},
"vector": vec
})
return results
except Exception as e:
raise ProcessingError(f"Failed to filter by metadata: {str(e)}") from e
def create_index(
self,
index_type: str = "hnsw",
+42 -51
View File
@@ -1185,75 +1185,29 @@ class VectorStore:
from datetime import datetime, timedelta
cutoff = datetime.now() - timedelta(days=7)
filters["timestamp"] = {"min": cutoff.isoformat()}
return filters
def _filter_by_metadata(self, filters: Dict[str, Any], limit: int) -> List[Dict[str, Any]]:
"""Filter decisions by metadata only."""
if self._backend_store is not None:
# No real backend wrapper implements filter_by_metadata; the only
# codebase hit is HybridSearch.filter_by_metadata which has a
# completely different signature (results, MetadataFilter) and is
# never stored in _backend_store. Silently returning [] here would
# be wrong — the caller (filter_decisions) would report zero matches
# for a query that simply isn't supported, indistinguishable from a
# genuine empty result. This is the same situation as get_vector()
# and get_metadata() (#843 fix): when a backend exists but cannot
# fulfil the request, raise NotImplementedError so the caller knows
# the backend lacks this capability rather than assuming no data.
if hasattr(self._backend_store, "filter_by_metadata"):
return self._backend_store.filter_by_metadata(filters, limit)
return self._backend_store.filter_by_metadata(filters=filters, limit=limit)
raise NotImplementedError(
f"Backend store {type(self._backend_store).__name__} does not "
"implement filter_by_metadata. Metadata-only filtering via "
"filter_decisions(query=None, ...) is only supported for the "
"inmemory backend. Pass a query string to use search_decisions() "
"filter_decisions(query=None, ...) is only supported for backends "
"that implement filter_by_metadata. Pass a query string to use search_decisions() "
"instead, which is supported by all backends."
)
results = []
for vector_id, metadata in self.metadata.items():
match = True
for key, value in filters.items():
if key not in metadata:
match = False
break
if isinstance(value, dict):
# Handle range filters
metadata_value = metadata[key]
if "min" in value and metadata_value < value["min"]:
match = False
break
if "max" in value and metadata_value > value["max"]:
match = False
break
elif isinstance(value, list):
# Handle list membership
metadata_value = metadata[key]
if isinstance(metadata_value, list):
# Both are lists - check for intersection
if not set(metadata_value) & set(value):
match = False
break
else:
# Metadata value is scalar, check if it's in the filter list
if metadata_value not in value:
match = False
break
else:
# Handle exact match
if metadata[key] != value:
match = False
break
if match:
if _matches_filter(metadata, filters):
results.append({
"id": vector_id,
"metadata": metadata,
"vector": self.get_vector(vector_id)
"vector": self.vectors.get(vector_id)
})
if len(results) >= limit:
@@ -1262,6 +1216,43 @@ class VectorStore:
return results
def _matches_filter(metadata: Dict[str, Any], filters: Dict[str, Any]) -> bool:
"""Check if metadata dictionary matches filter criteria."""
if not filters:
return True
if metadata is None:
return False
for key, value in filters.items():
if key not in metadata:
return False
metadata_value = metadata[key]
if isinstance(value, dict):
# Handle range filters
if "min" in value and value["min"] is not None:
if metadata_value is None or metadata_value < value["min"]:
return False
if "max" in value and value["max"] is not None:
if metadata_value is None or metadata_value > value["max"]:
return False
elif isinstance(value, list):
# Handle list membership
if isinstance(metadata_value, list):
if not (set(metadata_value) & set(value)):
return False
else:
if metadata_value not in value:
return False
else:
# Handle exact match
if metadata_value != value:
return False
return True
class VectorIndexer:
"""Vector indexing engine."""
+167
View File
@@ -445,6 +445,173 @@ class WeaviateStore:
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
return None
def _build_weaviate_filter(self, filters: Dict[str, Any]) -> Any:
"""Build native Weaviate Filter object from metadata filter dictionary."""
if not filters or not WEAVIATE_AVAILABLE:
return None
Filter = None
try:
from weaviate.classes.query import Filter
except (ImportError, AttributeError):
try:
if weaviate and hasattr(weaviate, "classes") and hasattr(weaviate.classes, "query"):
Filter = getattr(weaviate.classes.query, "Filter", None)
except AttributeError:
Filter = None
if Filter is None:
return None
try:
conditions = []
for key, value in filters.items():
if isinstance(value, dict):
if "min" in value and value["min"] is not None:
conditions.append(Filter.by_property(key).greater_or_equal(value["min"]))
if "max" in value and value["max"] is not None:
conditions.append(Filter.by_property(key).less_or_equal(value["max"]))
elif isinstance(value, list):
conditions.append(Filter.by_property(key).contains_any(value))
else:
conditions.append(Filter.by_property(key).equal(value))
if not conditions:
return None
weaviate_filter = conditions[0]
for cond in conditions[1:]:
weaviate_filter = weaviate_filter & cond
return weaviate_filter
except Exception as e:
self.logger.debug(f"Could not build native Weaviate filter: {e}")
return None
def filter_by_metadata(
self, filters: Dict[str, Any], limit: int = 10
) -> List[Dict[str, Any]]:
"""
Filter stored objects by metadata in Weaviate.
Args:
filters: Metadata filter criteria
limit: Maximum number of results
Returns:
List of matching result dicts with 'id', 'metadata', and 'vector'
"""
if self.collection is None or not WEAVIATE_AVAILABLE:
return []
from .vector_store import _matches_filter
native_filter = self._build_weaviate_filter(filters) if filters else None
results = []
seen_ids = set()
after_cursor = None
scanned_count = 0
page_size = max(limit, 100)
use_native_filter = native_filter is not None
try:
while len(results) < limit:
kwargs = {"limit": page_size, "include_vector": True}
if use_native_filter and native_filter is not None:
kwargs["filters"] = native_filter
if after_cursor is not None:
kwargs["after"] = after_cursor
try:
objs = self.collection.query.fetch_objects(**kwargs)
except TypeError as te:
# Handle kwargs incompatibility (e.g. mock or client version without filters/after)
if "filters" in kwargs:
use_native_filter = False
kwargs.pop("filters", None)
try:
objs = self.collection.query.fetch_objects(**kwargs)
except TypeError:
if "after" in kwargs:
kwargs.pop("after", None)
kwargs["offset"] = scanned_count
try:
objs = self.collection.query.fetch_objects(**kwargs)
except TypeError:
kwargs.pop("offset", None)
objs = self.collection.query.fetch_objects(**kwargs)
elif "after" in kwargs:
kwargs.pop("after", None)
kwargs["offset"] = scanned_count
try:
objs = self.collection.query.fetch_objects(**kwargs)
except TypeError:
kwargs.pop("offset", None)
objs = self.collection.query.fetch_objects(**kwargs)
else:
raise te
except Exception as fe:
if use_native_filter:
self.logger.warning(
f"Native Weaviate filter query failed, falling back to paginated fetch: {fe}"
)
use_native_filter = False
kwargs.pop("filters", None)
objs = self.collection.query.fetch_objects(**kwargs)
else:
raise fe
if not objs or not getattr(objs, "objects", None):
break
batch_objects = objs.objects
if not batch_objects:
break
new_objects_found = False
for obj in batch_objects:
obj_id = str(obj.uuid) if hasattr(obj, "uuid") and obj.uuid is not None else None
if obj_id:
if obj_id in seen_ids:
continue
seen_ids.add(obj_id)
new_objects_found = True
properties = getattr(obj, "properties", None) or {}
if _matches_filter(properties, filters):
vector = None
if hasattr(obj, "vector") and obj.vector:
vector = np.array(obj.vector)
results.append(
{
"id": obj_id,
"metadata": properties,
"vector": vector,
}
)
if len(results) >= limit:
break
if not new_objects_found:
break
scanned_count += len(batch_objects)
if len(batch_objects) < page_size:
break
last_obj = batch_objects[-1]
if hasattr(last_obj, "uuid") and last_obj.uuid is not None:
after_cursor = str(last_obj.uuid)
else:
break
return results
except Exception as e:
self.logger.warning(f"Failed to fetch Weaviate objects by metadata filter: {e}")
return results if results else []
def query_vectors(
self,
query_vector: np.ndarray,
@@ -0,0 +1,500 @@
import unittest
from unittest.mock import MagicMock, patch
import numpy as np
from semantica.vector_store.faiss_store import FAISSStore
from semantica.vector_store.qdrant_store import QdrantStore
from semantica.vector_store.pinecone_store import PineconeStore
from semantica.vector_store.milvus_store import MilvusStore
from semantica.vector_store.pgvector_store import PgVectorStore
from semantica.vector_store.weaviate_store import WeaviateStore
from semantica.utils.exceptions import ProcessingError, ValidationError
class TestBackendMetadataFiltering(unittest.TestCase):
def test_faiss_store_filter_by_metadata(self):
store = FAISSStore(dimension=2)
mock_index = MagicMock()
mock_index.metadata = {
"v1": {"category": "finance", "score": 10},
"v2": {"category": "tech", "score": 20},
}
mock_index.get_vector.side_effect = lambda vid: np.array([1.0, 0.0]) if vid == "v1" else np.array([0.0, 1.0])
store.index = mock_index
results = store.filter_by_metadata({"category": "finance"}, limit=10)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], "v1")
self.assertEqual(results[0]["metadata"], {"category": "finance", "score": 10})
@patch('semantica.vector_store.qdrant_store.FieldCondition', MagicMock())
@patch('semantica.vector_store.qdrant_store.MatchValue', MagicMock())
@patch('semantica.vector_store.qdrant_store.Filter', MagicMock())
@patch('semantica.vector_store.qdrant_store.QDRANT_AVAILABLE', True)
def test_qdrant_store_filter_by_metadata(self):
store = QdrantStore()
mock_collection = MagicMock()
mock_collection.collection_name = "test_coll"
store.collection = mock_collection
mock_client = MagicMock()
rec = MagicMock()
rec.id = "q1"
rec.payload = {"env": "prod"}
rec.vector = [0.1, 0.2]
mock_client.scroll.return_value = ([rec], None)
store.client = mock_client
results = store.filter_by_metadata({"env": "prod"}, limit=5)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], "q1")
self.assertEqual(results[0]["metadata"], {"env": "prod"})
mock_client.scroll.assert_called_once()
@patch('semantica.vector_store.qdrant_store.Range', MagicMock())
@patch('semantica.vector_store.qdrant_store.FieldCondition', MagicMock())
@patch('semantica.vector_store.qdrant_store.Filter', MagicMock())
@patch('semantica.vector_store.qdrant_store.QDRANT_AVAILABLE', True)
def test_qdrant_store_filter_by_metadata_range(self):
"""Range filters must construct Range objects and not raise NameError."""
store = QdrantStore()
mock_collection = MagicMock()
mock_collection.collection_name = "test_coll"
store.collection = mock_collection
mock_client = MagicMock()
rec = MagicMock()
rec.id = "r1"
rec.payload = {"score": 8}
rec.vector = [0.3, 0.4]
mock_client.scroll.return_value = ([rec], None)
store.client = mock_client
# min-only range
results = store.filter_by_metadata({"score": {"min": 5}}, limit=10)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], "r1")
mock_client.scroll.assert_called()
# Verify Range was actually called to build the condition (not skipped)
import semantica.vector_store.qdrant_store as qs_mod
qs_mod.Range.assert_called()
@patch('semantica.vector_store.qdrant_store.Range', MagicMock())
@patch('semantica.vector_store.qdrant_store.FieldCondition', MagicMock())
@patch('semantica.vector_store.qdrant_store.Filter', MagicMock())
@patch('semantica.vector_store.qdrant_store.QDRANT_AVAILABLE', True)
def test_qdrant_store_filter_by_metadata_range_min_and_max(self):
"""Range filters with both min and max must construct Range with both gte and lte."""
store = QdrantStore()
mock_collection = MagicMock()
mock_collection.collection_name = "test_coll"
store.collection = mock_collection
mock_client = MagicMock()
mock_client.scroll.return_value = ([], None)
store.client = mock_client
store.filter_by_metadata({"score": {"min": 5, "max": 10}}, limit=10)
import semantica.vector_store.qdrant_store as qs_mod
# Range must have been called with gte and lte
qs_mod.Range.assert_called_with(gte=5, lte=10)
@patch('semantica.vector_store.qdrant_store.MatchAny', MagicMock())
@patch('semantica.vector_store.qdrant_store.FieldCondition', MagicMock())
@patch('semantica.vector_store.qdrant_store.Filter', MagicMock())
@patch('semantica.vector_store.qdrant_store.QDRANT_AVAILABLE', True)
def test_qdrant_store_filter_by_metadata_list(self):
"""List filters must construct MatchAny objects and not raise NameError."""
store = QdrantStore()
mock_collection = MagicMock()
mock_collection.collection_name = "test_coll"
store.collection = mock_collection
mock_client = MagicMock()
rec = MagicMock()
rec.id = "l1"
rec.payload = {"tags": "python"}
rec.vector = [0.5, 0.6]
mock_client.scroll.return_value = ([rec], None)
store.client = mock_client
results = store.filter_by_metadata({"tags": ["python", "ml"]}, limit=10)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], "l1")
mock_client.scroll.assert_called()
# Verify MatchAny was actually called with the filter list
import semantica.vector_store.qdrant_store as qs_mod
qs_mod.MatchAny.assert_called_with(any=["python", "ml"])
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
def test_pinecone_store_filter_by_metadata(self):
store = PineconeStore(dimension=2)
mock_index_wrapper = MagicMock()
mock_inner_index = MagicMock()
match_obj = MagicMock()
match_obj.id = "p1"
match_obj.metadata = {"status": "active"}
match_obj.values = [0.1, 0.9]
response = MagicMock()
response.matches = [match_obj]
mock_inner_index.query.return_value = response
mock_index_wrapper.index = mock_inner_index
store.index = mock_index_wrapper
results = store.filter_by_metadata({"status": "active"}, limit=5)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], "p1")
self.assertEqual(results[0]["metadata"], {"status": "active"})
# Assert query vector dimension matches store.dimension (2)
mock_inner_index.query.assert_called_once()
query_kw = mock_inner_index.query.call_args[1]
self.assertEqual(len(query_kw["vector"]), 2)
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
def test_pinecone_store_filter_by_metadata_unknown_dimension_raises(self):
store = PineconeStore()
mock_index_wrapper = MagicMock()
mock_index_wrapper.describe_index_stats = MagicMock(return_value={})
store.index = mock_index_wrapper
with self.assertRaises(ProcessingError):
store.filter_by_metadata({"status": "active"}, limit=5)
@patch('semantica.vector_store.milvus_store.MILVUS_AVAILABLE', True)
def test_milvus_store_filter_by_metadata(self):
store = MilvusStore()
mock_coll_wrapper = MagicMock()
mock_inner_coll = MagicMock()
mock_inner_coll.query.return_value = [
{"id": "m1", "vector": [0.3, 0.4], "metadata": {"lang": "py"}}
]
mock_coll_wrapper.collection = mock_inner_coll
store.collection = mock_coll_wrapper
results = store.filter_by_metadata({"lang": "py"}, limit=5)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], "m1")
self.assertEqual(results[0]["metadata"], {"lang": "py"})
@patch('semantica.vector_store.milvus_store.MILVUS_AVAILABLE', True)
def test_milvus_store_filter_by_metadata_escaping(self):
store = MilvusStore()
mock_coll_wrapper = MagicMock()
mock_inner_coll = MagicMock()
mock_inner_coll.query.return_value = []
mock_coll_wrapper.collection = mock_inner_coll
store.collection = mock_coll_wrapper
store.filter_by_metadata(
{
"title": 'John "Jack" Doe',
"active": True,
"tags": ['python', 'c++ "v"'],
},
limit=5,
)
mock_inner_coll.query.assert_called_once()
expr = mock_inner_coll.query.call_args[1]["expr"]
self.assertIn('metadata["title"] == "John \\"Jack\\" Doe"', expr)
self.assertIn('metadata["active"] == true', expr)
self.assertIn('metadata["tags"] in ["python", "c++ \\"v\\""]', expr)
@patch('semantica.vector_store.milvus_store.MILVUS_AVAILABLE', True)
def test_milvus_store_filter_by_metadata_invalid_key_raises(self):
store = MilvusStore()
mock_coll_wrapper = MagicMock()
store.collection = mock_coll_wrapper
with self.assertRaises(ValidationError):
store.filter_by_metadata({'dept" || 1==1 || "': "val"}, limit=5)
@patch('semantica.vector_store.pgvector_store.PSYCOPG3_AVAILABLE', True)
@patch('semantica.vector_store.pgvector_store.psycopg_sql')
def test_pgvector_store_filter_by_metadata(self, mock_sql):
store = object.__new__(PgVectorStore)
store.table_name = "test_vectors"
store._is_safe_identifier = lambda k: True
mock_conn = MagicMock()
mock_cur = MagicMock()
mock_cur.fetchall.return_value = [
("pg1", [0.1, 0.2], {"org": "acme"})
]
mock_conn.cursor.return_value = mock_cur
with patch.object(PgVectorStore, '_get_connection', return_value=MagicMock(__enter__=MagicMock(return_value=mock_conn), __exit__=MagicMock())):
results = store.filter_by_metadata({"org": "acme"}, limit=10)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], "pg1")
self.assertEqual(results[0]["metadata"], {"org": "acme"})
@patch('semantica.vector_store.pgvector_store.PSYCOPG3_AVAILABLE', True)
@patch('semantica.vector_store.pgvector_store.psycopg_sql')
def test_pgvector_store_filter_by_metadata_bool_true(self, mock_sql):
"""Boolean True must become the string 'true' (lowercase) in the SQL parameter.
PostgreSQL JSONB ->> returns 'true' for a JSON boolean true.
str(True) == 'True' would never match; this test guards against regression.
"""
store = object.__new__(PgVectorStore)
store.table_name = "test_vectors"
store._is_safe_identifier = lambda k: True
mock_conn = MagicMock()
mock_cur = MagicMock()
mock_cur.fetchall.return_value = [
("pg2", [0.3, 0.4], {"active": True})
]
mock_conn.cursor.return_value = mock_cur
with patch.object(
PgVectorStore,
'_get_connection',
return_value=MagicMock(
__enter__=MagicMock(return_value=mock_conn),
__exit__=MagicMock(),
),
):
results = store.filter_by_metadata({"active": True}, limit=10)
# Result is returned correctly
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], "pg2")
# The critical assertion: 'true' (not 'True') was passed to execute()
execute_call_args = mock_cur.execute.call_args
self.assertIsNotNone(execute_call_args, "cursor.execute was not called")
params_passed = execute_call_args[0][1] # positional arg 1 is the params list/tuple
self.assertIn('true', params_passed,
"Expected lowercase 'true' in SQL params, got: {}".format(params_passed))
self.assertNotIn('True', params_passed,
"str(True)='True' must NOT appear in SQL params")
@patch('semantica.vector_store.pgvector_store.PSYCOPG3_AVAILABLE', True)
@patch('semantica.vector_store.pgvector_store.psycopg_sql')
def test_pgvector_store_filter_by_metadata_bool_false(self, mock_sql):
"""Boolean False must become the string 'false' (lowercase) in the SQL parameter.
PostgreSQL JSONB ->> returns 'false' for a JSON boolean false.
str(False) == 'False' would never match; this test guards against regression.
"""
store = object.__new__(PgVectorStore)
store.table_name = "test_vectors"
store._is_safe_identifier = lambda k: True
mock_conn = MagicMock()
mock_cur = MagicMock()
mock_cur.fetchall.return_value = [
("pg3", [0.5, 0.6], {"active": False})
]
mock_conn.cursor.return_value = mock_cur
with patch.object(
PgVectorStore,
'_get_connection',
return_value=MagicMock(
__enter__=MagicMock(return_value=mock_conn),
__exit__=MagicMock(),
),
):
results = store.filter_by_metadata({"active": False}, limit=10)
# Result is returned correctly
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], "pg3")
# The critical assertion: 'false' (not 'False') was passed to execute()
execute_call_args = mock_cur.execute.call_args
self.assertIsNotNone(execute_call_args, "cursor.execute was not called")
params_passed = execute_call_args[0][1] # positional arg 1 is the params list/tuple
self.assertIn('false', params_passed,
"Expected lowercase 'false' in SQL params, got: {}".format(params_passed))
self.assertNotIn('False', params_passed,
"str(False)='False' must NOT appear in SQL params")
@patch('semantica.vector_store.pgvector_store.PSYCOPG3_AVAILABLE', True)
@patch('semantica.vector_store.pgvector_store.psycopg_sql')
def test_pgvector_store_filter_by_metadata_bool_list(self, mock_sql):
"""List-valued boolean filters must use lowercase 'true'/'false', not
str(True)/str(False), matching the scalar branch's handling.
"""
store = object.__new__(PgVectorStore)
store.table_name = "test_vectors"
store._is_safe_identifier = lambda k: True
mock_conn = MagicMock()
mock_cur = MagicMock()
mock_cur.fetchall.return_value = [
("pg4", [0.7, 0.8], {"active": True})
]
mock_conn.cursor.return_value = mock_cur
with patch.object(
PgVectorStore,
'_get_connection',
return_value=MagicMock(
__enter__=MagicMock(return_value=mock_conn),
__exit__=MagicMock(),
),
):
results = store.filter_by_metadata({"active": [True, False]}, limit=10)
self.assertEqual(len(results), 1)
execute_call_args = mock_cur.execute.call_args
params_passed = execute_call_args[0][1]
flat_params = [v for p in params_passed for v in (p if isinstance(p, list) else [p])]
self.assertIn('true', flat_params)
self.assertIn('false', flat_params)
self.assertNotIn('True', flat_params)
self.assertNotIn('False', flat_params)
def test_faiss_store_filter_by_metadata_limit_zero(self):
"""limit=0 must return no results, not the first match."""
store = FAISSStore(dimension=2)
mock_index = MagicMock()
mock_index.metadata = {
"v1": {"category": "finance", "score": 10},
}
mock_index.get_vector.return_value = np.array([1.0, 0.0])
store.index = mock_index
results = store.filter_by_metadata({"category": "finance"}, limit=0)
self.assertEqual(results, [])
@patch('semantica.vector_store.milvus_store.MILVUS_AVAILABLE', True)
def test_milvus_store_filter_by_metadata_nan_raises(self):
"""NaN/Infinity are not valid Milvus expression literals and must be
rejected up front rather than silently producing an invalid expression
that gets swallowed by the broad except around the query() call.
"""
store = MilvusStore()
mock_coll_wrapper = MagicMock()
store.collection = mock_coll_wrapper
with self.assertRaises(ValidationError):
store.filter_by_metadata({"score": {"min": float("nan")}}, limit=5)
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
def test_pinecone_store_get_index_sets_dimension_from_stats(self):
"""get_index() must read stats from the returned PineconeIndex wrapper
(self.index), not from a nonexistent method on the store itself.
"""
store = PineconeStore()
mock_client = MagicMock()
mock_pinecone_index = MagicMock()
mock_client.get_index.return_value = mock_pinecone_index
store.client = mock_client
with patch(
'semantica.vector_store.pinecone_store.PineconeIndex'
) as mock_index_cls:
mock_index_instance = MagicMock()
mock_index_instance.describe_index_stats.return_value = {"dimension": 42}
mock_index_cls.return_value = mock_index_instance
store.get_index("my-index")
self.assertEqual(store.dimension, 42)
def test_weaviate_store_filter_by_metadata(self):
store = WeaviateStore()
mock_coll = MagicMock()
obj1 = MagicMock()
obj1.uuid = "w-uuid-1"
obj1.properties = {"dept": "eng"}
obj1.vector = [0.5, 0.5]
objs = MagicMock()
objs.objects = [obj1]
mock_coll.query.fetch_objects.return_value = objs
store.collection = mock_coll
with patch('semantica.vector_store.weaviate_store.WEAVIATE_AVAILABLE', True):
results = store.filter_by_metadata({"dept": "eng"}, limit=5)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], "w-uuid-1")
self.assertEqual(results[0]["metadata"], {"dept": "eng"})
def test_weaviate_store_filter_by_metadata_pagination(self):
"""Test that WeaviateStore.filter_by_metadata paginates beyond page 1 to find matching items."""
store = WeaviateStore()
mock_coll = MagicMock()
# Batch 1: 100 non-matching objects
batch1_objs = []
for i in range(100):
obj = MagicMock()
obj.uuid = f"batch1-uuid-{i}"
obj.properties = {"dept": "hr"}
obj.vector = [0.1, 0.1]
batch1_objs.append(obj)
res1 = MagicMock()
res1.objects = batch1_objs
# Batch 2: 2 matching objects
obj_match1 = MagicMock()
obj_match1.uuid = "match-uuid-1"
obj_match1.properties = {"dept": "eng"}
obj_match1.vector = [0.5, 0.5]
obj_match2 = MagicMock()
obj_match2.uuid = "match-uuid-2"
obj_match2.properties = {"dept": "eng"}
obj_match2.vector = [0.6, 0.6]
res2 = MagicMock()
res2.objects = [obj_match1, obj_match2]
def side_effect(**kwargs):
if kwargs.get("after") == "batch1-uuid-99":
return res2
return res1
mock_coll.query.fetch_objects.side_effect = side_effect
store.collection = mock_coll
with patch('semantica.vector_store.weaviate_store.WEAVIATE_AVAILABLE', True):
results = store.filter_by_metadata({"dept": "eng"}, limit=5)
self.assertEqual(len(results), 2)
self.assertEqual(results[0]["id"], "match-uuid-1")
self.assertEqual(results[1]["id"], "match-uuid-2")
def test_weaviate_store_filter_by_metadata_native_filter(self):
"""Test building native Weaviate filters for exact, range, and list criteria."""
store = WeaviateStore()
mock_filter_cls = MagicMock()
mock_filter_prop = MagicMock()
mock_filter_cls.by_property.return_value = mock_filter_prop
mock_module = MagicMock()
mock_module.classes.query.Filter = mock_filter_cls
with patch('semantica.vector_store.weaviate_store.WEAVIATE_AVAILABLE', True), \
patch('semantica.vector_store.weaviate_store.weaviate', mock_module):
# Test exact match
res = store._build_weaviate_filter({"dept": "eng"})
mock_filter_cls.by_property.assert_called_with("dept")
mock_filter_prop.equal.assert_called_with("eng")
# Test range filter
mock_filter_cls.reset_mock()
mock_filter_prop.reset_mock()
res = store._build_weaviate_filter({"age": {"min": 20, "max": 50}})
mock_filter_cls.by_property.assert_called_with("age")
mock_filter_prop.greater_or_equal.assert_called_with(20)
mock_filter_prop.less_or_equal.assert_called_with(50)
# Test list filter
mock_filter_cls.reset_mock()
mock_filter_prop.reset_mock()
res = store._build_weaviate_filter({"tags": ["a", "b"]})
mock_filter_cls.by_property.assert_called_with("tags")
mock_filter_prop.contains_any.assert_called_with(["a", "b"])
if __name__ == "__main__":
unittest.main()
@@ -773,19 +773,21 @@ class TestBuildDecisionContextFAISSBackend:
class TestFilterByMetadataBackendBehavior:
"""
Requirement (issue #848 follow-up): verify the chosen behavior of
Requirement (issue #848, superseded by #857): verify the behavior of
_filter_by_metadata when a non-inmemory backend is active.
The decision: raise NotImplementedError (matching get_vector / get_metadata
from #843) rather than silently returning [].
#848's original decision was to raise NotImplementedError (matching
get_vector / get_metadata from #843) rather than silently return [],
because at the time zero backend wrappers implemented
filter_by_metadata(filters, limit).
Rationale documented in the production comment:
- Zero backend wrappers implement filter_by_metadata(filters, limit).
- The only codebase hit (HybridSearch.filter_by_metadata) has a completely
different signature and is never stored in _backend_store.
- Returning [] would make filter_decisions(query=None, category="loan")
report "zero matches" when the truth is "capability not available"
indistinguishable from a real empty result and therefore wrong.
#857 gave every persistent backend (FAISS, Qdrant, Pinecone, Milvus,
PgVector, SQLiteVec, Weaviate) a real filter_by_metadata() implementation,
so FAISS-backed filter_decisions(query=None, ...) now returns actual
filtered results instead of raising. The NotImplementedError path itself
is still correct and still covered (see
test_filter_by_metadata_backend_not_implemented in test_vector_store.py)
for a backend that genuinely lacks the method.
"""
def _make_faiss_store(self):
@@ -809,34 +811,26 @@ class TestFilterByMetadataBackendBehavior:
)
return vs, ids
# ── FAISS backend: NotImplementedError, not AttributeError, not [] ── #
# ── FAISS backend: real results, not AttributeError, not [] ── #
def test_filter_by_metadata_faiss_raises_not_implemented(self):
def test_filter_by_metadata_faiss_returns_real_results(self):
"""
filter_decisions(query=None, category='loan') on a FAISS-backed store
must raise NotImplementedError, not AttributeError (old crash) and not
silently return [] (the wrong silent-failure fix).
This test pins the chosen behavior: explicit NotImplementedError matching
the get_vector/get_metadata precedent set by issue #843.
must return the actual matching decisions, not raise AttributeError
(old crash) and not silently return [] (the old NotImplementedError
stand-in from #848, superseded once #857 gave FAISSStore a real
filter_by_metadata()).
"""
vs, _ids = self._make_faiss_store()
with pytest.raises(NotImplementedError) as exc_info:
vs.filter_decisions(query=None, category="loan")
results = vs.filter_decisions(query=None, category="loan")
# Message must name the backend and point to the correct alternative
msg = str(exc_info.value)
assert "FAISSStore" in msg, (
f"Error message should name the backend class, got: {msg!r}"
)
assert "filter_decisions" in msg or "filter_by_metadata" in msg, (
f"Error message should mention the failing method, got: {msg!r}"
)
assert "search_decisions" in msg, (
f"Error message should suggest search_decisions() as the alternative, "
f"got: {msg!r}"
assert isinstance(results, list)
assert len(results) == 2, (
f"Expected 2 loan decisions, got {len(results)}: {results}"
)
for r in results:
assert r["metadata"]["category"] == "loan"
def test_filter_by_metadata_faiss_not_attribute_error(self):
"""
@@ -82,6 +82,7 @@ class TestPineconeStore(unittest.TestCase):
self.assertIsInstance(store.search_engine, PineconeSearch)
store.client.create_index.assert_called_once()
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
def test_upsert_vectors(self, mock_pinecone_client):
"""Test upserting vectors to Pinecone index."""
@@ -105,6 +106,7 @@ class TestPineconeStore(unittest.TestCase):
self.assertEqual(result["upserted_count"], 2)
store.index.upsert_vectors.assert_called_once()
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
def test_search_vectors(self, mock_pinecone_client):
"""Test searching vectors in Pinecone index."""
@@ -128,6 +130,7 @@ class TestPineconeStore(unittest.TestCase):
self.assertEqual(results[0]["id"], "id1")
store.search_engine.similarity_search.assert_called_once()
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
def test_delete_vectors(self, mock_pinecone_client):
"""Test deleting vectors from Pinecone index."""
@@ -148,6 +151,7 @@ class TestPineconeStore(unittest.TestCase):
# Fix: assert called without the empty dict
store.index.delete_vectors.assert_called_once_with(["id1", "id2"], "")
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
def test_fetch_vectors(self, mock_pinecone_client):
"""Test fetching vectors from Pinecone index."""
@@ -413,3 +413,51 @@ class TestSQLiteVecStoreStats:
stats = store.get_stats()
assert stats["vector_count"] == 4
class TestSQLiteVecStoreFilterByMetadata:
"""Test filter_by_metadata, including list-valued metadata handling."""
def test_filter_exact_match(self, store):
vectors = [np.random.rand(128).astype(np.float32) for _ in range(2)]
metadata = [{"category": "finance"}, {"category": "tech"}]
ids = store.add(vectors, metadata, ids=["v1", "v2"])
results = store.filter_by_metadata({"category": "finance"}, limit=10)
assert [r["id"] for r in results] == ["v1"]
def test_filter_scalar_field_against_list_filter(self, store):
"""A scalar metadata value should match via plain IN-list membership."""
vectors = [np.random.rand(128).astype(np.float32) for _ in range(2)]
metadata = [{"category": "finance"}, {"category": "tech"}]
store.add(vectors, metadata, ids=["v1", "v2"])
results = store.filter_by_metadata({"category": ["finance", "ops"]}, limit=10)
assert [r["id"] for r in results] == ["v1"]
def test_filter_array_field_intersects_list_filter(self, store):
"""A list-valued metadata field must match on set intersection with the
filter list, mirroring the in-memory backend's semantics -- not on a
literal comparison of the whole array's JSON text against each candidate.
"""
vectors = [np.random.rand(128).astype(np.float32) for _ in range(3)]
metadata = [
{"tags": ["python", "js"]},
{"tags": ["go"]},
{"tags": ["python", "ml"]},
]
store.add(vectors, metadata, ids=["v1", "v2", "v3"])
results = store.filter_by_metadata({"tags": ["python", "ml"]}, limit=10)
assert {r["id"] for r in results} == {"v1", "v3"}
def test_filter_limit_zero_returns_empty(self, store):
vectors = [np.random.rand(128).astype(np.float32)]
store.add(vectors, [{"category": "finance"}], ids=["v1"])
results = store.filter_by_metadata({"category": "finance"}, limit=0)
assert results == []
+63
View File
@@ -137,6 +137,69 @@ class TestVectorStore(unittest.TestCase):
self.assertTrue(mock_backend.called)
def test_filter_by_metadata_inmemory(self):
"""Test _filter_by_metadata on inmemory backend."""
store = VectorStore(backend="inmemory")
store.metadata = {
"v1": {"category": "finance", "amount": 100, "tags": ["a", "b"]},
"v2": {"category": "finance", "amount": 500, "tags": ["b", "c"]},
"v3": {"category": "tech", "amount": 200, "tags": ["c"]},
}
store.vectors = {
"v1": np.array([0.1]),
"v2": np.array([0.2]),
"v3": np.array([0.3]),
}
# Exact filter
results = store._filter_by_metadata({"category": "finance"}, limit=10)
self.assertEqual(len(results), 2)
res_ids = {r["id"] for r in results}
self.assertEqual(res_ids, {"v1", "v2"})
# Range filter
results = store._filter_by_metadata({"amount": {"min": 150}}, limit=10)
self.assertEqual(len(results), 2)
res_ids = {r["id"] for r in results}
self.assertEqual(res_ids, {"v2", "v3"})
# List intersection filter
results = store._filter_by_metadata({"tags": ["a"]}, limit=10)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], "v1")
def test_filter_by_metadata_persistent_backend_delegation(self):
"""Test that persistent backend delegates filter_by_metadata without AttributeError."""
store = VectorStore(backend="inmemory")
# Simulate persistent backend by deleting self.metadata attribute if any
if hasattr(store, "metadata"):
delattr(store, "metadata")
mock_backend = MagicMock()
mock_backend.filter_by_metadata.return_value = [
{"id": "p1", "metadata": {"category": "test"}, "vector": np.array([0.5])}
]
store._backend_store = mock_backend
store.backend = "faiss"
# Should NOT raise AttributeError: 'VectorStore' object has no attribute 'metadata'
results = store._filter_by_metadata({"category": "test"}, limit=5)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], "p1")
mock_backend.filter_by_metadata.assert_called_once_with(filters={"category": "test"}, limit=5)
def test_filter_by_metadata_backend_not_implemented(self):
"""Test that missing filter_by_metadata method raises NotImplementedError."""
store = VectorStore(backend="inmemory")
if hasattr(store, "metadata"):
delattr(store, "metadata")
store.backend = "unknown"
store._backend_store = object()
with self.assertRaises(NotImplementedError):
store._filter_by_metadata({"key": "val"}, limit=10)
def test_save_load_roundtrip_numpy_vectors(self):
"""save()/load() must handle numpy float32 vectors without raising.