mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Merge branch 'main' into fix/mcp-server-version
This commit is contained in:
@@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`VectorStore._filter_by_metadata()` `AttributeError` on all persistent backends** (#857, closes #849) by @TaherTadpatri
|
||||
- `_filter_by_metadata()` iterated `self.metadata` directly, which only exists on the `inmemory` backend — any persistent backend (`faiss`, `qdrant`, `pinecone`, `milvus`, `pgvector`, `sqlite`, `weaviate`) crashed with `AttributeError` on `filter_decisions(query=None, ...)` / metadata-only filtering. Filtering is now delegated to a native `filter_by_metadata()` implemented on each backend store, using backend-native payload/SQL/JSON filtering (Qdrant `scroll()`, Pinecone `query()`, Milvus expression filters, PostgreSQL JSONB, SQLite `json_extract()`, Weaviate collection filters)
|
||||
- **Fixed along the way**: `PineconeStore.get_index()` and `filter_by_metadata()` called a nonexistent `self.describe_index_stats()` on the store itself (the method only exists on the `PineconeIndex` wrapper returned by `self.index`); the resulting `AttributeError` was silently swallowed, so dimension auto-detection always failed quietly. Now correctly calls `self.index.describe_index_stats()`
|
||||
- **Fixed along the way**: `PineconeStore.filter_by_metadata()` probed for filter-only matches using an all-zero dummy query vector, which Pinecone rejects for cosine-metric indexes — the library's own default — making metadata-only filtering silently non-functional out of the box. Now uses a unit vector instead
|
||||
- **Fixed along the way**: `PgVectorStore.filter_by_metadata()`'s list-filter branch formatted boolean values with `str(v)` (`'True'`/`'False'`), never matching PostgreSQL JSONB's lowercase `'true'`/`'false'` text rendering, even though the equivalent scalar-filter branch already handled this correctly
|
||||
- **Fixed along the way**: list-valued metadata fields (e.g. `{"tags": ["python", "js"]}`) could never match a list filter on the SQLite or PostgreSQL backends, because both extracted the whole array as its JSON/text representation instead of matching individual elements — silently diverging from the in-memory backend's set-intersection semantics. SQLite now uses `json_each()` over a `json_type`-guarded array/scalar wrapper; PostgreSQL now uses the `?|` "any array element" operator alongside the existing scalar `= ANY(...)` path
|
||||
- **Fixed along the way**: `FAISSStore.filter_by_metadata(limit=0)` returned one result instead of zero, because the limit check ran after appending the current match
|
||||
- **Fixed along the way**: `MilvusStore`'s metadata expression builder rendered `NaN`/`Infinity` filter values as bare unquoted tokens, producing an invalid Milvus expression whose server-side rejection was then swallowed by a broad `except`, indistinguishable from "no matches"; these values are now rejected up front with a clear `ValidationError`
|
||||
- New/expanded test coverage in `tests/vector_store/test_backend_metadata_filtering.py` (all 7 backends, including the Pinecone dimension/zero-vector, PgVector boolean-list, FAISS `limit=0`, and Milvus `NaN` regressions) and `tests/vector_store/test_sqlite_vec_store.py` (new `TestSQLiteVecStoreFilterByMetadata`, run against the real `sqlite-vec` extension, including the array-vs-scalar intersection case)
|
||||
|
||||
## [0.6.5] - 2026-08-11
|
||||
|
||||
### Added
|
||||
|
||||
@@ -490,6 +490,42 @@ class FAISSStore:
|
||||
return self.index.get_metadata(vector_id)
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter stored vectors by metadata.
|
||||
|
||||
Args:
|
||||
filters: Metadata filter criteria
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of matching result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if self.index is None or not hasattr(self.index, "metadata"):
|
||||
return []
|
||||
|
||||
from .vector_store import _matches_filter
|
||||
|
||||
if limit <= 0:
|
||||
return []
|
||||
|
||||
results = []
|
||||
for vector_id, metadata in self.index.metadata.items():
|
||||
if _matches_filter(metadata, filters):
|
||||
results.append(
|
||||
{
|
||||
"id": vector_id,
|
||||
"metadata": metadata,
|
||||
"vector": self.get_vector(vector_id),
|
||||
}
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
return results
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""Get index statistics."""
|
||||
if self.index is None:
|
||||
|
||||
@@ -35,6 +35,8 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import math
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
@@ -43,6 +45,45 @@ from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
def _validate_milvus_key(key: str) -> str:
|
||||
"""Validate and escape a metadata filter key for Milvus queries."""
|
||||
if not key or not isinstance(key, str) or not re.match(r"^[a-zA-Z0-9_.-]+$", key):
|
||||
raise ValidationError(f"Invalid metadata filter key: '{key}'")
|
||||
return key.replace("\\", "\\\\").replace('"', '\\"')
|
||||
|
||||
|
||||
def _format_milvus_value(val: Any) -> str:
|
||||
"""Format and escape a filter value for Milvus expression syntax."""
|
||||
if isinstance(val, bool):
|
||||
return "true" if val else "false"
|
||||
elif isinstance(val, (int, float)):
|
||||
if isinstance(val, float) and not math.isfinite(val):
|
||||
raise ValidationError(
|
||||
f"Invalid metadata filter value: {val!r}. NaN/Infinity are not "
|
||||
"valid Milvus expression literals."
|
||||
)
|
||||
return str(val)
|
||||
elif isinstance(val, str):
|
||||
escaped = (
|
||||
val.replace("\\", "\\\\")
|
||||
.replace('"', '\\"')
|
||||
.replace("\n", "\\n")
|
||||
.replace("\r", "\\r")
|
||||
)
|
||||
return f'"{escaped}"'
|
||||
elif val is None:
|
||||
return "null"
|
||||
else:
|
||||
escaped = (
|
||||
str(val)
|
||||
.replace("\\", "\\\\")
|
||||
.replace('"', '\\"')
|
||||
.replace("\n", "\\n")
|
||||
.replace("\r", "\\r")
|
||||
)
|
||||
return f'"{escaped}"'
|
||||
|
||||
# Optional Milvus import
|
||||
try:
|
||||
from pymilvus import (
|
||||
@@ -550,11 +591,11 @@ class MilvusStore:
|
||||
"""Get vector by ID."""
|
||||
if not MILVUS_AVAILABLE or not self.collection:
|
||||
return None
|
||||
|
||||
|
||||
try:
|
||||
safe_id = vector_id.replace('"', '\\"')
|
||||
safe_id = vector_id.replace("\\", "\\\\").replace('"', '\\"')
|
||||
res = self.collection.collection.query(
|
||||
expr=f'id == "{safe_id}"',
|
||||
expr=f'id == "{safe_id}"',
|
||||
output_fields=["vector"]
|
||||
)
|
||||
if res and len(res) > 0:
|
||||
@@ -567,11 +608,11 @@ class MilvusStore:
|
||||
"""Get metadata by ID."""
|
||||
if not MILVUS_AVAILABLE or not self.collection:
|
||||
return None
|
||||
|
||||
|
||||
try:
|
||||
safe_id = vector_id.replace('"', '\\"')
|
||||
safe_id = vector_id.replace("\\", "\\\\").replace('"', '\\"')
|
||||
res = self.collection.collection.query(
|
||||
expr=f'id == "{safe_id}"',
|
||||
expr=f'id == "{safe_id}"',
|
||||
output_fields=["metadata"]
|
||||
)
|
||||
if res and len(res) > 0:
|
||||
@@ -580,6 +621,65 @@ class MilvusStore:
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter vectors by metadata using Milvus expression filtering.
|
||||
|
||||
Args:
|
||||
filters: Metadata filter criteria
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of matching result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if self.collection is None or not MILVUS_AVAILABLE:
|
||||
return []
|
||||
|
||||
expr_parts = []
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
safe_key = _validate_milvus_key(key)
|
||||
if isinstance(value, dict):
|
||||
if "min" in value and value["min"] is not None:
|
||||
min_val = _format_milvus_value(value["min"])
|
||||
expr_parts.append(f'metadata["{safe_key}"] >= {min_val}')
|
||||
if "max" in value and value["max"] is not None:
|
||||
max_val = _format_milvus_value(value["max"])
|
||||
expr_parts.append(f'metadata["{safe_key}"] <= {max_val}')
|
||||
elif isinstance(value, list):
|
||||
formatted_vals = [_format_milvus_value(v) for v in value]
|
||||
expr_parts.append(
|
||||
f'metadata["{safe_key}"] in [{", ".join(formatted_vals)}]'
|
||||
)
|
||||
else:
|
||||
formatted_val = _format_milvus_value(value)
|
||||
expr_parts.append(f'metadata["{safe_key}"] == {formatted_val}')
|
||||
|
||||
expr = " and ".join(expr_parts) if expr_parts else "id != ''"
|
||||
|
||||
try:
|
||||
query_results = self.collection.collection.query(
|
||||
expr=expr,
|
||||
limit=limit,
|
||||
output_fields=["id", "vector", "metadata"],
|
||||
)
|
||||
results = []
|
||||
for item in query_results:
|
||||
vec = item.get("vector")
|
||||
results.append(
|
||||
{
|
||||
"id": str(item.get("id")),
|
||||
"metadata": item.get("metadata") or {},
|
||||
"vector": np.array(vec) if vec is not None else None,
|
||||
}
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to query Milvus vectors by metadata expression: {e}")
|
||||
return []
|
||||
|
||||
def get_stats(self, collection_name: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Get collection statistics."""
|
||||
if self.collection is None and collection_name:
|
||||
|
||||
@@ -63,6 +63,7 @@ except (ImportError, OSError):
|
||||
except (ImportError, OSError):
|
||||
PSYCOPG2_AVAILABLE = False
|
||||
psycopg2 = None
|
||||
psycopg_sql = None
|
||||
|
||||
# Optional pgvector import
|
||||
try:
|
||||
@@ -655,6 +656,116 @@ class PgVectorStore:
|
||||
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter stored vectors by metadata using PostgreSQL JSONB queries.
|
||||
|
||||
Args:
|
||||
filters: Dictionary of metadata filter conditions
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of results containing id, metadata, and vector
|
||||
"""
|
||||
if not PSYCOPG3_AVAILABLE and not PSYCOPG2_AVAILABLE:
|
||||
raise ProcessingError(
|
||||
"Neither psycopg3 nor psycopg2 is available. "
|
||||
"Install with: pip install psycopg[binary] or psycopg2-binary"
|
||||
)
|
||||
|
||||
filter_conditions = []
|
||||
filter_values = []
|
||||
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
if not self._is_safe_identifier(key):
|
||||
raise ValidationError(
|
||||
f"Invalid filter key: {key!r}. "
|
||||
"Keys must be alphanumeric with underscores/hyphens only."
|
||||
)
|
||||
if isinstance(value, dict):
|
||||
if "min" in value and value["min"] is not None:
|
||||
filter_conditions.append(psycopg_sql.SQL("(metadata->>{})::numeric >= %s").format(
|
||||
psycopg_sql.Literal(key)
|
||||
))
|
||||
filter_values.append(value["min"])
|
||||
if "max" in value and value["max"] is not None:
|
||||
filter_conditions.append(psycopg_sql.SQL("(metadata->>{})::numeric <= %s").format(
|
||||
psycopg_sql.Literal(key)
|
||||
))
|
||||
filter_values.append(value["max"])
|
||||
elif isinstance(value, list):
|
||||
# Same lowercase-bool rule as the scalar branch below: ->> renders
|
||||
# JSON booleans as 'true'/'false', not str()'s 'True'/'False'.
|
||||
str_values = [
|
||||
('true' if v else 'false') if isinstance(v, bool) else str(v)
|
||||
for v in value
|
||||
]
|
||||
# If the metadata value at this key is itself a JSON array, match on
|
||||
# intersection (mirrors the in-memory backend's set-intersection
|
||||
# semantics) via the jsonb `?|` "any array element matches" operator;
|
||||
# otherwise fall back to plain scalar membership. `->>` renders an
|
||||
# array as its whole text representation, so it cannot be reused for
|
||||
# the array case.
|
||||
filter_conditions.append(psycopg_sql.SQL(
|
||||
"(CASE WHEN jsonb_typeof(metadata->{0}) = 'array' "
|
||||
"THEN metadata->{0} ?| %s "
|
||||
"ELSE metadata->>{0} = ANY(%s) END)"
|
||||
).format(psycopg_sql.Literal(key)))
|
||||
filter_values.append(str_values)
|
||||
filter_values.append(str_values)
|
||||
elif isinstance(value, bool):
|
||||
# PostgreSQL JSONB ->> returns lowercase 'true'/'false' for JSON booleans.
|
||||
# str(True)='True' and str(False)='False' would never match; use the
|
||||
# correct lowercase text that ->> actually produces.
|
||||
filter_conditions.append(psycopg_sql.SQL("metadata->>{} = %s").format(
|
||||
psycopg_sql.Literal(key)
|
||||
))
|
||||
filter_values.append('true' if value else 'false')
|
||||
else:
|
||||
filter_conditions.append(psycopg_sql.SQL("metadata->>{} = %s").format(
|
||||
psycopg_sql.Literal(key)
|
||||
))
|
||||
filter_values.append(str(value))
|
||||
|
||||
if filter_conditions:
|
||||
where_clause = psycopg_sql.SQL(" WHERE ") + psycopg_sql.SQL(" AND ").join(filter_conditions)
|
||||
else:
|
||||
where_clause = psycopg_sql.SQL("")
|
||||
|
||||
query_sql = psycopg_sql.SQL("""
|
||||
SELECT id, vector, metadata
|
||||
FROM {table}
|
||||
{where}
|
||||
LIMIT %s
|
||||
""").format(
|
||||
table=psycopg_sql.Identifier(self.table_name),
|
||||
where=where_clause
|
||||
)
|
||||
params = filter_values + [limit]
|
||||
|
||||
with self._get_connection() as conn:
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(query_sql, params)
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
|
||||
results = []
|
||||
for row in rows:
|
||||
vec_id, vector_data, meta = row
|
||||
vec = np.array(vector_data) if vector_data is not None else None
|
||||
results.append({
|
||||
"id": vec_id,
|
||||
"metadata": meta if isinstance(meta, dict) else json.loads(meta) if meta else {},
|
||||
"vector": vec
|
||||
})
|
||||
return results
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to filter vectors by metadata: {str(e)}") from e
|
||||
|
||||
def create_index(
|
||||
self,
|
||||
index_type: str = "hnsw",
|
||||
|
||||
@@ -75,7 +75,7 @@ class PineconeClient:
|
||||
|
||||
try:
|
||||
# Default to serverless spec if not provided
|
||||
if spec is None:
|
||||
if spec is None and ServerlessSpec is not None:
|
||||
spec = ServerlessSpec(cloud="aws", region="us-east-1")
|
||||
|
||||
# Map metric names
|
||||
@@ -327,6 +327,7 @@ class PineconeStore:
|
||||
|
||||
self.api_key = api_key or config.get("api_key")
|
||||
self.environment = environment or config.get("environment")
|
||||
self.dimension: Optional[int] = config.get("dimension")
|
||||
|
||||
self.client: Optional[PineconeClient] = None
|
||||
self.index: Optional[PineconeIndex] = None
|
||||
@@ -395,7 +396,7 @@ class PineconeStore:
|
||||
|
||||
try:
|
||||
# Create index spec if not provided
|
||||
if spec is None:
|
||||
if spec is None and ServerlessSpec is not None:
|
||||
spec = ServerlessSpec(cloud="aws", region="us-east-1")
|
||||
|
||||
self.client.create_index(index_name, dimension, metric, spec, **kwargs)
|
||||
@@ -404,6 +405,7 @@ class PineconeStore:
|
||||
pinecone_index = self.client.get_index(index_name)
|
||||
self.index = PineconeIndex(pinecone_index)
|
||||
self.search_engine = PineconeSearch(self.index)
|
||||
self.dimension = dimension
|
||||
|
||||
self.logger.info(f"Created Pinecone index: {index_name}")
|
||||
return self.index
|
||||
@@ -431,6 +433,13 @@ class PineconeStore:
|
||||
pinecone_index = self.client.get_index(index_name)
|
||||
self.index = PineconeIndex(pinecone_index)
|
||||
self.search_engine = PineconeSearch(self.index)
|
||||
if self.dimension is None:
|
||||
try:
|
||||
stats = self.index.describe_index_stats()
|
||||
if stats and isinstance(stats, dict) and stats.get("dimension"):
|
||||
self.dimension = int(stats["dimension"])
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Could not determine index dimension for '{index_name}': {e}")
|
||||
return self.index
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to get index: {str(e)}")
|
||||
@@ -516,6 +525,9 @@ class PineconeStore:
|
||||
else:
|
||||
vector_list.append(list(vector))
|
||||
|
||||
if self.dimension is None and vector_list:
|
||||
self.dimension = len(vector_list[0])
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Upserting vectors to index..."
|
||||
)
|
||||
@@ -582,6 +594,9 @@ class PineconeStore:
|
||||
else:
|
||||
query_vector = list(query_vector)
|
||||
|
||||
if self.dimension is None and query_vector:
|
||||
self.dimension = len(query_vector)
|
||||
|
||||
results = self.search_engine.similarity_search(
|
||||
np.array(query_vector), k, filter, namespace, **options
|
||||
)
|
||||
@@ -641,6 +656,85 @@ class PineconeStore:
|
||||
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10, namespace: str = ""
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter vectors by metadata using Pinecone metadata filters.
|
||||
|
||||
Args:
|
||||
filters: Metadata filter criteria
|
||||
limit: Maximum number of results
|
||||
namespace: Namespace to search in
|
||||
|
||||
Returns:
|
||||
List of matching result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if self.index is None or not PINECONE_AVAILABLE:
|
||||
return []
|
||||
|
||||
dimension = self.dimension
|
||||
if dimension is None:
|
||||
try:
|
||||
stats = self.index.describe_index_stats()
|
||||
if stats and isinstance(stats, dict) and stats.get("dimension"):
|
||||
dimension = int(stats["dimension"])
|
||||
self.dimension = dimension
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not dimension:
|
||||
raise ProcessingError(
|
||||
"Index dimension is unknown. Please specify 'dimension' when initializing PineconeStore "
|
||||
"or call create_index()/get_index() first."
|
||||
)
|
||||
|
||||
pinecone_filter = {}
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
if isinstance(value, dict):
|
||||
cond = {}
|
||||
if "min" in value and value["min"] is not None:
|
||||
cond["$gte"] = value["min"]
|
||||
if "max" in value and value["max"] is not None:
|
||||
cond["$lte"] = value["max"]
|
||||
if cond:
|
||||
pinecone_filter[key] = cond
|
||||
elif isinstance(value, list):
|
||||
pinecone_filter[key] = {"$in": value}
|
||||
else:
|
||||
pinecone_filter[key] = value
|
||||
|
||||
# A literal zero vector is rejected by Pinecone for cosine-metric indexes
|
||||
# ("Query vector must not be the zero vector"). Use a unit vector instead so
|
||||
# this works regardless of the index's distance metric; since this call only
|
||||
# cares about which vectors match `filter`, not similarity ranking, any
|
||||
# fixed non-zero query vector is an equally valid probe.
|
||||
dummy_vector = [1.0 / (dimension ** 0.5)] * dimension
|
||||
|
||||
try:
|
||||
response = self.index.index.query(
|
||||
vector=dummy_vector,
|
||||
top_k=limit,
|
||||
filter=pinecone_filter if pinecone_filter else None,
|
||||
namespace=namespace,
|
||||
include_metadata=True,
|
||||
include_values=True,
|
||||
)
|
||||
results = []
|
||||
for match in response.matches:
|
||||
results.append(
|
||||
{
|
||||
"id": match.id,
|
||||
"metadata": match.metadata or {},
|
||||
"vector": np.array(match.values) if match.values else None,
|
||||
}
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to filter Pinecone vectors by metadata: {e}")
|
||||
return []
|
||||
|
||||
def fetch_vectors(
|
||||
self, vector_ids: List[str], namespace: str = "", **options
|
||||
) -> Dict[str, Any]:
|
||||
|
||||
@@ -49,8 +49,10 @@ try:
|
||||
Distance,
|
||||
FieldCondition,
|
||||
Filter,
|
||||
MatchAny,
|
||||
MatchValue,
|
||||
PointStruct,
|
||||
Range,
|
||||
VectorParams,
|
||||
)
|
||||
|
||||
@@ -63,7 +65,9 @@ except (ImportError, OSError):
|
||||
PointStruct = None
|
||||
Filter = None
|
||||
FieldCondition = None
|
||||
MatchAny = None
|
||||
MatchValue = None
|
||||
Range = None
|
||||
CollectionStatus = None
|
||||
|
||||
|
||||
@@ -538,6 +542,68 @@ class QdrantStore:
|
||||
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter vectors by metadata using Qdrant payload filtering.
|
||||
|
||||
Args:
|
||||
filters: Metadata filter criteria
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of matching result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if self.collection is None or self.client is None or not QDRANT_AVAILABLE:
|
||||
return []
|
||||
|
||||
conditions = []
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
if isinstance(value, dict):
|
||||
cond_kwargs = {}
|
||||
if "min" in value and value["min"] is not None:
|
||||
cond_kwargs["gte"] = value["min"]
|
||||
if "max" in value and value["max"] is not None:
|
||||
cond_kwargs["lte"] = value["max"]
|
||||
if cond_kwargs:
|
||||
conditions.append(
|
||||
FieldCondition(key=key, range=Range(**cond_kwargs))
|
||||
)
|
||||
elif isinstance(value, list):
|
||||
conditions.append(
|
||||
FieldCondition(key=key, match=MatchAny(any=value))
|
||||
)
|
||||
else:
|
||||
conditions.append(
|
||||
FieldCondition(key=key, match=MatchValue(value=value))
|
||||
)
|
||||
|
||||
query_filter = Filter(must=conditions) if conditions else None
|
||||
|
||||
try:
|
||||
records, _ = self.client.scroll(
|
||||
collection_name=self.collection.collection_name,
|
||||
scroll_filter=query_filter,
|
||||
limit=limit,
|
||||
with_payload=True,
|
||||
with_vectors=True,
|
||||
)
|
||||
results = []
|
||||
for rec in records:
|
||||
results.append(
|
||||
{
|
||||
"id": str(rec.id),
|
||||
"metadata": rec.payload or {},
|
||||
"vector": np.array(rec.vector) if rec.vector is not None else None,
|
||||
}
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to scroll Qdrant points by metadata filter: {e}")
|
||||
return []
|
||||
|
||||
def delete_vectors(
|
||||
self, point_ids: List[Union[str, int]], **options
|
||||
) -> Dict[str, Any]:
|
||||
|
||||
@@ -616,6 +616,96 @@ class SQLiteVecStore:
|
||||
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter stored vectors by metadata using SQLite JSON functions.
|
||||
|
||||
Args:
|
||||
filters: Dictionary of metadata filter conditions
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of results containing id, metadata, and vector
|
||||
"""
|
||||
filter_conditions = []
|
||||
filter_params = []
|
||||
|
||||
if filters:
|
||||
for key, value in filters.items():
|
||||
if not self._is_safe_identifier(key):
|
||||
raise ValidationError(
|
||||
f"Invalid filter key: {key!r}. "
|
||||
"Keys must start with a letter or underscore and contain "
|
||||
"only alphanumeric characters and underscores."
|
||||
)
|
||||
if isinstance(value, dict):
|
||||
if "min" in value and value["min"] is not None:
|
||||
filter_conditions.append(f"CAST(json_extract(metadata, '$.{key}') AS NUMERIC) >= ?")
|
||||
filter_params.append(value["min"])
|
||||
if "max" in value and value["max"] is not None:
|
||||
filter_conditions.append(f"CAST(json_extract(metadata, '$.{key}') AS NUMERIC) <= ?")
|
||||
filter_params.append(value["max"])
|
||||
elif isinstance(value, list):
|
||||
# If the metadata value at this key is itself a JSON array, match on
|
||||
# intersection (mirrors the in-memory backend's set-intersection
|
||||
# semantics); otherwise fall back to plain scalar membership. Both
|
||||
# cases are handled uniformly via json_each: a non-array value is
|
||||
# wrapped in a one-element array first so json_each always sees a
|
||||
# valid JSON array to iterate.
|
||||
placeholders = ", ".join(["?"] * len(value))
|
||||
filter_conditions.append(
|
||||
f"EXISTS (SELECT 1 FROM json_each("
|
||||
f" CASE WHEN json_type(metadata, '$.{key}') = 'array'"
|
||||
f" THEN json_extract(metadata, '$.{key}')"
|
||||
f" ELSE json_array(json_extract(metadata, '$.{key}'))"
|
||||
f" END"
|
||||
f") je WHERE je.value IN ({placeholders}))"
|
||||
)
|
||||
filter_params.extend([str(v) if not isinstance(v, (int, float, bool)) else v for v in value])
|
||||
else:
|
||||
filter_conditions.append(f"json_extract(metadata, '$.{key}') = ?")
|
||||
if isinstance(value, bool):
|
||||
filter_params.append(1 if value else 0)
|
||||
else:
|
||||
filter_params.append(value)
|
||||
|
||||
where_clause = ""
|
||||
if filter_conditions:
|
||||
where_clause = " WHERE " + " AND ".join(filter_conditions)
|
||||
|
||||
query_sql = f"""
|
||||
SELECT id, embedding, metadata
|
||||
FROM {self.table_name}
|
||||
{where_clause}
|
||||
LIMIT ?
|
||||
"""
|
||||
params = filter_params + [limit]
|
||||
|
||||
with self._lock, self._get_connection() as conn:
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(query_sql, params)
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
|
||||
results = []
|
||||
for row in rows:
|
||||
vec_id, embedding_blob, meta_json = row
|
||||
vec = None
|
||||
if embedding_blob:
|
||||
vec = np.frombuffer(embedding_blob, dtype=np.float32).copy()
|
||||
|
||||
results.append({
|
||||
"id": vec_id,
|
||||
"metadata": json.loads(meta_json) if meta_json else {},
|
||||
"vector": vec
|
||||
})
|
||||
return results
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to filter by metadata: {str(e)}") from e
|
||||
|
||||
def create_index(
|
||||
self,
|
||||
index_type: str = "hnsw",
|
||||
|
||||
@@ -1185,75 +1185,29 @@ class VectorStore:
|
||||
from datetime import datetime, timedelta
|
||||
cutoff = datetime.now() - timedelta(days=7)
|
||||
filters["timestamp"] = {"min": cutoff.isoformat()}
|
||||
|
||||
return filters
|
||||
|
||||
def _filter_by_metadata(self, filters: Dict[str, Any], limit: int) -> List[Dict[str, Any]]:
|
||||
"""Filter decisions by metadata only."""
|
||||
if self._backend_store is not None:
|
||||
# No real backend wrapper implements filter_by_metadata; the only
|
||||
# codebase hit is HybridSearch.filter_by_metadata which has a
|
||||
# completely different signature (results, MetadataFilter) and is
|
||||
# never stored in _backend_store. Silently returning [] here would
|
||||
# be wrong — the caller (filter_decisions) would report zero matches
|
||||
# for a query that simply isn't supported, indistinguishable from a
|
||||
# genuine empty result. This is the same situation as get_vector()
|
||||
# and get_metadata() (#843 fix): when a backend exists but cannot
|
||||
# fulfil the request, raise NotImplementedError so the caller knows
|
||||
# the backend lacks this capability rather than assuming no data.
|
||||
if hasattr(self._backend_store, "filter_by_metadata"):
|
||||
return self._backend_store.filter_by_metadata(filters, limit)
|
||||
return self._backend_store.filter_by_metadata(filters=filters, limit=limit)
|
||||
raise NotImplementedError(
|
||||
f"Backend store {type(self._backend_store).__name__} does not "
|
||||
"implement filter_by_metadata. Metadata-only filtering via "
|
||||
"filter_decisions(query=None, ...) is only supported for the "
|
||||
"inmemory backend. Pass a query string to use search_decisions() "
|
||||
"filter_decisions(query=None, ...) is only supported for backends "
|
||||
"that implement filter_by_metadata. Pass a query string to use search_decisions() "
|
||||
"instead, which is supported by all backends."
|
||||
)
|
||||
|
||||
results = []
|
||||
|
||||
for vector_id, metadata in self.metadata.items():
|
||||
match = True
|
||||
|
||||
for key, value in filters.items():
|
||||
if key not in metadata:
|
||||
match = False
|
||||
break
|
||||
|
||||
if isinstance(value, dict):
|
||||
# Handle range filters
|
||||
metadata_value = metadata[key]
|
||||
if "min" in value and metadata_value < value["min"]:
|
||||
match = False
|
||||
break
|
||||
if "max" in value and metadata_value > value["max"]:
|
||||
match = False
|
||||
break
|
||||
elif isinstance(value, list):
|
||||
# Handle list membership
|
||||
metadata_value = metadata[key]
|
||||
if isinstance(metadata_value, list):
|
||||
# Both are lists - check for intersection
|
||||
if not set(metadata_value) & set(value):
|
||||
match = False
|
||||
break
|
||||
else:
|
||||
# Metadata value is scalar, check if it's in the filter list
|
||||
if metadata_value not in value:
|
||||
match = False
|
||||
break
|
||||
else:
|
||||
# Handle exact match
|
||||
if metadata[key] != value:
|
||||
match = False
|
||||
break
|
||||
|
||||
if match:
|
||||
if _matches_filter(metadata, filters):
|
||||
results.append({
|
||||
"id": vector_id,
|
||||
"metadata": metadata,
|
||||
"vector": self.get_vector(vector_id)
|
||||
"vector": self.vectors.get(vector_id)
|
||||
})
|
||||
|
||||
if len(results) >= limit:
|
||||
@@ -1262,6 +1216,43 @@ class VectorStore:
|
||||
return results
|
||||
|
||||
|
||||
def _matches_filter(metadata: Dict[str, Any], filters: Dict[str, Any]) -> bool:
|
||||
"""Check if metadata dictionary matches filter criteria."""
|
||||
if not filters:
|
||||
return True
|
||||
if metadata is None:
|
||||
return False
|
||||
|
||||
for key, value in filters.items():
|
||||
if key not in metadata:
|
||||
return False
|
||||
|
||||
metadata_value = metadata[key]
|
||||
|
||||
if isinstance(value, dict):
|
||||
# Handle range filters
|
||||
if "min" in value and value["min"] is not None:
|
||||
if metadata_value is None or metadata_value < value["min"]:
|
||||
return False
|
||||
if "max" in value and value["max"] is not None:
|
||||
if metadata_value is None or metadata_value > value["max"]:
|
||||
return False
|
||||
elif isinstance(value, list):
|
||||
# Handle list membership
|
||||
if isinstance(metadata_value, list):
|
||||
if not (set(metadata_value) & set(value)):
|
||||
return False
|
||||
else:
|
||||
if metadata_value not in value:
|
||||
return False
|
||||
else:
|
||||
# Handle exact match
|
||||
if metadata_value != value:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class VectorIndexer:
|
||||
"""Vector indexing engine."""
|
||||
|
||||
|
||||
@@ -445,6 +445,173 @@ class WeaviateStore:
|
||||
self.logger.warning(f"Failed to get metadata for {vector_id}: {e}")
|
||||
return None
|
||||
|
||||
def _build_weaviate_filter(self, filters: Dict[str, Any]) -> Any:
|
||||
"""Build native Weaviate Filter object from metadata filter dictionary."""
|
||||
if not filters or not WEAVIATE_AVAILABLE:
|
||||
return None
|
||||
|
||||
Filter = None
|
||||
try:
|
||||
from weaviate.classes.query import Filter
|
||||
except (ImportError, AttributeError):
|
||||
try:
|
||||
if weaviate and hasattr(weaviate, "classes") and hasattr(weaviate.classes, "query"):
|
||||
Filter = getattr(weaviate.classes.query, "Filter", None)
|
||||
except AttributeError:
|
||||
Filter = None
|
||||
|
||||
if Filter is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
conditions = []
|
||||
for key, value in filters.items():
|
||||
if isinstance(value, dict):
|
||||
if "min" in value and value["min"] is not None:
|
||||
conditions.append(Filter.by_property(key).greater_or_equal(value["min"]))
|
||||
if "max" in value and value["max"] is not None:
|
||||
conditions.append(Filter.by_property(key).less_or_equal(value["max"]))
|
||||
elif isinstance(value, list):
|
||||
conditions.append(Filter.by_property(key).contains_any(value))
|
||||
else:
|
||||
conditions.append(Filter.by_property(key).equal(value))
|
||||
|
||||
if not conditions:
|
||||
return None
|
||||
|
||||
weaviate_filter = conditions[0]
|
||||
for cond in conditions[1:]:
|
||||
weaviate_filter = weaviate_filter & cond
|
||||
|
||||
return weaviate_filter
|
||||
except Exception as e:
|
||||
self.logger.debug(f"Could not build native Weaviate filter: {e}")
|
||||
return None
|
||||
|
||||
def filter_by_metadata(
|
||||
self, filters: Dict[str, Any], limit: int = 10
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Filter stored objects by metadata in Weaviate.
|
||||
|
||||
Args:
|
||||
filters: Metadata filter criteria
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of matching result dicts with 'id', 'metadata', and 'vector'
|
||||
"""
|
||||
if self.collection is None or not WEAVIATE_AVAILABLE:
|
||||
return []
|
||||
|
||||
from .vector_store import _matches_filter
|
||||
|
||||
native_filter = self._build_weaviate_filter(filters) if filters else None
|
||||
|
||||
results = []
|
||||
seen_ids = set()
|
||||
after_cursor = None
|
||||
scanned_count = 0
|
||||
page_size = max(limit, 100)
|
||||
use_native_filter = native_filter is not None
|
||||
|
||||
try:
|
||||
while len(results) < limit:
|
||||
kwargs = {"limit": page_size, "include_vector": True}
|
||||
if use_native_filter and native_filter is not None:
|
||||
kwargs["filters"] = native_filter
|
||||
if after_cursor is not None:
|
||||
kwargs["after"] = after_cursor
|
||||
|
||||
try:
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
except TypeError as te:
|
||||
# Handle kwargs incompatibility (e.g. mock or client version without filters/after)
|
||||
if "filters" in kwargs:
|
||||
use_native_filter = False
|
||||
kwargs.pop("filters", None)
|
||||
try:
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
except TypeError:
|
||||
if "after" in kwargs:
|
||||
kwargs.pop("after", None)
|
||||
kwargs["offset"] = scanned_count
|
||||
try:
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
except TypeError:
|
||||
kwargs.pop("offset", None)
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
elif "after" in kwargs:
|
||||
kwargs.pop("after", None)
|
||||
kwargs["offset"] = scanned_count
|
||||
try:
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
except TypeError:
|
||||
kwargs.pop("offset", None)
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
else:
|
||||
raise te
|
||||
except Exception as fe:
|
||||
if use_native_filter:
|
||||
self.logger.warning(
|
||||
f"Native Weaviate filter query failed, falling back to paginated fetch: {fe}"
|
||||
)
|
||||
use_native_filter = False
|
||||
kwargs.pop("filters", None)
|
||||
objs = self.collection.query.fetch_objects(**kwargs)
|
||||
else:
|
||||
raise fe
|
||||
|
||||
if not objs or not getattr(objs, "objects", None):
|
||||
break
|
||||
|
||||
batch_objects = objs.objects
|
||||
if not batch_objects:
|
||||
break
|
||||
|
||||
new_objects_found = False
|
||||
for obj in batch_objects:
|
||||
obj_id = str(obj.uuid) if hasattr(obj, "uuid") and obj.uuid is not None else None
|
||||
if obj_id:
|
||||
if obj_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(obj_id)
|
||||
new_objects_found = True
|
||||
|
||||
properties = getattr(obj, "properties", None) or {}
|
||||
if _matches_filter(properties, filters):
|
||||
vector = None
|
||||
if hasattr(obj, "vector") and obj.vector:
|
||||
vector = np.array(obj.vector)
|
||||
results.append(
|
||||
{
|
||||
"id": obj_id,
|
||||
"metadata": properties,
|
||||
"vector": vector,
|
||||
}
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
if not new_objects_found:
|
||||
break
|
||||
|
||||
scanned_count += len(batch_objects)
|
||||
if len(batch_objects) < page_size:
|
||||
break
|
||||
|
||||
last_obj = batch_objects[-1]
|
||||
if hasattr(last_obj, "uuid") and last_obj.uuid is not None:
|
||||
after_cursor = str(last_obj.uuid)
|
||||
else:
|
||||
break
|
||||
|
||||
return results
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to fetch Weaviate objects by metadata filter: {e}")
|
||||
return results if results else []
|
||||
|
||||
|
||||
def query_vectors(
|
||||
self,
|
||||
query_vector: np.ndarray,
|
||||
|
||||
@@ -0,0 +1,500 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
import numpy as np
|
||||
|
||||
from semantica.vector_store.faiss_store import FAISSStore
|
||||
from semantica.vector_store.qdrant_store import QdrantStore
|
||||
from semantica.vector_store.pinecone_store import PineconeStore
|
||||
from semantica.vector_store.milvus_store import MilvusStore
|
||||
from semantica.vector_store.pgvector_store import PgVectorStore
|
||||
from semantica.vector_store.weaviate_store import WeaviateStore
|
||||
from semantica.utils.exceptions import ProcessingError, ValidationError
|
||||
|
||||
|
||||
class TestBackendMetadataFiltering(unittest.TestCase):
|
||||
|
||||
def test_faiss_store_filter_by_metadata(self):
|
||||
store = FAISSStore(dimension=2)
|
||||
mock_index = MagicMock()
|
||||
mock_index.metadata = {
|
||||
"v1": {"category": "finance", "score": 10},
|
||||
"v2": {"category": "tech", "score": 20},
|
||||
}
|
||||
mock_index.get_vector.side_effect = lambda vid: np.array([1.0, 0.0]) if vid == "v1" else np.array([0.0, 1.0])
|
||||
store.index = mock_index
|
||||
|
||||
results = store.filter_by_metadata({"category": "finance"}, limit=10)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "v1")
|
||||
self.assertEqual(results[0]["metadata"], {"category": "finance", "score": 10})
|
||||
|
||||
@patch('semantica.vector_store.qdrant_store.FieldCondition', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.MatchValue', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.Filter', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.QDRANT_AVAILABLE', True)
|
||||
def test_qdrant_store_filter_by_metadata(self):
|
||||
store = QdrantStore()
|
||||
mock_collection = MagicMock()
|
||||
mock_collection.collection_name = "test_coll"
|
||||
store.collection = mock_collection
|
||||
mock_client = MagicMock()
|
||||
rec = MagicMock()
|
||||
rec.id = "q1"
|
||||
rec.payload = {"env": "prod"}
|
||||
rec.vector = [0.1, 0.2]
|
||||
mock_client.scroll.return_value = ([rec], None)
|
||||
store.client = mock_client
|
||||
|
||||
results = store.filter_by_metadata({"env": "prod"}, limit=5)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "q1")
|
||||
self.assertEqual(results[0]["metadata"], {"env": "prod"})
|
||||
mock_client.scroll.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.qdrant_store.Range', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.FieldCondition', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.Filter', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.QDRANT_AVAILABLE', True)
|
||||
def test_qdrant_store_filter_by_metadata_range(self):
|
||||
"""Range filters must construct Range objects and not raise NameError."""
|
||||
store = QdrantStore()
|
||||
mock_collection = MagicMock()
|
||||
mock_collection.collection_name = "test_coll"
|
||||
store.collection = mock_collection
|
||||
mock_client = MagicMock()
|
||||
rec = MagicMock()
|
||||
rec.id = "r1"
|
||||
rec.payload = {"score": 8}
|
||||
rec.vector = [0.3, 0.4]
|
||||
mock_client.scroll.return_value = ([rec], None)
|
||||
store.client = mock_client
|
||||
|
||||
# min-only range
|
||||
results = store.filter_by_metadata({"score": {"min": 5}}, limit=10)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "r1")
|
||||
mock_client.scroll.assert_called()
|
||||
|
||||
# Verify Range was actually called to build the condition (not skipped)
|
||||
import semantica.vector_store.qdrant_store as qs_mod
|
||||
qs_mod.Range.assert_called()
|
||||
|
||||
@patch('semantica.vector_store.qdrant_store.Range', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.FieldCondition', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.Filter', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.QDRANT_AVAILABLE', True)
|
||||
def test_qdrant_store_filter_by_metadata_range_min_and_max(self):
|
||||
"""Range filters with both min and max must construct Range with both gte and lte."""
|
||||
store = QdrantStore()
|
||||
mock_collection = MagicMock()
|
||||
mock_collection.collection_name = "test_coll"
|
||||
store.collection = mock_collection
|
||||
mock_client = MagicMock()
|
||||
mock_client.scroll.return_value = ([], None)
|
||||
store.client = mock_client
|
||||
|
||||
store.filter_by_metadata({"score": {"min": 5, "max": 10}}, limit=10)
|
||||
|
||||
import semantica.vector_store.qdrant_store as qs_mod
|
||||
# Range must have been called with gte and lte
|
||||
qs_mod.Range.assert_called_with(gte=5, lte=10)
|
||||
|
||||
@patch('semantica.vector_store.qdrant_store.MatchAny', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.FieldCondition', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.Filter', MagicMock())
|
||||
@patch('semantica.vector_store.qdrant_store.QDRANT_AVAILABLE', True)
|
||||
def test_qdrant_store_filter_by_metadata_list(self):
|
||||
"""List filters must construct MatchAny objects and not raise NameError."""
|
||||
store = QdrantStore()
|
||||
mock_collection = MagicMock()
|
||||
mock_collection.collection_name = "test_coll"
|
||||
store.collection = mock_collection
|
||||
mock_client = MagicMock()
|
||||
rec = MagicMock()
|
||||
rec.id = "l1"
|
||||
rec.payload = {"tags": "python"}
|
||||
rec.vector = [0.5, 0.6]
|
||||
mock_client.scroll.return_value = ([rec], None)
|
||||
store.client = mock_client
|
||||
|
||||
results = store.filter_by_metadata({"tags": ["python", "ml"]}, limit=10)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "l1")
|
||||
mock_client.scroll.assert_called()
|
||||
|
||||
# Verify MatchAny was actually called with the filter list
|
||||
import semantica.vector_store.qdrant_store as qs_mod
|
||||
qs_mod.MatchAny.assert_called_with(any=["python", "ml"])
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
def test_pinecone_store_filter_by_metadata(self):
|
||||
store = PineconeStore(dimension=2)
|
||||
mock_index_wrapper = MagicMock()
|
||||
mock_inner_index = MagicMock()
|
||||
|
||||
match_obj = MagicMock()
|
||||
match_obj.id = "p1"
|
||||
match_obj.metadata = {"status": "active"}
|
||||
match_obj.values = [0.1, 0.9]
|
||||
|
||||
response = MagicMock()
|
||||
response.matches = [match_obj]
|
||||
mock_inner_index.query.return_value = response
|
||||
mock_index_wrapper.index = mock_inner_index
|
||||
store.index = mock_index_wrapper
|
||||
|
||||
results = store.filter_by_metadata({"status": "active"}, limit=5)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "p1")
|
||||
self.assertEqual(results[0]["metadata"], {"status": "active"})
|
||||
# Assert query vector dimension matches store.dimension (2)
|
||||
mock_inner_index.query.assert_called_once()
|
||||
query_kw = mock_inner_index.query.call_args[1]
|
||||
self.assertEqual(len(query_kw["vector"]), 2)
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
def test_pinecone_store_filter_by_metadata_unknown_dimension_raises(self):
|
||||
store = PineconeStore()
|
||||
mock_index_wrapper = MagicMock()
|
||||
mock_index_wrapper.describe_index_stats = MagicMock(return_value={})
|
||||
store.index = mock_index_wrapper
|
||||
with self.assertRaises(ProcessingError):
|
||||
store.filter_by_metadata({"status": "active"}, limit=5)
|
||||
|
||||
@patch('semantica.vector_store.milvus_store.MILVUS_AVAILABLE', True)
|
||||
def test_milvus_store_filter_by_metadata(self):
|
||||
store = MilvusStore()
|
||||
mock_coll_wrapper = MagicMock()
|
||||
mock_inner_coll = MagicMock()
|
||||
mock_inner_coll.query.return_value = [
|
||||
{"id": "m1", "vector": [0.3, 0.4], "metadata": {"lang": "py"}}
|
||||
]
|
||||
mock_coll_wrapper.collection = mock_inner_coll
|
||||
store.collection = mock_coll_wrapper
|
||||
|
||||
results = store.filter_by_metadata({"lang": "py"}, limit=5)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "m1")
|
||||
self.assertEqual(results[0]["metadata"], {"lang": "py"})
|
||||
|
||||
@patch('semantica.vector_store.milvus_store.MILVUS_AVAILABLE', True)
|
||||
def test_milvus_store_filter_by_metadata_escaping(self):
|
||||
store = MilvusStore()
|
||||
mock_coll_wrapper = MagicMock()
|
||||
mock_inner_coll = MagicMock()
|
||||
mock_inner_coll.query.return_value = []
|
||||
mock_coll_wrapper.collection = mock_inner_coll
|
||||
store.collection = mock_coll_wrapper
|
||||
|
||||
store.filter_by_metadata(
|
||||
{
|
||||
"title": 'John "Jack" Doe',
|
||||
"active": True,
|
||||
"tags": ['python', 'c++ "v"'],
|
||||
},
|
||||
limit=5,
|
||||
)
|
||||
|
||||
mock_inner_coll.query.assert_called_once()
|
||||
expr = mock_inner_coll.query.call_args[1]["expr"]
|
||||
self.assertIn('metadata["title"] == "John \\"Jack\\" Doe"', expr)
|
||||
self.assertIn('metadata["active"] == true', expr)
|
||||
self.assertIn('metadata["tags"] in ["python", "c++ \\"v\\""]', expr)
|
||||
|
||||
@patch('semantica.vector_store.milvus_store.MILVUS_AVAILABLE', True)
|
||||
def test_milvus_store_filter_by_metadata_invalid_key_raises(self):
|
||||
store = MilvusStore()
|
||||
mock_coll_wrapper = MagicMock()
|
||||
store.collection = mock_coll_wrapper
|
||||
|
||||
with self.assertRaises(ValidationError):
|
||||
store.filter_by_metadata({'dept" || 1==1 || "': "val"}, limit=5)
|
||||
|
||||
@patch('semantica.vector_store.pgvector_store.PSYCOPG3_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pgvector_store.psycopg_sql')
|
||||
def test_pgvector_store_filter_by_metadata(self, mock_sql):
|
||||
store = object.__new__(PgVectorStore)
|
||||
store.table_name = "test_vectors"
|
||||
store._is_safe_identifier = lambda k: True
|
||||
|
||||
mock_conn = MagicMock()
|
||||
mock_cur = MagicMock()
|
||||
mock_cur.fetchall.return_value = [
|
||||
("pg1", [0.1, 0.2], {"org": "acme"})
|
||||
]
|
||||
mock_conn.cursor.return_value = mock_cur
|
||||
|
||||
with patch.object(PgVectorStore, '_get_connection', return_value=MagicMock(__enter__=MagicMock(return_value=mock_conn), __exit__=MagicMock())):
|
||||
results = store.filter_by_metadata({"org": "acme"}, limit=10)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "pg1")
|
||||
self.assertEqual(results[0]["metadata"], {"org": "acme"})
|
||||
|
||||
@patch('semantica.vector_store.pgvector_store.PSYCOPG3_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pgvector_store.psycopg_sql')
|
||||
def test_pgvector_store_filter_by_metadata_bool_true(self, mock_sql):
|
||||
"""Boolean True must become the string 'true' (lowercase) in the SQL parameter.
|
||||
|
||||
PostgreSQL JSONB ->> returns 'true' for a JSON boolean true.
|
||||
str(True) == 'True' would never match; this test guards against regression.
|
||||
"""
|
||||
store = object.__new__(PgVectorStore)
|
||||
store.table_name = "test_vectors"
|
||||
store._is_safe_identifier = lambda k: True
|
||||
|
||||
mock_conn = MagicMock()
|
||||
mock_cur = MagicMock()
|
||||
mock_cur.fetchall.return_value = [
|
||||
("pg2", [0.3, 0.4], {"active": True})
|
||||
]
|
||||
mock_conn.cursor.return_value = mock_cur
|
||||
|
||||
with patch.object(
|
||||
PgVectorStore,
|
||||
'_get_connection',
|
||||
return_value=MagicMock(
|
||||
__enter__=MagicMock(return_value=mock_conn),
|
||||
__exit__=MagicMock(),
|
||||
),
|
||||
):
|
||||
results = store.filter_by_metadata({"active": True}, limit=10)
|
||||
|
||||
# Result is returned correctly
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "pg2")
|
||||
|
||||
# The critical assertion: 'true' (not 'True') was passed to execute()
|
||||
execute_call_args = mock_cur.execute.call_args
|
||||
self.assertIsNotNone(execute_call_args, "cursor.execute was not called")
|
||||
params_passed = execute_call_args[0][1] # positional arg 1 is the params list/tuple
|
||||
self.assertIn('true', params_passed,
|
||||
"Expected lowercase 'true' in SQL params, got: {}".format(params_passed))
|
||||
self.assertNotIn('True', params_passed,
|
||||
"str(True)='True' must NOT appear in SQL params")
|
||||
|
||||
@patch('semantica.vector_store.pgvector_store.PSYCOPG3_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pgvector_store.psycopg_sql')
|
||||
def test_pgvector_store_filter_by_metadata_bool_false(self, mock_sql):
|
||||
"""Boolean False must become the string 'false' (lowercase) in the SQL parameter.
|
||||
|
||||
PostgreSQL JSONB ->> returns 'false' for a JSON boolean false.
|
||||
str(False) == 'False' would never match; this test guards against regression.
|
||||
"""
|
||||
store = object.__new__(PgVectorStore)
|
||||
store.table_name = "test_vectors"
|
||||
store._is_safe_identifier = lambda k: True
|
||||
|
||||
mock_conn = MagicMock()
|
||||
mock_cur = MagicMock()
|
||||
mock_cur.fetchall.return_value = [
|
||||
("pg3", [0.5, 0.6], {"active": False})
|
||||
]
|
||||
mock_conn.cursor.return_value = mock_cur
|
||||
|
||||
with patch.object(
|
||||
PgVectorStore,
|
||||
'_get_connection',
|
||||
return_value=MagicMock(
|
||||
__enter__=MagicMock(return_value=mock_conn),
|
||||
__exit__=MagicMock(),
|
||||
),
|
||||
):
|
||||
results = store.filter_by_metadata({"active": False}, limit=10)
|
||||
|
||||
# Result is returned correctly
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "pg3")
|
||||
|
||||
# The critical assertion: 'false' (not 'False') was passed to execute()
|
||||
execute_call_args = mock_cur.execute.call_args
|
||||
self.assertIsNotNone(execute_call_args, "cursor.execute was not called")
|
||||
params_passed = execute_call_args[0][1] # positional arg 1 is the params list/tuple
|
||||
self.assertIn('false', params_passed,
|
||||
"Expected lowercase 'false' in SQL params, got: {}".format(params_passed))
|
||||
self.assertNotIn('False', params_passed,
|
||||
"str(False)='False' must NOT appear in SQL params")
|
||||
|
||||
@patch('semantica.vector_store.pgvector_store.PSYCOPG3_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pgvector_store.psycopg_sql')
|
||||
def test_pgvector_store_filter_by_metadata_bool_list(self, mock_sql):
|
||||
"""List-valued boolean filters must use lowercase 'true'/'false', not
|
||||
str(True)/str(False), matching the scalar branch's handling.
|
||||
"""
|
||||
store = object.__new__(PgVectorStore)
|
||||
store.table_name = "test_vectors"
|
||||
store._is_safe_identifier = lambda k: True
|
||||
|
||||
mock_conn = MagicMock()
|
||||
mock_cur = MagicMock()
|
||||
mock_cur.fetchall.return_value = [
|
||||
("pg4", [0.7, 0.8], {"active": True})
|
||||
]
|
||||
mock_conn.cursor.return_value = mock_cur
|
||||
|
||||
with patch.object(
|
||||
PgVectorStore,
|
||||
'_get_connection',
|
||||
return_value=MagicMock(
|
||||
__enter__=MagicMock(return_value=mock_conn),
|
||||
__exit__=MagicMock(),
|
||||
),
|
||||
):
|
||||
results = store.filter_by_metadata({"active": [True, False]}, limit=10)
|
||||
|
||||
self.assertEqual(len(results), 1)
|
||||
execute_call_args = mock_cur.execute.call_args
|
||||
params_passed = execute_call_args[0][1]
|
||||
flat_params = [v for p in params_passed for v in (p if isinstance(p, list) else [p])]
|
||||
self.assertIn('true', flat_params)
|
||||
self.assertIn('false', flat_params)
|
||||
self.assertNotIn('True', flat_params)
|
||||
self.assertNotIn('False', flat_params)
|
||||
|
||||
def test_faiss_store_filter_by_metadata_limit_zero(self):
|
||||
"""limit=0 must return no results, not the first match."""
|
||||
store = FAISSStore(dimension=2)
|
||||
mock_index = MagicMock()
|
||||
mock_index.metadata = {
|
||||
"v1": {"category": "finance", "score": 10},
|
||||
}
|
||||
mock_index.get_vector.return_value = np.array([1.0, 0.0])
|
||||
store.index = mock_index
|
||||
|
||||
results = store.filter_by_metadata({"category": "finance"}, limit=0)
|
||||
self.assertEqual(results, [])
|
||||
|
||||
@patch('semantica.vector_store.milvus_store.MILVUS_AVAILABLE', True)
|
||||
def test_milvus_store_filter_by_metadata_nan_raises(self):
|
||||
"""NaN/Infinity are not valid Milvus expression literals and must be
|
||||
rejected up front rather than silently producing an invalid expression
|
||||
that gets swallowed by the broad except around the query() call.
|
||||
"""
|
||||
store = MilvusStore()
|
||||
mock_coll_wrapper = MagicMock()
|
||||
store.collection = mock_coll_wrapper
|
||||
|
||||
with self.assertRaises(ValidationError):
|
||||
store.filter_by_metadata({"score": {"min": float("nan")}}, limit=5)
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
def test_pinecone_store_get_index_sets_dimension_from_stats(self):
|
||||
"""get_index() must read stats from the returned PineconeIndex wrapper
|
||||
(self.index), not from a nonexistent method on the store itself.
|
||||
"""
|
||||
store = PineconeStore()
|
||||
mock_client = MagicMock()
|
||||
mock_pinecone_index = MagicMock()
|
||||
mock_client.get_index.return_value = mock_pinecone_index
|
||||
store.client = mock_client
|
||||
|
||||
with patch(
|
||||
'semantica.vector_store.pinecone_store.PineconeIndex'
|
||||
) as mock_index_cls:
|
||||
mock_index_instance = MagicMock()
|
||||
mock_index_instance.describe_index_stats.return_value = {"dimension": 42}
|
||||
mock_index_cls.return_value = mock_index_instance
|
||||
|
||||
store.get_index("my-index")
|
||||
|
||||
self.assertEqual(store.dimension, 42)
|
||||
|
||||
def test_weaviate_store_filter_by_metadata(self):
|
||||
store = WeaviateStore()
|
||||
mock_coll = MagicMock()
|
||||
obj1 = MagicMock()
|
||||
obj1.uuid = "w-uuid-1"
|
||||
obj1.properties = {"dept": "eng"}
|
||||
obj1.vector = [0.5, 0.5]
|
||||
objs = MagicMock()
|
||||
objs.objects = [obj1]
|
||||
mock_coll.query.fetch_objects.return_value = objs
|
||||
store.collection = mock_coll
|
||||
|
||||
with patch('semantica.vector_store.weaviate_store.WEAVIATE_AVAILABLE', True):
|
||||
results = store.filter_by_metadata({"dept": "eng"}, limit=5)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "w-uuid-1")
|
||||
self.assertEqual(results[0]["metadata"], {"dept": "eng"})
|
||||
|
||||
def test_weaviate_store_filter_by_metadata_pagination(self):
|
||||
"""Test that WeaviateStore.filter_by_metadata paginates beyond page 1 to find matching items."""
|
||||
store = WeaviateStore()
|
||||
mock_coll = MagicMock()
|
||||
|
||||
# Batch 1: 100 non-matching objects
|
||||
batch1_objs = []
|
||||
for i in range(100):
|
||||
obj = MagicMock()
|
||||
obj.uuid = f"batch1-uuid-{i}"
|
||||
obj.properties = {"dept": "hr"}
|
||||
obj.vector = [0.1, 0.1]
|
||||
batch1_objs.append(obj)
|
||||
|
||||
res1 = MagicMock()
|
||||
res1.objects = batch1_objs
|
||||
|
||||
# Batch 2: 2 matching objects
|
||||
obj_match1 = MagicMock()
|
||||
obj_match1.uuid = "match-uuid-1"
|
||||
obj_match1.properties = {"dept": "eng"}
|
||||
obj_match1.vector = [0.5, 0.5]
|
||||
|
||||
obj_match2 = MagicMock()
|
||||
obj_match2.uuid = "match-uuid-2"
|
||||
obj_match2.properties = {"dept": "eng"}
|
||||
obj_match2.vector = [0.6, 0.6]
|
||||
|
||||
res2 = MagicMock()
|
||||
res2.objects = [obj_match1, obj_match2]
|
||||
|
||||
def side_effect(**kwargs):
|
||||
if kwargs.get("after") == "batch1-uuid-99":
|
||||
return res2
|
||||
return res1
|
||||
|
||||
mock_coll.query.fetch_objects.side_effect = side_effect
|
||||
store.collection = mock_coll
|
||||
|
||||
with patch('semantica.vector_store.weaviate_store.WEAVIATE_AVAILABLE', True):
|
||||
results = store.filter_by_metadata({"dept": "eng"}, limit=5)
|
||||
self.assertEqual(len(results), 2)
|
||||
self.assertEqual(results[0]["id"], "match-uuid-1")
|
||||
self.assertEqual(results[1]["id"], "match-uuid-2")
|
||||
|
||||
def test_weaviate_store_filter_by_metadata_native_filter(self):
|
||||
"""Test building native Weaviate filters for exact, range, and list criteria."""
|
||||
store = WeaviateStore()
|
||||
mock_filter_cls = MagicMock()
|
||||
mock_filter_prop = MagicMock()
|
||||
mock_filter_cls.by_property.return_value = mock_filter_prop
|
||||
|
||||
mock_module = MagicMock()
|
||||
mock_module.classes.query.Filter = mock_filter_cls
|
||||
|
||||
with patch('semantica.vector_store.weaviate_store.WEAVIATE_AVAILABLE', True), \
|
||||
patch('semantica.vector_store.weaviate_store.weaviate', mock_module):
|
||||
|
||||
# Test exact match
|
||||
res = store._build_weaviate_filter({"dept": "eng"})
|
||||
mock_filter_cls.by_property.assert_called_with("dept")
|
||||
mock_filter_prop.equal.assert_called_with("eng")
|
||||
|
||||
# Test range filter
|
||||
mock_filter_cls.reset_mock()
|
||||
mock_filter_prop.reset_mock()
|
||||
res = store._build_weaviate_filter({"age": {"min": 20, "max": 50}})
|
||||
mock_filter_cls.by_property.assert_called_with("age")
|
||||
mock_filter_prop.greater_or_equal.assert_called_with(20)
|
||||
mock_filter_prop.less_or_equal.assert_called_with(50)
|
||||
|
||||
# Test list filter
|
||||
mock_filter_cls.reset_mock()
|
||||
mock_filter_prop.reset_mock()
|
||||
res = store._build_weaviate_filter({"tags": ["a", "b"]})
|
||||
mock_filter_cls.by_property.assert_called_with("tags")
|
||||
mock_filter_prop.contains_any.assert_called_with(["a", "b"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -773,19 +773,21 @@ class TestBuildDecisionContextFAISSBackend:
|
||||
|
||||
class TestFilterByMetadataBackendBehavior:
|
||||
"""
|
||||
Requirement (issue #848 follow-up): verify the chosen behavior of
|
||||
Requirement (issue #848, superseded by #857): verify the behavior of
|
||||
_filter_by_metadata when a non-inmemory backend is active.
|
||||
|
||||
The decision: raise NotImplementedError (matching get_vector / get_metadata
|
||||
from #843) rather than silently returning [].
|
||||
#848's original decision was to raise NotImplementedError (matching
|
||||
get_vector / get_metadata from #843) rather than silently return [],
|
||||
because at the time zero backend wrappers implemented
|
||||
filter_by_metadata(filters, limit).
|
||||
|
||||
Rationale documented in the production comment:
|
||||
- Zero backend wrappers implement filter_by_metadata(filters, limit).
|
||||
- The only codebase hit (HybridSearch.filter_by_metadata) has a completely
|
||||
different signature and is never stored in _backend_store.
|
||||
- Returning [] would make filter_decisions(query=None, category="loan")
|
||||
report "zero matches" when the truth is "capability not available" —
|
||||
indistinguishable from a real empty result and therefore wrong.
|
||||
#857 gave every persistent backend (FAISS, Qdrant, Pinecone, Milvus,
|
||||
PgVector, SQLiteVec, Weaviate) a real filter_by_metadata() implementation,
|
||||
so FAISS-backed filter_decisions(query=None, ...) now returns actual
|
||||
filtered results instead of raising. The NotImplementedError path itself
|
||||
is still correct and still covered (see
|
||||
test_filter_by_metadata_backend_not_implemented in test_vector_store.py)
|
||||
for a backend that genuinely lacks the method.
|
||||
"""
|
||||
|
||||
def _make_faiss_store(self):
|
||||
@@ -809,34 +811,26 @@ class TestFilterByMetadataBackendBehavior:
|
||||
)
|
||||
return vs, ids
|
||||
|
||||
# ── FAISS backend: NotImplementedError, not AttributeError, not [] ── #
|
||||
# ── FAISS backend: real results, not AttributeError, not [] ── #
|
||||
|
||||
def test_filter_by_metadata_faiss_raises_not_implemented(self):
|
||||
def test_filter_by_metadata_faiss_returns_real_results(self):
|
||||
"""
|
||||
filter_decisions(query=None, category='loan') on a FAISS-backed store
|
||||
must raise NotImplementedError, not AttributeError (old crash) and not
|
||||
silently return [] (the wrong silent-failure fix).
|
||||
|
||||
This test pins the chosen behavior: explicit NotImplementedError matching
|
||||
the get_vector/get_metadata precedent set by issue #843.
|
||||
must return the actual matching decisions, not raise AttributeError
|
||||
(old crash) and not silently return [] (the old NotImplementedError
|
||||
stand-in from #848, superseded once #857 gave FAISSStore a real
|
||||
filter_by_metadata()).
|
||||
"""
|
||||
vs, _ids = self._make_faiss_store()
|
||||
|
||||
with pytest.raises(NotImplementedError) as exc_info:
|
||||
vs.filter_decisions(query=None, category="loan")
|
||||
results = vs.filter_decisions(query=None, category="loan")
|
||||
|
||||
# Message must name the backend and point to the correct alternative
|
||||
msg = str(exc_info.value)
|
||||
assert "FAISSStore" in msg, (
|
||||
f"Error message should name the backend class, got: {msg!r}"
|
||||
)
|
||||
assert "filter_decisions" in msg or "filter_by_metadata" in msg, (
|
||||
f"Error message should mention the failing method, got: {msg!r}"
|
||||
)
|
||||
assert "search_decisions" in msg, (
|
||||
f"Error message should suggest search_decisions() as the alternative, "
|
||||
f"got: {msg!r}"
|
||||
assert isinstance(results, list)
|
||||
assert len(results) == 2, (
|
||||
f"Expected 2 loan decisions, got {len(results)}: {results}"
|
||||
)
|
||||
for r in results:
|
||||
assert r["metadata"]["category"] == "loan"
|
||||
|
||||
def test_filter_by_metadata_faiss_not_attribute_error(self):
|
||||
"""
|
||||
|
||||
@@ -82,6 +82,7 @@ class TestPineconeStore(unittest.TestCase):
|
||||
self.assertIsInstance(store.search_engine, PineconeSearch)
|
||||
store.client.create_index.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_upsert_vectors(self, mock_pinecone_client):
|
||||
"""Test upserting vectors to Pinecone index."""
|
||||
@@ -105,6 +106,7 @@ class TestPineconeStore(unittest.TestCase):
|
||||
self.assertEqual(result["upserted_count"], 2)
|
||||
store.index.upsert_vectors.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_search_vectors(self, mock_pinecone_client):
|
||||
"""Test searching vectors in Pinecone index."""
|
||||
@@ -128,6 +130,7 @@ class TestPineconeStore(unittest.TestCase):
|
||||
self.assertEqual(results[0]["id"], "id1")
|
||||
store.search_engine.similarity_search.assert_called_once()
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_delete_vectors(self, mock_pinecone_client):
|
||||
"""Test deleting vectors from Pinecone index."""
|
||||
@@ -148,6 +151,7 @@ class TestPineconeStore(unittest.TestCase):
|
||||
# Fix: assert called without the empty dict
|
||||
store.index.delete_vectors.assert_called_once_with(["id1", "id2"], "")
|
||||
|
||||
@patch('semantica.vector_store.pinecone_store.PINECONE_AVAILABLE', True)
|
||||
@patch('semantica.vector_store.pinecone_store.PineconeClientLib')
|
||||
def test_fetch_vectors(self, mock_pinecone_client):
|
||||
"""Test fetching vectors from Pinecone index."""
|
||||
|
||||
@@ -413,3 +413,51 @@ class TestSQLiteVecStoreStats:
|
||||
|
||||
stats = store.get_stats()
|
||||
assert stats["vector_count"] == 4
|
||||
|
||||
|
||||
class TestSQLiteVecStoreFilterByMetadata:
|
||||
"""Test filter_by_metadata, including list-valued metadata handling."""
|
||||
|
||||
def test_filter_exact_match(self, store):
|
||||
vectors = [np.random.rand(128).astype(np.float32) for _ in range(2)]
|
||||
metadata = [{"category": "finance"}, {"category": "tech"}]
|
||||
ids = store.add(vectors, metadata, ids=["v1", "v2"])
|
||||
|
||||
results = store.filter_by_metadata({"category": "finance"}, limit=10)
|
||||
|
||||
assert [r["id"] for r in results] == ["v1"]
|
||||
|
||||
def test_filter_scalar_field_against_list_filter(self, store):
|
||||
"""A scalar metadata value should match via plain IN-list membership."""
|
||||
vectors = [np.random.rand(128).astype(np.float32) for _ in range(2)]
|
||||
metadata = [{"category": "finance"}, {"category": "tech"}]
|
||||
store.add(vectors, metadata, ids=["v1", "v2"])
|
||||
|
||||
results = store.filter_by_metadata({"category": ["finance", "ops"]}, limit=10)
|
||||
|
||||
assert [r["id"] for r in results] == ["v1"]
|
||||
|
||||
def test_filter_array_field_intersects_list_filter(self, store):
|
||||
"""A list-valued metadata field must match on set intersection with the
|
||||
filter list, mirroring the in-memory backend's semantics -- not on a
|
||||
literal comparison of the whole array's JSON text against each candidate.
|
||||
"""
|
||||
vectors = [np.random.rand(128).astype(np.float32) for _ in range(3)]
|
||||
metadata = [
|
||||
{"tags": ["python", "js"]},
|
||||
{"tags": ["go"]},
|
||||
{"tags": ["python", "ml"]},
|
||||
]
|
||||
store.add(vectors, metadata, ids=["v1", "v2", "v3"])
|
||||
|
||||
results = store.filter_by_metadata({"tags": ["python", "ml"]}, limit=10)
|
||||
|
||||
assert {r["id"] for r in results} == {"v1", "v3"}
|
||||
|
||||
def test_filter_limit_zero_returns_empty(self, store):
|
||||
vectors = [np.random.rand(128).astype(np.float32)]
|
||||
store.add(vectors, [{"category": "finance"}], ids=["v1"])
|
||||
|
||||
results = store.filter_by_metadata({"category": "finance"}, limit=0)
|
||||
|
||||
assert results == []
|
||||
|
||||
@@ -137,6 +137,69 @@ class TestVectorStore(unittest.TestCase):
|
||||
|
||||
self.assertTrue(mock_backend.called)
|
||||
|
||||
def test_filter_by_metadata_inmemory(self):
|
||||
"""Test _filter_by_metadata on inmemory backend."""
|
||||
store = VectorStore(backend="inmemory")
|
||||
store.metadata = {
|
||||
"v1": {"category": "finance", "amount": 100, "tags": ["a", "b"]},
|
||||
"v2": {"category": "finance", "amount": 500, "tags": ["b", "c"]},
|
||||
"v3": {"category": "tech", "amount": 200, "tags": ["c"]},
|
||||
}
|
||||
store.vectors = {
|
||||
"v1": np.array([0.1]),
|
||||
"v2": np.array([0.2]),
|
||||
"v3": np.array([0.3]),
|
||||
}
|
||||
|
||||
# Exact filter
|
||||
results = store._filter_by_metadata({"category": "finance"}, limit=10)
|
||||
self.assertEqual(len(results), 2)
|
||||
res_ids = {r["id"] for r in results}
|
||||
self.assertEqual(res_ids, {"v1", "v2"})
|
||||
|
||||
# Range filter
|
||||
results = store._filter_by_metadata({"amount": {"min": 150}}, limit=10)
|
||||
self.assertEqual(len(results), 2)
|
||||
res_ids = {r["id"] for r in results}
|
||||
self.assertEqual(res_ids, {"v2", "v3"})
|
||||
|
||||
# List intersection filter
|
||||
results = store._filter_by_metadata({"tags": ["a"]}, limit=10)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "v1")
|
||||
|
||||
def test_filter_by_metadata_persistent_backend_delegation(self):
|
||||
"""Test that persistent backend delegates filter_by_metadata without AttributeError."""
|
||||
store = VectorStore(backend="inmemory")
|
||||
# Simulate persistent backend by deleting self.metadata attribute if any
|
||||
if hasattr(store, "metadata"):
|
||||
delattr(store, "metadata")
|
||||
|
||||
mock_backend = MagicMock()
|
||||
mock_backend.filter_by_metadata.return_value = [
|
||||
{"id": "p1", "metadata": {"category": "test"}, "vector": np.array([0.5])}
|
||||
]
|
||||
store._backend_store = mock_backend
|
||||
store.backend = "faiss"
|
||||
|
||||
# Should NOT raise AttributeError: 'VectorStore' object has no attribute 'metadata'
|
||||
results = store._filter_by_metadata({"category": "test"}, limit=5)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "p1")
|
||||
mock_backend.filter_by_metadata.assert_called_once_with(filters={"category": "test"}, limit=5)
|
||||
|
||||
def test_filter_by_metadata_backend_not_implemented(self):
|
||||
"""Test that missing filter_by_metadata method raises NotImplementedError."""
|
||||
store = VectorStore(backend="inmemory")
|
||||
if hasattr(store, "metadata"):
|
||||
delattr(store, "metadata")
|
||||
|
||||
store.backend = "unknown"
|
||||
store._backend_store = object()
|
||||
|
||||
with self.assertRaises(NotImplementedError):
|
||||
store._filter_by_metadata({"key": "val"}, limit=10)
|
||||
|
||||
def test_save_load_roundtrip_numpy_vectors(self):
|
||||
"""save()/load() must handle numpy float32 vectors without raising.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user