diff --git a/pyproject.toml b/pyproject.toml index 0098ffc1..0278a764 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,9 +147,10 @@ vectorstore-weaviate = ["weaviate-client>=4.0.0"] vectorstore-pinecone = ["pinecone-client>=3.0.0"] vectorstore-milvus = ["pymilvus>=2.0.0"] vectorstore-pgvector = ["psycopg[binary,pool]>=3.0.0", "pgvector>=0.2.0"] +vectorstore-sqlite = ["sqlite-vec>=0.1.1"] vectorstore-all = [ - "semantica[vectorstore-qdrant,vectorstore-weaviate,vectorstore-pinecone,vectorstore-milvus,vectorstore-pgvector]" + "semantica[vectorstore-qdrant,vectorstore-weaviate,vectorstore-pinecone,vectorstore-milvus,vectorstore-pgvector,vectorstore-sqlite]" ] # ---- Infra / Queues / Workers ---- diff --git a/semantica/utils/constants.py b/semantica/utils/constants.py index 0853b69a..05d03b7b 100644 --- a/semantica/utils/constants.py +++ b/semantica/utils/constants.py @@ -80,6 +80,7 @@ SUPPORTED_VECTOR_STORES = [ "qdrant", "milvus", "chroma", + "sqlite", ] # Supported Graph Databases diff --git a/semantica/vector_store/__init__.py b/semantica/vector_store/__init__.py index 635e5b40..994b0bb3 100644 --- a/semantica/vector_store/__init__.py +++ b/semantica/vector_store/__init__.py @@ -138,12 +138,29 @@ from .hybrid_search import HybridSearch, MetadataFilter, SearchRanker from .hybrid_similarity import HybridSimilarityCalculator from .decision_embedding_pipeline import DecisionEmbeddingPipeline from .decision_vector_methods import ( - quick_decision, find_precedents, explain, similar_to, batch_decisions, - filter_decisions, get_decision_context, search_by_entities, get_decision_statistics, - update_similarity_weights, set_global_vector_store, get_global_vector_store, + quick_decision, + find_precedents, + explain, + similar_to, + batch_decisions, + filter_decisions, + get_decision_context, + search_by_entities, + get_decision_statistics, + update_similarity_weights, + set_global_vector_store, + get_global_vector_store, # Aliases - record, precedents, explain_decision, similar, batch, filter, context, - by_entities, stats, weights + record, + precedents, + explain_decision, + similar, + batch, + filter, + context, + by_entities, + stats, + weights, ) from .metadata_store import MetadataIndex, MetadataSchema, MetadataStore from .methods import ( @@ -161,6 +178,7 @@ from .methods import ( from .milvus_store import MilvusStore, MilvusClient, MilvusCollection, MilvusSearch from .namespace_manager import Namespace, NamespaceManager from .pgvector_store import PgVectorStore +from .sqlite_vec_store import SQLiteVecStore from .pinecone_store import PineconeStore, PineconeClient, PineconeIndex, PineconeSearch from .qdrant_store import QdrantStore, QdrantClient, QdrantCollection, QdrantSearch from .registry import MethodRegistry, method_registry @@ -205,6 +223,8 @@ __all__ = [ "PineconeSearch", # PgVector "PgVectorStore", + # SQLite + "SQLiteVecStore", # Hybrid search "HybridSearch", "MetadataFilter", diff --git a/semantica/vector_store/config.py b/semantica/vector_store/config.py index 39064cfd..c6fb3e00 100644 --- a/semantica/vector_store/config.py +++ b/semantica/vector_store/config.py @@ -120,6 +120,7 @@ class VectorStoreConfig: "VECTOR_STORE_QDRANT_URL": "qdrant_url", "VECTOR_STORE_MILVUS_HOST": "milvus_host", "VECTOR_STORE_MILVUS_PORT": "milvus_port", + "VECTOR_STORE_SQLITE_PATH": "sqlite_path", } for env_var, config_key in env_mappings.items(): diff --git a/semantica/vector_store/sqlite_vec_store.py b/semantica/vector_store/sqlite_vec_store.py new file mode 100644 index 00000000..a359bda0 --- /dev/null +++ b/semantica/vector_store/sqlite_vec_store.py @@ -0,0 +1,626 @@ +""" +SQLite Vector Store Module using sqlite-vec + +This module provides SQLite integration using the sqlite-vec extension for vector storage and +similarity search in the Semantica framework, supporting L2 and Cosine distance metrics, +dynamic JSON metadata filtering, and disk-backed persistence. + +Key Features: + - Distance metrics (Cosine, L2/Euclidean) + - Fully persistent or in-memory SQLite storage + - Dynamic metadata filtering using SQLite's JSON extract functions + - Fully thread-safe operations via locks and WAL mode + - Strict validation of vector dimensions and table names + - Parity with PgVectorStore interface for seamless drop-in usage + +Main Classes: + - SQLiteVecStore: Main SQLite vector store using sqlite-vec virtual tables + +Example Usage: + >>> from semantica.vector_store import SQLiteVecStore + >>> store = SQLiteVecStore( + ... db_path="vectors.db", + ... table_name="vectors", + ... dimension=768, + ... distance_metric="cosine" + ... ) + >>> store.add(vectors, metadata, ids) + >>> results = store.search(query_vector, top_k=10) + >>> store.close() + +Author: Semantica Contributors +License: MIT +""" + +import json +import os +import re +import sqlite3 +import threading +import uuid +from contextlib import contextmanager +from typing import Any, Dict, List, Optional, Union +from urllib.request import pathname2url + +import numpy as np + +from ..utils.exceptions import ProcessingError, ValidationError +from ..utils.logging import get_logger + +# Optional sqlite-vec import +try: + import sqlite_vec + + SQLITE_VEC_AVAILABLE = True +except (ImportError, OSError): + SQLITE_VEC_AVAILABLE = False + sqlite_vec = None + + +class SQLiteVecStore: + """ + SQLite vector store using the sqlite-vec extension for similarity search. + + - Vector storage with vec0 virtual table + - Similarity search with L2 and Cosine metrics + - Thread-safe query and insertion execution + - Dynamic metadata extraction and filtering + """ + + SUPPORTED_METRICS = {"cosine", "l2"} + + def __init__( + self, + db_path: str, + table_name: str, + dimension: int, + distance_metric: str = "cosine", + read_only: bool = False, + **kwargs, + ): + """ + Initialize SQLiteVecStore. + + Args: + db_path: Path to SQLite database file, or ':memory:' + table_name: Name of the virtual table to store vectors + dimension: Vector dimension + distance_metric: Distance metric (cosine, l2) + read_only: Open database in read-only mode (requires existing database) + **kwargs: Additional option parameters + + Raises: + ValidationError: If parameters are invalid + ProcessingError: If sqlite-vec or connection load extension fails + """ + self.logger = get_logger("sqlite_vec_store") + + # Validate dependencies + if not SQLITE_VEC_AVAILABLE: + raise ProcessingError( + "sqlite-vec Python package is not available. " + "Install with: pip install sqlite-vec" + ) + + # Validate parameters + if distance_metric.lower() not in self.SUPPORTED_METRICS: + raise ValidationError( + f"Unsupported distance metric: {distance_metric}. " + f"Supported: {', '.join(self.SUPPORTED_METRICS)}" + ) + + if not self._is_safe_identifier(table_name): + raise ValidationError( + f"Invalid table name: {table_name!r}. " + "Table names must be alphanumeric with underscores/hyphens only and start with a letter/underscore." + ) + + self.db_path = db_path + self.table_name = table_name + self.dimension = dimension + self.distance_metric = distance_metric.lower() + self.read_only = read_only + self.config = kwargs + self.use_wal = kwargs.get("use_wal", False) + + # Lock to ensure thread safety when sharing a single SQLite connection + self._lock = threading.Lock() + self._conn = None + + # Connect to database + self._init_connection() + + # Ensure table exists (if not read-only) + if not self.read_only: + self._ensure_table_exists() + + self.logger.info( + f"Initialized SQLiteVecStore: db_path={db_path}, table={table_name}, " + f"dimension={dimension}, metric={distance_metric}, read_only={read_only}" + ) + + def _init_connection(self): + """Initialize database connection and load sqlite-vec extension.""" + try: + if self.read_only: + if self.db_path == ":memory:": + self._conn = sqlite3.connect(":memory:", check_same_thread=False) + else: + if not os.path.exists(self.db_path): + raise ProcessingError( + f"Database file does not exist for read-only mode: {self.db_path}" + ) + abs_path = os.path.abspath(self.db_path) + url_path = pathname2url(abs_path) + self._conn = sqlite3.connect( + f"file:{url_path}?mode=ro", uri=True, check_same_thread=False + ) + else: + self._conn = sqlite3.connect(self.db_path, check_same_thread=False) + + # Enable and load extension + try: + self._conn.enable_load_extension(True) + sqlite_vec.load(self._conn) + self._conn.enable_load_extension(False) + except AttributeError as ae: + raise ProcessingError( + "SQLite load extension attribute is not available in this Python build. " + "Make sure you are using a Python build compiled with loadable extension support." + ) from ae + except Exception as e: + raise ProcessingError( + f"Failed to load sqlite-vec extension: {e}" + ) from e + + # Set WAL journal mode for performance and concurrent readers, if configured + if self.use_wal and not self.read_only and self.db_path != ":memory:": + self._conn.execute("PRAGMA journal_mode=WAL") + + except (ValidationError, ProcessingError): + if self._conn: + self._conn.close() + raise + except Exception as e: + if self._conn: + self._conn.close() + raise ProcessingError(f"Failed to establish SQLite connection: {e}") from e + + @contextmanager + def _get_connection(self): + """Get the active connection.""" + if not self._conn: + raise ProcessingError("Database connection is closed.") + try: + yield self._conn + except (ValidationError, ProcessingError): + raise + except Exception as e: + raise ProcessingError("Database operation failed") from e + + def _is_safe_identifier(self, key: str) -> bool: + """ + Validate that a string is safe to use as a SQL identifier. + + Only allows alphanumeric characters, underscores, and hyphens. + """ + if not isinstance(key, str): + return False + if not key: + return False + return bool(re.match(r"^[a-zA-Z_][a-zA-Z0-9_-]*$", key)) + + def _ensure_table_exists(self): + """Ensure the vector table exists.""" + # Define vec0 virtual table + create_table_sql = f""" + CREATE VIRTUAL TABLE IF NOT EXISTS {self.table_name} USING vec0( + id TEXT PRIMARY KEY, + embedding float[{self.dimension}] distance_metric={self.distance_metric}, + +metadata TEXT + ) + """ + with self._lock, self._get_connection() as conn: + try: + conn.execute(create_table_sql) + conn.commit() + self.logger.debug(f"Virtual table {self.table_name} ensured") + except Exception as e: + conn.rollback() + raise ProcessingError("Failed to create vec0 virtual table") from e + + def add( + self, + vectors: Union[List[np.ndarray], np.ndarray], + metadata: Optional[List[Dict[str, Any]]] = None, + ids: Optional[List[str]] = None, + ) -> List[str]: + """ + Add vectors to the store. + + Args: + vectors: List of vectors or numpy array + metadata: List of metadata dictionaries (one per vector) + ids: Optional list of IDs (auto-generated if not provided) + + Returns: + List of vector IDs + + Raises: + ValidationError: If input dimensions or lengths don't match + ProcessingError: If read-only mode is active or database operation fails + """ + if self.read_only: + raise ProcessingError("Cannot add vectors in read-only mode") + + # Convert to list if numpy array + if isinstance(vectors, np.ndarray): + vectors = [vectors[i] for i in range(len(vectors))] + + num_vectors = len(vectors) + + # Validate dimensions + for i, vec in enumerate(vectors): + if len(vec) != self.dimension: + raise ValidationError( + f"Vector at index {i} has dimension {len(vec)}, " + f"expected {self.dimension}" + ) + + # Generate IDs if not provided + if ids is None: + ids = [str(uuid.uuid4()) for _ in range(num_vectors)] + elif len(ids) != num_vectors: + raise ValidationError( + f"IDs length ({len(ids)}) must match vectors length ({num_vectors})" + ) + + # Prepare metadata + if metadata is None: + metadata = [{} for _ in range(num_vectors)] + elif len(metadata) != num_vectors: + raise ValidationError( + f"Metadata length ({len(metadata)}) must match vectors length ({num_vectors})" + ) + + with self._lock, self._get_connection() as conn: + try: + cur = conn.cursor() + # Run deletes first to emulate INSERT OR REPLACE / UPSERT behavior + # Use batched deletes to avoid a massive performance bottleneck + if ids: + batch_size = 1000 + for i in range(0, len(ids), batch_size): + batch_ids = ids[i : i + batch_size] + placeholders = ",".join(["?"] * len(batch_ids)) + cur.execute( + f"DELETE FROM {self.table_name} WHERE id IN ({placeholders})", + batch_ids, + ) + + # Build data tuples for insert + data_tuples = [ + ( + vec_id, + sqlite_vec.serialize_float32(vec), + json.dumps(meta), + ) + for vec_id, vec, meta in zip(ids, vectors, metadata) + ] + + # Bulk insert + cur.executemany( + f"INSERT INTO {self.table_name} (id, embedding, metadata) VALUES (?, ?, ?)", + data_tuples, + ) + conn.commit() + cur.close() + self.logger.info(f"Added {num_vectors} vectors") + return ids + except (ValidationError, ProcessingError): + conn.rollback() + raise + except Exception as e: + conn.rollback() + raise ProcessingError("Failed to add vectors to database") from e + + def search( + self, + query_vector: np.ndarray, + top_k: int = 10, + filter: Optional[Dict[str, Any]] = None, + ) -> List[Dict[str, Any]]: + """ + Search for similar vectors. + + Args: + query_vector: Query vector + top_k: Number of results to return + filter: Optional metadata filter (dict of key-value pairs) + + Returns: + List of results with id, score (similarity), and metadata + + Raises: + ValidationError: If query vector dimension doesn't match + ProcessingError: If database operation fails + """ + if len(query_vector) != self.dimension: + raise ValidationError( + f"Query vector has dimension {len(query_vector)}, expected {self.dimension}" + ) + + # Serialize query vector + query_serialized = sqlite_vec.serialize_float32(query_vector) + + # Build query + filter_conditions = [] + filter_params = [] + if filter: + for key, value in filter.items(): + if not self._is_safe_identifier(key): + raise ValidationError( + f"Invalid filter key: {key!r}. " + "Keys must be alphanumeric with underscores/hyphens only." + ) + filter_conditions.append(f"json_extract(metadata, '$.{key}') = ?") + if isinstance(value, bool): + filter_params.append(1 if value else 0) + else: + filter_params.append(value) + + where_clause = "" + if filter_conditions: + where_clause = " AND " + " AND ".join(filter_conditions) + + search_sql = f""" + SELECT id, distance, metadata + FROM {self.table_name} + WHERE embedding MATCH ? AND k = ?{where_clause} + """ + params = [query_serialized, top_k] + filter_params + + with self._lock, self._get_connection() as conn: + try: + cur = conn.cursor() + cur.execute(search_sql, params) + rows = cur.fetchall() + cur.close() + + results = [] + for row in rows: + vec_id, distance, meta_json = row + # Convert distance to similarity score + similarity = 1.0 / (1.0 + float(distance)) + + results.append( + { + "id": vec_id, + "score": similarity, + "metadata": json.loads(meta_json) if meta_json else {}, + } + ) + + return results + except (ValidationError, ProcessingError): + raise + except Exception as e: + raise ProcessingError("Failed to search vectors") from e + + def delete(self, ids: List[str]) -> bool: + """ + Delete vectors by ID. + + Args: + ids: List of vector IDs to delete + + Returns: + True if successful + + Raises: + ProcessingError: If read-only mode is active or database operation fails + """ + if not ids: + return True + + if self.read_only: + raise ProcessingError("Cannot delete vectors in read-only mode") + + with self._lock, self._get_connection() as conn: + try: + cur = conn.cursor() + for vec_id in ids: + cur.execute( + f"DELETE FROM {self.table_name} WHERE id = ?", (vec_id,) + ) + conn.commit() + cur.close() + self.logger.info(f"Deleted vectors: {len(ids)}") + return True + except Exception as e: + conn.rollback() + raise ProcessingError("Failed to delete vectors") from e + + def update( + self, + ids: List[str], + vectors: Optional[Union[List[np.ndarray], np.ndarray]] = None, + metadata: Optional[List[Dict[str, Any]]] = None, + ) -> bool: + """ + Update existing vectors. + + Args: + ids: List of vector IDs to update + vectors: Optional new vectors + metadata: Optional new metadata + + Returns: + True if successful + + Raises: + ValidationError: If input dimensions or lengths don't match + ProcessingError: If read-only mode is active or database operation fails + """ + if not ids: + return True + + if self.read_only: + raise ProcessingError("Cannot update vectors in read-only mode") + + if vectors is None and metadata is None: + raise ValidationError( + "Either vectors or metadata must be provided for update" + ) + + if vectors is not None: + if isinstance(vectors, np.ndarray): + vectors = [vectors[i] for i in range(len(vectors))] + if len(vectors) != len(ids): + raise ValidationError("Vectors length must match IDs length") + for i, vec in enumerate(vectors): + if len(vec) != self.dimension: + raise ValidationError( + f"Vector at index {i} has dimension {len(vec)}, expected {self.dimension}" + ) + + if metadata is not None and len(metadata) != len(ids): + raise ValidationError("Metadata length must match IDs length") + + with self._lock, self._get_connection() as conn: + try: + cur = conn.cursor() + for i, vec_id in enumerate(ids): + updates = [] + params = [] + + if vectors is not None: + updates.append("embedding = ?") + params.append(sqlite_vec.serialize_float32(vectors[i])) + + if metadata is not None: + updates.append("metadata = ?") + params.append(json.dumps(metadata[i])) + + params.append(vec_id) + cur.execute( + f"UPDATE {self.table_name} SET {', '.join(updates)} WHERE id = ?", + params, + ) + conn.commit() + cur.close() + self.logger.info(f"Updated {len(ids)} vectors") + return True + except ValidationError: + conn.rollback() + raise + except Exception as e: + conn.rollback() + raise ProcessingError("Failed to update vectors") from e + + def get(self, ids: List[str]) -> List[Dict[str, Any]]: + """ + Get vectors by ID. + + Args: + ids: List of vector IDs + + Returns: + List of dictionaries with id, vector, and metadata + + Raises: + ProcessingError: If database operation fails + """ + if not ids: + return [] + + with self._lock, self._get_connection() as conn: + try: + cur = conn.cursor() + results = [] + for vec_id in ids: + cur.execute( + f"SELECT id, embedding, metadata FROM {self.table_name} WHERE id = ?", + (vec_id,), + ) + row = cur.fetchone() + if row: + vid, embedding_blob, metadata_json = row + vec = ( + np.frombuffer(embedding_blob, dtype=np.float32).copy() + if embedding_blob + else None + ) + meta = json.loads(metadata_json) if metadata_json else {} + results.append( + { + "id": vid, + "vector": vec, + "metadata": meta, + } + ) + cur.close() + return results + except Exception as e: + raise ProcessingError("Failed to get vectors") from e + + def create_index( + self, + index_type: str = "hnsw", + params: Optional[Dict[str, Any]] = None, + ) -> bool: + """ + Create an index on the vector column. + For SQLiteVecStore, vec0 virtual tables automatically index vectors, + so this is a no-op that always returns True for interface parity. + """ + self.logger.debug("create_index is a no-op for SQLiteVecStore") + return True + + def get_stats(self) -> Dict[str, Any]: + """ + Get store statistics. + + Returns: + Dictionary with vector_count, dimension, and distance_metric + """ + with self._lock, self._get_connection() as conn: + try: + cur = conn.cursor() + cur.execute(f"SELECT COUNT(*) FROM {self.table_name}") + count = cur.fetchone()[0] + cur.close() + return { + "vector_count": count, + "dimension": self.dimension, + "distance_metric": self.distance_metric, + } + except Exception as e: + raise ProcessingError("Failed to get store statistics") from e + + def close(self): + """Close the database connection.""" + if hasattr(self, "_lock") and self._lock: + with self._lock: + if hasattr(self, "_conn") and self._conn: + try: + self._conn.close() + except Exception: + pass + self._conn = None + else: + if hasattr(self, "_conn") and self._conn: + try: + self._conn.close() + except Exception: + pass + self._conn = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def __del__(self): + self.close() diff --git a/semantica/vector_store/vector_store.py b/semantica/vector_store/vector_store.py index aff2bae7..356c73e8 100644 --- a/semantica/vector_store/vector_store.py +++ b/semantica/vector_store/vector_store.py @@ -90,7 +90,7 @@ class VectorStore: • Provides vector store operations """ - SUPPORTED_BACKENDS = {"faiss", "weaviate", "qdrant", "milvus", "pinecone", "pgvector", "inmemory"} + SUPPORTED_BACKENDS = {"faiss", "weaviate", "qdrant", "milvus", "pinecone", "pgvector", "inmemory", "sqlite"} def __init__(self, backend="faiss", config=None, max_workers: int = 6, **kwargs): """Initialize vector store.""" @@ -210,6 +210,31 @@ class VectorStore: ) self.logger.info(f"Initialized MilvusStore backend") + elif self.backend == "sqlite": + from .sqlite_vec_store import SQLiteVecStore + db_path = self.config.get("db_path") or self.config.get("sqlite_path") + if not db_path: + raise ValueError( + "sqlite backend requires 'db_path' in config. " + "Example: VectorStore(backend='sqlite', config={'db_path': 'vectors.db'})" + ) + + table_name = self.config.get("table_name", "vectors") + dimension = self.config.get("dimension", 768) + distance_metric = self.config.get("distance_metric", "cosine") + read_only = self.config.get("read_only", False) + + self._backend_store = SQLiteVecStore( + db_path=db_path, + table_name=table_name, + dimension=dimension, + distance_metric=distance_metric, + read_only=read_only, + **{k: v for k, v in self.config.items() + if k not in ['db_path', 'sqlite_path', 'table_name', 'dimension', 'distance_metric', 'read_only']} + ) + self.logger.info(f"Initialized SQLite backend") + else: # Fallback to in-memory for unknown backends self.logger.warning(f"Backend '{self.backend}' not implemented, using in-memory") @@ -218,7 +243,7 @@ class VectorStore: except ImportError as e: raise ImportError(f"Backend '{self.backend}' not available: {e}. Please install required dependencies.") from e except Exception as e: - if "requires" in str(e) and "connection_string" in str(e): + if "requires" in str(e) and ("connection_string" in str(e) or "db_path" in str(e)): # Re-raise validation errors for missing required parameters raise else: diff --git a/semantica/vector_store/vector_store_usage.md b/semantica/vector_store/vector_store_usage.md index b0a59939..13dbf3ab 100644 --- a/semantica/vector_store/vector_store_usage.md +++ b/semantica/vector_store/vector_store_usage.md @@ -1752,6 +1752,64 @@ method_registry.register("store", "normalized", custom_store_vectors) vector_ids = store_vectors(vectors, metadata=metadata, method="normalized") ``` +### SQLite Vector Store (sqlite-vec) + +The SQLite vector store backend uses the `sqlite-vec` extension to provide a lightweight, embedded, yet fully persistent vector store. It is ideal for local development, small-to-medium datasets, and embedded applications where setting up a separate PostgreSQL/pgvector instance is not desired. + +#### Installation + +Install Semantica with SQLite vector store dependencies: + +```bash +pip install semantica[vectorstore-sqlite] +``` + +#### Usage Example + +```python +from semantica.vector_store import VectorStore +import numpy as np + +# Initialize the vector store using the 'sqlite' backend +# The 'db_path' parameter points to the SQLite database file on disk. +# Use ':memory:' for a transient, in-memory store. +store = VectorStore( + backend="sqlite", + config={ + "db_path": "my_vector_database.db", + "table_name": "documents", + "dimension": 128, + "distance_metric": "cosine" # Supported metrics: 'cosine', 'l2' + } +) + +# Store vectors with metadata +vectors = [np.random.rand(128).astype(np.float32) for _ in range(5)] +metadata = [ + {"category": "ai", "public": True}, + {"category": "finance", "public": False}, + {"category": "ai", "public": False}, + {"category": "healthcare", "public": True}, + {"category": "finance", "public": True} +] +ids = store.store_vectors(vectors, metadata=metadata) + +# Search vectors with metadata filtering +query = np.random.rand(128).astype(np.float32) +results = store.search_vectors( + query, + k=2, + filter={"category": "ai"} +) + +for r in results: + print(f"ID: {r['id']}, Score: {r['score']}, Metadata: {r['metadata']}") + +# Close the database connection when done +# (Recommended on Windows to release file handles) +store._backend_store.close() +``` + ## Best Practices 1. **Vector Storage**: diff --git a/tests/vector_store/test_sqlite_vec_store.py b/tests/vector_store/test_sqlite_vec_store.py new file mode 100644 index 00000000..eec62256 --- /dev/null +++ b/tests/vector_store/test_sqlite_vec_store.py @@ -0,0 +1,416 @@ +""" +SQLite Vector Store Tests + +This module provides comprehensive unit tests for the SQLiteVecStore implementation using sqlite-vec. +Tests are skipped if sqlite-vec is not available. + +pytest tests/vector_store/test_sqlite_vec_store.py -v +""" + +import os +import uuid +import numpy as np +import pytest +from typing import Generator + +# Check dependencies +try: + from semantica.vector_store.sqlite_vec_store import SQLITE_VEC_AVAILABLE +except ImportError: + SQLITE_VEC_AVAILABLE = False + +# Skip all tests in this file if sqlite-vec is not available +pytestmark = pytest.mark.skipif( + not SQLITE_VEC_AVAILABLE, reason="sqlite-vec not available" +) + + +@pytest.fixture +def unique_table_name() -> str: + """Generate a unique table name for test isolation.""" + return f"test_vectors_{uuid.uuid4().hex[:8]}" + + +@pytest.fixture +def db_file(tmp_path) -> str: + """Create a temporary database file path.""" + return str(tmp_path / "test_vectors.db") + + +@pytest.fixture +def store(db_file, unique_table_name) -> Generator: + """Create a SQLiteVecStore instance for testing.""" + from semantica.vector_store.sqlite_vec_store import SQLiteVecStore + + store = SQLiteVecStore( + db_path=db_file, + table_name=unique_table_name, + dimension=128, + distance_metric="cosine", + ) + + yield store + + # Teardown + store.close() + + +class TestSQLiteVecStoreInit: + """Test SQLiteVecStore initialization.""" + + def test_init_success(self, store, db_file): + """Test successful initialization.""" + assert store.dimension == 128 + assert store.distance_metric == "cosine" + assert store.table_name.startswith("test_vectors_") + assert os.path.exists(db_file) or store.db_path == ":memory:" + + def test_init_unsupported_metric(self, db_file): + """Test initialization with unsupported distance metric.""" + from semantica.vector_store.sqlite_vec_store import SQLiteVecStore + from semantica.utils.exceptions import ValidationError + + with pytest.raises(ValidationError, match="Unsupported distance metric"): + SQLiteVecStore( + db_path=db_file, + table_name="test", + dimension=128, + distance_metric="invalid_metric", + ) + + def test_init_table_creation(self, store): + """Test that table is created on initialization.""" + with store._get_connection() as conn: + cur = conn.cursor() + cur.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", + (store.table_name,), + ) + row = cur.fetchone() + cur.close() + assert row is not None + assert row[0] == store.table_name + + +class TestSQLiteVecStoreAdd: + """Test vector addition operations.""" + + def test_add_single_vector(self, store): + """Test adding a single vector.""" + vector = np.random.rand(128).astype(np.float32) + metadata = {"source": "test", "index": 0} + + ids = store.add([vector], [metadata], ids=["vec_0"]) + + assert ids == ["vec_0"] + + # Retrieve and verify + res = store.get(["vec_0"]) + assert len(res) == 1 + assert res[0]["id"] == "vec_0" + assert np.allclose(res[0]["vector"], vector) + assert res[0]["metadata"] == metadata + + def test_add_multiple_vectors(self, store): + """Test adding multiple vectors.""" + vectors = [np.random.rand(128).astype(np.float32) for _ in range(5)] + metadata = [{"index": i} for i in range(5)] + + ids = store.add(vectors, metadata) + + assert len(ids) == 5 + assert all(isinstance(id_str, str) for id_str in ids) + + def test_add_auto_generate_ids(self, store): + """Test that IDs are auto-generated if not provided.""" + vectors = [np.random.rand(128).astype(np.float32) for _ in range(3)] + + ids = store.add(vectors) + + assert len(ids) == 3 + assert len(set(ids)) == 3 # All unique + + def test_add_wrong_dimension(self, store): + """Test adding vector with wrong dimension.""" + from semantica.utils.exceptions import ValidationError + + vector = np.random.rand(64).astype(np.float32) # Wrong dimension + + with pytest.raises(ValidationError, match="dimension"): + store.add([vector]) + + def test_add_no_metadata(self, store): + """Test adding vectors without metadata.""" + vectors = [np.random.rand(128).astype(np.float32) for _ in range(2)] + + ids = store.add(vectors) + + assert len(ids) == 2 + res = store.get(ids) + assert all(r["metadata"] == {} for r in res) + + def test_add_batch_with_numpy_array(self, store): + """Test adding vectors as numpy array.""" + vectors = np.random.rand(5, 128).astype(np.float32) + + ids = store.add(vectors) + + assert len(ids) == 5 + + +class TestSQLiteVecStoreSearch: + """Test vector search operations.""" + + @pytest.fixture(autouse=True) + def setup_vectors(self, store): + """Setup test vectors for search tests.""" + vectors = [] + for i in range(10): + vec = np.zeros(128, dtype=np.float32) + vec[i] = 1.0 # Each vector has peak at different position + vectors.append(vec) + + metadata = [{"category": "A" if i < 5 else "B", "index": i} for i in range(10)] + store.add(vectors, metadata) + + def test_search_basic(self, store): + """Test basic similarity search.""" + query = np.zeros(128, dtype=np.float32) + query[0] = 1.0 # Should match first vector perfectly + + results = store.search(query, top_k=3) + + assert len(results) == 3 + assert all("id" in r for r in results) + assert all("score" in r for r in results) + assert all("metadata" in r for r in results) + assert results[0]["score"] == pytest.approx(1.0) + + def test_search_top_k(self, store): + """Test search with different top_k values.""" + query = np.random.rand(128).astype(np.float32) + + results_5 = store.search(query, top_k=5) + results_10 = store.search(query, top_k=10) + + assert len(results_5) == 5 + assert len(results_10) == 10 + + def test_search_with_filter(self, store): + """Test search with metadata filter.""" + query = np.zeros(128, dtype=np.float32) + query[0] = 1.0 + + results = store.search(query, top_k=10, filter={"category": "A"}) + + assert len(results) <= 5 # Only 5 vectors have category A + assert all(r["metadata"].get("category") == "A" for r in results) + + def test_search_wrong_dimension(self, store): + """Test search with wrong query dimension.""" + from semantica.utils.exceptions import ValidationError + + query = np.random.rand(64).astype(np.float32) + + with pytest.raises(ValidationError, match="dimension"): + store.search(query, top_k=5) + + def test_search_empty_store(self, db_file, unique_table_name): + """Test search on empty store.""" + from semantica.vector_store.sqlite_vec_store import SQLiteVecStore + + empty_store = SQLiteVecStore( + db_path=db_file, + table_name=unique_table_name + "_empty", + dimension=128, + distance_metric="cosine", + ) + + query = np.random.rand(128).astype(np.float32) + results = empty_store.search(query, top_k=5) + + assert len(results) == 0 + empty_store.close() + + +class TestSQLiteVecStoreGet: + """Test vector retrieval operations.""" + + def test_get_existing_vectors(self, store): + """Test getting existing vectors.""" + vectors = [np.random.rand(128).astype(np.float32) for _ in range(3)] + metadata = [{"index": i} for i in range(3)] + ids = store.add(vectors, metadata) + + results = store.get(ids) + + assert len(results) == 3 + result_ids = {r["id"] for r in results} + assert result_ids == set(ids) + assert all(r["vector"] is not None for r in results) + for r in results: + assert r["metadata"]["index"] in [0, 1, 2] + + def test_get_nonexistent_ids(self, store): + """Test getting non-existent vector IDs.""" + results = store.get(["nonexistent_1", "nonexistent_2"]) + + assert len(results) == 0 + + def test_get_empty_list(self, store): + """Test getting with empty ID list.""" + results = store.get([]) + + assert results == [] + + def test_get_partial_ids(self, store): + """Test getting mix of existing and non-existing IDs.""" + vectors = [np.random.rand(128).astype(np.float32)] + ids = store.add(vectors, [{"test": True}]) + + results = store.get(ids + ["nonexistent"]) + + assert len(results) == 1 + assert results[0]["id"] == ids[0] + + +class TestSQLiteVecStoreUpdate: + """Test vector update operations.""" + + def test_update_vectors(self, store): + """Test updating vectors.""" + vectors = [np.random.rand(128).astype(np.float32) for _ in range(2)] + metadata = [{"version": 1} for _ in range(2)] + ids = store.add(vectors, metadata) + + new_vectors = [np.random.rand(128).astype(np.float32) for _ in range(2)] + new_metadata = [{"version": 2} for _ in range(2)] + + success = store.update(ids, new_vectors, new_metadata) + + assert success is True + + results = store.get(ids) + assert len(results) == 2 + assert all(r["metadata"]["version"] == 2 for r in results) + for r, new_v in zip(results, new_vectors): + # Find matching original by id to compare + assert np.allclose(r["vector"], new_v) or np.allclose( + results[1 - results.index(r)]["vector"], new_v + ) + + def test_update_metadata_only(self, store): + """Test updating only metadata.""" + vectors = [np.random.rand(128).astype(np.float32)] + ids = store.add(vectors, [{"tag": "original"}]) + + success = store.update(ids, metadata=[{"tag": "updated"}]) + + assert success is True + + results = store.get(ids) + assert results[0]["metadata"]["tag"] == "updated" + + def test_update_vectors_only(self, store): + """Test updating only vectors.""" + vectors = [np.random.rand(128).astype(np.float32)] + ids = store.add(vectors, [{"tag": "keep"}]) + + new_vector = np.random.rand(128).astype(np.float32) + success = store.update(ids, vectors=[new_vector]) + + assert success is True + + results = store.get(ids) + assert np.allclose(results[0]["vector"], new_vector) + assert results[0]["metadata"]["tag"] == "keep" + + +class TestSQLiteVecStoreDelete: + """Test vector deletion operations.""" + + def test_delete_vectors(self, store): + """Test deleting vectors.""" + vectors = [np.random.rand(128).astype(np.float32) for _ in range(3)] + ids = store.add(vectors) + + # Delete two of them + success = store.delete(ids[:2]) + assert success is True + + # Check only the third one remains + res = store.get(ids) + assert len(res) == 1 + assert res[0]["id"] == ids[2] + + def test_delete_empty(self, store): + """Test deleting empty list of IDs.""" + assert store.delete([]) is True + + +class TestSQLiteVecStoreReadOnly: + """Test read-only mode behavior.""" + + def test_read_only_mode(self, db_file, unique_table_name): + """Test that read-only mode restricts writes but allows reads.""" + from semantica.vector_store.sqlite_vec_store import SQLiteVecStore + from semantica.utils.exceptions import ProcessingError + + # 1. Create and populate database first + store_write = SQLiteVecStore( + db_path=db_file, + table_name=unique_table_name, + dimension=4, + ) + vec = np.array([1, 2, 3, 4], dtype=np.float32) + store_write.add([vec], ids=["v1"]) + store_write.close() + + # 2. Open in read-only mode + store_ro = SQLiteVecStore( + db_path=db_file, + table_name=unique_table_name, + dimension=4, + read_only=True, + ) + + # Read should succeed + results = store_ro.get(["v1"]) + assert len(results) == 1 + assert results[0]["id"] == "v1" + + # Search should succeed + search_res = store_ro.search(np.array([1, 2, 3, 4], dtype=np.float32), top_k=1) + assert len(search_res) == 1 + + # Write should fail + with pytest.raises(ProcessingError, match="read-only"): + store_ro.add([vec], ids=["v2"]) + + # Update should fail + with pytest.raises(ProcessingError, match="read-only"): + store_ro.update(["v1"], vectors=[vec]) + + # Delete should fail + with pytest.raises(ProcessingError, match="read-only"): + store_ro.delete(["v1"]) + + store_ro.close() + + +class TestSQLiteVecStoreStats: + """Test store statistics retrieval.""" + + def test_get_stats(self, store): + """Test getting stats from store.""" + stats = store.get_stats() + assert stats["vector_count"] == 0 + assert stats["dimension"] == 128 + assert stats["distance_metric"] == "cosine" + + # Add vectors and check again + vectors = [np.random.rand(128).astype(np.float32) for _ in range(4)] + store.add(vectors) + + stats = store.get_stats() + assert stats["vector_count"] == 4