From 03ed4b94e978a413aec69fc5430a4a6610de0a6c Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Sun, 9 Aug 2026 17:28:05 +0530 Subject: [PATCH] fix(vector-store): preserve ranking for unbounded scores in Pinecone/Qdrant The 1.0 / (1.0 + max(0.0, 1.0 - score)) normalization added in the last commit clamped every raw score >= 1.0 to an identical 1.0, collapsing result ranking for dot-product-metric indexes (unbounded), which cosine (bounded to [-1, 1]) never exercised. Replaced with x/(1+|x|) rescaled to (0, 1), which is strictly monotonic for any real score. Also adds regression tests for scores >= 1 and a CHANGELOG entry. --- CHANGELOG.md | 7 +++ semantica/vector_store/pinecone_store.py | 11 ++++- semantica/vector_store/qdrant_store.py | 9 +++- .../vector_store/test_search_result_schema.py | 48 +++++++++++++++++++ 4 files changed, 73 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 244b45fc..a77ec2f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`VectorStore.search_vectors()` returned inconsistent result shapes across backend implementations** (#853, closes #845) by @Sameer6305, reviewed by @KaifAhmad1 + - Every built-in backend (FAISS, Milvus, pgvector, Pinecone, Qdrant, SQLite-vec, Weaviate, in-memory) now returns the same canonical `SearchResult` shape (`id`, `score`, `metadata`, `vector`, `distance`), instead of some backends omitting `vector`/`metadata`/`distance` or, for Weaviate, returning a backend-specific `properties` key instead of `metadata` + - Added a `SearchResult` `TypedDict` (`semantica/vector_store/vector_store.py`, exported from `semantica.vector_store`) documenting the contract; `metadata` now always defaults to `{}` rather than being absent, and `id` accepts `Union[str, int]` to accommodate Milvus/Qdrant's native integer IDs without casting + - **Review fix**: the score-normalization formula added for Pinecone and Qdrant (`1.0 / (1.0 + max(0.0, 1.0 - score))`) clamped every raw score `>= 1.0` to an identical `1.0`, silently collapsing result ranking whenever the raw score could exceed 1 — which happens routinely for dot-product-metric indexes (unbounded), as opposed to cosine (bounded to `[-1, 1]`). Replaced with `(score / (1 + |score|) + 1) / 2`, which is strictly monotonic and bounded in `(0, 1)` for any real input, so ranking order is preserved regardless of metric or vector normalization + - Added `test_qdrant_unbounded_dot_product_scores_preserve_ranking` and `test_pinecone_unbounded_dotproduct_scores_preserve_ranking` (`tests/vector_store/test_search_result_schema.py`) asserting normalized scores stay strictly ordered and bounded for raw scores well above 1.0, the case the original formula silently collapsed and the existing tests (which only used scores `< 1`) never exercised + - Left out of scope, per the original PR: Weaviate's `similarity_search()` still isn't wired into `VectorStore.search_vectors()`'s backend dispatch; Milvus's collection schema still has no metadata column so its results always return `metadata: {}`; and `include_vectors` support (populating the `vector` field) is not yet implemented for any backend + - **`DecisionEmbeddingPipeline.find_similar_decisions()` crashed with `AttributeError` for any `VectorStore` backend other than `inmemory`** (#842, closes #839) by @Sameer6305 - `_get_candidate_embeddings()` iterated `VectorStore.vectors`/`VectorStore.metadata` directly, internal dicts only populated for `backend="inmemory"`; every persistent backend (FAISS, Pinecone, Qdrant, Milvus, ...) raised `AttributeError`. It now fetches candidates via the backend-agnostic `VectorStore.search_vectors()`, reading metadata via a `res.get("metadata") or res.get("payload")` fallback for backends that key it differently - Backends such as FAISS don't return the raw vector for each hit; `find_similar_decisions()` and `_find_semantic_similar()` now fall back to the search-provided score (normalized from `distance` when present) as the semantic similarity for those candidates instead of computing cosine similarity against a zero placeholder vector diff --git a/semantica/vector_store/pinecone_store.py b/semantica/vector_store/pinecone_store.py index 329ff316..69b60fe9 100644 --- a/semantica/vector_store/pinecone_store.py +++ b/semantica/vector_store/pinecone_store.py @@ -194,7 +194,16 @@ class PineconeIndex: results.append( { "id": match.id, - "score": 1.0 / (1.0 + max(0.0, 1.0 - float(match.score))), + # Pinecone's native score is already "higher is better" but its + # range depends on the configured metric (bounded for cosine, + # unbounded for dotproduct). Squash with x/(1+|x|) instead of + # clamping distance-to-zero, since the latter collapses every + # score >= 1.0 to an identical 1.0 and destroys ranking order + # for dotproduct/unnormalized-vector indexes. + "score": ( + float(match.score) / (1.0 + abs(float(match.score))) + 1.0 + ) + / 2.0, "metadata": match.metadata or {}, "vector": None, "distance": None, diff --git a/semantica/vector_store/qdrant_store.py b/semantica/vector_store/qdrant_store.py index aaf6b15e..49c71bfc 100644 --- a/semantica/vector_store/qdrant_store.py +++ b/semantica/vector_store/qdrant_store.py @@ -164,7 +164,14 @@ class QdrantCollection: results.append( { "id": result.id, - "score": 1.0 / (1.0 + max(0.0, 1.0 - float(result.score))), + # See pinecone_store.py PineconeIndex.search_vectors for why + # this uses x/(1+|x|) rather than clamping distance-to-zero: + # Qdrant's Dot distance metric is unbounded, and the old + # clamped formula collapsed every score >= 1.0 to 1.0. + "score": ( + float(result.score) / (1.0 + abs(float(result.score))) + 1.0 + ) + / 2.0, "metadata": result.payload or {}, "vector": None, "distance": None, diff --git a/tests/vector_store/test_search_result_schema.py b/tests/vector_store/test_search_result_schema.py index 392f2cbf..cdae2337 100644 --- a/tests/vector_store/test_search_result_schema.py +++ b/tests/vector_store/test_search_result_schema.py @@ -121,6 +121,29 @@ class TestQdrantSearchSchema(unittest.TestCase): self.assertIsNone(results[0]["vector"]) self.assertEqual(results[0]["metadata"], {"category": "x"}) + @patch("semantica.vector_store.qdrant_store.QDRANT_AVAILABLE", True) + def test_qdrant_unbounded_dot_product_scores_preserve_ranking(self): + """Qdrant's Dot distance metric is unbounded; normalized scores must + stay strictly ordered instead of collapsing once raw score >= 1.0 + (regression for #845 follow-up).""" + from semantica.vector_store.qdrant_store import QdrantCollection + + mock_client = MagicMock() + mock_hit_high = MagicMock(id="q_hi", score=50.0, payload={}) + mock_hit_mid = MagicMock(id="q_mid", score=2.0, payload={}) + mock_hit_low = MagicMock(id="q_lo", score=1.0, payload={}) + mock_client.search.return_value = [mock_hit_high, mock_hit_mid, mock_hit_low] + + coll = QdrantCollection(mock_client, "test_col") + results = coll.search_points(np.array([0.1, 0.2]), limit=3) + + _assert_canonical_schema(self, results) + scores = [r["score"] for r in results] + self.assertEqual(len(set(scores)), 3, "scores >= 1.0 must not collapse") + self.assertGreater(scores[0], scores[1]) + self.assertGreater(scores[1], scores[2]) + self.assertTrue(all(0.0 < s < 1.0 for s in scores)) + # --------------------------------------------------------------------------- # Pinecone @@ -163,6 +186,31 @@ class TestPineconeSearchSchema(unittest.TestCase): _assert_canonical_schema(self, results) self.assertEqual(results[0]["metadata"], {}) + @patch("semantica.vector_store.pinecone_store.PINECONE_AVAILABLE", True) + def test_pinecone_unbounded_dotproduct_scores_preserve_ranking(self): + """Pinecone's dotproduct metric is unbounded; normalized scores must + stay strictly ordered instead of collapsing once raw score >= 1.0 + (regression for #845 follow-up).""" + from semantica.vector_store.pinecone_store import PineconeIndex + + mock_index = MagicMock() + mock_match_high = MagicMock(id="p_hi", score=50.0, metadata={}) + mock_match_mid = MagicMock(id="p_mid", score=2.0, metadata={}) + mock_match_low = MagicMock(id="p_lo", score=1.0, metadata={}) + mock_index.query.return_value = MagicMock( + matches=[mock_match_high, mock_match_mid, mock_match_low] + ) + + pi = PineconeIndex(mock_index) + results = pi.search_vectors([0.1, 0.2], k=3) + + _assert_canonical_schema(self, results) + scores = [r["score"] for r in results] + self.assertEqual(len(set(scores)), 3, "scores >= 1.0 must not collapse") + self.assertGreater(scores[0], scores[1]) + self.assertGreater(scores[1], scores[2]) + self.assertTrue(all(0.0 < s < 1.0 for s in scores)) + # --------------------------------------------------------------------------- # Milvus