From 58125a0a93da6a31bae439417ab959d25d3ae1b2 Mon Sep 17 00:00:00 2001 From: Kevin Date: Sat, 22 Aug 2026 17:13:45 +0800 Subject: [PATCH] fix(dedup): never merge entities with different explicit types (closes #1137) (#1149) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dedup): never merge entities with different explicit types (fixes #1137) The duplicate candidate confidence scoring only rewarded same-type pairs but never penalized different-type pairs, so a Person 'Alice' and an Organization 'Acme' (different id, type, and name) passed the confidence threshold and were merged, silently dropping one entity. Add a type guard: when both entities carry a non-empty type and they differ, the pair is never a duplicate candidate (confidence 0, reason 'type_mismatch'). Untyped entities and genuinely duplicate same-type pairs keep their previous behavior. Regression tests cover all three cases. * fix(dedup): honor Entity.type and exclude mismatch structurally (review fixes) Two gaps from code review (#1149): 1. _get_entity_value mapped object 'type' exclusively to .label, which Entity objects never have — their type lives on .type. The mismatch guard therefore never saw the type of Entity objects, and differently typed objects could still merge. Read .type first, fall back to .label. 2. The mismatch branch returned a normal candidate with confidence 0.0, but detection filters with >= confidence_threshold, and 0.0 is a documented valid threshold, so mismatches slipped through. Exclude type_mismatch candidates structurally at both filter sites regardless of threshold. Adds tests for Entity objects with different types and for confidence_threshold=0.0. 94 dedup tests pass. --------- --- semantica/deduplication/duplicate_detector.py | 44 ++++++++-- tests/deduplication/test_deduplication.py | 80 +++++++++++++++++++ 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/semantica/deduplication/duplicate_detector.py b/semantica/deduplication/duplicate_detector.py index c7a29d49..e5b91c11 100644 --- a/semantica/deduplication/duplicate_detector.py +++ b/semantica/deduplication/duplicate_detector.py @@ -278,8 +278,12 @@ class DuplicateDetector: for i, (entity1, entity2, score) in enumerate(similarities): candidate = self._create_duplicate_candidate(entity1, entity2, score) - # Filter by confidence threshold - if candidate.confidence >= self.confidence_threshold: + # Filter by confidence threshold; type mismatches are excluded + # structurally so no threshold value can admit them. + if ( + candidate.confidence >= self.confidence_threshold + and "type_mismatch" not in candidate.reasons + ): candidates.append(candidate) remaining = total_similarities - (i + 1) @@ -624,8 +628,12 @@ class DuplicateDetector: new_entity, existing_entity, similarity.score ) - # Filter by confidence threshold - if candidate.confidence >= self.confidence_threshold: + # Filter by confidence threshold; type mismatches are + # excluded structurally regardless of the threshold. + if ( + candidate.confidence >= self.confidence_threshold + and "type_mismatch" not in candidate.reasons + ): candidates.append(candidate) processed += 1 @@ -723,7 +731,9 @@ class DuplicateDetector: if key == "name": return getattr(entity, "text", default) if key == "type": - return getattr(entity, "label", default) + # Entity objects store the type on .type; extraction entities + # may expose .label. Missing .label never means "no type". + return getattr(entity, "type", default) or getattr(entity, "label", default) if key == "properties": # Check metadata for properties metadata = getattr(entity, "metadata", {}) @@ -757,6 +767,25 @@ class DuplicateDetector: reasons = [] confidence = similarity_score + # Check entity type mismatch first: two entities with different + # explicit types are not duplicates, whatever their similarity. + entity_type1 = self._get_entity_value(entity1, "type") + entity_type2 = self._get_entity_value(entity2, "type") + if entity_type1 and entity_type2 and entity_type1 != entity_type2: + return DuplicateCandidate( + entity1=entity1, + entity2=entity2, + similarity_score=similarity_score, + confidence=0.0, + reasons=["type_mismatch"], + metadata={ + "name_match": False, + "common_properties": 0, + "type_match": False, + "type_mismatch": True, + }, + ) + # Check for exact name match (strong indicator) name1 = str(self._get_entity_value(entity1, "name", "")).lower().strip() name2 = str(self._get_entity_value(entity2, "name", "")).lower().strip() @@ -779,9 +808,8 @@ class DuplicateDetector: # Boost confidence for each matching property confidence += 0.05 * prop_matches - # Check entity type match - entity_type1 = self._get_entity_value(entity1, "type") - entity_type2 = self._get_entity_value(entity2, "type") + # Check entity type match (only boosts when types are equal; mismatch + # is handled above) if entity_type1 and entity_type2 and entity_type1 == entity_type2: reasons.append("same_type") confidence += 0.05 diff --git a/tests/deduplication/test_deduplication.py b/tests/deduplication/test_deduplication.py index dd351642..8dc7b7c2 100644 --- a/tests/deduplication/test_deduplication.py +++ b/tests/deduplication/test_deduplication.py @@ -12,6 +12,7 @@ from semantica.deduplication.cluster_builder import ClusterBuilder from semantica.deduplication.registry import MethodRegistry from semantica.deduplication.config import DeduplicationConfig from semantica.deduplication.methods import get_deduplication_method +from semantica.utils.types import Entity from semantica.utils.progress_tracker import ConsoleProgressDisplay class TestDeduplication(unittest.TestCase): @@ -87,6 +88,85 @@ class TestDeduplication(unittest.TestCase): # One group should have at least 2 entities (the Apple ones) apple_group = next((g for g in groups if len(g.entities) >= 2), None) self.assertIsNotNone(apple_group) + + def test_different_types_are_never_duplicates(self): + """Entities with different non-empty types must not merge (issue #1137).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Organization", "name": "Acme", "text": "Acme"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Person 'Alice' and Organization 'Acme' must not be duplicate candidates", + ) + # GraphBuilder with merge_entities=True must keep both entities + from semantica.kg import GraphBuilder + graph = GraphBuilder(merge_entities=True).build( + {"entities": entities, "relationships": []} + ) + self.assertEqual(len(graph["entities"]), 2) + + def test_same_type_same_name_still_merges(self): + """Type guard must not break legitimate dedup of same-type entities.""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Person", "name": "Alice", "text": "Alice"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertTrue( + duplicates, "Same-type same-name entities must still be detected as duplicates" + ) + + def test_untyped_same_name_still_merges(self): + """Entities with no type must retain previous behavior (merge on similarity).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "x1", "name": "Apple"}, + {"id": "x2", "name": "Apple"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertTrue( + duplicates, "Untyped same-name entities must still be detected as duplicates" + ) + + def test_entity_objects_different_types_not_duplicates(self): + """Entity objects expose their type via .type, not .label (issue #1137).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.0 + ) + entities = [ + Entity(id="e1", text="Alice", type="Person"), + Entity(id="e2", text="Acme", type="Organization"), + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Entity objects with different types must never be detected as duplicates", + ) + + def test_zero_threshold_still_excludes_type_mismatch(self): + """Type mismatch must be excluded structurally, not just by confidence 0.""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.0 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Organization", "name": "Acme", "text": "Acme"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Different-type candidates must be excluded even with confidence_threshold=0.0", + ) def test_entity_merger(self): """Test entity merging."""