diff --git a/semantica/deduplication/duplicate_detector.py b/semantica/deduplication/duplicate_detector.py index c7a29d49..e5b91c11 100644 --- a/semantica/deduplication/duplicate_detector.py +++ b/semantica/deduplication/duplicate_detector.py @@ -278,8 +278,12 @@ class DuplicateDetector: for i, (entity1, entity2, score) in enumerate(similarities): candidate = self._create_duplicate_candidate(entity1, entity2, score) - # Filter by confidence threshold - if candidate.confidence >= self.confidence_threshold: + # Filter by confidence threshold; type mismatches are excluded + # structurally so no threshold value can admit them. + if ( + candidate.confidence >= self.confidence_threshold + and "type_mismatch" not in candidate.reasons + ): candidates.append(candidate) remaining = total_similarities - (i + 1) @@ -624,8 +628,12 @@ class DuplicateDetector: new_entity, existing_entity, similarity.score ) - # Filter by confidence threshold - if candidate.confidence >= self.confidence_threshold: + # Filter by confidence threshold; type mismatches are + # excluded structurally regardless of the threshold. + if ( + candidate.confidence >= self.confidence_threshold + and "type_mismatch" not in candidate.reasons + ): candidates.append(candidate) processed += 1 @@ -723,7 +731,9 @@ class DuplicateDetector: if key == "name": return getattr(entity, "text", default) if key == "type": - return getattr(entity, "label", default) + # Entity objects store the type on .type; extraction entities + # may expose .label. Missing .label never means "no type". + return getattr(entity, "type", default) or getattr(entity, "label", default) if key == "properties": # Check metadata for properties metadata = getattr(entity, "metadata", {}) @@ -757,6 +767,25 @@ class DuplicateDetector: reasons = [] confidence = similarity_score + # Check entity type mismatch first: two entities with different + # explicit types are not duplicates, whatever their similarity. + entity_type1 = self._get_entity_value(entity1, "type") + entity_type2 = self._get_entity_value(entity2, "type") + if entity_type1 and entity_type2 and entity_type1 != entity_type2: + return DuplicateCandidate( + entity1=entity1, + entity2=entity2, + similarity_score=similarity_score, + confidence=0.0, + reasons=["type_mismatch"], + metadata={ + "name_match": False, + "common_properties": 0, + "type_match": False, + "type_mismatch": True, + }, + ) + # Check for exact name match (strong indicator) name1 = str(self._get_entity_value(entity1, "name", "")).lower().strip() name2 = str(self._get_entity_value(entity2, "name", "")).lower().strip() @@ -779,9 +808,8 @@ class DuplicateDetector: # Boost confidence for each matching property confidence += 0.05 * prop_matches - # Check entity type match - entity_type1 = self._get_entity_value(entity1, "type") - entity_type2 = self._get_entity_value(entity2, "type") + # Check entity type match (only boosts when types are equal; mismatch + # is handled above) if entity_type1 and entity_type2 and entity_type1 == entity_type2: reasons.append("same_type") confidence += 0.05 diff --git a/tests/deduplication/test_deduplication.py b/tests/deduplication/test_deduplication.py index dd351642..8dc7b7c2 100644 --- a/tests/deduplication/test_deduplication.py +++ b/tests/deduplication/test_deduplication.py @@ -12,6 +12,7 @@ from semantica.deduplication.cluster_builder import ClusterBuilder from semantica.deduplication.registry import MethodRegistry from semantica.deduplication.config import DeduplicationConfig from semantica.deduplication.methods import get_deduplication_method +from semantica.utils.types import Entity from semantica.utils.progress_tracker import ConsoleProgressDisplay class TestDeduplication(unittest.TestCase): @@ -87,6 +88,85 @@ class TestDeduplication(unittest.TestCase): # One group should have at least 2 entities (the Apple ones) apple_group = next((g for g in groups if len(g.entities) >= 2), None) self.assertIsNotNone(apple_group) + + def test_different_types_are_never_duplicates(self): + """Entities with different non-empty types must not merge (issue #1137).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Organization", "name": "Acme", "text": "Acme"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Person 'Alice' and Organization 'Acme' must not be duplicate candidates", + ) + # GraphBuilder with merge_entities=True must keep both entities + from semantica.kg import GraphBuilder + graph = GraphBuilder(merge_entities=True).build( + {"entities": entities, "relationships": []} + ) + self.assertEqual(len(graph["entities"]), 2) + + def test_same_type_same_name_still_merges(self): + """Type guard must not break legitimate dedup of same-type entities.""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Person", "name": "Alice", "text": "Alice"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertTrue( + duplicates, "Same-type same-name entities must still be detected as duplicates" + ) + + def test_untyped_same_name_still_merges(self): + """Entities with no type must retain previous behavior (merge on similarity).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.4 + ) + entities = [ + {"id": "x1", "name": "Apple"}, + {"id": "x2", "name": "Apple"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertTrue( + duplicates, "Untyped same-name entities must still be detected as duplicates" + ) + + def test_entity_objects_different_types_not_duplicates(self): + """Entity objects expose their type via .type, not .label (issue #1137).""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.0 + ) + entities = [ + Entity(id="e1", text="Alice", type="Person"), + Entity(id="e2", text="Acme", type="Organization"), + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Entity objects with different types must never be detected as duplicates", + ) + + def test_zero_threshold_still_excludes_type_mismatch(self): + """Type mismatch must be excluded structurally, not just by confidence 0.""" + detector = DuplicateDetector( + similarity_threshold=0.4, confidence_threshold=0.0 + ) + entities = [ + {"id": "e1", "type": "Person", "name": "Alice", "text": "Alice"}, + {"id": "e2", "type": "Organization", "name": "Acme", "text": "Acme"}, + ] + duplicates = detector.detect_duplicates(entities) + self.assertEqual( + duplicates, [], + "Different-type candidates must be excluded even with confidence_threshold=0.0", + ) def test_entity_merger(self): """Test entity merging."""