mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
KaifAhmad1andClaude Sonnet 4.6
246bcc96cd
Bug 1 — _parse_relation_result (methods.py): Relations whose subject/object weren't in the pre-extracted NER list were silently dropped because match_entity() returned None and the old code gated on `if subject_entity and object_entity`. Now unmatched names produce a synthetic UNKNOWN Entity so every LLM-returned relation is preserved (all three Apple co-founders are now returned). Bug 2 — _match_pattern (reasoner.py): Rewrote the regex builder to split on ?var placeholders first, then apply re.escape() only to the surrounding literal segments. The old approach (escape-then-sub) left edge cases where pre-bound variables and multi-word values with spaces could fail to unify. The new implementation also handles repeated variables via backreferences and uses non-greedy .+? to avoid over-consuming literal separators. Closes #354 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
87 lines
3.5 KiB
Python
87 lines
3.5 KiB
Python
"""Tests for RelationExtractor LLM multi-founder bug (#354)."""
|
|
|
|
import pytest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from semantica.semantic_extract.methods import _parse_relation_result
|
|
from semantica.semantic_extract.ner_extractor import Entity
|
|
|
|
|
|
APPLE_ENTITY = Entity(text="Apple Inc.", label="ORG", start_char=0, end_char=10)
|
|
JOBS_ENTITY = Entity(text="Steve Jobs", label="PERSON", start_char=0, end_char=10)
|
|
|
|
ALL_ENTITIES = [APPLE_ENTITY, JOBS_ENTITY]
|
|
|
|
TEXT = (
|
|
"Apple Inc. was founded by Steve Jobs, Steve Wozniak, "
|
|
"and Ronald Wayne on April 1, 1976."
|
|
)
|
|
|
|
LLM_RESPONSE = {
|
|
"relations": [
|
|
{"subject": "Steve Jobs", "predicate": "founded_by", "object": "Apple Inc.", "confidence": 0.95},
|
|
{"subject": "Steve Wozniak", "predicate": "founded_by", "object": "Apple Inc.", "confidence": 0.93},
|
|
{"subject": "Ronald Wayne", "predicate": "founded_by", "object": "Apple Inc.", "confidence": 0.91},
|
|
]
|
|
}
|
|
|
|
|
|
def test_all_three_founders_returned():
|
|
"""Bug #354 — all co-founders from LLM response must appear as Relation objects."""
|
|
relations = _parse_relation_result(LLM_RESPONSE, ALL_ENTITIES, TEXT, "openai", "gpt-4")
|
|
|
|
subjects = [r.subject.text for r in relations]
|
|
assert len(relations) == 3, f"Expected 3 relations, got {len(relations)}: {subjects}"
|
|
assert "Steve Jobs" in subjects
|
|
assert "Steve Wozniak" in subjects
|
|
assert "Ronald Wayne" in subjects
|
|
|
|
|
|
def test_unmatched_entity_becomes_synthetic():
|
|
"""Entities not found by NER must appear as synthetic UNKNOWN entities, not dropped."""
|
|
# Only Apple is in the pre-extracted list; all three founders are missing
|
|
relations = _parse_relation_result(LLM_RESPONSE, [APPLE_ENTITY], TEXT, "openai", "gpt-4")
|
|
|
|
assert len(relations) == 3
|
|
for r in relations:
|
|
if r.subject.text in ("Steve Jobs", "Steve Wozniak", "Ronald Wayne"):
|
|
assert r.subject.label == "UNKNOWN"
|
|
assert r.subject.metadata.get("synthetic") is True
|
|
|
|
|
|
def test_matched_entity_not_synthetic():
|
|
"""Entities that DO match pre-extracted NER results must not be marked synthetic."""
|
|
relations = _parse_relation_result(LLM_RESPONSE, ALL_ENTITIES, TEXT, "openai", "gpt-4")
|
|
|
|
jobs_relation = next(r for r in relations if r.subject.text == "Steve Jobs")
|
|
assert jobs_relation.subject.label == "PERSON"
|
|
assert not jobs_relation.subject.metadata.get("synthetic")
|
|
|
|
|
|
def test_predicate_and_confidence_preserved():
|
|
"""predicate and confidence from LLM response must be preserved."""
|
|
relations = _parse_relation_result(LLM_RESPONSE, ALL_ENTITIES, TEXT, "openai", "gpt-4")
|
|
|
|
for r in relations:
|
|
assert r.predicate == "founded_by"
|
|
assert r.confidence >= 0.9
|
|
|
|
|
|
def test_empty_llm_response_returns_empty():
|
|
"""Empty relations list from LLM must return empty list without error."""
|
|
relations = _parse_relation_result({"relations": []}, ALL_ENTITIES, TEXT, "openai", "gpt-4")
|
|
assert relations == []
|
|
|
|
|
|
def test_missing_subject_or_object_text_skipped():
|
|
"""Relations with empty subject or object text must be silently skipped."""
|
|
bad_response = {
|
|
"relations": [
|
|
{"subject": "", "predicate": "founded_by", "object": "Apple Inc."},
|
|
{"subject": "Steve Jobs", "predicate": "founded_by", "object": ""},
|
|
{"subject": "Steve Jobs", "predicate": "founded_by", "object": "Apple Inc."},
|
|
]
|
|
}
|
|
relations = _parse_relation_result(bad_response, ALL_ENTITIES, TEXT, "openai", "gpt-4")
|
|
assert len(relations) == 1
|