mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
fix(cli): wire deduplicate CLI through graph store and EntityMerger
This commit is contained in:
+49
-4
@@ -1038,19 +1038,64 @@ def deduplicate(
|
||||
"""
|
||||
cli_ctx = _require_ctx(cli_ctx)
|
||||
|
||||
strategy_map = {
|
||||
"blocking": "blocking_v2",
|
||||
"semantic": "legacy",
|
||||
"hybrid": "hybrid_v2",
|
||||
}
|
||||
|
||||
def _load_entities() -> List[Dict[str, Any]]:
|
||||
from .graph_store import get_nodes
|
||||
from .graph_store.config import graph_store_config
|
||||
|
||||
graph_db = dict(cli_ctx.config.to_dict().get("graph_db", {}))
|
||||
backend = cli_ctx.store_backend or graph_db.pop("backend", None)
|
||||
previous_graph_config = graph_store_config.get_all()
|
||||
graph_store_config.update(graph_db)
|
||||
if backend:
|
||||
graph_store_config.set("default_backend", backend)
|
||||
try:
|
||||
return get_nodes(limit=sys.maxsize)
|
||||
finally:
|
||||
graph_store_config.update(previous_graph_config)
|
||||
|
||||
def _action() -> None:
|
||||
if _is_dry(cli_ctx, local_dry):
|
||||
_dry(cli_ctx, "deduplicate", json_out=_is_json(cli_ctx, local_json),
|
||||
strategy=strategy, dedup_action=dedup_action)
|
||||
return
|
||||
try:
|
||||
from .deduplication import detect_duplicates, merge_entities
|
||||
from .deduplication import detect_duplicates
|
||||
from .deduplication.entity_merger import EntityMerger
|
||||
|
||||
entities = _load_entities()
|
||||
candidate_strategy = strategy_map.get(strategy, strategy)
|
||||
detector_sort_by = "similarity_score" if sort_by == "similarity" else "confidence"
|
||||
detection_kwargs: Dict[str, Any] = {
|
||||
"candidate_strategy": candidate_strategy,
|
||||
"sort_by": detector_sort_by,
|
||||
}
|
||||
if dedup_action == "detect":
|
||||
result = detect_duplicates(strategy=strategy, min_similarity=min_similarity)
|
||||
result = detect_duplicates(
|
||||
entities,
|
||||
method="group",
|
||||
similarity_threshold=min_similarity,
|
||||
**detection_kwargs,
|
||||
)
|
||||
elif dedup_action == "merge":
|
||||
result = merge_entities(strategy=strategy, min_similarity=min_similarity)
|
||||
merger = EntityMerger()
|
||||
result = merger.merge_duplicates(
|
||||
entities,
|
||||
threshold=min_similarity,
|
||||
**detection_kwargs,
|
||||
)
|
||||
else:
|
||||
result = detect_duplicates(strategy=strategy, min_similarity=min_similarity)
|
||||
result = detect_duplicates(
|
||||
entities,
|
||||
method="group",
|
||||
similarity_threshold=min_similarity,
|
||||
**detection_kwargs,
|
||||
)
|
||||
except ImportError as exc:
|
||||
raise click.ClickException(f"Deduplication module not available: {exc}") from exc
|
||||
if output:
|
||||
|
||||
@@ -544,6 +544,66 @@ class TestDeduplicate:
|
||||
data = _json_output(result)
|
||||
assert data["dry_run"] is True
|
||||
|
||||
def test_detect_runtime_path(self, runner, monkeypatch):
|
||||
entities = [
|
||||
{"id": "e1", "name": "Alice", "type": "Person"},
|
||||
{"id": "e2", "name": "Alice", "type": "Person"},
|
||||
{"id": "e3", "name": "Bob", "type": "Person"},
|
||||
]
|
||||
|
||||
class FakeStore:
|
||||
def get_nodes(self, labels=None, properties=None, limit=100, **options):
|
||||
return entities
|
||||
|
||||
monkeypatch.setattr("semantica.graph_store.methods._get_store", lambda: FakeStore())
|
||||
monkeypatch.setattr("semantica.graph_store.methods.get_nodes", lambda **kwargs: entities)
|
||||
|
||||
result = runner.invoke(
|
||||
main,
|
||||
["deduplicate", "--action", "detect", "--min-similarity", "0.1", "--json"],
|
||||
)
|
||||
|
||||
_ok(result)
|
||||
assert "Alice" in result.output
|
||||
assert "Bob" not in result.output or "entities" in result.output
|
||||
|
||||
def test_merge_runtime_path(self, runner, monkeypatch):
|
||||
entities = [
|
||||
{"id": "e1", "name": "Alice", "type": "Person"},
|
||||
{"id": "e2", "name": "Alice", "type": "Person"},
|
||||
]
|
||||
captured = {}
|
||||
|
||||
class FakeStore:
|
||||
def get_nodes(self, labels=None, properties=None, limit=100, **options):
|
||||
return entities
|
||||
|
||||
monkeypatch.setattr("semantica.graph_store.methods._get_store", lambda: FakeStore())
|
||||
monkeypatch.setattr("semantica.graph_store.methods.get_nodes", lambda **kwargs: entities)
|
||||
|
||||
def fake_merge(self, loaded_entities, **kwargs):
|
||||
captured["entities"] = loaded_entities
|
||||
captured["kwargs"] = kwargs
|
||||
return [{"merged": True, "count": len(loaded_entities)}]
|
||||
|
||||
monkeypatch.setattr(
|
||||
"semantica.deduplication.entity_merger.EntityMerger.merge_duplicates",
|
||||
fake_merge,
|
||||
)
|
||||
|
||||
result = runner.invoke(
|
||||
main,
|
||||
["deduplicate", "--action", "merge", "--json"],
|
||||
)
|
||||
|
||||
_ok(result)
|
||||
data = _json_output(result)
|
||||
assert data == [{"merged": True, "count": 2}]
|
||||
assert captured["entities"] == entities
|
||||
assert captured["kwargs"]["threshold"] == pytest.approx(0.7)
|
||||
assert captured["kwargs"]["candidate_strategy"] == "hybrid_v2"
|
||||
assert captured["kwargs"]["sort_by"] == "similarity_score"
|
||||
|
||||
def test_global_dry_run_triggers_dry(self, runner):
|
||||
result = runner.invoke(main, ["--dry-run", "--json", "deduplicate"])
|
||||
_ok(result)
|
||||
|
||||
Reference in New Issue
Block a user