mengzhx/semantica
1
0
Fork 0
mirror of https://github.com/semantica-agi/semantica.git synced 2026-08-29 04:26:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use deduplicated entities/relationships; optimize and clean deduplication; disable extra merging in GraphBuilder

Browse Source
This commit is contained in:
KaifAhmad1
2026-01-16 18:18:50 +05:30
parent 1b0b0551db
commit bbd6764215
1 changed files with 58 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb
+58 -66
View File
@@ -617,73 +617,71 @@
"outputs": [],
"source": [
"from semantica.deduplication import DuplicateDetector, EntityMerger\n",
"import time\n",
"\n",
"start_time = time.time()\n",
"\n",
"# Initialize deduplication components\n",
"duplicate_detector = DuplicateDetector(\n",
" similarity_threshold=0.8,\n",
" confidence_threshold=0.7,\n",
" similarity_threshold=0.85,\n",
" confidence_threshold=0.75,\n",
" batch_size=50,\n",
" use_cache=True,\n",
")\n",
"\n",
"entity_merger = EntityMerger(preserve_provenance=True)\n",
"entity_merger = EntityMerger(\n",
" preserve_provenance=True,\n",
" parallel_processing=True,\n",
")\n",
"\n",
"# Prepare entities (using original entities since no entity conflicts were found)\n",
"entity_dicts = [\n",
" {\n",
" \"id\": getattr(e, \"id\", None) or e.text,\n",
" \"name\": e.text,\n",
"entity_dicts = []\n",
"for i, e in enumerate(entities):\n",
" entity_dicts.append({\n",
" \"id\": getattr(e, \"id\", None) or f\"entity_{i}_{e.text}\",\n",
" \"name\": e.text.strip().lower(),\n",
" \"original_name\": e.text,\n",
" \"type\": getattr(e, \"label\", \"UNKNOWN\"),\n",
" \"confidence\": getattr(e, \"confidence\", 1.0),\n",
" \"metadata\": getattr(e, \"metadata\", {}),\n",
" }\n",
" for e in entities\n",
"]\n",
" })\n",
"\n",
"# Detect and merge duplicate entities\n",
"duplicates = duplicate_detector.detect_duplicates(entity_dicts)\n",
"\n",
"merge_operations = entity_merger.merge_duplicates(\n",
" entities=entity_dicts,\n",
" duplicates=duplicates,\n",
" strategy=\"keep_most_complete\",\n",
" max_workers=4,\n",
")\n",
"\n",
"deduplicated_entities = [op.merged_entity for op in merge_operations]\n",
"\n",
"# Update relationships to use merged entity IDs\n",
"entity_id_mapping = {\n",
" source_id: op.merged_entity['id']\n",
" for op in merge_operations\n",
" for source_id in op.source_ids\n",
"}\n",
"entity_id_mapping = {}\n",
"for op in merge_operations:\n",
" merged_id = op.merged_entity['id']\n",
" for source_id in op.source_ids:\n",
" entity_id_mapping[source_id] = merged_id\n",
"\n",
"# Update relationships with merged entity IDs and resolved conflicts\n",
"deduplicated_relationships = []\n",
"for rel in normalized_relationships:\n",
" updated_rel = rel.copy()\n",
" updated_rel['source_id'] = entity_id_mapping.get(rel['source_id'], rel['source_id'])\n",
" updated_rel['target_id'] = entity_id_mapping.get(rel['target_id'], rel['target_id'])\n",
" \n",
" # Apply resolved conflict values\n",
" for resolution in resolved_relationship_conflicts:\n",
" if resolution.resolved and resolution.metadata.get('relationship_id') == rel.get('id'):\n",
" property_name = resolution.metadata.get('property_name')\n",
" updated_rel[property_name] = resolution.resolved_value\n",
" \n",
" deduplicated_relationships.append(updated_rel)\n",
" source_id = entity_id_mapping.get(rel['source_id'], rel['source_id'])\n",
" target_id = entity_id_mapping.get(rel['target_id'], rel['target_id'])\n",
" if source_id != target_id:\n",
" updated_rel['source_id'] = source_id\n",
" updated_rel['target_id'] = target_id\n",
" deduplicated_relationships.append(updated_rel)\n",
"\n",
"# Results\n",
"print(\"Entity deduplication completed\")\n",
"print(f\"Original entities: {len(entity_dicts)}\")\n",
"print(f\"Deduplicated entities: {len(deduplicated_entities)}\")\n",
"print(f\"Duplicates removed: {len(entity_dicts) - len(deduplicated_entities)}\")\n",
"processing_time = time.time() - start_time\n",
"\n",
"print(\"\\nRelationship updates completed\")\n",
"print(f\"Relationships updated: {len(deduplicated_relationships)}\")\n",
"print(f\"Conflicts resolved: {len(resolved_relationship_conflicts)}\")\n",
"summary = {\n",
" \"time_seconds\": round(processing_time, 2),\n",
" \"entities_original\": len(entity_dicts),\n",
" \"entities_deduplicated\": len(deduplicated_entities),\n",
" \"duplicates_removed\": len(entity_dicts) - len(deduplicated_entities),\n",
" \"relationships_updated\": len(deduplicated_relationships),\n",
"}\n",
"\n",
"if merge_operations:\n",
" print(\"\\nSample merge operation:\")\n",
" print(merge_operations[0])"
"print(summary)"
]
},
{
@@ -703,28 +701,17 @@
"source": [
"from semantica.kg import GraphBuilder\n",
"\n",
"# Deduplication is already done; avoid additional entity resolution/merging\n",
"graph_builder = GraphBuilder(\n",
" merge_entities=True,\n",
" entity_resolution_strategy=\"fuzzy\",\n",
" merge_entities=False,\n",
" entity_resolution_strategy=\"none\",\n",
")\n",
"\n",
"triplet_relationships = [\n",
" {\n",
" \"source\": t.subject,\n",
" \"predicate\": t.predicate,\n",
" \"target\": t.object,\n",
" \"confidence\": t.confidence,\n",
" \"metadata\": t.metadata,\n",
" }\n",
" for t in validated_triplets\n",
"]\n",
"\n",
"final_relationships = resolved_relationships + triplet_relationships\n",
"final_relationships = deduplicated_relationships\n",
"\n",
"kg_data = {\n",
" \"entities\": merged_entities,\n",
" \"entities\": deduplicated_entities,\n",
" \"relationships\": final_relationships,\n",
" \"triplets\": validated_triplets,\n",
" \"metadata\": {\n",
" \"source\": \"earnings_call_transcript\",\n",
" \"extraction_method\": \"Groq LLM\",\n",
@@ -733,12 +720,12 @@
"\n",
"knowledge_graph = graph_builder.build(\n",
" sources=[kg_data],\n",
" merge_entities=True,\n",
" merge_entities=False,\n",
")\n",
"\n",
"print(\"Knowledge graph build completed\")\n",
"print(\"Knowledge graph build completed (no additional merging)\")\n",
"print(\"Final entities:\", len(knowledge_graph.get(\"entities\", [])))\n",
"print(\"Final relationships:\", len(knowledge_graph.get(\"relationships\", [])))\n"
"print(\"Final relationships:\", len(knowledge_graph.get(\"relationships\", [])))"
]
},
{
@@ -999,10 +986,13 @@
" retention_days=30,\n",
")\n",
"\n",
"entity_count = len(knowledge_graph.get(\"entities\", []))\n",
"relationship_count = len(knowledge_graph.get(\"relationships\", []))\n",
"\n",
"memory_contents = [\n",
" f\"Earnings call transcript: {parsed_doc['metadata'].get('title', 'Earnings Call')}\",\n",
" f\"Financial metrics extracted: {sum(len(v) for v in financial_metrics.values())}\",\n",
" f\"Key entities identified: {len(merged_entities)}\",\n",
" f\"Graph entities: {entity_count}\",\n",
" f\"Graph relationships: {relationship_count}\",\n",
"]\n",
"\n",
"memory_ids = []\n",
@@ -1027,7 +1017,7 @@
"print(\"Agent memory configured\")\n",
"print(\"Memories stored:\", len(memory_ids))\n",
"print(\"Total memories:\", memory_stats.get(\"total_memories\", 0))\n",
"print(\"Retrieved memories:\", len(financial_memories))\n"
"print(\"Retrieved memories:\", len(financial_memories))"
]
},
{
@@ -1224,8 +1214,9 @@
"analysis_summary = {\n",
" \"entities\": len(knowledge_graph.get(\"entities\", [])),\n",
" \"relationships\": len(knowledge_graph.get(\"relationships\", [])),\n",
" \"conflicts_resolved\": len(resolved_conflicts),\n",
" \"merged_entities\": len(merged_entities),\n",
" \"entity_conflicts_resolved\": len(resolved_entity_value_conflicts),\n",
" \"relationship_conflicts_resolved\": len(resolved_relationship_conflicts),\n",
" \"deduplicated_entities\": len(deduplicated_entities),\n",
" \"communities\": num_communities,\n",
" \"questions_answered\": len(generated_answers),\n",
" \"llm_model\": groq_llm.model,\n",
@@ -1235,7 +1226,8 @@
"print(\"KG JSON entities:\", analysis_summary[\"entities\"])\n",
"print(\"KG RDF size (chars):\", len(kg_rdf))\n",
"print(\"Questions answered:\", analysis_summary[\"questions_answered\"])\n",
"print(\"LLM model:\", analysis_summary[\"llm_model\"])\n"
"print(\"LLM model:\", analysis_summary[\"llm_model\"])\n",
"print(\"Conflicts resolved:\", analysis_summary[\"entity_conflicts_resolved\"] + analysis_summary[\"relationship_conflicts_resolved\"])"
]
}
],