From bbd6764215c1b8a48b10f5767b7cb8b412a0006a Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Fri, 16 Jan 2026 18:18:50 +0530 Subject: [PATCH] Use deduplicated entities/relationships; optimize and clean deduplication; disable extra merging in GraphBuilder --- .../finance/03_Earnings_Call_Analysis.ipynb | 124 ++++++++---------- 1 file changed, 58 insertions(+), 66 deletions(-) diff --git a/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb b/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb index dda18a38..62aa50ae 100644 --- a/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb +++ b/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb @@ -617,73 +617,71 @@ "outputs": [], "source": [ "from semantica.deduplication import DuplicateDetector, EntityMerger\n", + "import time\n", + "\n", + "start_time = time.time()\n", "\n", - "# Initialize deduplication components\n", "duplicate_detector = DuplicateDetector(\n", - " similarity_threshold=0.8,\n", - " confidence_threshold=0.7,\n", + " similarity_threshold=0.85,\n", + " confidence_threshold=0.75,\n", + " batch_size=50,\n", + " use_cache=True,\n", ")\n", "\n", - "entity_merger = EntityMerger(preserve_provenance=True)\n", + "entity_merger = EntityMerger(\n", + " preserve_provenance=True,\n", + " parallel_processing=True,\n", + ")\n", "\n", - "# Prepare entities (using original entities since no entity conflicts were found)\n", - "entity_dicts = [\n", - " {\n", - " \"id\": getattr(e, \"id\", None) or e.text,\n", - " \"name\": e.text,\n", + "entity_dicts = []\n", + "for i, e in enumerate(entities):\n", + " entity_dicts.append({\n", + " \"id\": getattr(e, \"id\", None) or f\"entity_{i}_{e.text}\",\n", + " \"name\": e.text.strip().lower(),\n", + " \"original_name\": e.text,\n", " \"type\": getattr(e, \"label\", \"UNKNOWN\"),\n", " \"confidence\": getattr(e, \"confidence\", 1.0),\n", " \"metadata\": getattr(e, \"metadata\", {}),\n", - " }\n", - " for e in entities\n", - "]\n", + " })\n", "\n", - "# Detect and merge duplicate entities\n", "duplicates = duplicate_detector.detect_duplicates(entity_dicts)\n", "\n", "merge_operations = entity_merger.merge_duplicates(\n", " entities=entity_dicts,\n", " duplicates=duplicates,\n", " strategy=\"keep_most_complete\",\n", + " max_workers=4,\n", ")\n", "\n", "deduplicated_entities = [op.merged_entity for op in merge_operations]\n", "\n", - "# Update relationships to use merged entity IDs\n", - "entity_id_mapping = {\n", - " source_id: op.merged_entity['id']\n", - " for op in merge_operations\n", - " for source_id in op.source_ids\n", - "}\n", + "entity_id_mapping = {}\n", + "for op in merge_operations:\n", + " merged_id = op.merged_entity['id']\n", + " for source_id in op.source_ids:\n", + " entity_id_mapping[source_id] = merged_id\n", "\n", - "# Update relationships with merged entity IDs and resolved conflicts\n", "deduplicated_relationships = []\n", "for rel in normalized_relationships:\n", " updated_rel = rel.copy()\n", - " updated_rel['source_id'] = entity_id_mapping.get(rel['source_id'], rel['source_id'])\n", - " updated_rel['target_id'] = entity_id_mapping.get(rel['target_id'], rel['target_id'])\n", - " \n", - " # Apply resolved conflict values\n", - " for resolution in resolved_relationship_conflicts:\n", - " if resolution.resolved and resolution.metadata.get('relationship_id') == rel.get('id'):\n", - " property_name = resolution.metadata.get('property_name')\n", - " updated_rel[property_name] = resolution.resolved_value\n", - " \n", - " deduplicated_relationships.append(updated_rel)\n", + " source_id = entity_id_mapping.get(rel['source_id'], rel['source_id'])\n", + " target_id = entity_id_mapping.get(rel['target_id'], rel['target_id'])\n", + " if source_id != target_id:\n", + " updated_rel['source_id'] = source_id\n", + " updated_rel['target_id'] = target_id\n", + " deduplicated_relationships.append(updated_rel)\n", "\n", - "# Results\n", - "print(\"Entity deduplication completed\")\n", - "print(f\"Original entities: {len(entity_dicts)}\")\n", - "print(f\"Deduplicated entities: {len(deduplicated_entities)}\")\n", - "print(f\"Duplicates removed: {len(entity_dicts) - len(deduplicated_entities)}\")\n", + "processing_time = time.time() - start_time\n", "\n", - "print(\"\\nRelationship updates completed\")\n", - "print(f\"Relationships updated: {len(deduplicated_relationships)}\")\n", - "print(f\"Conflicts resolved: {len(resolved_relationship_conflicts)}\")\n", + "summary = {\n", + " \"time_seconds\": round(processing_time, 2),\n", + " \"entities_original\": len(entity_dicts),\n", + " \"entities_deduplicated\": len(deduplicated_entities),\n", + " \"duplicates_removed\": len(entity_dicts) - len(deduplicated_entities),\n", + " \"relationships_updated\": len(deduplicated_relationships),\n", + "}\n", "\n", - "if merge_operations:\n", - " print(\"\\nSample merge operation:\")\n", - " print(merge_operations[0])" + "print(summary)" ] }, { @@ -703,28 +701,17 @@ "source": [ "from semantica.kg import GraphBuilder\n", "\n", + "# Deduplication is already done; avoid additional entity resolution/merging\n", "graph_builder = GraphBuilder(\n", - " merge_entities=True,\n", - " entity_resolution_strategy=\"fuzzy\",\n", + " merge_entities=False,\n", + " entity_resolution_strategy=\"none\",\n", ")\n", "\n", - "triplet_relationships = [\n", - " {\n", - " \"source\": t.subject,\n", - " \"predicate\": t.predicate,\n", - " \"target\": t.object,\n", - " \"confidence\": t.confidence,\n", - " \"metadata\": t.metadata,\n", - " }\n", - " for t in validated_triplets\n", - "]\n", - "\n", - "final_relationships = resolved_relationships + triplet_relationships\n", + "final_relationships = deduplicated_relationships\n", "\n", "kg_data = {\n", - " \"entities\": merged_entities,\n", + " \"entities\": deduplicated_entities,\n", " \"relationships\": final_relationships,\n", - " \"triplets\": validated_triplets,\n", " \"metadata\": {\n", " \"source\": \"earnings_call_transcript\",\n", " \"extraction_method\": \"Groq LLM\",\n", @@ -733,12 +720,12 @@ "\n", "knowledge_graph = graph_builder.build(\n", " sources=[kg_data],\n", - " merge_entities=True,\n", + " merge_entities=False,\n", ")\n", "\n", - "print(\"Knowledge graph build completed\")\n", + "print(\"Knowledge graph build completed (no additional merging)\")\n", "print(\"Final entities:\", len(knowledge_graph.get(\"entities\", [])))\n", - "print(\"Final relationships:\", len(knowledge_graph.get(\"relationships\", [])))\n" + "print(\"Final relationships:\", len(knowledge_graph.get(\"relationships\", [])))" ] }, { @@ -999,10 +986,13 @@ " retention_days=30,\n", ")\n", "\n", + "entity_count = len(knowledge_graph.get(\"entities\", []))\n", + "relationship_count = len(knowledge_graph.get(\"relationships\", []))\n", + "\n", "memory_contents = [\n", " f\"Earnings call transcript: {parsed_doc['metadata'].get('title', 'Earnings Call')}\",\n", - " f\"Financial metrics extracted: {sum(len(v) for v in financial_metrics.values())}\",\n", - " f\"Key entities identified: {len(merged_entities)}\",\n", + " f\"Graph entities: {entity_count}\",\n", + " f\"Graph relationships: {relationship_count}\",\n", "]\n", "\n", "memory_ids = []\n", @@ -1027,7 +1017,7 @@ "print(\"Agent memory configured\")\n", "print(\"Memories stored:\", len(memory_ids))\n", "print(\"Total memories:\", memory_stats.get(\"total_memories\", 0))\n", - "print(\"Retrieved memories:\", len(financial_memories))\n" + "print(\"Retrieved memories:\", len(financial_memories))" ] }, { @@ -1224,8 +1214,9 @@ "analysis_summary = {\n", " \"entities\": len(knowledge_graph.get(\"entities\", [])),\n", " \"relationships\": len(knowledge_graph.get(\"relationships\", [])),\n", - " \"conflicts_resolved\": len(resolved_conflicts),\n", - " \"merged_entities\": len(merged_entities),\n", + " \"entity_conflicts_resolved\": len(resolved_entity_value_conflicts),\n", + " \"relationship_conflicts_resolved\": len(resolved_relationship_conflicts),\n", + " \"deduplicated_entities\": len(deduplicated_entities),\n", " \"communities\": num_communities,\n", " \"questions_answered\": len(generated_answers),\n", " \"llm_model\": groq_llm.model,\n", @@ -1235,7 +1226,8 @@ "print(\"KG JSON entities:\", analysis_summary[\"entities\"])\n", "print(\"KG RDF size (chars):\", len(kg_rdf))\n", "print(\"Questions answered:\", analysis_summary[\"questions_answered\"])\n", - "print(\"LLM model:\", analysis_summary[\"llm_model\"])\n" + "print(\"LLM model:\", analysis_summary[\"llm_model\"])\n", + "print(\"Conflicts resolved:\", analysis_summary[\"entity_conflicts_resolved\"] + analysis_summary[\"relationship_conflicts_resolved\"])" ] } ],