diff --git a/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb b/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb index 5c27420a..74d0aeaa 100644 --- a/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb +++ b/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb @@ -10,15 +10,16 @@ "\n", "## Overview\n", "\n", - "This notebook demonstrates how to build knowledge graphs from entities and relationships using Semantica's graph building modules. You'll learn to use `GraphBuilder` and `EntityResolver`.\n", + "This notebook demonstrates how to build knowledge graphs from extracted entities and relationships using Semantica's graph building modules. You'll learn to use `GraphBuilder` and `EntityResolver`.\n", "\n", "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/kg/)\n", "\n", "### Learning Objectives\n", "\n", - "- Use `GraphBuilder` to construct knowledge graphs\n", - "- Use `EntityResolver` to resolve entity conflicts\n", - "**Note**: For deduplication, use the `semantica.deduplication` module.\n", + "- Extract entity mentions and relations, and map them into graph records\n", + "- Use `GraphBuilder` to construct a graph whose edges come from the actual extracted relations\n", + "- Use `EntityResolver` to merge duplicate mentions and remap relationship endpoints\n", + "- Use the `semantica.deduplication` module and report the complete deduplicated entity set\n", "\n", "## Installation\n", "\n", @@ -32,120 +33,199 @@ "\n", "---\n", "\n", - "## Step 1: Build Knowledge Graph\n", + "## Step 1: Extract Entities and Relations\n", "\n", - "Construct a knowledge graph from entities and relationships.\n" + "Extract entity mentions and relations from text. The sample text mentions `Apple Inc.` in two separate sentences, so we can later show how duplicate mentions are resolved into one canonical entity.\n" ] }, { "cell_type": "code", - "execution_count": null, "metadata": {}, - "outputs": [], "source": [ "!pip install semantica\n" - ] + ], + "execution_count": null, + "outputs": [] }, { "cell_type": "code", - "execution_count": null, "metadata": {}, - "outputs": [], "source": [ - "from semantica.kg import GraphBuilder\n", "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", "\n", - "builder = GraphBuilder()\n", + "text = (\n", + " \"Apple Inc. is headquartered in Cupertino, California. \"\n", + " \"Tim Cook is the CEO of Apple Inc. \"\n", + " \"The company is a technology company.\"\n", + ")\n", + "\n", "ner_extractor = NERExtractor()\n", "relation_extractor = RelationExtractor()\n", "\n", - "text = \"Apple Inc. is a technology company. Tim Cook is the CEO of Apple Inc. Apple Inc. is headquartered in Cupertino, California.\"\n", + "mentions = ner_extractor.extract(text)\n", + "relations = relation_extractor.extract(text, mentions)\n", "\n", - "entities_list = ner_extractor.extract(text)\n", - "relationships_list = relation_extractor.extract(text, entities_list)\n", + "print(\"Entity mentions:\")\n", + "for mention in mentions:\n", + " print(f\" {mention.text!r:<13} {mention.label:<7} span=[{mention.start_char}:{mention.end_char}]\")\n", "\n", - "entities = []\n", - "for i, entity in enumerate(entities_list[:5], 1):\n", - " entities.append({\n", - " \"id\": f\"e{i}\",\n", - " \"type\": entity.label,\n", - " \"name\": entity.text,\n", - " \"properties\": {}\n", - " })\n", - "\n", - "relationships = []\n", - "for i, rel in enumerate(relationships_list[:3], 1):\n", - " relationships.append({\n", - " \"source\": f\"e{1}\",\n", - " \"target\": f\"e{i+1}\",\n", - " \"type\": rel.predicate,\n", - " \"properties\": {}\n", - " })\n", - "\n", - "knowledge_graph = builder.build(entities, relationships)\n", - "\n", - "print(f\"Built knowledge graph with {len(knowledge_graph.get('entities', []))} entities\")\n", - "print(f\"Relationships: {len(knowledge_graph.get('relationships', []))}\")" - ] + "print(\"\\nExtracted relations:\")\n", + "for rel in relations:\n", + " print(f\" {rel.subject.text!r} --{rel.predicate}--> {rel.object.text!r}\")" + ], + "execution_count": null, + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 2: Entity Resolution\n", + "## Step 2: Build the Knowledge Graph\n", "\n", - "Resolve entity conflicts and duplicates.\n" + "Give every mention a graph ID, then translate each relation's `subject` and `object` into those IDs. Building edges from the actual relation endpoints — rather than guessing endpoints from list positions — is what keeps the graph faithful to the text.\n" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from semantica.kg import GraphBuilder\n", + "\n", + "entities = []\n", + "span_to_id = {}\n", + "for i, mention in enumerate(mentions, 1):\n", + " graph_id = f\"e{i}\"\n", + " span_to_id[(mention.start_char, mention.end_char)] = graph_id\n", + " entities.append({\n", + " \"id\": graph_id,\n", + " \"type\": mention.label,\n", + " \"name\": mention.text,\n", + " \"properties\": {},\n", + " })\n", + "\n", + "relationships = []\n", + "for rel in relations:\n", + " source_id = span_to_id.get((rel.subject.start_char, rel.subject.end_char))\n", + " target_id = span_to_id.get((rel.object.start_char, rel.object.end_char))\n", + " if source_id is None or target_id is None:\n", + " print(f\"Skipping relation with unmapped endpoint: \"\n", + " f\"{rel.subject.text!r} --{rel.predicate}--> {rel.object.text!r}\")\n", + " continue\n", + " relationships.append({\n", + " \"source\": source_id,\n", + " \"target\": target_id,\n", + " \"type\": rel.predicate,\n", + " \"properties\": {},\n", + " })\n", + "\n", + "builder = GraphBuilder()\n", + "knowledge_graph = builder.build({\"entities\": entities, \"relationships\": relationships})\n", + "\n", + "id_to_name = {entity[\"id\"]: entity[\"name\"] for entity in entities}\n", + "\n", + "print(f\"Graph entities ({len(knowledge_graph['entities'])}):\")\n", + "for entity in knowledge_graph[\"entities\"]:\n", + " print(f\" {entity['id']}: {entity['name']} ({entity['type']})\")\n", + "\n", + "print(f\"\\nGraph relationships ({len(knowledge_graph['relationships'])}):\")\n", + "for relationship in knowledge_graph[\"relationships\"]:\n", + " print(f\" {id_to_name[relationship['source']]} \"\n", + " f\"--{relationship['type']}--> {id_to_name[relationship['target']]}\")\n", + "\n", + "edges = {\n", + " (id_to_name[r[\"source\"]], r[\"type\"], id_to_name[r[\"target\"]])\n", + " for r in knowledge_graph[\"relationships\"]\n", + "}\n", + "assert (\"Apple Inc.\", \"located_in\", \"Cupertino\") in edges\n", + "assert (\"Tim Cook\", \"works_for\", \"Apple Inc.\") in edges" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Entity Resolution\n", + "\n", + "The graph currently contains two nodes for the same organization. `EntityResolver` merges duplicate mentions into one canonical entity and records which source IDs were merged (`merged_from`), so relationship endpoints can be remapped onto the canonical entity.\n" ] }, { "cell_type": "code", - "execution_count": null, "metadata": {}, - "outputs": [], "source": [ "from semantica.kg import EntityResolver\n", "\n", "entity_resolver = EntityResolver()\n", - "\n", "resolved_entities = entity_resolver.resolve_entities(entities)\n", "\n", - "print(f\"Original entities: {len(entities)}\")\n", - "print(f\"Resolved entities: {len(resolved_entities)}\")" - ] + "canonical_id = {}\n", + "for entity in resolved_entities:\n", + " for source_id in entity.get(\"merged_from\", [entity[\"id\"]]):\n", + " canonical_id[source_id] = entity[\"id\"]\n", + " if entity.get(\"merged_from\"):\n", + " print(f\"Merged {entity['merged_from']} -> {entity['id']}: {entity['name']}\")\n", + "\n", + "print(f\"\\nMentions in: {len(entities)}, resolved entities out: {len(resolved_entities)}\")\n", + "\n", + "resolved_names = {entity[\"id\"]: entity[\"name\"] for entity in resolved_entities}\n", + "print(\"\\nRelationships remapped onto canonical entities:\")\n", + "for relationship in relationships:\n", + " source = canonical_id[relationship[\"source\"]]\n", + " target = canonical_id[relationship[\"target\"]]\n", + " print(f\" {resolved_names[source]} --{relationship['type']}--> {resolved_names[target]}\")\n", + "\n", + "assert len(resolved_entities) == 4" + ], + "execution_count": null, + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 3: Deduplication\n", + "## Step 4: Deduplication\n", "\n", - "Remove duplicate entities from the graph.\n" + "The `semantica.deduplication` module gives finer control over the same problem. Note that `merge_duplicates` returns one `MergeOperation` per duplicate *group* — the complete deduplicated collection is those merged entities plus every entity that was not part of any group.\n" ] }, { "cell_type": "code", - "execution_count": null, "metadata": {}, - "outputs": [], "source": [ "from semantica.deduplication import DuplicateDetector, EntityMerger, MergeStrategy\n", "\n", - "# Detect duplicates\n", "detector = DuplicateDetector(similarity_threshold=0.8)\n", - "duplicate_groups = detector.detect_duplicate_groups(knowledge_graph.get('entities', []))\n", + "duplicate_groups = detector.detect_duplicate_groups(entities)\n", + "print(f\"Duplicate groups: {len(duplicate_groups)}\")\n", + "for group in duplicate_groups:\n", + " print(f\" {[entity['name'] for entity in group.entities]} \"\n", + " f\"(confidence={group.confidence:.2f})\")\n", "\n", - "# Merge duplicates\n", "merger = EntityMerger()\n", "merge_operations = merger.merge_duplicates(\n", - " knowledge_graph.get('entities', []),\n", - " strategy=MergeStrategy.KEEP_MOST_COMPLETE\n", + " entities, strategy=MergeStrategy.KEEP_MOST_COMPLETE\n", ")\n", "\n", - "deduplicated_entities = [op.merged_entity for op in merge_operations]\n", + "merged_source_ids = {\n", + " entity[\"id\"] for op in merge_operations for entity in op.source_entities\n", + "}\n", + "untouched_entities = [e for e in entities if e[\"id\"] not in merged_source_ids]\n", + "deduplicated_entities = untouched_entities + [\n", + " op.merged_entity for op in merge_operations\n", + "]\n", "\n", - "print(f\"Original entities: {len(knowledge_graph.get('entities', []))}\")\n", - "print(f\"Deduplicated entities: {len(deduplicated_entities)}\")\n" - ] + "print(f\"\\nMerge operations: {len(merge_operations)}\")\n", + "print(f\"Deduplicated entities ({len(deduplicated_entities)}):\")\n", + "for entity in deduplicated_entities:\n", + " print(f\" {entity['id']}: {entity['name']} ({entity['type']})\")\n", + "\n", + "assert len(merge_operations) == 1\n", + "assert len(deduplicated_entities) == 4" + ], + "execution_count": null, + "outputs": [] }, { "cell_type": "markdown", @@ -155,9 +235,10 @@ "\n", "You've learned how to build knowledge graphs:\n", "\n", - "- **GraphBuilder**: Construct knowledge graphs from entities and relationships\n", - "- **EntityResolver**: Resolve entity conflicts and duplicates\n", - "- **Deduplication**: Use `semantica.deduplication` module for removing duplicate entities\n", + "- **Extraction to graph**: map each mention to a graph ID and build edges from the actual `Relation.subject` / `Relation.object` endpoints\n", + "- **GraphBuilder**: construct knowledge graphs from explicit `{\"entities\": ..., \"relationships\": ...}` input\n", + "- **EntityResolver**: merge duplicate mentions into canonical entities and remap relationship endpoints\n", + "- **Deduplication**: combine `MergeOperation` results with untouched entities to get the complete deduplicated set\n", "\n", "Next: Learn how to analyze graphs in the Graph_Analytics notebook.\n" ]