mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-12 04:01:35 +00:00
287 lines
11 KiB
Plaintext
287 lines
11 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"[](https://colab.research.google.com/github/semantica-agi/semantica/blob/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb)\n",
|
|
"\n",
|
|
"# Building Knowledge Graphs\n",
|
|
"\n",
|
|
"## Overview\n",
|
|
"\n",
|
|
"This notebook demonstrates how to build knowledge graphs from extracted entities and relationships using Semantica's graph building modules. You'll learn to use `GraphBuilder` and `EntityResolver`.\n",
|
|
"\n",
|
|
"**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/kg/)\n",
|
|
"\n",
|
|
"### Learning Objectives\n",
|
|
"\n",
|
|
"- Extract entity mentions and relations, and map them into graph records\n",
|
|
"- Use `GraphBuilder` to construct a graph whose edges come from the actual extracted relations\n",
|
|
"- Use `EntityResolver` to merge duplicate mentions and remap relationship endpoints\n",
|
|
"- Use the `semantica.deduplication` module and report the complete deduplicated entity set\n",
|
|
"\n",
|
|
"## Installation\n",
|
|
"\n",
|
|
"Install Semantica from PyPI:\n",
|
|
"\n",
|
|
"```bash\n",
|
|
"pip install semantica\n",
|
|
"# Or with all optional dependencies:\n",
|
|
"pip install semantica[all]\n",
|
|
"```\n",
|
|
"\n",
|
|
"---\n",
|
|
"\n",
|
|
"## Step 1: Extract Entities and Relations\n",
|
|
"\n",
|
|
"Extract entity mentions and relations from text. The sample text mentions `Apple Inc.` in two separate sentences, so we can later show how duplicate mentions are resolved into one canonical entity.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"metadata": {},
|
|
"source": [
|
|
"%pip install semantica\n",
|
|
"\n",
|
|
"# spaCy models are distributed separately from the spaCy library. This lesson\n",
|
|
"# relies on the English model to recognize standalone places such as Cupertino.\n",
|
|
"import sys\n",
|
|
"import subprocess\n",
|
|
"import spacy\n",
|
|
"\n",
|
|
"try:\n",
|
|
" spacy.load(\"en_core_web_sm\")\n",
|
|
"except OSError:\n",
|
|
" subprocess.check_call([sys.executable, \"-m\", \"spacy\", \"download\", \"en_core_web_sm\"])\n"
|
|
],
|
|
"execution_count": null,
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"metadata": {},
|
|
"source": [
|
|
"from semantica.semantic_extract import NERExtractor, RelationExtractor\n",
|
|
"\n",
|
|
"text = (\n",
|
|
" \"Apple Inc. is headquartered in Cupertino, California. \"\n",
|
|
" \"Tim Cook is the CEO of Apple Inc. \"\n",
|
|
" \"The company is a technology company.\"\n",
|
|
")\n",
|
|
"\n",
|
|
"ner_extractor = NERExtractor()\n",
|
|
"relation_extractor = RelationExtractor()\n",
|
|
"\n",
|
|
"mentions = ner_extractor.extract(text)\n",
|
|
"relations = relation_extractor.extract(text, mentions)\n",
|
|
"\n",
|
|
"print(\"Entity mentions:\")\n",
|
|
"for mention in mentions:\n",
|
|
" print(f\" {mention.text!r:<13} {mention.label:<7} span=[{mention.start_char}:{mention.end_char}]\")\n",
|
|
"\n",
|
|
"print(\"\\nExtracted relations:\")\n",
|
|
"for rel in relations:\n",
|
|
" print(f\" {rel.subject.text!r} --{rel.predicate}--> {rel.object.text!r}\")"
|
|
],
|
|
"execution_count": null,
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Step 2: Build the Knowledge Graph\n",
|
|
"\n",
|
|
"Give every mention a graph ID, then translate each relation's `subject` and `object` into those IDs. Building edges from the actual relation endpoints — rather than guessing endpoints from list positions — is what keeps the graph faithful to the text.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"metadata": {},
|
|
"source": [
|
|
"from semantica.kg import GraphBuilder\n",
|
|
"\n",
|
|
"entities = []\n",
|
|
"span_to_id = {}\n",
|
|
"for i, mention in enumerate(mentions, 1):\n",
|
|
" graph_id = f\"e{i}\"\n",
|
|
" span_to_id[(mention.start_char, mention.end_char)] = graph_id\n",
|
|
" entities.append({\n",
|
|
" \"id\": graph_id,\n",
|
|
" \"type\": mention.label,\n",
|
|
" \"name\": mention.text,\n",
|
|
" \"properties\": {},\n",
|
|
" })\n",
|
|
"\n",
|
|
"relationships = []\n",
|
|
"for rel in relations:\n",
|
|
" source_id = span_to_id.get((rel.subject.start_char, rel.subject.end_char))\n",
|
|
" target_id = span_to_id.get((rel.object.start_char, rel.object.end_char))\n",
|
|
" if source_id is None or target_id is None:\n",
|
|
" print(f\"Skipping relation with unmapped endpoint: \"\n",
|
|
" f\"{rel.subject.text!r} --{rel.predicate}--> {rel.object.text!r}\")\n",
|
|
" continue\n",
|
|
" relationships.append({\n",
|
|
" \"source\": source_id,\n",
|
|
" \"target\": target_id,\n",
|
|
" \"type\": rel.predicate,\n",
|
|
" \"properties\": {},\n",
|
|
" })\n",
|
|
"\n",
|
|
"builder = GraphBuilder()\n",
|
|
"knowledge_graph = builder.build({\"entities\": entities, \"relationships\": relationships})\n",
|
|
"\n",
|
|
"id_to_name = {entity[\"id\"]: entity[\"name\"] for entity in entities}\n",
|
|
"\n",
|
|
"print(f\"Graph entities ({len(knowledge_graph['entities'])}):\")\n",
|
|
"for entity in knowledge_graph[\"entities\"]:\n",
|
|
" print(f\" {entity['id']}: {entity['name']} ({entity['type']})\")\n",
|
|
"\n",
|
|
"print(f\"\\nGraph relationships ({len(knowledge_graph['relationships'])}):\")\n",
|
|
"for relationship in knowledge_graph[\"relationships\"]:\n",
|
|
" print(f\" {id_to_name[relationship['source']]} \"\n",
|
|
" f\"--{relationship['type']}--> {id_to_name[relationship['target']]}\")\n",
|
|
"\n",
|
|
"edges = {\n",
|
|
" (id_to_name[r[\"source\"]], r[\"type\"], id_to_name[r[\"target\"]])\n",
|
|
" for r in knowledge_graph[\"relationships\"]\n",
|
|
"}\n",
|
|
"assert (\"Apple Inc.\", \"located_in\", \"Cupertino\") in edges\n",
|
|
"assert (\"Tim Cook\", \"works_for\", \"Apple Inc.\") in edges"
|
|
],
|
|
"execution_count": null,
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Step 3: Entity Resolution\n",
|
|
"\n",
|
|
"The graph currently contains two nodes for the same organization. `EntityResolver` merges duplicate mentions into one canonical entity and records which source IDs were merged (`merged_from`), so relationship endpoints can be remapped onto the canonical entity.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"metadata": {},
|
|
"source": [
|
|
"from semantica.kg import EntityResolver\n",
|
|
"\n",
|
|
"entity_resolver = EntityResolver()\n",
|
|
"resolved_entities = entity_resolver.resolve_entities(entities)\n",
|
|
"\n",
|
|
"canonical_id = {}\n",
|
|
"for entity in resolved_entities:\n",
|
|
" for source_id in entity.get(\"merged_from\", [entity[\"id\"]]):\n",
|
|
" canonical_id[source_id] = entity[\"id\"]\n",
|
|
" if entity.get(\"merged_from\"):\n",
|
|
" print(f\"Merged {entity['merged_from']} -> {entity['id']}: {entity['name']}\")\n",
|
|
"\n",
|
|
"print(f\"\\nMentions in: {len(entities)}, resolved entities out: {len(resolved_entities)}\")\n",
|
|
"\n",
|
|
"resolved_names = {entity[\"id\"]: entity[\"name\"] for entity in resolved_entities}\n",
|
|
"print(\"\\nRelationships remapped onto canonical entities:\")\n",
|
|
"for relationship in relationships:\n",
|
|
" source = canonical_id[relationship[\"source\"]]\n",
|
|
" target = canonical_id[relationship[\"target\"]]\n",
|
|
" print(f\" {resolved_names[source]} --{relationship['type']}--> {resolved_names[target]}\")\n",
|
|
"\n",
|
|
"canonical_entities = {(entity[\"name\"], entity[\"type\"]) for entity in resolved_entities}\n",
|
|
"assert canonical_entities == {\n",
|
|
" (\"Apple Inc.\", \"ORG\"),\n",
|
|
" (\"Tim Cook\", \"PERSON\"),\n",
|
|
" (\"Cupertino\", \"GPE\"),\n",
|
|
" (\"California\", \"GPE\"),\n",
|
|
"}\n",
|
|
"assert len(resolved_entities) == 4"
|
|
],
|
|
"execution_count": null,
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Step 4: Deduplication\n",
|
|
"\n",
|
|
"The `semantica.deduplication` module gives finer control over the same problem. Note that `merge_duplicates` returns one `MergeOperation` per duplicate *group* — the complete deduplicated collection is those merged entities plus every entity that was not part of any group.\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"metadata": {},
|
|
"source": [
|
|
"from semantica.deduplication import DuplicateDetector, EntityMerger, MergeStrategy\n",
|
|
"\n",
|
|
"detector = DuplicateDetector(similarity_threshold=0.8)\n",
|
|
"duplicate_groups = detector.detect_duplicate_groups(entities)\n",
|
|
"print(f\"Duplicate groups: {len(duplicate_groups)}\")\n",
|
|
"for group in duplicate_groups:\n",
|
|
" print(f\" {[entity['name'] for entity in group.entities]} \"\n",
|
|
" f\"(confidence={group.confidence:.2f})\")\n",
|
|
"\n",
|
|
"merger = EntityMerger()\n",
|
|
"merge_operations = merger.merge_duplicates(\n",
|
|
" entities, strategy=MergeStrategy.KEEP_MOST_COMPLETE\n",
|
|
")\n",
|
|
"\n",
|
|
"merged_source_ids = {\n",
|
|
" entity[\"id\"] for op in merge_operations for entity in op.source_entities\n",
|
|
"}\n",
|
|
"untouched_entities = [e for e in entities if e[\"id\"] not in merged_source_ids]\n",
|
|
"deduplicated_entities = untouched_entities + [\n",
|
|
" op.merged_entity for op in merge_operations\n",
|
|
"]\n",
|
|
"\n",
|
|
"print(f\"\\nMerge operations: {len(merge_operations)}\")\n",
|
|
"print(f\"Deduplicated entities ({len(deduplicated_entities)}):\")\n",
|
|
"for entity in deduplicated_entities:\n",
|
|
" print(f\" {entity['id']}: {entity['name']} ({entity['type']})\")\n",
|
|
"\n",
|
|
"assert len(merge_operations) == 1\n",
|
|
"assert len(deduplicated_entities) == 4"
|
|
],
|
|
"execution_count": null,
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Summary\n",
|
|
"\n",
|
|
"You've learned how to build knowledge graphs:\n",
|
|
"\n",
|
|
"- **Extraction to graph**: map each mention to a graph ID and build edges from the actual `Relation.subject` / `Relation.object` endpoints\n",
|
|
"- **GraphBuilder**: construct knowledge graphs from explicit `{\"entities\": ..., \"relationships\": ...}` input\n",
|
|
"- **EntityResolver**: merge duplicate mentions into canonical entities and remap relationship endpoints\n",
|
|
"- **Deduplication**: combine `MergeOperation` results with untouched entities to get the complete deduplicated set\n",
|
|
"\n",
|
|
"Next: Learn how to analyze graphs in the Graph_Analytics notebook.\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.9"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|