mengzhx/semantica
1
0
Fork 0
mirror of https://github.com/semantica-agi/semantica.git synced 2026-08-29 04:26:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update Unstructured_to_Ontology notebook to use OntologyValidator module from semantica

Browse Source
This commit is contained in:
KaifAhmad1
2025-11-13 13:54:39 +05:30
parent 4cca9c6ac5
commit dc7d239903
8 changed files with 1623 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cookbook/advanced_patterns/Complete_Visualization_Suite.ipynb
+218 -42
View File
@@ -8,59 +8,235 @@
"\n",
"## Overview\n",
"\n",
"Comprehensive visualization capabilities: visualize knowledge graphs, embeddings, quality metrics, analytics, and temporal data.\n",
"Comprehensive visualization capabilities: visualize knowledge graphs, embeddings, quality metrics, analytics, and temporal data.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from semantica.visualization import (\n",
" KGVisualizer,\n",
" EmbeddingVisualizer,\n",
" QualityVisualizer,\n",
" AnalyticsVisualizer,\n",
" TemporalVisualizer\n",
")\n",
"from semantica.kg import GraphBuilder, GraphAnalyzer\n",
"from semantica.embeddings import EmbeddingGenerator\n",
"from semantica.kg_qa import KGQualityAssessor\n",
"import numpy as np\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 1: Create Sample Knowledge Graph\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"builder = GraphBuilder()\n",
"\n",
"## Visualization Types\n",
"entities = [\n",
" {\"id\": \"e1\", \"type\": \"Person\", \"name\": \"Alice\", \"properties\": {\"age\": 30}},\n",
" {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Bob\", \"properties\": {\"age\": 35}},\n",
" {\"id\": \"e3\", \"type\": \"Organization\", \"name\": \"Tech Corp\", \"properties\": {\"founded\": 2010}},\n",
" {\"id\": \"e4\", \"type\": \"Location\", \"name\": \"San Francisco\", \"properties\": {\"country\": \"USA\"}},\n",
"]\n",
"\n",
"## Knowledge Graph Visualization\n",
"relationships = [\n",
" {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"knows\", \"properties\": {\"since\": 2020}},\n",
" {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"works_for\", \"properties\": {\"role\": \"Engineer\"}},\n",
" {\"source\": \"e3\", \"target\": \"e4\", \"type\": \"located_in\", \"properties\": {}},\n",
"]\n",
"\n",
"'''\n",
"# from semantica.visualization import KGVisualizer\n",
"# \n",
"# kg_visualizer = KGVisualizer()\n",
"# kg_visualizer.visualize(knowledge_graph)\n",
"'''\n",
"knowledge_graph = builder.build(entities, relationships)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2: Knowledge Graph Visualization\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"kg_visualizer = KGVisualizer()\n",
"kg_visualizer.visualize(knowledge_graph, layout=\"spring\", show_labels=True)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 3: Generate Embeddings and Visualize\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"embedding_generator = EmbeddingGenerator()\n",
"texts = [entity.get(\"name\", \"\") for entity in entities]\n",
"embeddings = embedding_generator.generate(texts)\n",
"\n",
"## Embedding Visualization\n",
"labels = [entity.get(\"type\", \"Unknown\") for entity in entities]\n",
"\n",
"'''\n",
"# from semantica.visualization import EmbeddingVisualizer\n",
"# \n",
"# embedding_visualizer = EmbeddingVisualizer()\n",
"# embedding_visualizer.visualize_tsne(embeddings, labels)\n",
"'''\n",
"embedding_visualizer = EmbeddingVisualizer()\n",
"embedding_visualizer.visualize_tsne(embeddings, labels, title=\"Entity Embeddings Visualization\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 4: Quality Metrics Visualization\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"quality_assessor = KGQualityAssessor()\n",
"quality_metrics = quality_assessor.assess(knowledge_graph)\n",
"\n",
"## Quality Visualization\n",
"quality_visualizer = QualityVisualizer()\n",
"quality_visualizer.visualize_metrics(quality_metrics, title=\"Knowledge Graph Quality Metrics\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 5: Graph Analytics Visualization\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"graph_analyzer = GraphAnalyzer()\n",
"\n",
"'''\n",
"# from semantica.visualization import QualityVisualizer\n",
"# \n",
"# quality_visualizer = QualityVisualizer()\n",
"# quality_visualizer.visualize_metrics(quality_metrics)\n",
"'''\n",
"centrality_results = graph_analyzer.calculate_centrality(\n",
" knowledge_graph, \n",
" centrality_type=\"degree\"\n",
")\n",
"\n",
"## Analytics Visualization\n",
"centrality_scores = {}\n",
"if centrality_results and \"centrality_measures\" in centrality_results:\n",
" degree_centrality = centrality_results[\"centrality_measures\"].get(\"degree\", {})\n",
" if isinstance(degree_centrality, dict) and \"centrality\" in degree_centrality:\n",
" centrality_scores = degree_centrality[\"centrality\"]\n",
" elif isinstance(degree_centrality, dict):\n",
" centrality_scores = degree_centrality\n",
"\n",
"'''\n",
"# from semantica.visualization import AnalyticsVisualizer\n",
"# \n",
"# analytics_visualizer = AnalyticsVisualizer()\n",
"# analytics_visualizer.visualize_centrality(centrality_scores)\n",
"# analytics_visualizer.visualize_communities(communities)\n",
"'''\n",
"communities_result = graph_analyzer.detect_communities(\n",
" knowledge_graph, \n",
" algorithm=\"louvain\"\n",
")\n",
"\n",
"## Temporal Data Visualization\n",
"communities = []\n",
"community_dict = {}\n",
"if communities_result and \"communities\" in communities_result:\n",
" communities_data = communities_result[\"communities\"]\n",
" if isinstance(communities_data, list):\n",
" communities = communities_data\n",
" for idx, community in enumerate(communities):\n",
" if isinstance(community, list):\n",
" for node in community:\n",
" community_dict[node] = idx\n",
" elif isinstance(community, dict) and \"nodes\" in community:\n",
" for node in community[\"nodes\"]:\n",
" community_dict[node] = idx\n",
"\n",
"'''\n",
"# from semantica.visualization import TemporalVisualizer\n",
"# \n",
"# temporal_visualizer = TemporalVisualizer()\n",
"# temporal_visualizer.visualize_timeline(temporal_kg)\n",
"# temporal_visualizer.visualize_evolution(entity_history)\n",
"# \n",
"# # Comprehensive visualization\n",
"# print(\"All visualizations generated\")\n",
"'''\n"
"analytics_visualizer = AnalyticsVisualizer()\n",
"analytics_visualizer.visualize_centrality(centrality_scores, title=\"Node Centrality Scores\")\n",
"\n",
"if community_dict:\n",
" analytics_visualizer.visualize_communities(\n",
" knowledge_graph, \n",
" community_dict, \n",
" title=\"Community Detection\"\n",
" )\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 6: Temporal Data Visualization\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"temporal_kg = {\n",
" \"entities\": entities,\n",
" \"relationships\": relationships,\n",
" \"timestamps\": {\n",
" \"e1\": [2020, 2021, 2022],\n",
" \"e2\": [2020, 2021],\n",
" \"e3\": [2010, 2015, 2020, 2022],\n",
" }\n",
"}\n",
"\n",
"entity_history = {\n",
" \"e1\": [\n",
" {\"timestamp\": 2020, \"properties\": {\"age\": 28}},\n",
" {\"timestamp\": 2021, \"properties\": {\"age\": 29}},\n",
" {\"timestamp\": 2022, \"properties\": {\"age\": 30}},\n",
" ]\n",
"}\n",
"\n",
"temporal_visualizer = TemporalVisualizer()\n",
"temporal_visualizer.visualize_timeline(temporal_kg, title=\"Temporal Knowledge Graph Timeline\")\n",
"temporal_visualizer.visualize_evolution(entity_history, entity_id=\"e1\", title=\"Entity Evolution\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary\n",
"\n",
"All visualization types demonstrated:\n",
"- Knowledge Graph Visualization\n",
"- Embedding Visualization (t-SNE)\n",
"- Quality Metrics Visualization\n",
"- Graph Analytics Visualization (Centrality & Communities)\n",
"- Temporal Data Visualization (Timeline & Evolution)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Complete Visualization Suite\")\n",
"print(\"All visualizations generated successfully\")\n"
]
}
],
cookbook/advanced_patterns/Conflict_Resolution_Strategies.ipynb
+282 -44
View File
@@ -10,58 +10,296 @@
"\n",
"Detect conflicts in knowledge graphs, apply multiple resolution strategies, track sources, and maintain audit trails.\n",
"\n",
"## Workflow: Detect Conflicts → Multiple Resolution Strategies → Track Sources → Audit\n",
"## Workflow: Detect Conflicts → Multiple Resolution Strategies → Track Sources → Audit\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from semantica.kg import GraphBuilder\n",
"from semantica.kg_qa import ConsistencyChecker\n",
"from datetime import datetime\n",
"import json\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 1: Create Knowledge Graph with Conflicting Data\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"builder = GraphBuilder()\n",
"\n",
"## Step 1: Detect Conflicts\n",
"entities = [\n",
" {\n",
" \"id\": \"e1\",\n",
" \"type\": \"Person\",\n",
" \"name\": \"John Doe\",\n",
" \"properties\": {\"age\": 30, \"location\": \"New York\"},\n",
" \"source\": \"source1\",\n",
" \"timestamp\": datetime(2023, 1, 1)\n",
" },\n",
" {\n",
" \"id\": \"e1\",\n",
" \"type\": \"Person\",\n",
" \"name\": \"John Doe\",\n",
" \"properties\": {\"age\": 32, \"location\": \"Boston\"},\n",
" \"source\": \"source2\",\n",
" \"timestamp\": datetime(2023, 6, 1)\n",
" },\n",
" {\n",
" \"id\": \"e2\",\n",
" \"type\": \"Organization\",\n",
" \"name\": \"Tech Corp\",\n",
" \"properties\": {\"founded\": 2010, \"employees\": 100},\n",
" \"source\": \"source1\",\n",
" \"timestamp\": datetime(2023, 1, 1)\n",
" },\n",
" {\n",
" \"id\": \"e2\",\n",
" \"type\": \"Organization\",\n",
" \"name\": \"Tech Corp\",\n",
" \"properties\": {\"founded\": 2012, \"employees\": 150},\n",
" \"source\": \"source2\",\n",
" \"timestamp\": datetime(2023, 3, 1)\n",
" },\n",
"]\n",
"\n",
"'''\n",
"# from semantica.conflicts import ConflictDetector\n",
"# \n",
"# detector = ConflictDetector()\n",
"# conflicts = detector.detect(knowledge_graph)\n",
"# \n",
"# print(f\"Found {len(conflicts)} conflicts\")\n",
"'''\n",
"relationships = [\n",
" {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"works_for\", \"source\": \"source1\"},\n",
" {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"founder_of\", \"source\": \"source2\"},\n",
"]\n",
"\n",
"## Step 2: Multiple Resolution Strategies\n",
"knowledge_graph = builder.build(entities, relationships)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2: Detect Conflicts\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"consistency_checker = ConsistencyChecker()\n",
"conflicts = consistency_checker.check_conflicts(knowledge_graph)\n",
"\n",
"'''\n",
"# from semantica.conflicts import ConflictResolver\n",
"# \n",
"# resolver = ConflictResolver()\n",
"# \n",
"# # Strategy 1: Most recent wins\n",
"# resolved_1 = resolver.resolve(conflicts, strategy=\"most_recent\")\n",
"# \n",
"# # Strategy 2: Most authoritative source wins\n",
"# resolved_2 = resolver.resolve(conflicts, strategy=\"authoritative\")\n",
"# \n",
"# # Strategy 3: Merge conflicting information\n",
"# resolved_3 = resolver.resolve(conflicts, strategy=\"merge\")\n",
"'''\n",
"for i, conflict in enumerate(conflicts, 1):\n",
" print(f\"Conflict {i}:\")\n",
" print(f\" Entity/Relationship: {conflict.get('entity_id', conflict.get('relationship_id'))}\")\n",
" print(f\" Type: {conflict.get('type')}\")\n",
" print(f\" Conflicting values: {conflict.get('values')}\")\n",
" print(f\" Sources: {conflict.get('sources')}\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 3: Multiple Resolution Strategies\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"class ConflictResolver:\n",
" def __init__(self):\n",
" self.audit_trail = []\n",
" \n",
" def resolve(self, conflicts, strategy=\"most_recent\"):\n",
" resolved = []\n",
" \n",
" for conflict in conflicts:\n",
" if strategy == \"most_recent\":\n",
" values = conflict.get('values', [])\n",
" timestamps = conflict.get('timestamps', [])\n",
" if timestamps:\n",
" most_recent_idx = timestamps.index(max(timestamps))\n",
" resolved_value = values[most_recent_idx]\n",
" else:\n",
" resolved_value = values[-1] if values else None\n",
" \n",
" elif strategy == \"authoritative\":\n",
" sources = conflict.get('sources', [])\n",
" authoritative_sources = [\"source1\", \"official_db\", \"verified\"]\n",
" resolved_value = None\n",
" for auth_source in authoritative_sources:\n",
" if auth_source in sources:\n",
" idx = sources.index(auth_source)\n",
" resolved_value = conflict.get('values', [])[idx]\n",
" break\n",
" if resolved_value is None:\n",
" resolved_value = conflict.get('values', [])[0] if conflict.get('values') else None\n",
" \n",
" elif strategy == \"merge\":\n",
" values = conflict.get('values', [])\n",
" if isinstance(values[0], dict):\n",
" merged = {}\n",
" for val in values:\n",
" merged.update(val)\n",
" resolved_value = merged\n",
" elif isinstance(values[0], (int, float)):\n",
" resolved_value = sum(values) / len(values)\n",
" else:\n",
" resolved_value = \", \".join(set(str(v) for v in values))\n",
" else:\n",
" resolved_value = conflict.get('values', [])[0] if conflict.get('values') else None\n",
" \n",
" resolved.append({\n",
" 'conflict_id': conflict.get('entity_id', conflict.get('relationship_id')),\n",
" 'resolved_value': resolved_value,\n",
" 'strategy': strategy,\n",
" 'timestamp': datetime.now()\n",
" })\n",
" \n",
" self.audit_trail.append({\n",
" 'conflict': conflict,\n",
" 'resolution': resolved[-1],\n",
" 'resolved_at': datetime.now()\n",
" })\n",
" \n",
" return resolved\n",
"\n",
"## Step 3: Track Sources\n",
"resolver = ConflictResolver()\n",
"\n",
"'''\n",
"# from semantica.conflicts import SourceTracker\n",
"# \n",
"# tracker = SourceTracker()\n",
"# \n",
"# # Track where each piece of information came from\n",
"# for conflict in conflicts:\n",
"# sources = tracker.get_sources(conflict)\n",
"# print(f\"Conflict sources: {sources}\")\n",
"'''\n",
"resolved_1 = resolver.resolve(conflicts, strategy=\"most_recent\")\n",
"print(\"Strategy 1: Most Recent Wins\")\n",
"for r in resolved_1:\n",
" print(f\" Resolved: {r['conflict_id']} = {r['resolved_value']}\")\n",
"\n",
"## Step 4: Audit\n",
"resolver2 = ConflictResolver()\n",
"resolved_2 = resolver2.resolve(conflicts, strategy=\"authoritative\")\n",
"print(\"\\nStrategy 2: Most Authoritative Source Wins\")\n",
"for r in resolved_2:\n",
" print(f\" Resolved: {r['conflict_id']} = {r['resolved_value']}\")\n",
"\n",
"'''\n",
"# # Maintain audit trail of all resolutions\n",
"# audit_log = tracker.get_audit_trail()\n",
"# \n",
"# # Handle data conflicts with full traceability\n",
"# print(f\"Audit log contains {len(audit_log)} entries\")\n",
"'''\n"
"resolver3 = ConflictResolver()\n",
"resolved_3 = resolver3.resolve(conflicts, strategy=\"merge\")\n",
"print(\"\\nStrategy 3: Merge Conflicting Information\")\n",
"for r in resolved_3:\n",
" print(f\" Resolved: {r['conflict_id']} = {r['resolved_value']}\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 4: Track Sources\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"class SourceTracker:\n",
" def __init__(self):\n",
" self.source_map = {}\n",
" \n",
" def track_sources(self, conflicts):\n",
" for conflict in conflicts:\n",
" conflict_id = conflict.get('entity_id', conflict.get('relationship_id'))\n",
" sources = conflict.get('sources', [])\n",
" timestamps = conflict.get('timestamps', [])\n",
" \n",
" self.source_map[conflict_id] = {\n",
" 'sources': sources,\n",
" 'timestamps': timestamps,\n",
" 'values': conflict.get('values', [])\n",
" }\n",
" \n",
" def get_sources(self, conflict):\n",
" conflict_id = conflict.get('entity_id', conflict.get('relationship_id'))\n",
" return self.source_map.get(conflict_id, {})\n",
"\n",
"tracker = SourceTracker()\n",
"tracker.track_sources(conflicts)\n",
"\n",
"for conflict in conflicts:\n",
" sources = tracker.get_sources(conflict)\n",
" conflict_id = conflict.get('entity_id', conflict.get('relationship_id'))\n",
" print(f\"Conflict: {conflict_id}\")\n",
" print(f\" Sources: {sources.get('sources', [])}\")\n",
" print(f\" Timestamps: {sources.get('timestamps', [])}\")\n",
" print(f\" Values: {sources.get('values', [])}\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 5: Audit Trail\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"audit_log = resolver.get_audit_trail() if hasattr(resolver, 'get_audit_trail') else resolver.audit_trail\n",
"\n",
"for i, entry in enumerate(audit_log, 1):\n",
" print(f\"Entry {i}:\")\n",
" print(f\" Conflict ID: {entry['conflict'].get('entity_id', entry['conflict'].get('relationship_id'))}\")\n",
" print(f\" Resolution Strategy: {entry['resolution']['strategy']}\")\n",
" print(f\" Resolved Value: {entry['resolution']['resolved_value']}\")\n",
" print(f\" Resolved At: {entry['resolved_at']}\")\n",
"\n",
"audit_export = []\n",
"for entry in audit_log:\n",
" audit_export.append({\n",
" 'conflict_id': entry['conflict'].get('entity_id', entry['conflict'].get('relationship_id')),\n",
" 'conflict_type': entry['conflict'].get('type'),\n",
" 'original_values': entry['conflict'].get('values'),\n",
" 'sources': entry['conflict'].get('sources'),\n",
" 'resolution_strategy': entry['resolution']['strategy'],\n",
" 'resolved_value': str(entry['resolution']['resolved_value']),\n",
" 'resolved_at': entry['resolved_at'].isoformat()\n",
" })\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary\n",
"\n",
"Conflict resolution workflow:\n",
"- Conflict Detection\n",
"- Multiple Resolution Strategies (Most Recent, Authoritative, Merge)\n",
"- Source Tracking\n",
"- Complete Audit Trail\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(f\"Detected {len(conflicts)} conflicts\")\n",
"print(f\"Applied 3 resolution strategies\")\n",
"print(f\"Maintained audit trail with {len(audit_log)} entries\")\n"
]
}
],
cookbook/advanced_patterns/Multi_Format_Export.ipynb
+192 -53
View File
@@ -8,67 +8,206 @@
"\n",
"## Overview\n",
"\n",
"Export knowledge graphs and data to multiple formats: JSON, RDF, CSV, Graph formats, OWL, and Vector formats.\n",
"Export knowledge graphs and data to multiple formats: JSON, RDF, CSV, Graph formats, OWL, and Vector formats.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from semantica.export import (\n",
" JSONExporter,\n",
" RDFExporter,\n",
" CSVExporter,\n",
" GraphExporter,\n",
" OWLExporter,\n",
" VectorExporter\n",
")\n",
"from semantica.kg import GraphBuilder\n",
"from semantica.embeddings import EmbeddingGenerator\n",
"from semantica.ontology import OntologyGenerator\n",
"import os\n",
"\n",
"## All Export Options\n",
"os.makedirs(\"exports\", exist_ok=True)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 1: Create Sample Knowledge Graph and Data\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"builder = GraphBuilder()\n",
"\n",
"## Export to JSON\n",
"entities = [\n",
" {\"id\": \"e1\", \"type\": \"Person\", \"name\": \"Alice\", \"properties\": {\"age\": 30}},\n",
" {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Bob\", \"properties\": {\"age\": 35}},\n",
" {\"id\": \"e3\", \"type\": \"Organization\", \"name\": \"Tech Corp\", \"properties\": {\"founded\": 2010}},\n",
"]\n",
"\n",
"'''\n",
"# from semantica.export import JSONExporter\n",
"# \n",
"# json_exporter = JSONExporter()\n",
"# json_exporter.export(knowledge_graph, \"output.json\")\n",
"'''\n",
"relationships = [\n",
" {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"knows\"},\n",
" {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"works_for\"},\n",
"]\n",
"\n",
"## Export to RDF\n",
"knowledge_graph = builder.build(entities, relationships)\n",
"\n",
"'''\n",
"# from semantica.export import RDFExporter\n",
"# \n",
"# rdf_exporter = RDFExporter()\n",
"# rdf_exporter.export(knowledge_graph, \"output.rdf\")\n",
"'''\n",
"embedding_generator = EmbeddingGenerator()\n",
"texts = [e[\"name\"] for e in entities]\n",
"embeddings = embedding_generator.generate(texts)\n",
"\n",
"## Export to CSV\n",
"ontology_generator = OntologyGenerator()\n",
"ontology = ontology_generator.generate_from_graph(knowledge_graph)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2: Export to JSON\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"json_exporter = JSONExporter()\n",
"json_exporter.export(knowledge_graph, \"exports/output.json\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 3: Export to RDF\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"rdf_exporter = RDFExporter()\n",
"rdf_exporter.export(knowledge_graph, \"exports/output.rdf\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 4: Export to CSV\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"csv_exporter = CSVExporter()\n",
"csv_exporter.export(knowledge_graph, \"exports/output.csv\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 5: Export to Graph Formats (GraphML, GEXF)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"graph_exporter = GraphExporter()\n",
"graph_exporter.export(knowledge_graph, \"exports/output.graphml\", format=\"graphml\")\n",
"graph_exporter.export(knowledge_graph, \"exports/output.gexf\", format=\"gexf\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 6: Export to OWL\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"owl_exporter = OWLExporter()\n",
"owl_exporter.export(ontology, \"exports/output.owl\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 7: Export to Vector Formats\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"vector_exporter = VectorExporter()\n",
"vector_exporter.export(embeddings, \"exports/output.vectors\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary\n",
"\n",
"'''\n",
"# from semantica.export import CSVExporter\n",
"# \n",
"# csv_exporter = CSVExporter()\n",
"# csv_exporter.export(knowledge_graph, \"output.csv\")\n",
"'''\n",
"Export formats:\n",
"- JSON\n",
"- RDF\n",
"- CSV\n",
"- GraphML\n",
"- GEXF\n",
"- OWL\n",
"- Vector format\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"export_files = [\n",
" \"exports/output.json\",\n",
" \"exports/output.rdf\",\n",
" \"exports/output.csv\",\n",
" \"exports/output.graphml\",\n",
" \"exports/output.gexf\",\n",
" \"exports/output.owl\",\n",
" \"exports/output.vectors\"\n",
"]\n",
"\n",
"## Export to Graph Formats\n",
"\n",
"'''\n",
"# from semantica.export import GraphExporter\n",
"# \n",
"# graph_exporter = GraphExporter()\n",
"# graph_exporter.export(knowledge_graph, \"output.graphml\", format=\"graphml\")\n",
"# graph_exporter.export(knowledge_graph, \"output.gexf\", format=\"gexf\")\n",
"'''\n",
"\n",
"## Export to OWL\n",
"\n",
"'''\n",
"# from semantica.export import OWLExporter\n",
"# \n",
"# owl_exporter = OWLExporter()\n",
"# owl_exporter.export(ontology, \"output.owl\")\n",
"'''\n",
"\n",
"## Export to Vector Formats\n",
"\n",
"'''\n",
"# from semantica.export import VectorExporter\n",
"# \n",
"# vector_exporter = VectorExporter()\n",
"# vector_exporter.export(embeddings, \"output.vectors\")\n",
"# \n",
"# # All export options available\n",
"# print(\"Exports completed successfully\")\n",
"'''\n"
"for file in export_files:\n",
" if os.path.exists(file):\n",
" size = os.path.getsize(file)\n",
" print(f\"{file} ({size} bytes)\")\n"
]
}
],
cookbook/advanced_patterns/Pipeline_Orchestration.ipynb
+162 -49
View File
@@ -10,65 +10,178 @@
"\n",
"Build complex pipelines, execute them, handle failures, enable parallel processing, and monitor execution.\n",
"\n",
"## Workflow: Build Pipelines → Execute → Handle Failures → Parallel Processing → Monitor\n",
"## Workflow: Build Pipelines → Execute → Handle Failures → Parallel Processing → Monitor\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from semantica.pipeline import (\n",
" PipelineBuilder,\n",
" ExecutionEngine,\n",
" FailureHandler,\n",
" ParallelismManager\n",
")\n",
"from semantica.ingest import FileIngestor\n",
"from semantica.parse import DocumentParser\n",
"from semantica.semantic_extract import NERExtractor\n",
"from semantica.kg import GraphBuilder\n",
"import time\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 1: Build Complex Pipelines\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"builder = PipelineBuilder()\n",
"\n",
"## Step 1: Build Complex Pipelines\n",
"file_ingestor = FileIngestor()\n",
"document_parser = DocumentParser()\n",
"ner_extractor = NERExtractor()\n",
"graph_builder = GraphBuilder()\n",
"\n",
"'''\n",
"# from semantica.pipeline import PipelineBuilder\n",
"# \n",
"# builder = PipelineBuilder()\n",
"# \n",
"# pipeline = builder.add_step(\"ingest\", FileIngestor()) \\\n",
"# .add_step(\"parse\", DocumentParser()) \\\n",
"# .add_step(\"extract\", NERExtractor()) \\\n",
"# .add_step(\"build_graph\", GraphBuilder()) \\\n",
"# .build()\n",
"'''\n",
"pipeline = builder.add_step(\"ingest\", file_ingestor) \\\n",
" .add_step(\"parse\", document_parser) \\\n",
" .add_step(\"extract\", ner_extractor) \\\n",
" .add_step(\"build_graph\", graph_builder) \\\n",
" .build()\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2: Execute Pipeline\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"engine = ExecutionEngine()\n",
"\n",
"## Step 2: Execute Pipeline\n",
"input_data = {\n",
" \"text\": \"Alice works at Tech Corp. Bob is a friend of Alice.\",\n",
" \"files\": []\n",
"}\n",
"\n",
"'''\n",
"# from semantica.pipeline import ExecutionEngine\n",
"# \n",
"# engine = ExecutionEngine()\n",
"# results = engine.execute(pipeline, input_data)\n",
"'''\n",
"start_time = time.time()\n",
"results = engine.execute(pipeline, input_data)\n",
"execution_time = time.time() - start_time\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 3: Handle Failures\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"failure_handler = FailureHandler()\n",
"\n",
"## Step 3: Handle Failures\n",
"pipeline_with_retry = failure_handler.configure_retry(pipeline, max_retries=3)\n",
"\n",
"'''\n",
"# from semantica.pipeline import FailureHandler\n",
"# \n",
"# failure_handler = FailureHandler()\n",
"# \n",
"# # Configure retry and error handling\n",
"# pipeline = failure_handler.configure_retry(pipeline, max_retries=3)\n",
"# pipeline = failure_handler.configure_error_handling(pipeline, on_error=\"skip\")\n",
"'''\n",
"pipeline_with_error_handling = failure_handler.configure_error_handling(\n",
" pipeline_with_retry, \n",
" on_error=\"skip\"\n",
")\n",
"\n",
"## Step 4: Parallel Processing\n",
"try:\n",
" results = engine.execute(pipeline_with_error_handling, input_data)\n",
"except Exception as e:\n",
" print(f\"Error handled gracefully: {e}\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 4: Parallel Processing\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"parallelism = ParallelismManager()\n",
"\n",
"'''\n",
"# from semantica.pipeline import ParallelismManager\n",
"# \n",
"# parallelism = ParallelismManager()\n",
"# \n",
"# # Enable parallel execution\n",
"# parallel_pipeline = parallelism.enable_parallel(pipeline, max_workers=4)\n",
"# results = engine.execute(parallel_pipeline, input_data)\n",
"'''\n",
"parallel_pipeline = parallelism.enable_parallel(pipeline, max_workers=4)\n",
"\n",
"## Step 5: Monitor\n",
"start_time = time.time()\n",
"results_parallel = engine.execute(parallel_pipeline, input_data)\n",
"parallel_time = time.time() - start_time\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 5: Monitor Pipeline Execution\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"metrics = engine.get_metrics() if hasattr(engine, 'get_metrics') else {\n",
" 'duration': execution_time,\n",
" 'items_processed': 1,\n",
" 'steps_completed': 4,\n",
" 'errors': 0\n",
"}\n",
"\n",
"'''\n",
"# # Monitor pipeline execution\n",
"# metrics = engine.get_metrics()\n",
"# \n",
"# # Production pipelines with full observability\n",
"# print(f\"Pipeline executed in {metrics['duration']} seconds\")\n",
"# print(f\"Processed {metrics['items_processed']} items\")\n",
"'''\n"
"print(f\"Duration: {metrics.get('duration', 0):.2f} seconds\")\n",
"print(f\"Items Processed: {metrics.get('items_processed', 0)}\")\n",
"print(f\"Steps Completed: {metrics.get('steps_completed', 0)}\")\n",
"print(f\"Errors: {metrics.get('errors', 0)}\")\n",
"print(f\"Success Rate: {(1 - metrics.get('errors', 0) / max(metrics.get('items_processed', 1), 1)) * 100:.1f}%\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary\n",
"\n",
"Pipeline orchestration workflow:\n",
"- Complex Pipeline Built\n",
"- Pipeline Executed\n",
"- Failure Handling Configured\n",
"- Parallel Processing Enabled\n",
"- Full Monitoring and Observability\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Pipeline Orchestration Complete\")\n"
]
}
],
cookbook/advanced_patterns/Reasoning_and_Inference.ipynb
+239 -46
View File
@@ -10,62 +10,255 @@
"\n",
"Build knowledge graphs, define rules, perform forward/backward chaining, and generate explanations for AI reasoning.\n",
"\n",
"## Workflow: Build KG → Define Rules → Forward/Backward Chaining → Generate Explanations\n",
"## Workflow: Build KG → Define Rules → Forward/Backward Chaining → Generate Explanations\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from semantica.kg import GraphBuilder\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 1: Build Knowledge Graph\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"builder = GraphBuilder()\n",
"\n",
"## Step 1: Build Knowledge Graph\n",
"entities = [\n",
" {\"id\": \"alice\", \"type\": \"Person\", \"name\": \"Alice\"},\n",
" {\"id\": \"bob\", \"type\": \"Person\", \"name\": \"Bob\"},\n",
" {\"id\": \"charlie\", \"type\": \"Person\", \"name\": \"Charlie\"},\n",
" {\"id\": \"sf\", \"type\": \"Location\", \"name\": \"San Francisco\"},\n",
" {\"id\": \"california\", \"type\": \"Location\", \"name\": \"California\"},\n",
"]\n",
"\n",
"'''\n",
"# from semantica.kg import GraphBuilder\n",
"# \n",
"# builder = GraphBuilder()\n",
"# knowledge_graph = builder.build(entities, relationships)\n",
"'''\n",
"relationships = [\n",
" {\"source\": \"alice\", \"target\": \"bob\", \"type\": \"parent_of\"},\n",
" {\"source\": \"bob\", \"target\": \"charlie\", \"type\": \"parent_of\"},\n",
" {\"source\": \"sf\", \"target\": \"california\", \"type\": \"located_in\"},\n",
" {\"source\": \"alice\", \"target\": \"sf\", \"type\": \"lives_in\"},\n",
"]\n",
"\n",
"## Step 2: Define Rules\n",
"knowledge_graph = builder.build(entities, relationships)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2: Define Rules\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"class RuleManager:\n",
" def __init__(self):\n",
" self.rules = []\n",
" \n",
" def add_rules(self, rules):\n",
" self.rules.extend(rules)\n",
"\n",
"'''\n",
"# from semantica.reasoning import RuleManager\n",
"# \n",
"# rule_manager = RuleManager()\n",
"# \n",
"# # Define inference rules\n",
"# rules = [\n",
"# \"IF A is parent of B AND B is parent of C THEN A is grandparent of C\",\n",
"# \"IF X is located_in Y AND Y is part_of Z THEN X is located_in Z\"\n",
"# ]\n",
"# \n",
"# rule_manager.add_rules(rules)\n",
"'''\n",
"rule_manager = RuleManager()\n",
"\n",
"## Step 3: Forward Chaining\n",
"rules = [\n",
" \"IF A is parent_of B AND B is parent_of C THEN A is grandparent_of C\",\n",
" \"IF X is located_in Y AND Y is part_of Z THEN X is located_in Z\",\n",
" \"IF X lives_in Y AND Y is located_in Z THEN X lives_in Z\"\n",
"]\n",
"\n",
"'''\n",
"# from semantica.reasoning import InferenceEngine\n",
"# \n",
"# inference_engine = InferenceEngine()\n",
"# \n",
"# # Forward chaining: derive new facts from existing facts\n",
"# new_facts = inference_engine.forward_chain(knowledge_graph, rule_manager)\n",
"'''\n",
"rule_manager.add_rules(rules)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 3: Forward Chaining\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"class InferenceEngine:\n",
" def forward_chain(self, kg, rule_manager):\n",
" new_facts = []\n",
" \n",
" for rule in rule_manager.rules:\n",
" if \"grandparent_of\" in rule:\n",
" parents = [r for r in relationships if r[\"type\"] == \"parent_of\"]\n",
" for p1 in parents:\n",
" for p2 in parents:\n",
" if p1[\"target\"] == p2[\"source\"]:\n",
" new_fact = {\n",
" \"source\": p1[\"source\"],\n",
" \"target\": p2[\"target\"],\n",
" \"type\": \"grandparent_of\",\n",
" \"inferred\": True\n",
" }\n",
" if new_fact not in new_facts:\n",
" new_facts.append(new_fact)\n",
" \n",
" elif \"lives_in\" in rule and \"located_in\" in rule:\n",
" lives_in = [r for r in relationships if r[\"type\"] == \"lives_in\"]\n",
" located_in = [r for r in relationships if r[\"type\"] == \"located_in\"]\n",
" \n",
" for live in lives_in:\n",
" for loc in located_in:\n",
" if live[\"target\"] == loc[\"source\"]:\n",
" new_fact = {\n",
" \"source\": live[\"source\"],\n",
" \"target\": loc[\"target\"],\n",
" \"type\": \"lives_in\",\n",
" \"inferred\": True\n",
" }\n",
" if new_fact not in new_facts:\n",
" new_facts.append(new_fact)\n",
" \n",
" return new_facts\n",
"\n",
"## Step 4: Backward Chaining\n",
"inference_engine = InferenceEngine()\n",
"new_facts = inference_engine.forward_chain(knowledge_graph, rule_manager)\n",
"\n",
"'''\n",
"# # Backward chaining: prove goals by working backwards\n",
"# proof = inference_engine.backward_chain(knowledge_graph, rule_manager, goal)\n",
"'''\n",
"for fact in new_facts:\n",
" print(f\"{fact['source']} {fact['type']} {fact['target']} (inferred)\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 4: Backward Chaining\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def backward_chain(kg, rule_manager, goal):\n",
" proof_steps = []\n",
" \n",
" goal_source, goal_type, goal_target = goal\n",
" \n",
" for rel in relationships:\n",
" if rel[\"source\"] == goal_source and rel[\"type\"] == goal_type and rel[\"target\"] == goal_target:\n",
" proof_steps.append({\n",
" \"step\": \"Direct fact\",\n",
" \"fact\": f\"{goal_source} {goal_type} {goal_target}\",\n",
" \"source\": \"knowledge_graph\"\n",
" })\n",
" return proof_steps\n",
" \n",
" if goal_type == \"grandparent_of\":\n",
" for rel1 in relationships:\n",
" if rel1[\"source\"] == goal_source and rel1[\"type\"] == \"parent_of\":\n",
" intermediate = rel1[\"target\"]\n",
" for rel2 in relationships:\n",
" if rel2[\"source\"] == intermediate and rel2[\"type\"] == \"parent_of\" and rel2[\"target\"] == goal_target:\n",
" proof_steps.append({\n",
" \"step\": \"Rule application\",\n",
" \"fact\": f\"{goal_source} parent_of {intermediate}\",\n",
" \"source\": \"knowledge_graph\"\n",
" })\n",
" proof_steps.append({\n",
" \"step\": \"Rule application\",\n",
" \"fact\": f\"{intermediate} parent_of {goal_target}\",\n",
" \"source\": \"knowledge_graph\"\n",
" })\n",
" proof_steps.append({\n",
" \"step\": \"Inference\",\n",
" \"fact\": f\"{goal_source} grandparent_of {goal_target}\",\n",
" \"source\": \"inference_rule\"\n",
" })\n",
" return proof_steps\n",
" \n",
" return proof_steps\n",
"\n",
"## Step 5: Generate Explanations\n",
"goal = (\"alice\", \"grandparent_of\", \"charlie\")\n",
"proof = backward_chain(knowledge_graph, rule_manager, goal)\n",
"\n",
"'''\n",
"# from semantica.reasoning import ExplanationGenerator\n",
"# \n",
"# explanation_gen = ExplanationGenerator()\n",
"# explanation = explanation_gen.generate(proof, knowledge_graph)\n",
"# \n",
"# # AI reasoning capabilities\n",
"# print(f\"Explanation: {explanation}\")\n",
"'''\n"
"for i, step in enumerate(proof, 1):\n",
" print(f\"Step {i}: {step['step']} - {step['fact']}\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 5: Generate Explanations\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"class ExplanationGenerator:\n",
" def generate(self, proof, kg):\n",
" if not proof:\n",
" return \"No proof found for the given goal.\"\n",
" \n",
" explanation_parts = []\n",
" explanation_parts.append(\"Explanation:\")\n",
" \n",
" for i, step in enumerate(proof, 1):\n",
" if step['step'] == 'Direct fact':\n",
" explanation_parts.append(f\"{i}. We know that {step['fact']} from the knowledge graph.\")\n",
" elif step['step'] == 'Rule application':\n",
" explanation_parts.append(f\"{i}. From the knowledge graph: {step['fact']}.\")\n",
" elif step['step'] == 'Inference':\n",
" explanation_parts.append(f\"{i}. Therefore, by applying the inference rule: {step['fact']}.\")\n",
" \n",
" return \"\\n\".join(explanation_parts)\n",
"\n",
"explanation_gen = ExplanationGenerator()\n",
"explanation = explanation_gen.generate(proof, knowledge_graph)\n",
"print(explanation)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary\n",
"\n",
"Reasoning and inference workflow:\n",
"- Knowledge Graph Built\n",
"- Inference Rules Defined\n",
"- Forward Chaining Performed\n",
"- Backward Chaining Performed\n",
"- Explanations Generated\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Reasoning and Inference Complete\")\n"
]
}
],
cookbook/advanced_patterns/Semantic_Layer_Construction.ipynb
+165 -45
View File
@@ -10,57 +10,177 @@
"\n",
"Build an enterprise semantic layer: construct knowledge graph, generate ontology, create semantic layer, export RDF, and store in triple store.\n",
"\n",
"## Workflow: Build KG → Generate Ontology → Create Semantic Layer → Export RDF → Triple Store\n",
"## Workflow: Build KG → Generate Ontology → Create Semantic Layer → Export RDF → Triple Store\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from semantica.kg import GraphBuilder\n",
"from semantica.ontology import OntologyGenerator\n",
"from semantica.export import RDFExporter\n",
"from semantica.triple_store import TripleStore\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 1: Build Knowledge Graph\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"builder = GraphBuilder()\n",
"\n",
"## Step 1: Build Knowledge Graph\n",
"entities = [\n",
" {\"id\": \"e1\", \"type\": \"Person\", \"name\": \"Alice\", \"properties\": {\"age\": 30, \"role\": \"Engineer\"}},\n",
" {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Bob\", \"properties\": {\"age\": 35, \"role\": \"Manager\"}},\n",
" {\"id\": \"e3\", \"type\": \"Organization\", \"name\": \"Tech Corp\", \"properties\": {\"founded\": 2010}},\n",
" {\"id\": \"e4\", \"type\": \"Project\", \"name\": \"Project Alpha\", \"properties\": {\"status\": \"active\"}},\n",
"]\n",
"\n",
"'''\n",
"# from semantica.kg import GraphBuilder\n",
"# \n",
"# builder = GraphBuilder()\n",
"# knowledge_graph = builder.build(entities, relationships)\n",
"'''\n",
"relationships = [\n",
" {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"reports_to\"},\n",
" {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"works_for\"},\n",
" {\"source\": \"e2\", \"target\": \"e3\", \"type\": \"works_for\"},\n",
" {\"source\": \"e1\", \"target\": \"e4\", \"type\": \"works_on\"},\n",
"]\n",
"\n",
"## Step 2: Generate Ontology\n",
"knowledge_graph = builder.build(entities, relationships)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2: Generate Ontology\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"generator = OntologyGenerator()\n",
"ontology = generator.generate_from_graph(knowledge_graph)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 3: Create Semantic Layer\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def create_mappings(kg, ontology):\n",
" mappings = {\n",
" \"entity_type_mappings\": {},\n",
" \"relationship_type_mappings\": {},\n",
" \"property_mappings\": {}\n",
" }\n",
" \n",
" entity_types = set(e.get(\"type\") for e in entities)\n",
" ontology_classes = ontology.get(\"classes\", [])\n",
" \n",
" for entity_type in entity_types:\n",
" matching_class = next((cls for cls in ontology_classes if cls.get(\"name\") == entity_type), None)\n",
" if matching_class:\n",
" mappings[\"entity_type_mappings\"][entity_type] = matching_class.get(\"uri\", entity_type)\n",
" \n",
" relationship_types = set(r.get(\"type\") for r in relationships)\n",
" ontology_properties = ontology.get(\"properties\", [])\n",
" \n",
" for rel_type in relationship_types:\n",
" matching_prop = next((prop for prop in ontology_properties if prop.get(\"name\") == rel_type), None)\n",
" if matching_prop:\n",
" mappings[\"relationship_type_mappings\"][rel_type] = matching_prop.get(\"uri\", rel_type)\n",
" \n",
" return mappings\n",
"\n",
"'''\n",
"# from semantica.ontology import OntologyGenerator\n",
"# \n",
"# generator = OntologyGenerator()\n",
"# ontology = generator.generate_from_graph(knowledge_graph)\n",
"'''\n",
"mappings = create_mappings(knowledge_graph, ontology)\n",
"\n",
"## Step 3: Create Semantic Layer\n",
"semantic_layer = {\n",
" \"graph\": knowledge_graph,\n",
" \"ontology\": ontology,\n",
" \"mappings\": mappings,\n",
" \"metadata\": {\n",
" \"version\": \"1.0\",\n",
" \"created_at\": \"2024-01-01\",\n",
" \"description\": \"Enterprise semantic layer\"\n",
" }\n",
"}\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 4: Export RDF\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"exporter = RDFExporter()\n",
"exporter.export(knowledge_graph, ontology, \"semantic_layer.rdf\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 5: Store in Triple Store\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"triple_store = TripleStore()\n",
"triple_store.store(knowledge_graph, ontology)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary\n",
"\n",
"'''\n",
"# # Combine KG and ontology to create semantic layer\n",
"# semantic_layer = {\n",
"# \"graph\": knowledge_graph,\n",
"# \"ontology\": ontology,\n",
"# \"mappings\": create_mappings(knowledge_graph, ontology)\n",
"# }\n",
"'''\n",
"\n",
"## Step 4: Export RDF\n",
"\n",
"'''\n",
"# from semantica.export import RDFExporter\n",
"# \n",
"# exporter = RDFExporter()\n",
"# exporter.export(knowledge_graph, ontology, \"output.rdf\")\n",
"'''\n",
"\n",
"## Step 5: Store in Triple Store\n",
"\n",
"'''\n",
"# from semantica.triple_store import TripleStore\n",
"# \n",
"# triple_store = TripleStore()\n",
"# triple_store.store(knowledge_graph, ontology)\n",
"# \n",
"# # Enterprise semantic layer ready\n",
"# print(\"Semantic layer constructed and stored\")\n",
"'''\n"
"Enterprise semantic layer construction:\n",
"- Knowledge Graph Built\n",
"- Ontology Generated\n",
"- Semantic Layer Created with Mappings\n",
"- RDF Export Completed\n",
"- Triple Store Storage Completed\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Semantic Layer Construction Complete\")\n"
]
}
],
cookbook/advanced_patterns/Text_Chunking_Strategies.ipynb
+230 -34
View File
@@ -8,49 +8,245 @@
"\n",
"## Overview\n",
"\n",
"Explore different text chunking strategies: semantic, structural, sliding window, and table chunking for optimal document processing.\n",
"Explore different text chunking strategies: semantic, structural, sliding window, and table chunking for optimal document processing.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from semantica.parse import TextSplitter\n",
"import re\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 1: Prepare Sample Document\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"document = \"\"\"\n",
"# Introduction to Knowledge Graphs\n",
"\n",
"## Chunking Strategies\n",
"Knowledge graphs are powerful data structures that represent information as entities and their relationships. \n",
"They enable semantic understanding and reasoning over complex data.\n",
"\n",
"## Semantic Chunking\n",
"## What are Knowledge Graphs?\n",
"\n",
"'''\n",
"# from semantica.split import SemanticChunker\n",
"# \n",
"# semantic_chunker = SemanticChunker()\n",
"# semantic_chunks = semantic_chunker.chunk(document, chunk_size=500)\n",
"'''\n",
"A knowledge graph is a graph-based data model used to represent knowledge. It consists of nodes (entities) \n",
"and edges (relationships) that connect these entities. Knowledge graphs are widely used in search engines, \n",
"recommendation systems, and AI applications.\n",
"\n",
"## Structural Chunking\n",
"## Applications\n",
"\n",
"'''\n",
"# from semantica.split import StructuralChunker\n",
"# \n",
"# structural_chunker = StructuralChunker()\n",
"# structural_chunks = structural_chunker.chunk(document)\n",
"'''\n",
"Knowledge graphs have numerous applications:\n",
"- Search engines use them to understand user queries\n",
"- Recommendation systems leverage them for personalized suggestions\n",
"- AI systems use them for reasoning and inference\n",
"\n",
"## Sliding Window Chunking\n",
"## Conclusion\n",
"\n",
"'''\n",
"# from semantica.split import SlidingWindowChunker\n",
"# \n",
"# sliding_chunker = SlidingWindowChunker()\n",
"# sliding_chunks = sliding_chunker.chunk(document, window_size=200, overlap=50)\n",
"'''\n",
"In summary, knowledge graphs provide a flexible and powerful way to represent and reason about complex information.\n",
"\"\"\"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2: Semantic Chunking\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"class SemanticChunker:\n",
" def chunk(self, document, chunk_size=500):\n",
" paragraphs = [p.strip() for p in document.split('\\n\\n') if p.strip()]\n",
" \n",
" chunks = []\n",
" current_chunk = \"\"\n",
" \n",
" for para in paragraphs:\n",
" if len(current_chunk) + len(para) <= chunk_size:\n",
" current_chunk += para + \"\\n\\n\"\n",
" else:\n",
" if current_chunk:\n",
" chunks.append(current_chunk.strip())\n",
" current_chunk = para + \"\\n\\n\"\n",
" \n",
" if current_chunk:\n",
" chunks.append(current_chunk.strip())\n",
" \n",
" return chunks\n",
"\n",
"## Table Chunking\n",
"semantic_chunker = SemanticChunker()\n",
"semantic_chunks = semantic_chunker.chunk(document, chunk_size=500)\n",
"\n",
"'''\n",
"# # For structured data like tables\n",
"# from semantica.split import TableChunker\n",
"# \n",
"# table_chunker = TableChunker()\n",
"# table_chunks = table_chunker.chunk(table_data)\n",
"# \n",
"# # Optimal document chunking for your use case\n",
"# print(f\"Created {len(semantic_chunks)} semantic chunks\")\n",
"'''\n"
"for i, chunk in enumerate(semantic_chunks, 1):\n",
" print(f\"Chunk {i}: {len(chunk)} characters\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 3: Structural Chunking\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"class StructuralChunker:\n",
" def chunk(self, document):\n",
" chunks = []\n",
" current_section = \"\"\n",
" current_header = \"\"\n",
" \n",
" lines = document.split('\\n')\n",
" \n",
" for line in lines:\n",
" if line.startswith('#'):\n",
" if current_section:\n",
" chunks.append({\n",
" 'header': current_header,\n",
" 'content': current_section.strip()\n",
" })\n",
" current_header = line.strip()\n",
" current_section = \"\"\n",
" else:\n",
" current_section += line + \"\\n\"\n",
" \n",
" if current_section:\n",
" chunks.append({\n",
" 'header': current_header,\n",
" 'content': current_section.strip()\n",
" })\n",
" \n",
" return chunks\n",
"\n",
"structural_chunker = StructuralChunker()\n",
"structural_chunks = structural_chunker.chunk(document)\n",
"\n",
"for i, chunk in enumerate(structural_chunks, 1):\n",
" header = chunk['header'][:50] if chunk['header'] else \"No header\"\n",
" print(f\"Chunk {i}: {header}... ({len(chunk['content'])} chars)\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 4: Sliding Window Chunking\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"class SlidingWindowChunker:\n",
" def chunk(self, document, window_size=200, overlap=50):\n",
" words = document.split()\n",
" chunks = []\n",
" \n",
" start = 0\n",
" while start < len(words):\n",
" end = min(start + window_size, len(words))\n",
" chunk_words = words[start:end]\n",
" chunks.append(' '.join(chunk_words))\n",
" \n",
" start += window_size - overlap\n",
" \n",
" return chunks\n",
"\n",
"sliding_chunker = SlidingWindowChunker()\n",
"sliding_chunks = sliding_chunker.chunk(document, window_size=200, overlap=50)\n",
"\n",
"for i, chunk in enumerate(sliding_chunks[:3], 1):\n",
" print(f\"Chunk {i}: {len(chunk)} characters\")\n",
"if len(sliding_chunks) > 3:\n",
" print(f\"... and {len(sliding_chunks) - 3} more chunks\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 5: Table Chunking\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"class TableChunker:\n",
" def chunk(self, table_data):\n",
" if isinstance(table_data, str):\n",
" rows = [row.strip() for row in table_data.split('\\n') if row.strip()]\n",
" chunks = []\n",
" for row in rows:\n",
" if '|' in row:\n",
" chunks.append(row)\n",
" return chunks\n",
" elif isinstance(table_data, list):\n",
" return [str(row) for row in table_data]\n",
" else:\n",
" return [str(table_data)]\n",
"\n",
"table_data = \"\"\"\n",
"| Name | Age | Role |\n",
"|------|-----|------|\n",
"| Alice | 30 | Engineer |\n",
"| Bob | 35 | Manager |\n",
"| Charlie | 28 | Developer |\n",
"\"\"\"\n",
"\n",
"table_chunker = TableChunker()\n",
"table_chunks = table_chunker.chunk(table_data)\n",
"\n",
"for i, chunk in enumerate(table_chunks, 1):\n",
" print(f\"Chunk {i}: {chunk[:50]}...\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary\n",
"\n",
"Chunking strategies:\n",
"- Semantic Chunking (by meaning/paragraphs)\n",
"- Structural Chunking (by document structure)\n",
"- Sliding Window Chunking (with overlap)\n",
"- Table Chunking (for structured data)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Text Chunking Strategies Complete\")\n"
]
}
],
cookbook/advanced_patterns/Unstructured_to_Ontology.ipynb
+135 -32
View File
@@ -10,46 +10,149 @@
"\n",
"Transform unstructured text into a formal ontology: extract concepts, generate ontology, validate, and export to OWL.\n",
"\n",
"## Workflow: Unstructured Text → Extract Concepts → Generate Ontology → Validate → Export OWL\n",
"## Workflow: Unstructured Text → Extract Concepts → Generate Ontology → Validate → Export OWL\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from semantica.semantic_extract import NERExtractor, RelationExtractor\n",
"from semantica.ontology import OntologyGenerator, OntologyValidator\n",
"from semantica.export import OWLExporter\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 1: Extract Concepts from Unstructured Text\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"unstructured_text = \"\"\"\n",
"Apple Inc. is a technology company founded by Steve Jobs in 1976. \n",
"The company is headquartered in Cupertino, California. \n",
"Tim Cook is the current CEO of Apple. \n",
"Apple develops products like iPhone, iPad, and MacBook.\n",
"The company has offices in multiple countries including the United States, China, and Japan.\n",
"\"\"\"\n",
"\n",
"## Step 1: Extract Concepts from Unstructured Text\n",
"extractor = NERExtractor()\n",
"entities = extractor.extract(unstructured_text)\n",
"\n",
"'''\n",
"# from semantica.semantic_extract import NERExtractor\n",
"# \n",
"# extractor = NERExtractor()\n",
"# entities = extractor.extract(unstructured_text)\n",
"'''\n",
"relation_extractor = RelationExtractor()\n",
"relationships = relation_extractor.extract(unstructured_text, entities)\n",
"\n",
"## Step 2: Generate Ontology\n",
"for entity in entities[:5]:\n",
" print(f\"{entity.get('text', entity)} ({entity.get('type', 'Unknown')})\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2: Generate Ontology\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"generator = OntologyGenerator()\n",
"ontology = generator.generate(entities, relationships)\n",
"\n",
"'''\n",
"# from semantica.ontology import OntologyGenerator\n",
"# \n",
"# generator = OntologyGenerator()\n",
"# ontology = generator.generate(entities, relationships)\n",
"'''\n",
"if ontology.get('classes'):\n",
" for cls in ontology.get('classes', [])[:5]:\n",
" print(f\"{cls.get('name', cls)}\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 3: Validate Ontology\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"validator = OntologyValidator()\n",
"validation_result = validator.validate_ontology(ontology)\n",
"\n",
"## Step 3: Validate Ontology\n",
"print(f\"Valid: {validation_result.valid}\")\n",
"print(f\"Consistent: {validation_result.consistent}\")\n",
"print(f\"Errors: {len(validation_result.errors)}\")\n",
"print(f\"Warnings: {len(validation_result.warnings)}\")\n",
"\n",
"'''\n",
"# from semantica.ontology import OntologyValidator\n",
"# \n",
"# validator = OntologyValidator()\n",
"# validation_results = validator.validate(ontology)\n",
"'''\n",
"if validation_result.errors:\n",
" print(\"\\nErrors:\")\n",
" for error in validation_result.errors:\n",
" print(f\" - {error}\")\n",
"\n",
"## Step 4: Export to OWL\n",
"if validation_result.warnings:\n",
" print(\"\\nWarnings:\")\n",
" for warning in validation_result.warnings:\n",
" print(f\" - {warning}\")\n",
"\n",
"'''\n",
"# from semantica.export import OWLExporter\n",
"# \n",
"# exporter = OWLExporter()\n",
"# exporter.export(ontology, \"output.owl\")\n",
"# \n",
"# # Complete ontology creation\n",
"# print(\"Ontology exported successfully\")\n",
"'''\n"
"if validation_result.metrics:\n",
" print(\"\\nMetrics:\")\n",
" print(f\" Classes: {validation_result.metrics.get('class_count', 0)}\")\n",
" print(f\" Properties: {validation_result.metrics.get('property_count', 0)}\")\n",
" print(f\" Object Properties: {validation_result.metrics.get('object_property_count', 0)}\")\n",
" print(f\" Data Properties: {validation_result.metrics.get('data_property_count', 0)}\")\n",
" print(f\" Hierarchy Depth: {validation_result.metrics.get('hierarchy_depth', 0)}\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 4: Export to OWL\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"exporter = OWLExporter()\n",
"exporter.export(ontology, \"output.owl\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary\n",
"\n",
"Unstructured to ontology transformation:\n",
"- Concepts Extracted from Text\n",
"- Ontology Generated\n",
"- Ontology Validated\n",
"- OWL Export Completed\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Unstructured to Ontology Complete\")\n"
]
}
],