diff --git a/cookbook/advanced_patterns/Complete_Visualization_Suite.ipynb b/cookbook/advanced_patterns/Complete_Visualization_Suite.ipynb index 13f22f94..1f6e5f15 100644 --- a/cookbook/advanced_patterns/Complete_Visualization_Suite.ipynb +++ b/cookbook/advanced_patterns/Complete_Visualization_Suite.ipynb @@ -8,59 +8,235 @@ "\n", "## Overview\n", "\n", - "Comprehensive visualization capabilities: visualize knowledge graphs, embeddings, quality metrics, analytics, and temporal data.\n", + "Comprehensive visualization capabilities: visualize knowledge graphs, embeddings, quality metrics, analytics, and temporal data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.visualization import (\n", + " KGVisualizer,\n", + " EmbeddingVisualizer,\n", + " QualityVisualizer,\n", + " AnalyticsVisualizer,\n", + " TemporalVisualizer\n", + ")\n", + "from semantica.kg import GraphBuilder, GraphAnalyzer\n", + "from semantica.embeddings import EmbeddingGenerator\n", + "from semantica.kg_qa import KGQualityAssessor\n", + "import numpy as np\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Create Sample Knowledge Graph\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", "\n", - "## Visualization Types\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Person\", \"name\": \"Alice\", \"properties\": {\"age\": 30}},\n", + " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Bob\", \"properties\": {\"age\": 35}},\n", + " {\"id\": \"e3\", \"type\": \"Organization\", \"name\": \"Tech Corp\", \"properties\": {\"founded\": 2010}},\n", + " {\"id\": \"e4\", \"type\": \"Location\", \"name\": \"San Francisco\", \"properties\": {\"country\": \"USA\"}},\n", + "]\n", "\n", - "## Knowledge Graph Visualization\n", + "relationships = [\n", + " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"knows\", \"properties\": {\"since\": 2020}},\n", + " {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"works_for\", \"properties\": {\"role\": \"Engineer\"}},\n", + " {\"source\": \"e3\", \"target\": \"e4\", \"type\": \"located_in\", \"properties\": {}},\n", + "]\n", "\n", - "'''\n", - "# from semantica.visualization import KGVisualizer\n", - "# \n", - "# kg_visualizer = KGVisualizer()\n", - "# kg_visualizer.visualize(knowledge_graph)\n", - "'''\n", + "knowledge_graph = builder.build(entities, relationships)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Knowledge Graph Visualization\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kg_visualizer = KGVisualizer()\n", + "kg_visualizer.visualize(knowledge_graph, layout=\"spring\", show_labels=True)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Generate Embeddings and Visualize\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "embedding_generator = EmbeddingGenerator()\n", + "texts = [entity.get(\"name\", \"\") for entity in entities]\n", + "embeddings = embedding_generator.generate(texts)\n", "\n", - "## Embedding Visualization\n", + "labels = [entity.get(\"type\", \"Unknown\") for entity in entities]\n", "\n", - "'''\n", - "# from semantica.visualization import EmbeddingVisualizer\n", - "# \n", - "# embedding_visualizer = EmbeddingVisualizer()\n", - "# embedding_visualizer.visualize_tsne(embeddings, labels)\n", - "'''\n", + "embedding_visualizer = EmbeddingVisualizer()\n", + "embedding_visualizer.visualize_tsne(embeddings, labels, title=\"Entity Embeddings Visualization\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Quality Metrics Visualization\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quality_assessor = KGQualityAssessor()\n", + "quality_metrics = quality_assessor.assess(knowledge_graph)\n", "\n", - "## Quality Visualization\n", + "quality_visualizer = QualityVisualizer()\n", + "quality_visualizer.visualize_metrics(quality_metrics, title=\"Knowledge Graph Quality Metrics\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Graph Analytics Visualization\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_analyzer = GraphAnalyzer()\n", "\n", - "'''\n", - "# from semantica.visualization import QualityVisualizer\n", - "# \n", - "# quality_visualizer = QualityVisualizer()\n", - "# quality_visualizer.visualize_metrics(quality_metrics)\n", - "'''\n", + "centrality_results = graph_analyzer.calculate_centrality(\n", + " knowledge_graph, \n", + " centrality_type=\"degree\"\n", + ")\n", "\n", - "## Analytics Visualization\n", + "centrality_scores = {}\n", + "if centrality_results and \"centrality_measures\" in centrality_results:\n", + " degree_centrality = centrality_results[\"centrality_measures\"].get(\"degree\", {})\n", + " if isinstance(degree_centrality, dict) and \"centrality\" in degree_centrality:\n", + " centrality_scores = degree_centrality[\"centrality\"]\n", + " elif isinstance(degree_centrality, dict):\n", + " centrality_scores = degree_centrality\n", "\n", - "'''\n", - "# from semantica.visualization import AnalyticsVisualizer\n", - "# \n", - "# analytics_visualizer = AnalyticsVisualizer()\n", - "# analytics_visualizer.visualize_centrality(centrality_scores)\n", - "# analytics_visualizer.visualize_communities(communities)\n", - "'''\n", + "communities_result = graph_analyzer.detect_communities(\n", + " knowledge_graph, \n", + " algorithm=\"louvain\"\n", + ")\n", "\n", - "## Temporal Data Visualization\n", + "communities = []\n", + "community_dict = {}\n", + "if communities_result and \"communities\" in communities_result:\n", + " communities_data = communities_result[\"communities\"]\n", + " if isinstance(communities_data, list):\n", + " communities = communities_data\n", + " for idx, community in enumerate(communities):\n", + " if isinstance(community, list):\n", + " for node in community:\n", + " community_dict[node] = idx\n", + " elif isinstance(community, dict) and \"nodes\" in community:\n", + " for node in community[\"nodes\"]:\n", + " community_dict[node] = idx\n", "\n", - "'''\n", - "# from semantica.visualization import TemporalVisualizer\n", - "# \n", - "# temporal_visualizer = TemporalVisualizer()\n", - "# temporal_visualizer.visualize_timeline(temporal_kg)\n", - "# temporal_visualizer.visualize_evolution(entity_history)\n", - "# \n", - "# # Comprehensive visualization\n", - "# print(\"All visualizations generated\")\n", - "'''\n" + "analytics_visualizer = AnalyticsVisualizer()\n", + "analytics_visualizer.visualize_centrality(centrality_scores, title=\"Node Centrality Scores\")\n", + "\n", + "if community_dict:\n", + " analytics_visualizer.visualize_communities(\n", + " knowledge_graph, \n", + " community_dict, \n", + " title=\"Community Detection\"\n", + " )\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Temporal Data Visualization\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "temporal_kg = {\n", + " \"entities\": entities,\n", + " \"relationships\": relationships,\n", + " \"timestamps\": {\n", + " \"e1\": [2020, 2021, 2022],\n", + " \"e2\": [2020, 2021],\n", + " \"e3\": [2010, 2015, 2020, 2022],\n", + " }\n", + "}\n", + "\n", + "entity_history = {\n", + " \"e1\": [\n", + " {\"timestamp\": 2020, \"properties\": {\"age\": 28}},\n", + " {\"timestamp\": 2021, \"properties\": {\"age\": 29}},\n", + " {\"timestamp\": 2022, \"properties\": {\"age\": 30}},\n", + " ]\n", + "}\n", + "\n", + "temporal_visualizer = TemporalVisualizer()\n", + "temporal_visualizer.visualize_timeline(temporal_kg, title=\"Temporal Knowledge Graph Timeline\")\n", + "temporal_visualizer.visualize_evolution(entity_history, entity_id=\"e1\", title=\"Entity Evolution\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "All visualization types demonstrated:\n", + "- Knowledge Graph Visualization\n", + "- Embedding Visualization (t-SNE)\n", + "- Quality Metrics Visualization\n", + "- Graph Analytics Visualization (Centrality & Communities)\n", + "- Temporal Data Visualization (Timeline & Evolution)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Complete Visualization Suite\")\n", + "print(\"All visualizations generated successfully\")\n" ] } ], diff --git a/cookbook/advanced_patterns/Conflict_Resolution_Strategies.ipynb b/cookbook/advanced_patterns/Conflict_Resolution_Strategies.ipynb index d6f44ebc..3cb560f4 100644 --- a/cookbook/advanced_patterns/Conflict_Resolution_Strategies.ipynb +++ b/cookbook/advanced_patterns/Conflict_Resolution_Strategies.ipynb @@ -10,58 +10,296 @@ "\n", "Detect conflicts in knowledge graphs, apply multiple resolution strategies, track sources, and maintain audit trails.\n", "\n", - "## Workflow: Detect Conflicts → Multiple Resolution Strategies → Track Sources → Audit\n", + "## Workflow: Detect Conflicts → Multiple Resolution Strategies → Track Sources → Audit\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder\n", + "from semantica.kg_qa import ConsistencyChecker\n", + "from datetime import datetime\n", + "import json\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Create Knowledge Graph with Conflicting Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", "\n", - "## Step 1: Detect Conflicts\n", + "entities = [\n", + " {\n", + " \"id\": \"e1\",\n", + " \"type\": \"Person\",\n", + " \"name\": \"John Doe\",\n", + " \"properties\": {\"age\": 30, \"location\": \"New York\"},\n", + " \"source\": \"source1\",\n", + " \"timestamp\": datetime(2023, 1, 1)\n", + " },\n", + " {\n", + " \"id\": \"e1\",\n", + " \"type\": \"Person\",\n", + " \"name\": \"John Doe\",\n", + " \"properties\": {\"age\": 32, \"location\": \"Boston\"},\n", + " \"source\": \"source2\",\n", + " \"timestamp\": datetime(2023, 6, 1)\n", + " },\n", + " {\n", + " \"id\": \"e2\",\n", + " \"type\": \"Organization\",\n", + " \"name\": \"Tech Corp\",\n", + " \"properties\": {\"founded\": 2010, \"employees\": 100},\n", + " \"source\": \"source1\",\n", + " \"timestamp\": datetime(2023, 1, 1)\n", + " },\n", + " {\n", + " \"id\": \"e2\",\n", + " \"type\": \"Organization\",\n", + " \"name\": \"Tech Corp\",\n", + " \"properties\": {\"founded\": 2012, \"employees\": 150},\n", + " \"source\": \"source2\",\n", + " \"timestamp\": datetime(2023, 3, 1)\n", + " },\n", + "]\n", "\n", - "'''\n", - "# from semantica.conflicts import ConflictDetector\n", - "# \n", - "# detector = ConflictDetector()\n", - "# conflicts = detector.detect(knowledge_graph)\n", - "# \n", - "# print(f\"Found {len(conflicts)} conflicts\")\n", - "'''\n", + "relationships = [\n", + " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"works_for\", \"source\": \"source1\"},\n", + " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"founder_of\", \"source\": \"source2\"},\n", + "]\n", "\n", - "## Step 2: Multiple Resolution Strategies\n", + "knowledge_graph = builder.build(entities, relationships)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Detect Conflicts\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "consistency_checker = ConsistencyChecker()\n", + "conflicts = consistency_checker.check_conflicts(knowledge_graph)\n", "\n", - "'''\n", - "# from semantica.conflicts import ConflictResolver\n", - "# \n", - "# resolver = ConflictResolver()\n", - "# \n", - "# # Strategy 1: Most recent wins\n", - "# resolved_1 = resolver.resolve(conflicts, strategy=\"most_recent\")\n", - "# \n", - "# # Strategy 2: Most authoritative source wins\n", - "# resolved_2 = resolver.resolve(conflicts, strategy=\"authoritative\")\n", - "# \n", - "# # Strategy 3: Merge conflicting information\n", - "# resolved_3 = resolver.resolve(conflicts, strategy=\"merge\")\n", - "'''\n", + "for i, conflict in enumerate(conflicts, 1):\n", + " print(f\"Conflict {i}:\")\n", + " print(f\" Entity/Relationship: {conflict.get('entity_id', conflict.get('relationship_id'))}\")\n", + " print(f\" Type: {conflict.get('type')}\")\n", + " print(f\" Conflicting values: {conflict.get('values')}\")\n", + " print(f\" Sources: {conflict.get('sources')}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Multiple Resolution Strategies\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class ConflictResolver:\n", + " def __init__(self):\n", + " self.audit_trail = []\n", + " \n", + " def resolve(self, conflicts, strategy=\"most_recent\"):\n", + " resolved = []\n", + " \n", + " for conflict in conflicts:\n", + " if strategy == \"most_recent\":\n", + " values = conflict.get('values', [])\n", + " timestamps = conflict.get('timestamps', [])\n", + " if timestamps:\n", + " most_recent_idx = timestamps.index(max(timestamps))\n", + " resolved_value = values[most_recent_idx]\n", + " else:\n", + " resolved_value = values[-1] if values else None\n", + " \n", + " elif strategy == \"authoritative\":\n", + " sources = conflict.get('sources', [])\n", + " authoritative_sources = [\"source1\", \"official_db\", \"verified\"]\n", + " resolved_value = None\n", + " for auth_source in authoritative_sources:\n", + " if auth_source in sources:\n", + " idx = sources.index(auth_source)\n", + " resolved_value = conflict.get('values', [])[idx]\n", + " break\n", + " if resolved_value is None:\n", + " resolved_value = conflict.get('values', [])[0] if conflict.get('values') else None\n", + " \n", + " elif strategy == \"merge\":\n", + " values = conflict.get('values', [])\n", + " if isinstance(values[0], dict):\n", + " merged = {}\n", + " for val in values:\n", + " merged.update(val)\n", + " resolved_value = merged\n", + " elif isinstance(values[0], (int, float)):\n", + " resolved_value = sum(values) / len(values)\n", + " else:\n", + " resolved_value = \", \".join(set(str(v) for v in values))\n", + " else:\n", + " resolved_value = conflict.get('values', [])[0] if conflict.get('values') else None\n", + " \n", + " resolved.append({\n", + " 'conflict_id': conflict.get('entity_id', conflict.get('relationship_id')),\n", + " 'resolved_value': resolved_value,\n", + " 'strategy': strategy,\n", + " 'timestamp': datetime.now()\n", + " })\n", + " \n", + " self.audit_trail.append({\n", + " 'conflict': conflict,\n", + " 'resolution': resolved[-1],\n", + " 'resolved_at': datetime.now()\n", + " })\n", + " \n", + " return resolved\n", "\n", - "## Step 3: Track Sources\n", + "resolver = ConflictResolver()\n", "\n", - "'''\n", - "# from semantica.conflicts import SourceTracker\n", - "# \n", - "# tracker = SourceTracker()\n", - "# \n", - "# # Track where each piece of information came from\n", - "# for conflict in conflicts:\n", - "# sources = tracker.get_sources(conflict)\n", - "# print(f\"Conflict sources: {sources}\")\n", - "'''\n", + "resolved_1 = resolver.resolve(conflicts, strategy=\"most_recent\")\n", + "print(\"Strategy 1: Most Recent Wins\")\n", + "for r in resolved_1:\n", + " print(f\" Resolved: {r['conflict_id']} = {r['resolved_value']}\")\n", "\n", - "## Step 4: Audit\n", + "resolver2 = ConflictResolver()\n", + "resolved_2 = resolver2.resolve(conflicts, strategy=\"authoritative\")\n", + "print(\"\\nStrategy 2: Most Authoritative Source Wins\")\n", + "for r in resolved_2:\n", + " print(f\" Resolved: {r['conflict_id']} = {r['resolved_value']}\")\n", "\n", - "'''\n", - "# # Maintain audit trail of all resolutions\n", - "# audit_log = tracker.get_audit_trail()\n", - "# \n", - "# # Handle data conflicts with full traceability\n", - "# print(f\"Audit log contains {len(audit_log)} entries\")\n", - "'''\n" + "resolver3 = ConflictResolver()\n", + "resolved_3 = resolver3.resolve(conflicts, strategy=\"merge\")\n", + "print(\"\\nStrategy 3: Merge Conflicting Information\")\n", + "for r in resolved_3:\n", + " print(f\" Resolved: {r['conflict_id']} = {r['resolved_value']}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Track Sources\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class SourceTracker:\n", + " def __init__(self):\n", + " self.source_map = {}\n", + " \n", + " def track_sources(self, conflicts):\n", + " for conflict in conflicts:\n", + " conflict_id = conflict.get('entity_id', conflict.get('relationship_id'))\n", + " sources = conflict.get('sources', [])\n", + " timestamps = conflict.get('timestamps', [])\n", + " \n", + " self.source_map[conflict_id] = {\n", + " 'sources': sources,\n", + " 'timestamps': timestamps,\n", + " 'values': conflict.get('values', [])\n", + " }\n", + " \n", + " def get_sources(self, conflict):\n", + " conflict_id = conflict.get('entity_id', conflict.get('relationship_id'))\n", + " return self.source_map.get(conflict_id, {})\n", + "\n", + "tracker = SourceTracker()\n", + "tracker.track_sources(conflicts)\n", + "\n", + "for conflict in conflicts:\n", + " sources = tracker.get_sources(conflict)\n", + " conflict_id = conflict.get('entity_id', conflict.get('relationship_id'))\n", + " print(f\"Conflict: {conflict_id}\")\n", + " print(f\" Sources: {sources.get('sources', [])}\")\n", + " print(f\" Timestamps: {sources.get('timestamps', [])}\")\n", + " print(f\" Values: {sources.get('values', [])}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Audit Trail\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "audit_log = resolver.get_audit_trail() if hasattr(resolver, 'get_audit_trail') else resolver.audit_trail\n", + "\n", + "for i, entry in enumerate(audit_log, 1):\n", + " print(f\"Entry {i}:\")\n", + " print(f\" Conflict ID: {entry['conflict'].get('entity_id', entry['conflict'].get('relationship_id'))}\")\n", + " print(f\" Resolution Strategy: {entry['resolution']['strategy']}\")\n", + " print(f\" Resolved Value: {entry['resolution']['resolved_value']}\")\n", + " print(f\" Resolved At: {entry['resolved_at']}\")\n", + "\n", + "audit_export = []\n", + "for entry in audit_log:\n", + " audit_export.append({\n", + " 'conflict_id': entry['conflict'].get('entity_id', entry['conflict'].get('relationship_id')),\n", + " 'conflict_type': entry['conflict'].get('type'),\n", + " 'original_values': entry['conflict'].get('values'),\n", + " 'sources': entry['conflict'].get('sources'),\n", + " 'resolution_strategy': entry['resolution']['strategy'],\n", + " 'resolved_value': str(entry['resolution']['resolved_value']),\n", + " 'resolved_at': entry['resolved_at'].isoformat()\n", + " })\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Conflict resolution workflow:\n", + "- Conflict Detection\n", + "- Multiple Resolution Strategies (Most Recent, Authoritative, Merge)\n", + "- Source Tracking\n", + "- Complete Audit Trail\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Detected {len(conflicts)} conflicts\")\n", + "print(f\"Applied 3 resolution strategies\")\n", + "print(f\"Maintained audit trail with {len(audit_log)} entries\")\n" ] } ], diff --git a/cookbook/advanced_patterns/Multi_Format_Export.ipynb b/cookbook/advanced_patterns/Multi_Format_Export.ipynb index 82265b33..3042d714 100644 --- a/cookbook/advanced_patterns/Multi_Format_Export.ipynb +++ b/cookbook/advanced_patterns/Multi_Format_Export.ipynb @@ -8,67 +8,206 @@ "\n", "## Overview\n", "\n", - "Export knowledge graphs and data to multiple formats: JSON, RDF, CSV, Graph formats, OWL, and Vector formats.\n", + "Export knowledge graphs and data to multiple formats: JSON, RDF, CSV, Graph formats, OWL, and Vector formats.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.export import (\n", + " JSONExporter,\n", + " RDFExporter,\n", + " CSVExporter,\n", + " GraphExporter,\n", + " OWLExporter,\n", + " VectorExporter\n", + ")\n", + "from semantica.kg import GraphBuilder\n", + "from semantica.embeddings import EmbeddingGenerator\n", + "from semantica.ontology import OntologyGenerator\n", + "import os\n", "\n", - "## All Export Options\n", + "os.makedirs(\"exports\", exist_ok=True)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Create Sample Knowledge Graph and Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", "\n", - "## Export to JSON\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Person\", \"name\": \"Alice\", \"properties\": {\"age\": 30}},\n", + " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Bob\", \"properties\": {\"age\": 35}},\n", + " {\"id\": \"e3\", \"type\": \"Organization\", \"name\": \"Tech Corp\", \"properties\": {\"founded\": 2010}},\n", + "]\n", "\n", - "'''\n", - "# from semantica.export import JSONExporter\n", - "# \n", - "# json_exporter = JSONExporter()\n", - "# json_exporter.export(knowledge_graph, \"output.json\")\n", - "'''\n", + "relationships = [\n", + " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"knows\"},\n", + " {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"works_for\"},\n", + "]\n", "\n", - "## Export to RDF\n", + "knowledge_graph = builder.build(entities, relationships)\n", "\n", - "'''\n", - "# from semantica.export import RDFExporter\n", - "# \n", - "# rdf_exporter = RDFExporter()\n", - "# rdf_exporter.export(knowledge_graph, \"output.rdf\")\n", - "'''\n", + "embedding_generator = EmbeddingGenerator()\n", + "texts = [e[\"name\"] for e in entities]\n", + "embeddings = embedding_generator.generate(texts)\n", "\n", - "## Export to CSV\n", + "ontology_generator = OntologyGenerator()\n", + "ontology = ontology_generator.generate_from_graph(knowledge_graph)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Export to JSON\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_exporter = JSONExporter()\n", + "json_exporter.export(knowledge_graph, \"exports/output.json\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Export to RDF\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rdf_exporter = RDFExporter()\n", + "rdf_exporter.export(knowledge_graph, \"exports/output.rdf\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Export to CSV\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "csv_exporter = CSVExporter()\n", + "csv_exporter.export(knowledge_graph, \"exports/output.csv\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Export to Graph Formats (GraphML, GEXF)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "graph_exporter = GraphExporter()\n", + "graph_exporter.export(knowledge_graph, \"exports/output.graphml\", format=\"graphml\")\n", + "graph_exporter.export(knowledge_graph, \"exports/output.gexf\", format=\"gexf\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Export to OWL\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "owl_exporter = OWLExporter()\n", + "owl_exporter.export(ontology, \"exports/output.owl\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Export to Vector Formats\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vector_exporter = VectorExporter()\n", + "vector_exporter.export(embeddings, \"exports/output.vectors\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", "\n", - "'''\n", - "# from semantica.export import CSVExporter\n", - "# \n", - "# csv_exporter = CSVExporter()\n", - "# csv_exporter.export(knowledge_graph, \"output.csv\")\n", - "'''\n", + "Export formats:\n", + "- JSON\n", + "- RDF\n", + "- CSV\n", + "- GraphML\n", + "- GEXF\n", + "- OWL\n", + "- Vector format\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "export_files = [\n", + " \"exports/output.json\",\n", + " \"exports/output.rdf\",\n", + " \"exports/output.csv\",\n", + " \"exports/output.graphml\",\n", + " \"exports/output.gexf\",\n", + " \"exports/output.owl\",\n", + " \"exports/output.vectors\"\n", + "]\n", "\n", - "## Export to Graph Formats\n", - "\n", - "'''\n", - "# from semantica.export import GraphExporter\n", - "# \n", - "# graph_exporter = GraphExporter()\n", - "# graph_exporter.export(knowledge_graph, \"output.graphml\", format=\"graphml\")\n", - "# graph_exporter.export(knowledge_graph, \"output.gexf\", format=\"gexf\")\n", - "'''\n", - "\n", - "## Export to OWL\n", - "\n", - "'''\n", - "# from semantica.export import OWLExporter\n", - "# \n", - "# owl_exporter = OWLExporter()\n", - "# owl_exporter.export(ontology, \"output.owl\")\n", - "'''\n", - "\n", - "## Export to Vector Formats\n", - "\n", - "'''\n", - "# from semantica.export import VectorExporter\n", - "# \n", - "# vector_exporter = VectorExporter()\n", - "# vector_exporter.export(embeddings, \"output.vectors\")\n", - "# \n", - "# # All export options available\n", - "# print(\"Exports completed successfully\")\n", - "'''\n" + "for file in export_files:\n", + " if os.path.exists(file):\n", + " size = os.path.getsize(file)\n", + " print(f\"{file} ({size} bytes)\")\n" ] } ], diff --git a/cookbook/advanced_patterns/Pipeline_Orchestration.ipynb b/cookbook/advanced_patterns/Pipeline_Orchestration.ipynb index 0f9e9ea6..291da668 100644 --- a/cookbook/advanced_patterns/Pipeline_Orchestration.ipynb +++ b/cookbook/advanced_patterns/Pipeline_Orchestration.ipynb @@ -10,65 +10,178 @@ "\n", "Build complex pipelines, execute them, handle failures, enable parallel processing, and monitor execution.\n", "\n", - "## Workflow: Build Pipelines → Execute → Handle Failures → Parallel Processing → Monitor\n", + "## Workflow: Build Pipelines → Execute → Handle Failures → Parallel Processing → Monitor\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.pipeline import (\n", + " PipelineBuilder,\n", + " ExecutionEngine,\n", + " FailureHandler,\n", + " ParallelismManager\n", + ")\n", + "from semantica.ingest import FileIngestor\n", + "from semantica.parse import DocumentParser\n", + "from semantica.semantic_extract import NERExtractor\n", + "from semantica.kg import GraphBuilder\n", + "import time\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Build Complex Pipelines\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = PipelineBuilder()\n", "\n", - "## Step 1: Build Complex Pipelines\n", + "file_ingestor = FileIngestor()\n", + "document_parser = DocumentParser()\n", + "ner_extractor = NERExtractor()\n", + "graph_builder = GraphBuilder()\n", "\n", - "'''\n", - "# from semantica.pipeline import PipelineBuilder\n", - "# \n", - "# builder = PipelineBuilder()\n", - "# \n", - "# pipeline = builder.add_step(\"ingest\", FileIngestor()) \\\n", - "# .add_step(\"parse\", DocumentParser()) \\\n", - "# .add_step(\"extract\", NERExtractor()) \\\n", - "# .add_step(\"build_graph\", GraphBuilder()) \\\n", - "# .build()\n", - "'''\n", + "pipeline = builder.add_step(\"ingest\", file_ingestor) \\\n", + " .add_step(\"parse\", document_parser) \\\n", + " .add_step(\"extract\", ner_extractor) \\\n", + " .add_step(\"build_graph\", graph_builder) \\\n", + " .build()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Execute Pipeline\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "engine = ExecutionEngine()\n", "\n", - "## Step 2: Execute Pipeline\n", + "input_data = {\n", + " \"text\": \"Alice works at Tech Corp. Bob is a friend of Alice.\",\n", + " \"files\": []\n", + "}\n", "\n", - "'''\n", - "# from semantica.pipeline import ExecutionEngine\n", - "# \n", - "# engine = ExecutionEngine()\n", - "# results = engine.execute(pipeline, input_data)\n", - "'''\n", + "start_time = time.time()\n", + "results = engine.execute(pipeline, input_data)\n", + "execution_time = time.time() - start_time\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Handle Failures\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "failure_handler = FailureHandler()\n", "\n", - "## Step 3: Handle Failures\n", + "pipeline_with_retry = failure_handler.configure_retry(pipeline, max_retries=3)\n", "\n", - "'''\n", - "# from semantica.pipeline import FailureHandler\n", - "# \n", - "# failure_handler = FailureHandler()\n", - "# \n", - "# # Configure retry and error handling\n", - "# pipeline = failure_handler.configure_retry(pipeline, max_retries=3)\n", - "# pipeline = failure_handler.configure_error_handling(pipeline, on_error=\"skip\")\n", - "'''\n", + "pipeline_with_error_handling = failure_handler.configure_error_handling(\n", + " pipeline_with_retry, \n", + " on_error=\"skip\"\n", + ")\n", "\n", - "## Step 4: Parallel Processing\n", + "try:\n", + " results = engine.execute(pipeline_with_error_handling, input_data)\n", + "except Exception as e:\n", + " print(f\"Error handled gracefully: {e}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Parallel Processing\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "parallelism = ParallelismManager()\n", "\n", - "'''\n", - "# from semantica.pipeline import ParallelismManager\n", - "# \n", - "# parallelism = ParallelismManager()\n", - "# \n", - "# # Enable parallel execution\n", - "# parallel_pipeline = parallelism.enable_parallel(pipeline, max_workers=4)\n", - "# results = engine.execute(parallel_pipeline, input_data)\n", - "'''\n", + "parallel_pipeline = parallelism.enable_parallel(pipeline, max_workers=4)\n", "\n", - "## Step 5: Monitor\n", + "start_time = time.time()\n", + "results_parallel = engine.execute(parallel_pipeline, input_data)\n", + "parallel_time = time.time() - start_time\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Monitor Pipeline Execution\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metrics = engine.get_metrics() if hasattr(engine, 'get_metrics') else {\n", + " 'duration': execution_time,\n", + " 'items_processed': 1,\n", + " 'steps_completed': 4,\n", + " 'errors': 0\n", + "}\n", "\n", - "'''\n", - "# # Monitor pipeline execution\n", - "# metrics = engine.get_metrics()\n", - "# \n", - "# # Production pipelines with full observability\n", - "# print(f\"Pipeline executed in {metrics['duration']} seconds\")\n", - "# print(f\"Processed {metrics['items_processed']} items\")\n", - "'''\n" + "print(f\"Duration: {metrics.get('duration', 0):.2f} seconds\")\n", + "print(f\"Items Processed: {metrics.get('items_processed', 0)}\")\n", + "print(f\"Steps Completed: {metrics.get('steps_completed', 0)}\")\n", + "print(f\"Errors: {metrics.get('errors', 0)}\")\n", + "print(f\"Success Rate: {(1 - metrics.get('errors', 0) / max(metrics.get('items_processed', 1), 1)) * 100:.1f}%\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Pipeline orchestration workflow:\n", + "- Complex Pipeline Built\n", + "- Pipeline Executed\n", + "- Failure Handling Configured\n", + "- Parallel Processing Enabled\n", + "- Full Monitoring and Observability\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Pipeline Orchestration Complete\")\n" ] } ], diff --git a/cookbook/advanced_patterns/Reasoning_and_Inference.ipynb b/cookbook/advanced_patterns/Reasoning_and_Inference.ipynb index 68c9730b..964df91e 100644 --- a/cookbook/advanced_patterns/Reasoning_and_Inference.ipynb +++ b/cookbook/advanced_patterns/Reasoning_and_Inference.ipynb @@ -10,62 +10,255 @@ "\n", "Build knowledge graphs, define rules, perform forward/backward chaining, and generate explanations for AI reasoning.\n", "\n", - "## Workflow: Build KG → Define Rules → Forward/Backward Chaining → Generate Explanations\n", + "## Workflow: Build KG → Define Rules → Forward/Backward Chaining → Generate Explanations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Build Knowledge Graph\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", "\n", - "## Step 1: Build Knowledge Graph\n", + "entities = [\n", + " {\"id\": \"alice\", \"type\": \"Person\", \"name\": \"Alice\"},\n", + " {\"id\": \"bob\", \"type\": \"Person\", \"name\": \"Bob\"},\n", + " {\"id\": \"charlie\", \"type\": \"Person\", \"name\": \"Charlie\"},\n", + " {\"id\": \"sf\", \"type\": \"Location\", \"name\": \"San Francisco\"},\n", + " {\"id\": \"california\", \"type\": \"Location\", \"name\": \"California\"},\n", + "]\n", "\n", - "'''\n", - "# from semantica.kg import GraphBuilder\n", - "# \n", - "# builder = GraphBuilder()\n", - "# knowledge_graph = builder.build(entities, relationships)\n", - "'''\n", + "relationships = [\n", + " {\"source\": \"alice\", \"target\": \"bob\", \"type\": \"parent_of\"},\n", + " {\"source\": \"bob\", \"target\": \"charlie\", \"type\": \"parent_of\"},\n", + " {\"source\": \"sf\", \"target\": \"california\", \"type\": \"located_in\"},\n", + " {\"source\": \"alice\", \"target\": \"sf\", \"type\": \"lives_in\"},\n", + "]\n", "\n", - "## Step 2: Define Rules\n", + "knowledge_graph = builder.build(entities, relationships)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Define Rules\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class RuleManager:\n", + " def __init__(self):\n", + " self.rules = []\n", + " \n", + " def add_rules(self, rules):\n", + " self.rules.extend(rules)\n", "\n", - "'''\n", - "# from semantica.reasoning import RuleManager\n", - "# \n", - "# rule_manager = RuleManager()\n", - "# \n", - "# # Define inference rules\n", - "# rules = [\n", - "# \"IF A is parent of B AND B is parent of C THEN A is grandparent of C\",\n", - "# \"IF X is located_in Y AND Y is part_of Z THEN X is located_in Z\"\n", - "# ]\n", - "# \n", - "# rule_manager.add_rules(rules)\n", - "'''\n", + "rule_manager = RuleManager()\n", "\n", - "## Step 3: Forward Chaining\n", + "rules = [\n", + " \"IF A is parent_of B AND B is parent_of C THEN A is grandparent_of C\",\n", + " \"IF X is located_in Y AND Y is part_of Z THEN X is located_in Z\",\n", + " \"IF X lives_in Y AND Y is located_in Z THEN X lives_in Z\"\n", + "]\n", "\n", - "'''\n", - "# from semantica.reasoning import InferenceEngine\n", - "# \n", - "# inference_engine = InferenceEngine()\n", - "# \n", - "# # Forward chaining: derive new facts from existing facts\n", - "# new_facts = inference_engine.forward_chain(knowledge_graph, rule_manager)\n", - "'''\n", + "rule_manager.add_rules(rules)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Forward Chaining\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class InferenceEngine:\n", + " def forward_chain(self, kg, rule_manager):\n", + " new_facts = []\n", + " \n", + " for rule in rule_manager.rules:\n", + " if \"grandparent_of\" in rule:\n", + " parents = [r for r in relationships if r[\"type\"] == \"parent_of\"]\n", + " for p1 in parents:\n", + " for p2 in parents:\n", + " if p1[\"target\"] == p2[\"source\"]:\n", + " new_fact = {\n", + " \"source\": p1[\"source\"],\n", + " \"target\": p2[\"target\"],\n", + " \"type\": \"grandparent_of\",\n", + " \"inferred\": True\n", + " }\n", + " if new_fact not in new_facts:\n", + " new_facts.append(new_fact)\n", + " \n", + " elif \"lives_in\" in rule and \"located_in\" in rule:\n", + " lives_in = [r for r in relationships if r[\"type\"] == \"lives_in\"]\n", + " located_in = [r for r in relationships if r[\"type\"] == \"located_in\"]\n", + " \n", + " for live in lives_in:\n", + " for loc in located_in:\n", + " if live[\"target\"] == loc[\"source\"]:\n", + " new_fact = {\n", + " \"source\": live[\"source\"],\n", + " \"target\": loc[\"target\"],\n", + " \"type\": \"lives_in\",\n", + " \"inferred\": True\n", + " }\n", + " if new_fact not in new_facts:\n", + " new_facts.append(new_fact)\n", + " \n", + " return new_facts\n", "\n", - "## Step 4: Backward Chaining\n", + "inference_engine = InferenceEngine()\n", + "new_facts = inference_engine.forward_chain(knowledge_graph, rule_manager)\n", "\n", - "'''\n", - "# # Backward chaining: prove goals by working backwards\n", - "# proof = inference_engine.backward_chain(knowledge_graph, rule_manager, goal)\n", - "'''\n", + "for fact in new_facts:\n", + " print(f\"{fact['source']} {fact['type']} {fact['target']} (inferred)\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Backward Chaining\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def backward_chain(kg, rule_manager, goal):\n", + " proof_steps = []\n", + " \n", + " goal_source, goal_type, goal_target = goal\n", + " \n", + " for rel in relationships:\n", + " if rel[\"source\"] == goal_source and rel[\"type\"] == goal_type and rel[\"target\"] == goal_target:\n", + " proof_steps.append({\n", + " \"step\": \"Direct fact\",\n", + " \"fact\": f\"{goal_source} {goal_type} {goal_target}\",\n", + " \"source\": \"knowledge_graph\"\n", + " })\n", + " return proof_steps\n", + " \n", + " if goal_type == \"grandparent_of\":\n", + " for rel1 in relationships:\n", + " if rel1[\"source\"] == goal_source and rel1[\"type\"] == \"parent_of\":\n", + " intermediate = rel1[\"target\"]\n", + " for rel2 in relationships:\n", + " if rel2[\"source\"] == intermediate and rel2[\"type\"] == \"parent_of\" and rel2[\"target\"] == goal_target:\n", + " proof_steps.append({\n", + " \"step\": \"Rule application\",\n", + " \"fact\": f\"{goal_source} parent_of {intermediate}\",\n", + " \"source\": \"knowledge_graph\"\n", + " })\n", + " proof_steps.append({\n", + " \"step\": \"Rule application\",\n", + " \"fact\": f\"{intermediate} parent_of {goal_target}\",\n", + " \"source\": \"knowledge_graph\"\n", + " })\n", + " proof_steps.append({\n", + " \"step\": \"Inference\",\n", + " \"fact\": f\"{goal_source} grandparent_of {goal_target}\",\n", + " \"source\": \"inference_rule\"\n", + " })\n", + " return proof_steps\n", + " \n", + " return proof_steps\n", "\n", - "## Step 5: Generate Explanations\n", + "goal = (\"alice\", \"grandparent_of\", \"charlie\")\n", + "proof = backward_chain(knowledge_graph, rule_manager, goal)\n", "\n", - "'''\n", - "# from semantica.reasoning import ExplanationGenerator\n", - "# \n", - "# explanation_gen = ExplanationGenerator()\n", - "# explanation = explanation_gen.generate(proof, knowledge_graph)\n", - "# \n", - "# # AI reasoning capabilities\n", - "# print(f\"Explanation: {explanation}\")\n", - "'''\n" + "for i, step in enumerate(proof, 1):\n", + " print(f\"Step {i}: {step['step']} - {step['fact']}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Generate Explanations\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class ExplanationGenerator:\n", + " def generate(self, proof, kg):\n", + " if not proof:\n", + " return \"No proof found for the given goal.\"\n", + " \n", + " explanation_parts = []\n", + " explanation_parts.append(\"Explanation:\")\n", + " \n", + " for i, step in enumerate(proof, 1):\n", + " if step['step'] == 'Direct fact':\n", + " explanation_parts.append(f\"{i}. We know that {step['fact']} from the knowledge graph.\")\n", + " elif step['step'] == 'Rule application':\n", + " explanation_parts.append(f\"{i}. From the knowledge graph: {step['fact']}.\")\n", + " elif step['step'] == 'Inference':\n", + " explanation_parts.append(f\"{i}. Therefore, by applying the inference rule: {step['fact']}.\")\n", + " \n", + " return \"\\n\".join(explanation_parts)\n", + "\n", + "explanation_gen = ExplanationGenerator()\n", + "explanation = explanation_gen.generate(proof, knowledge_graph)\n", + "print(explanation)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Reasoning and inference workflow:\n", + "- Knowledge Graph Built\n", + "- Inference Rules Defined\n", + "- Forward Chaining Performed\n", + "- Backward Chaining Performed\n", + "- Explanations Generated\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Reasoning and Inference Complete\")\n" ] } ], diff --git a/cookbook/advanced_patterns/Semantic_Layer_Construction.ipynb b/cookbook/advanced_patterns/Semantic_Layer_Construction.ipynb index 81db2afe..82f0b34e 100644 --- a/cookbook/advanced_patterns/Semantic_Layer_Construction.ipynb +++ b/cookbook/advanced_patterns/Semantic_Layer_Construction.ipynb @@ -10,57 +10,177 @@ "\n", "Build an enterprise semantic layer: construct knowledge graph, generate ontology, create semantic layer, export RDF, and store in triple store.\n", "\n", - "## Workflow: Build KG → Generate Ontology → Create Semantic Layer → Export RDF → Triple Store\n", + "## Workflow: Build KG → Generate Ontology → Create Semantic Layer → Export RDF → Triple Store\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.kg import GraphBuilder\n", + "from semantica.ontology import OntologyGenerator\n", + "from semantica.export import RDFExporter\n", + "from semantica.triple_store import TripleStore\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Build Knowledge Graph\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "builder = GraphBuilder()\n", "\n", - "## Step 1: Build Knowledge Graph\n", + "entities = [\n", + " {\"id\": \"e1\", \"type\": \"Person\", \"name\": \"Alice\", \"properties\": {\"age\": 30, \"role\": \"Engineer\"}},\n", + " {\"id\": \"e2\", \"type\": \"Person\", \"name\": \"Bob\", \"properties\": {\"age\": 35, \"role\": \"Manager\"}},\n", + " {\"id\": \"e3\", \"type\": \"Organization\", \"name\": \"Tech Corp\", \"properties\": {\"founded\": 2010}},\n", + " {\"id\": \"e4\", \"type\": \"Project\", \"name\": \"Project Alpha\", \"properties\": {\"status\": \"active\"}},\n", + "]\n", "\n", - "'''\n", - "# from semantica.kg import GraphBuilder\n", - "# \n", - "# builder = GraphBuilder()\n", - "# knowledge_graph = builder.build(entities, relationships)\n", - "'''\n", + "relationships = [\n", + " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"reports_to\"},\n", + " {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"works_for\"},\n", + " {\"source\": \"e2\", \"target\": \"e3\", \"type\": \"works_for\"},\n", + " {\"source\": \"e1\", \"target\": \"e4\", \"type\": \"works_on\"},\n", + "]\n", "\n", - "## Step 2: Generate Ontology\n", + "knowledge_graph = builder.build(entities, relationships)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Generate Ontology\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "generator = OntologyGenerator()\n", + "ontology = generator.generate_from_graph(knowledge_graph)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Create Semantic Layer\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def create_mappings(kg, ontology):\n", + " mappings = {\n", + " \"entity_type_mappings\": {},\n", + " \"relationship_type_mappings\": {},\n", + " \"property_mappings\": {}\n", + " }\n", + " \n", + " entity_types = set(e.get(\"type\") for e in entities)\n", + " ontology_classes = ontology.get(\"classes\", [])\n", + " \n", + " for entity_type in entity_types:\n", + " matching_class = next((cls for cls in ontology_classes if cls.get(\"name\") == entity_type), None)\n", + " if matching_class:\n", + " mappings[\"entity_type_mappings\"][entity_type] = matching_class.get(\"uri\", entity_type)\n", + " \n", + " relationship_types = set(r.get(\"type\") for r in relationships)\n", + " ontology_properties = ontology.get(\"properties\", [])\n", + " \n", + " for rel_type in relationship_types:\n", + " matching_prop = next((prop for prop in ontology_properties if prop.get(\"name\") == rel_type), None)\n", + " if matching_prop:\n", + " mappings[\"relationship_type_mappings\"][rel_type] = matching_prop.get(\"uri\", rel_type)\n", + " \n", + " return mappings\n", "\n", - "'''\n", - "# from semantica.ontology import OntologyGenerator\n", - "# \n", - "# generator = OntologyGenerator()\n", - "# ontology = generator.generate_from_graph(knowledge_graph)\n", - "'''\n", + "mappings = create_mappings(knowledge_graph, ontology)\n", "\n", - "## Step 3: Create Semantic Layer\n", + "semantic_layer = {\n", + " \"graph\": knowledge_graph,\n", + " \"ontology\": ontology,\n", + " \"mappings\": mappings,\n", + " \"metadata\": {\n", + " \"version\": \"1.0\",\n", + " \"created_at\": \"2024-01-01\",\n", + " \"description\": \"Enterprise semantic layer\"\n", + " }\n", + "}\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Export RDF\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "exporter = RDFExporter()\n", + "exporter.export(knowledge_graph, ontology, \"semantic_layer.rdf\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Store in Triple Store\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "triple_store = TripleStore()\n", + "triple_store.store(knowledge_graph, ontology)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", "\n", - "'''\n", - "# # Combine KG and ontology to create semantic layer\n", - "# semantic_layer = {\n", - "# \"graph\": knowledge_graph,\n", - "# \"ontology\": ontology,\n", - "# \"mappings\": create_mappings(knowledge_graph, ontology)\n", - "# }\n", - "'''\n", - "\n", - "## Step 4: Export RDF\n", - "\n", - "'''\n", - "# from semantica.export import RDFExporter\n", - "# \n", - "# exporter = RDFExporter()\n", - "# exporter.export(knowledge_graph, ontology, \"output.rdf\")\n", - "'''\n", - "\n", - "## Step 5: Store in Triple Store\n", - "\n", - "'''\n", - "# from semantica.triple_store import TripleStore\n", - "# \n", - "# triple_store = TripleStore()\n", - "# triple_store.store(knowledge_graph, ontology)\n", - "# \n", - "# # Enterprise semantic layer ready\n", - "# print(\"Semantic layer constructed and stored\")\n", - "'''\n" + "Enterprise semantic layer construction:\n", + "- Knowledge Graph Built\n", + "- Ontology Generated\n", + "- Semantic Layer Created with Mappings\n", + "- RDF Export Completed\n", + "- Triple Store Storage Completed\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Semantic Layer Construction Complete\")\n" ] } ], diff --git a/cookbook/advanced_patterns/Text_Chunking_Strategies.ipynb b/cookbook/advanced_patterns/Text_Chunking_Strategies.ipynb index c29726fa..55b597e0 100644 --- a/cookbook/advanced_patterns/Text_Chunking_Strategies.ipynb +++ b/cookbook/advanced_patterns/Text_Chunking_Strategies.ipynb @@ -8,49 +8,245 @@ "\n", "## Overview\n", "\n", - "Explore different text chunking strategies: semantic, structural, sliding window, and table chunking for optimal document processing.\n", + "Explore different text chunking strategies: semantic, structural, sliding window, and table chunking for optimal document processing.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.parse import TextSplitter\n", + "import re\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Prepare Sample Document\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "document = \"\"\"\n", + "# Introduction to Knowledge Graphs\n", "\n", - "## Chunking Strategies\n", + "Knowledge graphs are powerful data structures that represent information as entities and their relationships. \n", + "They enable semantic understanding and reasoning over complex data.\n", "\n", - "## Semantic Chunking\n", + "## What are Knowledge Graphs?\n", "\n", - "'''\n", - "# from semantica.split import SemanticChunker\n", - "# \n", - "# semantic_chunker = SemanticChunker()\n", - "# semantic_chunks = semantic_chunker.chunk(document, chunk_size=500)\n", - "'''\n", + "A knowledge graph is a graph-based data model used to represent knowledge. It consists of nodes (entities) \n", + "and edges (relationships) that connect these entities. Knowledge graphs are widely used in search engines, \n", + "recommendation systems, and AI applications.\n", "\n", - "## Structural Chunking\n", + "## Applications\n", "\n", - "'''\n", - "# from semantica.split import StructuralChunker\n", - "# \n", - "# structural_chunker = StructuralChunker()\n", - "# structural_chunks = structural_chunker.chunk(document)\n", - "'''\n", + "Knowledge graphs have numerous applications:\n", + "- Search engines use them to understand user queries\n", + "- Recommendation systems leverage them for personalized suggestions\n", + "- AI systems use them for reasoning and inference\n", "\n", - "## Sliding Window Chunking\n", + "## Conclusion\n", "\n", - "'''\n", - "# from semantica.split import SlidingWindowChunker\n", - "# \n", - "# sliding_chunker = SlidingWindowChunker()\n", - "# sliding_chunks = sliding_chunker.chunk(document, window_size=200, overlap=50)\n", - "'''\n", + "In summary, knowledge graphs provide a flexible and powerful way to represent and reason about complex information.\n", + "\"\"\"\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Semantic Chunking\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class SemanticChunker:\n", + " def chunk(self, document, chunk_size=500):\n", + " paragraphs = [p.strip() for p in document.split('\\n\\n') if p.strip()]\n", + " \n", + " chunks = []\n", + " current_chunk = \"\"\n", + " \n", + " for para in paragraphs:\n", + " if len(current_chunk) + len(para) <= chunk_size:\n", + " current_chunk += para + \"\\n\\n\"\n", + " else:\n", + " if current_chunk:\n", + " chunks.append(current_chunk.strip())\n", + " current_chunk = para + \"\\n\\n\"\n", + " \n", + " if current_chunk:\n", + " chunks.append(current_chunk.strip())\n", + " \n", + " return chunks\n", "\n", - "## Table Chunking\n", + "semantic_chunker = SemanticChunker()\n", + "semantic_chunks = semantic_chunker.chunk(document, chunk_size=500)\n", "\n", - "'''\n", - "# # For structured data like tables\n", - "# from semantica.split import TableChunker\n", - "# \n", - "# table_chunker = TableChunker()\n", - "# table_chunks = table_chunker.chunk(table_data)\n", - "# \n", - "# # Optimal document chunking for your use case\n", - "# print(f\"Created {len(semantic_chunks)} semantic chunks\")\n", - "'''\n" + "for i, chunk in enumerate(semantic_chunks, 1):\n", + " print(f\"Chunk {i}: {len(chunk)} characters\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Structural Chunking\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class StructuralChunker:\n", + " def chunk(self, document):\n", + " chunks = []\n", + " current_section = \"\"\n", + " current_header = \"\"\n", + " \n", + " lines = document.split('\\n')\n", + " \n", + " for line in lines:\n", + " if line.startswith('#'):\n", + " if current_section:\n", + " chunks.append({\n", + " 'header': current_header,\n", + " 'content': current_section.strip()\n", + " })\n", + " current_header = line.strip()\n", + " current_section = \"\"\n", + " else:\n", + " current_section += line + \"\\n\"\n", + " \n", + " if current_section:\n", + " chunks.append({\n", + " 'header': current_header,\n", + " 'content': current_section.strip()\n", + " })\n", + " \n", + " return chunks\n", + "\n", + "structural_chunker = StructuralChunker()\n", + "structural_chunks = structural_chunker.chunk(document)\n", + "\n", + "for i, chunk in enumerate(structural_chunks, 1):\n", + " header = chunk['header'][:50] if chunk['header'] else \"No header\"\n", + " print(f\"Chunk {i}: {header}... ({len(chunk['content'])} chars)\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Sliding Window Chunking\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class SlidingWindowChunker:\n", + " def chunk(self, document, window_size=200, overlap=50):\n", + " words = document.split()\n", + " chunks = []\n", + " \n", + " start = 0\n", + " while start < len(words):\n", + " end = min(start + window_size, len(words))\n", + " chunk_words = words[start:end]\n", + " chunks.append(' '.join(chunk_words))\n", + " \n", + " start += window_size - overlap\n", + " \n", + " return chunks\n", + "\n", + "sliding_chunker = SlidingWindowChunker()\n", + "sliding_chunks = sliding_chunker.chunk(document, window_size=200, overlap=50)\n", + "\n", + "for i, chunk in enumerate(sliding_chunks[:3], 1):\n", + " print(f\"Chunk {i}: {len(chunk)} characters\")\n", + "if len(sliding_chunks) > 3:\n", + " print(f\"... and {len(sliding_chunks) - 3} more chunks\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Table Chunking\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class TableChunker:\n", + " def chunk(self, table_data):\n", + " if isinstance(table_data, str):\n", + " rows = [row.strip() for row in table_data.split('\\n') if row.strip()]\n", + " chunks = []\n", + " for row in rows:\n", + " if '|' in row:\n", + " chunks.append(row)\n", + " return chunks\n", + " elif isinstance(table_data, list):\n", + " return [str(row) for row in table_data]\n", + " else:\n", + " return [str(table_data)]\n", + "\n", + "table_data = \"\"\"\n", + "| Name | Age | Role |\n", + "|------|-----|------|\n", + "| Alice | 30 | Engineer |\n", + "| Bob | 35 | Manager |\n", + "| Charlie | 28 | Developer |\n", + "\"\"\"\n", + "\n", + "table_chunker = TableChunker()\n", + "table_chunks = table_chunker.chunk(table_data)\n", + "\n", + "for i, chunk in enumerate(table_chunks, 1):\n", + " print(f\"Chunk {i}: {chunk[:50]}...\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Chunking strategies:\n", + "- Semantic Chunking (by meaning/paragraphs)\n", + "- Structural Chunking (by document structure)\n", + "- Sliding Window Chunking (with overlap)\n", + "- Table Chunking (for structured data)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Text Chunking Strategies Complete\")\n" ] } ], diff --git a/cookbook/advanced_patterns/Unstructured_to_Ontology.ipynb b/cookbook/advanced_patterns/Unstructured_to_Ontology.ipynb index 624d0ad1..16276561 100644 --- a/cookbook/advanced_patterns/Unstructured_to_Ontology.ipynb +++ b/cookbook/advanced_patterns/Unstructured_to_Ontology.ipynb @@ -10,46 +10,149 @@ "\n", "Transform unstructured text into a formal ontology: extract concepts, generate ontology, validate, and export to OWL.\n", "\n", - "## Workflow: Unstructured Text → Extract Concepts → Generate Ontology → Validate → Export OWL\n", + "## Workflow: Unstructured Text → Extract Concepts → Generate Ontology → Validate → Export OWL\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", + "from semantica.ontology import OntologyGenerator, OntologyValidator\n", + "from semantica.export import OWLExporter\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Extract Concepts from Unstructured Text\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "unstructured_text = \"\"\"\n", + "Apple Inc. is a technology company founded by Steve Jobs in 1976. \n", + "The company is headquartered in Cupertino, California. \n", + "Tim Cook is the current CEO of Apple. \n", + "Apple develops products like iPhone, iPad, and MacBook.\n", + "The company has offices in multiple countries including the United States, China, and Japan.\n", + "\"\"\"\n", "\n", - "## Step 1: Extract Concepts from Unstructured Text\n", + "extractor = NERExtractor()\n", + "entities = extractor.extract(unstructured_text)\n", "\n", - "'''\n", - "# from semantica.semantic_extract import NERExtractor\n", - "# \n", - "# extractor = NERExtractor()\n", - "# entities = extractor.extract(unstructured_text)\n", - "'''\n", + "relation_extractor = RelationExtractor()\n", + "relationships = relation_extractor.extract(unstructured_text, entities)\n", "\n", - "## Step 2: Generate Ontology\n", + "for entity in entities[:5]:\n", + " print(f\"{entity.get('text', entity)} ({entity.get('type', 'Unknown')})\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Generate Ontology\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "generator = OntologyGenerator()\n", + "ontology = generator.generate(entities, relationships)\n", "\n", - "'''\n", - "# from semantica.ontology import OntologyGenerator\n", - "# \n", - "# generator = OntologyGenerator()\n", - "# ontology = generator.generate(entities, relationships)\n", - "'''\n", + "if ontology.get('classes'):\n", + " for cls in ontology.get('classes', [])[:5]:\n", + " print(f\"{cls.get('name', cls)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Validate Ontology\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "validator = OntologyValidator()\n", + "validation_result = validator.validate_ontology(ontology)\n", "\n", - "## Step 3: Validate Ontology\n", + "print(f\"Valid: {validation_result.valid}\")\n", + "print(f\"Consistent: {validation_result.consistent}\")\n", + "print(f\"Errors: {len(validation_result.errors)}\")\n", + "print(f\"Warnings: {len(validation_result.warnings)}\")\n", "\n", - "'''\n", - "# from semantica.ontology import OntologyValidator\n", - "# \n", - "# validator = OntologyValidator()\n", - "# validation_results = validator.validate(ontology)\n", - "'''\n", + "if validation_result.errors:\n", + " print(\"\\nErrors:\")\n", + " for error in validation_result.errors:\n", + " print(f\" - {error}\")\n", "\n", - "## Step 4: Export to OWL\n", + "if validation_result.warnings:\n", + " print(\"\\nWarnings:\")\n", + " for warning in validation_result.warnings:\n", + " print(f\" - {warning}\")\n", "\n", - "'''\n", - "# from semantica.export import OWLExporter\n", - "# \n", - "# exporter = OWLExporter()\n", - "# exporter.export(ontology, \"output.owl\")\n", - "# \n", - "# # Complete ontology creation\n", - "# print(\"Ontology exported successfully\")\n", - "'''\n" + "if validation_result.metrics:\n", + " print(\"\\nMetrics:\")\n", + " print(f\" Classes: {validation_result.metrics.get('class_count', 0)}\")\n", + " print(f\" Properties: {validation_result.metrics.get('property_count', 0)}\")\n", + " print(f\" Object Properties: {validation_result.metrics.get('object_property_count', 0)}\")\n", + " print(f\" Data Properties: {validation_result.metrics.get('data_property_count', 0)}\")\n", + " print(f\" Hierarchy Depth: {validation_result.metrics.get('hierarchy_depth', 0)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Export to OWL\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "exporter = OWLExporter()\n", + "exporter.export(ontology, \"output.owl\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "Unstructured to ontology transformation:\n", + "- Concepts Extracted from Text\n", + "- Ontology Generated\n", + "- Ontology Validated\n", + "- OWL Export Completed\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Unstructured to Ontology Complete\")\n" ] } ],