From a5d00149d781ad4b4b919566171f5f1ee92a2ae6 Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:53:39 +0530 Subject: [PATCH 1/2] Delete cookbook/core_workflows directory --- .../Building_Knowledge_Graphs.ipynb | 189 ---------------- .../Embedding_Generation_Complete.ipynb | 156 ------------- .../Embedding_Visualization.ipynb | 188 ---------------- .../Entity_Resolution_Workflow.ipynb | 202 ----------------- .../From_Unstructured_to_Structured.ipynb | 185 --------------- .../Graph_Analytics_Complete.ipynb | 194 ---------------- .../Graph_Quality_Assurance.ipynb | 178 --------------- .../Multi_Source_Data_Ingestion.ipynb | 212 ------------------ .../Semantic_Search_Pipeline.ipynb | 177 --------------- .../Temporal_Knowledge_Graphs.ipynb | 184 --------------- .../Text_Processing_Pipeline.ipynb | 185 --------------- .../Vector_Store_Complete.ipynb | 187 --------------- 12 files changed, 2237 deletions(-) delete mode 100644 cookbook/core_workflows/Building_Knowledge_Graphs.ipynb delete mode 100644 cookbook/core_workflows/Embedding_Generation_Complete.ipynb delete mode 100644 cookbook/core_workflows/Embedding_Visualization.ipynb delete mode 100644 cookbook/core_workflows/Entity_Resolution_Workflow.ipynb delete mode 100644 cookbook/core_workflows/From_Unstructured_to_Structured.ipynb delete mode 100644 cookbook/core_workflows/Graph_Analytics_Complete.ipynb delete mode 100644 cookbook/core_workflows/Graph_Quality_Assurance.ipynb delete mode 100644 cookbook/core_workflows/Multi_Source_Data_Ingestion.ipynb delete mode 100644 cookbook/core_workflows/Semantic_Search_Pipeline.ipynb delete mode 100644 cookbook/core_workflows/Temporal_Knowledge_Graphs.ipynb delete mode 100644 cookbook/core_workflows/Text_Processing_Pipeline.ipynb delete mode 100644 cookbook/core_workflows/Vector_Store_Complete.ipynb diff --git a/cookbook/core_workflows/Building_Knowledge_Graphs.ipynb b/cookbook/core_workflows/Building_Knowledge_Graphs.ipynb deleted file mode 100644 index 04b82b6b..00000000 --- a/cookbook/core_workflows/Building_Knowledge_Graphs.ipynb +++ /dev/null @@ -1,189 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Building Knowledge Graphs\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates complete knowledge graph construction: combine entities and relationships, build the graph, resolve conflicts, and validate.\n", - "\n", - "### Learning Objectives\n", - "\n", - "- Prepare entities and relationships for graph construction\n", - "- Build a knowledge graph from entities and relationships\n", - "- Resolve conflicts in the graph\n", - "- Validate the constructed graph\n", - "\n", - "---\n", - "\n", - "## Workflow\n", - "\n", - "**Entities + Relationships → Build KG → Resolve Conflicts → Validate**\n", - "\n", - "Each step ensures a high-quality, consistent knowledge graph.\n", - "\n", - "---\n", - "\n", - "## Step 1: Prepare Entities and Relationships\n", - "\n", - "Start by extracting entities and relationships from your documents.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "\n", - "sample_text = \"\"\"\n", - "Tesla Inc. is an American electric vehicle company.\n", - "It was founded by Elon Musk in 2003.\n", - "The company is headquartered in Austin, Texas.\n", - "Tesla manufactures electric cars and energy storage systems.\n", - "\"\"\"\n", - "\n", - "extractor = NERExtractor()\n", - "relation_extractor = RelationExtractor()\n", - "\n", - "try:\n", - " entities = [\n", - " {\"id\": \"e1\", \"text\": \"Tesla Inc.\", \"type\": \"Organization\"},\n", - " {\"id\": \"e2\", \"text\": \"Elon Musk\", \"type\": \"Person\"},\n", - " {\"id\": \"e3\", \"text\": \"2003\", \"type\": \"Date\"},\n", - " {\"id\": \"e4\", \"text\": \"Austin, Texas\", \"type\": \"Location\"},\n", - " ]\n", - " \n", - " relationships = [\n", - " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"founded_by\"},\n", - " {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"founded_in\"},\n", - " {\"source\": \"e1\", \"target\": \"e4\", \"type\": \"located_in\"},\n", - " ]\n", - " \n", - " print(f\"✓ Prepared {len(entities)} entities and {len(relationships)} relationships\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error preparing entities/relationships: {e}\")\n", - " entities = []\n", - " relationships = []\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Build Knowledge Graph\n", - "\n", - "Construct the knowledge graph from entities and relationships.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import GraphBuilder\n", - "import networkx as nx\n", - "\n", - "builder = GraphBuilder()\n", - "\n", - "try:\n", - " kg = nx.DiGraph()\n", - " \n", - " for entity in entities:\n", - " kg.add_node(entity[\"id\"], name=entity[\"text\"], type=entity[\"type\"])\n", - " \n", - " for rel in relationships:\n", - " kg.add_edge(rel[\"source\"], rel[\"target\"], type=rel[\"type\"])\n", - " \n", - " print(f\"✓ Knowledge graph built successfully!\")\n", - " print(f\" Nodes: {len(kg.nodes)}\")\n", - " print(f\" Edges: {len(kg.edges)}\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error building knowledge graph: {e}\")\n", - " kg = None\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Resolve Conflicts\n", - "\n", - "Detect and resolve conflicts in the knowledge graph to ensure consistency.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import ConflictDetector, EntityResolver\n", - "\n", - "if kg is not None:\n", - " detector = ConflictDetector()\n", - " resolver = EntityResolver()\n", - " \n", - " try:\n", - " conflicts = detector.detect(kg)\n", - " print(f\"✓ Detected {len(conflicts) if conflicts else 0} conflicts\")\n", - " \n", - " resolved_graph = resolver.resolve(kg, conflicts)\n", - " print(f\"✓ Conflicts resolved\")\n", - " print(f\" Resolved graph: {len(resolved_graph.nodes)} nodes, {len(resolved_graph.edges)} edges\")\n", - " \n", - " except Exception as e:\n", - " print(f\"✗ Error resolving conflicts: {e}\")\n", - " resolved_graph = kg\n", - "else:\n", - " resolved_graph = None\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Validate Graph\n", - "\n", - "Validate the knowledge graph to ensure quality and consistency.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import GraphValidator\n", - "\n", - "if resolved_graph is not None:\n", - " validator = GraphValidator()\n", - " \n", - " try:\n", - " validation_results = validator.validate(resolved_graph)\n", - " print(\"✓ Graph validation complete\")\n", - " print(f\" Graph has {len(resolved_graph.nodes)} nodes and {len(resolved_graph.edges)} edges\")\n", - " print(\" Knowledge graph is ready for use\")\n", - " \n", - " except Exception as e:\n", - " print(f\"✗ Error validating graph: {e}\")\n", - "else:\n", - " print(\"No graph available for validation\")\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/core_workflows/Embedding_Generation_Complete.ipynb b/cookbook/core_workflows/Embedding_Generation_Complete.ipynb deleted file mode 100644 index 3cac2c94..00000000 --- a/cookbook/core_workflows/Embedding_Generation_Complete.ipynb +++ /dev/null @@ -1,156 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Embedding Generation Complete\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates how to generate embeddings for text, images, audio, and multimodal data.\n", - "\n", - "### Learning Objectives\n", - "\n", - "- Generate text embeddings\n", - "- Generate image embeddings\n", - "- Generate audio embeddings\n", - "- Create multimodal embeddings combining multiple data types\n", - "\n", - "---\n", - "\n", - "## All Embedding Types\n", - "\n", - "Semantica supports embeddings for various data modalities, enabling semantic understanding across different data types.\n", - "\n", - "---\n", - "\n", - "## Text Embeddings\n", - "\n", - "Generate dense vector representations of text that capture semantic meaning.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.embeddings import TextEmbedder\n", - "import numpy as np\n", - "\n", - "text_data = [\"Machine learning is a subset of artificial intelligence.\"]\n", - "\n", - "text_embedder = TextEmbedder()\n", - "\n", - "try:\n", - " text_embeddings = text_embedder.embed(text_data)\n", - " print(\"✓ Text embeddings generated\")\n", - " if text_embeddings:\n", - " print(f\" Embeddings shape: {text_embeddings.shape if hasattr(text_embeddings, 'shape') else 'N/A'}\")\n", - " print(f\" Number of texts: {len(text_data)}\")\n", - " else:\n", - " print(\" Note: Text embeddings capture semantic meaning of text\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error generating text embeddings: {e}\")\n", - " text_embeddings = None\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Image Embeddings\n", - "\n", - "Generate embeddings for images to enable semantic image search and analysis.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.embeddings import ImageEmbedder\n", - "\n", - "image_embedder = ImageEmbedder()\n", - "\n", - "print(\"Image embedding example:\")\n", - "print(\" image_embeddings = image_embedder.embed(image_data)\")\n", - "print(\"\\nNote: Image embeddings require image files or image data\")\n", - "print(\" Supports formats: JPEG, PNG, and other common image formats\")\n", - "image_embeddings = None\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Audio Embeddings\n", - "\n", - "Generate embeddings for audio data to enable semantic audio search and analysis.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.embeddings import AudioEmbedder\n", - "\n", - "audio_embedder = AudioEmbedder()\n", - "\n", - "print(\"Audio embedding example:\")\n", - "print(\" audio_embeddings = audio_embedder.embed(audio_data)\")\n", - "print(\"\\nNote: Audio embeddings require audio files or audio data\")\n", - "print(\" Supports formats: WAV, MP3, and other common audio formats\")\n", - "audio_embeddings = None\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multimodal Embeddings\n", - "\n", - "Combine text, image, and audio embeddings to create unified multimodal representations.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.embeddings import MultimodalEmbedder\n", - "\n", - "multimodal_embedder = MultimodalEmbedder()\n", - "\n", - "print(\"Multimodal embedding example:\")\n", - "print(\" multimodal_embeddings = multimodal_embedder.embed(\")\n", - "print(\" text=text_data,\")\n", - "print(\" image=image_data,\")\n", - "print(\" audio=audio_data\")\n", - "print(\" )\")\n", - "print(\"\\nNote: Multimodal embeddings combine multiple data types\")\n", - "print(\" into unified semantic representations\")\n", - "\n", - "if text_embeddings is not None:\n", - " print(f\"\\n✓ All embedding types demonstrated\")\n", - " print(\" Text embeddings: Available\")\n", - " print(\" Image embeddings: Available (with image data)\")\n", - " print(\" Audio embeddings: Available (with audio data)\")\n", - " print(\" Multimodal embeddings: Available (combining all types)\")\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/core_workflows/Embedding_Visualization.ipynb b/cookbook/core_workflows/Embedding_Visualization.ipynb deleted file mode 100644 index 6fab6017..00000000 --- a/cookbook/core_workflows/Embedding_Visualization.ipynb +++ /dev/null @@ -1,188 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Embedding Visualization\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates how to generate embeddings, optimize them, and visualize using t-SNE, PCA, and UMAP dimensionality reduction techniques.\n", - "\n", - "### Learning Objectives\n", - "\n", - "- Generate embeddings for documents\n", - "- Optimize embeddings for better quality\n", - "- Visualize embeddings using t-SNE\n", - "- Visualize embeddings using PCA\n", - "- Visualize embeddings using UMAP\n", - "\n", - "---\n", - "\n", - "## Workflow\n", - "\n", - "**Generate Embeddings → Optimize → Visualize**\n", - "\n", - "Visualization helps understand the structure and relationships in embedding spaces.\n", - "\n", - "---\n", - "\n", - "## Step 1: Generate Embeddings\n", - "\n", - "Start by generating embeddings for your documents.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.embeddings import EmbeddingGenerator\n", - "import numpy as np\n", - "\n", - "documents = [\n", - " \"Machine learning algorithms\",\n", - " \"Deep neural networks\",\n", - " \"Natural language processing\",\n", - " \"Computer vision\",\n", - " \"Reinforcement learning\",\n", - "]\n", - "\n", - "generator = EmbeddingGenerator()\n", - "\n", - "try:\n", - " embeddings = generator.generate(documents)\n", - " print(\"✓ Embeddings generated\")\n", - " print(f\" Documents: {len(documents)}\")\n", - " print(f\" Embedding dimension: {embeddings.shape[1] if hasattr(embeddings, 'shape') else 'N/A'}\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error generating embeddings: {e}\")\n", - " embeddings = np.random.rand(len(documents), 1536).astype(np.float32)\n", - " print(\" Using demo embeddings\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Optimize Embeddings\n", - "\n", - "Optimize embeddings to improve their quality and reduce noise.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.embeddings import EmbeddingOptimizer\n", - "\n", - "optimizer = EmbeddingOptimizer()\n", - "\n", - "try:\n", - " optimized_embeddings = optimizer.optimize(embeddings)\n", - " print(\"✓ Embeddings optimized\")\n", - " print(f\" Optimized embeddings ready for visualization\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error optimizing embeddings: {e}\")\n", - " optimized_embeddings = embeddings\n", - " print(\" Using original embeddings\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Visualize with t-SNE\n", - "\n", - "Use t-SNE (t-Distributed Stochastic Neighbor Embedding) to visualize embeddings in 2D space.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.visualization import EmbeddingVisualizer\n", - "\n", - "visualizer = EmbeddingVisualizer()\n", - "\n", - "labels = [f\"Doc {i+1}\" for i in range(len(documents))]\n", - "\n", - "try:\n", - " visualizer.visualize_tsne(optimized_embeddings, labels)\n", - " print(\"✓ t-SNE visualization complete\")\n", - " print(\" Note: t-SNE shows local structure and clusters in embedding space\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error visualizing with t-SNE: {e}\")\n", - " print(\" Note: t-SNE reduces high-dimensional embeddings to 2D for visualization\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Visualize with PCA\n", - "\n", - "Use PCA (Principal Component Analysis) to visualize embeddings, preserving global structure.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "try:\n", - " visualizer.visualize_pca(optimized_embeddings, labels)\n", - " print(\"✓ PCA visualization complete\")\n", - " print(\" Note: PCA preserves global structure and variance\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error visualizing with PCA: {e}\")\n", - " print(\" Note: PCA reduces dimensions while preserving maximum variance\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Visualize with UMAP\n", - "\n", - "Use UMAP (Uniform Manifold Approximation and Projection) for a balance between local and global structure.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "try:\n", - " visualizer.visualize_umap(optimized_embeddings, labels)\n", - " print(\"✓ UMAP visualization complete\")\n", - " print(\" Note: UMAP balances local and global structure preservation\")\n", - " print(\"\\n✓ Embedding visualization complete\")\n", - " print(\" All visualization methods demonstrate different aspects of embedding space\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error visualizing with UMAP: {e}\")\n", - " print(\" Note: UMAP provides a good balance between t-SNE and PCA\")\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/core_workflows/Entity_Resolution_Workflow.ipynb b/cookbook/core_workflows/Entity_Resolution_Workflow.ipynb deleted file mode 100644 index c548c0c0..00000000 --- a/cookbook/core_workflows/Entity_Resolution_Workflow.ipynb +++ /dev/null @@ -1,202 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Entity Resolution Workflow\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates the complete entity resolution workflow: extract entities, detect duplicates, resolve conflicts, merge entities, and validate the results.\n", - "\n", - "### Learning Objectives\n", - "\n", - "- Learn to extract entities from documents\n", - "- Detect duplicate entities\n", - "- Resolve entity conflicts\n", - "- Merge duplicate entities\n", - "- Validate resolved entities\n", - "\n", - "---\n", - "\n", - "## Workflow\n", - "\n", - "**Extract → Detect Duplicates → Resolve → Merge → Validate**\n", - "\n", - "This workflow ensures clean, deduplicated entities ready for knowledge graph construction.\n", - "\n", - "---\n", - "\n", - "## Step 1: Extract Entities\n", - "\n", - "Start by extracting entities from your documents.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.semantic_extract import NERExtractor\n", - "\n", - "sample_text = \"\"\"\n", - "Amazon.com Inc. is an American technology company.\n", - "Amazon was founded by Jeff Bezos in 1994.\n", - "The company is based in Seattle, Washington.\n", - "Andy Jassy is the current CEO of Amazon.\n", - "\"\"\"\n", - "\n", - "extractor = NERExtractor()\n", - "\n", - "try:\n", - " print(\"Extracting entities...\")\n", - " entities = [\n", - " {\"id\": \"e1\", \"text\": \"Amazon.com Inc.\", \"type\": \"Organization\"},\n", - " {\"id\": \"e2\", \"text\": \"Amazon\", \"type\": \"Organization\"},\n", - " {\"id\": \"e3\", \"text\": \"Jeff Bezos\", \"type\": \"Person\"},\n", - " {\"id\": \"e4\", \"text\": \"1994\", \"type\": \"Date\"},\n", - " {\"id\": \"e5\", \"text\": \"Seattle, Washington\", \"type\": \"Location\"},\n", - " {\"id\": \"e6\", \"text\": \"Andy Jassy\", \"type\": \"Person\"},\n", - " ]\n", - " \n", - " print(f\"✓ Extracted {len(entities)} entities\")\n", - " for entity in entities:\n", - " print(f\" - {entity['text']} ({entity['type']})\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error extracting entities: {e}\")\n", - " entities = []\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Detect Duplicates\n", - "\n", - "Detect duplicate entities that refer to the same real-world entity (e.g., \"Amazon.com Inc.\" and \"Amazon\").\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.deduplication import DuplicateDetector\n", - "\n", - "detector = DuplicateDetector()\n", - "\n", - "try:\n", - " duplicates = detector.detect(entities)\n", - " print(f\"✓ Detected {len(duplicates) if duplicates else 0} duplicate groups\")\n", - " if duplicates:\n", - " for dup_group in duplicates:\n", - " print(f\" Duplicate group: {dup_group}\")\n", - " else:\n", - " print(\" Note: 'Amazon.com Inc.' and 'Amazon' would be detected as duplicates\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error detecting duplicates: {e}\")\n", - " duplicates = []\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Resolve Entities\n", - "\n", - "Resolve entity conflicts and determine the canonical representation for each entity.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import EntityResolver\n", - "\n", - "resolver = EntityResolver()\n", - "\n", - "try:\n", - " resolved_entities = resolver.resolve(entities, duplicates)\n", - " print(f\"✓ Resolved {len(resolved_entities) if resolved_entities else len(entities)} entities\")\n", - " print(\" Note: Duplicate entities are resolved to canonical forms\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error resolving entities: {e}\")\n", - " resolved_entities = entities\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Merge Entities\n", - "\n", - "Merge duplicate entities into single canonical entities.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.deduplication import EntityMerger\n", - "\n", - "merger = EntityMerger()\n", - "\n", - "try:\n", - " merged_entities = merger.merge(resolved_entities)\n", - " print(f\"✓ Merged to {len(merged_entities) if merged_entities else len(resolved_entities)} unique entities\")\n", - " print(\" Note: Duplicates are merged into single entities\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error merging entities: {e}\")\n", - " merged_entities = resolved_entities\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Validate\n", - "\n", - "Validate the resolved and merged entities to ensure quality and consistency.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import GraphValidator\n", - "\n", - "validator = GraphValidator()\n", - "\n", - "try:\n", - " validation_results = validator.validate(merged_entities)\n", - " print(\"✓ Validation complete\")\n", - " print(f\" Validated {len(merged_entities)} unique entities\")\n", - " print(\" Entities are ready for knowledge graph construction\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error validating entities: {e}\")\n", - " print(f\" Using {len(merged_entities)} entities\")\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/core_workflows/From_Unstructured_to_Structured.ipynb b/cookbook/core_workflows/From_Unstructured_to_Structured.ipynb deleted file mode 100644 index 99c60f30..00000000 --- a/cookbook/core_workflows/From_Unstructured_to_Structured.ipynb +++ /dev/null @@ -1,185 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# From Unstructured to Structured\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates how to transform raw, unstructured documents into structured data through parsing, normalization, and entity extraction.\n", - "\n", - "### Learning Objectives\n", - "\n", - "- Learn to ingest documents from various formats\n", - "- Parse documents to extract content\n", - "- Normalize text for processing\n", - "- Extract structured entities from text\n", - "\n", - "---\n", - "\n", - "## Workflow\n", - "\n", - "**Raw Documents → Parsed → Normalized → Structured Data**\n", - "\n", - "Each step transforms the data further toward a structured format suitable for knowledge graph construction.\n", - "\n", - "---\n", - "\n", - "## Step 1: Ingest Raw Documents\n", - "\n", - "Start by ingesting documents from various sources. The `FileIngestor` supports multiple formats including PDF, DOCX, HTML, JSON, and more.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FileIngestor\n", - "from pathlib import Path\n", - "\n", - "ingestor = FileIngestor()\n", - "\n", - "sample_text = \"\"\"\n", - "Microsoft Corporation is an American multinational technology company.\n", - "It was founded by Bill Gates and Paul Allen in 1975.\n", - "The company is headquartered in Redmond, Washington.\n", - "Satya Nadella is the current CEO of Microsoft.\n", - "Microsoft develops software, services, and hardware products.\n", - "\"\"\"\n", - "\n", - "sample_file = Path(\"sample_document.txt\")\n", - "sample_file.write_text(sample_text)\n", - "\n", - "print(\"Sample document created\")\n", - "print(f\"File: {sample_file}\")\n", - "\n", - "try:\n", - " file_object = ingestor.ingest_file(sample_file, read_content=True)\n", - " print(f\"\\n✓ File ingested successfully!\")\n", - " print(f\" File name: {file_object.name}\")\n", - " print(f\" File type: {file_object.file_type}\")\n", - "except Exception as e:\n", - " print(f\"\\n✗ Error ingesting file: {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Parse Documents\n", - "\n", - "Parse the ingested documents to extract structured content. The `DocumentParser` handles various file formats and extracts text, metadata, and structure.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.parse import DocumentParser\n", - "\n", - "parser = DocumentParser()\n", - "\n", - "try:\n", - " parsed_content = parser.parse_document(str(sample_file))\n", - " print(\"✓ Document parsed successfully!\")\n", - " print(f\" Parsed content length: {len(parsed_content) if parsed_content else 0} characters\")\n", - " print(f\" Preview: {parsed_content[:150] if parsed_content else 'N/A'}...\")\n", - "except Exception as e:\n", - " print(f\"✗ Error parsing document: {e}\")\n", - " parsed_content = sample_text\n", - " print(\"Using raw text as fallback\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Normalize Text\n", - "\n", - "Normalize the parsed text to clean and standardize it for further processing. This includes fixing encoding, removing noise, and standardizing formats.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.normalize import TextNormalizer\n", - "\n", - "normalizer = TextNormalizer()\n", - "\n", - "try:\n", - " normalized_content = normalizer.normalize(parsed_content)\n", - " print(\"✓ Text normalized successfully!\")\n", - " print(f\" Normalized content length: {len(normalized_content) if normalized_content else 0} characters\")\n", - "except Exception as e:\n", - " print(f\"✗ Error normalizing text: {e}\")\n", - " normalized_content = parsed_content\n", - " print(\"Using parsed content as fallback\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Extract Entities\n", - "\n", - "Extract structured entities from the normalized text. This transforms unstructured text into structured entity data that can be used for knowledge graph construction.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.semantic_extract import NERExtractor\n", - "\n", - "extractor = NERExtractor()\n", - "\n", - "try:\n", - " print(\"Extracting entities from normalized text...\")\n", - " print(f\"\\nText: {normalized_content[:100]}...\")\n", - " \n", - " expected_entities = [\n", - " {\"text\": \"Microsoft Corporation\", \"type\": \"Organization\"},\n", - " {\"text\": \"Bill Gates\", \"type\": \"Person\"},\n", - " {\"text\": \"Paul Allen\", \"type\": \"Person\"},\n", - " {\"text\": \"1975\", \"type\": \"Date\"},\n", - " {\"text\": \"Redmond, Washington\", \"type\": \"Location\"},\n", - " {\"text\": \"Satya Nadella\", \"type\": \"Person\"},\n", - " ]\n", - " \n", - " print(f\"\\n✓ Found {len(expected_entities)} entities:\")\n", - " for entity in expected_entities:\n", - " print(f\" - {entity['text']} ({entity['type']})\")\n", - " \n", - " print(\"\\n✓ Transformation complete: Unstructured → Structured Data\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error extracting entities: {e}\")\n", - "\n", - "try:\n", - " if sample_file.exists():\n", - " sample_file.unlink()\n", - " print(\"\\n✓ Sample file cleaned up\")\n", - "except:\n", - " pass\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/core_workflows/Graph_Analytics_Complete.ipynb b/cookbook/core_workflows/Graph_Analytics_Complete.ipynb deleted file mode 100644 index a617c6ff..00000000 --- a/cookbook/core_workflows/Graph_Analytics_Complete.ipynb +++ /dev/null @@ -1,194 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Graph Analytics Complete\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates comprehensive graph analytics: build a knowledge graph, calculate centrality, detect communities, and analyze connectivity.\n", - "\n", - "### Learning Objectives\n", - "\n", - "- Build a knowledge graph for analysis\n", - "- Calculate node centrality to identify important entities\n", - "- Detect communities in the graph\n", - "- Analyze graph connectivity and structure\n", - "\n", - "---\n", - "\n", - "## Workflow\n", - "\n", - "**Build KG → Calculate Centrality → Detect Communities → Analyze Connectivity**\n", - "\n", - "Each step provides insights into your knowledge graph structure.\n", - "\n", - "---\n", - "\n", - "## Step 1: Build Knowledge Graph\n", - "\n", - "Start by building a knowledge graph from entities and relationships.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import GraphBuilder\n", - "import networkx as nx\n", - "\n", - "entities = [\n", - " {\"id\": \"e1\", \"name\": \"Company A\", \"type\": \"Organization\"},\n", - " {\"id\": \"e2\", \"name\": \"Person 1\", \"type\": \"Person\"},\n", - " {\"id\": \"e3\", \"name\": \"Person 2\", \"type\": \"Person\"},\n", - " {\"id\": \"e4\", \"name\": \"Location 1\", \"type\": \"Location\"},\n", - "]\n", - "\n", - "relationships = [\n", - " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"employs\"},\n", - " {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"employs\"},\n", - " {\"source\": \"e1\", \"target\": \"e4\", \"type\": \"located_in\"},\n", - " {\"source\": \"e2\", \"target\": \"e3\", \"type\": \"works_with\"},\n", - "]\n", - "\n", - "builder = GraphBuilder()\n", - "\n", - "try:\n", - " kg = nx.DiGraph()\n", - " for entity in entities:\n", - " kg.add_node(entity[\"id\"], name=entity[\"name\"], type=entity[\"type\"])\n", - " for rel in relationships:\n", - " kg.add_edge(rel[\"source\"], rel[\"target\"], type=rel[\"type\"])\n", - " \n", - " print(f\"✓ Knowledge graph built: {len(kg.nodes)} nodes, {len(kg.edges)} edges\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error building graph: {e}\")\n", - " kg = None\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Calculate Centrality\n", - "\n", - "Calculate centrality metrics to identify the most important nodes in the graph.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import CentralityCalculator\n", - "\n", - "if kg is not None:\n", - " centrality_calc = CentralityCalculator()\n", - " \n", - " try:\n", - " centrality_scores = centrality_calc.calculate(kg)\n", - " print(\"✓ Centrality calculated\")\n", - " \n", - " if centrality_scores:\n", - " top_nodes = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:5]\n", - " print(\"\\nTop nodes by centrality:\")\n", - " for node_id, score in top_nodes:\n", - " node_name = kg.nodes[node_id].get('name', node_id)\n", - " print(f\" - {node_name}: {score:.4f}\")\n", - " else:\n", - " print(\" Note: Centrality scores would show node importance\")\n", - " \n", - " except Exception as e:\n", - " print(f\"✗ Error calculating centrality: {e}\")\n", - "else:\n", - " print(\"No graph available for centrality calculation\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Detect Communities\n", - "\n", - "Detect communities (clusters) in the knowledge graph to identify groups of related entities.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import CommunityDetector\n", - "\n", - "if kg is not None:\n", - " detector = CommunityDetector()\n", - " \n", - " try:\n", - " communities = detector.detect(kg)\n", - " print(f\"✓ Detected {len(communities) if communities else 0} communities\")\n", - " if communities:\n", - " for i, community in enumerate(communities[:3]):\n", - " print(f\" Community {i+1}: {len(community)} nodes\")\n", - " else:\n", - " print(\" Note: Communities represent groups of closely connected nodes\")\n", - " \n", - " except Exception as e:\n", - " print(f\"✗ Error detecting communities: {e}\")\n", - "else:\n", - " print(\"No graph available for community detection\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Analyze Connectivity\n", - "\n", - "Analyze the connectivity of the graph to understand its structure and properties.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import ConnectivityAnalyzer\n", - "\n", - "if kg is not None:\n", - " analyzer = ConnectivityAnalyzer()\n", - " \n", - " try:\n", - " connectivity_metrics = analyzer.analyze(kg)\n", - " print(\"✓ Connectivity analysis complete\")\n", - " \n", - " if connectivity_metrics:\n", - " print(f\"\\nGraph Metrics:\")\n", - " print(f\" Density: {connectivity_metrics.get('density', 'N/A')}\")\n", - " print(f\" Average path length: {connectivity_metrics.get('avg_path_length', 'N/A')}\")\n", - " print(f\" Connected components: {connectivity_metrics.get('components', 'N/A')}\")\n", - " else:\n", - " print(\" Note: Connectivity metrics show graph structure properties\")\n", - " \n", - " except Exception as e:\n", - " print(f\"✗ Error analyzing connectivity: {e}\")\n", - "else:\n", - " print(\"No graph available for connectivity analysis\")\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/core_workflows/Graph_Quality_Assurance.ipynb b/cookbook/core_workflows/Graph_Quality_Assurance.ipynb deleted file mode 100644 index 2454620c..00000000 --- a/cookbook/core_workflows/Graph_Quality_Assurance.ipynb +++ /dev/null @@ -1,178 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Graph Quality Assurance\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates how to assess knowledge graph quality, validate structure, auto-fix issues, and generate quality reports.\n", - "\n", - "### Learning Objectives\n", - "\n", - "- Assess overall graph quality metrics\n", - "- Validate graph structure and consistency\n", - "- Automatically fix common issues\n", - "- Generate comprehensive quality reports\n", - "\n", - "---\n", - "\n", - "## Workflow\n", - "\n", - "**Assess Quality → Validate → Auto-Fix Issues → Generate Reports**\n", - "\n", - "This workflow ensures your knowledge graph meets quality standards.\n", - "\n", - "---\n", - "\n", - "## Step 1: Assess Quality\n", - "\n", - "Start by assessing the overall quality of your knowledge graph.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg_qa import KGQualityAssessor\n", - "import networkx as nx\n", - "\n", - "kg = nx.DiGraph()\n", - "kg.add_node(\"e1\", name=\"Entity 1\", type=\"Organization\")\n", - "kg.add_node(\"e2\", name=\"Entity 2\", type=\"Person\")\n", - "kg.add_edge(\"e1\", \"e2\", type=\"employs\")\n", - "\n", - "assessor = KGQualityAssessor()\n", - "\n", - "try:\n", - " quality_metrics = assessor.assess(kg)\n", - " print(\"✓ Quality assessment complete\")\n", - " \n", - " if quality_metrics:\n", - " print(f\"\\nQuality Metrics:\")\n", - " print(f\" Completeness: {quality_metrics.get('completeness', 'N/A')}\")\n", - " print(f\" Consistency: {quality_metrics.get('consistency', 'N/A')}\")\n", - " print(f\" Accuracy: {quality_metrics.get('accuracy', 'N/A')}\")\n", - " else:\n", - " print(\" Note: Quality metrics assess graph completeness, consistency, and accuracy\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error assessing quality: {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Validate\n", - "\n", - "Validate the graph structure and check for errors or inconsistencies.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg_qa import ValidationEngine\n", - "\n", - "validator = ValidationEngine()\n", - "\n", - "try:\n", - " validation_results = validator.validate(kg)\n", - " print(\"✓ Validation complete\")\n", - " \n", - " if validation_results:\n", - " if hasattr(validation_results, 'has_errors'):\n", - " if validation_results.has_errors():\n", - " errors = validation_results.errors if hasattr(validation_results, 'errors') else []\n", - " print(f\" Found {len(errors)} errors\")\n", - " else:\n", - " print(\" No errors detected\")\n", - " else:\n", - " print(\" Validation results available\")\n", - " else:\n", - " print(\" Note: Validation checks graph structure and consistency\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error validating graph: {e}\")\n", - " validation_results = None\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Auto-Fix Issues\n", - "\n", - "Automatically fix common issues detected during validation.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg_qa import AutomatedFixer\n", - "\n", - "fixer = AutomatedFixer()\n", - "\n", - "try:\n", - " fixed_graph = fixer.fix(kg, validation_results)\n", - " print(\"✓ Auto-fix complete\")\n", - " print(f\" Fixed graph: {len(fixed_graph.nodes) if fixed_graph else len(kg.nodes)} nodes\")\n", - " print(\" Note: Common issues are automatically resolved\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error auto-fixing: {e}\")\n", - " fixed_graph = kg\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Generate Reports\n", - "\n", - "Generate a comprehensive quality report summarizing all quality metrics and validation results.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg_qa import QualityReporter\n", - "\n", - "reporter = QualityReporter()\n", - "\n", - "try:\n", - " report = reporter.generate_report(kg, quality_metrics, validation_results)\n", - " print(\"✓ Quality report generated successfully\")\n", - " \n", - " if report:\n", - " print(f\" Report generated: {len(str(report))} characters\")\n", - " print(\" Report includes quality metrics, validation results, and recommendations\")\n", - " else:\n", - " print(\" Note: Quality reports provide comprehensive graph quality assessment\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error generating quality report: {e}\")\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/core_workflows/Multi_Source_Data_Ingestion.ipynb b/cookbook/core_workflows/Multi_Source_Data_Ingestion.ipynb deleted file mode 100644 index f0f485e3..00000000 --- a/cookbook/core_workflows/Multi_Source_Data_Ingestion.ipynb +++ /dev/null @@ -1,212 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Multi-Source Data Ingestion\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates how to ingest data from multiple sources (files, web, feeds, streams, and databases) and process them through a unified pipeline.\n", - "\n", - "### Learning Objectives\n", - "\n", - "- Learn to ingest from various data sources\n", - "- Combine data from multiple sources\n", - "- Process diverse data through a unified pipeline\n", - "\n", - "---\n", - "\n", - "## Unified Processing Pipeline\n", - "\n", - "Semantica provides specialized ingestors for different data sources, all producing a unified document format that can be processed together.\n", - "\n", - "---\n", - "\n", - "## Step 1: Ingest from Files\n", - "\n", - "Start by ingesting documents from local files or directories.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FileIngestor\n", - "from pathlib import Path\n", - "\n", - "file_ingestor = FileIngestor()\n", - "\n", - "sample_file = Path(\"sample_file.txt\")\n", - "sample_file.write_text(\"Sample file content for ingestion demonstration.\")\n", - "\n", - "try:\n", - " file_docs = file_ingestor.ingest_file(sample_file, read_content=True)\n", - " print(\"✓ Files ingested successfully!\")\n", - " print(f\" Document: {file_docs.name if hasattr(file_docs, 'name') else 'N/A'}\")\n", - "except Exception as e:\n", - " print(f\"✗ Error ingesting files: {e}\")\n", - " file_docs = []\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Ingest from Web\n", - "\n", - "Ingest content from web pages using the `WebIngestor`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import WebIngestor\n", - "\n", - "web_ingestor = WebIngestor()\n", - "\n", - "print(\"Web ingestion example:\")\n", - "print(\" web_docs = web_ingestor.ingest('https://example.com')\")\n", - "print(\"\\nNote: Actual web ingestion requires valid URLs and network access\")\n", - "web_docs = []\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Ingest from Feeds\n", - "\n", - "Ingest content from RSS/Atom feeds using the `FeedIngestor`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FeedIngestor\n", - "\n", - "feed_ingestor = FeedIngestor()\n", - "\n", - "print(\"Feed ingestion example:\")\n", - "print(\" feed_docs = feed_ingestor.ingest('https://example.com/feed.xml')\")\n", - "print(\"\\nNote: Actual feed ingestion requires valid feed URLs\")\n", - "feed_docs = []\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Ingest from Streams\n", - "\n", - "Ingest real-time data from streams using the `StreamIngestor`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import StreamIngestor\n", - "\n", - "stream_ingestor = StreamIngestor()\n", - "\n", - "print(\"Stream ingestion example:\")\n", - "print(\" stream_docs = stream_ingestor.ingest(stream_source)\")\n", - "print(\"\\nNote: Stream ingestion requires configured stream sources (Kafka, RabbitMQ, etc.)\")\n", - "stream_docs = []\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Ingest from Databases\n", - "\n", - "Ingest data from databases using the `DBIngestor`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import DBIngestor\n", - "\n", - "print(\"Database ingestion example:\")\n", - "print(\" db_ingestor = DBIngestor(connection_string='...')\")\n", - "print(\" db_docs = db_ingestor.ingest(query='SELECT * FROM table')\")\n", - "print(\"\\nNote: Database ingestion requires valid connection strings and queries\")\n", - "db_docs = []\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 6: Unified Processing\n", - "\n", - "Combine all documents from different sources and process them through a unified pipeline.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.parse import DocumentParser\n", - "\n", - "all_docs = []\n", - "if file_docs:\n", - " all_docs.append(file_docs)\n", - "all_docs.extend(web_docs)\n", - "all_docs.extend(feed_docs)\n", - "all_docs.extend(stream_docs)\n", - "all_docs.extend(db_docs)\n", - "\n", - "print(f\"Total documents from all sources: {len(all_docs)}\")\n", - "\n", - "parser = DocumentParser()\n", - "\n", - "if all_docs:\n", - " try:\n", - " parsed_docs = []\n", - " for doc in all_docs:\n", - " if hasattr(doc, 'content') and doc.content:\n", - " parsed = parser.parse_document(doc.content)\n", - " parsed_docs.append(parsed)\n", - " \n", - " print(f\"\\n✓ Processed {len(parsed_docs)} documents through unified pipeline\")\n", - " except Exception as e:\n", - " print(f\"\\n✗ Error processing documents: {e}\")\n", - "else:\n", - " print(\"\\nNote: Add documents from various sources to see unified processing\")\n", - "\n", - "try:\n", - " if sample_file.exists():\n", - " sample_file.unlink()\n", - "except:\n", - " pass\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/core_workflows/Semantic_Search_Pipeline.ipynb b/cookbook/core_workflows/Semantic_Search_Pipeline.ipynb deleted file mode 100644 index 976e238c..00000000 --- a/cookbook/core_workflows/Semantic_Search_Pipeline.ipynb +++ /dev/null @@ -1,177 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Semantic Search Pipeline\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates a production-ready semantic search pipeline: process documents, generate embeddings, store in vector database, and retrieve results.\n", - "\n", - "### Learning Objectives\n", - "\n", - "- Parse documents for search\n", - "- Generate embeddings for semantic search\n", - "- Store embeddings in a vector store\n", - "- Query and retrieve relevant results\n", - "\n", - "---\n", - "\n", - "## Workflow\n", - "\n", - "**Documents → Embeddings → Vector Store → Query → Results**\n", - "\n", - "This complete pipeline enables production-ready semantic search.\n", - "\n", - "---\n", - "\n", - "## Step 1: Parse Documents\n", - "\n", - "Start by parsing documents to extract searchable content.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.parse import DocumentParser\n", - "from pathlib import Path\n", - "\n", - "sample_docs = [\n", - " \"Python is a high-level programming language.\",\n", - " \"JavaScript is used for web development.\",\n", - " \"Machine learning algorithms learn from data.\",\n", - "]\n", - "\n", - "parser = DocumentParser()\n", - "\n", - "try:\n", - " parsed_docs = []\n", - " for doc in sample_docs:\n", - " parsed = parser.parse_document(doc)\n", - " parsed_docs.append(parsed)\n", - " \n", - " print(f\"✓ Parsed {len(parsed_docs)} documents\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error parsing documents: {e}\")\n", - " parsed_docs = sample_docs\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Generate Embeddings\n", - "\n", - "Generate embeddings for the parsed documents to enable semantic search.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.embeddings import EmbeddingGenerator\n", - "import numpy as np\n", - "\n", - "generator = EmbeddingGenerator()\n", - "\n", - "try:\n", - " embeddings = generator.generate(parsed_docs)\n", - " print(\"✓ Embeddings generated\")\n", - " print(f\" Documents: {len(parsed_docs)}\")\n", - " print(f\" Embeddings ready for storage\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error generating embeddings: {e}\")\n", - " embeddings = np.random.rand(len(parsed_docs), 1536).astype(np.float32)\n", - " print(\" Using demo embeddings\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Store in Vector Store\n", - "\n", - "Store the embeddings along with documents and metadata in a vector store.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import VectorStore\n", - "\n", - "vector_store = VectorStore()\n", - "\n", - "metadata = [{\"id\": i, \"source\": \"demo\"} for i in range(len(parsed_docs))]\n", - "\n", - "try:\n", - " vector_store.store(embeddings, parsed_docs, metadata)\n", - " print(\"✓ Documents stored in vector store\")\n", - " print(f\" Stored {len(parsed_docs)} documents\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error storing in vector store: {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Query and Retrieve\n", - "\n", - "Query the vector store and retrieve the most relevant results.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import VectorRetriever\n", - "\n", - "retriever = VectorRetriever(vector_store)\n", - "\n", - "query = \"programming language\"\n", - "query_embedding = generator.generate([query])[0] if hasattr(generator, 'generate') else np.random.rand(1536).astype(np.float32)\n", - "\n", - "try:\n", - " results = retriever.retrieve(query_embedding, top_k=3)\n", - " \n", - " print(\"✓ Production-ready semantic search complete\")\n", - " print(f\" Query: '{query}'\")\n", - " print(f\" Found {len(results) if results else 0} results\")\n", - " \n", - " if results:\n", - " print(\"\\nTop Results:\")\n", - " for i, result in enumerate(results):\n", - " score = result.score if hasattr(result, 'score') else 'N/A'\n", - " doc = result.document if hasattr(result, 'document') else 'N/A'\n", - " print(f\" {i+1}. Score: {score}, Document: {doc[:60]}...\")\n", - " else:\n", - " print(\" Note: Results would show most semantically similar documents\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error retrieving results: {e}\")\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/core_workflows/Temporal_Knowledge_Graphs.ipynb b/cookbook/core_workflows/Temporal_Knowledge_Graphs.ipynb deleted file mode 100644 index 68b74971..00000000 --- a/cookbook/core_workflows/Temporal_Knowledge_Graphs.ipynb +++ /dev/null @@ -1,184 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Temporal Knowledge Graphs\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates how to build time-aware knowledge graphs, create snapshots, perform time-point queries, and track history.\n", - "\n", - "### Learning Objectives\n", - "\n", - "- Build temporal knowledge graphs with time information\n", - "- Create snapshots at specific time points\n", - "- Query the graph at specific times\n", - "- Track entity and relationship history over time\n", - "\n", - "---\n", - "\n", - "## Workflow\n", - "\n", - "**Build Temporal KG → Create Snapshots → Time-Point Queries → Track History**\n", - "\n", - "Temporal graphs enable time-aware queries and historical analysis.\n", - "\n", - "---\n", - "\n", - "## Step 1: Build Temporal Knowledge Graph\n", - "\n", - "Build a knowledge graph with temporal support to track changes over time.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import GraphBuilder\n", - "import networkx as nx\n", - "\n", - "entities = [\n", - " {\"id\": \"e1\", \"name\": \"Company X\", \"type\": \"Organization\"},\n", - " {\"id\": \"e2\", \"name\": \"CEO A\", \"type\": \"Person\"},\n", - " {\"id\": \"e3\", \"name\": \"CEO B\", \"type\": \"Person\"},\n", - "]\n", - "\n", - "relationships = [\n", - " {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"has_ceo\", \"valid_from\": \"2020-01-01\", \"valid_to\": \"2023-12-31\"},\n", - " {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"has_ceo\", \"valid_from\": \"2024-01-01\", \"valid_to\": None},\n", - "]\n", - "\n", - "builder = GraphBuilder()\n", - "\n", - "try:\n", - " temporal_kg = nx.DiGraph()\n", - " for entity in entities:\n", - " temporal_kg.add_node(entity[\"id\"], name=entity[\"name\"], type=entity[\"type\"])\n", - " for rel in relationships:\n", - " temporal_kg.add_edge(rel[\"source\"], rel[\"target\"], type=rel[\"type\"], \n", - " valid_from=rel.get(\"valid_from\"), valid_to=rel.get(\"valid_to\"))\n", - " \n", - " print(f\"✓ Temporal knowledge graph built: {len(temporal_kg.nodes)} nodes, {len(temporal_kg.edges)} edges\")\n", - " print(\" Note: Relationships have temporal validity periods\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error building temporal graph: {e}\")\n", - " temporal_kg = None\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Create Snapshots\n", - "\n", - "Create snapshots of the graph at specific time points to capture the state at that moment.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import TemporalVersionManager\n", - "\n", - "if temporal_kg is not None:\n", - " version_manager = TemporalVersionManager()\n", - " \n", - " try:\n", - " snapshot = version_manager.create_snapshot(temporal_kg, timestamp=\"2024-01-01\")\n", - " print(\"✓ Snapshot created for 2024-01-01\")\n", - " print(f\" Snapshot nodes: {len(snapshot.nodes) if snapshot else 'N/A'}\")\n", - " print(\" Note: Snapshots capture graph state at specific time points\")\n", - " \n", - " except Exception as e:\n", - " print(f\"✗ Error creating snapshot: {e}\")\n", - "else:\n", - " print(\"No temporal graph available for snapshots\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Time-Point Queries\n", - "\n", - "Query the graph at specific time points to see what the graph looked like at that time.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import TemporalGraphQuery\n", - "\n", - "if temporal_kg is not None:\n", - " temporal_query = TemporalGraphQuery()\n", - " \n", - " try:\n", - " graph_at_time = temporal_query.query_at_time(temporal_kg, \"2024-01-01\")\n", - " print(\"✓ Queried graph at 2024-01-01\")\n", - " print(f\" Nodes at this time: {len(graph_at_time.nodes) if graph_at_time else 'N/A'}\")\n", - " \n", - " changes = temporal_query.query_changes(temporal_kg, \"2020-01-01\", \"2024-12-31\")\n", - " print(f\"\\n✓ Queried changes between 2020-01-01 and 2024-12-31\")\n", - " print(f\" Changes detected: {len(changes) if changes else 0}\")\n", - " \n", - " except Exception as e:\n", - " print(f\"✗ Error querying temporal graph: {e}\")\n", - "else:\n", - " print(\"No temporal graph available for queries\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Track History\n", - "\n", - "Track the evolution of specific entities and relationships over time.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if temporal_kg is not None and 'temporal_query' in locals():\n", - " try:\n", - " entity_history = temporal_query.get_entity_history(temporal_kg, \"e1\")\n", - " print(\"✓ Retrieved entity history\")\n", - " print(f\" History entries: {len(entity_history) if entity_history else 0}\")\n", - " \n", - " if temporal_kg.edges():\n", - " first_edge = list(temporal_kg.edges(data=True))[0]\n", - " rel_id = f\"{first_edge[0]}-{first_edge[1]}\"\n", - " relationship_history = temporal_query.get_relationship_history(temporal_kg, rel_id)\n", - " print(f\"\\n✓ Retrieved relationship history\")\n", - " print(f\" History entries: {len(relationship_history) if relationship_history else 0}\")\n", - " \n", - " print(f\"\\n✓ Temporal graph has {len(temporal_kg.nodes)} nodes across time\")\n", - " \n", - " except Exception as e:\n", - " print(f\"✗ Error tracking history: {e}\")\n", - "else:\n", - " print(\"No temporal graph available for history tracking\")\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/core_workflows/Text_Processing_Pipeline.ipynb b/cookbook/core_workflows/Text_Processing_Pipeline.ipynb deleted file mode 100644 index 2e724550..00000000 --- a/cookbook/core_workflows/Text_Processing_Pipeline.ipynb +++ /dev/null @@ -1,185 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Text Processing Pipeline\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates a complete text processing pipeline: normalize, clean, extract entities, and extract relationships from text.\n", - "\n", - "### Learning Objectives\n", - "\n", - "- Learn to normalize text for processing\n", - "- Clean and prepare text data\n", - "- Extract entities from text\n", - "- Extract relationships between entities\n", - "\n", - "---\n", - "\n", - "## Workflow\n", - "\n", - "**Normalize → Clean → Extract Entities → Extract Relationships**\n", - "\n", - "Each step prepares the text for the next stage of analysis.\n", - "\n", - "---\n", - "\n", - "## Step 1: Normalize Text\n", - "\n", - "Normalize text to standardize formats, fix encoding issues, and prepare for further processing.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.normalize import TextNormalizer\n", - "\n", - "sample_text = \"\"\"\n", - "Google LLC is an American multinational technology company.\n", - "It was founded by Larry Page and Sergey Brin in 1998.\n", - "The company is headquartered in Mountain View, California.\n", - "Sundar Pichai is the current CEO of Google.\n", - "\"\"\"\n", - "\n", - "normalizer = TextNormalizer()\n", - "\n", - "try:\n", - " normalized_text = normalizer.normalize(sample_text)\n", - " print(\"✓ Text normalized successfully!\")\n", - " print(f\" Normalized length: {len(normalized_text) if normalized_text else 0} characters\")\n", - "except Exception as e:\n", - " print(f\"✗ Error normalizing text: {e}\")\n", - " normalized_text = sample_text\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Clean Data\n", - "\n", - "Clean the normalized text to remove noise, fix formatting issues, and prepare for entity extraction.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.normalize import DataCleaner\n", - "\n", - "cleaner = DataCleaner()\n", - "\n", - "try:\n", - " cleaned_text = cleaner.clean(normalized_text)\n", - " print(\"✓ Text cleaned successfully!\")\n", - " print(f\" Cleaned length: {len(cleaned_text) if cleaned_text else 0} characters\")\n", - "except Exception as e:\n", - " print(f\"✗ Error cleaning text: {e}\")\n", - " cleaned_text = normalized_text\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Extract Entities\n", - "\n", - "Extract named entities from the cleaned text using Named Entity Recognition (NER).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.semantic_extract import NERExtractor\n", - "\n", - "extractor = NERExtractor()\n", - "\n", - "try:\n", - " print(\"Extracting entities...\")\n", - " print(f\"\\nText: {cleaned_text[:100]}...\")\n", - " \n", - " expected_entities = [\n", - " {\"text\": \"Google LLC\", \"type\": \"Organization\"},\n", - " {\"text\": \"Larry Page\", \"type\": \"Person\"},\n", - " {\"text\": \"Sergey Brin\", \"type\": \"Person\"},\n", - " {\"text\": \"1998\", \"type\": \"Date\"},\n", - " {\"text\": \"Mountain View, California\", \"type\": \"Location\"},\n", - " {\"text\": \"Sundar Pichai\", \"type\": \"Person\"},\n", - " ]\n", - " \n", - " print(f\"\\n✓ Found {len(expected_entities)} entities:\")\n", - " for entity in expected_entities:\n", - " print(f\" - {entity['text']} ({entity['type']})\")\n", - " \n", - " entities = expected_entities\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error extracting entities: {e}\")\n", - " entities = []\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Extract Relationships\n", - "\n", - "Extract relationships between the identified entities to understand how they connect.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.semantic_extract import RelationExtractor\n", - "\n", - "relation_extractor = RelationExtractor()\n", - "\n", - "try:\n", - " print(\"Extracting relationships...\")\n", - " \n", - " expected_relationships = [\n", - " {\"source\": \"Google LLC\", \"target\": \"Larry Page\", \"type\": \"founded_by\"},\n", - " {\"source\": \"Google LLC\", \"target\": \"Sergey Brin\", \"type\": \"founded_by\"},\n", - " {\"source\": \"Google LLC\", \"target\": \"1998\", \"type\": \"founded_in\"},\n", - " {\"source\": \"Google LLC\", \"target\": \"Mountain View, California\", \"type\": \"located_in\"},\n", - " {\"source\": \"Sundar Pichai\", \"target\": \"Google LLC\", \"type\": \"ceo_of\"},\n", - " ]\n", - " \n", - " print(f\"\\n✓ Found {len(expected_relationships)} relationships:\")\n", - " for rel in expected_relationships:\n", - " print(f\" - {rel['source']} --[{rel['type']}]--> {rel['target']}\")\n", - " \n", - " relationships = expected_relationships\n", - " \n", - " print(f\"\\n✓ Complete text analysis results:\")\n", - " print(f\" Entities: {len(entities)}\")\n", - " print(f\" Relationships: {len(relationships)}\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error extracting relationships: {e}\")\n", - " relationships = []\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/core_workflows/Vector_Store_Complete.ipynb b/cookbook/core_workflows/Vector_Store_Complete.ipynb deleted file mode 100644 index 4ca620c6..00000000 --- a/cookbook/core_workflows/Vector_Store_Complete.ipynb +++ /dev/null @@ -1,187 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Vector Store Complete\n", - "\n", - "## Overview\n", - "\n", - "This notebook demonstrates the complete vector store workflow: generate embeddings, store them in a vector database, perform searches, and use hybrid search.\n", - "\n", - "### Learning Objectives\n", - "\n", - "- Generate embeddings for documents\n", - "- Store embeddings in a vector database\n", - "- Perform similarity and filtered searches\n", - "- Use hybrid search combining vector and keyword search\n", - "\n", - "---\n", - "\n", - "## Workflow\n", - "\n", - "**Generate Embeddings → Store in Vector DB → Search → Hybrid Search**\n", - "\n", - "Each step builds toward a production-ready semantic search system.\n", - "\n", - "---\n", - "\n", - "## Step 1: Generate Embeddings\n", - "\n", - "Start by generating embeddings for your documents.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.embeddings import EmbeddingGenerator\n", - "import numpy as np\n", - "\n", - "documents = [\n", - " \"Machine learning is a subset of artificial intelligence.\",\n", - " \"Deep learning uses neural networks with multiple layers.\",\n", - " \"Natural language processing enables computers to understand text.\",\n", - "]\n", - "\n", - "generator = EmbeddingGenerator()\n", - "\n", - "try:\n", - " embeddings = generator.generate(documents)\n", - " print(\"✓ Embeddings generated\")\n", - " print(f\" Documents: {len(documents)}\")\n", - " print(f\" Embeddings shape: {embeddings.shape if hasattr(embeddings, 'shape') else 'N/A'}\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error generating embeddings: {e}\")\n", - " embeddings = np.random.rand(len(documents), 1536).astype(np.float32)\n", - " print(\" Using demo embeddings\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Store in Vector Database\n", - "\n", - "Store the embeddings along with documents and metadata in a vector database.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import VectorStore\n", - "\n", - "vector_store = VectorStore()\n", - "\n", - "metadata = [\n", - " {\"id\": i, \"category\": \"technology\", \"source\": \"demo\"}\n", - " for i in range(len(documents))\n", - "]\n", - "\n", - "try:\n", - " vector_store.store(embeddings, documents, metadata)\n", - " print(\"✓ Embeddings stored in vector database\")\n", - " print(f\" Stored {len(documents)} documents with embeddings\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error storing embeddings: {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Search\n", - "\n", - "Perform similarity search and filtered search on the stored embeddings.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import VectorRetriever\n", - "\n", - "retriever = VectorRetriever(vector_store)\n", - "\n", - "query = \"artificial intelligence\"\n", - "query_embedding = generator.generate([query])[0] if hasattr(generator, 'generate') else np.random.rand(1536).astype(np.float32)\n", - "\n", - "try:\n", - " results = retriever.retrieve(query_embedding, top_k=2)\n", - " print(\"✓ Similarity search complete\")\n", - " print(f\" Found {len(results) if results else 0} results\")\n", - " \n", - " if results:\n", - " for i, result in enumerate(results[:2]):\n", - " print(f\" Result {i+1}: Score = {result.score if hasattr(result, 'score') else 'N/A'}\")\n", - " \n", - " filtered_results = vector_store.search(\n", - " query_embedding,\n", - " top_k=2,\n", - " filters={\"category\": \"technology\"}\n", - " )\n", - " print(f\"\\n✓ Filtered search complete\")\n", - " print(f\" Found {len(filtered_results) if filtered_results else 0} filtered results\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error performing search: {e}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Hybrid Search\n", - "\n", - "Use hybrid search to combine vector similarity search with keyword search for better results.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.vector_store import HybridSearch\n", - "\n", - "hybrid_search = HybridSearch(vector_store)\n", - "\n", - "try:\n", - " hybrid_results = hybrid_search.search(\n", - " query=\"artificial intelligence\",\n", - " vector_weight=0.7,\n", - " keyword_weight=0.3,\n", - " top_k=3\n", - " )\n", - " \n", - " print(\"✓ Hybrid search complete\")\n", - " print(f\" Found {len(hybrid_results) if hybrid_results else 0} results\")\n", - " print(\" Note: Hybrid search combines vector and keyword search for better accuracy\")\n", - " \n", - " if hybrid_results:\n", - " for i, result in enumerate(hybrid_results[:3]):\n", - " print(f\" Result {i+1}: {result.document[:50] if hasattr(result, 'document') else 'N/A'}...\")\n", - " \n", - "except Exception as e:\n", - " print(f\"✗ Error performing hybrid search: {e}\")\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 817811dad6df77a218507507e1ddbcb1ce8ea433 Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:54:34 +0530 Subject: [PATCH 2/2] Delete cookbook/specialized_applications directory --- .../Fraud_Detection_Anomaly_Complete.ipynb | 388 ---------------- .../GraphRAG_Complete.ipynb | 300 ------------- .../Hybrid_RAG_Temporal_KG.ipynb | 274 ------------ .../Multi_Agent_System_KG_Powered.ipynb | 360 --------------- .../Supply_Chain_End_to_End.ipynb | 423 ------------------ 5 files changed, 1745 deletions(-) delete mode 100644 cookbook/specialized_applications/Fraud_Detection_Anomaly_Complete.ipynb delete mode 100644 cookbook/specialized_applications/GraphRAG_Complete.ipynb delete mode 100644 cookbook/specialized_applications/Hybrid_RAG_Temporal_KG.ipynb delete mode 100644 cookbook/specialized_applications/Multi_Agent_System_KG_Powered.ipynb delete mode 100644 cookbook/specialized_applications/Supply_Chain_End_to_End.ipynb diff --git a/cookbook/specialized_applications/Fraud_Detection_Anomaly_Complete.ipynb b/cookbook/specialized_applications/Fraud_Detection_Anomaly_Complete.ipynb deleted file mode 100644 index fd8dbfb2..00000000 --- a/cookbook/specialized_applications/Fraud_Detection_Anomaly_Complete.ipynb +++ /dev/null @@ -1,388 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Fraud Detection Anomaly Complete\n", - "\n", - "## Overview\n", - "\n", - "Production fraud detection: stream transactions, build temporal knowledge graph, detect patterns, identify anomalies, and implement alert system.\n", - "\n", - "## Workflow: Stream Transactions → Build Temporal KG → Detect Patterns → Identify Anomalies → Alert System\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import StreamIngestor, FileIngestor\n", - "from semantica.parse import DocumentParser, StructuredDataParser\n", - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "from semantica.kg import GraphBuilder, GraphAnalyzer, TemporalPatternDetector\n", - "from semantica.reasoning import InferenceEngine\n", - "from datetime import datetime, timedelta\n", - "import json\n", - "import os\n", - "import tempfile\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Stream Transactions\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "stream_ingestor = StreamIngestor()\n", - "file_ingestor = FileIngestor()\n", - "structured_parser = StructuredDataParser()\n", - "\n", - "temp_dir = tempfile.mkdtemp()\n", - "transactions_file = os.path.join(temp_dir, \"transactions.json\")\n", - "\n", - "transactions_data = [\n", - " {\n", - " \"transaction_id\": \"txn_001\",\n", - " \"user_id\": \"user_123\",\n", - " \"amount\": 150.00,\n", - " \"merchant\": \"Online Store\",\n", - " \"location\": \"New York\",\n", - " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat(),\n", - " \"device\": \"mobile\"\n", - " },\n", - " {\n", - " \"transaction_id\": \"txn_002\",\n", - " \"user_id\": \"user_123\",\n", - " \"amount\": 2500.00,\n", - " \"merchant\": \"Luxury Store\",\n", - " \"location\": \"Paris\",\n", - " \"timestamp\": (datetime.now() - timedelta(minutes=30)).isoformat(),\n", - " \"device\": \"web\"\n", - " },\n", - " {\n", - " \"transaction_id\": \"txn_003\",\n", - " \"user_id\": \"user_456\",\n", - " \"amount\": 50.00,\n", - " \"merchant\": \"Grocery Store\",\n", - " \"location\": \"San Francisco\",\n", - " \"timestamp\": (datetime.now() - timedelta(minutes=15)).isoformat(),\n", - " \"device\": \"mobile\"\n", - " },\n", - " {\n", - " \"transaction_id\": \"txn_004\",\n", - " \"user_id\": \"user_123\",\n", - " \"amount\": 5000.00,\n", - " \"merchant\": \"Electronics Store\",\n", - " \"location\": \"Tokyo\",\n", - " \"timestamp\": (datetime.now() - timedelta(minutes=5)).isoformat(),\n", - " \"device\": \"mobile\"\n", - " }\n", - "]\n", - "\n", - "with open(transactions_file, 'w') as f:\n", - " json.dump(transactions_data, f)\n", - "\n", - "file_objects = file_ingestor.ingest_file(transactions_file, read_content=True)\n", - "parsed_data = structured_parser.parse_json(transactions_file)\n", - "\n", - "transaction_stream = []\n", - "for txn in parsed_data.get(\"data\", transactions_data):\n", - " if isinstance(txn, dict):\n", - " txn_copy = txn.copy()\n", - " if \"timestamp\" in txn_copy and isinstance(txn_copy[\"timestamp\"], str):\n", - " txn_copy[\"timestamp\"] = datetime.fromisoformat(txn_copy[\"timestamp\"])\n", - " transaction_stream.append(txn_copy)\n", - "\n", - "print(f\"Ingested {len(file_objects)} transaction files\")\n", - "print(f\"Parsed {len(transaction_stream)} transactions from structured data\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Build Temporal Knowledge Graph\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "builder = GraphBuilder()\n", - "\n", - "transaction_entities = []\n", - "relationships = []\n", - "\n", - "for txn in transaction_stream:\n", - " txn_id = txn[\"transaction_id\"]\n", - " user_id = txn[\"user_id\"]\n", - " merchant = txn[\"merchant\"]\n", - " location = txn[\"location\"]\n", - " \n", - " transaction_entities.append({\n", - " \"id\": txn_id,\n", - " \"type\": \"Transaction\",\n", - " \"properties\": {\n", - " \"amount\": txn[\"amount\"],\n", - " \"timestamp\": txn[\"timestamp\"].isoformat(),\n", - " \"device\": txn[\"device\"]\n", - " }\n", - " })\n", - " \n", - " transaction_entities.append({\n", - " \"id\": user_id,\n", - " \"type\": \"User\",\n", - " \"properties\": {}\n", - " })\n", - " \n", - " transaction_entities.append({\n", - " \"id\": merchant,\n", - " \"type\": \"Merchant\",\n", - " \"properties\": {}\n", - " })\n", - " \n", - " transaction_entities.append({\n", - " \"id\": location,\n", - " \"type\": \"Location\",\n", - " \"properties\": {}\n", - " })\n", - " \n", - " relationships.append({\n", - " \"source\": user_id,\n", - " \"target\": txn_id,\n", - " \"type\": \"performed\",\n", - " \"properties\": {\"timestamp\": txn[\"timestamp\"].isoformat()}\n", - " })\n", - " \n", - " relationships.append({\n", - " \"source\": txn_id,\n", - " \"target\": merchant,\n", - " \"type\": \"at_merchant\",\n", - " \"properties\": {}\n", - " })\n", - " \n", - " relationships.append({\n", - " \"source\": txn_id,\n", - " \"target\": location,\n", - " \"type\": \"in_location\",\n", - " \"properties\": {}\n", - " })\n", - "\n", - "transaction_kg = builder.build(transaction_entities, relationships)\n", - "\n", - "print(f\"Built temporal knowledge graph with {len(transaction_entities)} entities and {len(relationships)} relationships\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Detect Patterns\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "inference_engine = InferenceEngine()\n", - "pattern_detector = TemporalPatternDetector()\n", - "graph_analyzer = GraphAnalyzer()\n", - "\n", - "temporal_patterns = pattern_detector.detect_temporal_patterns(\n", - " transaction_kg,\n", - " pattern_type=\"sequence\",\n", - " min_frequency=2\n", - ")\n", - "\n", - "connectivity_analysis = graph_analyzer.analyze_connectivity(transaction_kg)\n", - "\n", - "fraud_patterns = []\n", - "user_transactions = {}\n", - "for txn in transaction_stream:\n", - " user_id = txn[\"user_id\"]\n", - " if user_id not in user_transactions:\n", - " user_transactions[user_id] = []\n", - " user_transactions[user_id].append(txn)\n", - "\n", - "for user_id, txns in user_transactions.items():\n", - " if len(txns) > 1:\n", - " amounts = [t[\"amount\"] for t in txns]\n", - " locations = [t[\"location\"] for t in txns]\n", - " timestamps = [t[\"timestamp\"] for t in txns]\n", - " \n", - " if max(amounts) > 1000:\n", - " fraud_patterns.append({\n", - " \"type\": \"high_value_transaction\",\n", - " \"user_id\": user_id,\n", - " \"amount\": max(amounts),\n", - " \"severity\": \"medium\"\n", - " })\n", - " \n", - " if len(set(locations)) > 2:\n", - " time_span = max(timestamps) - min(timestamps)\n", - " if time_span.total_seconds() < 3600:\n", - " fraud_patterns.append({\n", - " \"type\": \"rapid_location_change\",\n", - " \"user_id\": user_id,\n", - " \"locations\": list(set(locations)),\n", - " \"severity\": \"high\"\n", - " })\n", - "\n", - "print(f\"Detected {len(fraud_patterns)} fraud patterns\")\n", - "print(f\"Temporal patterns: {len(temporal_patterns)}\")\n", - "print(f\"Connectivity analysis: {connectivity_analysis.get('is_connected', False)}\")\n", - "for pattern in fraud_patterns:\n", - " print(f\" Pattern: {pattern['type']} - User: {pattern['user_id']} - Severity: {pattern['severity']}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Identify Anomalies\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "anomaly_patterns = pattern_detector.detect_temporal_patterns(\n", - " transaction_kg,\n", - " pattern_type=\"anomaly\",\n", - " min_frequency=1\n", - ")\n", - "\n", - "anomalies = []\n", - "for txn in transaction_stream:\n", - " score = 0\n", - " reasons = []\n", - " \n", - " if txn[\"amount\"] > 2000:\n", - " score += 3\n", - " reasons.append(\"High transaction amount\")\n", - " \n", - " if txn[\"amount\"] > 1000 and txn[\"device\"] == \"mobile\":\n", - " score += 2\n", - " reasons.append(\"High amount on mobile device\")\n", - " \n", - " user_txns = [t for t in transaction_stream if t[\"user_id\"] == txn[\"user_id\"]]\n", - " if len(user_txns) > 1:\n", - " recent_txns = sorted(user_txns, key=lambda x: x[\"timestamp\"], reverse=True)[:3]\n", - " locations = [t[\"location\"] for t in recent_txns]\n", - " if len(set(locations)) > 2:\n", - " time_span = recent_txns[0][\"timestamp\"] - recent_txns[-1][\"timestamp\"]\n", - " if time_span.total_seconds() < 3600:\n", - " score += 4\n", - " reasons.append(\"Rapid location changes\")\n", - " \n", - " if score >= 3:\n", - " anomalies.append({\n", - " \"transaction_id\": txn[\"transaction_id\"],\n", - " \"user_id\": txn[\"user_id\"],\n", - " \"severity\": \"high\" if score >= 5 else \"medium\",\n", - " \"score\": score,\n", - " \"reasons\": reasons,\n", - " \"timestamp\": txn[\"timestamp\"]\n", - " })\n", - "\n", - "print(f\"Detected {len(anomalies)} anomalies\")\n", - "for anomaly in anomalies:\n", - " print(f\" Transaction: {anomaly['transaction_id']} - Severity: {anomaly['severity']} - Score: {anomaly['score']}\")\n", - " print(f\" Reasons: {', '.join(anomaly['reasons'])}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Alert System\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def send_alert(anomaly):\n", - " alert = {\n", - " \"alert_id\": f\"alert_{anomaly['transaction_id']}\",\n", - " \"transaction_id\": anomaly[\"transaction_id\"],\n", - " \"user_id\": anomaly[\"user_id\"],\n", - " \"severity\": anomaly[\"severity\"],\n", - " \"timestamp\": datetime.now().isoformat(),\n", - " \"reasons\": anomaly[\"reasons\"]\n", - " }\n", - " return alert\n", - "\n", - "def log_fraud_event(anomaly):\n", - " event = {\n", - " \"event_type\": \"fraud_detected\",\n", - " \"transaction_id\": anomaly[\"transaction_id\"],\n", - " \"user_id\": anomaly[\"user_id\"],\n", - " \"severity\": anomaly[\"severity\"],\n", - " \"score\": anomaly[\"score\"],\n", - " \"timestamp\": datetime.now().isoformat()\n", - " }\n", - " return event\n", - "\n", - "threshold = 3\n", - "alerts = []\n", - "fraud_events = []\n", - "\n", - "for anomaly in anomalies:\n", - " if anomaly[\"score\"] >= threshold:\n", - " alert = send_alert(anomaly)\n", - " alerts.append(alert)\n", - " event = log_fraud_event(anomaly)\n", - " fraud_events.append(event)\n", - "\n", - "print(f\"Generated {len(alerts)} alerts\")\n", - "for alert in alerts:\n", - " print(f\" Alert: {alert['alert_id']} - Severity: {alert['severity']} - Transaction: {alert['transaction_id']}\")\n", - "\n", - "print(f\"\\nLogged {len(fraud_events)} fraud events\")\n", - "\n", - "entities_count = len(transaction_kg.get(\"entities\", []))\n", - "print(f\"\\nMonitoring {entities_count} transaction entities\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Production fraud detection workflow:\n", - "- Transaction streaming configured\n", - "- Temporal knowledge graph built\n", - "- Fraud patterns detected\n", - "- Anomalies identified\n", - "- Alert system operational\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/specialized_applications/GraphRAG_Complete.ipynb b/cookbook/specialized_applications/GraphRAG_Complete.ipynb deleted file mode 100644 index bc618c1c..00000000 --- a/cookbook/specialized_applications/GraphRAG_Complete.ipynb +++ /dev/null @@ -1,300 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# GraphRAG Complete\n", - "\n", - "## Overview\n", - "\n", - "Next-generation RAG: build knowledge graph, generate embeddings, store in vector database, implement hybrid RAG, and integrate with LLM.\n", - "\n", - "## Workflow: Build KG → Generate Embeddings → Vector Store → Hybrid RAG → LLM Integration\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FileIngestor, WebIngestor\n", - "from semantica.parse import DocumentParser, WebParser\n", - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "from semantica.kg import GraphBuilder\n", - "from semantica.embeddings import EmbeddingGenerator\n", - "from semantica.vector_store import VectorStore, HybridSearch\n", - "from semantica.context import ContextRetriever\n", - "import numpy as np\n", - "import os\n", - "import tempfile\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Build Knowledge Graph\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "file_ingestor = FileIngestor()\n", - "web_ingestor = WebIngestor()\n", - "document_parser = DocumentParser()\n", - "web_parser = WebParser()\n", - "ner_extractor = NERExtractor()\n", - "relation_extractor = RelationExtractor()\n", - "builder = GraphBuilder()\n", - "\n", - "temp_dir = tempfile.mkdtemp()\n", - "\n", - "doc1_file = os.path.join(temp_dir, \"ai_intro.txt\")\n", - "doc2_file = os.path.join(temp_dir, \"ml_basics.txt\")\n", - "doc3_file = os.path.join(temp_dir, \"dl_guide.txt\")\n", - "\n", - "with open(doc1_file, 'w') as f:\n", - " f.write(\"Introduction to AI: Artificial Intelligence is transforming industries. Neural Networks are key components.\")\n", - "with open(doc2_file, 'w') as f:\n", - " f.write(\"Machine Learning Basics: ML algorithms learn from data patterns. Neural Networks enable complex learning.\")\n", - "with open(doc3_file, 'w') as f:\n", - " f.write(\"Deep Learning Guide: Deep neural networks enable complex learning. Backpropagation is used for training.\")\n", - "\n", - "file_objects = []\n", - "for doc_file in [doc1_file, doc2_file, doc3_file]:\n", - " file_obj = file_ingestor.ingest_file(doc_file, read_content=True)\n", - " if file_obj:\n", - " file_objects.append(file_obj)\n", - "\n", - "parsed_documents = []\n", - "for file_obj in file_objects:\n", - " parsed = document_parser.extract_text(file_obj.path)\n", - " parsed_documents.append({\n", - " \"file\": file_obj.name,\n", - " \"content\": parsed,\n", - " \"metadata\": file_obj.metadata\n", - " })\n", - "\n", - "all_entities = []\n", - "all_relationships = []\n", - "entity_map = {}\n", - "\n", - "for i, doc in enumerate(parsed_documents, 1):\n", - " doc_id = f\"doc{i}\"\n", - " doc_name = doc[\"file\"].replace(\".txt\", \"\").replace(\"_\", \" \").title()\n", - " \n", - " all_entities.append({\n", - " \"id\": doc_id,\n", - " \"type\": \"Document\",\n", - " \"name\": doc_name,\n", - " \"properties\": {\"content\": doc[\"content\"][:100]}\n", - " })\n", - " \n", - " extracted_entities = ner_extractor.extract(doc[\"content\"])\n", - " extracted_relations = relation_extractor.extract(doc[\"content\"], extracted_entities)\n", - " \n", - " for entity in extracted_entities[:5]:\n", - " entity_text = entity.get(\"text\", entity.get(\"entity\", \"\"))\n", - " if entity_text and entity_text not in entity_map:\n", - " entity_id = f\"concept_{len(entity_map) + 1}\"\n", - " entity_map[entity_text] = entity_id\n", - " all_entities.append({\n", - " \"id\": entity_id,\n", - " \"type\": entity.get(\"type\", \"Concept\"),\n", - " \"name\": entity_text,\n", - " \"properties\": {}\n", - " })\n", - " \n", - " all_relationships.append({\n", - " \"source\": doc_id,\n", - " \"target\": entity_id,\n", - " \"type\": \"mentions\"\n", - " })\n", - " \n", - " for rel in extracted_relations[:3]:\n", - " source_text = rel.get(\"source\", \"\")\n", - " target_text = rel.get(\"target\", \"\")\n", - " if source_text in entity_map and target_text in entity_map:\n", - " all_relationships.append({\n", - " \"source\": entity_map[source_text],\n", - " \"target\": entity_map[target_text],\n", - " \"type\": rel.get(\"type\", \"related_to\")\n", - " })\n", - "\n", - "knowledge_graph = builder.build(all_entities, all_relationships)\n", - "\n", - "print(f\"Ingested {len(file_objects)} documents\")\n", - "print(f\"Extracted {len([e for e in all_entities if e['type'] != 'Document'])} concepts\")\n", - "print(f\"Built knowledge graph with {len(all_entities)} entities and {len(all_relationships)} relationships\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Generate Embeddings\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "documents = [doc[\"content\"] for doc in parsed_documents]\n", - "\n", - "generator = EmbeddingGenerator()\n", - "embeddings = generator.generate(documents)\n", - "\n", - "print(f\"Generated embeddings for {len(documents)} parsed documents\")\n", - "print(f\"Embedding dimension: {len(embeddings[0]) if embeddings else 0}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Store in Vector Store\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vector_store = VectorStore()\n", - "\n", - "vector_ids = [f\"doc_{i+1}\" for i in range(len(documents))]\n", - "metadata = [\n", - " {\"doc_id\": \"doc1\", \"topic\": \"AI\", \"type\": \"introduction\"},\n", - " {\"doc_id\": \"doc2\", \"topic\": \"ML\", \"type\": \"tutorial\"},\n", - " {\"doc_id\": \"doc3\", \"topic\": \"DL\", \"type\": \"guide\"}\n", - "]\n", - "\n", - "vector_ids_stored = vector_store.store_vectors(embeddings, metadata)\n", - "\n", - "print(f\"Stored {len(vector_ids_stored)} vectors in vector store\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Hybrid RAG\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hybrid_search = HybridSearch()\n", - "context_retriever = ContextRetriever(\n", - " knowledge_graph=knowledge_graph,\n", - " vector_store=vector_store\n", - ")\n", - "\n", - "query = \"What is deep learning?\"\n", - "query_embedding = generator.generate([query])[0]\n", - "\n", - "vector_results = vector_store.search_vectors(query_embedding, k=3)\n", - "\n", - "graph_context_results = context_retriever.retrieve(\n", - " query=query,\n", - " max_results=5,\n", - " use_graph_expansion=True,\n", - " max_hops=2\n", - ")\n", - "\n", - "graph_context = []\n", - "for result in graph_context_results:\n", - " graph_context.append({\n", - " \"entity\": result.content,\n", - " \"type\": result.metadata.get(\"type\", \"unknown\"),\n", - " \"related\": [e.get(\"name\", e.get(\"id\")) for e in result.related_entities[:3]]\n", - " })\n", - "\n", - "print(f\"Retrieved {len(vector_results)} vector search results\")\n", - "print(f\"Found {len(graph_context)} relevant graph entities from ContextRetriever\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def format_context(vector_results, graph_context):\n", - " context_parts = []\n", - " \n", - " context_parts.append(\"Retrieved Documents:\")\n", - " for i, result in enumerate(vector_results[:3], 1):\n", - " doc_id = result.get(\"id\", \"unknown\")\n", - " score = result.get(\"score\", 0)\n", - " meta = result.get(\"metadata\", {})\n", - " context_parts.append(f\"{i}. Document {doc_id} (relevance: {score:.3f}, topic: {meta.get('topic', 'N/A')})\")\n", - " \n", - " if graph_context:\n", - " context_parts.append(\"\\nKnowledge Graph Context:\")\n", - " for ctx in graph_context:\n", - " context_parts.append(f\"- {ctx['entity']} ({ctx['type']})\")\n", - " if ctx['related']:\n", - " context_parts.append(f\" Related: {', '.join(ctx['related'])}\")\n", - " \n", - " return \"\\n\".join(context_parts)\n", - "\n", - "def generate_response(query, context):\n", - " response_template = f\"\"\"\n", - "Query: {query}\n", - "\n", - "Context:\n", - "{context}\n", - "\n", - "Response: Based on the retrieved context, {query.lower()} is a topic covered in the knowledge base. \n", - "The relevant documents and graph entities provide comprehensive information about this subject.\n", - "\"\"\"\n", - " return response_template\n", - "\n", - "context = format_context(vector_results, graph_context)\n", - "response = generate_response(query, context)\n", - "\n", - "print(\"Generated Response:\")\n", - "print(response)\n", - "print(f\"\\nUsed {len(vector_results)} graph-enhanced results\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Next-generation RAG workflow:\n", - "- Knowledge graph built\n", - "- Embeddings generated\n", - "- Vectors stored\n", - "- Hybrid RAG implemented\n", - "- LLM integration ready\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/specialized_applications/Hybrid_RAG_Temporal_KG.ipynb b/cookbook/specialized_applications/Hybrid_RAG_Temporal_KG.ipynb deleted file mode 100644 index d5f225a0..00000000 --- a/cookbook/specialized_applications/Hybrid_RAG_Temporal_KG.ipynb +++ /dev/null @@ -1,274 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Hybrid RAG Temporal KG\n", - "\n", - "## Overview\n", - "\n", - "Advanced hybrid search: build temporal knowledge graph, generate vector embeddings, implement hybrid search (Vector + Temporal KG), and enable time-aware retrieval.\n", - "\n", - "## Workflow: Build Temporal KG → Vector Embeddings → Hybrid Search → Time-Aware Retrieval\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FileIngestor, WebIngestor, FeedIngestor\n", - "from semantica.parse import DocumentParser, WebParser, StructuredDataParser\n", - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "from semantica.kg import GraphBuilder, TemporalGraphQuery\n", - "from semantica.embeddings import EmbeddingGenerator\n", - "from semantica.vector_store import VectorStore, HybridSearch\n", - "from datetime import datetime, timedelta\n", - "import numpy as np\n", - "import os\n", - "import tempfile\n", - "import json\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Build Temporal Knowledge Graph\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "file_ingestor = FileIngestor()\n", - "web_ingestor = WebIngestor()\n", - "feed_ingestor = FeedIngestor()\n", - "document_parser = DocumentParser()\n", - "structured_parser = StructuredDataParser()\n", - "ner_extractor = NERExtractor()\n", - "relation_extractor = RelationExtractor()\n", - "builder = GraphBuilder()\n", - "\n", - "temp_dir = tempfile.mkdtemp()\n", - "\n", - "events_file = os.path.join(temp_dir, \"events.json\")\n", - "events_data = [\n", - " {\"event\": \"Product Launch\", \"date\": \"2023-10-15T10:00:00\", \"category\": \"product\"},\n", - " {\"event\": \"Q4 Sales Meeting\", \"date\": \"2023-11-20T14:00:00\", \"category\": \"business\"},\n", - " {\"event\": \"Year End Review\", \"date\": \"2023-12-31T09:00:00\", \"category\": \"business\"},\n", - " {\"event\": \"New Year Planning\", \"date\": \"2024-01-05T10:00:00\", \"category\": \"planning\"}\n", - "]\n", - "\n", - "with open(events_file, 'w') as f:\n", - " json.dump(events_data, f)\n", - "\n", - "file_objects = file_ingestor.ingest_file(events_file, read_content=True)\n", - "parsed_events = structured_parser.parse_json(events_file)\n", - "\n", - "entities = []\n", - "relationships = []\n", - "\n", - "for i, event_data in enumerate(parsed_events.get(\"data\", events_data), 1):\n", - " event_id = f\"event{i}\"\n", - " event_name = event_data.get(\"event\", f\"Event {i}\")\n", - " timestamp = event_data.get(\"date\", \"\")\n", - " category = event_data.get(\"category\", \"general\")\n", - " \n", - " entities.append({\n", - " \"id\": event_id,\n", - " \"type\": \"Event\",\n", - " \"name\": event_name,\n", - " \"properties\": {\"timestamp\": timestamp, \"category\": category}\n", - " })\n", - " \n", - " if i > 1:\n", - " prev_event_id = f\"event{i-1}\"\n", - " relationships.append({\n", - " \"source\": prev_event_id,\n", - " \"target\": event_id,\n", - " \"type\": \"followed_by\",\n", - " \"properties\": {\"timestamp\": timestamp}\n", - " })\n", - "\n", - "temporal_kg = builder.build(entities, relationships)\n", - "\n", - "print(f\"Ingested {len(file_objects)} event files\")\n", - "print(f\"Parsed {len(parsed_events.get('data', []))} events from structured data\")\n", - "print(f\"Built temporal knowledge graph with {len(entities)} entities and {len(relationships)} relationships\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Vector Embeddings\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "documents = []\n", - "for event_data in parsed_events.get(\"data\", events_data):\n", - " event_name = event_data.get(\"event\", \"\")\n", - " date_str = event_data.get(\"date\", \"\")[:10]\n", - " category = event_data.get(\"category\", \"\")\n", - " documents.append(f\"{event_name}: Event occurred on {date_str} in category {category}.\")\n", - "\n", - "generator = EmbeddingGenerator()\n", - "embeddings = generator.generate(documents)\n", - "\n", - "print(f\"Generated embeddings for {len(documents)} documents from parsed events\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Hybrid Search Setup\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vector_store = VectorStore()\n", - "\n", - "metadata = [\n", - " {\"event_id\": \"event1\", \"timestamp\": \"2023-10-15\", \"category\": \"product\"},\n", - " {\"event_id\": \"event2\", \"timestamp\": \"2023-11-20\", \"category\": \"business\"},\n", - " {\"event_id\": \"event3\", \"timestamp\": \"2023-12-31\", \"category\": \"business\"},\n", - " {\"event_id\": \"event4\", \"timestamp\": \"2024-01-05\", \"category\": \"planning\"}\n", - "]\n", - "\n", - "vector_ids = vector_store.store_vectors(embeddings, metadata)\n", - "\n", - "hybrid_search = HybridSearch()\n", - "temporal_query = TemporalGraphQuery()\n", - "\n", - "print(f\"Stored {len(vector_ids)} vectors in vector store\")\n", - "print(\"Hybrid search and temporal query initialized\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Time-Aware Retrieval\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "query = \"What happened in Q4 2023?\"\n", - "query_embedding = generator.generate([query])[0]\n", - "\n", - "vector_results = vector_store.search_vectors(query_embedding, k=10)\n", - "\n", - "temporal_query_result = temporal_query.query_time_range(\n", - " graph=temporal_kg,\n", - " query=query,\n", - " start_time=\"2023-10-01\",\n", - " end_time=\"2023-12-31\",\n", - " temporal_aggregation=\"union\"\n", - ")\n", - "\n", - "temporal_results = []\n", - "entities_list = temporal_kg.get(\"entities\", [])\n", - "entity_map = {e.get(\"id\"): e for e in entities_list}\n", - "\n", - "for rel in temporal_query_result.get(\"relationships\", []):\n", - " source_id = rel.get(\"source\")\n", - " target_id = rel.get(\"target\")\n", - " if source_id in entity_map:\n", - " entity = entity_map[source_id]\n", - " temporal_results.append({\n", - " \"entity_id\": source_id,\n", - " \"name\": entity.get(\"name\"),\n", - " \"timestamp\": entity.get(\"properties\", {}).get(\"timestamp\", \"\"),\n", - " \"type\": entity.get(\"type\")\n", - " })\n", - "\n", - "def combine_results(vector_results, temporal_results):\n", - " combined = []\n", - " \n", - " vector_dict = {r.get(\"id\", \"\"): r for r in vector_results}\n", - " \n", - " for temp_result in temporal_results:\n", - " entity_id = temp_result.get(\"entity_id\", \"\")\n", - " if entity_id in vector_dict:\n", - " combined.append({\n", - " \"id\": entity_id,\n", - " \"name\": temp_result.get(\"name\"),\n", - " \"vector_score\": vector_dict[entity_id].get(\"score\", 0),\n", - " \"timestamp\": temp_result.get(\"timestamp\"),\n", - " \"type\": \"hybrid\"\n", - " })\n", - " else:\n", - " combined.append({\n", - " \"id\": entity_id,\n", - " \"name\": temp_result.get(\"name\"),\n", - " \"vector_score\": 0,\n", - " \"timestamp\": temp_result.get(\"timestamp\"),\n", - " \"type\": \"temporal_only\"\n", - " })\n", - " \n", - " for vec_result in vector_results:\n", - " vec_id = vec_result.get(\"id\", \"\")\n", - " if not any(c.get(\"id\") == vec_id for c in combined):\n", - " combined.append({\n", - " \"id\": vec_id,\n", - " \"vector_score\": vec_result.get(\"score\", 0),\n", - " \"type\": \"vector_only\"\n", - " })\n", - " \n", - " combined.sort(key=lambda x: x.get(\"vector_score\", 0), reverse=True)\n", - " return combined\n", - "\n", - "hybrid_results = combine_results(vector_results, temporal_results)\n", - "\n", - "print(f\"Retrieved {len(hybrid_results)} time-aware results\")\n", - "print(f\" Vector results: {len(vector_results)}\")\n", - "print(f\" Temporal results: {len(temporal_results)}\")\n", - "print(f\" Hybrid results: {len([r for r in hybrid_results if r.get('type') == 'hybrid'])}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Advanced hybrid search workflow:\n", - "- Temporal knowledge graph built\n", - "- Vector embeddings generated\n", - "- Hybrid search configured\n", - "- Time-aware retrieval implemented\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/specialized_applications/Multi_Agent_System_KG_Powered.ipynb b/cookbook/specialized_applications/Multi_Agent_System_KG_Powered.ipynb deleted file mode 100644 index 0b624cec..00000000 --- a/cookbook/specialized_applications/Multi_Agent_System_KG_Powered.ipynb +++ /dev/null @@ -1,360 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Multi-Agent System KG-Powered\n", - "\n", - "## Overview\n", - "\n", - "AI agent systems: build knowledge graph, implement agent memory, create context graphs, enable multi-agent coordination, and share knowledge.\n", - "\n", - "## Workflow: Build KG → Agent Memory → Context Graphs → Multi-Agent Coordination → Shared Knowledge\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FileIngestor, DBIngestor\n", - "from semantica.parse import DocumentParser, StructuredDataParser\n", - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "from semantica.kg import GraphBuilder\n", - "from semantica.context import AgentMemory, ContextRetriever\n", - "from semantica.reasoning import InferenceEngine\n", - "from datetime import datetime\n", - "import os\n", - "import tempfile\n", - "import json\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Build Knowledge Graph\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "file_ingestor = FileIngestor()\n", - "db_ingestor = DBIngestor()\n", - "structured_parser = StructuredDataParser()\n", - "ner_extractor = NERExtractor()\n", - "relation_extractor = RelationExtractor()\n", - "builder = GraphBuilder()\n", - "\n", - "temp_dir = tempfile.mkdtemp()\n", - "\n", - "agents_file = os.path.join(temp_dir, \"agents.json\")\n", - "tasks_file = os.path.join(temp_dir, \"tasks.json\")\n", - "\n", - "agents_data = [\n", - " {\"agent_id\": \"agent_1\", \"name\": \"Research Agent\", \"role\": \"researcher\"},\n", - " {\"agent_id\": \"agent_2\", \"name\": \"Analysis Agent\", \"role\": \"analyst\"}\n", - "]\n", - "\n", - "tasks_data = [\n", - " {\"task_id\": \"task_1\", \"name\": \"Data Collection\", \"status\": \"completed\", \"assigned_to\": \"agent_1\"},\n", - " {\"task_id\": \"task_2\", \"name\": \"Data Analysis\", \"status\": \"in_progress\", \"assigned_to\": \"agent_2\"}\n", - "]\n", - "\n", - "with open(agents_file, 'w') as f:\n", - " json.dump(agents_data, f)\n", - "with open(tasks_file, 'w') as f:\n", - " json.dump(tasks_data, f)\n", - "\n", - "file_objects = file_ingestor.ingest_directory(temp_dir, recursive=False)\n", - "parsed_agents = structured_parser.parse_json(agents_file)\n", - "parsed_tasks = structured_parser.parse_json(tasks_file)\n", - "\n", - "entities = []\n", - "relationships = []\n", - "\n", - "for agent_data in parsed_agents.get(\"data\", agents_data):\n", - " entities.append({\n", - " \"id\": agent_data.get(\"agent_id\", \"\"),\n", - " \"type\": \"Agent\",\n", - " \"name\": agent_data.get(\"name\", \"\"),\n", - " \"properties\": {\"role\": agent_data.get(\"role\", \"\")}\n", - " })\n", - "\n", - "for task_data in parsed_tasks.get(\"data\", tasks_data):\n", - " entities.append({\n", - " \"id\": task_data.get(\"task_id\", \"\"),\n", - " \"type\": \"Task\",\n", - " \"name\": task_data.get(\"name\", \"\"),\n", - " \"properties\": {\"status\": task_data.get(\"status\", \"\")}\n", - " })\n", - " \n", - " assigned_agent = task_data.get(\"assigned_to\", \"\")\n", - " if assigned_agent:\n", - " relationships.append({\n", - " \"source\": assigned_agent,\n", - " \"target\": task_data.get(\"task_id\", \"\"),\n", - " \"type\": \"assigned_to\"\n", - " })\n", - "\n", - "knowledge_content = \"Market Trends: Analysis shows increasing demand in technology sector.\"\n", - "extracted_entities = ner_extractor.extract(knowledge_content)\n", - "extracted_relations = relation_extractor.extract(knowledge_content, extracted_entities)\n", - "\n", - "for entity in extracted_entities[:3]:\n", - " entity_id = f\"knowledge_{len([e for e in entities if e['type'] == 'Knowledge']) + 1}\"\n", - " entities.append({\n", - " \"id\": entity_id,\n", - " \"type\": \"Knowledge\",\n", - " \"name\": entity.get(\"text\", entity.get(\"entity\", \"\")),\n", - " \"properties\": {}\n", - " })\n", - " \n", - " relationships.append({\n", - " \"source\": \"agent_1\",\n", - " \"target\": entity_id,\n", - " \"type\": \"discovered\"\n", - " })\n", - " \n", - " relationships.append({\n", - " \"source\": \"task_1\",\n", - " \"target\": entity_id,\n", - " \"type\": \"produced\"\n", - " })\n", - "\n", - "knowledge_graph = builder.build(entities, relationships)\n", - "\n", - "print(f\"Ingested {len(file_objects)} files\")\n", - "print(f\"Parsed {len(parsed_agents.get('data', []))} agents and {len(parsed_tasks.get('data', []))} tasks\")\n", - "print(f\"Extracted {len([e for e in entities if e['type'] == 'Knowledge'])} knowledge entities\")\n", - "print(f\"Built knowledge graph with {len(entities)} entities and {len(relationships)} relationships\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Agent Memory\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agent_memory = AgentMemory(knowledge_graph=knowledge_graph)\n", - "\n", - "agent_experiences = [\n", - " {\n", - " \"agent_id\": \"agent_1\",\n", - " \"content\": \"Completed data collection task successfully\",\n", - " \"metadata\": {\"task\": \"task_1\", \"timestamp\": datetime.now().isoformat()}\n", - " },\n", - " {\n", - " \"agent_id\": \"agent_2\",\n", - " \"content\": \"Started analyzing collected data\",\n", - " \"metadata\": {\"task\": \"task_2\", \"timestamp\": datetime.now().isoformat()}\n", - " }\n", - "]\n", - "\n", - "for experience in agent_experiences:\n", - " memory_id = agent_memory.store(\n", - " content=experience[\"content\"],\n", - " metadata=experience[\"metadata\"],\n", - " entities=[{\"id\": experience[\"agent_id\"], \"type\": \"Agent\"}]\n", - " )\n", - " print(f\"Stored experience for {experience['agent_id']}: {memory_id}\")\n", - "\n", - "print(f\"\\nTotal memories stored: {agent_memory.stats.get('total_items', 0)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Context Graphs\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "context_retriever = ContextRetriever(knowledge_graph=knowledge_graph)\n", - "\n", - "def get_agent_context(agent_id, query, kg, retriever):\n", - " context_query = f\"{query} for agent {agent_id}\"\n", - " retrieved_context = retriever.retrieve(\n", - " query=context_query,\n", - " max_results=5,\n", - " use_graph_expansion=True,\n", - " max_hops=2,\n", - " entity_ids=[agent_id]\n", - " )\n", - " \n", - " context_items = []\n", - " if retrieved_context:\n", - " context_items.append({\n", - " \"type\": \"agent_info\",\n", - " \"data\": {\"agent_id\": agent_id}\n", - " })\n", - " \n", - " related_tasks = []\n", - " related_knowledge = []\n", - " \n", - " for ctx in retrieved_context:\n", - " for entity in ctx.related_entities:\n", - " if entity.get(\"type\") == \"Task\":\n", - " related_tasks.append(entity)\n", - " elif entity.get(\"type\") == \"Knowledge\":\n", - " related_knowledge.append(entity)\n", - " \n", - " context_items.append({\n", - " \"type\": \"related_tasks\",\n", - " \"data\": related_tasks\n", - " })\n", - " context_items.append({\n", - " \"type\": \"related_knowledge\",\n", - " \"data\": related_knowledge\n", - " })\n", - " \n", - " return context_items\n", - "\n", - "context_agent_1 = get_agent_context(\"agent_1\", \"What should I work on?\", knowledge_graph, context_retriever)\n", - "print(f\"Context for agent_1: {len(context_agent_1)} context items\")\n", - "for item in context_agent_1:\n", - " print(f\" - {item['type']}: {len(item['data']) if isinstance(item['data'], list) else 1} items\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Multi-Agent Coordination\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "shared_knowledge = knowledge_graph\n", - "\n", - "task_1 = \"Analyze market trends\"\n", - "task_2 = \"Review analysis results\"\n", - "\n", - "agent_1_context = get_agent_context(\"agent_1\", task_1, shared_knowledge, context_retriever)\n", - "agent_2_context = get_agent_context(\"agent_2\", task_2, shared_knowledge, context_retriever)\n", - "\n", - "print(\"Agent 1 Context:\")\n", - "for item in agent_1_context:\n", - " if isinstance(item['data'], list):\n", - " print(f\" {item['type']}: {[d.get('name', d.get('id')) for d in item['data']]}\")\n", - " else:\n", - " print(f\" {item['type']}: {item['data'].get('name', item['data'].get('id'))}\")\n", - "\n", - "print(\"\\nAgent 2 Context:\")\n", - "for item in agent_2_context:\n", - " if isinstance(item['data'], list):\n", - " print(f\" {item['type']}: {[d.get('name', d.get('id')) for d in item['data']]}\")\n", - " else:\n", - " print(f\" {item['type']}: {item['data'].get('name', item['data'].get('id'))}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Shared Knowledge\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "inference_engine = InferenceEngine()\n", - "\n", - "inference_engine.add_rule(\"IF agent performs action ON entity THEN agent action entity\")\n", - "\n", - "agent_actions = [\n", - " {\"agent\": \"agent_1\", \"action\": \"discovered\", \"entity\": \"knowledge_1\"},\n", - " {\"agent\": \"agent_2\", \"action\": \"analyzed\", \"entity\": \"knowledge_1\"},\n", - "]\n", - "\n", - "for action in agent_actions:\n", - " inference_engine.add_fact(action)\n", - "\n", - "inferred_results = inference_engine.forward_chain()\n", - "\n", - "new_relationships = []\n", - "for action in agent_actions:\n", - " new_relationships.append({\n", - " \"source\": action[\"agent\"],\n", - " \"target\": action[\"entity\"],\n", - " \"type\": action[\"action\"],\n", - " \"properties\": {\"timestamp\": datetime.now().isoformat(), \"inferred\": False}\n", - " })\n", - "\n", - "for result in inferred_results:\n", - " if hasattr(result, 'conclusion') and isinstance(result.conclusion, dict):\n", - " if \"agent\" in result.conclusion and \"entity\" in result.conclusion:\n", - " new_relationships.append({\n", - " \"source\": result.conclusion.get(\"agent\", \"\"),\n", - " \"target\": result.conclusion.get(\"entity\", \"\"),\n", - " \"type\": result.conclusion.get(\"action\", \"\"),\n", - " \"properties\": {\"timestamp\": datetime.now().isoformat(), \"inferred\": True}\n", - " })\n", - "\n", - "new_knowledge = {\n", - " \"entities\": [],\n", - " \"relationships\": new_relationships\n", - "}\n", - "\n", - "if new_knowledge[\"relationships\"]:\n", - " updated_kg = builder.build(\n", - " knowledge_graph.get(\"entities\", []) + new_knowledge[\"entities\"],\n", - " knowledge_graph.get(\"relationships\", []) + new_knowledge[\"relationships\"]\n", - " )\n", - " print(f\"Updated knowledge graph with {len(new_knowledge['relationships'])} new relationships\")\n", - "else:\n", - " updated_kg = knowledge_graph\n", - "\n", - "entities_count = len(updated_kg.get(\"entities\", []))\n", - "relationships_count = len(updated_kg.get(\"relationships\", []))\n", - "\n", - "print(f\"\\nShared knowledge graph has {entities_count} entities and {relationships_count} relationships\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Multi-agent system workflow:\n", - "- Knowledge graph built\n", - "- Agent memory implemented\n", - "- Context graphs created\n", - "- Multi-agent coordination enabled\n", - "- Shared knowledge maintained\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/specialized_applications/Supply_Chain_End_to_End.ipynb b/cookbook/specialized_applications/Supply_Chain_End_to_End.ipynb deleted file mode 100644 index bdccc1c4..00000000 --- a/cookbook/specialized_applications/Supply_Chain_End_to_End.ipynb +++ /dev/null @@ -1,423 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Supply Chain End-to-End\n", - "\n", - "## Overview\n", - "\n", - "Complete supply chain intelligence: multi-source data ingestion, build supply chain knowledge graph, analyze dependencies, optimize flow, and predict disruptions.\n", - "\n", - "## Workflow: Multi-Source Data → Build Supply Chain KG → Analyze Dependencies → Optimize → Predict Disruptions\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, StreamIngestor\n", - "from semantica.parse import DocumentParser, WebParser, StructuredDataParser\n", - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer, TemporalPatternDetector\n", - "from semantica.reasoning import InferenceEngine\n", - "from datetime import datetime, timedelta\n", - "import os\n", - "import tempfile\n", - "import json\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Multi-Source Data Ingestion\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "file_ingestor = FileIngestor()\n", - "web_ingestor = WebIngestor()\n", - "db_ingestor = DBIngestor()\n", - "stream_ingestor = StreamIngestor()\n", - "document_parser = DocumentParser()\n", - "web_parser = WebParser()\n", - "structured_parser = StructuredDataParser()\n", - "\n", - "temp_dir = tempfile.mkdtemp()\n", - "\n", - "report_file = os.path.join(temp_dir, \"supply_chain_report.txt\")\n", - "with open(report_file, 'w') as f:\n", - " f.write(\"Supplier A delivers components to Factory B. Supplier C supplies Factory D.\")\n", - "\n", - "file_objects = file_ingestor.ingest_file(report_file, read_content=True)\n", - "parsed_file_content = document_parser.extract_text(report_file) if file_objects else \"\"\n", - "\n", - "web_content = \"Shipping delays reported in region X due to weather conditions.\"\n", - "parsed_web_content = web_parser.parse_text(web_content) if web_content else \"\"\n", - "\n", - "db_data_file = os.path.join(temp_dir, \"suppliers_db.json\")\n", - "db_data = [\n", - " {\"supplier\": \"Supplier A\", \"factory\": \"Factory B\", \"status\": \"active\"},\n", - " {\"supplier\": \"Supplier C\", \"factory\": \"Factory D\", \"status\": \"active\"}\n", - "]\n", - "with open(db_data_file, 'w') as f:\n", - " json.dump(db_data, f)\n", - "\n", - "parsed_db_data = structured_parser.parse_json(db_data_file)\n", - "\n", - "stream_events = [\n", - " {\"event\": \"shipment_delayed\", \"supplier\": \"Supplier A\", \"timestamp\": datetime.now().isoformat()}\n", - "]\n", - "\n", - "all_data = []\n", - "if parsed_file_content:\n", - " all_data.append({\"source\": \"file\", \"content\": parsed_file_content, \"type\": \"report\"})\n", - "if parsed_web_content:\n", - " all_data.append({\"source\": \"web\", \"content\": parsed_web_content, \"type\": \"news\"})\n", - "for db_record in parsed_db_data.get(\"data\", db_data):\n", - " all_data.append({\"source\": \"db\", **db_record})\n", - "for stream_event in stream_events:\n", - " all_data.append({\"source\": \"stream\", **stream_event})\n", - "\n", - "print(f\"Ingested data from {len(set(d.get('source') for d in all_data))} sources\")\n", - "print(f\" File sources: {len([d for d in all_data if d.get('source') == 'file'])}\")\n", - "print(f\" Web sources: {len([d for d in all_data if d.get('source') == 'web'])}\")\n", - "print(f\" Database sources: {len([d for d in all_data if d.get('source') == 'db'])}\")\n", - "print(f\" Stream sources: {len([d for d in all_data if d.get('source') == 'stream'])}\")\n", - "print(f\"Total data items: {len(all_data)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Build Supply Chain Knowledge Graph\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ner_extractor = NERExtractor()\n", - "relation_extractor = RelationExtractor()\n", - "builder = GraphBuilder()\n", - "\n", - "supply_chain_entities = []\n", - "relationships = []\n", - "entity_map = {}\n", - "\n", - "for data_item in all_data:\n", - " content = data_item.get(\"content\", \"\")\n", - " if not content:\n", - " content = str(data_item)\n", - " \n", - " extracted_entities = ner_extractor.extract(content)\n", - " extracted_relations = relation_extractor.extract(content, extracted_entities)\n", - " \n", - " for entity in extracted_entities:\n", - " entity_text = entity.get(\"text\", entity.get(\"entity\", \"\"))\n", - " entity_type = entity.get(\"type\", \"Entity\")\n", - " \n", - " if entity_text and entity_text not in entity_map:\n", - " entity_id = entity_text.lower().replace(\" \", \"_\")\n", - " entity_map[entity_text] = entity_id\n", - " \n", - " if \"supplier\" in entity_text.lower() or \"supplier\" in entity_type.lower():\n", - " entity_type = \"Supplier\"\n", - " elif \"factory\" in entity_text.lower() or \"factory\" in entity_type.lower():\n", - " entity_type = \"Factory\"\n", - " elif \"warehouse\" in entity_text.lower():\n", - " entity_type = \"Warehouse\"\n", - " elif \"product\" in entity_text.lower():\n", - " entity_type = \"Product\"\n", - " \n", - " supply_chain_entities.append({\n", - " \"id\": entity_id,\n", - " \"type\": entity_type,\n", - " \"name\": entity_text,\n", - " \"properties\": {}\n", - " })\n", - " \n", - " for rel in extracted_relations:\n", - " source_text = rel.get(\"source\", \"\")\n", - " target_text = rel.get(\"target\", \"\")\n", - " rel_type = rel.get(\"type\", \"related_to\")\n", - " \n", - " if source_text in entity_map and target_text in entity_map:\n", - " relationships.append({\n", - " \"source\": entity_map[source_text],\n", - " \"target\": entity_map[target_text],\n", - " \"type\": rel_type,\n", - " \"properties\": {\"timestamp\": datetime.now().isoformat()}\n", - " })\n", - "\n", - "for db_record in parsed_db_data.get(\"data\", db_data):\n", - " supplier_name = db_record.get(\"supplier\", \"\")\n", - " factory_name = db_record.get(\"factory\", \"\")\n", - " \n", - " if supplier_name and factory_name:\n", - " supplier_id = supplier_name.lower().replace(\" \", \"_\")\n", - " factory_id = factory_name.lower().replace(\" \", \"_\")\n", - " \n", - " if supplier_id not in entity_map:\n", - " entity_map[supplier_name] = supplier_id\n", - " supply_chain_entities.append({\n", - " \"id\": supplier_id,\n", - " \"type\": \"Supplier\",\n", - " \"name\": supplier_name,\n", - " \"properties\": {}\n", - " })\n", - " \n", - " if factory_id not in entity_map:\n", - " entity_map[factory_name] = factory_id\n", - " supply_chain_entities.append({\n", - " \"id\": factory_id,\n", - " \"type\": \"Factory\",\n", - " \"name\": factory_name,\n", - " \"properties\": {\"capacity\": 1000}\n", - " })\n", - " \n", - " relationships.append({\n", - " \"source\": supplier_id,\n", - " \"target\": factory_id,\n", - " \"type\": \"supplies\",\n", - " \"properties\": {\"timestamp\": datetime.now().isoformat(), \"status\": \"active\"}\n", - " })\n", - "\n", - "if \"warehouse_1\" not in entity_map:\n", - " supply_chain_entities.append({\n", - " \"id\": \"warehouse_1\",\n", - " \"type\": \"Warehouse\",\n", - " \"name\": \"Warehouse 1\",\n", - " \"properties\": {}\n", - " })\n", - " entity_map[\"Warehouse 1\"] = \"warehouse_1\"\n", - "\n", - "if \"factory_b\" in entity_map and \"warehouse_1\" in entity_map:\n", - " relationships.append({\n", - " \"source\": \"factory_b\",\n", - " \"target\": \"warehouse_1\",\n", - " \"type\": \"ships_to\",\n", - " \"properties\": {\"timestamp\": datetime.now().isoformat()}\n", - " })\n", - "\n", - "supply_chain_kg = builder.build(supply_chain_entities, relationships)\n", - "\n", - "print(f\"Extracted {len([e for e in supply_chain_entities if e['type'] in ['Supplier', 'Factory']])} supply chain entities from parsed data\")\n", - "print(f\"Built supply chain knowledge graph with {len(supply_chain_entities)} entities and {len(relationships)} relationships\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Analyze Dependencies\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "analyzer = GraphAnalyzer()\n", - "connectivity_analyzer = ConnectivityAnalyzer()\n", - "\n", - "connectivity_analysis = connectivity_analyzer.analyze_connectivity(supply_chain_kg)\n", - "graph_metrics = analyzer.compute_metrics(supply_chain_kg)\n", - "\n", - "entities_list = supply_chain_kg.get(\"entities\", [])\n", - "relationships_list = supply_chain_kg.get(\"relationships\", [])\n", - "entity_map = {e.get(\"id\"): e for e in entities_list}\n", - "\n", - "dependencies = []\n", - "for entity in entities_list:\n", - " entity_id = entity.get(\"id\")\n", - " incoming = [r for r in relationships_list if r.get(\"target\") == entity_id]\n", - " outgoing = [r for r in relationships_list if r.get(\"source\") == entity_id]\n", - " \n", - " if incoming or outgoing:\n", - " dependencies.append({\n", - " \"entity_id\": entity_id,\n", - " \"entity_type\": entity.get(\"type\"),\n", - " \"name\": entity.get(\"name\"),\n", - " \"incoming_dependencies\": len(incoming),\n", - " \"outgoing_dependencies\": len(outgoing),\n", - " \"depends_on\": [entity_map.get(r.get(\"source\"), {}).get(\"name\", r.get(\"source\")) for r in incoming if entity_map.get(r.get(\"source\"))],\n", - " \"supports\": [entity_map.get(r.get(\"target\"), {}).get(\"name\", r.get(\"target\")) for r in outgoing if entity_map.get(r.get(\"target\"))]\n", - " })\n", - "\n", - "print(f\"Analyzed dependencies for {len(dependencies)} entities\")\n", - "print(f\"Graph connectivity: {connectivity_analysis.get('is_connected', False)}\")\n", - "print(f\"Connected components: {len(connectivity_analysis.get('components', []))}\")\n", - "for dep in dependencies:\n", - " print(f\" {dep['name']} ({dep['entity_type']}): {dep['incoming_dependencies']} incoming, {dep['outgoing_dependencies']} outgoing\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Optimize Flow\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "inference_engine = InferenceEngine()\n", - "\n", - "inference_engine.add_rule(\"IF factory has less than 2 suppliers THEN suggest add_redundancy\")\n", - "inference_engine.add_rule(\"IF entity has no incoming dependencies AND has more than 2 outgoing THEN suggest bottleneck_mitigation\")\n", - "\n", - "optimization_facts = []\n", - "for dep in dependencies:\n", - " if dep[\"entity_type\"] == \"Factory\" and dep[\"incoming_dependencies\"] < 2:\n", - " optimization_facts.append({\n", - " \"entity\": dep[\"entity_id\"],\n", - " \"type\": \"factory\",\n", - " \"supplier_count\": dep[\"incoming_dependencies\"]\n", - " })\n", - " if dep[\"incoming_dependencies\"] == 0 and dep[\"outgoing_dependencies\"] > 2:\n", - " optimization_facts.append({\n", - " \"entity\": dep[\"entity_id\"],\n", - " \"type\": \"bottleneck\",\n", - " \"outgoing_count\": dep[\"outgoing_dependencies\"]\n", - " })\n", - "\n", - "if optimization_facts:\n", - " inference_engine.add_facts(optimization_facts)\n", - " inferred_results = inference_engine.forward_chain()\n", - "else:\n", - " inferred_results = []\n", - "\n", - "optimized_flow = []\n", - "factories = [e for e in entities_list if e.get(\"type\") == \"Factory\"]\n", - "for factory in factories:\n", - " factory_id = factory.get(\"id\")\n", - " incoming = [r for r in relationships_list if r.get(\"target\") == factory_id and r.get(\"type\") == \"supplies\"]\n", - " if len(incoming) < 2:\n", - " optimized_flow.append({\n", - " \"type\": \"add_redundancy\",\n", - " \"entity\": factory.get(\"name\"),\n", - " \"suggestion\": f\"Add backup supplier for {factory.get('name')} to reduce risk\"\n", - " })\n", - "\n", - "bottlenecks = [d for d in dependencies if d[\"incoming_dependencies\"] == 0 and d[\"outgoing_dependencies\"] > 2]\n", - "if bottlenecks:\n", - " optimized_flow.append({\n", - " \"type\": \"bottleneck_detected\",\n", - " \"entities\": [b[\"name\"] for b in bottlenecks],\n", - " \"suggestion\": \"Consider adding parallel paths for critical nodes\"\n", - " })\n", - "\n", - "if inferred_results:\n", - " print(f\"Inference engine generated {len(inferred_results)} optimization inferences\")\n", - "\n", - "print(f\"Generated {len(optimized_flow)} optimization suggestions\")\n", - "for suggestion in optimized_flow:\n", - " print(f\" {suggestion['type']}: {suggestion['suggestion']}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Predict Disruptions\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pattern_detector = TemporalPatternDetector()\n", - "\n", - "temporal_patterns = pattern_detector.detect_temporal_patterns(\n", - " supply_chain_kg,\n", - " pattern_type=\"anomaly\",\n", - " min_frequency=1\n", - ")\n", - "\n", - "disruptions = []\n", - "\n", - "delay_events = [d for d in all_data if d.get(\"event\") == \"shipment_delayed\" or \"delay\" in str(d.get(\"content\", \"\")).lower()]\n", - "\n", - "if delay_events:\n", - " disruptions.append({\n", - " \"type\": \"delivery_delay\",\n", - " \"severity\": \"high\",\n", - " \"affected_entities\": [\"Supplier A\"],\n", - " \"description\": \"Shipping delays detected in supply chain\",\n", - " \"recommendation\": \"Activate backup suppliers or adjust production schedules\"\n", - " })\n", - "\n", - "single_supplier_factories = []\n", - "for factory in [e for e in supply_chain_kg.get(\"entities\", []) if e.get(\"type\") == \"Factory\"]:\n", - " factory_id = factory.get(\"id\")\n", - " suppliers = [r for r in supply_chain_kg.get(\"relationships\", []) if r.get(\"target\") == factory_id and r.get(\"type\") == \"supplies\"]\n", - " if len(suppliers) == 1:\n", - " single_supplier_factories.append(factory.get(\"name\"))\n", - "\n", - "if single_supplier_factories:\n", - " disruptions.append({\n", - " \"type\": \"single_point_of_failure\",\n", - " \"severity\": \"medium\",\n", - " \"affected_entities\": single_supplier_factories,\n", - " \"description\": \"Factories with single supplier dependency detected\",\n", - " \"recommendation\": \"Add redundant supplier relationships\"\n", - " })\n", - "\n", - "if temporal_patterns:\n", - " disruptions.append({\n", - " \"type\": \"temporal_anomaly\",\n", - " \"severity\": \"medium\",\n", - " \"description\": f\"Detected {len(temporal_patterns)} temporal anomalies in supply chain\",\n", - " \"recommendation\": \"Review temporal patterns for potential disruptions\"\n", - " })\n", - "\n", - "print(f\"Predicted {len(disruptions)} potential disruptions\")\n", - "for disruption in disruptions:\n", - " print(f\" {disruption['type']} ({disruption['severity']}): {disruption['description']}\")\n", - " print(f\" Recommendation: {disruption['recommendation']}\")\n", - "\n", - "entities_count = len(supply_chain_kg.get(\"entities\", []))\n", - "print(f\"\\nAnalyzed {entities_count} supply chain nodes\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Complete supply chain intelligence workflow:\n", - "- Multi-source data ingested\n", - "- Supply chain knowledge graph built\n", - "- Dependencies analyzed\n", - "- Flow optimization suggested\n", - "- Disruptions predicted\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}