diff --git a/cookbook/specialized_applications/Fraud_Detection_Anomaly_Complete.ipynb b/cookbook/specialized_applications/Fraud_Detection_Anomaly_Complete.ipynb deleted file mode 100644 index fd8dbfb2..00000000 --- a/cookbook/specialized_applications/Fraud_Detection_Anomaly_Complete.ipynb +++ /dev/null @@ -1,388 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Fraud Detection Anomaly Complete\n", - "\n", - "## Overview\n", - "\n", - "Production fraud detection: stream transactions, build temporal knowledge graph, detect patterns, identify anomalies, and implement alert system.\n", - "\n", - "## Workflow: Stream Transactions → Build Temporal KG → Detect Patterns → Identify Anomalies → Alert System\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import StreamIngestor, FileIngestor\n", - "from semantica.parse import DocumentParser, StructuredDataParser\n", - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "from semantica.kg import GraphBuilder, GraphAnalyzer, TemporalPatternDetector\n", - "from semantica.reasoning import InferenceEngine\n", - "from datetime import datetime, timedelta\n", - "import json\n", - "import os\n", - "import tempfile\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Stream Transactions\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "stream_ingestor = StreamIngestor()\n", - "file_ingestor = FileIngestor()\n", - "structured_parser = StructuredDataParser()\n", - "\n", - "temp_dir = tempfile.mkdtemp()\n", - "transactions_file = os.path.join(temp_dir, \"transactions.json\")\n", - "\n", - "transactions_data = [\n", - " {\n", - " \"transaction_id\": \"txn_001\",\n", - " \"user_id\": \"user_123\",\n", - " \"amount\": 150.00,\n", - " \"merchant\": \"Online Store\",\n", - " \"location\": \"New York\",\n", - " \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat(),\n", - " \"device\": \"mobile\"\n", - " },\n", - " {\n", - " \"transaction_id\": \"txn_002\",\n", - " \"user_id\": \"user_123\",\n", - " \"amount\": 2500.00,\n", - " \"merchant\": \"Luxury Store\",\n", - " \"location\": \"Paris\",\n", - " \"timestamp\": (datetime.now() - timedelta(minutes=30)).isoformat(),\n", - " \"device\": \"web\"\n", - " },\n", - " {\n", - " \"transaction_id\": \"txn_003\",\n", - " \"user_id\": \"user_456\",\n", - " \"amount\": 50.00,\n", - " \"merchant\": \"Grocery Store\",\n", - " \"location\": \"San Francisco\",\n", - " \"timestamp\": (datetime.now() - timedelta(minutes=15)).isoformat(),\n", - " \"device\": \"mobile\"\n", - " },\n", - " {\n", - " \"transaction_id\": \"txn_004\",\n", - " \"user_id\": \"user_123\",\n", - " \"amount\": 5000.00,\n", - " \"merchant\": \"Electronics Store\",\n", - " \"location\": \"Tokyo\",\n", - " \"timestamp\": (datetime.now() - timedelta(minutes=5)).isoformat(),\n", - " \"device\": \"mobile\"\n", - " }\n", - "]\n", - "\n", - "with open(transactions_file, 'w') as f:\n", - " json.dump(transactions_data, f)\n", - "\n", - "file_objects = file_ingestor.ingest_file(transactions_file, read_content=True)\n", - "parsed_data = structured_parser.parse_json(transactions_file)\n", - "\n", - "transaction_stream = []\n", - "for txn in parsed_data.get(\"data\", transactions_data):\n", - " if isinstance(txn, dict):\n", - " txn_copy = txn.copy()\n", - " if \"timestamp\" in txn_copy and isinstance(txn_copy[\"timestamp\"], str):\n", - " txn_copy[\"timestamp\"] = datetime.fromisoformat(txn_copy[\"timestamp\"])\n", - " transaction_stream.append(txn_copy)\n", - "\n", - "print(f\"Ingested {len(file_objects)} transaction files\")\n", - "print(f\"Parsed {len(transaction_stream)} transactions from structured data\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Build Temporal Knowledge Graph\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "builder = GraphBuilder()\n", - "\n", - "transaction_entities = []\n", - "relationships = []\n", - "\n", - "for txn in transaction_stream:\n", - " txn_id = txn[\"transaction_id\"]\n", - " user_id = txn[\"user_id\"]\n", - " merchant = txn[\"merchant\"]\n", - " location = txn[\"location\"]\n", - " \n", - " transaction_entities.append({\n", - " \"id\": txn_id,\n", - " \"type\": \"Transaction\",\n", - " \"properties\": {\n", - " \"amount\": txn[\"amount\"],\n", - " \"timestamp\": txn[\"timestamp\"].isoformat(),\n", - " \"device\": txn[\"device\"]\n", - " }\n", - " })\n", - " \n", - " transaction_entities.append({\n", - " \"id\": user_id,\n", - " \"type\": \"User\",\n", - " \"properties\": {}\n", - " })\n", - " \n", - " transaction_entities.append({\n", - " \"id\": merchant,\n", - " \"type\": \"Merchant\",\n", - " \"properties\": {}\n", - " })\n", - " \n", - " transaction_entities.append({\n", - " \"id\": location,\n", - " \"type\": \"Location\",\n", - " \"properties\": {}\n", - " })\n", - " \n", - " relationships.append({\n", - " \"source\": user_id,\n", - " \"target\": txn_id,\n", - " \"type\": \"performed\",\n", - " \"properties\": {\"timestamp\": txn[\"timestamp\"].isoformat()}\n", - " })\n", - " \n", - " relationships.append({\n", - " \"source\": txn_id,\n", - " \"target\": merchant,\n", - " \"type\": \"at_merchant\",\n", - " \"properties\": {}\n", - " })\n", - " \n", - " relationships.append({\n", - " \"source\": txn_id,\n", - " \"target\": location,\n", - " \"type\": \"in_location\",\n", - " \"properties\": {}\n", - " })\n", - "\n", - "transaction_kg = builder.build(transaction_entities, relationships)\n", - "\n", - "print(f\"Built temporal knowledge graph with {len(transaction_entities)} entities and {len(relationships)} relationships\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Detect Patterns\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "inference_engine = InferenceEngine()\n", - "pattern_detector = TemporalPatternDetector()\n", - "graph_analyzer = GraphAnalyzer()\n", - "\n", - "temporal_patterns = pattern_detector.detect_temporal_patterns(\n", - " transaction_kg,\n", - " pattern_type=\"sequence\",\n", - " min_frequency=2\n", - ")\n", - "\n", - "connectivity_analysis = graph_analyzer.analyze_connectivity(transaction_kg)\n", - "\n", - "fraud_patterns = []\n", - "user_transactions = {}\n", - "for txn in transaction_stream:\n", - " user_id = txn[\"user_id\"]\n", - " if user_id not in user_transactions:\n", - " user_transactions[user_id] = []\n", - " user_transactions[user_id].append(txn)\n", - "\n", - "for user_id, txns in user_transactions.items():\n", - " if len(txns) > 1:\n", - " amounts = [t[\"amount\"] for t in txns]\n", - " locations = [t[\"location\"] for t in txns]\n", - " timestamps = [t[\"timestamp\"] for t in txns]\n", - " \n", - " if max(amounts) > 1000:\n", - " fraud_patterns.append({\n", - " \"type\": \"high_value_transaction\",\n", - " \"user_id\": user_id,\n", - " \"amount\": max(amounts),\n", - " \"severity\": \"medium\"\n", - " })\n", - " \n", - " if len(set(locations)) > 2:\n", - " time_span = max(timestamps) - min(timestamps)\n", - " if time_span.total_seconds() < 3600:\n", - " fraud_patterns.append({\n", - " \"type\": \"rapid_location_change\",\n", - " \"user_id\": user_id,\n", - " \"locations\": list(set(locations)),\n", - " \"severity\": \"high\"\n", - " })\n", - "\n", - "print(f\"Detected {len(fraud_patterns)} fraud patterns\")\n", - "print(f\"Temporal patterns: {len(temporal_patterns)}\")\n", - "print(f\"Connectivity analysis: {connectivity_analysis.get('is_connected', False)}\")\n", - "for pattern in fraud_patterns:\n", - " print(f\" Pattern: {pattern['type']} - User: {pattern['user_id']} - Severity: {pattern['severity']}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Identify Anomalies\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "anomaly_patterns = pattern_detector.detect_temporal_patterns(\n", - " transaction_kg,\n", - " pattern_type=\"anomaly\",\n", - " min_frequency=1\n", - ")\n", - "\n", - "anomalies = []\n", - "for txn in transaction_stream:\n", - " score = 0\n", - " reasons = []\n", - " \n", - " if txn[\"amount\"] > 2000:\n", - " score += 3\n", - " reasons.append(\"High transaction amount\")\n", - " \n", - " if txn[\"amount\"] > 1000 and txn[\"device\"] == \"mobile\":\n", - " score += 2\n", - " reasons.append(\"High amount on mobile device\")\n", - " \n", - " user_txns = [t for t in transaction_stream if t[\"user_id\"] == txn[\"user_id\"]]\n", - " if len(user_txns) > 1:\n", - " recent_txns = sorted(user_txns, key=lambda x: x[\"timestamp\"], reverse=True)[:3]\n", - " locations = [t[\"location\"] for t in recent_txns]\n", - " if len(set(locations)) > 2:\n", - " time_span = recent_txns[0][\"timestamp\"] - recent_txns[-1][\"timestamp\"]\n", - " if time_span.total_seconds() < 3600:\n", - " score += 4\n", - " reasons.append(\"Rapid location changes\")\n", - " \n", - " if score >= 3:\n", - " anomalies.append({\n", - " \"transaction_id\": txn[\"transaction_id\"],\n", - " \"user_id\": txn[\"user_id\"],\n", - " \"severity\": \"high\" if score >= 5 else \"medium\",\n", - " \"score\": score,\n", - " \"reasons\": reasons,\n", - " \"timestamp\": txn[\"timestamp\"]\n", - " })\n", - "\n", - "print(f\"Detected {len(anomalies)} anomalies\")\n", - "for anomaly in anomalies:\n", - " print(f\" Transaction: {anomaly['transaction_id']} - Severity: {anomaly['severity']} - Score: {anomaly['score']}\")\n", - " print(f\" Reasons: {', '.join(anomaly['reasons'])}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Alert System\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def send_alert(anomaly):\n", - " alert = {\n", - " \"alert_id\": f\"alert_{anomaly['transaction_id']}\",\n", - " \"transaction_id\": anomaly[\"transaction_id\"],\n", - " \"user_id\": anomaly[\"user_id\"],\n", - " \"severity\": anomaly[\"severity\"],\n", - " \"timestamp\": datetime.now().isoformat(),\n", - " \"reasons\": anomaly[\"reasons\"]\n", - " }\n", - " return alert\n", - "\n", - "def log_fraud_event(anomaly):\n", - " event = {\n", - " \"event_type\": \"fraud_detected\",\n", - " \"transaction_id\": anomaly[\"transaction_id\"],\n", - " \"user_id\": anomaly[\"user_id\"],\n", - " \"severity\": anomaly[\"severity\"],\n", - " \"score\": anomaly[\"score\"],\n", - " \"timestamp\": datetime.now().isoformat()\n", - " }\n", - " return event\n", - "\n", - "threshold = 3\n", - "alerts = []\n", - "fraud_events = []\n", - "\n", - "for anomaly in anomalies:\n", - " if anomaly[\"score\"] >= threshold:\n", - " alert = send_alert(anomaly)\n", - " alerts.append(alert)\n", - " event = log_fraud_event(anomaly)\n", - " fraud_events.append(event)\n", - "\n", - "print(f\"Generated {len(alerts)} alerts\")\n", - "for alert in alerts:\n", - " print(f\" Alert: {alert['alert_id']} - Severity: {alert['severity']} - Transaction: {alert['transaction_id']}\")\n", - "\n", - "print(f\"\\nLogged {len(fraud_events)} fraud events\")\n", - "\n", - "entities_count = len(transaction_kg.get(\"entities\", []))\n", - "print(f\"\\nMonitoring {entities_count} transaction entities\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Production fraud detection workflow:\n", - "- Transaction streaming configured\n", - "- Temporal knowledge graph built\n", - "- Fraud patterns detected\n", - "- Anomalies identified\n", - "- Alert system operational\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/specialized_applications/GraphRAG_Complete.ipynb b/cookbook/specialized_applications/GraphRAG_Complete.ipynb deleted file mode 100644 index bc618c1c..00000000 --- a/cookbook/specialized_applications/GraphRAG_Complete.ipynb +++ /dev/null @@ -1,300 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# GraphRAG Complete\n", - "\n", - "## Overview\n", - "\n", - "Next-generation RAG: build knowledge graph, generate embeddings, store in vector database, implement hybrid RAG, and integrate with LLM.\n", - "\n", - "## Workflow: Build KG → Generate Embeddings → Vector Store → Hybrid RAG → LLM Integration\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FileIngestor, WebIngestor\n", - "from semantica.parse import DocumentParser, WebParser\n", - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "from semantica.kg import GraphBuilder\n", - "from semantica.embeddings import EmbeddingGenerator\n", - "from semantica.vector_store import VectorStore, HybridSearch\n", - "from semantica.context import ContextRetriever\n", - "import numpy as np\n", - "import os\n", - "import tempfile\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Build Knowledge Graph\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "file_ingestor = FileIngestor()\n", - "web_ingestor = WebIngestor()\n", - "document_parser = DocumentParser()\n", - "web_parser = WebParser()\n", - "ner_extractor = NERExtractor()\n", - "relation_extractor = RelationExtractor()\n", - "builder = GraphBuilder()\n", - "\n", - "temp_dir = tempfile.mkdtemp()\n", - "\n", - "doc1_file = os.path.join(temp_dir, \"ai_intro.txt\")\n", - "doc2_file = os.path.join(temp_dir, \"ml_basics.txt\")\n", - "doc3_file = os.path.join(temp_dir, \"dl_guide.txt\")\n", - "\n", - "with open(doc1_file, 'w') as f:\n", - " f.write(\"Introduction to AI: Artificial Intelligence is transforming industries. Neural Networks are key components.\")\n", - "with open(doc2_file, 'w') as f:\n", - " f.write(\"Machine Learning Basics: ML algorithms learn from data patterns. Neural Networks enable complex learning.\")\n", - "with open(doc3_file, 'w') as f:\n", - " f.write(\"Deep Learning Guide: Deep neural networks enable complex learning. Backpropagation is used for training.\")\n", - "\n", - "file_objects = []\n", - "for doc_file in [doc1_file, doc2_file, doc3_file]:\n", - " file_obj = file_ingestor.ingest_file(doc_file, read_content=True)\n", - " if file_obj:\n", - " file_objects.append(file_obj)\n", - "\n", - "parsed_documents = []\n", - "for file_obj in file_objects:\n", - " parsed = document_parser.extract_text(file_obj.path)\n", - " parsed_documents.append({\n", - " \"file\": file_obj.name,\n", - " \"content\": parsed,\n", - " \"metadata\": file_obj.metadata\n", - " })\n", - "\n", - "all_entities = []\n", - "all_relationships = []\n", - "entity_map = {}\n", - "\n", - "for i, doc in enumerate(parsed_documents, 1):\n", - " doc_id = f\"doc{i}\"\n", - " doc_name = doc[\"file\"].replace(\".txt\", \"\").replace(\"_\", \" \").title()\n", - " \n", - " all_entities.append({\n", - " \"id\": doc_id,\n", - " \"type\": \"Document\",\n", - " \"name\": doc_name,\n", - " \"properties\": {\"content\": doc[\"content\"][:100]}\n", - " })\n", - " \n", - " extracted_entities = ner_extractor.extract(doc[\"content\"])\n", - " extracted_relations = relation_extractor.extract(doc[\"content\"], extracted_entities)\n", - " \n", - " for entity in extracted_entities[:5]:\n", - " entity_text = entity.get(\"text\", entity.get(\"entity\", \"\"))\n", - " if entity_text and entity_text not in entity_map:\n", - " entity_id = f\"concept_{len(entity_map) + 1}\"\n", - " entity_map[entity_text] = entity_id\n", - " all_entities.append({\n", - " \"id\": entity_id,\n", - " \"type\": entity.get(\"type\", \"Concept\"),\n", - " \"name\": entity_text,\n", - " \"properties\": {}\n", - " })\n", - " \n", - " all_relationships.append({\n", - " \"source\": doc_id,\n", - " \"target\": entity_id,\n", - " \"type\": \"mentions\"\n", - " })\n", - " \n", - " for rel in extracted_relations[:3]:\n", - " source_text = rel.get(\"source\", \"\")\n", - " target_text = rel.get(\"target\", \"\")\n", - " if source_text in entity_map and target_text in entity_map:\n", - " all_relationships.append({\n", - " \"source\": entity_map[source_text],\n", - " \"target\": entity_map[target_text],\n", - " \"type\": rel.get(\"type\", \"related_to\")\n", - " })\n", - "\n", - "knowledge_graph = builder.build(all_entities, all_relationships)\n", - "\n", - "print(f\"Ingested {len(file_objects)} documents\")\n", - "print(f\"Extracted {len([e for e in all_entities if e['type'] != 'Document'])} concepts\")\n", - "print(f\"Built knowledge graph with {len(all_entities)} entities and {len(all_relationships)} relationships\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Generate Embeddings\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "documents = [doc[\"content\"] for doc in parsed_documents]\n", - "\n", - "generator = EmbeddingGenerator()\n", - "embeddings = generator.generate(documents)\n", - "\n", - "print(f\"Generated embeddings for {len(documents)} parsed documents\")\n", - "print(f\"Embedding dimension: {len(embeddings[0]) if embeddings else 0}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Store in Vector Store\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vector_store = VectorStore()\n", - "\n", - "vector_ids = [f\"doc_{i+1}\" for i in range(len(documents))]\n", - "metadata = [\n", - " {\"doc_id\": \"doc1\", \"topic\": \"AI\", \"type\": \"introduction\"},\n", - " {\"doc_id\": \"doc2\", \"topic\": \"ML\", \"type\": \"tutorial\"},\n", - " {\"doc_id\": \"doc3\", \"topic\": \"DL\", \"type\": \"guide\"}\n", - "]\n", - "\n", - "vector_ids_stored = vector_store.store_vectors(embeddings, metadata)\n", - "\n", - "print(f\"Stored {len(vector_ids_stored)} vectors in vector store\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Hybrid RAG\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "hybrid_search = HybridSearch()\n", - "context_retriever = ContextRetriever(\n", - " knowledge_graph=knowledge_graph,\n", - " vector_store=vector_store\n", - ")\n", - "\n", - "query = \"What is deep learning?\"\n", - "query_embedding = generator.generate([query])[0]\n", - "\n", - "vector_results = vector_store.search_vectors(query_embedding, k=3)\n", - "\n", - "graph_context_results = context_retriever.retrieve(\n", - " query=query,\n", - " max_results=5,\n", - " use_graph_expansion=True,\n", - " max_hops=2\n", - ")\n", - "\n", - "graph_context = []\n", - "for result in graph_context_results:\n", - " graph_context.append({\n", - " \"entity\": result.content,\n", - " \"type\": result.metadata.get(\"type\", \"unknown\"),\n", - " \"related\": [e.get(\"name\", e.get(\"id\")) for e in result.related_entities[:3]]\n", - " })\n", - "\n", - "print(f\"Retrieved {len(vector_results)} vector search results\")\n", - "print(f\"Found {len(graph_context)} relevant graph entities from ContextRetriever\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def format_context(vector_results, graph_context):\n", - " context_parts = []\n", - " \n", - " context_parts.append(\"Retrieved Documents:\")\n", - " for i, result in enumerate(vector_results[:3], 1):\n", - " doc_id = result.get(\"id\", \"unknown\")\n", - " score = result.get(\"score\", 0)\n", - " meta = result.get(\"metadata\", {})\n", - " context_parts.append(f\"{i}. Document {doc_id} (relevance: {score:.3f}, topic: {meta.get('topic', 'N/A')})\")\n", - " \n", - " if graph_context:\n", - " context_parts.append(\"\\nKnowledge Graph Context:\")\n", - " for ctx in graph_context:\n", - " context_parts.append(f\"- {ctx['entity']} ({ctx['type']})\")\n", - " if ctx['related']:\n", - " context_parts.append(f\" Related: {', '.join(ctx['related'])}\")\n", - " \n", - " return \"\\n\".join(context_parts)\n", - "\n", - "def generate_response(query, context):\n", - " response_template = f\"\"\"\n", - "Query: {query}\n", - "\n", - "Context:\n", - "{context}\n", - "\n", - "Response: Based on the retrieved context, {query.lower()} is a topic covered in the knowledge base. \n", - "The relevant documents and graph entities provide comprehensive information about this subject.\n", - "\"\"\"\n", - " return response_template\n", - "\n", - "context = format_context(vector_results, graph_context)\n", - "response = generate_response(query, context)\n", - "\n", - "print(\"Generated Response:\")\n", - "print(response)\n", - "print(f\"\\nUsed {len(vector_results)} graph-enhanced results\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Next-generation RAG workflow:\n", - "- Knowledge graph built\n", - "- Embeddings generated\n", - "- Vectors stored\n", - "- Hybrid RAG implemented\n", - "- LLM integration ready\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/specialized_applications/Hybrid_RAG_Temporal_KG.ipynb b/cookbook/specialized_applications/Hybrid_RAG_Temporal_KG.ipynb deleted file mode 100644 index d5f225a0..00000000 --- a/cookbook/specialized_applications/Hybrid_RAG_Temporal_KG.ipynb +++ /dev/null @@ -1,274 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Hybrid RAG Temporal KG\n", - "\n", - "## Overview\n", - "\n", - "Advanced hybrid search: build temporal knowledge graph, generate vector embeddings, implement hybrid search (Vector + Temporal KG), and enable time-aware retrieval.\n", - "\n", - "## Workflow: Build Temporal KG → Vector Embeddings → Hybrid Search → Time-Aware Retrieval\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FileIngestor, WebIngestor, FeedIngestor\n", - "from semantica.parse import DocumentParser, WebParser, StructuredDataParser\n", - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "from semantica.kg import GraphBuilder, TemporalGraphQuery\n", - "from semantica.embeddings import EmbeddingGenerator\n", - "from semantica.vector_store import VectorStore, HybridSearch\n", - "from datetime import datetime, timedelta\n", - "import numpy as np\n", - "import os\n", - "import tempfile\n", - "import json\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Build Temporal Knowledge Graph\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "file_ingestor = FileIngestor()\n", - "web_ingestor = WebIngestor()\n", - "feed_ingestor = FeedIngestor()\n", - "document_parser = DocumentParser()\n", - "structured_parser = StructuredDataParser()\n", - "ner_extractor = NERExtractor()\n", - "relation_extractor = RelationExtractor()\n", - "builder = GraphBuilder()\n", - "\n", - "temp_dir = tempfile.mkdtemp()\n", - "\n", - "events_file = os.path.join(temp_dir, \"events.json\")\n", - "events_data = [\n", - " {\"event\": \"Product Launch\", \"date\": \"2023-10-15T10:00:00\", \"category\": \"product\"},\n", - " {\"event\": \"Q4 Sales Meeting\", \"date\": \"2023-11-20T14:00:00\", \"category\": \"business\"},\n", - " {\"event\": \"Year End Review\", \"date\": \"2023-12-31T09:00:00\", \"category\": \"business\"},\n", - " {\"event\": \"New Year Planning\", \"date\": \"2024-01-05T10:00:00\", \"category\": \"planning\"}\n", - "]\n", - "\n", - "with open(events_file, 'w') as f:\n", - " json.dump(events_data, f)\n", - "\n", - "file_objects = file_ingestor.ingest_file(events_file, read_content=True)\n", - "parsed_events = structured_parser.parse_json(events_file)\n", - "\n", - "entities = []\n", - "relationships = []\n", - "\n", - "for i, event_data in enumerate(parsed_events.get(\"data\", events_data), 1):\n", - " event_id = f\"event{i}\"\n", - " event_name = event_data.get(\"event\", f\"Event {i}\")\n", - " timestamp = event_data.get(\"date\", \"\")\n", - " category = event_data.get(\"category\", \"general\")\n", - " \n", - " entities.append({\n", - " \"id\": event_id,\n", - " \"type\": \"Event\",\n", - " \"name\": event_name,\n", - " \"properties\": {\"timestamp\": timestamp, \"category\": category}\n", - " })\n", - " \n", - " if i > 1:\n", - " prev_event_id = f\"event{i-1}\"\n", - " relationships.append({\n", - " \"source\": prev_event_id,\n", - " \"target\": event_id,\n", - " \"type\": \"followed_by\",\n", - " \"properties\": {\"timestamp\": timestamp}\n", - " })\n", - "\n", - "temporal_kg = builder.build(entities, relationships)\n", - "\n", - "print(f\"Ingested {len(file_objects)} event files\")\n", - "print(f\"Parsed {len(parsed_events.get('data', []))} events from structured data\")\n", - "print(f\"Built temporal knowledge graph with {len(entities)} entities and {len(relationships)} relationships\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Vector Embeddings\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "documents = []\n", - "for event_data in parsed_events.get(\"data\", events_data):\n", - " event_name = event_data.get(\"event\", \"\")\n", - " date_str = event_data.get(\"date\", \"\")[:10]\n", - " category = event_data.get(\"category\", \"\")\n", - " documents.append(f\"{event_name}: Event occurred on {date_str} in category {category}.\")\n", - "\n", - "generator = EmbeddingGenerator()\n", - "embeddings = generator.generate(documents)\n", - "\n", - "print(f\"Generated embeddings for {len(documents)} documents from parsed events\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Hybrid Search Setup\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vector_store = VectorStore()\n", - "\n", - "metadata = [\n", - " {\"event_id\": \"event1\", \"timestamp\": \"2023-10-15\", \"category\": \"product\"},\n", - " {\"event_id\": \"event2\", \"timestamp\": \"2023-11-20\", \"category\": \"business\"},\n", - " {\"event_id\": \"event3\", \"timestamp\": \"2023-12-31\", \"category\": \"business\"},\n", - " {\"event_id\": \"event4\", \"timestamp\": \"2024-01-05\", \"category\": \"planning\"}\n", - "]\n", - "\n", - "vector_ids = vector_store.store_vectors(embeddings, metadata)\n", - "\n", - "hybrid_search = HybridSearch()\n", - "temporal_query = TemporalGraphQuery()\n", - "\n", - "print(f\"Stored {len(vector_ids)} vectors in vector store\")\n", - "print(\"Hybrid search and temporal query initialized\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Time-Aware Retrieval\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "query = \"What happened in Q4 2023?\"\n", - "query_embedding = generator.generate([query])[0]\n", - "\n", - "vector_results = vector_store.search_vectors(query_embedding, k=10)\n", - "\n", - "temporal_query_result = temporal_query.query_time_range(\n", - " graph=temporal_kg,\n", - " query=query,\n", - " start_time=\"2023-10-01\",\n", - " end_time=\"2023-12-31\",\n", - " temporal_aggregation=\"union\"\n", - ")\n", - "\n", - "temporal_results = []\n", - "entities_list = temporal_kg.get(\"entities\", [])\n", - "entity_map = {e.get(\"id\"): e for e in entities_list}\n", - "\n", - "for rel in temporal_query_result.get(\"relationships\", []):\n", - " source_id = rel.get(\"source\")\n", - " target_id = rel.get(\"target\")\n", - " if source_id in entity_map:\n", - " entity = entity_map[source_id]\n", - " temporal_results.append({\n", - " \"entity_id\": source_id,\n", - " \"name\": entity.get(\"name\"),\n", - " \"timestamp\": entity.get(\"properties\", {}).get(\"timestamp\", \"\"),\n", - " \"type\": entity.get(\"type\")\n", - " })\n", - "\n", - "def combine_results(vector_results, temporal_results):\n", - " combined = []\n", - " \n", - " vector_dict = {r.get(\"id\", \"\"): r for r in vector_results}\n", - " \n", - " for temp_result in temporal_results:\n", - " entity_id = temp_result.get(\"entity_id\", \"\")\n", - " if entity_id in vector_dict:\n", - " combined.append({\n", - " \"id\": entity_id,\n", - " \"name\": temp_result.get(\"name\"),\n", - " \"vector_score\": vector_dict[entity_id].get(\"score\", 0),\n", - " \"timestamp\": temp_result.get(\"timestamp\"),\n", - " \"type\": \"hybrid\"\n", - " })\n", - " else:\n", - " combined.append({\n", - " \"id\": entity_id,\n", - " \"name\": temp_result.get(\"name\"),\n", - " \"vector_score\": 0,\n", - " \"timestamp\": temp_result.get(\"timestamp\"),\n", - " \"type\": \"temporal_only\"\n", - " })\n", - " \n", - " for vec_result in vector_results:\n", - " vec_id = vec_result.get(\"id\", \"\")\n", - " if not any(c.get(\"id\") == vec_id for c in combined):\n", - " combined.append({\n", - " \"id\": vec_id,\n", - " \"vector_score\": vec_result.get(\"score\", 0),\n", - " \"type\": \"vector_only\"\n", - " })\n", - " \n", - " combined.sort(key=lambda x: x.get(\"vector_score\", 0), reverse=True)\n", - " return combined\n", - "\n", - "hybrid_results = combine_results(vector_results, temporal_results)\n", - "\n", - "print(f\"Retrieved {len(hybrid_results)} time-aware results\")\n", - "print(f\" Vector results: {len(vector_results)}\")\n", - "print(f\" Temporal results: {len(temporal_results)}\")\n", - "print(f\" Hybrid results: {len([r for r in hybrid_results if r.get('type') == 'hybrid'])}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Advanced hybrid search workflow:\n", - "- Temporal knowledge graph built\n", - "- Vector embeddings generated\n", - "- Hybrid search configured\n", - "- Time-aware retrieval implemented\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/specialized_applications/Multi_Agent_System_KG_Powered.ipynb b/cookbook/specialized_applications/Multi_Agent_System_KG_Powered.ipynb deleted file mode 100644 index 0b624cec..00000000 --- a/cookbook/specialized_applications/Multi_Agent_System_KG_Powered.ipynb +++ /dev/null @@ -1,360 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Multi-Agent System KG-Powered\n", - "\n", - "## Overview\n", - "\n", - "AI agent systems: build knowledge graph, implement agent memory, create context graphs, enable multi-agent coordination, and share knowledge.\n", - "\n", - "## Workflow: Build KG → Agent Memory → Context Graphs → Multi-Agent Coordination → Shared Knowledge\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FileIngestor, DBIngestor\n", - "from semantica.parse import DocumentParser, StructuredDataParser\n", - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "from semantica.kg import GraphBuilder\n", - "from semantica.context import AgentMemory, ContextRetriever\n", - "from semantica.reasoning import InferenceEngine\n", - "from datetime import datetime\n", - "import os\n", - "import tempfile\n", - "import json\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Build Knowledge Graph\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "file_ingestor = FileIngestor()\n", - "db_ingestor = DBIngestor()\n", - "structured_parser = StructuredDataParser()\n", - "ner_extractor = NERExtractor()\n", - "relation_extractor = RelationExtractor()\n", - "builder = GraphBuilder()\n", - "\n", - "temp_dir = tempfile.mkdtemp()\n", - "\n", - "agents_file = os.path.join(temp_dir, \"agents.json\")\n", - "tasks_file = os.path.join(temp_dir, \"tasks.json\")\n", - "\n", - "agents_data = [\n", - " {\"agent_id\": \"agent_1\", \"name\": \"Research Agent\", \"role\": \"researcher\"},\n", - " {\"agent_id\": \"agent_2\", \"name\": \"Analysis Agent\", \"role\": \"analyst\"}\n", - "]\n", - "\n", - "tasks_data = [\n", - " {\"task_id\": \"task_1\", \"name\": \"Data Collection\", \"status\": \"completed\", \"assigned_to\": \"agent_1\"},\n", - " {\"task_id\": \"task_2\", \"name\": \"Data Analysis\", \"status\": \"in_progress\", \"assigned_to\": \"agent_2\"}\n", - "]\n", - "\n", - "with open(agents_file, 'w') as f:\n", - " json.dump(agents_data, f)\n", - "with open(tasks_file, 'w') as f:\n", - " json.dump(tasks_data, f)\n", - "\n", - "file_objects = file_ingestor.ingest_directory(temp_dir, recursive=False)\n", - "parsed_agents = structured_parser.parse_json(agents_file)\n", - "parsed_tasks = structured_parser.parse_json(tasks_file)\n", - "\n", - "entities = []\n", - "relationships = []\n", - "\n", - "for agent_data in parsed_agents.get(\"data\", agents_data):\n", - " entities.append({\n", - " \"id\": agent_data.get(\"agent_id\", \"\"),\n", - " \"type\": \"Agent\",\n", - " \"name\": agent_data.get(\"name\", \"\"),\n", - " \"properties\": {\"role\": agent_data.get(\"role\", \"\")}\n", - " })\n", - "\n", - "for task_data in parsed_tasks.get(\"data\", tasks_data):\n", - " entities.append({\n", - " \"id\": task_data.get(\"task_id\", \"\"),\n", - " \"type\": \"Task\",\n", - " \"name\": task_data.get(\"name\", \"\"),\n", - " \"properties\": {\"status\": task_data.get(\"status\", \"\")}\n", - " })\n", - " \n", - " assigned_agent = task_data.get(\"assigned_to\", \"\")\n", - " if assigned_agent:\n", - " relationships.append({\n", - " \"source\": assigned_agent,\n", - " \"target\": task_data.get(\"task_id\", \"\"),\n", - " \"type\": \"assigned_to\"\n", - " })\n", - "\n", - "knowledge_content = \"Market Trends: Analysis shows increasing demand in technology sector.\"\n", - "extracted_entities = ner_extractor.extract(knowledge_content)\n", - "extracted_relations = relation_extractor.extract(knowledge_content, extracted_entities)\n", - "\n", - "for entity in extracted_entities[:3]:\n", - " entity_id = f\"knowledge_{len([e for e in entities if e['type'] == 'Knowledge']) + 1}\"\n", - " entities.append({\n", - " \"id\": entity_id,\n", - " \"type\": \"Knowledge\",\n", - " \"name\": entity.get(\"text\", entity.get(\"entity\", \"\")),\n", - " \"properties\": {}\n", - " })\n", - " \n", - " relationships.append({\n", - " \"source\": \"agent_1\",\n", - " \"target\": entity_id,\n", - " \"type\": \"discovered\"\n", - " })\n", - " \n", - " relationships.append({\n", - " \"source\": \"task_1\",\n", - " \"target\": entity_id,\n", - " \"type\": \"produced\"\n", - " })\n", - "\n", - "knowledge_graph = builder.build(entities, relationships)\n", - "\n", - "print(f\"Ingested {len(file_objects)} files\")\n", - "print(f\"Parsed {len(parsed_agents.get('data', []))} agents and {len(parsed_tasks.get('data', []))} tasks\")\n", - "print(f\"Extracted {len([e for e in entities if e['type'] == 'Knowledge'])} knowledge entities\")\n", - "print(f\"Built knowledge graph with {len(entities)} entities and {len(relationships)} relationships\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Agent Memory\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agent_memory = AgentMemory(knowledge_graph=knowledge_graph)\n", - "\n", - "agent_experiences = [\n", - " {\n", - " \"agent_id\": \"agent_1\",\n", - " \"content\": \"Completed data collection task successfully\",\n", - " \"metadata\": {\"task\": \"task_1\", \"timestamp\": datetime.now().isoformat()}\n", - " },\n", - " {\n", - " \"agent_id\": \"agent_2\",\n", - " \"content\": \"Started analyzing collected data\",\n", - " \"metadata\": {\"task\": \"task_2\", \"timestamp\": datetime.now().isoformat()}\n", - " }\n", - "]\n", - "\n", - "for experience in agent_experiences:\n", - " memory_id = agent_memory.store(\n", - " content=experience[\"content\"],\n", - " metadata=experience[\"metadata\"],\n", - " entities=[{\"id\": experience[\"agent_id\"], \"type\": \"Agent\"}]\n", - " )\n", - " print(f\"Stored experience for {experience['agent_id']}: {memory_id}\")\n", - "\n", - "print(f\"\\nTotal memories stored: {agent_memory.stats.get('total_items', 0)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Context Graphs\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "context_retriever = ContextRetriever(knowledge_graph=knowledge_graph)\n", - "\n", - "def get_agent_context(agent_id, query, kg, retriever):\n", - " context_query = f\"{query} for agent {agent_id}\"\n", - " retrieved_context = retriever.retrieve(\n", - " query=context_query,\n", - " max_results=5,\n", - " use_graph_expansion=True,\n", - " max_hops=2,\n", - " entity_ids=[agent_id]\n", - " )\n", - " \n", - " context_items = []\n", - " if retrieved_context:\n", - " context_items.append({\n", - " \"type\": \"agent_info\",\n", - " \"data\": {\"agent_id\": agent_id}\n", - " })\n", - " \n", - " related_tasks = []\n", - " related_knowledge = []\n", - " \n", - " for ctx in retrieved_context:\n", - " for entity in ctx.related_entities:\n", - " if entity.get(\"type\") == \"Task\":\n", - " related_tasks.append(entity)\n", - " elif entity.get(\"type\") == \"Knowledge\":\n", - " related_knowledge.append(entity)\n", - " \n", - " context_items.append({\n", - " \"type\": \"related_tasks\",\n", - " \"data\": related_tasks\n", - " })\n", - " context_items.append({\n", - " \"type\": \"related_knowledge\",\n", - " \"data\": related_knowledge\n", - " })\n", - " \n", - " return context_items\n", - "\n", - "context_agent_1 = get_agent_context(\"agent_1\", \"What should I work on?\", knowledge_graph, context_retriever)\n", - "print(f\"Context for agent_1: {len(context_agent_1)} context items\")\n", - "for item in context_agent_1:\n", - " print(f\" - {item['type']}: {len(item['data']) if isinstance(item['data'], list) else 1} items\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Multi-Agent Coordination\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "shared_knowledge = knowledge_graph\n", - "\n", - "task_1 = \"Analyze market trends\"\n", - "task_2 = \"Review analysis results\"\n", - "\n", - "agent_1_context = get_agent_context(\"agent_1\", task_1, shared_knowledge, context_retriever)\n", - "agent_2_context = get_agent_context(\"agent_2\", task_2, shared_knowledge, context_retriever)\n", - "\n", - "print(\"Agent 1 Context:\")\n", - "for item in agent_1_context:\n", - " if isinstance(item['data'], list):\n", - " print(f\" {item['type']}: {[d.get('name', d.get('id')) for d in item['data']]}\")\n", - " else:\n", - " print(f\" {item['type']}: {item['data'].get('name', item['data'].get('id'))}\")\n", - "\n", - "print(\"\\nAgent 2 Context:\")\n", - "for item in agent_2_context:\n", - " if isinstance(item['data'], list):\n", - " print(f\" {item['type']}: {[d.get('name', d.get('id')) for d in item['data']]}\")\n", - " else:\n", - " print(f\" {item['type']}: {item['data'].get('name', item['data'].get('id'))}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Shared Knowledge\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "inference_engine = InferenceEngine()\n", - "\n", - "inference_engine.add_rule(\"IF agent performs action ON entity THEN agent action entity\")\n", - "\n", - "agent_actions = [\n", - " {\"agent\": \"agent_1\", \"action\": \"discovered\", \"entity\": \"knowledge_1\"},\n", - " {\"agent\": \"agent_2\", \"action\": \"analyzed\", \"entity\": \"knowledge_1\"},\n", - "]\n", - "\n", - "for action in agent_actions:\n", - " inference_engine.add_fact(action)\n", - "\n", - "inferred_results = inference_engine.forward_chain()\n", - "\n", - "new_relationships = []\n", - "for action in agent_actions:\n", - " new_relationships.append({\n", - " \"source\": action[\"agent\"],\n", - " \"target\": action[\"entity\"],\n", - " \"type\": action[\"action\"],\n", - " \"properties\": {\"timestamp\": datetime.now().isoformat(), \"inferred\": False}\n", - " })\n", - "\n", - "for result in inferred_results:\n", - " if hasattr(result, 'conclusion') and isinstance(result.conclusion, dict):\n", - " if \"agent\" in result.conclusion and \"entity\" in result.conclusion:\n", - " new_relationships.append({\n", - " \"source\": result.conclusion.get(\"agent\", \"\"),\n", - " \"target\": result.conclusion.get(\"entity\", \"\"),\n", - " \"type\": result.conclusion.get(\"action\", \"\"),\n", - " \"properties\": {\"timestamp\": datetime.now().isoformat(), \"inferred\": True}\n", - " })\n", - "\n", - "new_knowledge = {\n", - " \"entities\": [],\n", - " \"relationships\": new_relationships\n", - "}\n", - "\n", - "if new_knowledge[\"relationships\"]:\n", - " updated_kg = builder.build(\n", - " knowledge_graph.get(\"entities\", []) + new_knowledge[\"entities\"],\n", - " knowledge_graph.get(\"relationships\", []) + new_knowledge[\"relationships\"]\n", - " )\n", - " print(f\"Updated knowledge graph with {len(new_knowledge['relationships'])} new relationships\")\n", - "else:\n", - " updated_kg = knowledge_graph\n", - "\n", - "entities_count = len(updated_kg.get(\"entities\", []))\n", - "relationships_count = len(updated_kg.get(\"relationships\", []))\n", - "\n", - "print(f\"\\nShared knowledge graph has {entities_count} entities and {relationships_count} relationships\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Multi-agent system workflow:\n", - "- Knowledge graph built\n", - "- Agent memory implemented\n", - "- Context graphs created\n", - "- Multi-agent coordination enabled\n", - "- Shared knowledge maintained\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/specialized_applications/Supply_Chain_End_to_End.ipynb b/cookbook/specialized_applications/Supply_Chain_End_to_End.ipynb deleted file mode 100644 index bdccc1c4..00000000 --- a/cookbook/specialized_applications/Supply_Chain_End_to_End.ipynb +++ /dev/null @@ -1,423 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Supply Chain End-to-End\n", - "\n", - "## Overview\n", - "\n", - "Complete supply chain intelligence: multi-source data ingestion, build supply chain knowledge graph, analyze dependencies, optimize flow, and predict disruptions.\n", - "\n", - "## Workflow: Multi-Source Data → Build Supply Chain KG → Analyze Dependencies → Optimize → Predict Disruptions\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, StreamIngestor\n", - "from semantica.parse import DocumentParser, WebParser, StructuredDataParser\n", - "from semantica.semantic_extract import NERExtractor, RelationExtractor\n", - "from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer, TemporalPatternDetector\n", - "from semantica.reasoning import InferenceEngine\n", - "from datetime import datetime, timedelta\n", - "import os\n", - "import tempfile\n", - "import json\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Multi-Source Data Ingestion\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "file_ingestor = FileIngestor()\n", - "web_ingestor = WebIngestor()\n", - "db_ingestor = DBIngestor()\n", - "stream_ingestor = StreamIngestor()\n", - "document_parser = DocumentParser()\n", - "web_parser = WebParser()\n", - "structured_parser = StructuredDataParser()\n", - "\n", - "temp_dir = tempfile.mkdtemp()\n", - "\n", - "report_file = os.path.join(temp_dir, \"supply_chain_report.txt\")\n", - "with open(report_file, 'w') as f:\n", - " f.write(\"Supplier A delivers components to Factory B. Supplier C supplies Factory D.\")\n", - "\n", - "file_objects = file_ingestor.ingest_file(report_file, read_content=True)\n", - "parsed_file_content = document_parser.extract_text(report_file) if file_objects else \"\"\n", - "\n", - "web_content = \"Shipping delays reported in region X due to weather conditions.\"\n", - "parsed_web_content = web_parser.parse_text(web_content) if web_content else \"\"\n", - "\n", - "db_data_file = os.path.join(temp_dir, \"suppliers_db.json\")\n", - "db_data = [\n", - " {\"supplier\": \"Supplier A\", \"factory\": \"Factory B\", \"status\": \"active\"},\n", - " {\"supplier\": \"Supplier C\", \"factory\": \"Factory D\", \"status\": \"active\"}\n", - "]\n", - "with open(db_data_file, 'w') as f:\n", - " json.dump(db_data, f)\n", - "\n", - "parsed_db_data = structured_parser.parse_json(db_data_file)\n", - "\n", - "stream_events = [\n", - " {\"event\": \"shipment_delayed\", \"supplier\": \"Supplier A\", \"timestamp\": datetime.now().isoformat()}\n", - "]\n", - "\n", - "all_data = []\n", - "if parsed_file_content:\n", - " all_data.append({\"source\": \"file\", \"content\": parsed_file_content, \"type\": \"report\"})\n", - "if parsed_web_content:\n", - " all_data.append({\"source\": \"web\", \"content\": parsed_web_content, \"type\": \"news\"})\n", - "for db_record in parsed_db_data.get(\"data\", db_data):\n", - " all_data.append({\"source\": \"db\", **db_record})\n", - "for stream_event in stream_events:\n", - " all_data.append({\"source\": \"stream\", **stream_event})\n", - "\n", - "print(f\"Ingested data from {len(set(d.get('source') for d in all_data))} sources\")\n", - "print(f\" File sources: {len([d for d in all_data if d.get('source') == 'file'])}\")\n", - "print(f\" Web sources: {len([d for d in all_data if d.get('source') == 'web'])}\")\n", - "print(f\" Database sources: {len([d for d in all_data if d.get('source') == 'db'])}\")\n", - "print(f\" Stream sources: {len([d for d in all_data if d.get('source') == 'stream'])}\")\n", - "print(f\"Total data items: {len(all_data)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Build Supply Chain Knowledge Graph\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ner_extractor = NERExtractor()\n", - "relation_extractor = RelationExtractor()\n", - "builder = GraphBuilder()\n", - "\n", - "supply_chain_entities = []\n", - "relationships = []\n", - "entity_map = {}\n", - "\n", - "for data_item in all_data:\n", - " content = data_item.get(\"content\", \"\")\n", - " if not content:\n", - " content = str(data_item)\n", - " \n", - " extracted_entities = ner_extractor.extract(content)\n", - " extracted_relations = relation_extractor.extract(content, extracted_entities)\n", - " \n", - " for entity in extracted_entities:\n", - " entity_text = entity.get(\"text\", entity.get(\"entity\", \"\"))\n", - " entity_type = entity.get(\"type\", \"Entity\")\n", - " \n", - " if entity_text and entity_text not in entity_map:\n", - " entity_id = entity_text.lower().replace(\" \", \"_\")\n", - " entity_map[entity_text] = entity_id\n", - " \n", - " if \"supplier\" in entity_text.lower() or \"supplier\" in entity_type.lower():\n", - " entity_type = \"Supplier\"\n", - " elif \"factory\" in entity_text.lower() or \"factory\" in entity_type.lower():\n", - " entity_type = \"Factory\"\n", - " elif \"warehouse\" in entity_text.lower():\n", - " entity_type = \"Warehouse\"\n", - " elif \"product\" in entity_text.lower():\n", - " entity_type = \"Product\"\n", - " \n", - " supply_chain_entities.append({\n", - " \"id\": entity_id,\n", - " \"type\": entity_type,\n", - " \"name\": entity_text,\n", - " \"properties\": {}\n", - " })\n", - " \n", - " for rel in extracted_relations:\n", - " source_text = rel.get(\"source\", \"\")\n", - " target_text = rel.get(\"target\", \"\")\n", - " rel_type = rel.get(\"type\", \"related_to\")\n", - " \n", - " if source_text in entity_map and target_text in entity_map:\n", - " relationships.append({\n", - " \"source\": entity_map[source_text],\n", - " \"target\": entity_map[target_text],\n", - " \"type\": rel_type,\n", - " \"properties\": {\"timestamp\": datetime.now().isoformat()}\n", - " })\n", - "\n", - "for db_record in parsed_db_data.get(\"data\", db_data):\n", - " supplier_name = db_record.get(\"supplier\", \"\")\n", - " factory_name = db_record.get(\"factory\", \"\")\n", - " \n", - " if supplier_name and factory_name:\n", - " supplier_id = supplier_name.lower().replace(\" \", \"_\")\n", - " factory_id = factory_name.lower().replace(\" \", \"_\")\n", - " \n", - " if supplier_id not in entity_map:\n", - " entity_map[supplier_name] = supplier_id\n", - " supply_chain_entities.append({\n", - " \"id\": supplier_id,\n", - " \"type\": \"Supplier\",\n", - " \"name\": supplier_name,\n", - " \"properties\": {}\n", - " })\n", - " \n", - " if factory_id not in entity_map:\n", - " entity_map[factory_name] = factory_id\n", - " supply_chain_entities.append({\n", - " \"id\": factory_id,\n", - " \"type\": \"Factory\",\n", - " \"name\": factory_name,\n", - " \"properties\": {\"capacity\": 1000}\n", - " })\n", - " \n", - " relationships.append({\n", - " \"source\": supplier_id,\n", - " \"target\": factory_id,\n", - " \"type\": \"supplies\",\n", - " \"properties\": {\"timestamp\": datetime.now().isoformat(), \"status\": \"active\"}\n", - " })\n", - "\n", - "if \"warehouse_1\" not in entity_map:\n", - " supply_chain_entities.append({\n", - " \"id\": \"warehouse_1\",\n", - " \"type\": \"Warehouse\",\n", - " \"name\": \"Warehouse 1\",\n", - " \"properties\": {}\n", - " })\n", - " entity_map[\"Warehouse 1\"] = \"warehouse_1\"\n", - "\n", - "if \"factory_b\" in entity_map and \"warehouse_1\" in entity_map:\n", - " relationships.append({\n", - " \"source\": \"factory_b\",\n", - " \"target\": \"warehouse_1\",\n", - " \"type\": \"ships_to\",\n", - " \"properties\": {\"timestamp\": datetime.now().isoformat()}\n", - " })\n", - "\n", - "supply_chain_kg = builder.build(supply_chain_entities, relationships)\n", - "\n", - "print(f\"Extracted {len([e for e in supply_chain_entities if e['type'] in ['Supplier', 'Factory']])} supply chain entities from parsed data\")\n", - "print(f\"Built supply chain knowledge graph with {len(supply_chain_entities)} entities and {len(relationships)} relationships\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Analyze Dependencies\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "analyzer = GraphAnalyzer()\n", - "connectivity_analyzer = ConnectivityAnalyzer()\n", - "\n", - "connectivity_analysis = connectivity_analyzer.analyze_connectivity(supply_chain_kg)\n", - "graph_metrics = analyzer.compute_metrics(supply_chain_kg)\n", - "\n", - "entities_list = supply_chain_kg.get(\"entities\", [])\n", - "relationships_list = supply_chain_kg.get(\"relationships\", [])\n", - "entity_map = {e.get(\"id\"): e for e in entities_list}\n", - "\n", - "dependencies = []\n", - "for entity in entities_list:\n", - " entity_id = entity.get(\"id\")\n", - " incoming = [r for r in relationships_list if r.get(\"target\") == entity_id]\n", - " outgoing = [r for r in relationships_list if r.get(\"source\") == entity_id]\n", - " \n", - " if incoming or outgoing:\n", - " dependencies.append({\n", - " \"entity_id\": entity_id,\n", - " \"entity_type\": entity.get(\"type\"),\n", - " \"name\": entity.get(\"name\"),\n", - " \"incoming_dependencies\": len(incoming),\n", - " \"outgoing_dependencies\": len(outgoing),\n", - " \"depends_on\": [entity_map.get(r.get(\"source\"), {}).get(\"name\", r.get(\"source\")) for r in incoming if entity_map.get(r.get(\"source\"))],\n", - " \"supports\": [entity_map.get(r.get(\"target\"), {}).get(\"name\", r.get(\"target\")) for r in outgoing if entity_map.get(r.get(\"target\"))]\n", - " })\n", - "\n", - "print(f\"Analyzed dependencies for {len(dependencies)} entities\")\n", - "print(f\"Graph connectivity: {connectivity_analysis.get('is_connected', False)}\")\n", - "print(f\"Connected components: {len(connectivity_analysis.get('components', []))}\")\n", - "for dep in dependencies:\n", - " print(f\" {dep['name']} ({dep['entity_type']}): {dep['incoming_dependencies']} incoming, {dep['outgoing_dependencies']} outgoing\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Optimize Flow\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "inference_engine = InferenceEngine()\n", - "\n", - "inference_engine.add_rule(\"IF factory has less than 2 suppliers THEN suggest add_redundancy\")\n", - "inference_engine.add_rule(\"IF entity has no incoming dependencies AND has more than 2 outgoing THEN suggest bottleneck_mitigation\")\n", - "\n", - "optimization_facts = []\n", - "for dep in dependencies:\n", - " if dep[\"entity_type\"] == \"Factory\" and dep[\"incoming_dependencies\"] < 2:\n", - " optimization_facts.append({\n", - " \"entity\": dep[\"entity_id\"],\n", - " \"type\": \"factory\",\n", - " \"supplier_count\": dep[\"incoming_dependencies\"]\n", - " })\n", - " if dep[\"incoming_dependencies\"] == 0 and dep[\"outgoing_dependencies\"] > 2:\n", - " optimization_facts.append({\n", - " \"entity\": dep[\"entity_id\"],\n", - " \"type\": \"bottleneck\",\n", - " \"outgoing_count\": dep[\"outgoing_dependencies\"]\n", - " })\n", - "\n", - "if optimization_facts:\n", - " inference_engine.add_facts(optimization_facts)\n", - " inferred_results = inference_engine.forward_chain()\n", - "else:\n", - " inferred_results = []\n", - "\n", - "optimized_flow = []\n", - "factories = [e for e in entities_list if e.get(\"type\") == \"Factory\"]\n", - "for factory in factories:\n", - " factory_id = factory.get(\"id\")\n", - " incoming = [r for r in relationships_list if r.get(\"target\") == factory_id and r.get(\"type\") == \"supplies\"]\n", - " if len(incoming) < 2:\n", - " optimized_flow.append({\n", - " \"type\": \"add_redundancy\",\n", - " \"entity\": factory.get(\"name\"),\n", - " \"suggestion\": f\"Add backup supplier for {factory.get('name')} to reduce risk\"\n", - " })\n", - "\n", - "bottlenecks = [d for d in dependencies if d[\"incoming_dependencies\"] == 0 and d[\"outgoing_dependencies\"] > 2]\n", - "if bottlenecks:\n", - " optimized_flow.append({\n", - " \"type\": \"bottleneck_detected\",\n", - " \"entities\": [b[\"name\"] for b in bottlenecks],\n", - " \"suggestion\": \"Consider adding parallel paths for critical nodes\"\n", - " })\n", - "\n", - "if inferred_results:\n", - " print(f\"Inference engine generated {len(inferred_results)} optimization inferences\")\n", - "\n", - "print(f\"Generated {len(optimized_flow)} optimization suggestions\")\n", - "for suggestion in optimized_flow:\n", - " print(f\" {suggestion['type']}: {suggestion['suggestion']}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Predict Disruptions\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pattern_detector = TemporalPatternDetector()\n", - "\n", - "temporal_patterns = pattern_detector.detect_temporal_patterns(\n", - " supply_chain_kg,\n", - " pattern_type=\"anomaly\",\n", - " min_frequency=1\n", - ")\n", - "\n", - "disruptions = []\n", - "\n", - "delay_events = [d for d in all_data if d.get(\"event\") == \"shipment_delayed\" or \"delay\" in str(d.get(\"content\", \"\")).lower()]\n", - "\n", - "if delay_events:\n", - " disruptions.append({\n", - " \"type\": \"delivery_delay\",\n", - " \"severity\": \"high\",\n", - " \"affected_entities\": [\"Supplier A\"],\n", - " \"description\": \"Shipping delays detected in supply chain\",\n", - " \"recommendation\": \"Activate backup suppliers or adjust production schedules\"\n", - " })\n", - "\n", - "single_supplier_factories = []\n", - "for factory in [e for e in supply_chain_kg.get(\"entities\", []) if e.get(\"type\") == \"Factory\"]:\n", - " factory_id = factory.get(\"id\")\n", - " suppliers = [r for r in supply_chain_kg.get(\"relationships\", []) if r.get(\"target\") == factory_id and r.get(\"type\") == \"supplies\"]\n", - " if len(suppliers) == 1:\n", - " single_supplier_factories.append(factory.get(\"name\"))\n", - "\n", - "if single_supplier_factories:\n", - " disruptions.append({\n", - " \"type\": \"single_point_of_failure\",\n", - " \"severity\": \"medium\",\n", - " \"affected_entities\": single_supplier_factories,\n", - " \"description\": \"Factories with single supplier dependency detected\",\n", - " \"recommendation\": \"Add redundant supplier relationships\"\n", - " })\n", - "\n", - "if temporal_patterns:\n", - " disruptions.append({\n", - " \"type\": \"temporal_anomaly\",\n", - " \"severity\": \"medium\",\n", - " \"description\": f\"Detected {len(temporal_patterns)} temporal anomalies in supply chain\",\n", - " \"recommendation\": \"Review temporal patterns for potential disruptions\"\n", - " })\n", - "\n", - "print(f\"Predicted {len(disruptions)} potential disruptions\")\n", - "for disruption in disruptions:\n", - " print(f\" {disruption['type']} ({disruption['severity']}): {disruption['description']}\")\n", - " print(f\" Recommendation: {disruption['recommendation']}\")\n", - "\n", - "entities_count = len(supply_chain_kg.get(\"entities\", []))\n", - "print(f\"\\nAnalyzed {entities_count} supply chain nodes\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Complete supply chain intelligence workflow:\n", - "- Multi-source data ingested\n", - "- Supply chain knowledge graph built\n", - "- Dependencies analyzed\n", - "- Flow optimization suggested\n", - "- Disruptions predicted\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}