mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Delete cookbook/specialized_applications directory
This commit is contained in:
Mohd Kaif
@@ -1,388 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Fraud Detection Anomaly Complete\n",
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Production fraud detection: stream transactions, build temporal knowledge graph, detect patterns, identify anomalies, and implement alert system.\n",
|
||||
"\n",
|
||||
"## Workflow: Stream Transactions → Build Temporal KG → Detect Patterns → Identify Anomalies → Alert System\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import StreamIngestor, FileIngestor\n",
|
||||
"from semantica.parse import DocumentParser, StructuredDataParser\n",
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor\n",
|
||||
"from semantica.kg import GraphBuilder, GraphAnalyzer, TemporalPatternDetector\n",
|
||||
"from semantica.reasoning import InferenceEngine\n",
|
||||
"from datetime import datetime, timedelta\n",
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"import tempfile\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 1: Stream Transactions\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"stream_ingestor = StreamIngestor()\n",
|
||||
"file_ingestor = FileIngestor()\n",
|
||||
"structured_parser = StructuredDataParser()\n",
|
||||
"\n",
|
||||
"temp_dir = tempfile.mkdtemp()\n",
|
||||
"transactions_file = os.path.join(temp_dir, \"transactions.json\")\n",
|
||||
"\n",
|
||||
"transactions_data = [\n",
|
||||
" {\n",
|
||||
" \"transaction_id\": \"txn_001\",\n",
|
||||
" \"user_id\": \"user_123\",\n",
|
||||
" \"amount\": 150.00,\n",
|
||||
" \"merchant\": \"Online Store\",\n",
|
||||
" \"location\": \"New York\",\n",
|
||||
" \"timestamp\": (datetime.now() - timedelta(hours=1)).isoformat(),\n",
|
||||
" \"device\": \"mobile\"\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"transaction_id\": \"txn_002\",\n",
|
||||
" \"user_id\": \"user_123\",\n",
|
||||
" \"amount\": 2500.00,\n",
|
||||
" \"merchant\": \"Luxury Store\",\n",
|
||||
" \"location\": \"Paris\",\n",
|
||||
" \"timestamp\": (datetime.now() - timedelta(minutes=30)).isoformat(),\n",
|
||||
" \"device\": \"web\"\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"transaction_id\": \"txn_003\",\n",
|
||||
" \"user_id\": \"user_456\",\n",
|
||||
" \"amount\": 50.00,\n",
|
||||
" \"merchant\": \"Grocery Store\",\n",
|
||||
" \"location\": \"San Francisco\",\n",
|
||||
" \"timestamp\": (datetime.now() - timedelta(minutes=15)).isoformat(),\n",
|
||||
" \"device\": \"mobile\"\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"transaction_id\": \"txn_004\",\n",
|
||||
" \"user_id\": \"user_123\",\n",
|
||||
" \"amount\": 5000.00,\n",
|
||||
" \"merchant\": \"Electronics Store\",\n",
|
||||
" \"location\": \"Tokyo\",\n",
|
||||
" \"timestamp\": (datetime.now() - timedelta(minutes=5)).isoformat(),\n",
|
||||
" \"device\": \"mobile\"\n",
|
||||
" }\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"with open(transactions_file, 'w') as f:\n",
|
||||
" json.dump(transactions_data, f)\n",
|
||||
"\n",
|
||||
"file_objects = file_ingestor.ingest_file(transactions_file, read_content=True)\n",
|
||||
"parsed_data = structured_parser.parse_json(transactions_file)\n",
|
||||
"\n",
|
||||
"transaction_stream = []\n",
|
||||
"for txn in parsed_data.get(\"data\", transactions_data):\n",
|
||||
" if isinstance(txn, dict):\n",
|
||||
" txn_copy = txn.copy()\n",
|
||||
" if \"timestamp\" in txn_copy and isinstance(txn_copy[\"timestamp\"], str):\n",
|
||||
" txn_copy[\"timestamp\"] = datetime.fromisoformat(txn_copy[\"timestamp\"])\n",
|
||||
" transaction_stream.append(txn_copy)\n",
|
||||
"\n",
|
||||
"print(f\"Ingested {len(file_objects)} transaction files\")\n",
|
||||
"print(f\"Parsed {len(transaction_stream)} transactions from structured data\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Build Temporal Knowledge Graph\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"builder = GraphBuilder()\n",
|
||||
"\n",
|
||||
"transaction_entities = []\n",
|
||||
"relationships = []\n",
|
||||
"\n",
|
||||
"for txn in transaction_stream:\n",
|
||||
" txn_id = txn[\"transaction_id\"]\n",
|
||||
" user_id = txn[\"user_id\"]\n",
|
||||
" merchant = txn[\"merchant\"]\n",
|
||||
" location = txn[\"location\"]\n",
|
||||
" \n",
|
||||
" transaction_entities.append({\n",
|
||||
" \"id\": txn_id,\n",
|
||||
" \"type\": \"Transaction\",\n",
|
||||
" \"properties\": {\n",
|
||||
" \"amount\": txn[\"amount\"],\n",
|
||||
" \"timestamp\": txn[\"timestamp\"].isoformat(),\n",
|
||||
" \"device\": txn[\"device\"]\n",
|
||||
" }\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" transaction_entities.append({\n",
|
||||
" \"id\": user_id,\n",
|
||||
" \"type\": \"User\",\n",
|
||||
" \"properties\": {}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" transaction_entities.append({\n",
|
||||
" \"id\": merchant,\n",
|
||||
" \"type\": \"Merchant\",\n",
|
||||
" \"properties\": {}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" transaction_entities.append({\n",
|
||||
" \"id\": location,\n",
|
||||
" \"type\": \"Location\",\n",
|
||||
" \"properties\": {}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" relationships.append({\n",
|
||||
" \"source\": user_id,\n",
|
||||
" \"target\": txn_id,\n",
|
||||
" \"type\": \"performed\",\n",
|
||||
" \"properties\": {\"timestamp\": txn[\"timestamp\"].isoformat()}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" relationships.append({\n",
|
||||
" \"source\": txn_id,\n",
|
||||
" \"target\": merchant,\n",
|
||||
" \"type\": \"at_merchant\",\n",
|
||||
" \"properties\": {}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" relationships.append({\n",
|
||||
" \"source\": txn_id,\n",
|
||||
" \"target\": location,\n",
|
||||
" \"type\": \"in_location\",\n",
|
||||
" \"properties\": {}\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"transaction_kg = builder.build(transaction_entities, relationships)\n",
|
||||
"\n",
|
||||
"print(f\"Built temporal knowledge graph with {len(transaction_entities)} entities and {len(relationships)} relationships\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Detect Patterns\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"inference_engine = InferenceEngine()\n",
|
||||
"pattern_detector = TemporalPatternDetector()\n",
|
||||
"graph_analyzer = GraphAnalyzer()\n",
|
||||
"\n",
|
||||
"temporal_patterns = pattern_detector.detect_temporal_patterns(\n",
|
||||
" transaction_kg,\n",
|
||||
" pattern_type=\"sequence\",\n",
|
||||
" min_frequency=2\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"connectivity_analysis = graph_analyzer.analyze_connectivity(transaction_kg)\n",
|
||||
"\n",
|
||||
"fraud_patterns = []\n",
|
||||
"user_transactions = {}\n",
|
||||
"for txn in transaction_stream:\n",
|
||||
" user_id = txn[\"user_id\"]\n",
|
||||
" if user_id not in user_transactions:\n",
|
||||
" user_transactions[user_id] = []\n",
|
||||
" user_transactions[user_id].append(txn)\n",
|
||||
"\n",
|
||||
"for user_id, txns in user_transactions.items():\n",
|
||||
" if len(txns) > 1:\n",
|
||||
" amounts = [t[\"amount\"] for t in txns]\n",
|
||||
" locations = [t[\"location\"] for t in txns]\n",
|
||||
" timestamps = [t[\"timestamp\"] for t in txns]\n",
|
||||
" \n",
|
||||
" if max(amounts) > 1000:\n",
|
||||
" fraud_patterns.append({\n",
|
||||
" \"type\": \"high_value_transaction\",\n",
|
||||
" \"user_id\": user_id,\n",
|
||||
" \"amount\": max(amounts),\n",
|
||||
" \"severity\": \"medium\"\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" if len(set(locations)) > 2:\n",
|
||||
" time_span = max(timestamps) - min(timestamps)\n",
|
||||
" if time_span.total_seconds() < 3600:\n",
|
||||
" fraud_patterns.append({\n",
|
||||
" \"type\": \"rapid_location_change\",\n",
|
||||
" \"user_id\": user_id,\n",
|
||||
" \"locations\": list(set(locations)),\n",
|
||||
" \"severity\": \"high\"\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"print(f\"Detected {len(fraud_patterns)} fraud patterns\")\n",
|
||||
"print(f\"Temporal patterns: {len(temporal_patterns)}\")\n",
|
||||
"print(f\"Connectivity analysis: {connectivity_analysis.get('is_connected', False)}\")\n",
|
||||
"for pattern in fraud_patterns:\n",
|
||||
" print(f\" Pattern: {pattern['type']} - User: {pattern['user_id']} - Severity: {pattern['severity']}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Identify Anomalies\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"anomaly_patterns = pattern_detector.detect_temporal_patterns(\n",
|
||||
" transaction_kg,\n",
|
||||
" pattern_type=\"anomaly\",\n",
|
||||
" min_frequency=1\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"anomalies = []\n",
|
||||
"for txn in transaction_stream:\n",
|
||||
" score = 0\n",
|
||||
" reasons = []\n",
|
||||
" \n",
|
||||
" if txn[\"amount\"] > 2000:\n",
|
||||
" score += 3\n",
|
||||
" reasons.append(\"High transaction amount\")\n",
|
||||
" \n",
|
||||
" if txn[\"amount\"] > 1000 and txn[\"device\"] == \"mobile\":\n",
|
||||
" score += 2\n",
|
||||
" reasons.append(\"High amount on mobile device\")\n",
|
||||
" \n",
|
||||
" user_txns = [t for t in transaction_stream if t[\"user_id\"] == txn[\"user_id\"]]\n",
|
||||
" if len(user_txns) > 1:\n",
|
||||
" recent_txns = sorted(user_txns, key=lambda x: x[\"timestamp\"], reverse=True)[:3]\n",
|
||||
" locations = [t[\"location\"] for t in recent_txns]\n",
|
||||
" if len(set(locations)) > 2:\n",
|
||||
" time_span = recent_txns[0][\"timestamp\"] - recent_txns[-1][\"timestamp\"]\n",
|
||||
" if time_span.total_seconds() < 3600:\n",
|
||||
" score += 4\n",
|
||||
" reasons.append(\"Rapid location changes\")\n",
|
||||
" \n",
|
||||
" if score >= 3:\n",
|
||||
" anomalies.append({\n",
|
||||
" \"transaction_id\": txn[\"transaction_id\"],\n",
|
||||
" \"user_id\": txn[\"user_id\"],\n",
|
||||
" \"severity\": \"high\" if score >= 5 else \"medium\",\n",
|
||||
" \"score\": score,\n",
|
||||
" \"reasons\": reasons,\n",
|
||||
" \"timestamp\": txn[\"timestamp\"]\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"print(f\"Detected {len(anomalies)} anomalies\")\n",
|
||||
"for anomaly in anomalies:\n",
|
||||
" print(f\" Transaction: {anomaly['transaction_id']} - Severity: {anomaly['severity']} - Score: {anomaly['score']}\")\n",
|
||||
" print(f\" Reasons: {', '.join(anomaly['reasons'])}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 5: Alert System\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def send_alert(anomaly):\n",
|
||||
" alert = {\n",
|
||||
" \"alert_id\": f\"alert_{anomaly['transaction_id']}\",\n",
|
||||
" \"transaction_id\": anomaly[\"transaction_id\"],\n",
|
||||
" \"user_id\": anomaly[\"user_id\"],\n",
|
||||
" \"severity\": anomaly[\"severity\"],\n",
|
||||
" \"timestamp\": datetime.now().isoformat(),\n",
|
||||
" \"reasons\": anomaly[\"reasons\"]\n",
|
||||
" }\n",
|
||||
" return alert\n",
|
||||
"\n",
|
||||
"def log_fraud_event(anomaly):\n",
|
||||
" event = {\n",
|
||||
" \"event_type\": \"fraud_detected\",\n",
|
||||
" \"transaction_id\": anomaly[\"transaction_id\"],\n",
|
||||
" \"user_id\": anomaly[\"user_id\"],\n",
|
||||
" \"severity\": anomaly[\"severity\"],\n",
|
||||
" \"score\": anomaly[\"score\"],\n",
|
||||
" \"timestamp\": datetime.now().isoformat()\n",
|
||||
" }\n",
|
||||
" return event\n",
|
||||
"\n",
|
||||
"threshold = 3\n",
|
||||
"alerts = []\n",
|
||||
"fraud_events = []\n",
|
||||
"\n",
|
||||
"for anomaly in anomalies:\n",
|
||||
" if anomaly[\"score\"] >= threshold:\n",
|
||||
" alert = send_alert(anomaly)\n",
|
||||
" alerts.append(alert)\n",
|
||||
" event = log_fraud_event(anomaly)\n",
|
||||
" fraud_events.append(event)\n",
|
||||
"\n",
|
||||
"print(f\"Generated {len(alerts)} alerts\")\n",
|
||||
"for alert in alerts:\n",
|
||||
" print(f\" Alert: {alert['alert_id']} - Severity: {alert['severity']} - Transaction: {alert['transaction_id']}\")\n",
|
||||
"\n",
|
||||
"print(f\"\\nLogged {len(fraud_events)} fraud events\")\n",
|
||||
"\n",
|
||||
"entities_count = len(transaction_kg.get(\"entities\", []))\n",
|
||||
"print(f\"\\nMonitoring {entities_count} transaction entities\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Summary\n",
|
||||
"\n",
|
||||
"Production fraud detection workflow:\n",
|
||||
"- Transaction streaming configured\n",
|
||||
"- Temporal knowledge graph built\n",
|
||||
"- Fraud patterns detected\n",
|
||||
"- Anomalies identified\n",
|
||||
"- Alert system operational\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -1,300 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# GraphRAG Complete\n",
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Next-generation RAG: build knowledge graph, generate embeddings, store in vector database, implement hybrid RAG, and integrate with LLM.\n",
|
||||
"\n",
|
||||
"## Workflow: Build KG → Generate Embeddings → Vector Store → Hybrid RAG → LLM Integration\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import FileIngestor, WebIngestor\n",
|
||||
"from semantica.parse import DocumentParser, WebParser\n",
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor\n",
|
||||
"from semantica.kg import GraphBuilder\n",
|
||||
"from semantica.embeddings import EmbeddingGenerator\n",
|
||||
"from semantica.vector_store import VectorStore, HybridSearch\n",
|
||||
"from semantica.context import ContextRetriever\n",
|
||||
"import numpy as np\n",
|
||||
"import os\n",
|
||||
"import tempfile\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 1: Build Knowledge Graph\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"file_ingestor = FileIngestor()\n",
|
||||
"web_ingestor = WebIngestor()\n",
|
||||
"document_parser = DocumentParser()\n",
|
||||
"web_parser = WebParser()\n",
|
||||
"ner_extractor = NERExtractor()\n",
|
||||
"relation_extractor = RelationExtractor()\n",
|
||||
"builder = GraphBuilder()\n",
|
||||
"\n",
|
||||
"temp_dir = tempfile.mkdtemp()\n",
|
||||
"\n",
|
||||
"doc1_file = os.path.join(temp_dir, \"ai_intro.txt\")\n",
|
||||
"doc2_file = os.path.join(temp_dir, \"ml_basics.txt\")\n",
|
||||
"doc3_file = os.path.join(temp_dir, \"dl_guide.txt\")\n",
|
||||
"\n",
|
||||
"with open(doc1_file, 'w') as f:\n",
|
||||
" f.write(\"Introduction to AI: Artificial Intelligence is transforming industries. Neural Networks are key components.\")\n",
|
||||
"with open(doc2_file, 'w') as f:\n",
|
||||
" f.write(\"Machine Learning Basics: ML algorithms learn from data patterns. Neural Networks enable complex learning.\")\n",
|
||||
"with open(doc3_file, 'w') as f:\n",
|
||||
" f.write(\"Deep Learning Guide: Deep neural networks enable complex learning. Backpropagation is used for training.\")\n",
|
||||
"\n",
|
||||
"file_objects = []\n",
|
||||
"for doc_file in [doc1_file, doc2_file, doc3_file]:\n",
|
||||
" file_obj = file_ingestor.ingest_file(doc_file, read_content=True)\n",
|
||||
" if file_obj:\n",
|
||||
" file_objects.append(file_obj)\n",
|
||||
"\n",
|
||||
"parsed_documents = []\n",
|
||||
"for file_obj in file_objects:\n",
|
||||
" parsed = document_parser.extract_text(file_obj.path)\n",
|
||||
" parsed_documents.append({\n",
|
||||
" \"file\": file_obj.name,\n",
|
||||
" \"content\": parsed,\n",
|
||||
" \"metadata\": file_obj.metadata\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"all_entities = []\n",
|
||||
"all_relationships = []\n",
|
||||
"entity_map = {}\n",
|
||||
"\n",
|
||||
"for i, doc in enumerate(parsed_documents, 1):\n",
|
||||
" doc_id = f\"doc{i}\"\n",
|
||||
" doc_name = doc[\"file\"].replace(\".txt\", \"\").replace(\"_\", \" \").title()\n",
|
||||
" \n",
|
||||
" all_entities.append({\n",
|
||||
" \"id\": doc_id,\n",
|
||||
" \"type\": \"Document\",\n",
|
||||
" \"name\": doc_name,\n",
|
||||
" \"properties\": {\"content\": doc[\"content\"][:100]}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" extracted_entities = ner_extractor.extract(doc[\"content\"])\n",
|
||||
" extracted_relations = relation_extractor.extract(doc[\"content\"], extracted_entities)\n",
|
||||
" \n",
|
||||
" for entity in extracted_entities[:5]:\n",
|
||||
" entity_text = entity.get(\"text\", entity.get(\"entity\", \"\"))\n",
|
||||
" if entity_text and entity_text not in entity_map:\n",
|
||||
" entity_id = f\"concept_{len(entity_map) + 1}\"\n",
|
||||
" entity_map[entity_text] = entity_id\n",
|
||||
" all_entities.append({\n",
|
||||
" \"id\": entity_id,\n",
|
||||
" \"type\": entity.get(\"type\", \"Concept\"),\n",
|
||||
" \"name\": entity_text,\n",
|
||||
" \"properties\": {}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" all_relationships.append({\n",
|
||||
" \"source\": doc_id,\n",
|
||||
" \"target\": entity_id,\n",
|
||||
" \"type\": \"mentions\"\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" for rel in extracted_relations[:3]:\n",
|
||||
" source_text = rel.get(\"source\", \"\")\n",
|
||||
" target_text = rel.get(\"target\", \"\")\n",
|
||||
" if source_text in entity_map and target_text in entity_map:\n",
|
||||
" all_relationships.append({\n",
|
||||
" \"source\": entity_map[source_text],\n",
|
||||
" \"target\": entity_map[target_text],\n",
|
||||
" \"type\": rel.get(\"type\", \"related_to\")\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"knowledge_graph = builder.build(all_entities, all_relationships)\n",
|
||||
"\n",
|
||||
"print(f\"Ingested {len(file_objects)} documents\")\n",
|
||||
"print(f\"Extracted {len([e for e in all_entities if e['type'] != 'Document'])} concepts\")\n",
|
||||
"print(f\"Built knowledge graph with {len(all_entities)} entities and {len(all_relationships)} relationships\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Generate Embeddings\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"documents = [doc[\"content\"] for doc in parsed_documents]\n",
|
||||
"\n",
|
||||
"generator = EmbeddingGenerator()\n",
|
||||
"embeddings = generator.generate(documents)\n",
|
||||
"\n",
|
||||
"print(f\"Generated embeddings for {len(documents)} parsed documents\")\n",
|
||||
"print(f\"Embedding dimension: {len(embeddings[0]) if embeddings else 0}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Store in Vector Store\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"vector_store = VectorStore()\n",
|
||||
"\n",
|
||||
"vector_ids = [f\"doc_{i+1}\" for i in range(len(documents))]\n",
|
||||
"metadata = [\n",
|
||||
" {\"doc_id\": \"doc1\", \"topic\": \"AI\", \"type\": \"introduction\"},\n",
|
||||
" {\"doc_id\": \"doc2\", \"topic\": \"ML\", \"type\": \"tutorial\"},\n",
|
||||
" {\"doc_id\": \"doc3\", \"topic\": \"DL\", \"type\": \"guide\"}\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"vector_ids_stored = vector_store.store_vectors(embeddings, metadata)\n",
|
||||
"\n",
|
||||
"print(f\"Stored {len(vector_ids_stored)} vectors in vector store\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Hybrid RAG\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"hybrid_search = HybridSearch()\n",
|
||||
"context_retriever = ContextRetriever(\n",
|
||||
" knowledge_graph=knowledge_graph,\n",
|
||||
" vector_store=vector_store\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"query = \"What is deep learning?\"\n",
|
||||
"query_embedding = generator.generate([query])[0]\n",
|
||||
"\n",
|
||||
"vector_results = vector_store.search_vectors(query_embedding, k=3)\n",
|
||||
"\n",
|
||||
"graph_context_results = context_retriever.retrieve(\n",
|
||||
" query=query,\n",
|
||||
" max_results=5,\n",
|
||||
" use_graph_expansion=True,\n",
|
||||
" max_hops=2\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"graph_context = []\n",
|
||||
"for result in graph_context_results:\n",
|
||||
" graph_context.append({\n",
|
||||
" \"entity\": result.content,\n",
|
||||
" \"type\": result.metadata.get(\"type\", \"unknown\"),\n",
|
||||
" \"related\": [e.get(\"name\", e.get(\"id\")) for e in result.related_entities[:3]]\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"print(f\"Retrieved {len(vector_results)} vector search results\")\n",
|
||||
"print(f\"Found {len(graph_context)} relevant graph entities from ContextRetriever\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def format_context(vector_results, graph_context):\n",
|
||||
" context_parts = []\n",
|
||||
" \n",
|
||||
" context_parts.append(\"Retrieved Documents:\")\n",
|
||||
" for i, result in enumerate(vector_results[:3], 1):\n",
|
||||
" doc_id = result.get(\"id\", \"unknown\")\n",
|
||||
" score = result.get(\"score\", 0)\n",
|
||||
" meta = result.get(\"metadata\", {})\n",
|
||||
" context_parts.append(f\"{i}. Document {doc_id} (relevance: {score:.3f}, topic: {meta.get('topic', 'N/A')})\")\n",
|
||||
" \n",
|
||||
" if graph_context:\n",
|
||||
" context_parts.append(\"\\nKnowledge Graph Context:\")\n",
|
||||
" for ctx in graph_context:\n",
|
||||
" context_parts.append(f\"- {ctx['entity']} ({ctx['type']})\")\n",
|
||||
" if ctx['related']:\n",
|
||||
" context_parts.append(f\" Related: {', '.join(ctx['related'])}\")\n",
|
||||
" \n",
|
||||
" return \"\\n\".join(context_parts)\n",
|
||||
"\n",
|
||||
"def generate_response(query, context):\n",
|
||||
" response_template = f\"\"\"\n",
|
||||
"Query: {query}\n",
|
||||
"\n",
|
||||
"Context:\n",
|
||||
"{context}\n",
|
||||
"\n",
|
||||
"Response: Based on the retrieved context, {query.lower()} is a topic covered in the knowledge base. \n",
|
||||
"The relevant documents and graph entities provide comprehensive information about this subject.\n",
|
||||
"\"\"\"\n",
|
||||
" return response_template\n",
|
||||
"\n",
|
||||
"context = format_context(vector_results, graph_context)\n",
|
||||
"response = generate_response(query, context)\n",
|
||||
"\n",
|
||||
"print(\"Generated Response:\")\n",
|
||||
"print(response)\n",
|
||||
"print(f\"\\nUsed {len(vector_results)} graph-enhanced results\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Summary\n",
|
||||
"\n",
|
||||
"Next-generation RAG workflow:\n",
|
||||
"- Knowledge graph built\n",
|
||||
"- Embeddings generated\n",
|
||||
"- Vectors stored\n",
|
||||
"- Hybrid RAG implemented\n",
|
||||
"- LLM integration ready\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -1,274 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Hybrid RAG Temporal KG\n",
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Advanced hybrid search: build temporal knowledge graph, generate vector embeddings, implement hybrid search (Vector + Temporal KG), and enable time-aware retrieval.\n",
|
||||
"\n",
|
||||
"## Workflow: Build Temporal KG → Vector Embeddings → Hybrid Search → Time-Aware Retrieval\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import FileIngestor, WebIngestor, FeedIngestor\n",
|
||||
"from semantica.parse import DocumentParser, WebParser, StructuredDataParser\n",
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor\n",
|
||||
"from semantica.kg import GraphBuilder, TemporalGraphQuery\n",
|
||||
"from semantica.embeddings import EmbeddingGenerator\n",
|
||||
"from semantica.vector_store import VectorStore, HybridSearch\n",
|
||||
"from datetime import datetime, timedelta\n",
|
||||
"import numpy as np\n",
|
||||
"import os\n",
|
||||
"import tempfile\n",
|
||||
"import json\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 1: Build Temporal Knowledge Graph\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"file_ingestor = FileIngestor()\n",
|
||||
"web_ingestor = WebIngestor()\n",
|
||||
"feed_ingestor = FeedIngestor()\n",
|
||||
"document_parser = DocumentParser()\n",
|
||||
"structured_parser = StructuredDataParser()\n",
|
||||
"ner_extractor = NERExtractor()\n",
|
||||
"relation_extractor = RelationExtractor()\n",
|
||||
"builder = GraphBuilder()\n",
|
||||
"\n",
|
||||
"temp_dir = tempfile.mkdtemp()\n",
|
||||
"\n",
|
||||
"events_file = os.path.join(temp_dir, \"events.json\")\n",
|
||||
"events_data = [\n",
|
||||
" {\"event\": \"Product Launch\", \"date\": \"2023-10-15T10:00:00\", \"category\": \"product\"},\n",
|
||||
" {\"event\": \"Q4 Sales Meeting\", \"date\": \"2023-11-20T14:00:00\", \"category\": \"business\"},\n",
|
||||
" {\"event\": \"Year End Review\", \"date\": \"2023-12-31T09:00:00\", \"category\": \"business\"},\n",
|
||||
" {\"event\": \"New Year Planning\", \"date\": \"2024-01-05T10:00:00\", \"category\": \"planning\"}\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"with open(events_file, 'w') as f:\n",
|
||||
" json.dump(events_data, f)\n",
|
||||
"\n",
|
||||
"file_objects = file_ingestor.ingest_file(events_file, read_content=True)\n",
|
||||
"parsed_events = structured_parser.parse_json(events_file)\n",
|
||||
"\n",
|
||||
"entities = []\n",
|
||||
"relationships = []\n",
|
||||
"\n",
|
||||
"for i, event_data in enumerate(parsed_events.get(\"data\", events_data), 1):\n",
|
||||
" event_id = f\"event{i}\"\n",
|
||||
" event_name = event_data.get(\"event\", f\"Event {i}\")\n",
|
||||
" timestamp = event_data.get(\"date\", \"\")\n",
|
||||
" category = event_data.get(\"category\", \"general\")\n",
|
||||
" \n",
|
||||
" entities.append({\n",
|
||||
" \"id\": event_id,\n",
|
||||
" \"type\": \"Event\",\n",
|
||||
" \"name\": event_name,\n",
|
||||
" \"properties\": {\"timestamp\": timestamp, \"category\": category}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" if i > 1:\n",
|
||||
" prev_event_id = f\"event{i-1}\"\n",
|
||||
" relationships.append({\n",
|
||||
" \"source\": prev_event_id,\n",
|
||||
" \"target\": event_id,\n",
|
||||
" \"type\": \"followed_by\",\n",
|
||||
" \"properties\": {\"timestamp\": timestamp}\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"temporal_kg = builder.build(entities, relationships)\n",
|
||||
"\n",
|
||||
"print(f\"Ingested {len(file_objects)} event files\")\n",
|
||||
"print(f\"Parsed {len(parsed_events.get('data', []))} events from structured data\")\n",
|
||||
"print(f\"Built temporal knowledge graph with {len(entities)} entities and {len(relationships)} relationships\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Vector Embeddings\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"documents = []\n",
|
||||
"for event_data in parsed_events.get(\"data\", events_data):\n",
|
||||
" event_name = event_data.get(\"event\", \"\")\n",
|
||||
" date_str = event_data.get(\"date\", \"\")[:10]\n",
|
||||
" category = event_data.get(\"category\", \"\")\n",
|
||||
" documents.append(f\"{event_name}: Event occurred on {date_str} in category {category}.\")\n",
|
||||
"\n",
|
||||
"generator = EmbeddingGenerator()\n",
|
||||
"embeddings = generator.generate(documents)\n",
|
||||
"\n",
|
||||
"print(f\"Generated embeddings for {len(documents)} documents from parsed events\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Hybrid Search Setup\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"vector_store = VectorStore()\n",
|
||||
"\n",
|
||||
"metadata = [\n",
|
||||
" {\"event_id\": \"event1\", \"timestamp\": \"2023-10-15\", \"category\": \"product\"},\n",
|
||||
" {\"event_id\": \"event2\", \"timestamp\": \"2023-11-20\", \"category\": \"business\"},\n",
|
||||
" {\"event_id\": \"event3\", \"timestamp\": \"2023-12-31\", \"category\": \"business\"},\n",
|
||||
" {\"event_id\": \"event4\", \"timestamp\": \"2024-01-05\", \"category\": \"planning\"}\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"vector_ids = vector_store.store_vectors(embeddings, metadata)\n",
|
||||
"\n",
|
||||
"hybrid_search = HybridSearch()\n",
|
||||
"temporal_query = TemporalGraphQuery()\n",
|
||||
"\n",
|
||||
"print(f\"Stored {len(vector_ids)} vectors in vector store\")\n",
|
||||
"print(\"Hybrid search and temporal query initialized\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Time-Aware Retrieval\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"query = \"What happened in Q4 2023?\"\n",
|
||||
"query_embedding = generator.generate([query])[0]\n",
|
||||
"\n",
|
||||
"vector_results = vector_store.search_vectors(query_embedding, k=10)\n",
|
||||
"\n",
|
||||
"temporal_query_result = temporal_query.query_time_range(\n",
|
||||
" graph=temporal_kg,\n",
|
||||
" query=query,\n",
|
||||
" start_time=\"2023-10-01\",\n",
|
||||
" end_time=\"2023-12-31\",\n",
|
||||
" temporal_aggregation=\"union\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"temporal_results = []\n",
|
||||
"entities_list = temporal_kg.get(\"entities\", [])\n",
|
||||
"entity_map = {e.get(\"id\"): e for e in entities_list}\n",
|
||||
"\n",
|
||||
"for rel in temporal_query_result.get(\"relationships\", []):\n",
|
||||
" source_id = rel.get(\"source\")\n",
|
||||
" target_id = rel.get(\"target\")\n",
|
||||
" if source_id in entity_map:\n",
|
||||
" entity = entity_map[source_id]\n",
|
||||
" temporal_results.append({\n",
|
||||
" \"entity_id\": source_id,\n",
|
||||
" \"name\": entity.get(\"name\"),\n",
|
||||
" \"timestamp\": entity.get(\"properties\", {}).get(\"timestamp\", \"\"),\n",
|
||||
" \"type\": entity.get(\"type\")\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"def combine_results(vector_results, temporal_results):\n",
|
||||
" combined = []\n",
|
||||
" \n",
|
||||
" vector_dict = {r.get(\"id\", \"\"): r for r in vector_results}\n",
|
||||
" \n",
|
||||
" for temp_result in temporal_results:\n",
|
||||
" entity_id = temp_result.get(\"entity_id\", \"\")\n",
|
||||
" if entity_id in vector_dict:\n",
|
||||
" combined.append({\n",
|
||||
" \"id\": entity_id,\n",
|
||||
" \"name\": temp_result.get(\"name\"),\n",
|
||||
" \"vector_score\": vector_dict[entity_id].get(\"score\", 0),\n",
|
||||
" \"timestamp\": temp_result.get(\"timestamp\"),\n",
|
||||
" \"type\": \"hybrid\"\n",
|
||||
" })\n",
|
||||
" else:\n",
|
||||
" combined.append({\n",
|
||||
" \"id\": entity_id,\n",
|
||||
" \"name\": temp_result.get(\"name\"),\n",
|
||||
" \"vector_score\": 0,\n",
|
||||
" \"timestamp\": temp_result.get(\"timestamp\"),\n",
|
||||
" \"type\": \"temporal_only\"\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" for vec_result in vector_results:\n",
|
||||
" vec_id = vec_result.get(\"id\", \"\")\n",
|
||||
" if not any(c.get(\"id\") == vec_id for c in combined):\n",
|
||||
" combined.append({\n",
|
||||
" \"id\": vec_id,\n",
|
||||
" \"vector_score\": vec_result.get(\"score\", 0),\n",
|
||||
" \"type\": \"vector_only\"\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" combined.sort(key=lambda x: x.get(\"vector_score\", 0), reverse=True)\n",
|
||||
" return combined\n",
|
||||
"\n",
|
||||
"hybrid_results = combine_results(vector_results, temporal_results)\n",
|
||||
"\n",
|
||||
"print(f\"Retrieved {len(hybrid_results)} time-aware results\")\n",
|
||||
"print(f\" Vector results: {len(vector_results)}\")\n",
|
||||
"print(f\" Temporal results: {len(temporal_results)}\")\n",
|
||||
"print(f\" Hybrid results: {len([r for r in hybrid_results if r.get('type') == 'hybrid'])}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Summary\n",
|
||||
"\n",
|
||||
"Advanced hybrid search workflow:\n",
|
||||
"- Temporal knowledge graph built\n",
|
||||
"- Vector embeddings generated\n",
|
||||
"- Hybrid search configured\n",
|
||||
"- Time-aware retrieval implemented\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -1,360 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Multi-Agent System KG-Powered\n",
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"AI agent systems: build knowledge graph, implement agent memory, create context graphs, enable multi-agent coordination, and share knowledge.\n",
|
||||
"\n",
|
||||
"## Workflow: Build KG → Agent Memory → Context Graphs → Multi-Agent Coordination → Shared Knowledge\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import FileIngestor, DBIngestor\n",
|
||||
"from semantica.parse import DocumentParser, StructuredDataParser\n",
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor\n",
|
||||
"from semantica.kg import GraphBuilder\n",
|
||||
"from semantica.context import AgentMemory, ContextRetriever\n",
|
||||
"from semantica.reasoning import InferenceEngine\n",
|
||||
"from datetime import datetime\n",
|
||||
"import os\n",
|
||||
"import tempfile\n",
|
||||
"import json\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 1: Build Knowledge Graph\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"file_ingestor = FileIngestor()\n",
|
||||
"db_ingestor = DBIngestor()\n",
|
||||
"structured_parser = StructuredDataParser()\n",
|
||||
"ner_extractor = NERExtractor()\n",
|
||||
"relation_extractor = RelationExtractor()\n",
|
||||
"builder = GraphBuilder()\n",
|
||||
"\n",
|
||||
"temp_dir = tempfile.mkdtemp()\n",
|
||||
"\n",
|
||||
"agents_file = os.path.join(temp_dir, \"agents.json\")\n",
|
||||
"tasks_file = os.path.join(temp_dir, \"tasks.json\")\n",
|
||||
"\n",
|
||||
"agents_data = [\n",
|
||||
" {\"agent_id\": \"agent_1\", \"name\": \"Research Agent\", \"role\": \"researcher\"},\n",
|
||||
" {\"agent_id\": \"agent_2\", \"name\": \"Analysis Agent\", \"role\": \"analyst\"}\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"tasks_data = [\n",
|
||||
" {\"task_id\": \"task_1\", \"name\": \"Data Collection\", \"status\": \"completed\", \"assigned_to\": \"agent_1\"},\n",
|
||||
" {\"task_id\": \"task_2\", \"name\": \"Data Analysis\", \"status\": \"in_progress\", \"assigned_to\": \"agent_2\"}\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"with open(agents_file, 'w') as f:\n",
|
||||
" json.dump(agents_data, f)\n",
|
||||
"with open(tasks_file, 'w') as f:\n",
|
||||
" json.dump(tasks_data, f)\n",
|
||||
"\n",
|
||||
"file_objects = file_ingestor.ingest_directory(temp_dir, recursive=False)\n",
|
||||
"parsed_agents = structured_parser.parse_json(agents_file)\n",
|
||||
"parsed_tasks = structured_parser.parse_json(tasks_file)\n",
|
||||
"\n",
|
||||
"entities = []\n",
|
||||
"relationships = []\n",
|
||||
"\n",
|
||||
"for agent_data in parsed_agents.get(\"data\", agents_data):\n",
|
||||
" entities.append({\n",
|
||||
" \"id\": agent_data.get(\"agent_id\", \"\"),\n",
|
||||
" \"type\": \"Agent\",\n",
|
||||
" \"name\": agent_data.get(\"name\", \"\"),\n",
|
||||
" \"properties\": {\"role\": agent_data.get(\"role\", \"\")}\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"for task_data in parsed_tasks.get(\"data\", tasks_data):\n",
|
||||
" entities.append({\n",
|
||||
" \"id\": task_data.get(\"task_id\", \"\"),\n",
|
||||
" \"type\": \"Task\",\n",
|
||||
" \"name\": task_data.get(\"name\", \"\"),\n",
|
||||
" \"properties\": {\"status\": task_data.get(\"status\", \"\")}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" assigned_agent = task_data.get(\"assigned_to\", \"\")\n",
|
||||
" if assigned_agent:\n",
|
||||
" relationships.append({\n",
|
||||
" \"source\": assigned_agent,\n",
|
||||
" \"target\": task_data.get(\"task_id\", \"\"),\n",
|
||||
" \"type\": \"assigned_to\"\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"knowledge_content = \"Market Trends: Analysis shows increasing demand in technology sector.\"\n",
|
||||
"extracted_entities = ner_extractor.extract(knowledge_content)\n",
|
||||
"extracted_relations = relation_extractor.extract(knowledge_content, extracted_entities)\n",
|
||||
"\n",
|
||||
"for entity in extracted_entities[:3]:\n",
|
||||
" entity_id = f\"knowledge_{len([e for e in entities if e['type'] == 'Knowledge']) + 1}\"\n",
|
||||
" entities.append({\n",
|
||||
" \"id\": entity_id,\n",
|
||||
" \"type\": \"Knowledge\",\n",
|
||||
" \"name\": entity.get(\"text\", entity.get(\"entity\", \"\")),\n",
|
||||
" \"properties\": {}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" relationships.append({\n",
|
||||
" \"source\": \"agent_1\",\n",
|
||||
" \"target\": entity_id,\n",
|
||||
" \"type\": \"discovered\"\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" relationships.append({\n",
|
||||
" \"source\": \"task_1\",\n",
|
||||
" \"target\": entity_id,\n",
|
||||
" \"type\": \"produced\"\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"knowledge_graph = builder.build(entities, relationships)\n",
|
||||
"\n",
|
||||
"print(f\"Ingested {len(file_objects)} files\")\n",
|
||||
"print(f\"Parsed {len(parsed_agents.get('data', []))} agents and {len(parsed_tasks.get('data', []))} tasks\")\n",
|
||||
"print(f\"Extracted {len([e for e in entities if e['type'] == 'Knowledge'])} knowledge entities\")\n",
|
||||
"print(f\"Built knowledge graph with {len(entities)} entities and {len(relationships)} relationships\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Agent Memory\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"agent_memory = AgentMemory(knowledge_graph=knowledge_graph)\n",
|
||||
"\n",
|
||||
"agent_experiences = [\n",
|
||||
" {\n",
|
||||
" \"agent_id\": \"agent_1\",\n",
|
||||
" \"content\": \"Completed data collection task successfully\",\n",
|
||||
" \"metadata\": {\"task\": \"task_1\", \"timestamp\": datetime.now().isoformat()}\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"agent_id\": \"agent_2\",\n",
|
||||
" \"content\": \"Started analyzing collected data\",\n",
|
||||
" \"metadata\": {\"task\": \"task_2\", \"timestamp\": datetime.now().isoformat()}\n",
|
||||
" }\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for experience in agent_experiences:\n",
|
||||
" memory_id = agent_memory.store(\n",
|
||||
" content=experience[\"content\"],\n",
|
||||
" metadata=experience[\"metadata\"],\n",
|
||||
" entities=[{\"id\": experience[\"agent_id\"], \"type\": \"Agent\"}]\n",
|
||||
" )\n",
|
||||
" print(f\"Stored experience for {experience['agent_id']}: {memory_id}\")\n",
|
||||
"\n",
|
||||
"print(f\"\\nTotal memories stored: {agent_memory.stats.get('total_items', 0)}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Context Graphs\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"context_retriever = ContextRetriever(knowledge_graph=knowledge_graph)\n",
|
||||
"\n",
|
||||
"def get_agent_context(agent_id, query, kg, retriever):\n",
|
||||
" context_query = f\"{query} for agent {agent_id}\"\n",
|
||||
" retrieved_context = retriever.retrieve(\n",
|
||||
" query=context_query,\n",
|
||||
" max_results=5,\n",
|
||||
" use_graph_expansion=True,\n",
|
||||
" max_hops=2,\n",
|
||||
" entity_ids=[agent_id]\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" context_items = []\n",
|
||||
" if retrieved_context:\n",
|
||||
" context_items.append({\n",
|
||||
" \"type\": \"agent_info\",\n",
|
||||
" \"data\": {\"agent_id\": agent_id}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" related_tasks = []\n",
|
||||
" related_knowledge = []\n",
|
||||
" \n",
|
||||
" for ctx in retrieved_context:\n",
|
||||
" for entity in ctx.related_entities:\n",
|
||||
" if entity.get(\"type\") == \"Task\":\n",
|
||||
" related_tasks.append(entity)\n",
|
||||
" elif entity.get(\"type\") == \"Knowledge\":\n",
|
||||
" related_knowledge.append(entity)\n",
|
||||
" \n",
|
||||
" context_items.append({\n",
|
||||
" \"type\": \"related_tasks\",\n",
|
||||
" \"data\": related_tasks\n",
|
||||
" })\n",
|
||||
" context_items.append({\n",
|
||||
" \"type\": \"related_knowledge\",\n",
|
||||
" \"data\": related_knowledge\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" return context_items\n",
|
||||
"\n",
|
||||
"context_agent_1 = get_agent_context(\"agent_1\", \"What should I work on?\", knowledge_graph, context_retriever)\n",
|
||||
"print(f\"Context for agent_1: {len(context_agent_1)} context items\")\n",
|
||||
"for item in context_agent_1:\n",
|
||||
" print(f\" - {item['type']}: {len(item['data']) if isinstance(item['data'], list) else 1} items\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Multi-Agent Coordination\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"shared_knowledge = knowledge_graph\n",
|
||||
"\n",
|
||||
"task_1 = \"Analyze market trends\"\n",
|
||||
"task_2 = \"Review analysis results\"\n",
|
||||
"\n",
|
||||
"agent_1_context = get_agent_context(\"agent_1\", task_1, shared_knowledge, context_retriever)\n",
|
||||
"agent_2_context = get_agent_context(\"agent_2\", task_2, shared_knowledge, context_retriever)\n",
|
||||
"\n",
|
||||
"print(\"Agent 1 Context:\")\n",
|
||||
"for item in agent_1_context:\n",
|
||||
" if isinstance(item['data'], list):\n",
|
||||
" print(f\" {item['type']}: {[d.get('name', d.get('id')) for d in item['data']]}\")\n",
|
||||
" else:\n",
|
||||
" print(f\" {item['type']}: {item['data'].get('name', item['data'].get('id'))}\")\n",
|
||||
"\n",
|
||||
"print(\"\\nAgent 2 Context:\")\n",
|
||||
"for item in agent_2_context:\n",
|
||||
" if isinstance(item['data'], list):\n",
|
||||
" print(f\" {item['type']}: {[d.get('name', d.get('id')) for d in item['data']]}\")\n",
|
||||
" else:\n",
|
||||
" print(f\" {item['type']}: {item['data'].get('name', item['data'].get('id'))}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 5: Shared Knowledge\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"inference_engine = InferenceEngine()\n",
|
||||
"\n",
|
||||
"inference_engine.add_rule(\"IF agent performs action ON entity THEN agent action entity\")\n",
|
||||
"\n",
|
||||
"agent_actions = [\n",
|
||||
" {\"agent\": \"agent_1\", \"action\": \"discovered\", \"entity\": \"knowledge_1\"},\n",
|
||||
" {\"agent\": \"agent_2\", \"action\": \"analyzed\", \"entity\": \"knowledge_1\"},\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for action in agent_actions:\n",
|
||||
" inference_engine.add_fact(action)\n",
|
||||
"\n",
|
||||
"inferred_results = inference_engine.forward_chain()\n",
|
||||
"\n",
|
||||
"new_relationships = []\n",
|
||||
"for action in agent_actions:\n",
|
||||
" new_relationships.append({\n",
|
||||
" \"source\": action[\"agent\"],\n",
|
||||
" \"target\": action[\"entity\"],\n",
|
||||
" \"type\": action[\"action\"],\n",
|
||||
" \"properties\": {\"timestamp\": datetime.now().isoformat(), \"inferred\": False}\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"for result in inferred_results:\n",
|
||||
" if hasattr(result, 'conclusion') and isinstance(result.conclusion, dict):\n",
|
||||
" if \"agent\" in result.conclusion and \"entity\" in result.conclusion:\n",
|
||||
" new_relationships.append({\n",
|
||||
" \"source\": result.conclusion.get(\"agent\", \"\"),\n",
|
||||
" \"target\": result.conclusion.get(\"entity\", \"\"),\n",
|
||||
" \"type\": result.conclusion.get(\"action\", \"\"),\n",
|
||||
" \"properties\": {\"timestamp\": datetime.now().isoformat(), \"inferred\": True}\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"new_knowledge = {\n",
|
||||
" \"entities\": [],\n",
|
||||
" \"relationships\": new_relationships\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"if new_knowledge[\"relationships\"]:\n",
|
||||
" updated_kg = builder.build(\n",
|
||||
" knowledge_graph.get(\"entities\", []) + new_knowledge[\"entities\"],\n",
|
||||
" knowledge_graph.get(\"relationships\", []) + new_knowledge[\"relationships\"]\n",
|
||||
" )\n",
|
||||
" print(f\"Updated knowledge graph with {len(new_knowledge['relationships'])} new relationships\")\n",
|
||||
"else:\n",
|
||||
" updated_kg = knowledge_graph\n",
|
||||
"\n",
|
||||
"entities_count = len(updated_kg.get(\"entities\", []))\n",
|
||||
"relationships_count = len(updated_kg.get(\"relationships\", []))\n",
|
||||
"\n",
|
||||
"print(f\"\\nShared knowledge graph has {entities_count} entities and {relationships_count} relationships\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Summary\n",
|
||||
"\n",
|
||||
"Multi-agent system workflow:\n",
|
||||
"- Knowledge graph built\n",
|
||||
"- Agent memory implemented\n",
|
||||
"- Context graphs created\n",
|
||||
"- Multi-agent coordination enabled\n",
|
||||
"- Shared knowledge maintained\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -1,423 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Supply Chain End-to-End\n",
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Complete supply chain intelligence: multi-source data ingestion, build supply chain knowledge graph, analyze dependencies, optimize flow, and predict disruptions.\n",
|
||||
"\n",
|
||||
"## Workflow: Multi-Source Data → Build Supply Chain KG → Analyze Dependencies → Optimize → Predict Disruptions\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, StreamIngestor\n",
|
||||
"from semantica.parse import DocumentParser, WebParser, StructuredDataParser\n",
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor\n",
|
||||
"from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer, TemporalPatternDetector\n",
|
||||
"from semantica.reasoning import InferenceEngine\n",
|
||||
"from datetime import datetime, timedelta\n",
|
||||
"import os\n",
|
||||
"import tempfile\n",
|
||||
"import json\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 1: Multi-Source Data Ingestion\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"file_ingestor = FileIngestor()\n",
|
||||
"web_ingestor = WebIngestor()\n",
|
||||
"db_ingestor = DBIngestor()\n",
|
||||
"stream_ingestor = StreamIngestor()\n",
|
||||
"document_parser = DocumentParser()\n",
|
||||
"web_parser = WebParser()\n",
|
||||
"structured_parser = StructuredDataParser()\n",
|
||||
"\n",
|
||||
"temp_dir = tempfile.mkdtemp()\n",
|
||||
"\n",
|
||||
"report_file = os.path.join(temp_dir, \"supply_chain_report.txt\")\n",
|
||||
"with open(report_file, 'w') as f:\n",
|
||||
" f.write(\"Supplier A delivers components to Factory B. Supplier C supplies Factory D.\")\n",
|
||||
"\n",
|
||||
"file_objects = file_ingestor.ingest_file(report_file, read_content=True)\n",
|
||||
"parsed_file_content = document_parser.extract_text(report_file) if file_objects else \"\"\n",
|
||||
"\n",
|
||||
"web_content = \"Shipping delays reported in region X due to weather conditions.\"\n",
|
||||
"parsed_web_content = web_parser.parse_text(web_content) if web_content else \"\"\n",
|
||||
"\n",
|
||||
"db_data_file = os.path.join(temp_dir, \"suppliers_db.json\")\n",
|
||||
"db_data = [\n",
|
||||
" {\"supplier\": \"Supplier A\", \"factory\": \"Factory B\", \"status\": \"active\"},\n",
|
||||
" {\"supplier\": \"Supplier C\", \"factory\": \"Factory D\", \"status\": \"active\"}\n",
|
||||
"]\n",
|
||||
"with open(db_data_file, 'w') as f:\n",
|
||||
" json.dump(db_data, f)\n",
|
||||
"\n",
|
||||
"parsed_db_data = structured_parser.parse_json(db_data_file)\n",
|
||||
"\n",
|
||||
"stream_events = [\n",
|
||||
" {\"event\": \"shipment_delayed\", \"supplier\": \"Supplier A\", \"timestamp\": datetime.now().isoformat()}\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"all_data = []\n",
|
||||
"if parsed_file_content:\n",
|
||||
" all_data.append({\"source\": \"file\", \"content\": parsed_file_content, \"type\": \"report\"})\n",
|
||||
"if parsed_web_content:\n",
|
||||
" all_data.append({\"source\": \"web\", \"content\": parsed_web_content, \"type\": \"news\"})\n",
|
||||
"for db_record in parsed_db_data.get(\"data\", db_data):\n",
|
||||
" all_data.append({\"source\": \"db\", **db_record})\n",
|
||||
"for stream_event in stream_events:\n",
|
||||
" all_data.append({\"source\": \"stream\", **stream_event})\n",
|
||||
"\n",
|
||||
"print(f\"Ingested data from {len(set(d.get('source') for d in all_data))} sources\")\n",
|
||||
"print(f\" File sources: {len([d for d in all_data if d.get('source') == 'file'])}\")\n",
|
||||
"print(f\" Web sources: {len([d for d in all_data if d.get('source') == 'web'])}\")\n",
|
||||
"print(f\" Database sources: {len([d for d in all_data if d.get('source') == 'db'])}\")\n",
|
||||
"print(f\" Stream sources: {len([d for d in all_data if d.get('source') == 'stream'])}\")\n",
|
||||
"print(f\"Total data items: {len(all_data)}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Build Supply Chain Knowledge Graph\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"ner_extractor = NERExtractor()\n",
|
||||
"relation_extractor = RelationExtractor()\n",
|
||||
"builder = GraphBuilder()\n",
|
||||
"\n",
|
||||
"supply_chain_entities = []\n",
|
||||
"relationships = []\n",
|
||||
"entity_map = {}\n",
|
||||
"\n",
|
||||
"for data_item in all_data:\n",
|
||||
" content = data_item.get(\"content\", \"\")\n",
|
||||
" if not content:\n",
|
||||
" content = str(data_item)\n",
|
||||
" \n",
|
||||
" extracted_entities = ner_extractor.extract(content)\n",
|
||||
" extracted_relations = relation_extractor.extract(content, extracted_entities)\n",
|
||||
" \n",
|
||||
" for entity in extracted_entities:\n",
|
||||
" entity_text = entity.get(\"text\", entity.get(\"entity\", \"\"))\n",
|
||||
" entity_type = entity.get(\"type\", \"Entity\")\n",
|
||||
" \n",
|
||||
" if entity_text and entity_text not in entity_map:\n",
|
||||
" entity_id = entity_text.lower().replace(\" \", \"_\")\n",
|
||||
" entity_map[entity_text] = entity_id\n",
|
||||
" \n",
|
||||
" if \"supplier\" in entity_text.lower() or \"supplier\" in entity_type.lower():\n",
|
||||
" entity_type = \"Supplier\"\n",
|
||||
" elif \"factory\" in entity_text.lower() or \"factory\" in entity_type.lower():\n",
|
||||
" entity_type = \"Factory\"\n",
|
||||
" elif \"warehouse\" in entity_text.lower():\n",
|
||||
" entity_type = \"Warehouse\"\n",
|
||||
" elif \"product\" in entity_text.lower():\n",
|
||||
" entity_type = \"Product\"\n",
|
||||
" \n",
|
||||
" supply_chain_entities.append({\n",
|
||||
" \"id\": entity_id,\n",
|
||||
" \"type\": entity_type,\n",
|
||||
" \"name\": entity_text,\n",
|
||||
" \"properties\": {}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" for rel in extracted_relations:\n",
|
||||
" source_text = rel.get(\"source\", \"\")\n",
|
||||
" target_text = rel.get(\"target\", \"\")\n",
|
||||
" rel_type = rel.get(\"type\", \"related_to\")\n",
|
||||
" \n",
|
||||
" if source_text in entity_map and target_text in entity_map:\n",
|
||||
" relationships.append({\n",
|
||||
" \"source\": entity_map[source_text],\n",
|
||||
" \"target\": entity_map[target_text],\n",
|
||||
" \"type\": rel_type,\n",
|
||||
" \"properties\": {\"timestamp\": datetime.now().isoformat()}\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"for db_record in parsed_db_data.get(\"data\", db_data):\n",
|
||||
" supplier_name = db_record.get(\"supplier\", \"\")\n",
|
||||
" factory_name = db_record.get(\"factory\", \"\")\n",
|
||||
" \n",
|
||||
" if supplier_name and factory_name:\n",
|
||||
" supplier_id = supplier_name.lower().replace(\" \", \"_\")\n",
|
||||
" factory_id = factory_name.lower().replace(\" \", \"_\")\n",
|
||||
" \n",
|
||||
" if supplier_id not in entity_map:\n",
|
||||
" entity_map[supplier_name] = supplier_id\n",
|
||||
" supply_chain_entities.append({\n",
|
||||
" \"id\": supplier_id,\n",
|
||||
" \"type\": \"Supplier\",\n",
|
||||
" \"name\": supplier_name,\n",
|
||||
" \"properties\": {}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" if factory_id not in entity_map:\n",
|
||||
" entity_map[factory_name] = factory_id\n",
|
||||
" supply_chain_entities.append({\n",
|
||||
" \"id\": factory_id,\n",
|
||||
" \"type\": \"Factory\",\n",
|
||||
" \"name\": factory_name,\n",
|
||||
" \"properties\": {\"capacity\": 1000}\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" relationships.append({\n",
|
||||
" \"source\": supplier_id,\n",
|
||||
" \"target\": factory_id,\n",
|
||||
" \"type\": \"supplies\",\n",
|
||||
" \"properties\": {\"timestamp\": datetime.now().isoformat(), \"status\": \"active\"}\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"if \"warehouse_1\" not in entity_map:\n",
|
||||
" supply_chain_entities.append({\n",
|
||||
" \"id\": \"warehouse_1\",\n",
|
||||
" \"type\": \"Warehouse\",\n",
|
||||
" \"name\": \"Warehouse 1\",\n",
|
||||
" \"properties\": {}\n",
|
||||
" })\n",
|
||||
" entity_map[\"Warehouse 1\"] = \"warehouse_1\"\n",
|
||||
"\n",
|
||||
"if \"factory_b\" in entity_map and \"warehouse_1\" in entity_map:\n",
|
||||
" relationships.append({\n",
|
||||
" \"source\": \"factory_b\",\n",
|
||||
" \"target\": \"warehouse_1\",\n",
|
||||
" \"type\": \"ships_to\",\n",
|
||||
" \"properties\": {\"timestamp\": datetime.now().isoformat()}\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"supply_chain_kg = builder.build(supply_chain_entities, relationships)\n",
|
||||
"\n",
|
||||
"print(f\"Extracted {len([e for e in supply_chain_entities if e['type'] in ['Supplier', 'Factory']])} supply chain entities from parsed data\")\n",
|
||||
"print(f\"Built supply chain knowledge graph with {len(supply_chain_entities)} entities and {len(relationships)} relationships\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Analyze Dependencies\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"analyzer = GraphAnalyzer()\n",
|
||||
"connectivity_analyzer = ConnectivityAnalyzer()\n",
|
||||
"\n",
|
||||
"connectivity_analysis = connectivity_analyzer.analyze_connectivity(supply_chain_kg)\n",
|
||||
"graph_metrics = analyzer.compute_metrics(supply_chain_kg)\n",
|
||||
"\n",
|
||||
"entities_list = supply_chain_kg.get(\"entities\", [])\n",
|
||||
"relationships_list = supply_chain_kg.get(\"relationships\", [])\n",
|
||||
"entity_map = {e.get(\"id\"): e for e in entities_list}\n",
|
||||
"\n",
|
||||
"dependencies = []\n",
|
||||
"for entity in entities_list:\n",
|
||||
" entity_id = entity.get(\"id\")\n",
|
||||
" incoming = [r for r in relationships_list if r.get(\"target\") == entity_id]\n",
|
||||
" outgoing = [r for r in relationships_list if r.get(\"source\") == entity_id]\n",
|
||||
" \n",
|
||||
" if incoming or outgoing:\n",
|
||||
" dependencies.append({\n",
|
||||
" \"entity_id\": entity_id,\n",
|
||||
" \"entity_type\": entity.get(\"type\"),\n",
|
||||
" \"name\": entity.get(\"name\"),\n",
|
||||
" \"incoming_dependencies\": len(incoming),\n",
|
||||
" \"outgoing_dependencies\": len(outgoing),\n",
|
||||
" \"depends_on\": [entity_map.get(r.get(\"source\"), {}).get(\"name\", r.get(\"source\")) for r in incoming if entity_map.get(r.get(\"source\"))],\n",
|
||||
" \"supports\": [entity_map.get(r.get(\"target\"), {}).get(\"name\", r.get(\"target\")) for r in outgoing if entity_map.get(r.get(\"target\"))]\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"print(f\"Analyzed dependencies for {len(dependencies)} entities\")\n",
|
||||
"print(f\"Graph connectivity: {connectivity_analysis.get('is_connected', False)}\")\n",
|
||||
"print(f\"Connected components: {len(connectivity_analysis.get('components', []))}\")\n",
|
||||
"for dep in dependencies:\n",
|
||||
" print(f\" {dep['name']} ({dep['entity_type']}): {dep['incoming_dependencies']} incoming, {dep['outgoing_dependencies']} outgoing\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Optimize Flow\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"inference_engine = InferenceEngine()\n",
|
||||
"\n",
|
||||
"inference_engine.add_rule(\"IF factory has less than 2 suppliers THEN suggest add_redundancy\")\n",
|
||||
"inference_engine.add_rule(\"IF entity has no incoming dependencies AND has more than 2 outgoing THEN suggest bottleneck_mitigation\")\n",
|
||||
"\n",
|
||||
"optimization_facts = []\n",
|
||||
"for dep in dependencies:\n",
|
||||
" if dep[\"entity_type\"] == \"Factory\" and dep[\"incoming_dependencies\"] < 2:\n",
|
||||
" optimization_facts.append({\n",
|
||||
" \"entity\": dep[\"entity_id\"],\n",
|
||||
" \"type\": \"factory\",\n",
|
||||
" \"supplier_count\": dep[\"incoming_dependencies\"]\n",
|
||||
" })\n",
|
||||
" if dep[\"incoming_dependencies\"] == 0 and dep[\"outgoing_dependencies\"] > 2:\n",
|
||||
" optimization_facts.append({\n",
|
||||
" \"entity\": dep[\"entity_id\"],\n",
|
||||
" \"type\": \"bottleneck\",\n",
|
||||
" \"outgoing_count\": dep[\"outgoing_dependencies\"]\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"if optimization_facts:\n",
|
||||
" inference_engine.add_facts(optimization_facts)\n",
|
||||
" inferred_results = inference_engine.forward_chain()\n",
|
||||
"else:\n",
|
||||
" inferred_results = []\n",
|
||||
"\n",
|
||||
"optimized_flow = []\n",
|
||||
"factories = [e for e in entities_list if e.get(\"type\") == \"Factory\"]\n",
|
||||
"for factory in factories:\n",
|
||||
" factory_id = factory.get(\"id\")\n",
|
||||
" incoming = [r for r in relationships_list if r.get(\"target\") == factory_id and r.get(\"type\") == \"supplies\"]\n",
|
||||
" if len(incoming) < 2:\n",
|
||||
" optimized_flow.append({\n",
|
||||
" \"type\": \"add_redundancy\",\n",
|
||||
" \"entity\": factory.get(\"name\"),\n",
|
||||
" \"suggestion\": f\"Add backup supplier for {factory.get('name')} to reduce risk\"\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"bottlenecks = [d for d in dependencies if d[\"incoming_dependencies\"] == 0 and d[\"outgoing_dependencies\"] > 2]\n",
|
||||
"if bottlenecks:\n",
|
||||
" optimized_flow.append({\n",
|
||||
" \"type\": \"bottleneck_detected\",\n",
|
||||
" \"entities\": [b[\"name\"] for b in bottlenecks],\n",
|
||||
" \"suggestion\": \"Consider adding parallel paths for critical nodes\"\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"if inferred_results:\n",
|
||||
" print(f\"Inference engine generated {len(inferred_results)} optimization inferences\")\n",
|
||||
"\n",
|
||||
"print(f\"Generated {len(optimized_flow)} optimization suggestions\")\n",
|
||||
"for suggestion in optimized_flow:\n",
|
||||
" print(f\" {suggestion['type']}: {suggestion['suggestion']}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 5: Predict Disruptions\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"pattern_detector = TemporalPatternDetector()\n",
|
||||
"\n",
|
||||
"temporal_patterns = pattern_detector.detect_temporal_patterns(\n",
|
||||
" supply_chain_kg,\n",
|
||||
" pattern_type=\"anomaly\",\n",
|
||||
" min_frequency=1\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"disruptions = []\n",
|
||||
"\n",
|
||||
"delay_events = [d for d in all_data if d.get(\"event\") == \"shipment_delayed\" or \"delay\" in str(d.get(\"content\", \"\")).lower()]\n",
|
||||
"\n",
|
||||
"if delay_events:\n",
|
||||
" disruptions.append({\n",
|
||||
" \"type\": \"delivery_delay\",\n",
|
||||
" \"severity\": \"high\",\n",
|
||||
" \"affected_entities\": [\"Supplier A\"],\n",
|
||||
" \"description\": \"Shipping delays detected in supply chain\",\n",
|
||||
" \"recommendation\": \"Activate backup suppliers or adjust production schedules\"\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"single_supplier_factories = []\n",
|
||||
"for factory in [e for e in supply_chain_kg.get(\"entities\", []) if e.get(\"type\") == \"Factory\"]:\n",
|
||||
" factory_id = factory.get(\"id\")\n",
|
||||
" suppliers = [r for r in supply_chain_kg.get(\"relationships\", []) if r.get(\"target\") == factory_id and r.get(\"type\") == \"supplies\"]\n",
|
||||
" if len(suppliers) == 1:\n",
|
||||
" single_supplier_factories.append(factory.get(\"name\"))\n",
|
||||
"\n",
|
||||
"if single_supplier_factories:\n",
|
||||
" disruptions.append({\n",
|
||||
" \"type\": \"single_point_of_failure\",\n",
|
||||
" \"severity\": \"medium\",\n",
|
||||
" \"affected_entities\": single_supplier_factories,\n",
|
||||
" \"description\": \"Factories with single supplier dependency detected\",\n",
|
||||
" \"recommendation\": \"Add redundant supplier relationships\"\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"if temporal_patterns:\n",
|
||||
" disruptions.append({\n",
|
||||
" \"type\": \"temporal_anomaly\",\n",
|
||||
" \"severity\": \"medium\",\n",
|
||||
" \"description\": f\"Detected {len(temporal_patterns)} temporal anomalies in supply chain\",\n",
|
||||
" \"recommendation\": \"Review temporal patterns for potential disruptions\"\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"print(f\"Predicted {len(disruptions)} potential disruptions\")\n",
|
||||
"for disruption in disruptions:\n",
|
||||
" print(f\" {disruption['type']} ({disruption['severity']}): {disruption['description']}\")\n",
|
||||
" print(f\" Recommendation: {disruption['recommendation']}\")\n",
|
||||
"\n",
|
||||
"entities_count = len(supply_chain_kg.get(\"entities\", []))\n",
|
||||
"print(f\"\\nAnalyzed {entities_count} supply chain nodes\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Summary\n",
|
||||
"\n",
|
||||
"Complete supply chain intelligence workflow:\n",
|
||||
"- Multi-source data ingested\n",
|
||||
"- Supply chain knowledge graph built\n",
|
||||
"- Dependencies analyzed\n",
|
||||
"- Flow optimization suggested\n",
|
||||
"- Disruptions predicted\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Reference in New Issue
Block a user