mengzhx/semantica
1
0
Fork 0
mirror of https://github.com/semantica-agi/semantica.git synced 2026-09-13 04:04:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: remove 07_Pipeline_Orchestration notebook and all references

Browse Source
This commit is contained in:
KaifAhmad1
2025-12-20 20:18:14 +05:30
parent bd3bc7de7d
commit 1fceb634ae
6 changed files with 614 additions and 1023 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+612 -613
View File
File diff suppressed because it is too large Load Diff
cookbook/advanced/07_Pipeline_Orchestration.ipynb
-242
View File
@@ -1,242 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/07_Pipeline_Orchestration.ipynb)\n",
"\n",
"# Pipeline Orchestration\n",
"\n",
"## Overview\n",
"\n",
"Build complex pipelines, execute them, handle failures, enable parallel processing, and monitor execution.\n",
"\n",
"\n",
"**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/pipeline/)\n",
"\n",
"## Installation\n",
"\n",
"Install Semantica from PyPI:\n",
"\n",
"```bash\n",
"pip install semantica\n",
"# Or with all optional dependencies:\n",
"pip install semantica[all]\n",
"```\n",
"\n",
"## Workflow: Build Pipelines \u2192 Execute \u2192 Handle Failures \u2192 Parallel Processing \u2192 Monitor\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!pip install semantica\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from semantica.pipeline import (\n",
" PipelineBuilder,\n",
" ExecutionEngine,\n",
" FailureHandler,\n",
" ParallelismManager,\n",
" RetryPolicy,\n",
" RetryStrategy\n",
")\n",
"from semantica.ingest import FileIngestor\n",
"from semantica.parse import DocumentParser\n",
"from semantica.semantic_extract import NERExtractor\n",
"from semantica.kg import GraphBuilder\n",
"import time\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 1: Build Complex Pipelines\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"builder = PipelineBuilder()\n",
"\n",
"file_ingestor = FileIngestor()\n",
"document_parser = DocumentParser()\n",
"ner_extractor = NERExtractor()\n",
"graph_builder = GraphBuilder()\n",
"\n",
"# Define handlers for each pipeline step\n",
"def ingest_handler(data, **config):\n",
" files = data.get(\"files\", [])\n",
" if files:\n",
" # Ingest first file as example\n",
" file_obj = file_ingestor.ingest_file(files[0], read_content=True)\n",
" return {**data, \"file\": file_obj}\n",
" return data\n",
"\n",
"def parse_handler(data, **config):\n",
" # If a file was ingested, try parsing; otherwise pass text through\n",
" file_obj = data.get(\"file\")\n",
" if file_obj and getattr(file_obj, \"path\", None):\n",
" parsed = document_parser.parse_document(file_obj.path)\n",
" text = parsed.get(\"text\") if isinstance(parsed, dict) else None\n",
" return {**data, \"text\": text or data.get(\"text\")}\n",
" return data\n",
"\n",
"def extract_handler(data, **config):\n",
" text = data.get(\"text\", \"\")\n",
" entities = ner_extractor.extract_entities(text)\n",
" # Normalize to dict list for graph builder\n",
" entity_dicts = [\n",
" {\"id\": f\"e{i}\", \"name\": e.text, \"type\": e.label} for i, e in enumerate(entities)\n",
" ]\n",
" return {**data, \"entities\": entity_dicts}\n",
"\n",
"def build_graph_handler(data, **config):\n",
" entities = data.get(\"entities\", [])\n",
" graph = graph_builder.build({\"entities\": entities})\n",
" return {**data, \"graph\": graph}\n",
"\n",
"# Build pipeline with proper handlers and dependencies\n",
"pipeline = (\n",
" builder\n",
" .add_step(\"ingest\", \"ingest\", handler=ingest_handler)\n",
" .add_step(\"parse\", \"parse\", dependencies=[\"ingest\"], handler=parse_handler)\n",
" .add_step(\"extract\", \"extract\", dependencies=[\"parse\"], handler=extract_handler)\n",
" .add_step(\"build_graph\", \"build_graph\", dependencies=[\"extract\"], handler=build_graph_handler)\n",
").build()\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2: Execute Pipeline\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"engine = ExecutionEngine()\n",
"\n",
"input_data = {\n",
" \"text\": \"Alice works at Tech Corp. Bob is a friend of Alice.\",\n",
" \"files\": []\n",
"}\n",
"\n",
"start_time = time.time()\n",
"result = engine.execute_pipeline(pipeline, input_data)\n",
"execution_time = result.metrics.get(\"execution_time\", time.time() - start_time)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 3: Handle Failures\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Configure retry policy for the 'extract' step type\n",
"engine.failure_handler.set_retry_policy(\n",
" \"extract\",\n",
" RetryPolicy(max_retries=3, backoff_factor=2.0, strategy=RetryStrategy.EXPONENTIAL)\n",
")\n",
"\n",
"result = engine.execute_pipeline(pipeline, input_data)\n",
"print(\"Pipeline executed with retry policy configured\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 4: Parallel Processing\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"parallelism = ParallelismManager(max_workers=4)\n",
"\n",
"# Identify groups of steps that can run in parallel\n",
"groups = parallelism.identify_parallelizable_steps(pipeline)\n",
"\n",
"# Execute first parallelizable group as a demonstration\n",
"start_time = time.time()\n",
"parallel_results = []\n",
"for group in groups:\n",
" parallel_results.extend(parallelism.execute_pipeline_steps_parallel(group, input_data, max_workers=4))\n",
"parallel_time = time.time() - start_time\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 5: Monitor Pipeline Execution\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Metrics from execution engine\n",
"metrics = result.metrics\n",
"progress = engine.get_progress(pipeline.name)\n",
"\n",
"print(f\"Duration: {metrics.get('execution_time', 0):.2f} seconds\")\n",
"print(f\"Steps Executed: {metrics.get('steps_executed', 0)}\")\n",
"print(f\"Steps Failed: {metrics.get('steps_failed', 0)}\")\n",
"print(f\"Progress: {progress.get('progress_percentage', 0):.1f}% (status: {progress.get('status')})\")\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary\n",
"\n",
"Pipeline orchestration workflow:\n",
"- Complex Pipeline Built\n",
"- Pipeline Executed\n",
"- Failure Handling Configured\n",
"- Parallel Processing Enabled\n",
"- Full Monitoring and Observability\n"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
docs/cookbook.md
-9
View File
@@ -242,15 +242,6 @@ Deep dive into advanced features, customization, and complex workflows.
[Open Notebook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb)
- :material-pipe: **Pipeline Orchestration**
---
Building robust, automated data processing pipelines.
**Topics**: Workflows, Automation, Error Handling
**Difficulty**: Advanced
[Open Notebook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/07_Pipeline_Orchestration.ipynb)
- :material-brain: **Reasoning and Inference**
---
docs/reference/pipeline.md
-3
View File
@@ -381,6 +381,3 @@ result = engine.execute_pipeline(pipeline, data={"path": "document.pdf"})
- [Split Module](split.md) - Common processing step
- [Vector Store Module](vector_store.md) - Common sink step
## Cookbook
- [Pipeline Orchestration](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/07_Pipeline_Orchestration.ipynb)
tests/pipeline/test_notebook_07.py
-154
View File
@@ -1,154 +0,0 @@
import unittest
from unittest.mock import MagicMock, patch
import time
import pytest
from semantica.pipeline import (
PipelineBuilder,
ExecutionEngine,
FailureHandler,
ParallelismManager,
RetryPolicy,
RetryStrategy
)
pytestmark = pytest.mark.integration
class TestNotebook07(unittest.TestCase):
def setUp(self):
# Mock external dependencies used in the notebook
self.mock_file_ingestor = MagicMock()
self.mock_document_parser = MagicMock()
self.mock_ner_extractor = MagicMock()
self.mock_graph_builder = MagicMock()
# Setup return values
self.mock_file_ingestor.ingest_file.return_value = MagicMock(path="test.txt")
self.mock_document_parser.parse_document.return_value = {"text": "Alice works at Tech Corp."}
# Mock NER entities
mock_entity = MagicMock()
mock_entity.text = "Alice"
mock_entity.label = "PERSON"
self.mock_ner_extractor.extract_entities.return_value = [mock_entity]
self.mock_graph_builder.build.return_value = {"nodes": [], "edges": []}
def test_pipeline_orchestration_workflow(self):
"""Replicates the workflow in 07_Pipeline_Orchestration.ipynb"""
builder = PipelineBuilder()
# Define handlers (logic copied from notebook)
def ingest_handler(data, **config):
files = data.get("files", [])
if files:
# Ingest first file as example
file_obj = self.mock_file_ingestor.ingest_file(files[0], read_content=True)
return {**data, "file": file_obj}
return data
def parse_handler(data, **config):
# If a file was ingested, try parsing; otherwise pass text through
file_obj = data.get("file")
# Mock object path check
if file_obj and getattr(file_obj, "path", None):
parsed = self.mock_document_parser.parse_document(file_obj.path)
text = parsed.get("text") if isinstance(parsed, dict) else None
return {**data, "text": text or data.get("text")}
return data
def extract_handler(data, **config):
text = data.get("text", "")
entities = self.mock_ner_extractor.extract_entities(text)
# Normalize to dict list for graph builder
entity_dicts = [
{"id": f"e{i}", "name": e.text, "type": e.label} for i, e in enumerate(entities)
]
return {**data, "entities": entity_dicts}
def build_graph_handler(data, **config):
entities = data.get("entities", [])
graph = self.mock_graph_builder.build({"entities": entities})
return {**data, "graph": graph}
# Build pipeline
pipeline = (
builder
.add_step("ingest", "ingest", handler=ingest_handler)
.add_step("parse", "parse", dependencies=["ingest"], handler=parse_handler)
.add_step("extract", "extract", dependencies=["parse"], handler=extract_handler)
.add_step("build_graph", "build_graph", dependencies=["extract"], handler=build_graph_handler)
).build()
# Step 2: Execute Pipeline
engine = ExecutionEngine()
input_data = {
"text": "Alice works at Tech Corp. Bob is a friend of Alice.",
"files": ["sample.txt"]
}
result = engine.execute_pipeline(pipeline, input_data)
self.assertTrue(result.success)
self.assertIn("graph", result.output)
# Verify mocks called
self.mock_file_ingestor.ingest_file.assert_called()
self.mock_document_parser.parse_document.assert_called()
self.mock_ner_extractor.extract_entities.assert_called()
self.mock_graph_builder.build.assert_called()
# Step 3: Handle Failures
# Configure retry policy
engine.failure_handler.set_retry_policy(
"extract",
RetryPolicy(max_retries=3, backoff_factor=1.0, strategy=RetryStrategy.LINEAR)
)
# Execute again (should still pass)
result_retry = engine.execute_pipeline(pipeline, input_data)
self.assertTrue(result_retry.success)
# Step 4: Parallel Processing
parallelism = ParallelismManager(max_workers=4)
groups = parallelism.identify_parallelizable_steps(pipeline)
# The pipeline is sequential (ingest->parse->extract->build_graph), so groups should be single steps
# [[ingest], [parse], [extract], [build_graph]]
self.assertEqual(len(groups), 4)
# Execute parallel steps (simulated)
parallel_results = []
for group in groups:
# We mock the execution here or just call the manager's method
# Since execute_pipeline_steps_parallel needs Task objects or similar logic,
# and the notebook uses it slightly differently (it seems to assume integration with engine).
# Let's check how the notebook uses it:
# parallel_results.extend(parallelism.execute_pipeline_steps_parallel(group, input_data, max_workers=4))
# The ParallelismManager.execute_pipeline_steps_parallel likely takes PipelineStep objects and data
# We need to ensure input_data flows correctly. In a real pipeline, output of one step is input to next.
# The notebook example simplifies this by passing `input_data` to all, which works if steps are independent or data is static.
# But here steps depend on previous output.
# So we'll just verify the method runs without error.
try:
parallelism.execute_pipeline_steps_parallel(group, input_data, max_workers=2)
except Exception as e:
# It might fail if handlers expect data from previous steps which is not in 'input_data'
# For this test, we accept that or catch it.
# Actually, let's just verify `identify_parallelizable_steps` works as expected.
pass
# Step 5: Monitor
metrics = result.metrics
progress = engine.get_progress(pipeline.name)
self.assertIn("execution_time", metrics)
self.assertEqual(metrics.get("steps_failed", 0), 0)
# Progress might be cleared or 100% depending on implementation
if __name__ == '__main__':
unittest.main()
tests/test_pipeline_orchestration.py
+2 -2
View File
@@ -260,10 +260,10 @@ def test_parallelism_manager_identify_parallelizable_steps(parallelism_manager,
assert "s2" in names
assert "s3" in names
# --- End-to-End Notebook Simulation ---
# --- End-to-End Pipeline Orchestration Test ---
def test_end_to_end_pipeline_orchestration(pipeline_builder, execution_engine):
# This simulates the logic in 07_Pipeline_Orchestration.ipynb
# This simulates a complete pipeline orchestration workflow
# Mocks for actual components to avoid file I/O and heavy processing
file_ingestor_mock = MagicMock()