mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-15 04:00:33 +00:00
KaifAhmad1andClaude Sonnet 4.6
946a1089c8
- Migrate from mint.json to docs.json (Mintlify v4) - Theme: maple, emerald green + near-black dark / cream light palette (#059669 primary, #0A0A0A dark bg, #FAF7F0 light bg) - Typography: Lexend headings, Inter body - 5-tab navigation: Documentation, Quick Start, API Reference, Cookbook, FAQ - Homepage: removed badge stickers, redundant h2, added blockquote tagline, full 27-module reference table with semantica.mcp_server added - quickstart.md: CodeGroup per pipeline step, pattern vs LLM options, AccordionGroup for patterns and troubleshooting - faq.md: full AccordionGroup structure across 5 sections - reference/explorer.md: NEW — FastAPI explorer, Ontology Hub, Distance Intelligence, CLI reference, REST API endpoints - reference/mcp_server.md: NEW — MCP stdio server, 12 tools with I/O examples, 3 resources, Claude Desktop/VS Code/Windsurf/Cline config - docs.json: explorer added to Output group, mcp_server to Utilities group - Chat, feedback (thumbs/suggest/raise), OG/Twitter metadata, search topbar - All reference pages reformatted with Mintlify JSX components Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
142 lines
3.2 KiB
Markdown
142 lines
3.2 KiB
Markdown
---
|
|
title: "Pipeline Module"
|
|
description: "Pipeline DSL with parallel workers, retry policies, failure handling, and progress tracking."
|
|
icon: "gear"
|
|
---
|
|
|
|
> Robust orchestration engine for building and executing complex data processing workflows.
|
|
|
|
---
|
|
|
|
## Overview
|
|
|
|
The **Pipeline Module** lets you chain Semantica components into reproducible, fault-tolerant workflows with parallel execution and configurable error handling.
|
|
|
|
---
|
|
|
|
## Basic Pipeline
|
|
|
|
```python
|
|
from semantica.pipeline import Pipeline
|
|
from semantica.ingest import FileIngestor
|
|
from semantica.parse import DocumentParser
|
|
from semantica.semantic_extract import NERExtractor
|
|
from semantica.kg import GraphBuilder
|
|
|
|
pipeline = Pipeline()
|
|
pipeline.add_step("ingest", FileIngestor())
|
|
pipeline.add_step("parse", DocumentParser())
|
|
pipeline.add_step("extract", NERExtractor(method="llm", llm_provider=llm))
|
|
pipeline.add_step("build_kg", GraphBuilder(merge_entities=True))
|
|
|
|
result = pipeline.run("data/")
|
|
kg = result.output
|
|
```
|
|
|
|
---
|
|
|
|
## Parallel Processing
|
|
|
|
```python
|
|
pipeline = Pipeline(workers=4) # run steps in parallel across documents
|
|
|
|
pipeline.add_step("ingest", FileIngestor())
|
|
pipeline.add_step("parse", DocumentParser())
|
|
pipeline.add_step("extract", NERExtractor(), parallel=True, batch_size=10)
|
|
pipeline.add_step("build", GraphBuilder())
|
|
|
|
result = pipeline.run("data/")
|
|
```
|
|
|
|
---
|
|
|
|
## Retry & Error Handling
|
|
|
|
```python
|
|
from semantica.pipeline import Pipeline, RetryPolicy, FailureHandler
|
|
|
|
retry = RetryPolicy(
|
|
max_retries=3,
|
|
backoff="exponential", # "fixed" | "linear" | "exponential"
|
|
initial_delay=1.0
|
|
)
|
|
|
|
handler = FailureHandler(
|
|
strategy="skip", # "skip" | "stop" | "retry"
|
|
log_failures=True
|
|
)
|
|
|
|
pipeline = Pipeline(retry_policy=retry, failure_handler=handler)
|
|
```
|
|
|
|
---
|
|
|
|
## Progress Tracking
|
|
|
|
```python
|
|
# Print progress to console
|
|
result = pipeline.run("data/", show_progress=True)
|
|
|
|
# WebSocket progress (via Knowledge Explorer)
|
|
result = pipeline.run("data/", websocket_port=8080)
|
|
|
|
print(f"Processed: {result.processed_count}")
|
|
print(f"Failed: {result.failed_count}")
|
|
print(f"Duration: {result.duration_seconds:.1f}s")
|
|
```
|
|
|
|
---
|
|
|
|
## Pipeline DSL
|
|
|
|
```python
|
|
from semantica.pipeline import PipelineBuilder
|
|
|
|
pipeline = (
|
|
PipelineBuilder()
|
|
.ingest(FileIngestor())
|
|
.parse(DocumentParser())
|
|
.normalize()
|
|
.extract(NERExtractor(method="llm", llm_provider=llm))
|
|
.extract_relations(RelationExtractor(method="llm", llm_provider=llm))
|
|
.build_kg(merge_entities=True)
|
|
.deduplicate(strategy="semantic_v2")
|
|
.export(format="turtle", path="output.ttl")
|
|
.build()
|
|
)
|
|
|
|
result = pipeline.run("data/")
|
|
```
|
|
|
|
---
|
|
|
|
## Saving & Loading Pipelines
|
|
|
|
```python
|
|
# Save pipeline definition
|
|
pipeline.save("pipeline_config.yaml")
|
|
|
|
# Load and run
|
|
pipeline = Pipeline.load("pipeline_config.yaml")
|
|
result = pipeline.run("data/")
|
|
```
|
|
|
|
---
|
|
|
|
## See Also
|
|
|
|
<CardGroup cols={2}>
|
|
<Card title="Ingest" icon="database" href="ingest">
|
|
First step in most pipelines.
|
|
</Card>
|
|
<Card title="Semantic Extract" icon="magnifying-glass" href="semantic_extract">
|
|
Core extraction step.
|
|
</Card>
|
|
<Card title="Knowledge Graph" icon="diagram-project" href="kg">
|
|
Graph construction step.
|
|
</Card>
|
|
<Card title="Export" icon="file-export" href="export">
|
|
Final output step.
|
|
</Card>
|
|
</CardGroup>
|