mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
cookbook(trading): migrate to StructuredDataParser.parse_data and dict access; fix backtesting notebook parsing and iteration; minor doc updates
This commit is contained in:
KaifAhmad1
@@ -224,8 +224,8 @@
|
||||
"\n",
|
||||
"structured_parser = StructuredDataParser()\n",
|
||||
"\n",
|
||||
"parsed_json = structured_parser.parse_json(json_file)\n",
|
||||
"parsed_csv = structured_parser.parse_csv(csv_file)\n",
|
||||
"parsed_json = structured_parser.parse_data(json_file, data_format=\"json\")\n",
|
||||
"parsed_csv = structured_parser.parse_data(csv_file, data_format=\"csv\")\n",
|
||||
"\n",
|
||||
"print(f\"Structured parser parsed JSON: {len(parsed_json.get('data', {}).get('companies', []))} companies\")\n",
|
||||
"print(f\"Structured parser parsed CSV: {len(parsed_csv.get('rows', []))} rows\")\n"
|
||||
|
||||
@@ -67,7 +67,6 @@
|
||||
"from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n",
|
||||
"from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n",
|
||||
"from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
|
||||
"from semantica.conflicts import ConflictDetector\n",
|
||||
"from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n",
|
||||
"from semantica.visualization import KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n",
|
||||
@@ -125,7 +124,7 @@
|
||||
" json.dump(report_data, f, indent=2)\n",
|
||||
"\n",
|
||||
"file_objects = file_ingestor.ingest_file(financial_report_file, read_content=True)\n",
|
||||
"parsed_data = structured_parser.parse_json(financial_report_file)\n",
|
||||
"parsed_data = structured_parser.parse_data(financial_report_file, data_format=\"json\")\n",
|
||||
"\n",
|
||||
"# Ingest from financial feeds\n",
|
||||
"financial_feed_list = []\n",
|
||||
@@ -338,8 +337,8 @@
|
||||
"inference_engine.add_rule(\"IF company has_segment Services AND revenue > 20000000000 THEN services_growth\")\n",
|
||||
"\n",
|
||||
"# Add facts from financial data\n",
|
||||
"if parsed_data and parsed_data.data:\n",
|
||||
" report = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n",
|
||||
"if parsed_data and parsed_data.get(\"data\"):\n",
|
||||
" report = parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), dict) else parsed_data.get(\"data\")[0] if isinstance(parsed_data.get(\"data\"), list) else {}\n",
|
||||
" if isinstance(report, dict):\n",
|
||||
" inference_engine.add_fact({\n",
|
||||
" \"company\": report.get(\"symbol\", \"\"),\n",
|
||||
|
||||
@@ -121,7 +121,7 @@
|
||||
" json.dump(clinical_data, f, indent=2)\n",
|
||||
"\n",
|
||||
"file_objects = file_ingestor.ingest_file(clinical_report_file, read_content=True)\n",
|
||||
"parsed_data = structured_parser.parse_json(clinical_report_file)\n",
|
||||
"parsed_data = structured_parser.parse_data(clinical_report_file, data_format=\"json\")\n",
|
||||
"\n",
|
||||
"# Ingest from FHIR APIs\n",
|
||||
"fhir_content_list = []\n",
|
||||
@@ -177,10 +177,10 @@
|
||||
" if isinstance(parsed_mcp, dict):\n",
|
||||
" if \"patient_records\" in parsed_mcp:\n",
|
||||
" # Merge patient records from MCP\n",
|
||||
" if isinstance(parsed_data.data, list):\n",
|
||||
" parsed_data.data.extend(parsed_mcp.get(\"patient_records\", []))\n",
|
||||
" elif isinstance(parsed_data.data, dict):\n",
|
||||
" parsed_data.data = [parsed_data.data] + parsed_mcp.get(\"patient_records\", [])\n",
|
||||
" if isinstance(parsed_data.get(\"data\"), list):\n",
|
||||
" parsed_data[\"data\"].extend(parsed_mcp.get(\"patient_records\", []))\n",
|
||||
" elif isinstance(parsed_data.get(\"data\"), dict):\n",
|
||||
" parsed_data[\"data\"] = [parsed_data.get(\"data\")] + parsed_mcp.get(\"patient_records\", [])\n",
|
||||
" print(f\" Parsed MCP item\")\n",
|
||||
"\n",
|
||||
"mcp_ingestor.disconnect(\"clinical_mcp_server\")\n",
|
||||
|
||||
@@ -218,16 +218,16 @@
|
||||
"data_normalizer = DataNormalizer()\n",
|
||||
"\n",
|
||||
"# Parse BOM data using Semantica\n",
|
||||
"parsed_bom = structured_parser.parse_json(bom_file)\n",
|
||||
"bom_data_parsed = parsed_bom.data if hasattr(parsed_bom, 'data') else parsed_bom\n",
|
||||
"parsed_bom = structured_parser.parse_data(bom_file, data_format=\"json\")\n",
|
||||
"bom_data_parsed = parsed_bom.get(\"data\") if isinstance(parsed_bom, dict) else parsed_bom\n",
|
||||
"\n",
|
||||
"# Parse supplier data using Semantica\n",
|
||||
"parsed_suppliers = structured_parser.parse_json(supplier_file)\n",
|
||||
"supplier_data_parsed = parsed_suppliers.data if hasattr(parsed_suppliers, 'data') else parsed_suppliers\n",
|
||||
"parsed_suppliers = structured_parser.parse_data(supplier_file, data_format=\"json\")\n",
|
||||
"supplier_data_parsed = parsed_suppliers.get(\"data\") if isinstance(parsed_suppliers, dict) else parsed_suppliers\n",
|
||||
"\n",
|
||||
"# Parse tariff data using Semantica\n",
|
||||
"parsed_tariffs = structured_parser.parse_json(tariff_file)\n",
|
||||
"tariff_data_parsed = parsed_tariffs.data if hasattr(parsed_tariffs, 'data') else parsed_tariffs\n",
|
||||
"parsed_tariffs = structured_parser.parse_data(tariff_file, data_format=\"json\")\n",
|
||||
"tariff_data_parsed = parsed_tariffs.get(\"data\") if isinstance(parsed_tariffs, dict) else parsed_tariffs\n",
|
||||
"\n",
|
||||
"# Normalize supplier names using Semantica\n",
|
||||
"if isinstance(supplier_data_parsed, dict):\n",
|
||||
|
||||
@@ -146,7 +146,7 @@
|
||||
" json.dump(market_stream, f, indent=2)\n",
|
||||
"\n",
|
||||
"file_objects = file_ingestor.ingest_file(market_data_file, read_content=True)\n",
|
||||
"parsed_data = structured_parser.parse_json(market_data_file)\n",
|
||||
"parsed_data = structured_parser.parse_data(market_data_file, data_format=\"json\")\n",
|
||||
"\n",
|
||||
"# Ingest from financial feeds\n",
|
||||
"financial_feed_list = []\n",
|
||||
@@ -187,8 +187,8 @@
|
||||
"market_relationships = []\n",
|
||||
"\n",
|
||||
"# Extract from market data\n",
|
||||
"if parsed_data and parsed_data.data:\n",
|
||||
" for market_entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n",
|
||||
"if parsed_data and parsed_data.get(\"data\"):\n",
|
||||
" for market_entry in parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else [parsed_data.get(\"data\")]:\n",
|
||||
" if isinstance(market_entry, dict):\n",
|
||||
" symbol = market_entry.get(\"symbol\", \"\")\n",
|
||||
" \n",
|
||||
@@ -300,8 +300,8 @@
|
||||
"inference_engine.add_rule(\"IF multiple stocks show same pattern THEN market_trend\")\n",
|
||||
"\n",
|
||||
"# Add facts from market data\n",
|
||||
"if parsed_data and parsed_data.data:\n",
|
||||
" for market_entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n",
|
||||
"if parsed_data and parsed_data.get(\"data\"):\n",
|
||||
" for market_entry in parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else [parsed_data.get(\"data\")]:\n",
|
||||
" if isinstance(market_entry, dict):\n",
|
||||
" inference_engine.add_fact({\n",
|
||||
" \"symbol\": market_entry.get(\"symbol\", \"\"),\n",
|
||||
|
||||
@@ -133,7 +133,7 @@
|
||||
" json.dump(news_data, f, indent=2)\n",
|
||||
"\n",
|
||||
"file_objects = file_ingestor.ingest_file(news_file, read_content=True)\n",
|
||||
"parsed_data = structured_parser.parse_json(news_file)\n",
|
||||
"parsed_data = structured_parser.parse_data(news_file, data_format=\"json\")\n",
|
||||
"\n",
|
||||
"# Ingest from financial feeds\n",
|
||||
"financial_feed_list = []\n",
|
||||
@@ -175,8 +175,8 @@
|
||||
"all_news_texts = []\n",
|
||||
"\n",
|
||||
"# Extract from news data\n",
|
||||
"if parsed_data and parsed_data.data:\n",
|
||||
" articles = parsed_data.data.get(\"articles\", []) if isinstance(parsed_data.data, dict) else parsed_data.data if isinstance(parsed_data.data, list) else []\n",
|
||||
"if parsed_data and parsed_data.get(\"data\"):\n",
|
||||
" articles = parsed_data.get(\"data\").get(\"articles\", []) if isinstance(parsed_data.get(\"data\"), dict) else parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else []\n",
|
||||
" \n",
|
||||
" for article in articles:\n",
|
||||
" if isinstance(article, dict):\n",
|
||||
@@ -250,7 +250,7 @@
|
||||
"\n",
|
||||
"# Analyze sentiment from embeddings and article properties\n",
|
||||
"sentiment_scores = []\n",
|
||||
"for i, article in enumerate(parsed_data.data.get(\"articles\", []) if parsed_data and parsed_data.data and isinstance(parsed_data.data, dict) else []):\n",
|
||||
"for i, article in enumerate(parsed_data.get(\"data\").get(\"articles\", []) if parsed_data and parsed_data.get(\"data\") and isinstance(parsed_data.get(\"data\"), dict) else []):\n",
|
||||
" if isinstance(article, dict):\n",
|
||||
" sentiment = article.get(\"sentiment\", \"neutral\")\n",
|
||||
" sentiment_value = 1.0 if sentiment == \"positive\" else -1.0 if sentiment == \"negative\" else 0.0\n",
|
||||
|
||||
@@ -145,7 +145,7 @@
|
||||
" json.dump(trading_stream, f, indent=2)\n",
|
||||
"\n",
|
||||
"file_objects = file_ingestor.ingest_file(trading_stream_file, read_content=True)\n",
|
||||
"parsed_data = structured_parser.parse_json(trading_stream_file)\n",
|
||||
"parsed_data = structured_parser.parse_data(trading_stream_file, data_format=\"json\")\n",
|
||||
"\n",
|
||||
"print(f\"\\n📊 Ingestion Summary:\")\n",
|
||||
"print(f\" Trading stream files: {len([file_objects]) if file_objects else 0}\")\n",
|
||||
@@ -177,8 +177,8 @@
|
||||
"trading_relationships = []\n",
|
||||
"\n",
|
||||
"# Extract from trading stream data\n",
|
||||
"if parsed_data and parsed_data.data:\n",
|
||||
" for position in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n",
|
||||
"if parsed_data and parsed_data.get(\"data\"):\n",
|
||||
" for position in parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else [parsed_data.get(\"data\")]:\n",
|
||||
" if isinstance(position, dict):\n",
|
||||
" position_id = position.get(\"position_id\", \"\")\n",
|
||||
" symbol = position.get(\"symbol\", \"\")\n",
|
||||
@@ -258,8 +258,8 @@
|
||||
"\n",
|
||||
"# Monitor position changes\n",
|
||||
"position_changes = []\n",
|
||||
"if parsed_data and parsed_data.data:\n",
|
||||
" for position in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n",
|
||||
"if parsed_data and parsed_data.get(\"data\"):\n",
|
||||
" for position in parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else [parsed_data.get(\"data\")]:\n",
|
||||
" if isinstance(position, dict):\n",
|
||||
" price_change = position.get(\"current_price\", 0) - position.get(\"entry_price\", 0)\n",
|
||||
" price_change_percent = (price_change / position.get(\"entry_price\", 1)) * 100 if position.get(\"entry_price\", 0) > 0 else 0\n",
|
||||
|
||||
@@ -140,7 +140,7 @@
|
||||
" json.dump(portfolio_risk_data, f, indent=2)\n",
|
||||
"\n",
|
||||
"file_objects = file_ingestor.ingest_file(risk_data_file, read_content=True)\n",
|
||||
"parsed_data = structured_parser.parse_json(risk_data_file)\n",
|
||||
"parsed_data = structured_parser.parse_data(risk_data_file, data_format=\"json\")\n",
|
||||
"\n",
|
||||
"print(f\"\\n📊 Ingestion Summary:\")\n",
|
||||
"print(f\" Risk data files: {len([file_objects]) if file_objects else 0}\")\n",
|
||||
@@ -171,8 +171,8 @@
|
||||
"risk_relationships = []\n",
|
||||
"\n",
|
||||
"# Extract from portfolio risk data\n",
|
||||
"if parsed_data and parsed_data.data:\n",
|
||||
" portfolio = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n",
|
||||
"if parsed_data and parsed_data.get(\"data\"):\n",
|
||||
" portfolio = parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), dict) else parsed_data.get(\"data\")[0] if isinstance(parsed_data.get(\"data\"), list) else {}\n",
|
||||
" \n",
|
||||
" if isinstance(portfolio, dict):\n",
|
||||
" portfolio_id = portfolio.get(\"portfolio_id\", \"\")\n",
|
||||
@@ -271,8 +271,8 @@
|
||||
"\n",
|
||||
"# Analyze risk concentration\n",
|
||||
"risk_concentration = {}\n",
|
||||
"if parsed_data and parsed_data.data:\n",
|
||||
" portfolio = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n",
|
||||
"if parsed_data and parsed_data.get(\"data\"):\n",
|
||||
" portfolio = parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), dict) else parsed_data.get(\"data\")[0] if isinstance(parsed_data.get(\"data\"), list) else {}\n",
|
||||
" if isinstance(portfolio, dict):\n",
|
||||
" total_value = portfolio.get(\"total_value\", 1)\n",
|
||||
" for position in portfolio.get(\"positions\", []):\n",
|
||||
@@ -318,8 +318,8 @@
|
||||
"\n",
|
||||
"# Assess portfolio risk\n",
|
||||
"portfolio_risk_assessment = {}\n",
|
||||
"if parsed_data and parsed_data.data:\n",
|
||||
" portfolio = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n",
|
||||
"if parsed_data and parsed_data.get(\"data\"):\n",
|
||||
" portfolio = parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), dict) else parsed_data.get(\"data\")[0] if isinstance(parsed_data.get(\"data\"), list) else {}\n",
|
||||
" if isinstance(portfolio, dict):\n",
|
||||
" portfolio_risk_score = portfolio.get(\"portfolio_risk_score\", 0)\n",
|
||||
" \n",
|
||||
|
||||
@@ -67,7 +67,6 @@
|
||||
"from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n",
|
||||
"from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n",
|
||||
"from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
|
||||
"from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n",
|
||||
"from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n",
|
||||
"import tempfile\n",
|
||||
@@ -113,11 +112,11 @@
|
||||
" {\"symbol\": \"MSFT\", \"date\": \"2023-01-16\", \"open\": 351.25, \"high\": 353.50, \"low\": 350.75, \"close\": 352.50, \"volume\": 31000000}\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"with open(historical_data_file, 'w') as f:\n",
|
||||
"with open(historical_data_file, 'w') as f,\n",
|
||||
" json.dump(historical_data, f, indent=2)\n",
|
||||
"\n",
|
||||
"file_objects = file_ingestor.ingest_file(historical_data_file, read_content=True)\n",
|
||||
"parsed_data = structured_parser.parse_json(historical_data_file)\n",
|
||||
"parsed_data = structured_parser.parse_data(historical_data_file, data_format=\"json\")\n",
|
||||
"\n",
|
||||
"# Ingest from historical market APIs\n",
|
||||
"historical_api_list = []\n",
|
||||
@@ -166,8 +165,8 @@
|
||||
"historical_relationships = []\n",
|
||||
"\n",
|
||||
"# Extract from historical data\n",
|
||||
"if parsed_data and parsed_data.data:\n",
|
||||
" for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n",
|
||||
"if parsed_data and parsed_data.get(\"data\"):\n",
|
||||
" for entry in parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else [parsed_data.get(\"data\")]:\n",
|
||||
" if isinstance(entry, dict):\n",
|
||||
" symbol = entry.get(\"symbol\", \"\")\n",
|
||||
" date = entry.get(\"date\", \"\")\n",
|
||||
@@ -251,8 +250,8 @@
|
||||
" trades = []\n",
|
||||
" positions = {}\n",
|
||||
" \n",
|
||||
" if parsed_data and parsed_data.data:\n",
|
||||
" sorted_data = sorted(parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data], \n",
|
||||
" if parsed_data and parsed_data.get(\"data\"):\n",
|
||||
" sorted_data = sorted(parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else [parsed_data.get(\"data\")], \n",
|
||||
" key=lambda x: x.get(\"date\", \"\"))\n",
|
||||
" \n",
|
||||
" for entry in sorted_data:\n",
|
||||
|
||||
+127
-45
@@ -81,11 +81,12 @@ Unified interface for document formats.
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description | Supported Formats |
|
||||
|--------|-------------|-------------------|
|
||||
| `parse_document(path)` | Auto-detect and parse | PDF, DOCX, PPTX, TXT |
|
||||
| `parse_pdf(path)` | PDF specific parsing | PDF |
|
||||
| `parse_docx(path)` | Word specific parsing | DOCX |
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `parse_document(path)` | Auto-detect format and parse |
|
||||
| `extract_text(path)` | Extract text from PDF/DOCX/HTML/TXT |
|
||||
| `extract_metadata(path)` | Extract document metadata |
|
||||
| `parse_batch(paths)` | Parse multiple documents |
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -94,8 +95,8 @@ from semantica.parse import DocumentParser
|
||||
|
||||
parser = DocumentParser()
|
||||
doc = parser.parse_document("report.pdf")
|
||||
print(f"Title: {doc.metadata.title}")
|
||||
print(f"Text: {doc.text[:100]}...")
|
||||
print(doc.get("metadata", {}).get("title"))
|
||||
print(doc.get("full_text", "")[:100])
|
||||
```
|
||||
|
||||
### WebParser
|
||||
@@ -106,8 +107,10 @@ Parses web content.
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `parse_html(url)` | Static HTML parsing |
|
||||
| `parse_dynamic(url)` | JS-rendered parsing |
|
||||
| `parse_web_content(content, content_type)` | Parse HTML/XML |
|
||||
| `extract_text(content)` | Clean text from HTML |
|
||||
| `extract_links(content)` | Extract hyperlinks |
|
||||
| `render_javascript(url)` | Render JS for dynamic pages |
|
||||
|
||||
### StructuredDataParser
|
||||
|
||||
@@ -117,8 +120,17 @@ Parses data files.
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `parse_json(path)` | JSON with nesting |
|
||||
| `parse_csv(path)` | CSV with type inference |
|
||||
| `parse_data(path, data_format)` | Parse JSON/CSV/XML/YAML |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.parse import StructuredDataParser
|
||||
|
||||
parser = StructuredDataParser()
|
||||
data = parser.parse_data("data.json", data_format="json")
|
||||
print(type(data.get("data"))).__name__
|
||||
```
|
||||
|
||||
### CodeParser
|
||||
|
||||
@@ -128,22 +140,111 @@ Parses source code.
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `parse_code(path)` | Extract AST & symbols |
|
||||
| `get_dependencies(path)` | Find imports |
|
||||
| `parse_code(path)` | Parse code file; returns structure, comments, dependencies |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.parse import CodeParser
|
||||
|
||||
parser = CodeParser()
|
||||
data = parser.parse_code("script.py", language="python")
|
||||
print(data.get("structure", {}).get("functions", []))
|
||||
print(data.get("dependencies", {}))
|
||||
```
|
||||
|
||||
### EmailParser
|
||||
|
||||
Parses email messages.
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `parse_email(path)` | Parse full email (headers/body/attachments) |
|
||||
| `parse_headers(path)` | Extract headers only |
|
||||
| `extract_body(path)` | Extract text/HTML body |
|
||||
| `analyze_thread(path)` | Thread reconstruction |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.parse import EmailParser
|
||||
|
||||
parser = EmailParser()
|
||||
email = parser.parse_email("email.eml", extract_attachments=True)
|
||||
print(email.headers.subject)
|
||||
print(email.body.text[:120])
|
||||
```
|
||||
|
||||
### MediaParser
|
||||
|
||||
Parses media files.
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `parse_media(path, media_type)` | Parse image/audio/video |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.parse import MediaParser
|
||||
|
||||
parser = MediaParser()
|
||||
image = parser.parse_media("image.jpg", media_type="image")
|
||||
print(image.get("metadata", {}))
|
||||
```
|
||||
|
||||
### Format-Specific Parsers
|
||||
|
||||
- `PDFParser`, `DOCXParser`, `PPTXParser`, `ExcelParser`
|
||||
- `HTMLParser`, `XMLParser`
|
||||
- `JSONParser`, `CSVParser`
|
||||
- `ImageParser`
|
||||
|
||||
**Examples:**
|
||||
|
||||
```python
|
||||
from semantica.parse import DocumentParser, WebParser, StructuredDataParser
|
||||
|
||||
# Document
|
||||
doc = DocumentParser().parse_document("document.pdf")
|
||||
print(doc.get("full_text", "")[:120])
|
||||
|
||||
# Web
|
||||
web = WebParser().parse_web_content("https://example.com", content_type="html")
|
||||
print(web.get("text", "")[:120])
|
||||
|
||||
# Structured Data (JSON)
|
||||
data = StructuredDataParser().parse_data("data.json", data_format="json")
|
||||
print(list(data.get("data", {}).keys()))
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Convenience Functions
|
||||
## Usage Examples
|
||||
|
||||
### WebParser
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_document, parse_json, parse_web_content
|
||||
from semantica.parse import WebParser
|
||||
|
||||
# Auto-detect format
|
||||
doc = parse_document("file.pdf")
|
||||
parser = WebParser()
|
||||
html = parser.parse_web_content("https://example.com", content_type="html")
|
||||
links = parser.extract_links("https://example.com")
|
||||
```
|
||||
|
||||
# Parse specific types
|
||||
data = parse_json("data.json")
|
||||
web = parse_web_content("https://google.com")
|
||||
### StructuredDataParser
|
||||
|
||||
```python
|
||||
from semantica.parse import StructuredDataParser
|
||||
|
||||
parser = StructuredDataParser()
|
||||
json = parser.parse_data("data.json", data_format="json")
|
||||
csv = parser.parse_data("data.csv", data_format="csv")
|
||||
xml = parser.parse_data("data.xml", data_format="xml")
|
||||
```
|
||||
|
||||
---
|
||||
@@ -177,37 +278,18 @@ parse:
|
||||
|
||||
---
|
||||
|
||||
## Integration Examples
|
||||
## Integration
|
||||
|
||||
### Ingest & Parse Pipeline
|
||||
|
||||
```python
|
||||
from semantica.ingest import Ingestor
|
||||
from semantica.parse import DocumentParser, ImageParser
|
||||
|
||||
# 1. Ingest Raw File
|
||||
ingestor = Ingestor()
|
||||
file_path = ingestor.ingest("scan.png")
|
||||
|
||||
# 2. Parse (with OCR)
|
||||
if file_path.endswith(".png"):
|
||||
parser = ImageParser(ocr_enabled=True)
|
||||
content = parser.parse_image(file_path)
|
||||
else:
|
||||
parser = DocumentParser()
|
||||
content = parser.parse_document(file_path)
|
||||
|
||||
print(content.text)
|
||||
```
|
||||
Use parser classes directly in pipelines and services. Avoid convenience functions for stronger type clarity and consistency.
|
||||
|
||||
---
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Disable OCR if not needed**: OCR is slow. Only enable it (`ocr_enabled=True`) if you expect scanned documents.
|
||||
2. **Use Specific Parsers**: If you know the format, use `parse_json` or `parse_pdf` directly for better type hinting.
|
||||
3. **Handle Encodings**: The parser tries to auto-detect encoding, but for CSV/TXT, explicitly specifying it is safer.
|
||||
4. **Clean Web Content**: Use `parse_web_content` which includes boilerplate removal, rather than raw HTML parsing.
|
||||
1. Disable OCR if not needed; enable only for scanned documents.
|
||||
2. Use specific parser classes like `JSONParser` or `PDFParser` when format is known.
|
||||
3. Handle encodings explicitly for CSV/TXT where auto-detect may fail.
|
||||
4. Clean web content using `WebParser` utilities rather than raw HTML parsing.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+98
-227
@@ -19,25 +19,6 @@ This comprehensive guide demonstrates how to use the data parsing module for doc
|
||||
|
||||
## Basic Usage
|
||||
|
||||
### Using the Convenience Functions
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_document, parse_web_content, parse_json
|
||||
|
||||
# Parse a PDF document
|
||||
doc = parse_document("document.pdf", method="default")
|
||||
print(f"Text: {doc.get('full_text', '')}")
|
||||
print(f"Pages: {doc.get('total_pages', 0)}")
|
||||
|
||||
# Parse web content
|
||||
web = parse_web_content("https://example.com", method="default")
|
||||
print(f"Content: {web.get('text', '')}")
|
||||
|
||||
# Parse JSON data
|
||||
data = parse_json("data.json", method="default")
|
||||
print(f"Data: {data.data}")
|
||||
```
|
||||
|
||||
### Using Main Classes
|
||||
|
||||
```python
|
||||
@@ -66,15 +47,9 @@ print(f"Data: {data}")
|
||||
### PDF Parsing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_pdf, PDFParser
|
||||
from semantica.parse import PDFParser
|
||||
|
||||
# Using convenience function
|
||||
pdf = parse_pdf("document.pdf", method="default", extract_tables=True)
|
||||
print(f"Text: {pdf.get('full_text', '')}")
|
||||
print(f"Pages: {pdf.get('total_pages', 0)}")
|
||||
print(f"Tables: {pdf.get('pages', [{}])[0].get('tables', [])}")
|
||||
|
||||
# Using PDFParser directly
|
||||
# Using PDFParser
|
||||
pdf_parser = PDFParser()
|
||||
pdf_data = pdf_parser.parse("document.pdf", extract_text=True, extract_tables=True)
|
||||
|
||||
@@ -91,14 +66,9 @@ tables = pdf_parser.extract_tables("document.pdf")
|
||||
### DOCX Parsing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_docx, DOCXParser
|
||||
from semantica.parse import DOCXParser
|
||||
|
||||
# Using convenience function
|
||||
docx = parse_docx("document.docx", method="default")
|
||||
print(f"Text: {docx.get('text', '')}")
|
||||
print(f"Sections: {docx.get('sections', [])}")
|
||||
|
||||
# Using DOCXParser directly
|
||||
# Using DOCXParser
|
||||
docx_parser = DOCXParser()
|
||||
docx_data = docx_parser.parse("document.docx", extract_tables=True)
|
||||
|
||||
@@ -148,13 +118,9 @@ for sheet_name, sheet in excel_data.sheets.items():
|
||||
### HTML Document Parsing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_document, HTMLParser
|
||||
from semantica.parse import HTMLParser
|
||||
|
||||
# Using convenience function
|
||||
html_doc = parse_document("page.html", method="default")
|
||||
print(f"Text: {html_doc.get('text', '')}")
|
||||
|
||||
# Using HTMLParser directly
|
||||
# Using HTMLParser
|
||||
html_parser = HTMLParser()
|
||||
html_data = html_parser.parse("page.html", extract_links=True, extract_images=True)
|
||||
|
||||
@@ -172,10 +138,11 @@ for link in links:
|
||||
### Text File Parsing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_document
|
||||
from semantica.parse import DocumentParser
|
||||
|
||||
# Parse plain text file
|
||||
text_doc = parse_document("document.txt", method="default")
|
||||
doc_parser = DocumentParser()
|
||||
text_doc = doc_parser.parse_document("document.txt")
|
||||
print(f"Text: {text_doc.get('text', '')}")
|
||||
```
|
||||
|
||||
@@ -184,13 +151,9 @@ print(f"Text: {text_doc.get('text', '')}")
|
||||
### HTML Content Parsing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_web_content, WebParser
|
||||
from semantica.parse import WebParser
|
||||
|
||||
# Using convenience function
|
||||
web = parse_web_content("https://example.com", content_type="html", method="default")
|
||||
print(f"Text: {web.get('text', '')}")
|
||||
|
||||
# Using WebParser directly
|
||||
# Using WebParser
|
||||
web_parser = WebParser()
|
||||
html_data = web_parser.parse_web_content("https://example.com", content_type="html")
|
||||
|
||||
@@ -209,12 +172,9 @@ rendered = web_parser.render_javascript("https://example.com", wait_time=5)
|
||||
### XML Content Parsing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_web_content, XMLParser
|
||||
from semantica.parse import XMLParser
|
||||
|
||||
# Using convenience function
|
||||
xml_data = parse_web_content("data.xml", content_type="xml", method="default")
|
||||
|
||||
# Using XMLParser directly
|
||||
# Using XMLParser
|
||||
xml_parser = XMLParser()
|
||||
xml_data = xml_parser.parse("data.xml")
|
||||
|
||||
@@ -248,14 +208,9 @@ rendered_content = web_parser.parse_web_content(
|
||||
### JSON Parsing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_json, JSONParser
|
||||
from semantica.parse import JSONParser
|
||||
|
||||
# Using convenience function
|
||||
json_data = parse_json("data.json", method="default")
|
||||
print(f"Data: {json_data.data}")
|
||||
print(f"Type: {json_data.type}")
|
||||
|
||||
# Using JSONParser directly
|
||||
# Using JSONParser
|
||||
json_parser = JSONParser()
|
||||
json_data = json_parser.parse("data.json", flatten=True)
|
||||
|
||||
@@ -268,14 +223,9 @@ for path in paths:
|
||||
### CSV Parsing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_csv, CSVParser
|
||||
from semantica.parse import CSVParser
|
||||
|
||||
# Using convenience function
|
||||
csv_data = parse_csv("data.csv", delimiter=",", method="default")
|
||||
print(f"Headers: {csv_data.headers}")
|
||||
print(f"Rows: {csv_data.row_count}")
|
||||
|
||||
# Using CSVParser directly
|
||||
# Using CSVParser
|
||||
csv_parser = CSVParser()
|
||||
csv_data = csv_parser.parse("data.csv", delimiter=",", has_header=True)
|
||||
|
||||
@@ -288,13 +238,9 @@ for row in rows:
|
||||
### XML Parsing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_xml, XMLParser
|
||||
from semantica.parse import XMLParser
|
||||
|
||||
# Using convenience function
|
||||
xml_data = parse_xml("data.xml", method="default")
|
||||
print(f"Root: {xml_data.root.tag}")
|
||||
|
||||
# Using XMLParser directly
|
||||
# Using XMLParser
|
||||
xml_parser = XMLParser()
|
||||
xml_data = xml_parser.parse("data.xml", engine="lxml")
|
||||
|
||||
@@ -319,26 +265,19 @@ print(f"Data: {yaml_data}")
|
||||
### Basic Email Parsing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_email, EmailParser
|
||||
from semantica.parse import EmailParser
|
||||
|
||||
# Using convenience function
|
||||
email = parse_email("email.eml", method="default")
|
||||
print(f"Subject: {email.headers.subject}")
|
||||
print(f"From: {email.headers.from_address}")
|
||||
print(f"To: {email.headers.to_addresses}")
|
||||
print(f"Body: {email.body.text}")
|
||||
|
||||
# Using EmailParser directly
|
||||
# Using EmailParser
|
||||
email_parser = EmailParser()
|
||||
email_data = email_parser.parse_email("email.eml", extract_attachments=True)
|
||||
|
||||
# Extract headers only
|
||||
headers = email_parser.parse_headers("email.eml")
|
||||
# Access headers from parsed email
|
||||
headers = email_data.headers
|
||||
print(f"Subject: {headers.subject}")
|
||||
print(f"Date: {headers.date}")
|
||||
|
||||
# Extract body only
|
||||
body = email_parser.extract_body("email.eml")
|
||||
# Access body from parsed email
|
||||
body = email_data.body
|
||||
print(f"Text: {body.text}")
|
||||
print(f"HTML: {body.html}")
|
||||
```
|
||||
@@ -351,15 +290,12 @@ from semantica.parse import EmailParser
|
||||
email_parser = EmailParser()
|
||||
|
||||
# Analyze email thread
|
||||
thread = email_parser.analyze_thread("email.eml")
|
||||
print(f"Thread ID: {thread.thread_id}")
|
||||
print(f"Messages: {len(thread.messages)}")
|
||||
print(f"Subject: {thread.subject}")
|
||||
|
||||
# Parse multiple emails in a thread
|
||||
emails = []
|
||||
for email_file in ["email1.eml", "email2.eml", "email3.eml"]:
|
||||
email = email_parser.parse_email(email_file)
|
||||
print(f"Email: {email.headers.subject}")
|
||||
emails.append(email_parser.parse_email(email_file))
|
||||
|
||||
thread = email_parser.analyze_thread(emails)
|
||||
print(f"Messages: {len(thread.get('messages', []))}")
|
||||
```
|
||||
|
||||
### Email Attachment Extraction
|
||||
@@ -382,30 +318,23 @@ for attachment in email_data.body.attachments:
|
||||
### Basic Code Parsing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_code, CodeParser
|
||||
from semantica.parse import CodeParser
|
||||
|
||||
# Using convenience function
|
||||
code = parse_code("script.py", method="default")
|
||||
print(f"Functions: {code.get('structure', {}).get('functions', [])}")
|
||||
print(f"Classes: {code.get('structure', {}).get('classes', [])}")
|
||||
print(f"Imports: {code.get('structure', {}).get('imports', [])}")
|
||||
|
||||
# Using CodeParser directly
|
||||
# Using CodeParser
|
||||
code_parser = CodeParser()
|
||||
code_data = code_parser.parse_code("script.py", language="python")
|
||||
|
||||
# Extract structure
|
||||
structure = code_parser.extract_structure("script.py", language="python")
|
||||
print(f"Functions: {structure.functions}")
|
||||
print(f"Classes: {structure.classes}")
|
||||
# Access structure, comments, dependencies
|
||||
structure = code_data.get("structure", {})
|
||||
print(f"Functions: {structure.get('functions', [])}")
|
||||
print(f"Classes: {structure.get('classes', [])}")
|
||||
print(f"Imports: {structure.get('imports', [])}")
|
||||
|
||||
# Extract comments
|
||||
comments = code_parser.extract_comments("script.py", language="python")
|
||||
comments = code_data.get("comments", [])
|
||||
for comment in comments:
|
||||
print(f"Comment: {comment.text} (Line {comment.line_number})")
|
||||
print(f"Comment: {comment.get('text', '')} (Line {comment.get('line_number', 0)})")
|
||||
|
||||
# Analyze dependencies
|
||||
dependencies = code_parser.analyze_dependencies("script.py", language="python")
|
||||
dependencies = code_data.get("dependencies", {})
|
||||
print(f"Dependencies: {dependencies}")
|
||||
```
|
||||
|
||||
@@ -417,23 +346,18 @@ from semantica.parse import CodeParser, SyntaxTreeParser
|
||||
code_parser = CodeParser()
|
||||
syntax_parser = SyntaxTreeParser()
|
||||
|
||||
# Parse syntax tree
|
||||
tree = syntax_parser.parse_syntax_tree("script.py", language="python")
|
||||
# Parse file content then build syntax tree
|
||||
with open("script.py", "r", encoding="utf-8", errors="ignore") as f:
|
||||
content = f.read()
|
||||
|
||||
# Extract functions
|
||||
functions = syntax_parser.extract_functions("script.py", language="python")
|
||||
for func in functions:
|
||||
print(f"Function: {func.get('name', '')}")
|
||||
tree = syntax_parser.parse_syntax_tree(content, language="python")
|
||||
|
||||
# Extract classes
|
||||
classes = syntax_parser.extract_classes("script.py", language="python")
|
||||
for cls in classes:
|
||||
print(f"Class: {cls.get('name', '')}")
|
||||
|
||||
# Extract imports
|
||||
imports = syntax_parser.extract_imports("script.py", language="python")
|
||||
for imp in imports:
|
||||
print(f"Import: {imp}")
|
||||
# Structure and imports are provided via parse_code
|
||||
code_data = code_parser.parse_code("script.py", language="python")
|
||||
structure = code_data.get("structure", {})
|
||||
print(f"Functions: {structure.get('functions', [])}")
|
||||
print(f"Classes: {structure.get('classes', [])}")
|
||||
print(f"Imports: {structure.get('imports', [])}")
|
||||
```
|
||||
|
||||
### Multi-Language Code Parsing
|
||||
@@ -458,19 +382,9 @@ java_code = code_parser.parse_code("Main.java", language="java")
|
||||
### Image Parsing with OCR
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_image, ImageParser
|
||||
from semantica.parse import ImageParser
|
||||
|
||||
# Using convenience function
|
||||
image = parse_image("image.jpg", method="default", extract_text=True)
|
||||
print(f"Format: {image.get('metadata', {}).get('format', '')}")
|
||||
print(f"Size: {image.get('metadata', {}).get('size', (0, 0))}")
|
||||
|
||||
# Extract OCR text
|
||||
if "ocr_result" in image:
|
||||
print(f"OCR Text: {image['ocr_result'].text}")
|
||||
print(f"Confidence: {image['ocr_result'].confidence}")
|
||||
|
||||
# Using ImageParser directly
|
||||
# Using ImageParser
|
||||
image_parser = ImageParser()
|
||||
image_data = image_parser.parse("image.jpg", extract_text=True, ocr_language="eng")
|
||||
|
||||
@@ -489,20 +403,15 @@ print(f"Confidence: {ocr_result.confidence}")
|
||||
### Media File Parsing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_media, MediaParser
|
||||
from semantica.parse import MediaParser
|
||||
|
||||
# Using convenience function
|
||||
media = parse_media("video.mp4", method="default")
|
||||
print(f"Type: {media.get('media_type', '')}")
|
||||
print(f"Metadata: {media.get('metadata', {})}")
|
||||
|
||||
# Using MediaParser directly
|
||||
# Using MediaParser
|
||||
media_parser = MediaParser()
|
||||
media_data = media_parser.parse_media("image.jpg", media_type="image")
|
||||
|
||||
# Get supported formats
|
||||
formats = media_parser.get_supported_formats()
|
||||
print(f"Supported formats: {formats}")
|
||||
# Parse audio/video similarly by specifying media_type
|
||||
video = media_parser.parse_media("video.mp4", media_type="video")
|
||||
audio = media_parser.parse_media("audio.mp3", media_type="audio")
|
||||
```
|
||||
|
||||
## Format-Specific Parsers
|
||||
@@ -647,42 +556,7 @@ if "ocr_result" in image_data:
|
||||
print(f"Language: {ocr.language}")
|
||||
```
|
||||
|
||||
## Using Methods
|
||||
|
||||
### Method Selection
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_document, parse_web_content, parse_json
|
||||
|
||||
# Use default method
|
||||
doc = parse_document("document.pdf", method="default")
|
||||
|
||||
# Use specific method (if registered)
|
||||
doc = parse_document("document.pdf", method="custom_pdf_parser")
|
||||
|
||||
# List available methods
|
||||
from semantica.parse import list_available_methods
|
||||
|
||||
methods = list_available_methods("document")
|
||||
print(f"Available document methods: {methods}")
|
||||
```
|
||||
|
||||
### Custom Method Registration
|
||||
|
||||
```python
|
||||
from semantica.parse import method_registry, parse_document
|
||||
|
||||
# Define custom parsing method
|
||||
def custom_document_parser(file_path, file_type=None, **kwargs):
|
||||
# Custom parsing logic
|
||||
return {"text": "Custom parsed text", "metadata": {}}
|
||||
|
||||
# Register custom method
|
||||
method_registry.register("document", "custom", custom_document_parser)
|
||||
|
||||
# Use custom method
|
||||
doc = parse_document("document.pdf", method="custom")
|
||||
```
|
||||
|
||||
|
||||
## Using Registry
|
||||
|
||||
@@ -726,15 +600,10 @@ print(f"Web methods: {web_methods}")
|
||||
### Getting Methods
|
||||
|
||||
```python
|
||||
from semantica.parse import method_registry, get_parse_method
|
||||
from semantica.parse import method_registry
|
||||
|
||||
# Get method directly
|
||||
method = method_registry.get("document", "default")
|
||||
if method:
|
||||
result = method("document.pdf")
|
||||
|
||||
# Using convenience function
|
||||
method = get_parse_method("document", "default")
|
||||
if method:
|
||||
result = method("document.pdf")
|
||||
```
|
||||
@@ -838,16 +707,18 @@ print(f"All config: {all_config}")
|
||||
### Batch Document Processing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_document
|
||||
from semantica.parse import DocumentParser
|
||||
from pathlib import Path
|
||||
|
||||
# Process multiple documents
|
||||
documents = ["doc1.pdf", "doc2.docx", "doc3.html"]
|
||||
results = []
|
||||
|
||||
doc_parser = DocumentParser()
|
||||
|
||||
for doc_path in documents:
|
||||
try:
|
||||
result = parse_document(doc_path, method="default")
|
||||
result = doc_parser.parse_document(doc_path)
|
||||
results.append({
|
||||
"file": doc_path,
|
||||
"text": result.get("full_text", ""),
|
||||
@@ -861,35 +732,36 @@ pdf_dir = Path("documents")
|
||||
pdf_files = list(pdf_dir.glob("*.pdf"))
|
||||
|
||||
for pdf_file in pdf_files:
|
||||
result = parse_document(pdf_file, method="default")
|
||||
result = doc_parser.parse_document(pdf_file)
|
||||
print(f"Processed: {pdf_file.name} ({result.get('total_pages', 0)} pages)")
|
||||
```
|
||||
|
||||
### Multi-Format Data Extraction
|
||||
|
||||
```python
|
||||
from semantica.parse import (
|
||||
parse_document, parse_json, parse_csv, parse_xml
|
||||
)
|
||||
from semantica.parse import DocumentParser, JSONParser, CSVParser, XMLParser
|
||||
|
||||
# Extract data from multiple formats
|
||||
files = {
|
||||
"document.pdf": parse_document,
|
||||
"data.json": parse_json,
|
||||
"data.csv": parse_csv,
|
||||
"data.xml": parse_xml,
|
||||
}
|
||||
doc_parser = DocumentParser()
|
||||
json_parser = JSONParser()
|
||||
csv_parser = CSVParser()
|
||||
xml_parser = XMLParser()
|
||||
|
||||
files = [
|
||||
("document.pdf", lambda p: doc_parser.parse_document(p)),
|
||||
("data.json", lambda p: json_parser.parse(p)),
|
||||
("data.csv", lambda p: csv_parser.parse(p)),
|
||||
("data.xml", lambda p: xml_parser.parse(p)),
|
||||
]
|
||||
|
||||
extracted_data = {}
|
||||
|
||||
for file_path, parser_func in files.items():
|
||||
for file_path, parse_fn in files:
|
||||
try:
|
||||
data = parser_func(file_path, method="default")
|
||||
data = parse_fn(file_path)
|
||||
extracted_data[file_path] = data
|
||||
except Exception as e:
|
||||
print(f"Error parsing {file_path}: {e}")
|
||||
|
||||
# Process extracted data
|
||||
for file_path, data in extracted_data.items():
|
||||
print(f"File: {file_path}")
|
||||
if isinstance(data, dict):
|
||||
@@ -966,7 +838,7 @@ print(f"Total imports: {len(set(all_imports))}")
|
||||
### OCR Batch Processing
|
||||
|
||||
```python
|
||||
from semantica.parse import parse_image
|
||||
from semantica.parse import ImageParser
|
||||
from pathlib import Path
|
||||
|
||||
# Process all images in a directory
|
||||
@@ -975,9 +847,11 @@ image_files = list(image_dir.glob("*.jpg")) + list(image_dir.glob("*.png"))
|
||||
|
||||
ocr_results = []
|
||||
|
||||
parser = ImageParser()
|
||||
|
||||
for image_file in image_files:
|
||||
try:
|
||||
image_data = parse_image(image_file, method="default", extract_text=True)
|
||||
image_data = parser.parse(image_file, extract_text=True)
|
||||
|
||||
if "ocr_result" in image_data:
|
||||
ocr_results.append({
|
||||
@@ -998,36 +872,33 @@ for result in ocr_results:
|
||||
### Custom Parser Pipeline
|
||||
|
||||
```python
|
||||
from semantica.parse import (
|
||||
parse_document, parse_json, parse_csv,
|
||||
method_registry
|
||||
)
|
||||
from semantica.parse import DocumentParser, StructuredDataParser, method_registry
|
||||
|
||||
doc_parser = DocumentParser()
|
||||
data_parser = StructuredDataParser()
|
||||
|
||||
# Define custom pipeline
|
||||
def pipeline_parser(file_path, file_type=None, **kwargs):
|
||||
# Step 1: Parse document
|
||||
doc = parse_document(file_path, file_type=file_type, **kwargs)
|
||||
|
||||
# Step 2: Extract structured data if available
|
||||
doc = doc_parser.parse_document(file_path, file_type=file_type, **kwargs)
|
||||
structured_data = {}
|
||||
if "json" in str(file_path):
|
||||
structured_data = parse_json(file_path, **kwargs)
|
||||
elif "csv" in str(file_path):
|
||||
structured_data = parse_csv(file_path, **kwargs)
|
||||
|
||||
# Step 3: Combine results
|
||||
suffix = str(file_path).lower()
|
||||
if suffix.endswith(".json"):
|
||||
structured_data = data_parser.parse_data(file_path, data_format="json")
|
||||
elif suffix.endswith(".csv"):
|
||||
structured_data = data_parser.parse_data(file_path, data_format="csv")
|
||||
elif suffix.endswith(".xml"):
|
||||
structured_data = data_parser.parse_data(file_path, data_format="xml")
|
||||
return {
|
||||
"document": doc,
|
||||
"structured": structured_data,
|
||||
"combined_text": doc.get("full_text", "") + str(structured_data)
|
||||
}
|
||||
|
||||
# Register pipeline
|
||||
method_registry.register("document", "pipeline", pipeline_parser)
|
||||
|
||||
# Use pipeline
|
||||
result = parse_document("document.pdf", method="pipeline")
|
||||
print(f"Combined text: {result['combined_text']}")
|
||||
method = method_registry.get("document", "pipeline")
|
||||
if method:
|
||||
result = method("document.pdf")
|
||||
print(f"Combined text: {result['combined_text']}")
|
||||
```
|
||||
|
||||
This comprehensive guide covers all major features of the data parsing module. For more specific use cases or advanced scenarios, refer to the individual parser class documentation or explore the source code.
|
||||
|
||||
Reference in New Issue
Block a user