diff --git a/cookbook/introduction/03_Document_Parsing.ipynb b/cookbook/introduction/03_Document_Parsing.ipynb index fec81962..cfc7486b 100644 --- a/cookbook/introduction/03_Document_Parsing.ipynb +++ b/cookbook/introduction/03_Document_Parsing.ipynb @@ -224,8 +224,8 @@ "\n", "structured_parser = StructuredDataParser()\n", "\n", - "parsed_json = structured_parser.parse_json(json_file)\n", - "parsed_csv = structured_parser.parse_csv(csv_file)\n", + "parsed_json = structured_parser.parse_data(json_file, data_format=\"json\")\n", + "parsed_csv = structured_parser.parse_data(csv_file, data_format=\"csv\")\n", "\n", "print(f\"Structured parser parsed JSON: {len(parsed_json.get('data', {}).get('companies', []))} companies\")\n", "print(f\"Structured parser parsed CSV: {len(parsed_csv.get('rows', []))} rows\")\n" diff --git a/cookbook/use_cases/finance/02_Financial_Reports_Analysis.ipynb b/cookbook/use_cases/finance/02_Financial_Reports_Analysis.ipynb index 62e60391..91fb751c 100644 --- a/cookbook/use_cases/finance/02_Financial_Reports_Analysis.ipynb +++ b/cookbook/use_cases/finance/02_Financial_Reports_Analysis.ipynb @@ -67,7 +67,6 @@ "from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector\n", "from semantica.kg import ConnectivityAnalyzer, TemporalGraphQuery, TemporalPatternDetector\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.conflicts import ConflictDetector\n", "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, AnalyticsVisualizer, TemporalVisualizer\n", @@ -125,7 +124,7 @@ " json.dump(report_data, f, indent=2)\n", "\n", "file_objects = file_ingestor.ingest_file(financial_report_file, read_content=True)\n", - "parsed_data = structured_parser.parse_json(financial_report_file)\n", + "parsed_data = structured_parser.parse_data(financial_report_file, data_format=\"json\")\n", "\n", "# Ingest from financial feeds\n", "financial_feed_list = []\n", @@ -338,8 +337,8 @@ "inference_engine.add_rule(\"IF company has_segment Services AND revenue > 20000000000 THEN services_growth\")\n", "\n", "# Add facts from financial data\n", - "if parsed_data and parsed_data.data:\n", - " report = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + "if parsed_data and parsed_data.get(\"data\"):\n", + " report = parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), dict) else parsed_data.get(\"data\")[0] if isinstance(parsed_data.get(\"data\"), list) else {}\n", " if isinstance(report, dict):\n", " inference_engine.add_fact({\n", " \"company\": report.get(\"symbol\", \"\"),\n", diff --git a/cookbook/use_cases/healthcare/01_Clinical_Reports_Processing.ipynb b/cookbook/use_cases/healthcare/01_Clinical_Reports_Processing.ipynb index 83d84ebf..c726e0a8 100644 --- a/cookbook/use_cases/healthcare/01_Clinical_Reports_Processing.ipynb +++ b/cookbook/use_cases/healthcare/01_Clinical_Reports_Processing.ipynb @@ -121,7 +121,7 @@ " json.dump(clinical_data, f, indent=2)\n", "\n", "file_objects = file_ingestor.ingest_file(clinical_report_file, read_content=True)\n", - "parsed_data = structured_parser.parse_json(clinical_report_file)\n", + "parsed_data = structured_parser.parse_data(clinical_report_file, data_format=\"json\")\n", "\n", "# Ingest from FHIR APIs\n", "fhir_content_list = []\n", @@ -177,10 +177,10 @@ " if isinstance(parsed_mcp, dict):\n", " if \"patient_records\" in parsed_mcp:\n", " # Merge patient records from MCP\n", - " if isinstance(parsed_data.data, list):\n", - " parsed_data.data.extend(parsed_mcp.get(\"patient_records\", []))\n", - " elif isinstance(parsed_data.data, dict):\n", - " parsed_data.data = [parsed_data.data] + parsed_mcp.get(\"patient_records\", [])\n", + " if isinstance(parsed_data.get(\"data\"), list):\n", + " parsed_data[\"data\"].extend(parsed_mcp.get(\"patient_records\", []))\n", + " elif isinstance(parsed_data.get(\"data\"), dict):\n", + " parsed_data[\"data\"] = [parsed_data.get(\"data\")] + parsed_mcp.get(\"patient_records\", [])\n", " print(f\" Parsed MCP item\")\n", "\n", "mcp_ingestor.disconnect(\"clinical_mcp_server\")\n", diff --git a/cookbook/use_cases/supply_chain/02_Supply_Chain_Risk_Management.ipynb b/cookbook/use_cases/supply_chain/02_Supply_Chain_Risk_Management.ipynb index 74212c92..73f869af 100644 --- a/cookbook/use_cases/supply_chain/02_Supply_Chain_Risk_Management.ipynb +++ b/cookbook/use_cases/supply_chain/02_Supply_Chain_Risk_Management.ipynb @@ -218,16 +218,16 @@ "data_normalizer = DataNormalizer()\n", "\n", "# Parse BOM data using Semantica\n", - "parsed_bom = structured_parser.parse_json(bom_file)\n", - "bom_data_parsed = parsed_bom.data if hasattr(parsed_bom, 'data') else parsed_bom\n", + "parsed_bom = structured_parser.parse_data(bom_file, data_format=\"json\")\n", + "bom_data_parsed = parsed_bom.get(\"data\") if isinstance(parsed_bom, dict) else parsed_bom\n", "\n", "# Parse supplier data using Semantica\n", - "parsed_suppliers = structured_parser.parse_json(supplier_file)\n", - "supplier_data_parsed = parsed_suppliers.data if hasattr(parsed_suppliers, 'data') else parsed_suppliers\n", + "parsed_suppliers = structured_parser.parse_data(supplier_file, data_format=\"json\")\n", + "supplier_data_parsed = parsed_suppliers.get(\"data\") if isinstance(parsed_suppliers, dict) else parsed_suppliers\n", "\n", "# Parse tariff data using Semantica\n", - "parsed_tariffs = structured_parser.parse_json(tariff_file)\n", - "tariff_data_parsed = parsed_tariffs.data if hasattr(parsed_tariffs, 'data') else parsed_tariffs\n", + "parsed_tariffs = structured_parser.parse_data(tariff_file, data_format=\"json\")\n", + "tariff_data_parsed = parsed_tariffs.get(\"data\") if isinstance(parsed_tariffs, dict) else parsed_tariffs\n", "\n", "# Normalize supplier names using Semantica\n", "if isinstance(supplier_data_parsed, dict):\n", diff --git a/cookbook/use_cases/trading/01_Market_Data_Analysis.ipynb b/cookbook/use_cases/trading/01_Market_Data_Analysis.ipynb index 477ee767..32a2e3c7 100644 --- a/cookbook/use_cases/trading/01_Market_Data_Analysis.ipynb +++ b/cookbook/use_cases/trading/01_Market_Data_Analysis.ipynb @@ -146,7 +146,7 @@ " json.dump(market_stream, f, indent=2)\n", "\n", "file_objects = file_ingestor.ingest_file(market_data_file, read_content=True)\n", - "parsed_data = structured_parser.parse_json(market_data_file)\n", + "parsed_data = structured_parser.parse_data(market_data_file, data_format=\"json\")\n", "\n", "# Ingest from financial feeds\n", "financial_feed_list = []\n", @@ -187,8 +187,8 @@ "market_relationships = []\n", "\n", "# Extract from market data\n", - "if parsed_data and parsed_data.data:\n", - " for market_entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + "if parsed_data and parsed_data.get(\"data\"):\n", + " for market_entry in parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else [parsed_data.get(\"data\")]:\n", " if isinstance(market_entry, dict):\n", " symbol = market_entry.get(\"symbol\", \"\")\n", " \n", @@ -300,8 +300,8 @@ "inference_engine.add_rule(\"IF multiple stocks show same pattern THEN market_trend\")\n", "\n", "# Add facts from market data\n", - "if parsed_data and parsed_data.data:\n", - " for market_entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + "if parsed_data and parsed_data.get(\"data\"):\n", + " for market_entry in parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else [parsed_data.get(\"data\")]:\n", " if isinstance(market_entry, dict):\n", " inference_engine.add_fact({\n", " \"symbol\": market_entry.get(\"symbol\", \"\"),\n", diff --git a/cookbook/use_cases/trading/02_News_Sentiment_Analysis.ipynb b/cookbook/use_cases/trading/02_News_Sentiment_Analysis.ipynb index c78be5d9..02889cdb 100644 --- a/cookbook/use_cases/trading/02_News_Sentiment_Analysis.ipynb +++ b/cookbook/use_cases/trading/02_News_Sentiment_Analysis.ipynb @@ -133,7 +133,7 @@ " json.dump(news_data, f, indent=2)\n", "\n", "file_objects = file_ingestor.ingest_file(news_file, read_content=True)\n", - "parsed_data = structured_parser.parse_json(news_file)\n", + "parsed_data = structured_parser.parse_data(news_file, data_format=\"json\")\n", "\n", "# Ingest from financial feeds\n", "financial_feed_list = []\n", @@ -175,8 +175,8 @@ "all_news_texts = []\n", "\n", "# Extract from news data\n", - "if parsed_data and parsed_data.data:\n", - " articles = parsed_data.data.get(\"articles\", []) if isinstance(parsed_data.data, dict) else parsed_data.data if isinstance(parsed_data.data, list) else []\n", + "if parsed_data and parsed_data.get(\"data\"):\n", + " articles = parsed_data.get(\"data\").get(\"articles\", []) if isinstance(parsed_data.get(\"data\"), dict) else parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else []\n", " \n", " for article in articles:\n", " if isinstance(article, dict):\n", @@ -250,7 +250,7 @@ "\n", "# Analyze sentiment from embeddings and article properties\n", "sentiment_scores = []\n", - "for i, article in enumerate(parsed_data.data.get(\"articles\", []) if parsed_data and parsed_data.data and isinstance(parsed_data.data, dict) else []):\n", + "for i, article in enumerate(parsed_data.get(\"data\").get(\"articles\", []) if parsed_data and parsed_data.get(\"data\") and isinstance(parsed_data.get(\"data\"), dict) else []):\n", " if isinstance(article, dict):\n", " sentiment = article.get(\"sentiment\", \"neutral\")\n", " sentiment_value = 1.0 if sentiment == \"positive\" else -1.0 if sentiment == \"negative\" else 0.0\n", diff --git a/cookbook/use_cases/trading/03_Real_Time_Monitoring.ipynb b/cookbook/use_cases/trading/03_Real_Time_Monitoring.ipynb index 67fbe361..bce67666 100644 --- a/cookbook/use_cases/trading/03_Real_Time_Monitoring.ipynb +++ b/cookbook/use_cases/trading/03_Real_Time_Monitoring.ipynb @@ -145,7 +145,7 @@ " json.dump(trading_stream, f, indent=2)\n", "\n", "file_objects = file_ingestor.ingest_file(trading_stream_file, read_content=True)\n", - "parsed_data = structured_parser.parse_json(trading_stream_file)\n", + "parsed_data = structured_parser.parse_data(trading_stream_file, data_format=\"json\")\n", "\n", "print(f\"\\nšŸ“Š Ingestion Summary:\")\n", "print(f\" Trading stream files: {len([file_objects]) if file_objects else 0}\")\n", @@ -177,8 +177,8 @@ "trading_relationships = []\n", "\n", "# Extract from trading stream data\n", - "if parsed_data and parsed_data.data:\n", - " for position in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + "if parsed_data and parsed_data.get(\"data\"):\n", + " for position in parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else [parsed_data.get(\"data\")]:\n", " if isinstance(position, dict):\n", " position_id = position.get(\"position_id\", \"\")\n", " symbol = position.get(\"symbol\", \"\")\n", @@ -258,8 +258,8 @@ "\n", "# Monitor position changes\n", "position_changes = []\n", - "if parsed_data and parsed_data.data:\n", - " for position in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + "if parsed_data and parsed_data.get(\"data\"):\n", + " for position in parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else [parsed_data.get(\"data\")]:\n", " if isinstance(position, dict):\n", " price_change = position.get(\"current_price\", 0) - position.get(\"entry_price\", 0)\n", " price_change_percent = (price_change / position.get(\"entry_price\", 1)) * 100 if position.get(\"entry_price\", 0) > 0 else 0\n", diff --git a/cookbook/use_cases/trading/04_Risk_Assessment.ipynb b/cookbook/use_cases/trading/04_Risk_Assessment.ipynb index 2e63875a..197c143c 100644 --- a/cookbook/use_cases/trading/04_Risk_Assessment.ipynb +++ b/cookbook/use_cases/trading/04_Risk_Assessment.ipynb @@ -140,7 +140,7 @@ " json.dump(portfolio_risk_data, f, indent=2)\n", "\n", "file_objects = file_ingestor.ingest_file(risk_data_file, read_content=True)\n", - "parsed_data = structured_parser.parse_json(risk_data_file)\n", + "parsed_data = structured_parser.parse_data(risk_data_file, data_format=\"json\")\n", "\n", "print(f\"\\nšŸ“Š Ingestion Summary:\")\n", "print(f\" Risk data files: {len([file_objects]) if file_objects else 0}\")\n", @@ -171,8 +171,8 @@ "risk_relationships = []\n", "\n", "# Extract from portfolio risk data\n", - "if parsed_data and parsed_data.data:\n", - " portfolio = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + "if parsed_data and parsed_data.get(\"data\"):\n", + " portfolio = parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), dict) else parsed_data.get(\"data\")[0] if isinstance(parsed_data.get(\"data\"), list) else {}\n", " \n", " if isinstance(portfolio, dict):\n", " portfolio_id = portfolio.get(\"portfolio_id\", \"\")\n", @@ -271,8 +271,8 @@ "\n", "# Analyze risk concentration\n", "risk_concentration = {}\n", - "if parsed_data and parsed_data.data:\n", - " portfolio = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + "if parsed_data and parsed_data.get(\"data\"):\n", + " portfolio = parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), dict) else parsed_data.get(\"data\")[0] if isinstance(parsed_data.get(\"data\"), list) else {}\n", " if isinstance(portfolio, dict):\n", " total_value = portfolio.get(\"total_value\", 1)\n", " for position in portfolio.get(\"positions\", []):\n", @@ -318,8 +318,8 @@ "\n", "# Assess portfolio risk\n", "portfolio_risk_assessment = {}\n", - "if parsed_data and parsed_data.data:\n", - " portfolio = parsed_data.data if isinstance(parsed_data.data, dict) else parsed_data.data[0] if isinstance(parsed_data.data, list) else {}\n", + "if parsed_data and parsed_data.get(\"data\"):\n", + " portfolio = parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), dict) else parsed_data.get(\"data\")[0] if isinstance(parsed_data.get(\"data\"), list) else {}\n", " if isinstance(portfolio, dict):\n", " portfolio_risk_score = portfolio.get(\"portfolio_risk_score\", 0)\n", " \n", diff --git a/cookbook/use_cases/trading/05_Strategy_Backtesting.ipynb b/cookbook/use_cases/trading/05_Strategy_Backtesting.ipynb index 3f62b34b..4b9a9939 100644 --- a/cookbook/use_cases/trading/05_Strategy_Backtesting.ipynb +++ b/cookbook/use_cases/trading/05_Strategy_Backtesting.ipynb @@ -67,7 +67,6 @@ "from semantica.kg import GraphBuilder, TemporalGraphQuery, TemporalPatternDetector, GraphAnalyzer\n", "from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n", "from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n", - "from semantica.export import JSONExporter, CSVExporter, RDFExporter, ReportGenerator\n", "from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n", "import tempfile\n", @@ -113,11 +112,11 @@ " {\"symbol\": \"MSFT\", \"date\": \"2023-01-16\", \"open\": 351.25, \"high\": 353.50, \"low\": 350.75, \"close\": 352.50, \"volume\": 31000000}\n", "]\n", "\n", - "with open(historical_data_file, 'w') as f:\n", + "with open(historical_data_file, 'w') as f,\n", " json.dump(historical_data, f, indent=2)\n", "\n", "file_objects = file_ingestor.ingest_file(historical_data_file, read_content=True)\n", - "parsed_data = structured_parser.parse_json(historical_data_file)\n", + "parsed_data = structured_parser.parse_data(historical_data_file, data_format=\"json\")\n", "\n", "# Ingest from historical market APIs\n", "historical_api_list = []\n", @@ -166,8 +165,8 @@ "historical_relationships = []\n", "\n", "# Extract from historical data\n", - "if parsed_data and parsed_data.data:\n", - " for entry in parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data]:\n", + "if parsed_data and parsed_data.get(\"data\"):\n", + " for entry in parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else [parsed_data.get(\"data\")]:\n", " if isinstance(entry, dict):\n", " symbol = entry.get(\"symbol\", \"\")\n", " date = entry.get(\"date\", \"\")\n", @@ -251,8 +250,8 @@ " trades = []\n", " positions = {}\n", " \n", - " if parsed_data and parsed_data.data:\n", - " sorted_data = sorted(parsed_data.data if isinstance(parsed_data.data, list) else [parsed_data.data], \n", + " if parsed_data and parsed_data.get(\"data\"):\n", + " sorted_data = sorted(parsed_data.get(\"data\") if isinstance(parsed_data.get(\"data\"), list) else [parsed_data.get(\"data\")], \n", " key=lambda x: x.get(\"date\", \"\"))\n", " \n", " for entry in sorted_data:\n", diff --git a/docs/reference/parse.md b/docs/reference/parse.md index d2904fe7..16e96921 100644 --- a/docs/reference/parse.md +++ b/docs/reference/parse.md @@ -81,11 +81,12 @@ Unified interface for document formats. **Methods:** -| Method | Description | Supported Formats | -|--------|-------------|-------------------| -| `parse_document(path)` | Auto-detect and parse | PDF, DOCX, PPTX, TXT | -| `parse_pdf(path)` | PDF specific parsing | PDF | -| `parse_docx(path)` | Word specific parsing | DOCX | +| Method | Description | +|--------|-------------| +| `parse_document(path)` | Auto-detect format and parse | +| `extract_text(path)` | Extract text from PDF/DOCX/HTML/TXT | +| `extract_metadata(path)` | Extract document metadata | +| `parse_batch(paths)` | Parse multiple documents | **Example:** @@ -94,8 +95,8 @@ from semantica.parse import DocumentParser parser = DocumentParser() doc = parser.parse_document("report.pdf") -print(f"Title: {doc.metadata.title}") -print(f"Text: {doc.text[:100]}...") +print(doc.get("metadata", {}).get("title")) +print(doc.get("full_text", "")[:100]) ``` ### WebParser @@ -106,8 +107,10 @@ Parses web content. | Method | Description | |--------|-------------| -| `parse_html(url)` | Static HTML parsing | -| `parse_dynamic(url)` | JS-rendered parsing | +| `parse_web_content(content, content_type)` | Parse HTML/XML | +| `extract_text(content)` | Clean text from HTML | +| `extract_links(content)` | Extract hyperlinks | +| `render_javascript(url)` | Render JS for dynamic pages | ### StructuredDataParser @@ -117,8 +120,17 @@ Parses data files. | Method | Description | |--------|-------------| -| `parse_json(path)` | JSON with nesting | -| `parse_csv(path)` | CSV with type inference | +| `parse_data(path, data_format)` | Parse JSON/CSV/XML/YAML | + +**Example:** + +```python +from semantica.parse import StructuredDataParser + +parser = StructuredDataParser() +data = parser.parse_data("data.json", data_format="json") +print(type(data.get("data"))).__name__ +``` ### CodeParser @@ -128,22 +140,111 @@ Parses source code. | Method | Description | |--------|-------------| -| `parse_code(path)` | Extract AST & symbols | -| `get_dependencies(path)` | Find imports | +| `parse_code(path)` | Parse code file; returns structure, comments, dependencies | + +**Example:** + +```python +from semantica.parse import CodeParser + +parser = CodeParser() +data = parser.parse_code("script.py", language="python") +print(data.get("structure", {}).get("functions", [])) +print(data.get("dependencies", {})) +``` + +### EmailParser + +Parses email messages. + +**Methods:** + +| Method | Description | +|--------|-------------| +| `parse_email(path)` | Parse full email (headers/body/attachments) | +| `parse_headers(path)` | Extract headers only | +| `extract_body(path)` | Extract text/HTML body | +| `analyze_thread(path)` | Thread reconstruction | + +**Example:** + +```python +from semantica.parse import EmailParser + +parser = EmailParser() +email = parser.parse_email("email.eml", extract_attachments=True) +print(email.headers.subject) +print(email.body.text[:120]) +``` + +### MediaParser + +Parses media files. + +**Methods:** + +| Method | Description | +|--------|-------------| +| `parse_media(path, media_type)` | Parse image/audio/video | + +**Example:** + +```python +from semantica.parse import MediaParser + +parser = MediaParser() +image = parser.parse_media("image.jpg", media_type="image") +print(image.get("metadata", {})) +``` + +### Format-Specific Parsers + +- `PDFParser`, `DOCXParser`, `PPTXParser`, `ExcelParser` +- `HTMLParser`, `XMLParser` +- `JSONParser`, `CSVParser` +- `ImageParser` + +**Examples:** + +```python +from semantica.parse import DocumentParser, WebParser, StructuredDataParser + +# Document +doc = DocumentParser().parse_document("document.pdf") +print(doc.get("full_text", "")[:120]) + +# Web +web = WebParser().parse_web_content("https://example.com", content_type="html") +print(web.get("text", "")[:120]) + +# Structured Data (JSON) +data = StructuredDataParser().parse_data("data.json", data_format="json") +print(list(data.get("data", {}).keys())) +``` --- -## Convenience Functions +## Usage Examples + +### WebParser ```python -from semantica.parse import parse_document, parse_json, parse_web_content +from semantica.parse import WebParser -# Auto-detect format -doc = parse_document("file.pdf") +parser = WebParser() +html = parser.parse_web_content("https://example.com", content_type="html") +links = parser.extract_links("https://example.com") +``` -# Parse specific types -data = parse_json("data.json") -web = parse_web_content("https://google.com") +### StructuredDataParser + +```python +from semantica.parse import StructuredDataParser + +parser = StructuredDataParser() +json = parser.parse_data("data.json", data_format="json") +csv = parser.parse_data("data.csv", data_format="csv") +xml = parser.parse_data("data.xml", data_format="xml") ``` --- @@ -177,37 +278,18 @@ parse: --- -## Integration Examples +## Integration -### Ingest & Parse Pipeline - -```python -from semantica.ingest import Ingestor -from semantica.parse import DocumentParser, ImageParser - -# 1. Ingest Raw File -ingestor = Ingestor() -file_path = ingestor.ingest("scan.png") - -# 2. Parse (with OCR) -if file_path.endswith(".png"): - parser = ImageParser(ocr_enabled=True) - content = parser.parse_image(file_path) -else: - parser = DocumentParser() - content = parser.parse_document(file_path) - -print(content.text) -``` +Use parser classes directly in pipelines and services. Avoid convenience functions for stronger type clarity and consistency. --- ## Best Practices -1. **Disable OCR if not needed**: OCR is slow. Only enable it (`ocr_enabled=True`) if you expect scanned documents. -2. **Use Specific Parsers**: If you know the format, use `parse_json` or `parse_pdf` directly for better type hinting. -3. **Handle Encodings**: The parser tries to auto-detect encoding, but for CSV/TXT, explicitly specifying it is safer. -4. **Clean Web Content**: Use `parse_web_content` which includes boilerplate removal, rather than raw HTML parsing. +1. Disable OCR if not needed; enable only for scanned documents. +2. Use specific parser classes like `JSONParser` or `PDFParser` when format is known. +3. Handle encodings explicitly for CSV/TXT where auto-detect may fail. +4. Clean web content using `WebParser` utilities rather than raw HTML parsing. --- diff --git a/semantica/parse/parse_usage.md b/semantica/parse/parse_usage.md index e0271acb..54f81565 100644 --- a/semantica/parse/parse_usage.md +++ b/semantica/parse/parse_usage.md @@ -19,25 +19,6 @@ This comprehensive guide demonstrates how to use the data parsing module for doc ## Basic Usage -### Using the Convenience Functions - -```python -from semantica.parse import parse_document, parse_web_content, parse_json - -# Parse a PDF document -doc = parse_document("document.pdf", method="default") -print(f"Text: {doc.get('full_text', '')}") -print(f"Pages: {doc.get('total_pages', 0)}") - -# Parse web content -web = parse_web_content("https://example.com", method="default") -print(f"Content: {web.get('text', '')}") - -# Parse JSON data -data = parse_json("data.json", method="default") -print(f"Data: {data.data}") -``` - ### Using Main Classes ```python @@ -66,15 +47,9 @@ print(f"Data: {data}") ### PDF Parsing ```python -from semantica.parse import parse_pdf, PDFParser +from semantica.parse import PDFParser -# Using convenience function -pdf = parse_pdf("document.pdf", method="default", extract_tables=True) -print(f"Text: {pdf.get('full_text', '')}") -print(f"Pages: {pdf.get('total_pages', 0)}") -print(f"Tables: {pdf.get('pages', [{}])[0].get('tables', [])}") - -# Using PDFParser directly +# Using PDFParser pdf_parser = PDFParser() pdf_data = pdf_parser.parse("document.pdf", extract_text=True, extract_tables=True) @@ -91,14 +66,9 @@ tables = pdf_parser.extract_tables("document.pdf") ### DOCX Parsing ```python -from semantica.parse import parse_docx, DOCXParser +from semantica.parse import DOCXParser -# Using convenience function -docx = parse_docx("document.docx", method="default") -print(f"Text: {docx.get('text', '')}") -print(f"Sections: {docx.get('sections', [])}") - -# Using DOCXParser directly +# Using DOCXParser docx_parser = DOCXParser() docx_data = docx_parser.parse("document.docx", extract_tables=True) @@ -148,13 +118,9 @@ for sheet_name, sheet in excel_data.sheets.items(): ### HTML Document Parsing ```python -from semantica.parse import parse_document, HTMLParser +from semantica.parse import HTMLParser -# Using convenience function -html_doc = parse_document("page.html", method="default") -print(f"Text: {html_doc.get('text', '')}") - -# Using HTMLParser directly +# Using HTMLParser html_parser = HTMLParser() html_data = html_parser.parse("page.html", extract_links=True, extract_images=True) @@ -172,10 +138,11 @@ for link in links: ### Text File Parsing ```python -from semantica.parse import parse_document +from semantica.parse import DocumentParser # Parse plain text file -text_doc = parse_document("document.txt", method="default") +doc_parser = DocumentParser() +text_doc = doc_parser.parse_document("document.txt") print(f"Text: {text_doc.get('text', '')}") ``` @@ -184,13 +151,9 @@ print(f"Text: {text_doc.get('text', '')}") ### HTML Content Parsing ```python -from semantica.parse import parse_web_content, WebParser +from semantica.parse import WebParser -# Using convenience function -web = parse_web_content("https://example.com", content_type="html", method="default") -print(f"Text: {web.get('text', '')}") - -# Using WebParser directly +# Using WebParser web_parser = WebParser() html_data = web_parser.parse_web_content("https://example.com", content_type="html") @@ -209,12 +172,9 @@ rendered = web_parser.render_javascript("https://example.com", wait_time=5) ### XML Content Parsing ```python -from semantica.parse import parse_web_content, XMLParser +from semantica.parse import XMLParser -# Using convenience function -xml_data = parse_web_content("data.xml", content_type="xml", method="default") - -# Using XMLParser directly +# Using XMLParser xml_parser = XMLParser() xml_data = xml_parser.parse("data.xml") @@ -248,14 +208,9 @@ rendered_content = web_parser.parse_web_content( ### JSON Parsing ```python -from semantica.parse import parse_json, JSONParser +from semantica.parse import JSONParser -# Using convenience function -json_data = parse_json("data.json", method="default") -print(f"Data: {json_data.data}") -print(f"Type: {json_data.type}") - -# Using JSONParser directly +# Using JSONParser json_parser = JSONParser() json_data = json_parser.parse("data.json", flatten=True) @@ -268,14 +223,9 @@ for path in paths: ### CSV Parsing ```python -from semantica.parse import parse_csv, CSVParser +from semantica.parse import CSVParser -# Using convenience function -csv_data = parse_csv("data.csv", delimiter=",", method="default") -print(f"Headers: {csv_data.headers}") -print(f"Rows: {csv_data.row_count}") - -# Using CSVParser directly +# Using CSVParser csv_parser = CSVParser() csv_data = csv_parser.parse("data.csv", delimiter=",", has_header=True) @@ -288,13 +238,9 @@ for row in rows: ### XML Parsing ```python -from semantica.parse import parse_xml, XMLParser +from semantica.parse import XMLParser -# Using convenience function -xml_data = parse_xml("data.xml", method="default") -print(f"Root: {xml_data.root.tag}") - -# Using XMLParser directly +# Using XMLParser xml_parser = XMLParser() xml_data = xml_parser.parse("data.xml", engine="lxml") @@ -319,26 +265,19 @@ print(f"Data: {yaml_data}") ### Basic Email Parsing ```python -from semantica.parse import parse_email, EmailParser +from semantica.parse import EmailParser -# Using convenience function -email = parse_email("email.eml", method="default") -print(f"Subject: {email.headers.subject}") -print(f"From: {email.headers.from_address}") -print(f"To: {email.headers.to_addresses}") -print(f"Body: {email.body.text}") - -# Using EmailParser directly +# Using EmailParser email_parser = EmailParser() email_data = email_parser.parse_email("email.eml", extract_attachments=True) -# Extract headers only -headers = email_parser.parse_headers("email.eml") +# Access headers from parsed email +headers = email_data.headers print(f"Subject: {headers.subject}") print(f"Date: {headers.date}") -# Extract body only -body = email_parser.extract_body("email.eml") +# Access body from parsed email +body = email_data.body print(f"Text: {body.text}") print(f"HTML: {body.html}") ``` @@ -351,15 +290,12 @@ from semantica.parse import EmailParser email_parser = EmailParser() # Analyze email thread -thread = email_parser.analyze_thread("email.eml") -print(f"Thread ID: {thread.thread_id}") -print(f"Messages: {len(thread.messages)}") -print(f"Subject: {thread.subject}") - -# Parse multiple emails in a thread +emails = [] for email_file in ["email1.eml", "email2.eml", "email3.eml"]: - email = email_parser.parse_email(email_file) - print(f"Email: {email.headers.subject}") + emails.append(email_parser.parse_email(email_file)) + +thread = email_parser.analyze_thread(emails) +print(f"Messages: {len(thread.get('messages', []))}") ``` ### Email Attachment Extraction @@ -382,30 +318,23 @@ for attachment in email_data.body.attachments: ### Basic Code Parsing ```python -from semantica.parse import parse_code, CodeParser +from semantica.parse import CodeParser -# Using convenience function -code = parse_code("script.py", method="default") -print(f"Functions: {code.get('structure', {}).get('functions', [])}") -print(f"Classes: {code.get('structure', {}).get('classes', [])}") -print(f"Imports: {code.get('structure', {}).get('imports', [])}") - -# Using CodeParser directly +# Using CodeParser code_parser = CodeParser() code_data = code_parser.parse_code("script.py", language="python") -# Extract structure -structure = code_parser.extract_structure("script.py", language="python") -print(f"Functions: {structure.functions}") -print(f"Classes: {structure.classes}") +# Access structure, comments, dependencies +structure = code_data.get("structure", {}) +print(f"Functions: {structure.get('functions', [])}") +print(f"Classes: {structure.get('classes', [])}") +print(f"Imports: {structure.get('imports', [])}") -# Extract comments -comments = code_parser.extract_comments("script.py", language="python") +comments = code_data.get("comments", []) for comment in comments: - print(f"Comment: {comment.text} (Line {comment.line_number})") + print(f"Comment: {comment.get('text', '')} (Line {comment.get('line_number', 0)})") -# Analyze dependencies -dependencies = code_parser.analyze_dependencies("script.py", language="python") +dependencies = code_data.get("dependencies", {}) print(f"Dependencies: {dependencies}") ``` @@ -417,23 +346,18 @@ from semantica.parse import CodeParser, SyntaxTreeParser code_parser = CodeParser() syntax_parser = SyntaxTreeParser() -# Parse syntax tree -tree = syntax_parser.parse_syntax_tree("script.py", language="python") +# Parse file content then build syntax tree +with open("script.py", "r", encoding="utf-8", errors="ignore") as f: + content = f.read() -# Extract functions -functions = syntax_parser.extract_functions("script.py", language="python") -for func in functions: - print(f"Function: {func.get('name', '')}") +tree = syntax_parser.parse_syntax_tree(content, language="python") -# Extract classes -classes = syntax_parser.extract_classes("script.py", language="python") -for cls in classes: - print(f"Class: {cls.get('name', '')}") - -# Extract imports -imports = syntax_parser.extract_imports("script.py", language="python") -for imp in imports: - print(f"Import: {imp}") +# Structure and imports are provided via parse_code +code_data = code_parser.parse_code("script.py", language="python") +structure = code_data.get("structure", {}) +print(f"Functions: {structure.get('functions', [])}") +print(f"Classes: {structure.get('classes', [])}") +print(f"Imports: {structure.get('imports', [])}") ``` ### Multi-Language Code Parsing @@ -458,19 +382,9 @@ java_code = code_parser.parse_code("Main.java", language="java") ### Image Parsing with OCR ```python -from semantica.parse import parse_image, ImageParser +from semantica.parse import ImageParser -# Using convenience function -image = parse_image("image.jpg", method="default", extract_text=True) -print(f"Format: {image.get('metadata', {}).get('format', '')}") -print(f"Size: {image.get('metadata', {}).get('size', (0, 0))}") - -# Extract OCR text -if "ocr_result" in image: - print(f"OCR Text: {image['ocr_result'].text}") - print(f"Confidence: {image['ocr_result'].confidence}") - -# Using ImageParser directly +# Using ImageParser image_parser = ImageParser() image_data = image_parser.parse("image.jpg", extract_text=True, ocr_language="eng") @@ -489,20 +403,15 @@ print(f"Confidence: {ocr_result.confidence}") ### Media File Parsing ```python -from semantica.parse import parse_media, MediaParser +from semantica.parse import MediaParser -# Using convenience function -media = parse_media("video.mp4", method="default") -print(f"Type: {media.get('media_type', '')}") -print(f"Metadata: {media.get('metadata', {})}") - -# Using MediaParser directly +# Using MediaParser media_parser = MediaParser() media_data = media_parser.parse_media("image.jpg", media_type="image") -# Get supported formats -formats = media_parser.get_supported_formats() -print(f"Supported formats: {formats}") +# Parse audio/video similarly by specifying media_type +video = media_parser.parse_media("video.mp4", media_type="video") +audio = media_parser.parse_media("audio.mp3", media_type="audio") ``` ## Format-Specific Parsers @@ -647,42 +556,7 @@ if "ocr_result" in image_data: print(f"Language: {ocr.language}") ``` -## Using Methods - -### Method Selection - -```python -from semantica.parse import parse_document, parse_web_content, parse_json - -# Use default method -doc = parse_document("document.pdf", method="default") - -# Use specific method (if registered) -doc = parse_document("document.pdf", method="custom_pdf_parser") - -# List available methods -from semantica.parse import list_available_methods - -methods = list_available_methods("document") -print(f"Available document methods: {methods}") -``` - -### Custom Method Registration - -```python -from semantica.parse import method_registry, parse_document - -# Define custom parsing method -def custom_document_parser(file_path, file_type=None, **kwargs): - # Custom parsing logic - return {"text": "Custom parsed text", "metadata": {}} - -# Register custom method -method_registry.register("document", "custom", custom_document_parser) - -# Use custom method -doc = parse_document("document.pdf", method="custom") -``` + ## Using Registry @@ -726,15 +600,10 @@ print(f"Web methods: {web_methods}") ### Getting Methods ```python -from semantica.parse import method_registry, get_parse_method +from semantica.parse import method_registry # Get method directly method = method_registry.get("document", "default") -if method: - result = method("document.pdf") - -# Using convenience function -method = get_parse_method("document", "default") if method: result = method("document.pdf") ``` @@ -838,16 +707,18 @@ print(f"All config: {all_config}") ### Batch Document Processing ```python -from semantica.parse import parse_document +from semantica.parse import DocumentParser from pathlib import Path # Process multiple documents documents = ["doc1.pdf", "doc2.docx", "doc3.html"] results = [] +doc_parser = DocumentParser() + for doc_path in documents: try: - result = parse_document(doc_path, method="default") + result = doc_parser.parse_document(doc_path) results.append({ "file": doc_path, "text": result.get("full_text", ""), @@ -861,35 +732,36 @@ pdf_dir = Path("documents") pdf_files = list(pdf_dir.glob("*.pdf")) for pdf_file in pdf_files: - result = parse_document(pdf_file, method="default") + result = doc_parser.parse_document(pdf_file) print(f"Processed: {pdf_file.name} ({result.get('total_pages', 0)} pages)") ``` ### Multi-Format Data Extraction ```python -from semantica.parse import ( - parse_document, parse_json, parse_csv, parse_xml -) +from semantica.parse import DocumentParser, JSONParser, CSVParser, XMLParser -# Extract data from multiple formats -files = { - "document.pdf": parse_document, - "data.json": parse_json, - "data.csv": parse_csv, - "data.xml": parse_xml, -} +doc_parser = DocumentParser() +json_parser = JSONParser() +csv_parser = CSVParser() +xml_parser = XMLParser() + +files = [ + ("document.pdf", lambda p: doc_parser.parse_document(p)), + ("data.json", lambda p: json_parser.parse(p)), + ("data.csv", lambda p: csv_parser.parse(p)), + ("data.xml", lambda p: xml_parser.parse(p)), +] extracted_data = {} -for file_path, parser_func in files.items(): +for file_path, parse_fn in files: try: - data = parser_func(file_path, method="default") + data = parse_fn(file_path) extracted_data[file_path] = data except Exception as e: print(f"Error parsing {file_path}: {e}") -# Process extracted data for file_path, data in extracted_data.items(): print(f"File: {file_path}") if isinstance(data, dict): @@ -966,7 +838,7 @@ print(f"Total imports: {len(set(all_imports))}") ### OCR Batch Processing ```python -from semantica.parse import parse_image +from semantica.parse import ImageParser from pathlib import Path # Process all images in a directory @@ -975,9 +847,11 @@ image_files = list(image_dir.glob("*.jpg")) + list(image_dir.glob("*.png")) ocr_results = [] +parser = ImageParser() + for image_file in image_files: try: - image_data = parse_image(image_file, method="default", extract_text=True) + image_data = parser.parse(image_file, extract_text=True) if "ocr_result" in image_data: ocr_results.append({ @@ -998,36 +872,33 @@ for result in ocr_results: ### Custom Parser Pipeline ```python -from semantica.parse import ( - parse_document, parse_json, parse_csv, - method_registry -) +from semantica.parse import DocumentParser, StructuredDataParser, method_registry + +doc_parser = DocumentParser() +data_parser = StructuredDataParser() -# Define custom pipeline def pipeline_parser(file_path, file_type=None, **kwargs): - # Step 1: Parse document - doc = parse_document(file_path, file_type=file_type, **kwargs) - - # Step 2: Extract structured data if available + doc = doc_parser.parse_document(file_path, file_type=file_type, **kwargs) structured_data = {} - if "json" in str(file_path): - structured_data = parse_json(file_path, **kwargs) - elif "csv" in str(file_path): - structured_data = parse_csv(file_path, **kwargs) - - # Step 3: Combine results + suffix = str(file_path).lower() + if suffix.endswith(".json"): + structured_data = data_parser.parse_data(file_path, data_format="json") + elif suffix.endswith(".csv"): + structured_data = data_parser.parse_data(file_path, data_format="csv") + elif suffix.endswith(".xml"): + structured_data = data_parser.parse_data(file_path, data_format="xml") return { "document": doc, "structured": structured_data, "combined_text": doc.get("full_text", "") + str(structured_data) } -# Register pipeline method_registry.register("document", "pipeline", pipeline_parser) -# Use pipeline -result = parse_document("document.pdf", method="pipeline") -print(f"Combined text: {result['combined_text']}") +method = method_registry.get("document", "pipeline") +if method: + result = method("document.pdf") + print(f"Combined text: {result['combined_text']}") ``` This comprehensive guide covers all major features of the data parsing module. For more specific use cases or advanced scenarios, refer to the individual parser class documentation or explore the source code.