mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-10 04:00:35 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ce4e68b465 |
+1
-1
@@ -173,7 +173,7 @@ This includes PyTorch with CUDA, FAISS GPU, and CuPy.
|
|||||||
<Accordion title="How does Semantica handle large datasets?" icon="layer-group">
|
<Accordion title="How does Semantica handle large datasets?" icon="layer-group">
|
||||||
|
|
||||||
- **Batching**: process documents in configurable chunks to control memory usage
|
- **Batching**: process documents in configurable chunks to control memory usage
|
||||||
- **Parallel processing**: `Pipeline(workers=N)` runs extraction steps concurrently
|
- **Parallel processing**: `PipelineBuilder().set_parallelism(N)` runs independent pipeline steps concurrently
|
||||||
- **Delta processing**: update graphs incrementally without full recompute on new data
|
- **Delta processing**: update graphs incrementally without full recompute on new data
|
||||||
- **Persistent backends**: swap in-memory NetworkX for Neo4j, FalkorDB, or Apache AGE for large-scale production graphs
|
- **Persistent backends**: swap in-memory NetworkX for Neo4j, FalkorDB, or Apache AGE for large-scale production graphs
|
||||||
|
|
||||||
|
|||||||
+67
-41
@@ -62,18 +62,19 @@ sources = ingestor.ingest("data/report.pdf")
|
|||||||
```python Web
|
```python Web
|
||||||
from semantica.ingest import WebIngestor
|
from semantica.ingest import WebIngestor
|
||||||
|
|
||||||
ingestor = WebIngestor(max_depth=2)
|
ingestor = WebIngestor()
|
||||||
sources = ingestor.ingest("https://example.com/article")
|
page = ingestor.ingest_url("https://example.com/article")
|
||||||
|
# WebContent: page.text, page.title, page.html, page.links, page.metadata
|
||||||
```
|
```
|
||||||
|
|
||||||
```python Parquet / XML (v0.5.0)
|
```python Parquet / XML
|
||||||
from semantica.ingest import ParquetIngestor, XMLIngestor
|
from semantica.ingest import ParquetIngestor, XMLIngestor
|
||||||
|
|
||||||
# Single file or Hive-partitioned directory
|
# Single file or Hive-partitioned directory
|
||||||
sources = ParquetIngestor().ingest("data/events.parquet")
|
sources = ParquetIngestor().ingest("data/events.parquet")
|
||||||
|
|
||||||
# XML with XSD schema validation
|
# XML; pass an XSD to validate against during ingestion
|
||||||
sources = XMLIngestor(validate_xsd="schema.xsd").ingest("data/records/")
|
sources = XMLIngestor().ingest("data/records/", schema_path="schema.xsd")
|
||||||
```
|
```
|
||||||
|
|
||||||
</CodeGroup>
|
</CodeGroup>
|
||||||
@@ -88,22 +89,24 @@ Extract structured text and layout from raw documents.
|
|||||||
from semantica.parse import DocumentParser
|
from semantica.parse import DocumentParser
|
||||||
|
|
||||||
parser = DocumentParser()
|
parser = DocumentParser()
|
||||||
parsed = parser.parse(sources[0])
|
parsed = parser.parse(sources[0].path) # parse() takes a path string
|
||||||
|
|
||||||
print(parsed.text[:200]) # extracted text
|
print(parsed["text"][:200]) # extracted text
|
||||||
print(parsed.metadata) # title, author, date, source
|
print(parsed["metadata"]) # file_path, encoding, size, and format-specific keys
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`parse()` returns a `dict` with `text`, `full_text`, and `metadata` keys.
|
||||||
|
|
||||||
<Tip>
|
<Tip>
|
||||||
For PDFs with tables, charts, or multi-column layouts, use `DoclingParser`: it applies advanced layout analysis and returns structured table data alongside text.
|
For PDFs with tables, charts, or multi-column layouts, use `DoclingParser` (`pip install semantica[parse-docling]`): it applies advanced layout analysis and returns structured table data alongside text.
|
||||||
</Tip>
|
</Tip>
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from semantica.parse import DoclingParser
|
from semantica.parse import DoclingParser
|
||||||
|
|
||||||
parser = DoclingParser()
|
parser = DoclingParser()
|
||||||
parsed = parser.parse(sources[0])
|
parsed = parser.parse(sources[0].path)
|
||||||
print(parsed.tables) # structured table objects
|
print(parsed["tables"]) # structured table data
|
||||||
```
|
```
|
||||||
|
|
||||||
</Step>
|
</Step>
|
||||||
@@ -117,26 +120,28 @@ Identify named entities and extract typed relationships between them.
|
|||||||
```python Pattern-based (fast, no API key)
|
```python Pattern-based (fast, no API key)
|
||||||
from semantica.semantic_extract import NERExtractor, RelationExtractor
|
from semantica.semantic_extract import NERExtractor, RelationExtractor
|
||||||
|
|
||||||
ner = NERExtractor(method="pattern")
|
text = parsed["text"]
|
||||||
entities = ner.extract(parsed)
|
|
||||||
# Returns: [{"text": "Apple Inc.", "type": "ORGANIZATION", "confidence": 0.98}, ...]
|
|
||||||
|
|
||||||
rel = RelationExtractor(method="rule")
|
ner = NERExtractor(method="pattern")
|
||||||
relationships = rel.extract(parsed, entities=entities)
|
entities = ner.extract(text)
|
||||||
# Returns: [{"subject": "Steve Jobs", "predicate": "founded", "object": "Apple Inc."}, ...]
|
# Returns: [Entity(text="Apple Inc.", label="ORG", start_char=0, end_char=10, confidence=0.7), ...]
|
||||||
|
|
||||||
|
rel = RelationExtractor(method="pattern")
|
||||||
|
relationships = rel.extract(text, entities=entities)
|
||||||
|
# Returns: [Relation(subject=Entity(...), predicate="founded_by", object=Entity(...), confidence=0.7), ...]
|
||||||
```
|
```
|
||||||
|
|
||||||
```python LLM-powered (higher accuracy)
|
```python LLM-powered (higher accuracy)
|
||||||
from semantica.semantic_extract import NERExtractor, RelationExtractor
|
from semantica.semantic_extract import NERExtractor, RelationExtractor
|
||||||
from semantica.llms import Groq
|
|
||||||
|
|
||||||
llm = Groq(model="llama-3.3-70b-versatile")
|
# Reads GROQ_API_KEY from the environment; provider/llm_model select the backend
|
||||||
|
text = parsed["text"]
|
||||||
|
|
||||||
ner = NERExtractor(method="llm", llm_provider=llm)
|
ner = NERExtractor(method="llm", provider="groq", llm_model="llama-3.3-70b-versatile")
|
||||||
entities = ner.extract(parsed)
|
entities = ner.extract(text)
|
||||||
|
|
||||||
rel = RelationExtractor(method="llm", llm_provider=llm)
|
rel = RelationExtractor(method="llm", provider="groq", llm_model="llama-3.3-70b-versatile")
|
||||||
relationships = rel.extract(parsed, entities=entities)
|
relationships = rel.extract(text, entities=entities)
|
||||||
```
|
```
|
||||||
|
|
||||||
</CodeGroup>
|
</CodeGroup>
|
||||||
@@ -198,16 +203,17 @@ exporter.export(graph, file_path="graph.nt", format="nt")
|
|||||||
from semantica.export import ParquetExporter
|
from semantica.export import ParquetExporter
|
||||||
|
|
||||||
exporter = ParquetExporter()
|
exporter = ParquetExporter()
|
||||||
exporter.export(graph, file_path="output/graph.parquet")
|
exporter.export(graph, file_path="output/graph")
|
||||||
# Writes nodes.parquet + edges.parquet: ready for Spark, BigQuery, Databricks
|
# Dict input writes one file per key: output/graph_entities.parquet and
|
||||||
|
# output/graph_relationships.parquet: ready for Spark, BigQuery, Databricks
|
||||||
```
|
```
|
||||||
|
|
||||||
```python ArangoDB
|
```python ArangoDB
|
||||||
from semantica.export import ArangoAQLExporter
|
from semantica.export import ArangoAQLExporter
|
||||||
|
|
||||||
exporter = ArangoAQLExporter()
|
exporter = ArangoAQLExporter()
|
||||||
aql = exporter.export(graph)
|
exporter.export(graph, file_path="graph.aql")
|
||||||
# Returns ready-to-run AQL INSERT statements
|
# Writes ready-to-run AQL INSERT statements to graph.aql
|
||||||
```
|
```
|
||||||
|
|
||||||
</CodeGroup>
|
</CodeGroup>
|
||||||
@@ -272,14 +278,21 @@ relationships = rel.extract(text, entities=entities)
|
|||||||
<Accordion title="Multi-source incremental graph build" icon="layer-group">
|
<Accordion title="Multi-source incremental graph build" icon="layer-group">
|
||||||
|
|
||||||
```python
|
```python
|
||||||
|
from semantica.ingest import FileIngestor
|
||||||
|
from semantica.parse import DocumentParser
|
||||||
|
from semantica.semantic_extract import NERExtractor, RelationExtractor
|
||||||
from semantica.kg import GraphBuilder
|
from semantica.kg import GraphBuilder
|
||||||
|
|
||||||
builder = GraphBuilder(merge_entities=True)
|
parser = DocumentParser()
|
||||||
all_entities, all_rels = [], []
|
ner = NERExtractor(method="pattern")
|
||||||
|
rel = RelationExtractor(method="pattern")
|
||||||
|
builder = GraphBuilder(merge_entities=True)
|
||||||
|
|
||||||
for doc in parsed_docs:
|
all_entities, all_rels = [], []
|
||||||
entities = ner.extract(doc)
|
for source in FileIngestor().ingest("data/reports/"):
|
||||||
rels = rel.extract(doc, entities=entities)
|
text = parser.parse(source.path)["text"]
|
||||||
|
entities = ner.extract(text)
|
||||||
|
rels = rel.extract(text, entities=entities)
|
||||||
all_entities.extend(entities)
|
all_entities.extend(entities)
|
||||||
all_rels.extend(rels)
|
all_rels.extend(rels)
|
||||||
|
|
||||||
@@ -359,7 +372,8 @@ graph = builder.build({"entities": entities, "relationships": relationships})
|
|||||||
# Retrieve full lineage for any entity
|
# Retrieve full lineage for any entity
|
||||||
sources = prov.get_all_sources("Apple Inc.")
|
sources = prov.get_all_sources("Apple Inc.")
|
||||||
print(sources[0])
|
print(sources[0])
|
||||||
# {"source": "data/report.pdf", "location": None, "timestamp": "...", "confidence": 0.98}
|
# {"source": "data/report.pdf", "location": None, "timestamp": "...",
|
||||||
|
# "confidence": 1.0, "metadata": {"confidence": 0.98}}
|
||||||
```
|
```
|
||||||
|
|
||||||
</Accordion>
|
</Accordion>
|
||||||
@@ -373,30 +387,42 @@ print(sources[0])
|
|||||||
|
|
||||||
<Accordion title="No entities extracted" icon="magnifying-glass">
|
<Accordion title="No entities extracted" icon="magnifying-glass">
|
||||||
|
|
||||||
The document likely contains scanned images rather than machine-readable text. Enable OCR:
|
The document likely contains scanned images rather than machine-readable text. `DocumentParser` warns when a PDF has no text layer; switch to `DoclingParser` with OCR enabled:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from semantica.parse import DocumentParser
|
from semantica.parse import DoclingParser # pip install semantica[parse-docling]
|
||||||
|
|
||||||
parser = DocumentParser(ocr=True) # enables Tesseract OCR
|
parser = DoclingParser(enable_ocr=True)
|
||||||
parsed = parser.parse(sources[0])
|
parsed = parser.parse(sources[0].path)
|
||||||
```
|
```
|
||||||
|
|
||||||
</Accordion>
|
</Accordion>
|
||||||
|
|
||||||
<Accordion title="Slow processing on large corpora" icon="gauge">
|
<Accordion title="Slow processing on large corpora" icon="gauge">
|
||||||
|
|
||||||
Enable parallel processing and GPU acceleration:
|
Enable GPU acceleration and run pipeline steps in parallel:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install semantica[gpu]
|
pip install semantica[gpu]
|
||||||
```
|
```
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from semantica.pipeline import Pipeline
|
from semantica.pipeline import PipelineBuilder, ExecutionEngine
|
||||||
|
|
||||||
pipeline = Pipeline(workers=8, batch_size=32)
|
builder = PipelineBuilder()
|
||||||
pipeline.run(sources)
|
builder.add_step("ingest", step_type="ingest", source="data/reports/", recursive=True)
|
||||||
|
builder.add_step("extract", step_type="ner_extract")
|
||||||
|
builder.add_step("build", step_type="kg_build", merge_entities=True)
|
||||||
|
|
||||||
|
pipeline = (
|
||||||
|
builder
|
||||||
|
.connect_steps("ingest", "extract")
|
||||||
|
.connect_steps("extract", "build")
|
||||||
|
.set_parallelism(8)
|
||||||
|
.build(name="reports_pipeline")
|
||||||
|
)
|
||||||
|
|
||||||
|
result = ExecutionEngine().execute_pipeline(pipeline)
|
||||||
```
|
```
|
||||||
|
|
||||||
</Accordion>
|
</Accordion>
|
||||||
|
|||||||
Reference in New Issue
Block a user