From 5d70d0c10dca0053818c54fca3e8b5e81285933f Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Sun, 24 May 2026 15:49:58 +0530 Subject: [PATCH] docs: replace Exported Classes import blocks with summary tables (all 25 modules) (#567) * docs: replace Exported Classes import blocks with summary tables across all 25 modules * docs: add method/parameter tables to parse, ingest, ontology, normalize, triplet_store, change_management, conflicts, export, graph_store, provenance, and semantic_extract modules --- docs/reference/change_management.md | 39 ++++++++-------- docs/reference/conflicts.md | 52 +++++++++------------ docs/reference/context.md | 48 +++++--------------- docs/reference/core.md | 27 +++-------- docs/reference/deduplication.md | 41 ++++++++--------- docs/reference/embeddings.md | 38 +++++----------- docs/reference/evals.md | 15 ++++--- docs/reference/explorer.md | 9 ++++ docs/reference/export.md | 43 +++++++++--------- docs/reference/graph_store.md | 51 +++++++++------------ docs/reference/ingest.md | 70 +++++++++++++---------------- docs/reference/kg.md | 48 ++++++-------------- docs/reference/normalize.md | 61 +++++++++---------------- docs/reference/ontology.md | 65 ++++++++++++++------------- docs/reference/parse.md | 65 +++++++++++++++------------ docs/reference/pipeline.md | 42 ++++------------- docs/reference/provenance.md | 44 ++++++++---------- docs/reference/reasoning.md | 47 +++++-------------- docs/reference/seed.md | 12 +++-- docs/reference/semantic_extract.md | 46 ++++++++----------- docs/reference/split.md | 49 +++++++++----------- docs/reference/triplet_store.md | 46 ++++++++++--------- docs/reference/utils.md | 36 ++++++--------- docs/reference/vector_store.md | 44 +++++------------- docs/reference/visualization.md | 27 ++++------- 25 files changed, 423 insertions(+), 642 deletions(-) diff --git a/docs/reference/change_management.md b/docs/reference/change_management.md index 8d45dcc1..4278afae 100644 --- a/docs/reference/change_management.md +++ b/docs/reference/change_management.md @@ -12,26 +12,14 @@ icon: "clock-rotate-left" ## Exported Classes -```python -from semantica.change_management import ( - # Change metadata - ChangeLogEntry, # snapshot record: version, author, message, checksum, changes - # Storage backends - VersionStorage, # abstract storage interface - InMemoryVersionStorage, # fast in-memory backend (dev/test only) - SQLiteVersionStorage, # persistent SQLite backend (production) - # Integrity utilities - compute_checksum, # SHA-256 checksum of a graph state - verify_checksum, # verify graph against a stored checksum - # Version managers - TemporalVersionManager, # KG version management: snapshot, diff, rollback - OntologyVersionManager, # ontology version management - BaseVersionManager, # base class for custom version managers - # Ontology versioning (moved from ontology module) - VersionManager, # OWL ontology version control - OntologyVersion, # ontology version metadata dataclass -) -``` +| Class | Role | +| --- | --- | +| `TemporalVersionManager` | Snapshot, diff, rollback, and per-entity audit trail for temporal KGs | +| `OntologyVersionManager` | Schema versioning with backward-compatible migration support | +| `InMemoryVersionStorage` | Fast in-memory storage for dev and testing — no persistence | +| `SQLiteVersionStorage` | Production storage — persists to a local SQLite file | +| `compute_checksum()` | Returns SHA-256 fingerprint of a graph or ontology state | +| `verify_checksum()` | Detects tampering by comparing stored vs recomputed checksum | ## What You Get @@ -125,6 +113,17 @@ for v in versions: kg_v1 = manager.get_version("v1.0") ``` +### TemporalVersionManager Methods + +| Method | Returns | Description | +| ------ | ------- | ----------- | +| `create_snapshot(graph, version, author, message)` | `str` | Create a version snapshot, returns snapshot ID | +| `get_version(version_id)` | `KnowledgeGraph` | Retrieve a graph at a specific version | +| `list_versions()` | `List[Version]` | List all versions with metadata | +| `diff(from_version, to_version)` | `DiffResult` | Compare two snapshots | +| `rollback(version_id)` | `KnowledgeGraph` | Restore graph to a previous version | +| `get_checksum(snapshot_id)` | `str` | Get SHA-256 checksum of a snapshot | + ## Diff Analysis Compare any two snapshots to see exactly what changed — useful for code review, incident investigation, and regulatory audit: diff --git a/docs/reference/conflicts.md b/docs/reference/conflicts.md index 5d75d637..3cc1ea30 100644 --- a/docs/reference/conflicts.md +++ b/docs/reference/conflicts.md @@ -20,38 +20,15 @@ Semantica's conflict detection makes disagreements explicit and actionable: ## Exported Classes -```python -from semantica.conflicts import ( - # Detection - ConflictDetector, # detect value, type, temporal, logical, relationship conflicts - Conflict, # {id, entity_id, attribute, values, sources, conflict_type, severity} - ConflictType, # enum: VALUE_CONFLICT, TYPE_CONFLICT, TEMPORAL_CONFLICT, ... - # Resolution - ConflictResolver, # resolve conflicts with configurable strategy - ResolutionStrategy, # enum: VOTING, CREDIBILITY_WEIGHTED, MOST_RECENT, FIRST_SEEN, ... - ResolutionResult, # outcome of a resolve_conflicts() call - # Convenience strategy aliases - voting, credibility_weighted, most_recent, first_seen, highest_confidence, - manual_review, expert_review, - # Source tracking - SourceTracker, # track which source contributed each property value - SourceReference, # {source_id, credibility, timestamp} - PropertySource, # per-property source attribution record - # Analysis - ConflictAnalyzer, # analyze patterns, severity distribution, source stats - ConflictPattern, # recurring conflict pattern detected across entities - # Investigation - InvestigationGuideGenerator, # generate step-by-step checklists for manual review - InvestigationGuide, # {title, context, steps} - InvestigationStep, # {order, description, check, priority} - # Convenience functions - detect_conflicts, # quick: detect_conflicts(entities, attribute="name") - resolve_conflicts, # quick: resolve_conflicts(conflicts, strategy=voting) - analyze_conflicts, # quick: analyze_conflicts(conflicts) - track_sources, # quick: track_sources(entities) - generate_investigation_guide,# quick: generate_investigation_guide(conflict) -) -``` +| Class | Role | +| --- | --- | +| `ConflictDetector` | Detects value, type, temporal, logical, and relationship conflicts across entity pairs | +| `ConflictResolver` | Resolves conflicts with configurable strategy: `voting`, `credibility_weighted`, `most_recent`, `first_seen`, `highest_confidence`, `manual_review` | +| `ConflictType` | Enum: `VALUE_CONFLICT`, `TYPE_CONFLICT`, `TEMPORAL_CONFLICT`, `LOGICAL_CONFLICT`, `RELATIONSHIP_CONFLICT` | +| `ResolutionStrategy` | Enum of available resolution strategies passed to `ConflictResolver` | +| `SourceTracker` | Tracks which source contributed each property value on each entity | +| `ConflictAnalyzer` | Analyzes conflict patterns, severity distribution, and per-source statistics | +| `InvestigationGuideGenerator` | Generates step-by-step checklists for human review of unresolvable conflicts | ## What You Get @@ -174,6 +151,17 @@ relation_conflicts = detector.detect_relationship_conflicts(kg) - `LOGICAL` conflicts require an ontology or SHACL schema to be loaded; without one, they are not detected - Detection runs in O(n·sources) time — it groups by entity+attribute and checks disagreement within each group +### ConflictDetector Methods + +| Method | Returns | Description | +| ------ | ------- | ----------- | +| `detect_conflicts(kg)` | `List[Conflict]` | Detect all conflict types at once | +| `detect_value_conflicts(entities, attribute)` | `List[Conflict]` | Detect value disagreements on a specific attribute | +| `detect_type_conflicts(entities)` | `List[Conflict]` | Detect type classification conflicts | +| `detect_temporal_conflicts(entities)` | `List[Conflict]` | Detect overlapping validity window conflicts | +| `detect_logical_conflicts(kg)` | `List[Conflict]` | Detect ontology/SHACL constraint violations | +| `detect_relationship_conflicts(kg)` | `List[Conflict]` | Detect relationship property conflicts | + ## ConflictResolver ```python diff --git a/docs/reference/context.md b/docs/reference/context.md index e448e7e9..235a654e 100644 --- a/docs/reference/context.md +++ b/docs/reference/context.md @@ -8,44 +8,16 @@ icon: "brain" ## Exported Classes -```python -from semantica.context import ( - # High-level interfaces - AgentContext, # primary entry point: store, retrieve, record_decision, find_precedents - DecisionContext, # decision-focused facade (wraps AgentContext + DecisionRecorder) - # Graph primitives - ContextGraph, # in-memory graph: add/get entities, record decisions, find precedents - ContextNode, # {id, label, node_type, properties, embedding, confidence} - ContextEdge, # {source, target, edge_type, weight, properties} - # Memory - AgentMemory, # RAG memory: store(text), retrieve(query, max_results) - MemoryItem, # {id, content, timestamp, conversation_id, embedding, metadata} - # Retrieval - ContextRetriever, # retrieve(query, max_results, use_graph, min_score) - RetrievedContext, # {content, score, source, metadata} - TemporalGraphRetriever, # retrieval with temporal decay weighting - # Entity linking - EntityLinker, # link_entity(text, entity_type) -> LinkedEntity with URI - EntityLink, # {entity_id, uri, source_text, confidence} - LinkedEntity, # {canonical_id, uri, aliases, type, properties} - # Decision tracking models - Decision, # {id, category, scenario, reasoning, outcome, confidence, timestamp} - Policy, # {id, name, conditions, action, priority} - PolicyException, # {policy_id, decision_id, reason, override_authority} - Precedent, # {decision_id, scenario, outcome, similarity, timestamp} - ApprovalChain, # ordered list of approvers for escalation - # Decision tracking classes - DecisionRecorder, # record and persist decisions with embeddings - DecisionQuery, # query decisions: by_category, by_outcome, by_date_range - CausalChainAnalyzer, # trace causality: get_causal_chain, analyze_impact - PolicyEngine, # check_compliance, get_applicable_policies, enforce_policy - # Convenience functions - record_decision, # record_decision(category, scenario, reasoning, outcome, confidence) - find_precedents, # find_precedents(scenario, category, limit) - analyze_decision_impact, # analyze_decision_impact(decision_id) - check_decision_compliance, # check_decision_compliance(decision, policies) -) -``` +| Class | Role | +| --- | --- | +| `AgentContext` | Primary entry point — memory, retrieval, decisions, graph traversal, checkpoints | +| `ContextGraph` | In-memory knowledge graph with centrality, community detection, and decision tracking | +| `AgentMemory` | RAG-backed persistent memory: `store(text)`, `retrieve(query, max_results)` | +| `EntityLinker` | Link entity mentions to canonical URIs across multiple sources | +| `ContextRetriever` | Hybrid vector + graph retrieval with min-score and temporal decay options | +| `DecisionRecorder` | Record decisions with embeddings, causal chains, and metadata | +| `PolicyEngine` | Compliance checking: `check_compliance()`, `get_applicable_policies()` | +| `CausalChainAnalyzer` | Trace how decisions influenced each other: `get_causal_chain(decision_id)` | ## What You Get diff --git a/docs/reference/core.md b/docs/reference/core.md index b8f1d30d..02087102 100644 --- a/docs/reference/core.md +++ b/docs/reference/core.md @@ -8,26 +8,13 @@ icon: "gear" ## Exported Classes -```python -from semantica.core import ( - Semantica, # orchestration class — coordinates full KG pipeline - ConfigManager, # YAML config loading, deep-merge, env var overrides - LifecycleManager,# startup/shutdown state machine + health monitoring - PluginRegistry, # plugin discovery, registration, and loading - method_registry, # global MethodRegistry instance for custom dispatch -) - -# For custom build methods: -from semantica.core.methods import build_knowledge_base -``` - -## What You Get - -- **`Semantica`** — orchestration class for coordinating complex multi-module workflows -- **`ConfigManager`** — unified config loading, merging, and validation with environment variable overrides -- **`LifecycleManager`** — startup/shutdown hooks and component health monitoring -- **`PluginRegistry`** — dynamic plugin discovery, registration, and loading -- **`method_registry`** — global `MethodRegistry` instance — register and dispatch custom orchestration methods +| Class | Role | +| --- | --- | +| `Semantica` | Orchestration entry point — coordinates the full KG construction pipeline | +| `ConfigManager` | YAML config loading, deep-merge, validation, and env var overrides | +| `LifecycleManager` | Startup/shutdown state machine with health monitoring and lifecycle hooks | +| `PluginRegistry` | Dynamic plugin discovery, registration, and loading | +| `method_registry` | Global `MethodRegistry` instance — register and dispatch custom orchestration methods | **Use individual modules directly** for the vast majority of use cases. Use the `Semantica` orchestration class only when you need application-level lifecycle management or a plugin system. diff --git a/docs/reference/deduplication.md b/docs/reference/deduplication.md index ccf5a943..79e420e5 100644 --- a/docs/reference/deduplication.md +++ b/docs/reference/deduplication.md @@ -8,30 +8,27 @@ icon: "copy" ## Exported Classes -```python -from semantica.deduplication import ( - DuplicateDetector, # pairwise + batch duplicate detection - EntityMerger, # merge duplicate groups with per-property policies - SimilarityCalculator, # Levenshtein, Jaro-Winkler, cosine, Jaccard, embedding - ClusterBuilder, # Union-Find + hierarchical clustering - PropertyMergeRule, # enum: KEEP_FIRST, KEEP_LONGEST, UNION, VOTING, ... - MergeStrategyManager, # manage and apply merge strategies - # Convenience functions - detect_duplicates, # quick: detect_duplicates(entities, method="semantic_v2") - merge_entities, # quick: merge_entities(entities, duplicates, method="union") - calculate_similarity, # quick: calculate_similarity(a, b, method="hybrid_v2") - # Registry - method_registry, # register custom similarity functions -) -``` +| Class | Role | +| --- | --- | +| `DuplicateDetector` | Pairwise and batch detection with configurable strategy and threshold | +| `EntityMerger` | Merge duplicate groups with per-property merge policies (`KEEP_FIRST`, `UNION`, `VOTING`, ...) | +| `SimilarityCalculator` | Levenshtein, Jaro-Winkler, cosine, Jaccard, and embedding similarity | +| `ClusterBuilder` | Union-Find and hierarchical clustering for large-scale batch deduplication | +| `PropertyMergeRule` | Enum of merge policies used by `EntityMerger` | +| `MergeStrategyManager` | Manage and apply named merge strategies across entity types | +| `detect_duplicates()` | Quick function: `detect_duplicates(entities, method="semantic_v2")` | +| `merge_entities()` | Quick function: `merge_entities(entities, duplicates, method="union")` | -## What You Get +**Available `method=` values:** -- **`DuplicateDetector`** — pairwise and batch duplicate detection with configurable strategies -- **`EntityMerger`** — merge duplicate groups with configurable property-level merge policies -- **`SimilarityCalculator`** — multi-factor similarity: Levenshtein, Jaro-Winkler, cosine, Jaccard, embedding -- **`ClusterBuilder`** — Union-Find and hierarchical clustering for large-scale batch deduplication -- **Convenience functions** — `detect_duplicates`, `merge_entities`, `calculate_similarity` +| Method | Speed | Notes | +| --- | --- | --- | +| `exact` | Fastest | Exact string match only | +| `fuzzy` | Fast | Levenshtein + Jaro-Winkler | +| `semantic` | Moderate | Embedding cosine similarity | +| `blocking_v2` | Fast | Sorted neighborhood blocking (v2) | +| `hybrid_v2` | Balanced | Blocking + multi-feature scoring (v2) | +| `semantic_v2` | Best accuracy | Embedding + structural features, up to 7× faster than v1 | ## DuplicateDetector diff --git a/docs/reference/embeddings.md b/docs/reference/embeddings.md index 7685353b..664d03a1 100644 --- a/docs/reference/embeddings.md +++ b/docs/reference/embeddings.md @@ -21,34 +21,16 @@ Semantica uses embeddings for: ## Exported Classes -```python -from semantica.embeddings import ( - # Core generators - EmbeddingGenerator, # main handler: generate_embeddings(text, data_type="text") - TextEmbedder, # text embedding: embed(text), embed_batch(texts) - GraphEmbeddingManager, # embed KG nodes/subgraphs for GraphRAG - VectorEmbeddingManager, # embedding management for vector databases - # Provider stores - OpenAIStore, # OpenAI text-embedding-* API - BGEStore, # BAAI/bge-* via sentence-transformers - FastEmbedStore, # ONNX-accelerated, no CUDA required - LlamaStore, # Ollama local embedding models - ProviderStoreFactory, # create(provider="bge", model="...") factory - # Pooling strategies - MeanPooling, # default — best for retrieval and clustering - MaxPooling, # captures presence of any feature - CLSPooling, # CLS token (BERT-style classification models) - AttentionPooling, # softmax-weighted sum - HierarchicalPooling, # for long documents exceeding context length - PoolingStrategyFactory, # create(strategy="mean") factory - # Convenience functions - embed_text, # embed_text(text, method="sentence_transformers") - generate_embeddings, # generate_embeddings(texts, method="openai") - calculate_similarity, # calculate_similarity(a, b, method="cosine") - pool_embeddings, # pool_embeddings(token_embeddings, strategy="mean") - check_available_providers, # returns {"sentence_transformers": True, ...} -) -``` +| Class | Role | +| --- | --- | +| `EmbeddingGenerator` | Provider-agnostic entry point — handles batching, caching, and provider selection | +| `TextEmbedder` | Text embedding with disk caching and automatic batch splitting | +| `GraphEmbeddingManager` | Embed KG nodes and subgraphs for GraphRAG and distance intelligence | +| `OpenAIStore` | OpenAI `text-embedding-3-small` / `text-embedding-3-large` provider | +| `BGEStore` | BAAI/bge models via `sentence-transformers` — free, high-quality | +| `FastEmbedStore` | ONNX-accelerated local embeddings — no CUDA required | +| `LlamaStore` | Local Ollama embedding models | +| `MeanPooling` | Default pooling strategy — best for retrieval and clustering | ## What You Get diff --git a/docs/reference/evals.md b/docs/reference/evals.md index 4da8b3db..29f76af0 100644 --- a/docs/reference/evals.md +++ b/docs/reference/evals.md @@ -14,12 +14,15 @@ icon: "chart-line" When released, `semantica.evals` will provide: -- **KG quality metrics** — completeness, consistency, schema compliance, coverage, and orphan node detection -- **Extraction accuracy** — NER precision / recall / F1 and relation extraction metrics against gold-standard datasets -- **Pipeline benchmarking** — throughput (docs/sec), per-step latency, peak memory, and error rate -- **Regression tracking** — record runs and compare metrics across commits or config changes -- **Deduplication accuracy** — merge precision, false positive / false negative rates -- **Reasoning correctness** — inference accuracy, rule coverage, and derivation depth +| Planned Class | Role | +| --- | --- | +| `KGEvaluator` | Completeness, consistency, schema compliance, coverage, and orphan node detection | +| `ExtractionEvaluator` | NER precision / recall / F1 and relation extraction metrics against gold datasets | +| `PipelineBenchmark` | Throughput (docs/sec), per-step latency, peak memory, and error rate | +| `RegressionTracker` | Record runs and compare metrics across commits or config changes | +| `EvalReport` | Structured report: `{scores, regressions, recommendations}` | +| `DeduplicationEvaluator` | Merge precision, false positive / false negative rates | +| `ReasoningEvaluator` | Inference accuracy, rule coverage, and derivation depth | ## Current Workaround diff --git a/docs/reference/explorer.md b/docs/reference/explorer.md index 3174081b..9690007a 100644 --- a/docs/reference/explorer.md +++ b/docs/reference/explorer.md @@ -23,6 +23,15 @@ python -m semantica.explorer --graph my_graph.json --port 8000 --host 0.0.0.0 `semantica.explorer` is a **server process**, not a Python library. It exposes no importable classes. Use the CLI or `python -m semantica.explorer` to launch. +| CLI Flag | Default | Description | +| --- | --- | --- | +| `--graph` | required | Path to a graph JSON/pickle file to load on startup | +| `--port` | `8000` | HTTP port to listen on | +| `--host` | `127.0.0.1` | Bind address — use `0.0.0.0` to expose on the network | +| `--no-browser` | off | Suppress auto-opening the browser tab | +| `--config` | — | Path to a YAML config file for graph store and auth settings | +| `--debug` | off | Enable debug mode with hot-reload | + ## What You Get diff --git a/docs/reference/export.md b/docs/reference/export.md index 8c4d67f7..0aa7fdf2 100644 --- a/docs/reference/export.md +++ b/docs/reference/export.md @@ -8,27 +8,20 @@ icon: "file-export" ## Exported Classes -```python -from semantica.export import ( - RDFExporter, # Turtle, JSON-LD, N-Triples, RDF/XML - ParquetExporter, # columnar Parquet (Spark, BigQuery, Databricks, Snowflake) - LPGExporter, # Cypher CREATE/MERGE for Neo4j / Memgraph - ArangoAQLExporter, # AQL INSERT for ArangoDB - GraphExporter, # GraphML, GEXF, Graphviz DOT - OWLExporter, # OWL 2.0 in Turtle, XML, JSON-LD - CSVExporter, # flat CSV nodes + edges - VectorExporter, # embedding vectors as JSON, NumPy, or FAISS - ArrowExporter, # Apache Arrow IPC (zero-copy transfer) - DistanceExporter, # semantic distance matrices and ego-graphs - ReportGenerator, # human-readable analytics reports (HTML, Markdown, JSON) - NamespaceManager, # register and resolve RDF namespace prefixes - SemanticNetworkYAMLExporter, # YAML semantic network export - # Convenience functions - export_rdf, export_parquet, export_csv, export_lpg, - export_arango, export_graph, export_owl, export_vector, - export_arrow, generate_report, -) -``` +| Class | Output formats | Notes | +| --- | --- | --- | +| `RDFExporter` | Turtle, JSON-LD, N-Triples, RDF/XML | Optional PROV-O provenance embedding | +| `ParquetExporter` | `.parquet` | PyArrow-typed, Hive-partition support | +| `LPGExporter` | Cypher `CREATE`/`MERGE` | Neo4j and Memgraph compatible | +| `ArangoAQLExporter` | AQL `INSERT` | Vertex and edge collections | +| `GraphExporter` | GraphML, GEXF, Graphviz DOT | Standard graph interchange formats | +| `OWLExporter` | OWL 2.0 in Turtle/XML/JSON-LD | Full ontology serialization | +| `CSVExporter` | `.csv` | Flat nodes + edges tables | +| `VectorExporter` | JSON, NumPy `.npy`, FAISS index | Embedding vector export | +| `ArrowExporter` | Apache Arrow IPC | Zero-copy transfer to Pandas/Polars/Spark | +| `DistanceExporter` | JSON/CSV matrix | Semantic distance matrices and ego-graphs | +| `ReportGenerator` | HTML, Markdown, JSON | Human-readable analytics reports | +| `NamespaceManager` | — | Register and resolve RDF namespace prefixes | ## What You Get @@ -90,6 +83,14 @@ from semantica.export import ( +## RDFExporter Constructor Parameters + +| Parameter | Type | Default | Description | +| --------- | ---- | ------- | ----------- | +| `namespace_manager` | `NamespaceManager` | `None` | Custom namespace prefix manager | +| `include_provenance` | `bool` | `False` | Embed W3C PROV-O lineage triples | +| `provenance_manager` | `ProvenanceManager` | `None` | Provenance source when `include_provenance=True` | + ## Exporters diff --git a/docs/reference/graph_store.md b/docs/reference/graph_store.md index 819fb64a..37008913 100644 --- a/docs/reference/graph_store.md +++ b/docs/reference/graph_store.md @@ -8,35 +8,15 @@ icon: "server" ## Exported Classes -```python -from semantica.graph_store import ( - # Core interface - GraphStore, # unified interface: add_node, add_edge, query, find_paths - GraphManager, # store management and operations - NodeManager, # node CRUD operations - RelationshipManager, # relationship CRUD operations - QueryEngine, # Cypher query execution with caching - GraphAnalytics, # centrality, community detection, shortest path - # Backend stores - Neo4jStore, # Neo4j via Bolt — production workloads - ApacheAgeStore, # PostgreSQL + AGE extension - AmazonNeptuneStore, # AWS Neptune — SPARQL/Gremlin/openCypher - FalkorDBStore, # Redis-based — ultra-low latency - # Convenience functions - create_node, # create_node(labels, properties) - create_nodes, # bulk: create_nodes(entities) - create_relationship, # create_relationship(start_id, end_id, rel_type) - create_relationships, # bulk: create_relationships(rels) - get_nodes, # get_nodes(labels, filters) - get_relationships, # get_relationships(start_id, rel_type) - get_neighbors, # get_neighbors(node_id, direction="both") - update_node, # update_node(node_id, properties) - delete_node, # delete_node(node_id) - execute_query, # execute_query(cypher, parameters) - shortest_path, # shortest_path(source, target) - run_analytics, # run_analytics(graph, algorithm) -) -``` +| Class | Role | +| --- | --- | +| `GraphStore` | Unified interface: `add_node`, `add_edge`, `query`, `find_paths`, `get_neighbors` | +| `QueryEngine` | Parameterized Cypher execution with result caching and explain plans | +| `GraphAnalytics` | Centrality, community detection, shortest path, and PageRank on stored graphs | +| `Neo4jStore` | Production workloads via Bolt — supports APOC and GDS plugins | +| `ApacheAgeStore` | PostgreSQL + AGE extension — no separate graph server needed | +| `AmazonNeptuneStore` | AWS Neptune — SPARQL, Gremlin, and openCypher endpoints | +| `FalkorDBStore` | Redis-based — sub-millisecond latency for real-time applications | ## What You Get @@ -98,6 +78,19 @@ from semantica.graph_store import ( +## GraphStore Methods + +| Method | Returns | Description | +| ------ | ------- | ----------- | +| `create_nodes(entities)` | `List[str]` | Create nodes from entity list, returns node IDs | +| `add_edges(relationships)` | `List[str]` | Add edges from relationship list, returns edge IDs | +| `query(cypher, parameters)` | `List[dict]` | Execute Cypher query with optional parameters | +| `create_index(label, property)` | `None` | Create an index for faster lookups | +| `delete_node(node_id)` | `bool` | Delete a node by ID | +| `delete_edge(edge_id)` | `bool` | Delete an edge by ID | +| `get_node(node_id)` | `dict` | Retrieve a node by ID | +| `get_neighbors(node_id)` | `List[dict]` | Get all neighbors of a node | + ## Backends diff --git a/docs/reference/ingest.md b/docs/reference/ingest.md index eec13563..036f3e79 100644 --- a/docs/reference/ingest.md +++ b/docs/reference/ingest.md @@ -8,44 +8,17 @@ icon: "database" ## Exported Classes -```python -from semantica.ingest import ( - # File ingestion (always available) - FileIngestor, # local files and directories: ingest(path, recursive=True) - CloudStorageIngestor, # AWS S3, Google Cloud Storage, Azure Blob Storage - FileObject, # {content, source_id, source_type, metadata, raw_bytes} - FileTypeDetector, # auto-detect file type from extension and magic bytes - ParquetIngestor, # Apache Parquet files and partitioned datasets - XMLIngestor, # XXE-safe lxml XML parsing with optional XSD validation - # Web ingestion (requires beautifulsoup4) - WebIngestor, # web scraping: ingest_url(url), crawl(url, max_pages) - FeedIngestor, # RSS/Atom feeds: ingest_feed(url), monitor_feeds(...) - FeedMonitor, # live feed monitoring with callback on new items - # Stream ingestion - StreamIngestor, # real-time: ingest_kafka/rabbitmq/kinesis/pulsar - KafkaProcessor, # Kafka consumer group processor - RabbitMQProcessor, # AMQP queue processor - KinesisProcessor, # AWS Kinesis stream processor - PulsarProcessor, # Apache Pulsar consumer - # Repository ingestion (requires gitpython) - RepoIngestor, # Git repos: ingest(url_or_path), include_commits=True - # Email ingestion - EmailIngestor, # IMAP/POP3: ingest() with attachment extraction - # Database ingestion - DBIngestor, # SQL: ingest_database(connection_string, include_tables) - SnowflakeIngestor, # Snowflake: ingest_query(sql), ingest_table(name) - OntologyIngestor, # OWL/RDF ontology files: ingest_ontology(path) - # Convenience functions - ingest, # ingest(source, source_type="file") — unified dispatcher - ingest_file, # ingest_file(path, method="directory") - ingest_web, # ingest_web(url, method="url") - ingest_feed, # ingest_feed(url) - ingest_stream, # ingest_stream(topic, ...) - ingest_database, # ingest_database(connection_string, ...) - ingest_parquet, # ingest_parquet(path, columns=[...]) - ingest_xml, # ingest_xml(path, validate_xsd=None) -) -``` +| Class | Role | +| --- | --- | +| `FileIngestor` | PDF, DOCX, HTML, JSON, CSV, Excel, PPTX, ZIP/TAR — type auto-detected from extension | +| `WebIngestor` | Web scraping and crawling with JavaScript rendering support | +| `FeedIngestor` | RSS/Atom feed ingestion with live monitoring via `FeedMonitor` | +| `StreamIngestor` | Real-time ingestion from Kafka, RabbitMQ, AWS Kinesis, and Apache Pulsar | +| `RepoIngestor` | Git repositories — source files, commit history, README, and metadata | +| `DBIngestor` | SQL databases via SQLAlchemy — tables, views, and custom queries | +| `ParquetIngestor` | Apache Parquet files and partitioned datasets with column selection | +| `XMLIngestor` | XXE-safe XML parsing with optional XSD schema validation | +| `ingest()` | Unified dispatcher — detects type automatically from source path or URL | ## What You Get @@ -363,6 +336,27 @@ from semantica.ingest import ( +## Convenience Function + +| Parameter | Type | Default | Description | +| --------- | ---- | ------- | ----------- | +| `source` | `str` | required | File path, directory, URL, or connection string | +| `source_type` | `str` | `"auto"` | `"file"`, `"web"`, `"db"`, `"stream"`, `"feed"`, `"repo"` — auto-detected from path if omitted | +| `recursive` | `bool` | `False` | Scan subdirectories for file-based sources | +| `metadata` | `dict` | `{}` | Extra metadata attached to every returned `DataSource` | + +Returns `List[DataSource]` — each item has `content`, `metadata`, `source_id`, and `source_type`. + +## DataSource Fields + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `content` | `str` | Extracted or loaded text content | +| `metadata` | `dict` | Title, author, URL, date, page count, etc. | +| `source_id` | `str` | Unique identifier for this source | +| `source_type` | `str` | `"file"`, `"web"`, `"database"`, `"stream"`, ... | +| `raw_bytes` | `Optional[bytes]` | Original binary content (if available) | + ## OntologyIngestor Ingest existing OWL or RDF ontology files as structured knowledge sources: diff --git a/docs/reference/kg.md b/docs/reference/kg.md index ec928734..e01808ab 100644 --- a/docs/reference/kg.md +++ b/docs/reference/kg.md @@ -8,40 +8,20 @@ icon: "diagram-project" ## Exported Classes -```python -from semantica.kg import ( - KnowledgeGraph, # core graph data structure - GraphBuilder, # construct from entities + relationships - GraphBuilderWithProvenance, # auto-tracks provenance for every node/edge - EntityResolver, # entity deduplication during construction - GraphAnalyzer, # temporal evolution, diversity metrics - GraphValidator, # schema and constraint validation - TemporalGraphQuery, # point-in-time snapshots, diffs, interval queries - TemporalPatternDetector, # sequence/cycle/trend detection - TemporalVersionManager, # snapshot creation and version comparison - TemporalNormalizer, # normalize timestamps across granularities - BiTemporalFact, # bi-temporal fact model (transaction + valid time) - CentralityCalculator, # degree, betweenness, closeness, PageRank, eigenvector - CommunityDetector, # Louvain, Leiden, Label Propagation, K-Clique - PathFinder, # Dijkstra, A*, BFS, K-Shortest paths - LinkPredictor, # Preferential Attachment, Jaccard, Adamic-Adar - NodeEmbedder, # Node2Vec, DeepWalk structural embeddings - SimilarityCalculator, # cosine, Euclidean, Manhattan, correlation similarity - ConnectivityAnalyzer, # connected components, bridges, density - ProvenanceTracker, # source tracking and lineage management -) -``` - -## What You Get - -- **`GraphBuilder`** — construct graphs from entities and relationships with automatic entity merging -- **`TemporalGraphQuery`** — time-aware point-in-time snapshots, diffs, and Allen interval queries (v0.4.0) -- **`SimilarityCalculator`** — cosine, Euclidean, Manhattan, and correlation similarity scoring -- **`CentralityCalculator`** — PageRank, degree, betweenness, closeness, eigenvector centrality -- **`CommunityDetector`** — Louvain, Leiden, Label Propagation, K-Clique community detection -- **`PathFinder`** — Dijkstra, A\*, BFS, K-Shortest path algorithms -- **`LinkPredictor`** — Preferential Attachment, Jaccard, Adamic-Adar link prediction -- **`NodeEmbedder`** — Node2Vec, DeepWalk structural embeddings +| Class | Role | +| --- | --- | +| `KnowledgeGraph` | Core graph data structure — nodes, edges, properties, temporal validity | +| `GraphBuilder` | Construct from entities + relationships with automatic entity merging | +| `GraphBuilderWithProvenance` | Drop-in replacement that auto-tracks provenance for every node and edge | +| `EntityResolver` | Entity deduplication and merging during graph construction | +| `TemporalGraphQuery` | Point-in-time snapshots, temporal diffs, and all 13 Allen interval queries | +| `CentralityCalculator` | PageRank, degree, betweenness, closeness, eigenvector centrality | +| `CommunityDetector` | Louvain, Leiden, Label Propagation, and K-Clique community detection | +| `PathFinder` | Dijkstra, A*, BFS, and K-Shortest path algorithms | +| `LinkPredictor` | Preferential Attachment, Jaccard, Adamic-Adar link prediction | +| `NodeEmbedder` | Node2Vec, DeepWalk structural embeddings for downstream ML | +| `SimilarityCalculator` | Cosine, Euclidean, Manhattan, and correlation similarity scoring | +| `GraphValidator` | Schema and constraint validation before persistence | For conflict detection and advanced entity resolution, use `semantica.conflicts` and `semantica.deduplication` alongside this module. diff --git a/docs/reference/normalize.md b/docs/reference/normalize.md index 0d1abba2..2fae5436 100644 --- a/docs/reference/normalize.md +++ b/docs/reference/normalize.md @@ -19,47 +19,14 @@ Normalization collapses these variants before any extractor, deduplicator, or gr ## Exported Classes -```python -from semantica.normalize import ( - # Text normalization - TextNormalizer, # coordinator: strip_html, normalize_unicode, fix_encoding - UnicodeNormalizer, # NFC/NFD/NFKC/NFKD normalization - WhitespaceNormalizer, # collapse spaces, normalize line endings - SpecialCharacterProcessor, # smart quotes, dashes, diacritics - TextCleaner, # general text cleaning utilities - # Entity normalization - EntityNormalizer, # coordinator: normalize_entity(text, entity_type) - AliasResolver, # resolve "ML" -> "Machine Learning" via dictionary - EntityDisambiguator, # disambiguate("Apple", context=...) with confidence - NameVariantHandler, # normalize("Dr. JOHN P. SMITH Jr.") -> "John P. Smith" - # Date/time normalization - DateNormalizer, # normalize_date(str) -> ISO 8601 - TimeZoneNormalizer, # normalize to UTC or target timezone - RelativeDateProcessor, # "3 days ago" -> datetime - TemporalExpressionParser, # "Q2 2023" -> {start, end, type} - # Number normalization - NumberNormalizer, # normalize_number("$1.2B") -> 1200000000.0 - UnitConverter, # convert(100, from_unit="km/h", to_unit="m/s") - CurrencyNormalizer, # normalize("$42.50") -> {amount, currency, raw} - ScientificNotationHandler, # parse scientific notation strings - # Data cleaning - DataCleaner, # remove_duplicates, fill_missing - DataValidator, # validate(records, schema={"name": str, "age": int}) - DuplicateDetector, # detect duplicate records by similarity threshold - MissingValueHandler, # fill missing values: mean/median/mode/constant - # Language & encoding - LanguageDetector, # detect(text) -> {language, confidence} - EncodingHandler, # detect_encoding, to_utf8, remove_bom - # Convenience functions - normalize_text, # normalize_text(text, method="default") - normalize_entity, # normalize_entity(name, entity_type="Person") - normalize_date, # normalize_date("Jan 1st, 2020") - normalize_number, # normalize_number("$1,234.56") - clean_text, # clean_text(text) - detect_language, # detect_language(text) - resolve_aliases, # resolve_aliases(text, aliases_dict) -) -``` +| Class | Role | +| --- | --- | +| `TextNormalizer` | Unicode forms (NFC/NFKC), whitespace collapse, HTML stripping, smart-quote and dash normalization | +| `EntityNormalizer` | Corporate suffixes, honorifics, alias resolution, and entity disambiguation | +| `DateNormalizer` | Parses any date string format → ISO 8601; handles relative dates and fiscal quarters | +| `NumberNormalizer` | `"$1.2B"` → `1200000000.0`; unit conversion (`km/h` → `m/s`); currency parsing | +| `DataCleaner` | Remove duplicates, fill missing values, validate records against a schema | +| `LanguageDetector` | `detect(text)` → `{language, confidence}` using statistical n-gram models | ## What You Get @@ -157,6 +124,18 @@ num = normalize_number("$1.2B") # → 1200000000.0 lang = detect_language("Bonjour le monde") # → {"language": "fr", "confidence": 0.98} ``` +## TextNormalizer Constructor Parameters + +| Parameter | Type | Default | Description | +| --------- | ---- | ------- | ----------- | +| `lowercase` | `bool` | `False` | Convert to lowercase | +| `remove_punctuation` | `bool` | `False` | Strip all punctuation | +| `remove_extra_whitespace` | `bool` | `True` | Collapse tabs, newlines, non-breaking spaces | +| `strip_html` | `bool` | `False` | Remove HTML tags and decode entities | +| `normalize_unicode` | `bool` | `True` | Apply Unicode normal form | +| `fix_encoding` | `bool` | `True` | Repair cp1252/latin-1 mojibake | +| `form` | `str` | `"NFC"` | Unicode form: `"NFC"` / `"NFD"` / `"NFKC"` / `"NFKD"` | + ## Normalizers diff --git a/docs/reference/ontology.md b/docs/reference/ontology.md index bd7a3f2c..07602b76 100644 --- a/docs/reference/ontology.md +++ b/docs/reference/ontology.md @@ -8,38 +8,18 @@ icon: "sitemap" ## Exported Classes -```python -from semantica.ontology import ( - OntologyGenerator, # auto-generate from KG data (6-stage pipeline) - LLMOntologyGenerator, # LLM-powered ontology generation - OntologyEngine, # unified orchestration facade - ClassInferrer, # class discovery and hierarchy building - PropertyGenerator, # property inference and XSD type mapping - SHACLGenerator, # generate SHACL shapes from ontology - OntologyValidator, # validate graphs against SHACL shapes - SHACLValidationReport, # validation report with violations list - SHACLViolation, # individual constraint violation - OWLGenerator, # OWL/RDF serialization (Turtle, XML, JSON-LD) - OntologyEvaluator, # quality evaluation: coverage, completeness - NamespaceManager, # IRI generation and namespace prefix management - OntologyAligner, # align and merge ontologies across schemas (use OntologyEngine) - AssociativeClassBuilder, # N-ary relationship intermediate class creation - NamingConventions, # PascalCase/camelCase enforcement - DomainOntologies, # pre-built domain ontologies - ingest_ontology, # load ontology from file -) -``` - -## What You Get - -- **`OntologyGenerator`** — auto-generate ontologies from existing knowledge graph data (6-stage pipeline) -- **`LLMOntologyGenerator`** — LLM-powered ontology generation for complex domains -- **`OntologyEngine`** — unified facade that orchestrates the full ontology lifecycle -- **`SHACLGenerator`** / **`OntologyValidator`** — generate SHACL shapes and validate any graph -- **`OWLGenerator`** — serialize ontologies to Turtle, RDF/XML, JSON-LD -- **`NamespaceManager`** — IRI generation, prefix management, namespace binding -- **`OntologyEvaluator`** — coverage, completeness, and granularity quality metrics -- **`AssociativeClassBuilder`** — model N-ary relationships as intermediate OWL classes +| Class | Role | +| --- | --- | +| `OntologyEngine` | Unified facade orchestrating the full ontology lifecycle | +| `OntologyGenerator` | Auto-generate ontologies from KG data (6-stage pipeline) | +| `LLMOntologyGenerator` | LLM-powered ontology generation for complex domains | +| `SHACLGenerator` | Generate SHACL shapes from an ontology or KG schema | +| `OntologyValidator` | Validate any graph against SHACL shapes — returns `SHACLValidationReport` | +| `OWLGenerator` | Serialize ontologies to Turtle, RDF/XML, JSON-LD | +| `NamespaceManager` | IRI generation, prefix management, and namespace binding | +| `OntologyEvaluator` | Coverage, completeness, and granularity quality metrics | +| `OntologyAligner` | Align and merge ontologies across schemas | +| `AssociativeClassBuilder` | Model N-ary relationships as intermediate OWL classes | ## OntologyEngine (Unified Facade) @@ -63,6 +43,16 @@ if not report.conforms: engine.export(ontology, "ontology.ttl", format="turtle") ``` +### OntologyEngine Methods + +| Method | Description | +| ------ | ----------- | +| `generate_ontology(data)` | Run the 6-stage pipeline on entity/relationship data | +| `validate(kg)` | Check a knowledge graph against generated SHACL shapes | +| `export(ontology, path, format)` | Serialize to `"turtle"`, `"xml"`, or `"json-ld"` | +| `align(other_ontology)` | Align and merge with another ontology | +| `evaluate(ontology, kg)` | Compute coverage, completeness, and granularity metrics | + ## OntologyGenerator (6-Stage Pipeline) Generate a formal ontology automatically from your knowledge graph entities and relationships: @@ -110,6 +100,17 @@ if not report.conforms: print(f" Path: {violation.path}") ``` +### Validation Report Fields + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `conforms` | `bool` | `True` if the graph passes all SHACL constraints | +| `violations` | `List[SHACLViolation]` | Detailed failure records | +| `severity` | `str` | `"violation"`, `"warning"`, or `"info"` | +| `message` | `str` | Human-readable constraint failure description | +| `node` | `str` | IRI of the violating graph node | +| `path` | `str` | IRI of the violating property path | + ## LLM-Powered Ontology Generation For complex or novel domains where schema patterns are hard to infer statistically: diff --git a/docs/reference/parse.md b/docs/reference/parse.md index 6aa9830a..68e689ab 100644 --- a/docs/reference/parse.md +++ b/docs/reference/parse.md @@ -8,35 +8,16 @@ icon: "file-lines" ## Exported Classes -```python -from semantica.parse import ( - DocumentParser, # auto-detect format — delegates to format-specific parser - PDFParser, # PDF text extraction - DOCXParser, # Word .docx documents - HTMLParser, # HTML / web pages - MarkdownParser, # Markdown files - TXTParser, # plain text - JSONParser, # JSON documents - XMLParser, # XML documents - CSVParser, # CSV / TSV files - WebParser, # URL fetch + HTML parsing - EmailParser, # .eml / .msg email files - CodeParser, # source code files - # Data types - ParsedDocument, # {text, sections, tables, metadata, source_id} - DocumentMetadata, # {title, author, created_date, page_count, language, ...} -) - -# Optional — requires: pip install "semantica[docling]" -from semantica.parse import DoclingParser # advanced OCR + layout analysis -``` - -## What You Get - -- **`DocumentParser`** — standard parser for PDF, DOCX, HTML, TXT, JSON, CSV, PPTX, XLSX — auto-detects format -- **`DoclingParser`** — advanced parser for complex layouts, merged-cell tables, multi-column PDFs, and OCR (optional dep) -- **`ParsedDocument`** — structured output with `text`, `sections`, `tables`, and `metadata` -- **Format-specific parsers** — `PDFParser`, `DOCXParser`, `HTMLParser`, `WebParser`, `EmailParser`, `CodeParser`, etc. +| Class | Role | +| --- | --- | +| `DocumentParser` | Auto-detects format — delegates to format-specific parser (PDF, DOCX, HTML, JSON, CSV, ...) | +| `DoclingParser` | Complex layouts, merged-cell tables, multi-column PDFs, and OCR (`pip install semantica[docling]`) | +| `ParsedDocument` | `{text, sections, tables, metadata, source_id}` — structured output from any parser | +| `DocumentMetadata` | `{title, author, created_date, page_count, language, word_count}` | +| `PDFParser` | PDF text and metadata extraction | +| `WebParser` | URL fetch + HTML parsing | +| `EmailParser` | `.eml` / `.msg` email files with attachment extraction | +| `CodeParser` | Source code files with syntax-aware block detection | ## DocumentParser @@ -100,6 +81,24 @@ parser = DoclingParser( parsed = parser.parse("data/scanned_contract.pdf") ``` +## Supported Formats + +| Format | Extension | Parser Used | Notes | +| ------ | --------- | ----------- | ----- | +| PDF | `.pdf` | `PDFParser` / `DoclingParser` | Text, tables, metadata; Docling adds OCR | +| Word | `.docx` | Built-in | Text, headings, tables, metadata | +| HTML | `.html`, `.htm` | `HTMLParser` / `WebParser` | `WebParser` fetches remote URLs | +| Markdown | `.md` | Built-in | Preserves heading hierarchy | +| Plain text | `.txt` | `TXTParser` | Minimal metadata | +| JSON | `.json` | `JSONParser` | One object per line or array | +| CSV / TSV | `.csv`, `.tsv` | `CSVParser` | Header auto-detected | +| Excel | `.xlsx`, `.xls` | Built-in | Sheet selection supported | +| PowerPoint | `.pptx` | Built-in | `DoclingParser` for embedded charts | +| Email | `.eml`, `.msg` | `EmailParser` | Attachments extracted | +| XML | `.xml` | `XMLIngestor` | XXE-safe, optional XSD validation | +| Archive | `.zip`, `.tar` | `FileIngestor` | Recursive extraction | +| Source code | `.py`, `.js`, `.java`, ... | `CodeParser` | AST-aware block detection | + ## Parsed Document Object Both parsers return a `ParsedDocument` with the same structure: @@ -126,6 +125,14 @@ class DocumentMetadata: format: str # "pdf" | "docx" | "pptx" | ... ``` +## DocumentParser Methods + +| Method | Returns | Description | +| ------ | ------- | ----------- | +| `parse(source)` | `ParsedDocument` | Auto-detect format and extract text, sections, metadata | +| `parse_batch(sources)` | `List[ParsedDocument]` | Process multiple sources in parallel | +| `is_supported(path)` | `bool` | Check if the file extension is supported | + ## Integration with FileIngestor The most common pattern — ingest a directory then parse each source: diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md index 10194d9f..bc61d915 100644 --- a/docs/reference/pipeline.md +++ b/docs/reference/pipeline.md @@ -8,39 +8,15 @@ icon: "gear" ## Exported Classes -```python -from semantica.pipeline import ( - # Pipeline construction - PipelineBuilder, # DSL: add_step, connect_steps, build - Pipeline, # pipeline definition dataclass - PipelineStep, # step definition: name, step_type, handler, dependencies - StepStatus, # enum: PENDING, RUNNING, COMPLETED, FAILED, SKIPPED - PipelineSerializer, # serialize/deserialize pipeline to JSON/YAML - # Execution - ExecutionEngine, # execute_pipeline(pipeline, data) -> ExecutionResult - ExecutionResult, # {success, output, metadata, metrics, errors} - PipelineStatus, # enum: RUNNING, PAUSED, STOPPED - ProgressTracker, # get_progress(pipeline_id) -> {completed, total, pct} - # Failure handling - FailureHandler, # configure strategy: skip/retry/abort - RetryHandler, # retry with exponential backoff - FallbackHandler, # fall back to alternative step on failure - RetryPolicy, # {max_retries, backoff, jitter} - RetryStrategy, # enum: FIXED, EXPONENTIAL, LINEAR - ErrorSeverity, # enum: LOW, MEDIUM, HIGH, CRITICAL - # Parallelism - ParallelismManager, # execute_parallel(tasks, timeout) — thread or process pool - ParallelExecutionResult, # {success, result, error, task_id} - # Resource management - ResourceScheduler, # allocate_resources / release_resources - ResourceType, # enum: CPU, MEMORY, GPU, NETWORK, DISK - # Validation - PipelineValidator, # validate_pipeline(pipeline) -> ValidationResult - # Templates - PipelineTemplateManager, # get_template("full-qa") -> pre-wired Pipeline - PipelineTemplate, # template metadata dataclass -) -``` +| Class | Role | +| --- | --- | +| `PipelineBuilder` | DSL for wiring steps: `add_step`, `connect_steps`, `set_parallel`, `build` | +| `ExecutionEngine` | Runs a built pipeline: `execute_pipeline(pipeline, data)` → `ExecutionResult` | +| `ExecutionResult` | `{success, output, metadata, metrics, errors}` — full run summary | +| `FailureHandler` | Per-step strategy: `skip`, `retry`, `abort`, or `fallback` on failure | +| `ParallelismManager` | Thread or process pool for concurrent step execution with configurable workers | +| `PipelineValidator` | Catches dependency cycles, missing handlers, and config errors before running | +| `PipelineTemplateManager` | Pre-built templates: `"full-qa"`, `"extract-only"`, `"kg-build"` | ## Why Use a Pipeline? diff --git a/docs/reference/provenance.md b/docs/reference/provenance.md index d83ddb3f..6702df62 100644 --- a/docs/reference/provenance.md +++ b/docs/reference/provenance.md @@ -8,31 +8,15 @@ icon: "link" ## Exported Classes -```python -from semantica.provenance import ( - ProvenanceManager, # track entities, get lineage, export PROV-O - ProvenanceEntry, # single provenance record (entity_id, source, method, ...) - SourceReference, # rich source pointer (DOI, page, quote, URL) - ProvenanceStorage, # abstract storage backend - InMemoryStorage, # default in-memory backend - SQLiteStorage, # persistent SQLite backend for production - compute_checksum, # compute tamper-evident hash for an entry - verify_checksum, # verify integrity of a stored entry -) - -# GraphBuilderWithProvenance is in semantica.kg, not semantica.provenance -from semantica.kg import GraphBuilderWithProvenance -``` - -## What You Get - -- **`ProvenanceManager`** — track entities and relationships with source attribution and lineage retrieval -- **`ProvenanceEntry`** / **`SourceReference`** — structured records with DOI, page, quote, confidence, and timestamp -- **`InMemoryStorage`** / **`SQLiteStorage`** — swappable persistence backends -- **`compute_checksum`** / **`verify_checksum`** — tamper-evident integrity verification -- **Lineage graph** — full upstream lineage from any entity back to its source document -- **W3C PROV-O export** — serialize lineage as Turtle RDF or JSON-LD for compliance reporting -- **`GraphBuilderWithProvenance`** (in `semantica.kg`) — drop-in replacement that auto-tracks every node and edge +| Class | Role | +| --- | --- | +| `ProvenanceManager` | Track entities and get lineage: `track_entity`, `get_lineage`, `export_provenance` | +| `ProvenanceEntry` | Single record: `{entity_id, source, method, confidence, timestamp, checksum}` | +| `SourceReference` | Rich source pointer: `{url, doi, page, quote, author, publication_date}` | +| `InMemoryStorage` | Default backend — fast, not persisted across restarts | +| `SQLiteStorage` | Production backend — persists to a local SQLite file | +| `compute_checksum()` | Returns SHA-256 fingerprint of a provenance entry | +| `verify_checksum()` | Detects tampering by comparing stored vs recomputed hash | ## ProvenanceManager @@ -92,6 +76,16 @@ manager.track_entity( ) ``` +## ProvenanceManager Methods + +| Method | Returns | Description | +| ------ | ------- | ----------- | +| `track_entity(entity_id, source_reference, confidence)` | `str` | Record a provenance entry, returns entry ID | +| `get_lineage(entity_id)` | `ProvenanceEntry` | Retrieve full lineage for an entity | +| `export_prov_o(entity_id, format)` | `str` | Export single entity as W3C PROV-O Turtle/JSON-LD | +| `export_all(path, format)` | `None` | Export full provenance graph to file | +| `verify_checksum(entry, checksum)` | `bool` | Verify entry hasn't been tampered with | + ## Tamper-Evident Checksums Verify that provenance records have not been modified after creation: diff --git a/docs/reference/reasoning.md b/docs/reference/reasoning.md index 203c9f46..d357a68c 100644 --- a/docs/reference/reasoning.md +++ b/docs/reference/reasoning.md @@ -8,42 +8,17 @@ icon: "microchip" ## Exported Classes -```python -from semantica.reasoning import ( - # Engines - Reasoner, # IF/THEN forward-chaining facade - GraphReasoner, # inference over full KG structure - ReteEngine, # high-performance Rete pattern matching - SPARQLReasoner, # SPARQL-based RDF inference - DatalogReasoner, # recursive Horn clause fixpoint evaluation - TemporalReasoningEngine, # Allen interval algebra (13 relations) - ExplanationGenerator, # structured step-by-step explanations - # Data types - Rule, # IF/THEN rule definition - Fact, # base fact (subject, predicate, obj) - RuleType, # enum: FORWARD_CHAIN, BACKWARD_CHAIN, ... - InferenceResult, # result of infer() — contains derived_facts list - DatalogFact, # Datalog base fact (predicate, args tuple) - DatalogRule, # Datalog Horn clause ("head :- body.") - TemporalInterval, # time interval with start/end - IntervalRelation, # enum of 13 Allen relations - # Explanation types - Explanation, # conclusion + confidence + reasoning_path - ReasoningPath, # ordered list of ReasoningSteps - ReasoningStep, # single step: fact + rule_name + depth - Justification, # full justification record -) -``` - -## What You Get - -- **`Reasoner`** — main facade for IF/THEN forward-chaining with variable substitution -- **`GraphReasoner`** — inference over full knowledge graph structure (transitivity, symmetry, inverses) -- **`ReteEngine`** — high-performance pattern matching via the Rete algorithm for large rule sets -- **`SPARQLReasoner`** — query expansion and property chain inference over RDF graphs -- **`DatalogReasoner`** — recursive Horn clause rules with guaranteed fixpoint termination (v0.4.0) -- **`TemporalReasoningEngine`** — all 13 Allen interval algebra relations for time-aware inference -- **`ExplanationGenerator`** — structured explanation paths for every derived conclusion +| Class | Role | +| --- | --- | +| `Reasoner` | IF/THEN forward-chaining facade with variable substitution | +| `GraphReasoner` | Inference over full KG structure (transitivity, symmetry, inverses, property chains) | +| `ReteEngine` | High-performance Rete pattern matching for large rule sets | +| `SPARQLReasoner` | Query expansion and property chain inference over RDF graphs | +| `DatalogReasoner` | Recursive Horn clause rules with guaranteed fixpoint termination | +| `TemporalReasoningEngine` | All 13 Allen interval algebra relations for time-aware inference | +| `ExplanationGenerator` | Structured step-by-step explanations with confidence and reasoning path | +| `Rule` | IF/THEN rule definition: `{conditions, actions, confidence, rule_type}` | +| `InferenceResult` | Result of `infer()` — contains `derived_facts` and metadata | ## Quick Start diff --git a/docs/reference/seed.md b/docs/reference/seed.md index 47174346..631e5e6c 100644 --- a/docs/reference/seed.md +++ b/docs/reference/seed.md @@ -8,13 +8,11 @@ icon: "database" ## Exported Classes -```python -from semantica.seed import ( - SeedDataManager, # coordinator: register_source, create_foundation_graph, integrate_with_extracted - SeedDataSource, # {name, source_type, path, config} — dataclass for a registered source - SeedData, # {entities, relationships, metadata} — loaded seed data container -) -``` +| Class | Role | +| --- | --- | +| `SeedDataManager` | Coordinator: `register_source`, `load_all`, `create_foundation_graph`, `integrate_with_extracted` | +| `SeedDataSource` | Config dataclass: `{name, source_type, path, config}` — one per registered source | +| `SeedData` | Loaded data container: `{entities, relationships, metadata}` returned by `load_all` | ## What You Get diff --git a/docs/reference/semantic_extract.md b/docs/reference/semantic_extract.md index c1efe0b8..ca7f596a 100644 --- a/docs/reference/semantic_extract.md +++ b/docs/reference/semantic_extract.md @@ -8,38 +8,21 @@ icon: "magnifying-glass-chart" ## Exported Classes -```python -from semantica.semantic_extract import ( - # Primary extractors - NamedEntityRecognizer, # full NER coordinator (confidence_threshold, merge_overlapping) - NERExtractor, # core NER implementation used by NamedEntityRecognizer - RelationExtractor, # typed relationship extraction - TripletExtractor, # (subject, predicate, object) triplet generation - EventDetector, # event detection with participants and temporal context - CoreferenceResolver, # resolve pronouns and aliases to canonical entities - # Data types - Entity, # {id, text, type, confidence, start, end} - Relation, # {subject, predicate, object, confidence} - Event, # {type, participants, temporal, location, confidence} - CoreferenceChain, # list of mentions resolving to the same entity - # Advanced - EntityClassifier, # classify entity candidates by type - CustomEntityDetector, # pattern/dictionary-based custom entity detection - TemporalEventProcessor, # extract temporal information from events -) -``` - `NamedEntityRecognizer` is the high-level coordinator with confidence thresholding and overlap merging. `NERExtractor` is the lower-level implementation. For most use cases, start with `NERExtractor` for simplicity or `NamedEntityRecognizer` for fine-grained control. -## What You Get - -- **`NERExtractor`** / **`NamedEntityRecognizer`** — named entity recognition: Person, Organization, Location, Date, and custom types -- **`RelationExtractor`** — typed semantic relationships between entities (`founded_by`, `located_in`, etc.) -- **`TripletExtractor`** — direct `(subject, predicate, object)` triplet generation for RDF-ready output -- **`EventDetector`** — event detection with participants, temporal context, and confidence scores -- **`CoreferenceResolver`** — resolve "Apple" and "the company" to the same entity across a document +| Class | Role | +| --- | --- | +| `NamedEntityRecognizer` | High-level NER with confidence thresholding and overlap merging | +| `NERExtractor` | Core NER implementation — use directly for simplicity | +| `RelationExtractor` | Typed relationship extraction (`founded_by`, `located_in`, ...) | +| `TripletExtractor` | Direct `(subject, predicate, object)` triplet generation for RDF output | +| `EventDetector` | Event detection with participants, temporal context, and confidence scores | +| `CoreferenceResolver` | Resolve "Apple" and "the company" to the same canonical entity | +| `Entity` | `{id, text, type, confidence, start, end}` | +| `Relation` | `{subject, predicate, object, confidence}` | +| `Event` | `{type, participants, temporal, location, confidence}` | ## Quick Start @@ -58,6 +41,13 @@ triplets = TripletExtractor(method="llm", llm_provider=llm).extract(text) Semantic extraction pipeline: raw text fans into NER, Relation, and Coreference extractors, then merges into a Triplet Generator +## Extractor Methods + +| Method | Returns | Description | +| ------ | ------- | ----------- | +| `extract(text)` | `List[Entity]` / `List[Relation]` / `List[Triplet]` / `List[Event]` | Extract from single text input | +| `extract_batch(texts, batch_size)` | `List[List[...]]` | Process multiple texts in parallel | + ## NERExtractor ```python diff --git a/docs/reference/split.md b/docs/reference/split.md index fbfda6dc..e14d0b12 100644 --- a/docs/reference/split.md +++ b/docs/reference/split.md @@ -18,34 +18,27 @@ Semantica's chunking methods are designed to avoid these failure modes. ## Exported Classes -```python -from semantica.split import ( - # Unified splitter (start here) - TextSplitter, # method=: recursive, sentence, token, semantic_transformer, - # entity_aware, relation_aware, code, structural, markdown - Splitter, # alias for TextSplitter (backward compat) - # Data type - Chunk, # {text, start_char, end_char, token_count, metadata, entities, relationships} - # Specialized chunkers - SemanticChunker, # embedding-based semantic boundary detection - StructuralChunker, # heading/section-based splits from ParsedDocument - SlidingWindowChunker, # fixed-size sliding window with overlap - TableChunker, # table-specific chunking - EntityAwareChunker, # KG: preserves named entities across chunk boundaries - RelationAwareChunker, # KG: keeps subject-predicate-object triplets intact - GraphBasedChunker, # splits based on graph community structure - OntologyAwareChunker, # splits respecting ontology concept boundaries - HierarchicalChunker, # multi-level hierarchical chunking - ProvenanceTracker, # track chunk provenance back to source document - # Convenience split functions - split_recursive, # split_recursive(text, chunk_size, chunk_overlap) - split_by_sentences, # split_by_sentences(text) - split_by_tokens, # split_by_tokens(text, chunk_size, tokenizer) - split_semantic_transformer, # split_semantic_transformer(text, threshold) - split_entity_aware, # split_entity_aware(text, entities) - split_relation_aware, # split_relation_aware(text, relationships) -) -``` +| Class | Role | +| --- | --- | +| `TextSplitter` | Unified entry point — swap `method=` without changing downstream code | +| `Chunk` | `{text, start_char, end_char, token_count, metadata, entities, relationships}` | +| `SemanticChunker` | Embedding-based topic-shift detection — splits only when content actually changes | +| `StructuralChunker` | Heading/section-based splits from a `ParsedDocument` | +| `EntityAwareChunker` | Prevents named entity mentions from being split across chunk boundaries | +| `RelationAwareChunker` | Keeps subject-predicate-object triplets intact within a single chunk | +| `HierarchicalChunker` | Multi-level chunking producing parent/child chunk relationships | + +**Available `method=` values for `TextSplitter`:** + +| Method | Best for | +| --- | --- | +| `recursive` | General text — splits on paragraphs, sentences, words in order | +| `sentence` | Conversational text, QA | +| `token` | LLM context window enforcement | +| `semantic_transformer` | Long documents with topic shifts | +| `entity_aware` | KG extraction pipelines | +| `code` | Source code files | +| `structural` | PDFs and DOCX with heading hierarchy | ## What You Get diff --git a/docs/reference/triplet_store.md b/docs/reference/triplet_store.md index 63f9dfe7..b232c1ca 100644 --- a/docs/reference/triplet_store.md +++ b/docs/reference/triplet_store.md @@ -8,27 +8,14 @@ icon: "table" ## Exported Classes -```python -from semantica.triplet_store import ( - # Core interface - TripletStore, # unified: add_triplet, get_triplets, execute_query, bulk_load - QueryEngine, # SPARQL execution: execute_query, optimize_query, plan_query - BulkLoader, # high-volume loading with progress tracking and transaction support - # Backend stores - BlazegraphStore, # Blazegraph REST API (HTTP/HTTPS, Named Graphs, SPARQL 1.1) - JenaStore, # Apache Jena Fuseki (SPARQL 1.1, TDB2, GeoSPARQL) - RDF4JStore, # Eclipse RDF4J (SailRepository, in-memory or native) - # Convenience functions - add_triplet, # add_triplet(subject, predicate, obj) - add_triplets, # bulk: add_triplets(triplets) - get_triplets, # get_triplets(subject=None, predicate=None, obj=None) - delete_triplet, # delete_triplet(subject, predicate, obj) - execute_query, # execute_query(sparql, result_format="json") - optimize_query, # optimize_query(sparql) -> optimized SPARQL string - bulk_load, # bulk_load(file_path, format="turtle") - validate_triplets,# validate_triplets(triplets) -> ValidationResult -) -``` +| Class | Role | +| --- | --- | +| `TripletStore` | Unified interface: `add_triplet`, `get_triplets`, `delete_triplet`, `execute_query`, `bulk_load` | +| `QueryEngine` | SPARQL 1.1 execution with query optimization and result streaming | +| `BulkLoader` | High-volume RDF loading with progress tracking and transaction batching | +| `BlazegraphStore` | Blazegraph REST API — Named Graphs, SPARQL 1.1 Update, GeoSPARQL | +| `JenaStore` | Apache Jena Fuseki — TDB2 backend, GeoSPARQL, SPARQL 1.1 | +| `RDF4JStore` | Eclipse RDF4J — SailRepository, in-memory or native store | ## What You Get @@ -198,6 +185,23 @@ results = store.sparql(""" """) ``` +## TripletStore Methods + +| Method | Returns | Description | +| ------ | ------- | ----------- | +| `add_triplet(s, p, o, graph=None)` | `str` | Add a single triplet, returns triplet ID | +| `add_triplets_bulk(triplets)` | `List[str]` | Batch add triplets with transaction support | +| `get_triplets(graph=None)` | `List[dict]` | Retrieve all triplets or from a named graph | +| `delete_triplet(triplet_id)` | `bool` | Delete a triplet by ID | +| `sparql(query)` | `List[dict]` | Execute SPARQL SELECT query | +| `sparql_construct(query)` | `Graph` | Execute SPARQL CONSTRUCT query | +| `sparql_ask(query)` | `bool` | Execute SPARQL ASK query | +| `sparql_update(query)` | `None` | Execute SPARQL UPDATE (INSERT/DELETE) | +| `bulk_load(file, format)` | `None` | Load RDF file (turtle, nt, xml) | +| `export(path, format)` | `None` | Export to turtle, nt, xml | +| `list_graphs()` | `List[str]` | List all named graphs | +| `clear_graph(graph_uri)` | `None` | Delete all triples from a named graph | + ## SPARQL Queries ```python diff --git a/docs/reference/utils.md b/docs/reference/utils.md index c123bdfd..25f64773 100644 --- a/docs/reference/utils.md +++ b/docs/reference/utils.md @@ -8,28 +8,20 @@ icon: "wrench" ## Exported Classes -```python -from semantica.utils import ( - # Logging - setup_logging, # configure root logger — level, format (json/text) - get_logger, # get a named logger instance - log_performance, # @decorator — logs function name, duration, exception - # Validation - validate_entity, # validate entity dict structure, raises ValidationError - validate_config, # validate config dict against schema, raises ValidationError - # Progress tracking - ProgressTracker, # class-based tracker with ETA - track_progress, # wraps any iterable with live progress bar - # Helpers - clean_text, # normalize whitespace, strip control characters - hash_data, # deterministic SHA-256 hash of any serializable object - safe_filename, # sanitize a string for use as a filename - # Exceptions - SemanticaError, # base exception for all Semantica errors - ValidationError, # raised when input fails validation - ProcessingError, # raised during extraction, graph build, or pipeline step -) -``` +| Name | Type | Role | +| --- | --- | --- | +| `setup_logging` | function | Configure root logger — `level`, `format` (`"json"` or `"text"`) | +| `get_logger` | function | Get a named logger instance | +| `log_performance` | decorator | Logs function name, duration, and any exception | +| `validate_entity` | function | Validate entity dict structure — raises `ValidationError` on failure | +| `validate_config` | function | Validate config dict against schema — raises `ValidationError` on failure | +| `ProgressTracker` | class | Class-based progress tracker with ETA and step callbacks | +| `track_progress` | function | Wrap any iterable with a live progress bar | +| `clean_text` | function | Normalize whitespace and strip control characters | +| `hash_data` | function | Deterministic SHA-256 hash of any serializable object | +| `SemanticaError` | exception | Base exception for all Semantica errors | +| `ValidationError` | exception | Raised when input fails validation | +| `ProcessingError` | exception | Raised during extraction, graph build, or pipeline step | ## What You Get diff --git a/docs/reference/vector_store.md b/docs/reference/vector_store.md index 0fbd1c68..99b5a350 100644 --- a/docs/reference/vector_store.md +++ b/docs/reference/vector_store.md @@ -8,39 +8,17 @@ icon: "database" ## Exported Classes -```python -from semantica.vector_store import ( - # Core interface - VectorStore, # unified: store_vectors, search_vectors, update_vectors, delete_vectors - VectorIndexer, # build/rebuild FAISS/ANN indices - VectorRetriever, # kNN and hybrid search - VectorManager, # store management and CRUD operations - # Backend stores - FAISSStore, # local disk / in-memory (Flat, IVF, HNSW, PQ index types) - WeaviateStore, # cloud/self-hosted, schema-aware, GraphQL queries - QdrantStore, # cloud/self-hosted, payload filtering - MilvusStore, # highly scalable, partitioning and complex queries - PineconeStore, # managed cloud vector database - PgVectorStore, # PostgreSQL with pgvector extension - # Hybrid & metadata search - HybridSearch, # fuse vector + metadata results (RRF or weighted average) - MetadataFilter, # MetadataFilter().eq("category", "science").gt("year", 2020) - SearchRanker, # configurable re-ranking after fusion - MetadataStore, # inverted index for fast metadata filtering - NamespaceManager, # multi-tenant namespace isolation - # Decision-specific helpers - DecisionEmbeddingPipeline, # end-to-end: record + embed + store + retrieve - quick_decision, # quick_decision(text, entities, outcome) — shorthand record - find_precedents, # find_precedents(scenario, k=5) — similarity search - # Convenience functions - store_vectors, # store_vectors(vectors, metadata) - search_vectors, # search_vectors(query_vector, k=10) - hybrid_search, # hybrid_search(query_vector, filter=...) - update_vectors, # update_vectors(ids, new_vectors) - delete_vectors, # delete_vectors(ids) - create_index, # create_index(index_type="hnsw", dimension=768) -) -``` +| Class | Role | +| --- | --- | +| `VectorStore` | Unified interface: `store_vectors`, `search_vectors`, `update_vectors`, `delete_vectors` | +| `HybridSearch` | Fuses dense vector similarity with sparse BM25/keyword results via RRF or weighted average | +| `MetadataFilter` | Chainable filter builder: `.eq("type", "person").gt("year", 2020).in_("tag", [...])` | +| `NamespaceManager` | Multi-tenant isolation — separate index namespaces per project or user | +| `FAISSStore` | Local disk or in-memory — Flat, IVF, HNSW, and PQ index types | +| `WeaviateStore` | Cloud or self-hosted, schema-aware, GraphQL queries | +| `QdrantStore` | Cloud or self-hosted with payload-based filtering | +| `PineconeStore` | Managed cloud vector database with serverless and pod modes | +| `PgVectorStore` | PostgreSQL with `pgvector` extension — no extra infrastructure | ## What You Get diff --git a/docs/reference/visualization.md b/docs/reference/visualization.md index 136f99b0..966e5c82 100644 --- a/docs/reference/visualization.md +++ b/docs/reference/visualization.md @@ -8,25 +8,14 @@ icon: "chart-bar" ## Exported Classes -```python -from semantica.visualization import ( - # Visualizers - KGVisualizer, # visualize_network(graph), visualize_communities(graph, communities) - OntologyVisualizer, # visualize_hierarchy(ontology), visualize_structure(ontology) - EmbeddingVisualizer, # visualize_2d_projection(embeddings, labels, method="umap") - SemanticNetworkVisualizer, # visualize_network(semantic_network) - AnalyticsVisualizer, # visualize_centrality(analytics), visualize_communities(analytics) - TemporalVisualizer, # visualize_timeline(events), visualize_evolution(snapshots) - # D3Visualizer is listed in __all__ but loaded lazily (requires d3js dependency) - # Convenience functions - visualize_kg, # visualize_kg(graph, output="interactive", method="default") - visualize_ontology, # visualize_ontology(ontology, output="interactive") - visualize_embeddings, # visualize_embeddings(embeddings, labels, method="umap") - visualize_semantic_network, # visualize_semantic_network(network) - visualize_analytics, # visualize_analytics(analytics_result) - visualize_temporal, # visualize_temporal(temporal_data) -) -``` +| Class | Role | +| --- | --- | +| `KGVisualizer` | Interactive network, community, and subgraph rendering with force/hierarchical/circular layouts | +| `OntologyVisualizer` | Class hierarchy and property relationship diagrams from any ontology | +| `EmbeddingVisualizer` | 2D/3D UMAP or t-SNE projection of embedding spaces with cluster labels | +| `SemanticNetworkVisualizer` | Weighted semantic network rendering | +| `AnalyticsVisualizer` | Centrality scores and community distribution charts | +| `TemporalVisualizer` | Timeline views and graph evolution animations across snapshots | ## What You Get