diff --git a/PLATFORM_REFERENCE.md b/PLATFORM_REFERENCE.md new file mode 100644 index 00000000..af75e034 --- /dev/null +++ b/PLATFORM_REFERENCE.md @@ -0,0 +1,976 @@ +# Platform Reference + +The full module-by-module API reference, additional recipes, the complete integrations matrix, and the capability table. If you're new to Semantica, start with the [README](README.md#quick-start); this doc is the deep end. + +**Jump to:** [Module Reference](#module-reference) · [More Recipes](#more-recipes) · [Features at a Glance](#features-at-a-glance) · [Integrations](#integrations) · [MCP Server](#mcp-server) · [REST API](#rest-api) · [Plugin Bundles](#plugin-bundles) + +--- + +## Module Reference + +Every module is independently importable and composable. Below are working examples for each. + +### `semantica.ingest`: Multi-Source Ingestion + +Ingest from files, web, databases, APIs, streams, email, Git repos, Parquet, Databricks, Snowflake, or MCP servers, all through a unified interface. + +```python +from semantica.ingest import FileIngestor, WebIngestor, ParquetIngestor, DBIngestor + +# Ingest an entire directory of contracts (PDF, DOCX, HTML, TXT) +docs = FileIngestor().ingest_directory("./contracts/", recursive=True) + +# Ingest live web content with robots.txt compliance +pages = WebIngestor().ingest_url("https://example.com/reports/annual-2024.html") + +# Ingest structured data from Parquet with Snappy compression +records = ParquetIngestor().ingest("./data/transactions.parquet") + +# Ingest from a SQL database - specify which tables to pull +rows = DBIngestor().ingest_database( + connection_string="postgresql://user:pass@localhost/mydb", + include_tables=["customer_events"], + max_rows_per_table=50_000, +) +``` + +**Supported sources:** Local files (PDF, DOCX, PPTX, HTML, TXT, CSV, JSON, YAML, Excel, XML) · Web pages · RSS/Atom feeds · REST APIs · Databases (PostgreSQL, MySQL, SQLite, Oracle, SQL Server) · Parquet datasets · Snowflake · Git repositories · Email (IMAP/POP3) · Message streams (Kafka, RabbitMQ, Kinesis, Pulsar) · MCP resources + +--- + +### `semantica.semantic_extract`: NER, Relations, Events, Triplets + +Extract structured knowledge from raw text in one pass. + +```python +from semantica.semantic_extract import ( + NamedEntityRecognizer, + RelationExtractor, + EventDetector, + TripletExtractor, +) + +text = """ +Anthropic CEO Dario Amodei announced a $7.3B Series E funding round in partnership +with Google and Spark Capital, valuing the company at $61.5B as of Q4 2024. +""" + +# Named entity recognition with confidence thresholding +ner = NamedEntityRecognizer(confidence_threshold=0.7) +entities = ner.extract_entities(text) +# → [Entity(name="Dario Amodei", type="PERSON"), Entity(name="Anthropic", type="ORG"), +# Entity(name="Google", type="ORG"), Entity(name="$7.3B", type="MONEY"), ...] + +# Relationship extraction - bidirectional support +rel_extractor = RelationExtractor(confidence_threshold=0.6, bidirectional=True) +relations = rel_extractor.extract_relations(text, entities=entities) +# → [Relation(subject="Dario Amodei", predicate="ceo_of", object="Anthropic"), +# Relation(subject="Anthropic", predicate="raised", object="$7.3B Series E"), ...] + +# Event detection with temporal processing +events = EventDetector(extract_participants=True, extract_time=True).detect_events(text) +# → [Event(type="FUNDING", participants=["Anthropic","Google","Spark Capital"], +# amount="$7.3B", date="Q4 2024")] + +# RDF triplets with optional provenance metadata +triplets = TripletExtractor(include_temporal=True, include_provenance=True).extract_triplets(text) +# → [("Anthropic", "valuation", "$61.5B"), ("Dario Amodei", "is_ceo_of", "Anthropic"), ...] +``` + +--- + +### `semantica.kg`: Knowledge Graph Construction & Analysis + +Build a production knowledge graph from documents and run graph algorithms over it. + +```python +from semantica.ingest import FileIngestor +from semantica.kg import ( + GraphBuilder, + GraphAnalyzer, + CentralityCalculator, + CommunityDetector, + PathFinder, + LinkPredictor, + BiTemporalFact, +) +from datetime import datetime + +# Build KG - merge duplicate entities, track temporal edges +sources = FileIngestor().ingest_directory("./contracts/", recursive=True) +kg = GraphBuilder(merge_entities=True, enable_temporal=True).build(sources) + +# Graph analytics +analyzer = GraphAnalyzer() +analysis = analyzer.analyze_graph(kg) # full graph metrics + +centrality = CentralityCalculator() +degree = centrality.calculate_degree_centrality(kg) # most-connected entities +betweenness = centrality.calculate_betweenness_centrality(kg) + +communities = CommunityDetector().detect_communities(kg, method="louvain") # natural clusters +path = PathFinder().find_shortest_path(kg, "alice_chen", "contract_001") +predictions = LinkPredictor().predict_links(kg, top_k=10) # relationship predictions + +# Bi-temporal facts - track valid time vs. recorded time independently +fact = BiTemporalFact( + valid_from=datetime(2024, 3, 1), + valid_until=datetime(2025, 1, 1), + recorded_at=datetime(2024, 3, 5), +) +``` + +--- + +### `semantica.reasoning`: Forward Chaining, Rete, Datalog, SPARQL + +Run explainable rule-based inference, not a black box. + +```python +from semantica.reasoning import ReteEngine, Rule, Fact, RuleType + +rete = ReteEngine() +rete.build_network([ + Rule( + rule_id="aml_flag", + name="Flag high-risk transactions", + conditions=[ + {"field": "amount", "operator": ">", "value": 10_000}, + {"field": "country", "operator": "in", "value": ["IR", "KP", "SY"]}, + ], + conclusion="flag_for_compliance_review", + rule_type=RuleType.IMPLICATION, + ), + Rule( + rule_id="velocity_check", + name="Flag rapid sequential transfers", + conditions=[ + {"field": "transfers_in_1h", "operator": ">", "value": 5}, + {"field": "total_amount", "operator": ">", "value": 50_000}, + ], + conclusion="flag_velocity_breach", + rule_type=RuleType.IMPLICATION, + ), +]) + +rete.add_fact(Fact("tx_001", "transaction", [{"amount": 15_000, "country": "IR"}])) +flagged = rete.match_patterns() +# → [{"rule": "aml_flag", "matched_facts": ["tx_001"], "conclusion": "flag_for_compliance_review"}] +``` + +```python +# Recursive Datalog - natural language for graph queries +from semantica.reasoning import DatalogReasoner + +engine = DatalogReasoner() +engine.add_fact("parent(tom, bob)") +engine.add_fact("parent(bob, ann)") +engine.add_fact("parent(ann, pat)") +engine.add_rule("ancestor(X, Y) :- parent(X, Y).") +engine.add_rule("ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).") +ancestors = engine.query("ancestor(tom, ?X)") +# → [{"X": "bob"}, {"X": "ann"}, {"X": "pat"}] +``` + +```python +# Explainable reasoning - trace the path, not just the answer +from semantica.reasoning import ExplanationGenerator, Reasoner + +reasoner = Reasoner() +result = reasoner.infer(kg, rules=[...]) + +explainer = ExplanationGenerator() +explanation = explainer.generate(result) +# → Explanation(conclusion="...", steps=[ReasoningStep(...)], justification=Justification(...)) +``` + +--- + +### `semantica.vector_store`: Hybrid & Filtered Semantic Search + +Drop-in vector store with multiple backends, hybrid search, and decision-aware retrieval. + +```python +from semantica.vector_store import VectorStore, HybridSearch + +# Works with FAISS, Qdrant, Weaviate, Milvus, Pinecone, PgVector, or in-memory +vs = VectorStore(backend="qdrant", dimension=1536) + +# Store a decision with scenario description and outcome +vs.store_decision( + scenario="Personal loan A-7291, $85k income, 31% DTI, 3yr employment", + outcome="approved", + confidence=0.94, + category="loan_underwriting", +) + +# Semantic similarity search +results = vs.search( + query="personal loan approval with low DTI", + limit=10, +) + +# Hybrid search - dense + sparse retrieval in one pass with RRF fusion +hs = HybridSearch(vector_store=vs) +hits = hs.search("high-risk transactions 2024") + +# Explain why a decision was retrieved +explanation = vs.explain_decision(results[0]["id"]) +``` + +--- + +### `semantica.split`: GraphRAG-Native Document Chunking + +KG-aware splitting that preserves entity boundaries, relation triplets, and ontology concepts, essential for GraphRAG pipelines. + +```python +from semantica.split import TextSplitter, EntityAwareChunker, RelationAwareChunker + +text = open("contracts/master_agreement.txt").read() + +# Standard recursive chunking +chunks = TextSplitter(method="recursive", chunk_size=1000, chunk_overlap=200).split(text) + +# Entity-aware chunking - never splits a named entity across chunks (GraphRAG) +chunks = TextSplitter(method="entity_aware", ner_method="llm", chunk_size=1000).split(text) + +# Relation-aware chunking - preserves (subject, predicate, object) triplets intact +chunks = RelationAwareChunker(chunk_size=1000, preserve_triplets=True).chunk(text) + +# Graph-based chunking - uses centrality to find natural community boundaries +chunks = TextSplitter(method="graph_based", chunk_size=1000).split(text) + +# Hierarchical chunking - multi-level (section → paragraph → sentence) +chunks = TextSplitter(method="hierarchical", levels=["section", "paragraph"]).split(text) +``` + +**Supported methods:** `recursive` · `token` · `sentence` · `paragraph` · `semantic_transformer` · `entity_aware` · `relation_aware` · `graph_based` · `ontology_aware` · `hierarchical` · `community_detection` · `centrality_based` · `llm` + +--- + +### `semantica.provenance`: W3C PROV-O Lineage + +Every fact is linked to its source. No black boxes, no mystery outputs. + +```python +from semantica.provenance import ProvenanceManager + +prov = ProvenanceManager(storage_path="./provenance.db") + +# Track where every entity came from +prov.track_entity( + entity_id="acme_corp", + source="contracts/acme_master_agreement_2024.pdf", + metadata={"page": 1, "confidence": 0.97, "extractor": "NamedEntityRecognizer"}, +) + +prov.track_relationship( + relationship_id="alice_works_for_acme", + source_entity_id="alice_chen", + target_entity_id="acme_corp", + source="hr_records/employees_q1_2024.csv", +) + +# Answer "where did this come from?" +lineage = prov.get_lineage("acme_corp") +trail = prov.trace_lineage("alice_chen") # full ancestor chain +entry = prov.get_provenance("acme_corp") +``` + +--- + +### `semantica.ontology`: OWL Generation, SHACL Validation + +Generate ontologies from data, validate shapes, and manage your vocabulary. + +```python +from semantica.ontology import OntologyGenerator, OntologyValidator + +data = { + "entities": [ + {"id": "acme_corp", "type": "Organization", "industry": "SaaS", "founded": 2012}, + {"id": "alice_chen", "type": "Person", "role": "CTO", "since": 2019}, + ], + "relationships": [ + {"source": "alice_chen", "target": "acme_corp", "type": "works_for"}, + ], +} + +gen = OntologyGenerator(base_uri="https://semantica.dev/ontology/") +ontology = gen.generate_ontology(data) +classes = gen.infer_classes(data) +props = gen.infer_properties(data, classes) +optimized = gen.optimize_ontology(ontology) + +# Validate against SHACL shapes +validator = OntologyValidator() +report = validator.validate(ontology) +# → ValidationResult(conforms=True, errors=[], warnings=[]) +``` + +--- + +### `semantica.conflicts`: Conflict Detection & Resolution + +Detect and resolve conflicting facts from multiple sources before they corrupt your knowledge base. + +```python +from semantica.conflicts import ConflictDetector, ConflictResolver, SourceTracker + +entities_from_source_a = [ + {"id": "alice_chen", "role": "CTO", "salary": 250_000, "start_date": "2019-03-01"}, +] +entities_from_source_b = [ + {"id": "alice_chen", "role": "VP Eng", "salary": 275_000, "start_date": "2019-03-01"}, +] + +# Detect all conflict types: value, type, relationship, temporal, logical +detector = ConflictDetector() +conflicts = detector.detect_conflicts(entities_from_source_a + entities_from_source_b) +# → [Conflict(entity="alice_chen", field="role", values=["CTO","VP Eng"], severity="HIGH"), +# Conflict(entity="alice_chen", field="salary", values=[250000,275000], severity="MEDIUM")] + +# Resolve using multiple strategies +resolver = ConflictResolver() +resolved = resolver.resolve(conflicts, strategy="credibility_weighted") # weighted by source trust +resolved = resolver.resolve(conflicts, strategy="temporal") # prefer most recent +resolved = resolver.resolve(conflicts, strategy="voting") # majority wins + +# Track source credibility over time +tracker = SourceTracker() +tracker.track("source_a", credibility=0.85) +tracker.track("source_b", credibility=0.72) +``` + +--- + +### `semantica.deduplication`: Entity Resolution at Scale + +Block, cluster, and merge duplicates with semantic similarity. **6.98× faster** than baseline. + +```python +from semantica.deduplication import DuplicateDetector, EntityMerger + +entities = [ + {"id": "e1", "name": "Acme Corporation", "domain": "acme.com"}, + {"id": "e2", "name": "Acme Corp.", "domain": "acme.com"}, + {"id": "e3", "name": "ACME Corp", "domain": "acme.co"}, + {"id": "e4", "name": "Globex Industries", "domain": "globex.com"}, +] + +detector = DuplicateDetector(similarity_threshold=0.75, use_clustering=True) +candidates = detector.detect_duplicates(entities) +groups = detector.detect_duplicate_groups(entities) +# → DuplicateGroup(entities=["e1","e2","e3"], confidence=0.91, strategy="semantic+blocking") + +merger = EntityMerger(preserve_provenance=True) +ops = merger.merge_duplicates(entities, strategy="keep_most_complete") +history = merger.get_merge_history() +``` + +--- + +### `semantica.normalize`: Data Normalization & Cleaning + +Standardize text, entities, dates, numbers, and encodings before building your knowledge graph. + +```python +from semantica.normalize import ( + TextNormalizer, + EntityNormalizer, + DateNormalizer, + NumberNormalizer, + DataCleaner, +) + +# Unicode, whitespace, casing, HTML tags, smart quotes +text = TextNormalizer().normalize(" Acme Corp.’s Q4 report… ") +# → "Acme Corp.'s Q4 report..." + +# Alias resolution + entity disambiguation with confidence scores +names = EntityNormalizer().normalize_entity("ACME Corp.") +# → NormalizedEntity(canonical="Acme Corporation", type="Organization", confidence=0.91) + +# Natural language date parsing with timezone conversion +dt = DateNormalizer().normalize_date("3 weeks ago") +# → datetime(2026, 5, 22, tzinfo=UTC) + +# Unit conversion and currency normalization +price = NumberNormalizer().normalize("$1.25M USD") +# → NormalizedNumber(value=1_250_000, currency="USD") + +# Deduplicate and impute missing values across a dataset +clean = DataCleaner().clean(records, dedup_threshold=0.9, fill_missing="mean") +``` + +--- + +### `semantica.pipeline`: Pipeline DSL + +Compose ingestion, extraction, and graph-building into a declarative, parallel pipeline. + +```python +from semantica.pipeline import PipelineBuilder, ExecutionEngine + +pipeline = ( + PipelineBuilder() + .add_step("ingest", step_type="ingest", source="./contracts/", recursive=True) + .add_step("extract", step_type="ner_extract") + .add_step("relations", step_type="relation_extract") + .add_step("build_kg", step_type="kg_build", merge_entities=True) + .add_step("deduplicate", step_type="deduplicate", threshold=0.75) + .add_step("export", step_type="export", format="turtle", output="kg.ttl") + .connect_steps("ingest", "extract") + .connect_steps("extract", "relations") + .connect_steps("relations", "build_kg") + .connect_steps("build_kg", "deduplicate") + .connect_steps("deduplicate", "export") + .set_parallelism(4) + .build(name="contracts_pipeline") +) + +engine = ExecutionEngine() +result = engine.execute(pipeline) +status = engine.get_status(pipeline) +progress = engine.get_progress(pipeline) +``` + +--- + +### Temporal Intelligence: Bi-Temporal Graphs & Time Travel + +Track when facts were true *in the world* vs. when they were *recorded*, and query either axis. + +```python +from semantica.context import ContextGraph +from semantica.kg import ( + BiTemporalFact, + TemporalGraphQuery, + TemporalVersionManager, + TemporalNormalizer, +) +from datetime import datetime + +graph = ContextGraph(advanced_analytics=True) +graph.add_node("alice_chen", "Person", role="VP Engineering") +graph.add_node("acme_corp", "Organization", valuation=1_200_000_000) + +# Point-in-time snapshots - replay history without reprocessing +snapshot_2023 = graph.state_at("2023-06-01") +snapshot_2024 = graph.state_at("2024-01-01") + +# Bi-temporal facts - valid_time is when true in the world; +# recorded_at is when you learned about it +fact = BiTemporalFact( + valid_from=datetime(2024, 3, 1), + valid_until=datetime(2025, 1, 1), + recorded_at=datetime(2024, 3, 5), +) + +# Allen interval algebra - 13 temporal relations (before, during, overlaps, etc.) +tq = TemporalGraphQuery(graph) +facts_in_window = tq.query_time_range("2024-01-01", "2024-12-31") + +# Normalize natural language temporal expressions +norm = TemporalNormalizer() +dt = norm.normalize("last quarter") # → datetime range for Q1 2026 +``` + +--- + +### `semantica.export`: RDF, OWL, Parquet, Cypher, JSON-LD + +Export to any format required by regulators, graph databases, or downstream systems. + +```python +from semantica.export import ( + RDFExporter, + JSONExporter, + ParquetExporter, + LPGExporter, + ReportGenerator, +) + +kg = {"entities": [...], "relationships": [...]} + +rdf = RDFExporter() +turtle_str = rdf.export_to_rdf(kg, format="turtle") # returns string +jsonld_str = rdf.export_to_rdf(kg, format="json-ld") + +rdf.export(kg, "kg_audit.ttl", format="turtle") +rdf.export(kg, "kg_audit.jsonld", format="json-ld") +rdf.export(kg, "kg_audit.nt", format="n-triples") + +# Columnar analytics - Snappy-compressed Parquet +ParquetExporter().export(kg, "kg_snapshot.parquet", compression="snappy") + +# JSON knowledge graph +JSONExporter().export_knowledge_graph(kg, "kg.json") + +# Neo4j / Memgraph Cypher statements for graph database import +LPGExporter().export(kg, "kg_import.cypher", method="cypher") + +# Human-readable HTML / Markdown report +ReportGenerator().generate(kg, "audit_report.html", format="html") +``` + +--- + +### `semantica.visualization`: Interactive Graph Workbench + +Render force-directed graphs, community maps, ontology hierarchies, and temporal dashboards. + +```python +from semantica.visualization import ( + KGVisualizer, + OntologyVisualizer, + EmbeddingVisualizer, + TemporalVisualizer, +) +import numpy as np + +kg = {"entities": [...], "relationships": [...]} + +# Interactive force-directed graph (opens in browser) +viz = KGVisualizer(layout="force", color_scheme="default") +viz.visualize_network(kg, output="interactive", file_path="kg.html") +viz.visualize_communities(kg, communities, output="interactive") +viz.visualize_centrality(kg, centrality, centrality_type="degree") +viz.visualize_entity_types(kg, output="html", file_path="entity_types.html") + +# Ontology class hierarchy +OntologyVisualizer().visualize_hierarchy(ontology, output="interactive") + +# 2D embedding projection (UMAP / t-SNE / PCA) +EmbeddingVisualizer().visualize_2d_projection( + embeddings=np.array([...]), + labels=["entity_a", "entity_b"], + method="umap", +) + +# Timeline scrubber - watch the graph evolve +TemporalVisualizer().visualize_timeline(kg, output="interactive") +``` + +--- + +### Multi-Agent Shared Context with Agno + +One shared intelligence layer. All agents read and write to the same context graph. + +```python +# pip install semantica[agno] +from agno.agent import Agent +from agno.team import Team +from agno.models.anthropic import Claude +from semantica.context import ContextGraph +from semantica.vector_store import VectorStore +from integrations.agno import AgnoSharedContext, AgnoDecisionKit, AgnoKGToolkit + +shared = AgnoSharedContext( + vector_store=VectorStore(backend="faiss"), + knowledge_graph=ContextGraph(advanced_analytics=True), + decision_tracking=True, +) + +researcher = Agent( + name="Researcher", + model=Claude(id="claude-sonnet-4-6"), + memory=shared.bind_agent("researcher"), + tools=[AgnoKGToolkit(context=shared)], +) +analyst = Agent( + name="Analyst", + model=Claude(id="claude-sonnet-4-6"), + memory=shared.bind_agent("analyst"), + tools=[AgnoDecisionKit(context=shared)], +) + +team = Team(agents=[researcher, analyst], mode="coordinate") +# Researcher's findings are instantly available to the Analyst - no copy, no sync +``` + +→ [runnable notebooks in the cookbook](https://github.com/semantica-agi/semantica/tree/main/cookbook), each self-contained and runnable in under 5 minutes + +--- + +## More Recipes + +The README covers the flagship audit-trail recipe. Here are three more common patterns. + +### End-to-End GraphRAG Pipeline + +```python +from semantica.ingest import FileIngestor +from semantica.split import TextSplitter +from semantica.semantic_extract import NamedEntityRecognizer, RelationExtractor +from semantica.kg import GraphBuilder +from semantica.vector_store import VectorStore, HybridSearch +from semantica.context import AgentContext + +# 1. Ingest +docs = FileIngestor().ingest_directory("./docs/", recursive=True) + +# 2. Entity-aware chunking - never splits an entity across a chunk boundary +splitter = TextSplitter(method="entity_aware", chunk_size=1000) +chunks = [splitter.split(doc["text"]) for doc in docs] + +# 3. Extract entities and relations +ner = NamedEntityRecognizer(confidence_threshold=0.7) +rel_ext = RelationExtractor(confidence_threshold=0.6) +entities = [ner.extract_entities(chunk) for chunk_group in chunks for chunk in chunk_group] + +# 4. Build KG +kg = GraphBuilder(merge_entities=True, enable_temporal=True).build(docs) + +# 5. Hybrid retrieval +vs = VectorStore(backend="faiss") +ctx = AgentContext(vector_store=vs, knowledge_graph=kg) +ctx.store("Alice approved the Acme renewal in Q1 2024", conversation_id="c1") + +results = HybridSearch(vector_store=vs).search("who approved the renewal?") +``` + +--- + +### AML Rules Engine + +```python +from semantica.reasoning import ReteEngine, Rule, Fact, RuleType + +rete = ReteEngine() +rete.build_network([ + Rule( + rule_id="sanctions_check", + name="Flag sanctioned-country transactions", + conditions=[ + {"field": "amount", "operator": ">", "value": 10_000}, + {"field": "country", "operator": "in", "value": ["IR", "KP", "SY", "CU"]}, + ], + conclusion="flag_for_compliance_review", + rule_type=RuleType.IMPLICATION, + ), +]) +rete.add_fact(Fact("tx_99", "transaction", [{"amount": 25_000, "country": "IR"}])) +matches = rete.match_patterns() +# → [{"rule": "sanctions_check", "matched_facts": ["tx_99"], +# "conclusion": "flag_for_compliance_review"}] +``` + +--- + +### Ontology-to-Knowledge-Graph in One Pass + +```python +from semantica.ingest import FileIngestor +from semantica.semantic_extract import NamedEntityRecognizer, RelationExtractor +from semantica.kg import GraphBuilder +from semantica.ontology import OntologyGenerator, OntologyValidator +from semantica.export import RDFExporter + +sources = FileIngestor().ingest_directory("./contracts/") +ner = NamedEntityRecognizer(confidence_threshold=0.7) +entities = ner.extract_entities_batch([s["text"] for s in sources]) + +kg = GraphBuilder(merge_entities=True).build(sources) +gen = OntologyGenerator(base_uri="https://myco.dev/ontology/") +ont = gen.generate_ontology({"entities": entities[0], "relationships": []}) + +report = OntologyValidator().validate(ont) +if report.conforms: + RDFExporter().export({"entities": entities[0]}, "ontology.ttl", format="turtle") +``` + +--- + +## Features at a Glance + +| Capability | Highlights | +| --- | --- | +| **Context Graphs** | Queryable graph of entities, decisions, relationships; causal links; cross-graph navigation | +| **Decision Intelligence** | `record_decision` · `trace_decision_chain` · `find_similar_decisions` · `analyze_decision_impact` · `check_decision_rules` | +| **Temporal Intelligence** | Point-in-time snapshots · Allen interval algebra (13 relations) · `TemporalNormalizer` · bi-temporal provenance | +| **Distance Intelligence** | N×N semantic distance matrices · ego-mode visualization · distance bands · 10× embedding cache | +| **Semantic Extraction** | NER · relation extraction · event detection · triplet generation · coreference · **6.98×** faster dedup | +| **Reasoning Engines** | Forward chaining · Rete · deductive · abductive · SPARQL · Datalog with explainable output | +| **GraphRAG Chunking** | Entity-aware · relation-aware · graph-based · ontology-aware · community-detection chunking | +| **Conflict Detection** | Value / type / relationship / temporal / logical conflicts · 5 resolution strategies | +| **Provenance** | W3C PROV-O · every fact traced to source · audit log export JSON/CSV/RDF | +| **Ontology Hub** | SHACL Studio · visual editor · cross-ontology alignments · 5-dimension health dashboard | +| **Vector Store** | FAISS · Pinecone · Weaviate · Qdrant · Milvus · PgVector · hybrid + filtered search | +| **Graph Databases (LPG)** | Neo4j · FalkorDB · Apache AGE · AWS Neptune | +| **Triple Stores (RDF)** | Blazegraph · Apache Jena · Eclipse RDF4J · unified `TripletStore` interface · SPARQL query & bulk load | +| **LLM Providers** | **All already supported today:** OpenAI (GPT-4o, o1, o3) · Anthropic (Claude 4) · Google Gemini · Mistral · Meta Llama · Groq · Cohere · Azure OpenAI · AWS Bedrock · Ollama · DeepSeek · Perplexity · Together AI · Fireworks AI · Replicate · HuggingFace · via `semantica.llms` and LiteLLM | + +--- + +## Integrations + +Native plugin bundles across major editors, a full-featured MCP server, a comprehensive REST API, and first-class Agno support. All LLM providers already supported: OpenAI · Anthropic · Gemini · Mistral · Llama · Groq · Cohere · Azure · Bedrock · Ollama · DeepSeek · HuggingFace and more via LiteLLM + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Native Plugin BundleMCP Server + Plugin
+Claude Code
+Claude Code
+Skills · agents · hooks +
+Cursor
+Cursor
+Skills · agents +
+Codex CLI
+Codex CLI
+Skills · agents +
+Windsurf
+Windsurf
+plugin +
+Cline
+Cline
+plugin +
+Continue
+Continue
+plugin +
+VS Code
+VS Code
+plugin +
+OpenClaw
+OpenClaw
+MCP + plugin +
MCP ServerREST API
+Claude Desktop
+Claude Desktop
+MCP server +
+GitHub Copilot
+GitHub Copilot
+REST API +
+Roo Code
+Roo Code
+REST API +
+Goose
+Goose
+REST API +
+Kilo Code
+Kilo Code
+REST API +
+Aider
+Aider
+REST API +
+Amazon Q
+Amazon Q
+REST API +
+Zed
+Zed
+REST API +
+ +### Agentic Frameworks + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Native Integration
+Agno
+Agno
+First-class · pip install semantica[agno] +
Already Supported via REST API & MCP
+LangChain
+LangChain
+REST API · MCP +
+LangGraph
+LangGraph
+REST API · MCP +
+CrewAI
+CrewAI
+REST API · MCP +
+LlamaIndex
+LlamaIndex
+REST API · MCP +
+AutoGen
+AutoGen
+REST API · MCP +
+OpenAI Agents SDK
+OpenAI Agents
+REST API · MCP +
+Google ADK
+Google ADK
+REST API · MCP +
Native SDK Integration (Coming Soon)
+LangChain
+LangChain
+Dedicated toolkit +
+CrewAI
+CrewAI
+Dedicated toolkit +
+LlamaIndex
+LlamaIndex
+Dedicated toolkit +
+AutoGen
+AutoGen
+Dedicated toolkit +
+OpenAI Agents SDK
+OpenAI Agents
+Dedicated toolkit +
+Google ADK
+Google ADK
+Dedicated toolkit +
+ +--- + +## MCP Server + +Connect any MCP-compatible client (Claude Desktop, Windsurf, Cline, VS Code) in 30 seconds: + +```bash +python -m semantica.mcp_server +# or via the installed entry point +semantica-mcp +``` + +```json +{ + "mcpServers": { + "semantica": { "command": "python", "args": ["-m", "semantica.mcp_server"] } + } +} +``` + +**Tools exposed over MCP:** + +| Tool | What it does | +| --- | --- | +| `extract_entities` | NER on any text | +| `extract_relations` | Relation extraction | +| `record_decision` | Persist a decision node | +| `query_decisions` | Search decision history | +| `find_precedents` | Semantic precedent lookup | +| `get_causal_chain` | Full causal ancestry | +| `add_entity` | Add a KG node | +| `add_relationship` | Add a KG edge | +| `run_reasoning` | Execute rule set | +| `get_graph_analytics` | Centrality, communities | +| `export_graph` | Export to RDF/JSON/Parquet | +| `get_graph_summary` | Graph statistics | + +--- + +## REST API + +```bash +# Start the backend +python -m semantica.server # port 8000 + +# Extract entities via REST +curl -X POST http://localhost:8000/api/extract/entities \ + -H "Content-Type: application/json" \ + -d '{"text": "Apple CEO Tim Cook announced record earnings."}' + +# Record a decision +curl -X POST http://localhost:8000/api/decisions \ + -H "Content-Type: application/json" \ + -d '{ + "category": "vendor_selection", + "scenario": "Choose ML cloud provider", + "reasoning": "Best GPU availability and pricing", + "outcome": "selected_aws", + "confidence": 0.91 + }' + +# Query the knowledge graph +curl http://localhost:8000/api/graph/neighbors/acme_corp?hops=2 +``` + +**REST endpoints span:** `extract` · `kg` · `decisions` · `reasoning` · `provenance` · `ontology` · `embeddings` · `search` · `export` · `pipeline` · `temporal` · `deduplication` + +--- + +## Plugin Bundles + +**Domain skills:** `extract` · `ingest` · `query` · `ontology` · `validate` · `deduplicate` · `embed` · `reason` · `decision` · `causal` · `temporal` · `provenance` · `policy` · `explain` · `export` · `change` · `visualize` + +**Specialized agents:** `kg-assistant` · `decision-advisor` · `explainability` + +Bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw in [`plugins/`](plugins/). + +--- + +[← Back to README](README.md) diff --git a/README.md b/README.md index ead03825..cd24c42f 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,17 @@ Semantica -### The Context and Accountability Layer for AI Agents +### Context and Accountability Infrastructure for AI Agents -> Semantica is the Context and Accountability Layer for AI agents: context graphs, decision intelligence, explainable reasoning, and policy governance, with a permanent audit trail behind every decision. -> -> It's the intelligence category **Palantir** built for the enterprise, delivered **100% open source** and self-hostable, with zero black boxes and zero vendor lock-in, from startup to Fortune 500, without the seven-figure contract. +[![The Open Source Palantir for AI Agents](https://img.shields.io/badge/The%20Open%20Source%20Palantir-for%20AI%20Agents-000000?style=for-the-badge)](https://github.com/semantica-agi/semantica) + +> Ingest your enterprise data, extract what matters, build a Context Graph and knowledge graph (KG), and run graph analytics and causal reasoning over all of it, with full decision provenance baked in. Explainable, traceable, and trustworthy by design. **Decision Intelligence  ·  Context Management  ·  Deterministic Reasoning  ·  Traceability** -**Open Source  ·  Auditable  ·  Governed  ·  Self-Hostable** +![Open Source](https://img.shields.io/badge/Open%20Source-0EA5E9?style=flat-square) ![Auditable](https://img.shields.io/badge/Auditable-0EA5E9?style=flat-square) ![Governed](https://img.shields.io/badge/Governed-0EA5E9?style=flat-square) ![Self--Hostable](https://img.shields.io/badge/Self--Hostable-0EA5E9?style=flat-square) ![Zero Vendor Lock--In](https://img.shields.io/badge/Zero%20Vendor%20Lock--In-0EA5E9?style=flat-square) -**Polyglot Graph Storage** - -[![GitHub Stars](https://img.shields.io/github/stars/semantica-agi/semantica?style=flat-square&color=FFD700&logo=github&logoColor=white&label=Stars)](https://github.com/semantica-agi/semantica) [![PyPI](https://img.shields.io/pypi/v/semantica.svg?style=flat-square&color=0066CC&logo=pypi&logoColor=white)](https://pypi.org/project/semantica/) [![Total Downloads](https://static.pepy.tech/badge/semantica?style=flat-square)](https://pepy.tech/project/semantica) [![Python 3.8+](https://img.shields.io/badge/python-3.8+-3776AB?style=flat-square&logo=python&logoColor=white)](https://www.python.org/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=flat-square)](https://opensource.org/licenses/MIT) [![CI](https://img.shields.io/github/actions/workflow/status/semantica-agi/semantica/ci.yml?style=flat-square&label=CI)](https://github.com/semantica-agi/semantica/actions) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/semantica-agi/semantica) +[![GitHub Stars](https://img.shields.io/github/stars/semantica-agi/semantica?style=flat-square&color=FFD700&logo=github&logoColor=white&label=Stars)](https://github.com/semantica-agi/semantica) [![GitHub Forks](https://img.shields.io/github/forks/semantica-agi/semantica?style=flat-square&color=6E40C9&logo=github&logoColor=white&label=Forks)](https://github.com/semantica-agi/semantica/network/members) [![Contributors](https://img.shields.io/github/contributors/semantica-agi/semantica?style=flat-square&color=2EA043&logo=github&logoColor=white)](https://github.com/semantica-agi/semantica/graphs/contributors) [![PyPI](https://img.shields.io/pypi/v/semantica.svg?style=flat-square&color=0066CC&logo=pypi&logoColor=white)](https://pypi.org/project/semantica/) [![Total Downloads](https://static.pepy.tech/badge/semantica?style=flat-square)](https://pepy.tech/project/semantica) [![Python 3.8+](https://img.shields.io/badge/python-3.8+-3776AB?style=flat-square&logo=python&logoColor=white)](https://www.python.org/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=flat-square)](https://opensource.org/licenses/MIT) [![CI](https://img.shields.io/github/actions/workflow/status/semantica-agi/semantica/ci.yml?style=flat-square&label=CI)](https://github.com/semantica-agi/semantica/actions) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/semantica-agi/semantica) [![Website](https://img.shields.io/badge/Website-getsemantica.ai-000000?style=flat-square&logo=googlechrome&logoColor=white)](https://getsemantica.ai/) [![Docs](https://img.shields.io/badge/Docs-docs.getsemantica.ai-0099FF?style=flat-square&logo=readthedocs&logoColor=white)](https://docs.getsemantica.ai/) [![Discord](https://img.shields.io/badge/Discord-Join%20Community-5865F2?style=flat-square&logo=discord&logoColor=white)](https://discord.gg/sV34vps5hH) [![Twitter/X](https://img.shields.io/badge/Follow-%40BuildSemantica-000000?style=flat-square&logo=x&logoColor=white)](https://x.com/BuildSemantica) [![YouTube](https://img.shields.io/badge/YouTube-Watch%20Demos-FF0000?style=flat-square&logo=youtube&logoColor=white)](https://www.youtube.com/watch?v=QfnNZg4-dZA) [![Changelog](https://img.shields.io/badge/Changelog-View-6E40C9?style=flat-square&logo=keepachangelog&logoColor=white)](CHANGELOG.md) @@ -22,51 +20,71 @@ --- -> Most AI agents act without a trail. They store embeddings, not meaning: context that can't be explained, decisions that can't be audited. Regulators, auditors, and enterprise risk teams all ask the same question: **can you prove what your AI did and why?** -> -> Semantica answers that question. It sits alongside your LLM, vector store, and agent framework as a dedicated context and accountability layer, adding structured intelligence, causal reasoning, and a full audit trail to every decision your agents make. Built for teams running AI agents in production, especially in healthcare, finance, legal, and government, where every decision must be explainable and defensible. +
-**What Semantica gives you:** + +Semantica Knowledge Explorer: live graph, decisions, entity resolution, ontology hub + + +*Knowledge Explorer · Context Graphs · Reasoning Engine · Decision Intelligence · Ontology Hub* + +**[▶ Watch the full platform walkthrough](https://www.youtube.com/watch?v=QfnNZg4-dZA)** + +
+ +--- + +Most AI agents act without a trail. They store embeddings, not meaning: context that can't be explained, decisions that can't be audited. In lending, that gap is a compliance exposure, not an inconvenience: an underwriting agent's approval has to survive a regulator's "why" months later. + +Semantica sits underneath your LLM, vector store, and agent framework as a deterministic infrastructure layer: no LLM required for graph construction, reasoning, or provenance. + +**Who it's for:** + +- **AI/ML platform teams** shipping agents that make consequential decisions and need structured, queryable context built from fragmented raw data, not just a vector index +- **Compliance, risk, and audit teams** who need a straight answer to "why did the AI do that?" in a format a regulator will actually accept +- **Regulated enterprises** (finance, healthcare, legal, government, defense) that can't ship a black box, and can't send their data to someone else's SaaS to get one +- **Platform and infra engineers** who want the KG, reasoning, and provenance stack self-hosted and swappable, not locked to one vendor's backend +- **Data and knowledge engineers** building a KG from messy, multi-source data: entities and relationships get extracted, conflicting or contradictory facts are flagged instead of silently overwritten, and duplicates are merged before they turn into noise + +**[Quick Start](#quick-start)**  ·  **[Architecture](#architecture)**  ·  **[What You Get](#what-semantica-gives-you)**  ·  **[Why Semantica](#why-semantica)**  ·  **[Decision Intelligence](#decision-intelligence)**  ·  **[Context Graphs](#context-graphs)**  ·  **[Recipe: Audit Trail](#recipe-audit-trail-for-a-regulated-decision)**  ·  **[Platform Reference](PLATFORM_REFERENCE.md)**  ·  **[CLI](#cli)**  ·  **[Performance](#performance)**  ·  **[Install](#installation)** + +--- + +## What Semantica Gives You - **Context Graphs:** A structured, queryable graph of everything your agent knows, decides, and reasons about - **Decision Intelligence:** Every decision is a first-class object: traceable, searchable by precedent, and causally linked - **AI Governance & Ontology:** SHACL constraints, conflict detection, compliance rules, OWL generation, and SKOS vocabulary management with a visual editor - **Full Auditability:** W3C PROV-O provenance on every fact, with audit trails exportable to JSON, CSV, or RDF - **Deterministic Reasoning:** Forward chaining, Rete network, Datalog, and SPARQL with fully explainable paths, not black boxes -- **Knowledge Pipeline:** Multi-source ingestion, entity-aware chunking, and semantic deduplication with provenance-preserving merges -- **Polyglot Graph Storage:** Native support for both RDF (Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code -- **Flexible Storage & Visualization:** Swap storage backends freely; explore it all in an interactive browser workbench +- **Knowledge Pipeline:** Multi-source ingestion, entity-aware chunking, NER/relation/event extraction, and knowledge graph construction, with semantic deduplication and provenance-preserving merges throughout +- **Graph Analytics:** Centrality, community detection, link prediction, and shortest-path queries over the graph you just built +- **Polyglot Graph Storage:** Native RDF (Blazegraph, Apache Jena, Eclipse RDF4J via SPARQL) and Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune via Cypher), plus vector stores, all swappable without touching your code +- **Visualization:** Explore any graph, ontology, or timeline in an interactive browser workbench - **Drop-in Integrations:** Native Agno support, a full-featured MCP server, a comprehensive CLI, a REST API, and plugins across major editors --- -**[Quick Start](#quick-start)**  ·  **[Architecture](ARCHITECTURE.md)**  ·  **[Why Semantica](#why-semantica)**  ·  **[Context Graphs](#context-graphs)**  ·  **[Decision Intelligence](#decision-intelligence)**  ·  **[Module Reference](#module-reference)**  ·  **[Recipes](#recipes)**  ·  **[CLI](#cli)**  ·  **[Integrations](#integrations)**  ·  **[Performance](#performance)**  ·  **[Install](#installation)** +## Why Semantica ---- +| | Vector DB + RAG | Plain LLM Memory | **Semantica** | +| --- | --- | --- | --- | +| **Recall method** | Embedding similarity | Token window | Graph traversal + semantic search | +| **Decision history** | Not stored | Not stored | First-class queryable objects | +| **Provenance** | None | None | W3C PROV-O, source-linked | +| **Reasoning** | None | Black box | Forward chain, Rete, Datalog, SPARQL | +| **Conflict detection** | Silent overwrite | Silent overwrite | Detected, flagged, resolved | +| **Time travel** | No | No | Point-in-time graph snapshots | +| **Compliance export** | None | None | PROV-O, SHACL, OWL, RDF | +| **Policy enforcement** | None | None | Built-in rule engine + SHACL | +| **Entity resolution** | No | No | Blocking + semantic deduplication | +| **Multi-agent context** | Separate per agent | Separate per agent | Single shared intelligence layer | -## See It in Action - -
- -Semantica Knowledge Explorer: live graph, decisions, entity resolution, ontology hub - - - Semantica: Full Platform Walkthrough on YouTube - - -**[Watch the full platform walkthrough →](https://www.youtube.com/watch?v=QfnNZg4-dZA)** - -*Knowledge Explorer · Context Graphs · Reasoning Engine · Decision Intelligence · Ontology Hub* - -
+Semantica complements your existing stack rather than replacing it. Keep your LLM, vector store, and agent framework exactly as they are; Semantica adds the decision records, causal reasoning, provenance, ontology governance, conflict detection, and audit trails on top. The reasoning engines, KG construction, and provenance layer are fully deterministic; no LLM is required to use them. --- @@ -119,82 +137,29 @@ If Semantica solves a real problem for you, a star helps others find it. ## Architecture -The full data pipeline and decision intelligence lifecycle are documented with Mermaid flowcharts in **[ARCHITECTURE.md](ARCHITECTURE.md)**: +Semantica is a real end-to-end pipeline, not a single library with a marketing name. Every stage below is a shipping module, independently importable: -- [Full data pipeline](ARCHITECTURE.md#full-data-pipeline): all sources → ingest → parse → normalize → split → extract → deduplication → KG → storage → export -- [Decision intelligence lifecycle](ARCHITECTURE.md#decision-intelligence-lifecycle): record → link → query → govern → audit - -**→ [View architecture →](ARCHITECTURE.md)** - -Every component is independently importable. Use one module or all of them. - ---- - -## Why Semantica - -| | Vector DB + RAG | Plain LLM Memory | **Semantica** | -| --- | --- | --- | --- | -| **Recall method** | Embedding similarity | Token window | Graph traversal + semantic search | -| **Decision history** | Not stored | Not stored | First-class queryable objects | -| **Provenance** | None | None | W3C PROV-O, source-linked | -| **Reasoning** | None | Black box | Forward chain, Rete, Datalog, SPARQL | -| **Conflict detection** | Silent overwrite | Silent overwrite | Detected, flagged, resolved | -| **Time travel** | No | No | Point-in-time graph snapshots | -| **Compliance export** | None | None | PROV-O, SHACL, OWL, RDF | -| **Policy enforcement** | None | None | Built-in rule engine + SHACL | -| **Entity resolution** | No | No | Blocking + semantic deduplication | -| **Multi-agent context** | Separate per agent | Separate per agent | Single shared intelligence layer | - -Semantica complements your existing stack rather than replacing it. Keep your LLM, vector store, and agent framework exactly as they are. Semantica sits alongside them as the accountability and intelligence layer, adding structured decision records, causal reasoning, W3C PROV-O provenance, ontology governance, conflict detection, and compliance-grade audit trails. The reasoning engines, KG construction, and provenance layer are fully deterministic; no LLM is required to use them. - ---- - -## Context Graphs - -A Context Graph is the structured memory layer that traditional RAG is missing. Instead of flat embeddings that answer *"what is similar?"*, a Context Graph answers *"what is connected, why, and how?"* - -Every entity, relationship, decision, and fact is a first-class node, queryable by graph traversal and neighbor expansion. Entities link to source documents. Decisions link to evidence and consequences. Facts carry full provenance. Conflicts are detected, not silently overwritten. - -```python -from semantica.context import ContextGraph, AgentContext -from semantica.vector_store import VectorStore - -graph = ContextGraph(advanced_analytics=True) - -# Add nodes with typed properties -graph.add_node("acme_corp", "Organization", name="Acme Corp", industry="SaaS") -graph.add_node("alice_chen", "Person", name="Alice Chen", role="CTO") -graph.add_node("contract_001", "Contract", value=2_400_000, currency="USD") - -# Add typed, weighted edges (extra kwargs become edge metadata) -graph.add_edge("alice_chen", "acme_corp", edge_type="works_for", since="2019-03-01") -graph.add_edge("acme_corp", "contract_001", edge_type="party_to", signed="2024-01-15") - -# BFS traversal - hop through the graph from any node -neighbors = graph.get_neighbors("acme_corp", hops=2) - -# Point-in-time snapshot - the graph as it existed on any past date -snapshot = graph.state_at("2024-01-01") - -# AgentContext - high-level API for agent memory workflows -vs = VectorStore(backend="faiss") -ctx = AgentContext(vector_store=vs, knowledge_graph=graph) -ctx.store("Alice approved the Acme renewal in Q1 2024", conversation_id="conv_001") -retrieved = ctx.retrieve("who approved the Acme contract?") +``` +Sources → Ingest → Parse → Normalize → Split → Extract → Conflict Detection → Deduplication + → Knowledge Graph → [ Ontology · Reasoning · Provenance · Decisions ] → Enriched KG + → Vector Store + Polyglot Graph Store (RDF & LPG) → Export / Visualize / REST · MCP · CLI ``` -**Why graph over embeddings:** +- **Ingest:** files, web, databases, enterprise data platforms (Databricks, Snowflake), cloud (Drive, Elasticsearch), streams (Kafka, Kinesis), Git, email, MCP +- **Parse → Normalize → Split:** document parsing, text/entity/date normalization, GraphRAG-native entity-aware chunking +- **Extract → Conflict Detection → Deduplication:** NER, relations, events, triplets; conflicting facts flagged and resolved before they merge +- **Knowledge Graph:** `GraphBuilder` constructs the graph; bi-temporal facts and full graph analytics (centrality, communities, link prediction) run on top of it +- **Ontology · Reasoning · Provenance · Decisions:** the intelligence layer sitting on the KG, with SHACL/OWL governance, Rete/Datalog/SPARQL inference, W3C PROV-O lineage, and first-class decision records +- **Storage:** polyglot by design, with RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J), Labeled Property Graphs (Neo4j, FalkorDB, Apache AGE, AWS Neptune), and vector stores, all swappable without touching your code +- **Outputs:** export (RDF, OWL, Parquet, Cypher, JSON-LD), interactive visualization, and access via REST API, MCP server, or CLI -- Traversal finds connections embeddings miss, including a person 3 hops from a contract -- Every node carries provenance so you can always ask *"where did this come from?"* -- Conflicts are detected and flagged before they corrupt your knowledge base -- Point-in-time snapshots let you replay history without reprocessing +**→ [Full Mermaid diagrams for the pipeline and the decision intelligence lifecycle](ARCHITECTURE.md)** --- ## Decision Intelligence -Decision Intelligence turns every AI choice from an ephemeral inference into a permanent, auditable, queryable record. It answers *"what did your AI decide, why, and what happened next?"* The question regulators and enterprise risk teams are asking with increasing urgency. +Decision Intelligence turns every AI choice from an ephemeral inference into a permanent, auditable, queryable record. It answers *"what did your AI decide, why, and what happened next?"*: the question regulators and enterprise risk teams ask with increasing urgency. In Semantica, a decision is not a log line. It is a first-class graph node with a full lifecycle. In regulated domains, every AI decision must be traceable to a source and defensible to an auditor: `record_decision()` creates a permanent, structured record exportable as W3C PROV-O, the format most compliance frameworks accept for regulator submission. @@ -251,635 +216,45 @@ insights = graph.get_decision_insights() --- -## Module Reference +## Context Graphs -Semantica is a full platform. Every module is independently importable and composable. Below are working examples for each. - -### `semantica.ingest`: Multi-Source Ingestion - -Ingest from files, web, databases, APIs, streams, email, Git repos, Parquet, Snowflake, or MCP servers, all through a unified interface. +A Context Graph is the structured memory layer that traditional RAG is missing. Instead of flat embeddings that answer *"what is similar?"*, a Context Graph answers *"what is connected, why, and how?"* Every entity, relationship, decision, and fact is a first-class node, queryable by graph traversal. Entities link to source documents, decisions link to evidence and consequences, facts carry full provenance, and conflicts are detected, not silently overwritten. ```python -from semantica.ingest import FileIngestor, WebIngestor, ParquetIngestor, DBIngestor - -# Ingest an entire directory of contracts (PDF, DOCX, HTML, TXT) -docs = FileIngestor().ingest_directory("./contracts/", recursive=True) - -# Ingest live web content with robots.txt compliance -pages = WebIngestor().ingest_url("https://example.com/reports/annual-2024.html") - -# Ingest structured data from Parquet with Snappy compression -records = ParquetIngestor().ingest("./data/transactions.parquet") - -# Ingest from a SQL database - specify which tables to pull -rows = DBIngestor().ingest_database( - connection_string="postgresql://user:pass@localhost/mydb", - include_tables=["customer_events"], - max_rows_per_table=50_000, -) -``` - -**Supported sources:** Local files (PDF, DOCX, PPTX, HTML, TXT, CSV, JSON, YAML, Excel, XML) · Web pages · RSS/Atom feeds · REST APIs · Databases (PostgreSQL, MySQL, SQLite, Oracle, SQL Server) · Parquet datasets · Snowflake · Git repositories · Email (IMAP/POP3) · Message streams (Kafka, RabbitMQ, Kinesis, Pulsar) · MCP resources - ---- - -### `semantica.semantic_extract`: NER, Relations, Events, Triplets - -Extract structured knowledge from raw text in one pass. - -```python -from semantica.semantic_extract import ( - NamedEntityRecognizer, - RelationExtractor, - EventDetector, - TripletExtractor, -) - -text = """ -Anthropic CEO Dario Amodei announced a $7.3B Series E funding round in partnership -with Google and Spark Capital, valuing the company at $61.5B as of Q4 2024. -""" - -# Named entity recognition with confidence thresholding -ner = NamedEntityRecognizer(confidence_threshold=0.7) -entities = ner.extract_entities(text) -# → [Entity(name="Dario Amodei", type="PERSON"), Entity(name="Anthropic", type="ORG"), -# Entity(name="Google", type="ORG"), Entity(name="$7.3B", type="MONEY"), ...] - -# Relationship extraction - bidirectional support -rel_extractor = RelationExtractor(confidence_threshold=0.6, bidirectional=True) -relations = rel_extractor.extract_relations(text, entities=entities) -# → [Relation(subject="Dario Amodei", predicate="ceo_of", object="Anthropic"), -# Relation(subject="Anthropic", predicate="raised", object="$7.3B Series E"), ...] - -# Event detection with temporal processing -events = EventDetector(extract_participants=True, extract_time=True).detect_events(text) -# → [Event(type="FUNDING", participants=["Anthropic","Google","Spark Capital"], -# amount="$7.3B", date="Q4 2024")] - -# RDF triplets with optional provenance metadata -triplets = TripletExtractor(include_temporal=True, include_provenance=True).extract_triplets(text) -# → [("Anthropic", "valuation", "$61.5B"), ("Dario Amodei", "is_ceo_of", "Anthropic"), ...] -``` - ---- - -### `semantica.kg`: Knowledge Graph Construction & Analysis - -Build a production knowledge graph from documents and run graph algorithms over it. - -```python -from semantica.ingest import FileIngestor -from semantica.kg import ( - GraphBuilder, - GraphAnalyzer, - CentralityCalculator, - CommunityDetector, - PathFinder, - LinkPredictor, - BiTemporalFact, -) -from datetime import datetime - -# Build KG - merge duplicate entities, track temporal edges -sources = FileIngestor().ingest_directory("./contracts/", recursive=True) -kg = GraphBuilder(merge_entities=True, enable_temporal=True).build(sources) - -# Graph analytics -analyzer = GraphAnalyzer() -analysis = analyzer.analyze_graph(kg) # full graph metrics - -centrality = CentralityCalculator() -degree = centrality.calculate_degree_centrality(kg) # most-connected entities -betweenness = centrality.calculate_betweenness_centrality(kg) - -communities = CommunityDetector().detect_communities(kg, method="louvain") # natural clusters -path = PathFinder().find_shortest_path(kg, "alice_chen", "contract_001") -predictions = LinkPredictor().predict_links(kg, top_k=10) # relationship predictions - -# Bi-temporal facts - track valid time vs. recorded time independently -fact = BiTemporalFact( - valid_from=datetime(2024, 3, 1), - valid_until=datetime(2025, 1, 1), - recorded_at=datetime(2024, 3, 5), -) -``` - ---- - -### `semantica.reasoning`: Forward Chaining, Rete, Datalog, SPARQL - -Run explainable rule-based inference, not a black box. - -```python -from semantica.reasoning import ReteEngine, Rule, Fact, RuleType - -rete = ReteEngine() -rete.build_network([ - Rule( - rule_id="aml_flag", - name="Flag high-risk transactions", - conditions=[ - {"field": "amount", "operator": ">", "value": 10_000}, - {"field": "country", "operator": "in", "value": ["IR", "KP", "SY"]}, - ], - conclusion="flag_for_compliance_review", - rule_type=RuleType.IMPLICATION, - ), - Rule( - rule_id="velocity_check", - name="Flag rapid sequential transfers", - conditions=[ - {"field": "transfers_in_1h", "operator": ">", "value": 5}, - {"field": "total_amount", "operator": ">", "value": 50_000}, - ], - conclusion="flag_velocity_breach", - rule_type=RuleType.IMPLICATION, - ), -]) - -rete.add_fact(Fact("tx_001", "transaction", [{"amount": 15_000, "country": "IR"}])) -flagged = rete.match_patterns() -# → [{"rule": "aml_flag", "matched_facts": ["tx_001"], "conclusion": "flag_for_compliance_review"}] -``` - -```python -# Recursive Datalog - natural language for graph queries -from semantica.reasoning import DatalogReasoner - -engine = DatalogReasoner() -engine.add_fact("parent(tom, bob)") -engine.add_fact("parent(bob, ann)") -engine.add_fact("parent(ann, pat)") -engine.add_rule("ancestor(X, Y) :- parent(X, Y).") -engine.add_rule("ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).") -ancestors = engine.query("ancestor(tom, ?X)") -# → [{"X": "bob"}, {"X": "ann"}, {"X": "pat"}] -``` - -```python -# Explainable reasoning - trace the path, not just the answer -from semantica.reasoning import ExplanationGenerator, Reasoner - -reasoner = Reasoner() -result = reasoner.infer(kg, rules=[...]) - -explainer = ExplanationGenerator() -explanation = explainer.generate(result) -# → Explanation(conclusion="...", steps=[ReasoningStep(...)], justification=Justification(...)) -``` - ---- - -### `semantica.vector_store`: Hybrid & Filtered Semantic Search - -Drop-in vector store with multiple backends, hybrid search, and decision-aware retrieval. - -```python -from semantica.vector_store import VectorStore, HybridSearch - -# Works with FAISS, Qdrant, Weaviate, Milvus, Pinecone, PgVector, or in-memory -vs = VectorStore(backend="qdrant", dimension=1536) - -# Store a decision with scenario description and outcome -vs.store_decision( - scenario="Personal loan A-7291, $85k income, 31% DTI, 3yr employment", - outcome="approved", - confidence=0.94, - category="loan_underwriting", -) - -# Semantic similarity search -results = vs.search( - query="personal loan approval with low DTI", - limit=10, -) - -# Hybrid search - dense + sparse retrieval in one pass with RRF fusion -hs = HybridSearch(vector_store=vs) -hits = hs.search("high-risk transactions 2024") - -# Explain why a decision was retrieved -explanation = vs.explain_decision(results[0]["id"]) -``` - ---- - -### `semantica.split`: GraphRAG-Native Document Chunking - -KG-aware splitting that preserves entity boundaries, relation triplets, and ontology concepts, essential for GraphRAG pipelines. - -```python -from semantica.split import TextSplitter, EntityAwareChunker, RelationAwareChunker - -text = open("contracts/master_agreement.txt").read() - -# Standard recursive chunking -chunks = TextSplitter(method="recursive", chunk_size=1000, chunk_overlap=200).split(text) - -# Entity-aware chunking - never splits a named entity across chunks (GraphRAG) -chunks = TextSplitter(method="entity_aware", ner_method="llm", chunk_size=1000).split(text) - -# Relation-aware chunking - preserves (subject, predicate, object) triplets intact -chunks = RelationAwareChunker(chunk_size=1000, preserve_triplets=True).chunk(text) - -# Graph-based chunking - uses centrality to find natural community boundaries -chunks = TextSplitter(method="graph_based", chunk_size=1000).split(text) - -# Hierarchical chunking - multi-level (section → paragraph → sentence) -chunks = TextSplitter(method="hierarchical", levels=["section", "paragraph"]).split(text) -``` - -**Supported methods:** `recursive` · `token` · `sentence` · `paragraph` · `semantic_transformer` · `entity_aware` · `relation_aware` · `graph_based` · `ontology_aware` · `hierarchical` · `community_detection` · `centrality_based` · `llm` - ---- - -### `semantica.provenance`: W3C PROV-O Lineage - -Every fact is linked to its source. No black boxes, no mystery outputs. - -```python -from semantica.provenance import ProvenanceManager - -prov = ProvenanceManager(storage_path="./provenance.db") - -# Track where every entity came from -prov.track_entity( - entity_id="acme_corp", - source="contracts/acme_master_agreement_2024.pdf", - metadata={"page": 1, "confidence": 0.97, "extractor": "NamedEntityRecognizer"}, -) - -prov.track_relationship( - relationship_id="alice_works_for_acme", - source_entity_id="alice_chen", - target_entity_id="acme_corp", - source="hr_records/employees_q1_2024.csv", -) - -# Answer "where did this come from?" -lineage = prov.get_lineage("acme_corp") -trail = prov.trace_lineage("alice_chen") # full ancestor chain -entry = prov.get_provenance("acme_corp") -``` - ---- - -### `semantica.ontology`: OWL Generation, SHACL Validation - -Generate ontologies from data, validate shapes, and manage your vocabulary. - -```python -from semantica.ontology import OntologyGenerator, OntologyValidator - -data = { - "entities": [ - {"id": "acme_corp", "type": "Organization", "industry": "SaaS", "founded": 2012}, - {"id": "alice_chen", "type": "Person", "role": "CTO", "since": 2019}, - ], - "relationships": [ - {"source": "alice_chen", "target": "acme_corp", "type": "works_for"}, - ], -} - -gen = OntologyGenerator(base_uri="https://semantica.dev/ontology/") -ontology = gen.generate_ontology(data) -classes = gen.infer_classes(data) -props = gen.infer_properties(data, classes) -optimized = gen.optimize_ontology(ontology) - -# Validate against SHACL shapes -validator = OntologyValidator() -report = validator.validate(ontology) -# → ValidationResult(conforms=True, errors=[], warnings=[]) -``` - ---- - -### `semantica.conflicts`: Conflict Detection & Resolution - -Detect and resolve conflicting facts from multiple sources before they corrupt your knowledge base. - -```python -from semantica.conflicts import ConflictDetector, ConflictResolver, SourceTracker - -entities_from_source_a = [ - {"id": "alice_chen", "role": "CTO", "salary": 250_000, "start_date": "2019-03-01"}, -] -entities_from_source_b = [ - {"id": "alice_chen", "role": "VP Eng", "salary": 275_000, "start_date": "2019-03-01"}, -] - -# Detect all conflict types: value, type, relationship, temporal, logical -detector = ConflictDetector() -conflicts = detector.detect_conflicts(entities_from_source_a + entities_from_source_b) -# → [Conflict(entity="alice_chen", field="role", values=["CTO","VP Eng"], severity="HIGH"), -# Conflict(entity="alice_chen", field="salary", values=[250000,275000], severity="MEDIUM")] - -# Resolve using multiple strategies -resolver = ConflictResolver() -resolved = resolver.resolve(conflicts, strategy="credibility_weighted") # weighted by source trust -resolved = resolver.resolve(conflicts, strategy="temporal") # prefer most recent -resolved = resolver.resolve(conflicts, strategy="voting") # majority wins - -# Track source credibility over time -tracker = SourceTracker() -tracker.track("source_a", credibility=0.85) -tracker.track("source_b", credibility=0.72) -``` - ---- - -### `semantica.deduplication`: Entity Resolution at Scale - -Block, cluster, and merge duplicates with semantic similarity. **6.98× faster** than baseline. - -```python -from semantica.deduplication import DuplicateDetector, EntityMerger - -entities = [ - {"id": "e1", "name": "Acme Corporation", "domain": "acme.com"}, - {"id": "e2", "name": "Acme Corp.", "domain": "acme.com"}, - {"id": "e3", "name": "ACME Corp", "domain": "acme.co"}, - {"id": "e4", "name": "Globex Industries", "domain": "globex.com"}, -] - -detector = DuplicateDetector(similarity_threshold=0.75, use_clustering=True) -candidates = detector.detect_duplicates(entities) -groups = detector.detect_duplicate_groups(entities) -# → DuplicateGroup(entities=["e1","e2","e3"], confidence=0.91, strategy="semantic+blocking") - -merger = EntityMerger(preserve_provenance=True) -ops = merger.merge_duplicates(entities, strategy="keep_most_complete") -history = merger.get_merge_history() -``` - ---- - -### `semantica.normalize`: Data Normalization & Cleaning - -Standardize text, entities, dates, numbers, and encodings before building your knowledge graph. - -```python -from semantica.normalize import ( - TextNormalizer, - EntityNormalizer, - DateNormalizer, - NumberNormalizer, - DataCleaner, -) - -# Unicode, whitespace, casing, HTML tags, smart quotes -text = TextNormalizer().normalize(" Acme Corp.’s Q4 report… ") -# → "Acme Corp.'s Q4 report..." - -# Alias resolution + entity disambiguation with confidence scores -names = EntityNormalizer().normalize_entity("ACME Corp.") -# → NormalizedEntity(canonical="Acme Corporation", type="Organization", confidence=0.91) - -# Natural language date parsing with timezone conversion -dt = DateNormalizer().normalize_date("3 weeks ago") -# → datetime(2026, 5, 22, tzinfo=UTC) - -# Unit conversion and currency normalization -price = NumberNormalizer().normalize("$1.25M USD") -# → NormalizedNumber(value=1_250_000, currency="USD") - -# Deduplicate and impute missing values across a dataset -clean = DataCleaner().clean(records, dedup_threshold=0.9, fill_missing="mean") -``` - ---- - -### `semantica.pipeline`: Pipeline DSL - -Compose ingestion, extraction, and graph-building into a declarative, parallel pipeline. - -```python -from semantica.pipeline import PipelineBuilder, ExecutionEngine - -pipeline = ( - PipelineBuilder() - .add_step("ingest", step_type="ingest", source="./contracts/", recursive=True) - .add_step("extract", step_type="ner_extract") - .add_step("relations", step_type="relation_extract") - .add_step("build_kg", step_type="kg_build", merge_entities=True) - .add_step("deduplicate", step_type="deduplicate", threshold=0.75) - .add_step("export", step_type="export", format="turtle", output="kg.ttl") - .connect_steps("ingest", "extract") - .connect_steps("extract", "relations") - .connect_steps("relations", "build_kg") - .connect_steps("build_kg", "deduplicate") - .connect_steps("deduplicate", "export") - .set_parallelism(4) - .build(name="contracts_pipeline") -) - -engine = ExecutionEngine() -result = engine.execute(pipeline) -status = engine.get_status(pipeline) -progress = engine.get_progress(pipeline) -``` - ---- - -### Temporal Intelligence: Bi-Temporal Graphs & Time Travel - -Track when facts were true *in the world* vs. when they were *recorded*, and query either axis. - -```python -from semantica.context import ContextGraph -from semantica.kg import ( - BiTemporalFact, - TemporalGraphQuery, - TemporalVersionManager, - TemporalNormalizer, -) -from datetime import datetime +from semantica.context import ContextGraph, AgentContext +from semantica.vector_store import VectorStore graph = ContextGraph(advanced_analytics=True) -graph.add_node("alice_chen", "Person", role="VP Engineering") -graph.add_node("acme_corp", "Organization", valuation=1_200_000_000) -# Point-in-time snapshots - replay history without reprocessing -snapshot_2023 = graph.state_at("2023-06-01") -snapshot_2024 = graph.state_at("2024-01-01") +# Add nodes with typed properties +graph.add_node("acme_corp", "Organization", name="Acme Corp", industry="SaaS") +graph.add_node("alice_chen", "Person", name="Alice Chen", role="CTO") +graph.add_node("contract_001", "Contract", value=2_400_000, currency="USD") -# Bi-temporal facts - valid_time is when true in the world; -# recorded_at is when you learned about it -fact = BiTemporalFact( - valid_from=datetime(2024, 3, 1), - valid_until=datetime(2025, 1, 1), - recorded_at=datetime(2024, 3, 5), -) +# Add typed, weighted edges (extra kwargs become edge metadata) +graph.add_edge("alice_chen", "acme_corp", edge_type="works_for", since="2019-03-01") +graph.add_edge("acme_corp", "contract_001", edge_type="party_to", signed="2024-01-15") -# Allen interval algebra - 13 temporal relations (before, during, overlaps, etc.) -tq = TemporalGraphQuery(graph) -facts_in_window = tq.query_time_range("2024-01-01", "2024-12-31") +# BFS traversal - hop through the graph from any node +neighbors = graph.get_neighbors("acme_corp", hops=2) -# Normalize natural language temporal expressions -norm = TemporalNormalizer() -dt = norm.normalize("last quarter") # → datetime range for Q1 2026 -``` +# Point-in-time snapshot - the graph as it existed on any past date +snapshot = graph.state_at("2024-01-01") ---- - -### `semantica.export`: RDF, OWL, Parquet, Cypher, JSON-LD - -Export to any format required by regulators, graph databases, or downstream systems. - -```python -from semantica.export import ( - RDFExporter, - JSONExporter, - ParquetExporter, - LPGExporter, - ReportGenerator, -) - -kg = {"entities": [...], "relationships": [...]} - -rdf = RDFExporter() -turtle_str = rdf.export_to_rdf(kg, format="turtle") # returns string -jsonld_str = rdf.export_to_rdf(kg, format="json-ld") - -rdf.export(kg, "kg_audit.ttl", format="turtle") -rdf.export(kg, "kg_audit.jsonld", format="json-ld") -rdf.export(kg, "kg_audit.nt", format="n-triples") - -# Columnar analytics - Snappy-compressed Parquet -ParquetExporter().export(kg, "kg_snapshot.parquet", compression="snappy") - -# JSON knowledge graph -JSONExporter().export_knowledge_graph(kg, "kg.json") - -# Neo4j / Memgraph Cypher statements for graph database import -LPGExporter().export(kg, "kg_import.cypher", method="cypher") - -# Human-readable HTML / Markdown report -ReportGenerator().generate(kg, "audit_report.html", format="html") -``` - ---- - -### `semantica.visualization`: Interactive Graph Workbench - -Render force-directed graphs, community maps, ontology hierarchies, and temporal dashboards. - -```python -from semantica.visualization import ( - KGVisualizer, - OntologyVisualizer, - EmbeddingVisualizer, - TemporalVisualizer, -) -import numpy as np - -kg = {"entities": [...], "relationships": [...]} - -# Interactive force-directed graph (opens in browser) -viz = KGVisualizer(layout="force", color_scheme="default") -viz.visualize_network(kg, output="interactive", file_path="kg.html") -viz.visualize_communities(kg, communities, output="interactive") -viz.visualize_centrality(kg, centrality, centrality_type="degree") -viz.visualize_entity_types(kg, output="html", file_path="entity_types.html") - -# Ontology class hierarchy -OntologyVisualizer().visualize_hierarchy(ontology, output="interactive") - -# 2D embedding projection (UMAP / t-SNE / PCA) -EmbeddingVisualizer().visualize_2d_projection( - embeddings=np.array([...]), - labels=["entity_a", "entity_b"], - method="umap", -) - -# Timeline scrubber - watch the graph evolve -TemporalVisualizer().visualize_timeline(kg, output="interactive") -``` - ---- - -### Multi-Agent Shared Context with Agno - -One shared intelligence layer. All agents read and write to the same context graph. - -```python -# pip install semantica[agno] -from agno.agent import Agent -from agno.team import Team -from agno.models.anthropic import Claude -from semantica.context import ContextGraph -from semantica.vector_store import VectorStore -from integrations.agno import AgnoSharedContext, AgnoDecisionKit, AgnoKGToolkit - -shared = AgnoSharedContext( - vector_store=VectorStore(backend="faiss"), - knowledge_graph=ContextGraph(advanced_analytics=True), - decision_tracking=True, -) - -researcher = Agent( - name="Researcher", - model=Claude(id="claude-sonnet-4-6"), - memory=shared.bind_agent("researcher"), - tools=[AgnoKGToolkit(context=shared)], -) -analyst = Agent( - name="Analyst", - model=Claude(id="claude-sonnet-4-6"), - memory=shared.bind_agent("analyst"), - tools=[AgnoDecisionKit(context=shared)], -) - -team = Team(agents=[researcher, analyst], mode="coordinate") -# Researcher's findings are instantly available to the Analyst - no copy, no sync -``` - -→ [runnable notebooks in the cookbook](https://github.com/semantica-agi/semantica/tree/main/cookbook), each self-contained and runnable in under 5 minutes - ---- - -## Recipes - -Copy-paste patterns for the most common use cases. - -### End-to-End GraphRAG Pipeline - -```python -from semantica.ingest import FileIngestor -from semantica.split import TextSplitter -from semantica.semantic_extract import NamedEntityRecognizer, RelationExtractor -from semantica.kg import GraphBuilder -from semantica.vector_store import VectorStore, HybridSearch -from semantica.context import AgentContext - -# 1. Ingest -docs = FileIngestor().ingest_directory("./docs/", recursive=True) - -# 2. Entity-aware chunking - never splits an entity across a chunk boundary -splitter = TextSplitter(method="entity_aware", chunk_size=1000) -chunks = [splitter.split(doc["text"]) for doc in docs] - -# 3. Extract entities and relations -ner = NamedEntityRecognizer(confidence_threshold=0.7) -rel_ext = RelationExtractor(confidence_threshold=0.6) -entities = [ner.extract_entities(chunk) for chunk_group in chunks for chunk in chunk_group] - -# 4. Build KG -kg = GraphBuilder(merge_entities=True, enable_temporal=True).build(docs) - -# 5. Hybrid retrieval +# AgentContext - high-level API for agent memory workflows vs = VectorStore(backend="faiss") -ctx = AgentContext(vector_store=vs, knowledge_graph=kg) -ctx.store("Alice approved the Acme renewal in Q1 2024", conversation_id="c1") - -results = HybridSearch(vector_store=vs).search("who approved the renewal?") +ctx = AgentContext(vector_store=vs, knowledge_graph=graph) +ctx.store("Alice approved the Acme renewal in Q1 2024", conversation_id="conv_001") +retrieved = ctx.retrieve("who approved the Acme contract?") ``` +**Why graph over embeddings:** traversal finds connections embeddings miss (a person 3 hops from a contract); every node carries provenance so you can always ask *"where did this come from?"*; conflicts are flagged before they corrupt your knowledge base; point-in-time snapshots let you replay history without reprocessing. + --- -### Audit Trail for a Regulated Decision +## Recipe: Audit Trail for a Regulated Decision + +The flagship pattern: record a causally-linked decision chain, attach provenance to every entity, and export a regulator-ready audit trail. ```python from semantica.context import ContextGraph @@ -909,55 +284,32 @@ kg = graph.export_graph() RDFExporter().export(kg, "audit_trail.ttl", format="turtle") ``` ---- - -### AML Rules Engine - -```python -from semantica.reasoning import ReteEngine, Rule, Fact, RuleType - -rete = ReteEngine() -rete.build_network([ - Rule( - rule_id="sanctions_check", - name="Flag sanctioned-country transactions", - conditions=[ - {"field": "amount", "operator": ">", "value": 10_000}, - {"field": "country", "operator": "in", "value": ["IR", "KP", "SY", "CU"]}, - ], - conclusion="flag_for_compliance_review", - rule_type=RuleType.IMPLICATION, - ), -]) -rete.add_fact(Fact("tx_99", "transaction", [{"amount": 25_000, "country": "IR"}])) -matches = rete.match_patterns() -# → [{"rule": "sanctions_check", "matched_facts": ["tx_99"], -# "conclusion": "flag_for_compliance_review"}] -``` +More recipes (GraphRAG pipelines, an AML rules engine, ontology-to-KG in one pass) are in **[PLATFORM_REFERENCE.md](PLATFORM_REFERENCE.md#more-recipes)**. --- -### Ontology-to-Knowledge-Graph in One Pass +## Explore the Platform -```python -from semantica.ingest import FileIngestor -from semantica.semantic_extract import NamedEntityRecognizer, RelationExtractor -from semantica.kg import GraphBuilder -from semantica.ontology import OntologyGenerator, OntologyValidator -from semantica.export import RDFExporter +Every module below is independently importable, with working code samples; use one or all of them. -sources = FileIngestor().ingest_directory("./contracts/") -ner = NamedEntityRecognizer(confidence_threshold=0.7) -entities = ner.extract_entities_batch([s["text"] for s in sources]) +| Module | What it does | +| --- | --- | +| [`semantica.ingest`](PLATFORM_REFERENCE.md#semanticaingest-multi-source-ingestion) | Files, web, databases, APIs, streams, email, Git, Parquet, Snowflake, MCP | +| [`semantica.semantic_extract`](PLATFORM_REFERENCE.md#semanticasemantic_extract-ner-relations-events-triplets) | NER, relation extraction, event detection, triplet generation | +| [`semantica.kg`](PLATFORM_REFERENCE.md#semanticakg-knowledge-graph-construction--analysis) | Graph construction, centrality, communities, link prediction | +| [`semantica.reasoning`](PLATFORM_REFERENCE.md#semanticareasoning-forward-chaining-rete-datalog-sparql) | Forward chaining, Rete, Datalog, SPARQL, fully explainable | +| [`semantica.vector_store`](PLATFORM_REFERENCE.md#semanticavector_store-hybrid--filtered-semantic-search) | FAISS, Qdrant, Weaviate, Milvus, Pinecone, PgVector, hybrid search | +| [`semantica.split`](PLATFORM_REFERENCE.md#semanticasplit-graphrag-native-document-chunking) | Entity-aware, relation-aware, ontology-aware chunking for GraphRAG | +| [`semantica.provenance`](PLATFORM_REFERENCE.md#semanticaprovenance-w3c-prov-o-lineage) | W3C PROV-O lineage on every fact | +| [`semantica.ontology`](PLATFORM_REFERENCE.md#semanticaontology-owl-generation-shacl-validation) | OWL generation, SHACL validation, SKOS vocabularies | +| [`semantica.conflicts`](PLATFORM_REFERENCE.md#semanticaconflicts-conflict-detection--resolution) | Detect and resolve conflicting facts across sources | +| [`semantica.deduplication`](PLATFORM_REFERENCE.md#semanticadeduplication-entity-resolution-at-scale) | Entity resolution at scale, 6.98× faster than baseline | +| [`semantica.export`](PLATFORM_REFERENCE.md#semanticaexport-rdf-owl-parquet-cypher-json-ld) | RDF, OWL, Parquet, Cypher, JSON-LD | +| [`semantica.visualization`](PLATFORM_REFERENCE.md#semanticavisualization-interactive-graph-workbench) | Force-directed graphs, ontology hierarchies, temporal dashboards | +| [Temporal Intelligence](PLATFORM_REFERENCE.md#temporal-intelligence-bi-temporal-graphs--time-travel) | Bi-temporal facts, Allen interval algebra, time travel | +| [Multi-Agent (Agno)](PLATFORM_REFERENCE.md#multi-agent-shared-context-with-agno) | One shared context graph across every agent on a team | -kg = GraphBuilder(merge_entities=True).build(sources) -gen = OntologyGenerator(base_uri="https://myco.dev/ontology/") -ont = gen.generate_ontology({"entities": entities[0], "relationships": []}) - -report = OntologyValidator().validate(ont) -if report.conforms: - RDFExporter().export({"entities": entities[0]}, "ontology.ttl", format="turtle") -``` +**→ [Full Platform Reference](PLATFORM_REFERENCE.md)**: every module, more recipes, the full integrations matrix, MCP tool list, and REST endpoints. --- @@ -997,203 +349,13 @@ Start with `semantica`, verify with `doctor`, build a graph, and explore the com ## Integrations -Native plugin bundles across major editors, a full-featured MCP server, a comprehensive REST API, and first-class Agno support. All LLM providers already supported: OpenAI · Anthropic · Gemini · Mistral · Llama · Groq · Cohere · Azure · Bedrock · Ollama · DeepSeek · HuggingFace and more via LiteLLM +Native plugin bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw; a full-featured MCP server for any MCP-compatible client; a comprehensive REST API; and first-class Agno support for multi-agent shared context. Every major LLM provider is already supported via `semantica.llms` and LiteLLM: OpenAI, Anthropic, Gemini, Mistral, Llama, Groq, Cohere, Azure, Bedrock, Ollama, DeepSeek, HuggingFace, and more. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Native Plugin BundleMCP Server + Plugin
-Claude Code
-Claude Code
-Skills · agents · hooks -
-Cursor
-Cursor
-Skills · agents -
-Codex CLI
-Codex CLI
-Skills · agents -
-Windsurf
-Windsurf
-plugin -
-Cline
-Cline
-plugin -
-Continue
-Continue
-plugin -
-VS Code
-VS Code
-plugin -
-OpenClaw
-OpenClaw
-MCP + plugin -
MCP ServerREST API
-Claude Desktop
-Claude Desktop
-MCP server -
-GitHub Copilot
-GitHub Copilot
-REST API -
-Roo Code
-Roo Code
-REST API -
-Goose
-Goose
-REST API -
-Kilo Code
-Kilo Code
-REST API -
-Aider
-Aider
-REST API -
-Amazon Q
-Amazon Q
-REST API -
-Zed
-Zed
-REST API -
- -### Agentic Frameworks - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Native Integration
-Agno
-Agno
-First-class · pip install semantica[agno] -
Already Supported via REST API & MCP
-LangChain
-LangChain
-REST API · MCP -
-LangGraph
-LangGraph
-REST API · MCP -
-CrewAI
-CrewAI
-REST API · MCP -
-LlamaIndex
-LlamaIndex
-REST API · MCP -
-AutoGen
-AutoGen
-REST API · MCP -
-OpenAI Agents SDK
-OpenAI Agents
-REST API · MCP -
-Google ADK
-Google ADK
-REST API · MCP -
Native SDK Integration (Coming Soon)
-LangChain
-LangChain
-Dedicated toolkit -
-CrewAI
-CrewAI
-Dedicated toolkit -
-LlamaIndex
-LlamaIndex
-Dedicated toolkit -
-AutoGen
-AutoGen
-Dedicated toolkit -
-OpenAI Agents SDK
-OpenAI Agents
-Dedicated toolkit -
-Google ADK
-Google ADK
-Dedicated toolkit -
- ---- - -### MCP Server - -Connect any MCP-compatible client (Claude Desktop, Windsurf, Cline, VS Code) in 30 seconds: +MCP setup in 30 seconds: ```bash python -m semantica.mcp_server -# or via the installed entry point -semantica-mcp +# or: semantica-mcp ``` ```json @@ -1204,62 +366,7 @@ semantica-mcp } ``` -**Tools exposed over MCP:** - -| Tool | What it does | -| --- | --- | -| `extract_entities` | NER on any text | -| `extract_relations` | Relation extraction | -| `record_decision` | Persist a decision node | -| `query_decisions` | Search decision history | -| `find_precedents` | Semantic precedent lookup | -| `get_causal_chain` | Full causal ancestry | -| `add_entity` | Add a KG node | -| `add_relationship` | Add a KG edge | -| `run_reasoning` | Execute rule set | -| `get_graph_analytics` | Centrality, communities | -| `export_graph` | Export to RDF/JSON/Parquet | -| `get_graph_summary` | Graph statistics | - ---- - -### REST API - -```bash -# Start the backend -python -m semantica.server # port 8000 - -# Extract entities via REST -curl -X POST http://localhost:8000/api/extract/entities \ - -H "Content-Type: application/json" \ - -d '{"text": "Apple CEO Tim Cook announced record earnings."}' - -# Record a decision -curl -X POST http://localhost:8000/api/decisions \ - -H "Content-Type: application/json" \ - -d '{ - "category": "vendor_selection", - "scenario": "Choose ML cloud provider", - "reasoning": "Best GPU availability and pricing", - "outcome": "selected_aws", - "confidence": 0.91 - }' - -# Query the knowledge graph -curl http://localhost:8000/api/graph/neighbors/acme_corp?hops=2 -``` - -**REST endpoints span:** `extract` · `kg` · `decisions` · `reasoning` · `provenance` · `ontology` · `embeddings` · `search` · `export` · `pipeline` · `temporal` · `deduplication` - ---- - -### Plugin Bundles - -**Domain skills:** `extract` · `ingest` · `query` · `ontology` · `validate` · `deduplicate` · `embed` · `reason` · `decision` · `causal` · `temporal` · `provenance` · `policy` · `explain` · `export` · `change` · `visualize` - -**Specialized agents:** `kg-assistant` · `decision-advisor` · `explainability` - -Bundles for Claude Code, Cursor, Codex, Windsurf, Cline, Continue, VS Code, and OpenClaw in [`plugins/`](plugins/). +**→ [Full integrations matrix, MCP tool list, and REST endpoints](PLATFORM_REFERENCE.md#integrations)** --- @@ -1285,30 +392,7 @@ semantica-explorer --graph my_graph.json # Dashboard opens at http://127.0.0.1:8000 ``` -For contributor / dev-server setup, see the full local setup guide: - -→ **[explorer/README.md: Local Setup Guide](explorer/README.md)** - ---- - -## Features at a Glance - -| Capability | Highlights | -| --- | --- | -| **Context Graphs** | Queryable graph of entities, decisions, relationships; causal links; cross-graph navigation | -| **Decision Intelligence** | `record_decision` · `trace_decision_chain` · `find_similar_decisions` · `analyze_decision_impact` · `check_decision_rules` | -| **Temporal Intelligence** | Point-in-time snapshots · Allen interval algebra (13 relations) · `TemporalNormalizer` · bi-temporal provenance | -| **Distance Intelligence** | N×N semantic distance matrices · ego-mode visualization · distance bands · 10× embedding cache | -| **Semantic Extraction** | NER · relation extraction · event detection · triplet generation · coreference · **6.98×** faster dedup | -| **Reasoning Engines** | Forward chaining · Rete · deductive · abductive · SPARQL · Datalog with explainable output | -| **GraphRAG Chunking** | Entity-aware · relation-aware · graph-based · ontology-aware · community-detection chunking | -| **Conflict Detection** | Value / type / relationship / temporal / logical conflicts · 5 resolution strategies | -| **Provenance** | W3C PROV-O · every fact traced to source · audit log export JSON/CSV/RDF | -| **Ontology Hub** | SHACL Studio · visual editor · cross-ontology alignments · 5-dimension health dashboard | -| **Vector Store** | FAISS · Pinecone · Weaviate · Qdrant · Milvus · PgVector · hybrid + filtered search | -| **Graph Databases (LPG)** | Neo4j · FalkorDB · Apache AGE · AWS Neptune | -| **Triple Stores (RDF)** | Blazegraph · Apache Jena · Eclipse RDF4J · unified `TripletStore` interface · SPARQL query & bulk load | -| **LLM Providers** | **All already supported today:** OpenAI (GPT-4o, o1, o3) · Anthropic (Claude 4) · Google Gemini · Mistral · Meta Llama · Groq · Cohere · Azure OpenAI · AWS Bedrock · Ollama · DeepSeek · Perplexity · Together AI · Fireworks AI · Replicate · HuggingFace · via `semantica.llms` and LiteLLM | +For contributor / dev-server setup: **[explorer/README.md: Local Setup Guide](explorer/README.md)** --- @@ -1333,13 +417,14 @@ For contributor / dev-server setup, see the full local setup guide: ## Built for High-Stakes Domains -Semantica is designed for environments where AI outputs must be explainable, auditable, and defensible. +Semantica is designed for environments where AI outputs must be explainable, auditable, and defensible, and where the data itself can't leave your infrastructure. Self-hostable with zero vendor lock-in, it's built as much for organizations handling confidential or classified data as for regulated industries chasing an audit trail: +- **Finance:** Loan underwriting audit trails, fraud detection, AML compliance, regulatory risk knowledge graphs - **Healthcare:** Clinical decision support, drug interaction graphs, and patient safety audit trails -- **Finance:** Fraud detection, AML compliance, regulatory risk knowledge graphs, and loan decision audit trails - **Legal:** Evidence-backed research, contract analysis, case law reasoning, and privilege tracking +- **Government & Defense:** Policy decision records, classified information governance, and regulatory reporting, fully self-hosted with no data leaving your perimeter +- **Law Enforcement:** Case linkage, evidence provenance chains, and investigative knowledge graphs that hold up under legal scrutiny - **Cybersecurity:** Threat attribution, incident response timelines, and IOC provenance tracking -- **Government:** Policy decision records, classified information governance, and regulatory reporting - **Autonomous Systems:** Decision logs, safety validation, and explainable AI for certification --- @@ -1358,11 +443,12 @@ pip install semantica[graph-neo4j] # Neo4j graph store (LPG) pip install semantica[graph-falkordb] # FalkorDB graph store (LPG) pip install semantica[graph-apache-age] # Apache AGE graph store (LPG) pip install semantica[graph-amazon-neptune] # AWS Neptune graph store (LPG) -# RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J) need no extra — +# RDF triple stores (Blazegraph, Apache Jena, Eclipse RDF4J) need no extra: # semantica.triplet_store talks SPARQL over HTTP using the core `requests` dependency pip install semantica[vectorstore-qdrant] # Qdrant vector store pip install semantica[vectorstore-pinecone] # Pinecone vector store pip install semantica[db-snowflake] # Snowflake +pip install semantica[db-databricks] # Databricks (SDK + SQL connector) pip install semantica[ingest-parquet] # Parquet / PyArrow pip install semantica[ingest-arrow] # Apache Arrow, Feather, IPC pip install semantica[viz] # HTML interactive visualization @@ -1381,7 +467,7 @@ cd semantica && pip install -e ".[dev]" && pytest tests/ ## Enterprise -On-premises deployment · Private cloud · Custom domain implementations · SLA-backed support · Professional services for regulated industries (healthcare, finance, legal, government). +On-premises deployment · Private cloud · Custom domain implementations · SLA-backed support · Professional services for regulated industries (finance, healthcare, legal, government). **[getsemantica.ai](https://getsemantica.ai/)** for enterprise solutions and pricing.