mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a047ebf74f | ||
|
|
88c12b1867 | ||
|
|
094bb8d82b | ||
|
|
7ff2fd9981 | ||
|
|
0a555145e4 | ||
|
|
994e58a170 | ||
|
|
244144dee3 | ||
|
|
5dfca85500 | ||
|
|
f3dd7a05bd | ||
|
|
d03a237278 | ||
|
|
f3ac9fbffa | ||
|
|
6856580a7a | ||
|
|
4a282628ea | ||
|
|
c73e35a2fe | ||
|
|
a99f18b71b | ||
|
|
84b90b45a2 | ||
|
|
d7d589f64e | ||
|
|
d3366bbcf0 | ||
|
|
95c5486d22 | ||
|
|
8b6e8608c3 | ||
|
|
315e2edb14 | ||
|
|
a93ed8f13a | ||
|
|
921bf18041 | ||
|
|
971b42631e | ||
|
|
3e4bc8521f | ||
|
|
521e2e27d8 | ||
|
|
c8f745cef0 | ||
|
|
c307011311 | ||
|
|
79ff296001 | ||
|
|
2a28e833b9 | ||
|
|
30cede84c7 | ||
|
|
9c8d0c032b | ||
|
|
e0e42dc539 | ||
|
|
f59fe1d689 | ||
|
|
e7e67bd673 | ||
|
|
1cfbf626d0 | ||
|
|
5d5928badf | ||
|
|
2f94986b01 | ||
|
|
3e7863aa23 | ||
|
|
d23ca2d743 | ||
|
|
507a1f9c71 | ||
|
|
afc94ad059 | ||
|
|
bad6bd0326 | ||
|
|
3207eb3b41 | ||
|
|
3457f4d7c8 | ||
|
|
7bbf8e9881 | ||
|
|
a163a46c56 | ||
|
|
6ee19d971e | ||
|
|
0f48b5bc87 | ||
|
|
e7bf664868 | ||
|
|
ff7768f1ad | ||
|
|
e0fce67ab2 | ||
|
|
ea477f9b32 | ||
|
|
36e94cdbbc |
@@ -106,3 +106,6 @@ sample_data/
|
||||
.personal/
|
||||
.local/
|
||||
*.local
|
||||
|
||||
# Test Results
|
||||
test_results.txt
|
||||
|
||||
+1
-1
@@ -58,7 +58,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Production-ready quality assurance modules
|
||||
- Comprehensive documentation with MkDocs
|
||||
- Cookbook with interactive tutorials
|
||||
- Support for multiple vector stores (Pinecone, Weaviate, Qdrant, FAISS)
|
||||
- Support for multiple vector stores (Weaviate, Qdrant, FAISS)
|
||||
- Support for multiple graph databases (Neo4j, NetworkX, RDFLib)
|
||||
- Temporal knowledge graph support
|
||||
- Conflict detection and resolution
|
||||
|
||||
-237
@@ -1,237 +0,0 @@
|
||||
# Add Intelligence Cookbook Notebooks with MCP, Agents, and Orchestrator-Worker Pattern
|
||||
|
||||
## Overview
|
||||
Add comprehensive intelligence-focused notebooks to `cookbook/use_cases/intelligence/` with complete end-to-end pipelines. The **Intelligence Analysis** notebook will use the **Orchestrator-Worker Pattern** with detailed graph analytics, hybrid RAG, and ontology building. Update documentation in `docs/cookbook.md` and `docs/use-cases.md`.
|
||||
|
||||
## New Notebooks to Create
|
||||
|
||||
### 1. Criminal Network Analysis (`Criminal_Network_Analysis.ipynb`)
|
||||
Complete pipeline from data sources to GraphRAG with agent-based workflows:
|
||||
- **Data Sources**: Ingest from police reports, court records, surveillance data, communication logs
|
||||
- **MCP Integration**: Utilize MCP for accessing public records databases, court records APIs, and real-time data streams
|
||||
- **Semantica Agents**:
|
||||
- Data Gathering Agent (autonomous data collection with AgentMemory)
|
||||
- Network Analysis Agent (graph analytics and community detection)
|
||||
- Pattern Detection Agent (identifying suspicious patterns)
|
||||
- Report Generation Agent (compiling intelligence reports)
|
||||
- **Agent Coordination**: Use Pipeline module for parallel agent workflows
|
||||
- **Agent Memory**: AgentMemory for persistent context across interactions
|
||||
- **Complete Pipeline**: Data sources → MCP → Parsing → Extraction → KG → Graph Analytics → GraphRAG → Agent Analysis → Visualization → Reporting
|
||||
|
||||
### 2. Law Enforcement and Forensics (`Law_Enforcement_Forensics.ipynb`)
|
||||
Complete forensic analysis pipeline with agent-based workflows:
|
||||
- **Data Sources**: Case files, evidence logs, witness statements, forensic reports, crime scene data
|
||||
- **Semantica Agents**:
|
||||
- Evidence Collection Agent (autonomous evidence gathering)
|
||||
- Timeline Analysis Agent (temporal case timelines)
|
||||
- Cross-Case Correlation Agent (connections across cases)
|
||||
- Forensic Report Agent (comprehensive report generation)
|
||||
- **Agent Coordination**: Multi-agent pipeline for parallel evidence processing
|
||||
- **Agent Memory**: Persistent memory for case context and evidence chains
|
||||
- **Complete Pipeline**: Case files → Parsing → Evidence Extraction → Temporal KG → Graph Analytics → GraphRAG → Agent Analysis → Visualization → Reporting
|
||||
|
||||
### 3. Intelligence Analysis (`Intelligence_Analysis.ipynb`) - **ORCHESTRATOR-WORKER PATTERN**
|
||||
Comprehensive intelligence analysis using **Orchestrator-Worker Pattern** with detailed implementation:
|
||||
|
||||
#### Orchestrator-Worker Architecture:
|
||||
- **Orchestrator**: ExecutionEngine coordinates all workers using PipelineBuilder and ParallelismManager
|
||||
- **Worker 1 - Data Ingestion Worker**: Handles multi-source data ingestion (FileIngestor, WebIngestor, StreamIngestor, FeedIngestor, DBIngestor)
|
||||
- **Worker 2 - Ontology Building Worker**: Complete 6-stage ontology generation pipeline
|
||||
- Stage 1: Semantic Network Parsing (extract domain concepts)
|
||||
- Stage 2: YAML-to-Definition (transform concepts to class definitions)
|
||||
- Stage 3: Definition-to-Types (map to OWL types)
|
||||
- Stage 4: Hierarchy Generation (build taxonomic structures)
|
||||
- Stage 5: TTL Generation (generate OWL/Turtle syntax)
|
||||
- Stage 6: Symbolic Validation (HermiT/Pellet reasoning)
|
||||
- **Worker 3 - Graph Construction Worker**: Builds knowledge graphs (GraphBuilder, TemporalGraphQuery)
|
||||
- **Worker 4 - Graph Analytics Worker**: Comprehensive graph analytics including:
|
||||
- Centrality Measures: PageRank, Betweenness, Closeness, Eigenvector
|
||||
- Community Detection: Louvain algorithm
|
||||
- Connectivity Analysis: Path finding, shortest paths, connectivity metrics
|
||||
- Graph Metrics: Density, clustering coefficient, diameter, radius
|
||||
- **Worker 5 - Hybrid RAG Worker**: Complete hybrid RAG implementation:
|
||||
- Vector Store setup with embeddings
|
||||
- Knowledge Graph queries
|
||||
- Hybrid Search (combining vector similarity + graph traversal)
|
||||
- Context Retrieval (ContextRetriever)
|
||||
- Query Orchestration across KG and vector store
|
||||
- **Worker 6 - Intelligence Analysis Worker**: Threat assessment, geospatial analysis, pattern detection
|
||||
- **Worker 7 - Report Generation Worker**: Compiles comprehensive intelligence reports
|
||||
|
||||
#### Complete Features:
|
||||
- **Data Sources**: OSINT feeds, threat intelligence, social media, news, public records, geospatial data
|
||||
- **MCP Integration**: Real-time data fetching, web scraping, API integration, browser automation for OSINT
|
||||
- **Agent Memory**: Persistent memory for threat context and intelligence history
|
||||
- **Complete Pipeline**: OSINT sources → MCP → Orchestrator → Parallel Workers → Ontology → KG → Graph Analytics → Hybrid RAG → Intelligence Analysis → Visualization → Reporting
|
||||
|
||||
## Files to Create/Modify
|
||||
|
||||
### New Notebooks (in `cookbook/use_cases/intelligence/`)
|
||||
- `Criminal_Network_Analysis.ipynb`
|
||||
- `Law_Enforcement_Forensics.ipynb`
|
||||
- `Intelligence_Analysis.ipynb` (with Orchestrator-Worker Pattern)
|
||||
|
||||
### Documentation Updates
|
||||
- `docs/cookbook.md` - Add new notebooks to Intelligence section
|
||||
- `docs/use-cases.md` - Add use case cards for Criminal Network Analysis and Law Enforcement & Forensics
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Intelligence Analysis - Orchestrator-Worker Pipeline Structure:
|
||||
|
||||
1. **Orchestrator Setup** - Initialize ExecutionEngine, PipelineBuilder, ParallelismManager
|
||||
2. **Data Sources** - Multiple ingestion (FileIngestor, DBIngestor, WebIngestor, StreamIngestor, FeedIngestor)
|
||||
3. **MCP Integration** - External data access, web scraping, browser automation
|
||||
4. **Worker 1 - Data Ingestion Worker** - Parallel data gathering from multiple sources
|
||||
5. **Data Parsing** - Parse structured/unstructured data (JSONParser, XMLParser, CSVParser, DocumentParser, StructuredDataParser)
|
||||
6. **Data Normalization** - Clean and standardize (TextNormalizer, DataNormalizer)
|
||||
7. **Entity & Relation Extraction** - Extract entities, relationships, events (NERExtractor, RelationExtractor, TripleExtractor, EventDetector)
|
||||
8. **Worker 2 - Ontology Building Worker** - Complete 6-stage ontology generation:
|
||||
- Use OntologyGenerator, ClassInferrer, PropertyGenerator
|
||||
- Generate OWL/Turtle with OWLGenerator
|
||||
- Validate with OntologyValidator (HermiT/Pellet)
|
||||
9. **Worker 3 - Graph Construction Worker** - Build knowledge graphs:
|
||||
- GraphBuilder for entity/relationship graphs
|
||||
- TemporalGraphQuery for time-aware graphs
|
||||
10. **Worker 4 - Graph Analytics Worker** - All graph analytics:
|
||||
- GraphAnalyzer: PageRank, Betweenness, Closeness, Eigenvector centrality
|
||||
- CommunityDetector: Louvain community detection
|
||||
- ConnectivityAnalyzer: Path finding, shortest paths, connectivity
|
||||
- CentralityCalculator: All centrality measures
|
||||
- Graph metrics: density, clustering, diameter, radius
|
||||
11. **Worker 5 - Hybrid RAG Worker** - Complete hybrid RAG:
|
||||
- EmbeddingGenerator: Generate embeddings for entities and text
|
||||
- VectorStore: Store and index embeddings
|
||||
- HybridSearch: Combine vector similarity + graph queries
|
||||
- ContextRetriever: Retrieve relevant context from KG and vectors
|
||||
- Query orchestration: Coordinate queries across KG and vector store
|
||||
12. **Worker 6 - Intelligence Analysis Worker** - Threat assessment, geospatial analysis, pattern detection
|
||||
13. **Agent Memory Integration** - Store and retrieve agent context using AgentMemory
|
||||
14. **Orchestrator Coordination** - Coordinate all workers with parallel execution
|
||||
15. **Visualization** - Network graphs, analytics dashboards, maps (KGVisualizer, AnalyticsVisualizer, TemporalVisualizer)
|
||||
16. **Worker 7 - Report Generation Worker** - Compile comprehensive intelligence reports
|
||||
17. **Report Generation** - Professional HTML reports (ReportGenerator, HTMLExporter)
|
||||
|
||||
### Other Notebooks - Standard Pipeline Structure:
|
||||
|
||||
1. **Data Sources** - Multiple ingestion
|
||||
2. **MCP Integration** - (Criminal Network Analysis only)
|
||||
3. **Semantica Agent Setup** - Initialize AgentMemory, create specialized agents
|
||||
4. **Agent-Based Data Gathering** - Autonomous agents gather data
|
||||
5. **Data Parsing** - Parse structured/unstructured data
|
||||
6. **Data Normalization** - Clean and standardize
|
||||
7. **Entity & Relation Extraction** - Extract entities, relationships, events
|
||||
8. **Knowledge Graph Construction** - Build graphs
|
||||
9. **Agent-Based Analysis** - Specialized agents perform parallel analysis
|
||||
10. **Graph Analytics** - Community detection, centrality, connectivity
|
||||
11. **GraphRAG Implementation** - Embeddings, vector store, hybrid search
|
||||
12. **Agent Memory Integration** - Store and retrieve agent context
|
||||
13. **Detailed Analysis** - Reasoning, inference, pattern detection
|
||||
14. **Agent Coordination** - Pipeline module for multi-agent workflow orchestration
|
||||
15. **Visualization** - Network graphs, analytics dashboards, maps
|
||||
16. **Agent-Based Report Generation** - Agents compile comprehensive reports
|
||||
17. **Report Generation** - Professional HTML reports
|
||||
|
||||
### Semantica Agent Implementation:
|
||||
|
||||
- **AgentMemory**: Persistent context storage, memory retrieval, conversation history
|
||||
- **Pipeline Coordination**: PipelineBuilder, ExecutionEngine, ParallelismManager for multi-agent workflows
|
||||
- **Specialized Agents**: Each agent has specific role (data gathering, analysis, reporting)
|
||||
- **Agent Examples**: Code demonstrations of agent workflows with memory integration
|
||||
|
||||
### MCP Integration:
|
||||
|
||||
- **Intelligence Analysis**: MCP browser tools for OSINT, resources for external feeds
|
||||
- **Criminal Network Analysis**: MCP for public records, court databases, API integration
|
||||
- **Agent-MCP Coordination**: Agents use MCP for autonomous data gathering
|
||||
|
||||
### Notebook Structure:
|
||||
|
||||
#### Intelligence Analysis (Orchestrator-Worker Pattern):
|
||||
- Overview with Orchestrator-Worker pattern explanation
|
||||
- Semantica modules used (30+ modules including Orchestrator, Workers, Ontology, Graph Analytics, Hybrid RAG)
|
||||
- **Orchestrator Architecture**: Detailed explanation of orchestrator and worker roles
|
||||
- **Worker Implementation**: Detailed code for each worker (7 workers)
|
||||
- **Ontology Building**: Complete 6-stage ontology generation pipeline demonstration
|
||||
- **Graph Analytics**: All analytics methods (PageRank, Betweenness, Closeness, Eigenvector, Louvain, connectivity, paths)
|
||||
- **Hybrid RAG**: Complete implementation with KG queries + vector search, query orchestration
|
||||
- MCP integration demonstration
|
||||
- Step-by-step implementation with orchestrator coordinating workers
|
||||
- Parallel worker execution examples
|
||||
- Agent memory integration
|
||||
- Best practices for orchestrator-worker pattern
|
||||
- Best practices for agents and MCP
|
||||
- Conclusion with key takeaways
|
||||
|
||||
#### Other Notebooks:
|
||||
- Overview with complete pipeline description
|
||||
- Semantica modules used (20+ modules including AgentMemory, Pipeline)
|
||||
- Agent Architecture explanation
|
||||
- MCP integration demonstration (Criminal Network Analysis)
|
||||
- Step-by-step implementation with agent workflows
|
||||
- Agent memory integration examples
|
||||
- Multi-agent pipeline orchestration
|
||||
- Best practices for agents and MCP
|
||||
- Conclusion with key takeaways
|
||||
|
||||
## Key Implementation Details for Orchestrator-Worker Pattern:
|
||||
|
||||
### Orchestrator Code Example:
|
||||
```python
|
||||
from semantica.pipeline import PipelineBuilder, ExecutionEngine, ParallelismManager
|
||||
from semantica.ontology import OntologyGenerator
|
||||
from semantica.kg import GraphBuilder, GraphAnalyzer
|
||||
from semantica.vector_store import VectorStore, HybridSearch
|
||||
from semantica.context import AgentMemory
|
||||
|
||||
# Initialize orchestrator
|
||||
orchestrator = ExecutionEngine()
|
||||
parallelism_manager = ParallelismManager(max_workers=7)
|
||||
|
||||
# Define workers
|
||||
def data_ingestion_worker(sources):
|
||||
# Worker 1: Multi-source data ingestion
|
||||
pass
|
||||
|
||||
def ontology_building_worker(entities, relationships):
|
||||
# Worker 2: Complete 6-stage ontology generation
|
||||
ontology_gen = OntologyGenerator()
|
||||
ontology = ontology_gen.generate_ontology({"entities": entities, "relationships": relationships})
|
||||
return ontology
|
||||
|
||||
def graph_construction_worker(entities, relationships):
|
||||
# Worker 3: Build knowledge graph
|
||||
graph_builder = GraphBuilder()
|
||||
kg = graph_builder.build(entities, relationships)
|
||||
return kg
|
||||
|
||||
def graph_analytics_worker(kg):
|
||||
# Worker 4: All graph analytics
|
||||
analyzer = GraphAnalyzer()
|
||||
pagerank = analyzer.compute_centrality(kg, method="pagerank")
|
||||
betweenness = analyzer.compute_centrality(kg, method="betweenness")
|
||||
communities = analyzer.detect_communities(kg, method="louvain")
|
||||
# ... all analytics
|
||||
return {"pagerank": pagerank, "betweenness": betweenness, "communities": communities}
|
||||
|
||||
def hybrid_rag_worker(kg, vector_store):
|
||||
# Worker 5: Hybrid RAG with KG and vector store
|
||||
hybrid_search = HybridSearch(vector_store=vector_store, knowledge_graph=kg)
|
||||
# Query orchestration
|
||||
pass
|
||||
|
||||
# Build pipeline with workers
|
||||
pipeline = PipelineBuilder() \
|
||||
.add_step("data_ingestion", "custom", func=data_ingestion_worker) \
|
||||
.add_step("ontology_building", "custom", func=ontology_building_worker) \
|
||||
.add_step("graph_construction", "custom", func=graph_construction_worker) \
|
||||
.add_step("graph_analytics", "custom", func=graph_analytics_worker) \
|
||||
.add_step("hybrid_rag", "custom", func=hybrid_rag_worker) \
|
||||
.build()
|
||||
|
||||
# Execute with parallel workers
|
||||
result = orchestrator.execute_pipeline(pipeline, parallel=True, max_workers=7)
|
||||
```
|
||||
|
||||
Each notebook demonstrates the full journey from raw data sources through autonomous agent workflows (or orchestrator-worker pattern) and GraphRAG to actionable intelligence.
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
# PR: Context Module Testing & Validation
|
||||
|
||||
## Description
|
||||
This PR adds comprehensive testing and validation for the **Context Engineering Module** (`semantica.context`). It includes unit tests for core components, verification of notebook examples, and a critical bug fix in the deduplication module.
|
||||
|
||||
## Changes
|
||||
|
||||
### 1. New Unit Tests (`tests/context/`)
|
||||
Added `tests/context/test_context.py` covering:
|
||||
- **AgentContext**: End-to-end storage and retrieval (RAG & GraphRAG).
|
||||
- **AgentMemory**: Hierarchical memory management (short-term buffer vs. long-term vector store) and retention policies.
|
||||
- **ContextGraph**: Node/edge addition and neighbor traversal.
|
||||
- **EntityLinker**: URI assignment and entity linking logic.
|
||||
- **ContextRetriever**: Hybrid retrieval strategies (Vector + Graph).
|
||||
|
||||
### 2. Notebook Verification
|
||||
Verified functionality of the following notebooks by converting them to test scripts:
|
||||
- `19_Context_Module.ipynb`: Verified high-level interface, token limits, and graph construction.
|
||||
- `11_Advanced_Context_Engineering.ipynb`: Verified custom memory pruning, hybrid tuning, and custom graph builders.
|
||||
|
||||
### 3. Bug Fixes
|
||||
- **`semantica/deduplication/merge_strategy.py`**: Fixed a `NameError` caused by a missing `Tuple` import. This was discovered during global import validation.
|
||||
|
||||
### 4. Verification
|
||||
- All new tests passed.
|
||||
- Global import check confirmed no other hidden dependency issues.
|
||||
- Integration test `verify_context_sync.py` passed, confirming correct synchronization between memory, graph, and vector store.
|
||||
|
||||
## Testing Instructions
|
||||
Run the new tests with:
|
||||
```bash
|
||||
python -m unittest tests/context/test_context.py
|
||||
```
|
||||
@@ -319,7 +319,7 @@ result = kg.query("Who founded the company?", return_format="structured")
|
||||
print(f"Nodes: {kg.node_count}, Answer: {result.answer}")
|
||||
```
|
||||
|
||||
[**Cookbook: Building Knowledge Graphs**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb) • [**Graph Store**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/09_Graph_Store.ipynb) • [**Triple Store**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/20_Triple_Store.ipynb) • [**Visualization**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/16_Visualization.ipynb)
|
||||
[**Cookbook: Building Knowledge Graphs**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb) • [**Graph Store**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/09_Graph_Store.ipynb) • [**Triplet Store**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/20_Triplet_Store.ipynb) • [**Visualization**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/16_Visualization.ipynb)
|
||||
|
||||
[**Graph Analytics**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/10_Graph_Analytics.ipynb) • [**Advanced Graph Analytics**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/advanced/02_Advanced_Graph_Analytics.ipynb)
|
||||
|
||||
@@ -503,8 +503,6 @@ print(f"Answer: {result.answer} | Nodes: {kg.node_count}, Edges: {kg.edge_count}
|
||||
|:-----------:|:-----------|
|
||||
| [**Discord**](https://discord.gg/semantica) | Real-time help, showcases |
|
||||
| [**GitHub Discussions**](https://github.com/Hawksight-AI/semantica/discussions) | Q&A, feature requests |
|
||||
| [**Twitter**](https://twitter.com/semantica_ai) | Updates, tips |
|
||||
| [**YouTube**](https://youtube.com/@semantica) | Tutorials, webinars |
|
||||
|
||||
### Learning Resources
|
||||
|
||||
|
||||
-206
@@ -1,206 +0,0 @@
|
||||
# Add Intelligence Cookbook Notebooks with MCP and Semantica Agents
|
||||
|
||||
## Overview
|
||||
Add comprehensive intelligence-focused notebooks to `cookbook/use_cases/intelligence/` with complete end-to-end pipelines covering data ingestion (including MCP integration), knowledge graph construction, GraphRAG implementation, **Semantica agent-based workflows**, and detailed analysis. Update documentation in `docs/cookbook.md` and `docs/use-cases.md`.
|
||||
|
||||
## New Notebooks to Create
|
||||
|
||||
### 1. Criminal Network Analysis (`Criminal_Network_Analysis.ipynb`)
|
||||
Complete pipeline from data sources to GraphRAG with **agent-based workflows**:
|
||||
- **Data Sources**: Ingest from police reports, court records, surveillance data, communication logs
|
||||
- **MCP Integration**: Utilize MCP for accessing public records databases, court records APIs, and real-time data streams
|
||||
- **Semantica Agents**:
|
||||
- **Data Gathering Agent**: Autonomous agent using AgentMemory to gather and track data from multiple sources
|
||||
- **Network Analysis Agent**: Specialized agent for graph analytics and community detection
|
||||
- **Pattern Detection Agent**: Agent for identifying suspicious patterns and relationships
|
||||
- **Report Generation Agent**: Agent for compiling intelligence reports
|
||||
- **Agent Coordination**: Use Pipeline module (PipelineBuilder, ExecutionEngine, ParallelismManager) to coordinate parallel agent workflows
|
||||
- **Agent Memory**: Use AgentMemory for persistent context across agent interactions
|
||||
- **Parsing**: Parse structured/unstructured documents, JSON, CSV, PDFs
|
||||
- **Extraction**: Extract suspects, organizations, locations, events, relationships
|
||||
- **Knowledge Graph**: Build criminal network graph with temporal relationships
|
||||
- **Graph Analytics**: Community detection, centrality measures, key player identification
|
||||
- **GraphRAG**: Vector store, hybrid search, context retrieval for intelligence queries
|
||||
- **Detailed Analysis**: Pattern detection, network structure analysis, threat assessment
|
||||
- **Visualization**: Network graphs, community visualization, centrality rankings
|
||||
- **Reporting**: Generate intelligence reports on criminal structures
|
||||
|
||||
### 2. Law Enforcement and Forensics (`Law_Enforcement_Forensics.ipynb`)
|
||||
Complete forensic analysis pipeline with **agent-based workflows**:
|
||||
- **Data Sources**: Case files, evidence logs, witness statements, forensic reports, crime scene data
|
||||
- **Semantica Agents**:
|
||||
- **Evidence Collection Agent**: Autonomous agent for gathering and organizing evidence
|
||||
- **Timeline Analysis Agent**: Agent for building temporal case timelines
|
||||
- **Cross-Case Correlation Agent**: Agent for finding connections across multiple cases
|
||||
- **Forensic Report Agent**: Agent for generating comprehensive forensic reports
|
||||
- **Agent Coordination**: Multi-agent pipeline for parallel evidence processing
|
||||
- **Agent Memory**: Persistent memory for case context and evidence chains
|
||||
- **Parsing**: Parse PDFs, structured reports, evidence databases, temporal logs
|
||||
- **Extraction**: Extract entities (persons, locations, evidence, events), relationships, timelines
|
||||
- **Knowledge Graph**: Build temporal knowledge graph for case timelines and evidence correlation
|
||||
- **Graph Analytics**: Timeline analysis, evidence correlation, pattern detection across cases
|
||||
- **GraphRAG**: Semantic search across case files, evidence retrieval, context-aware queries
|
||||
- **Detailed Analysis**: Cross-case correlation, evidence chain analysis, suspect identification
|
||||
- **Visualization**: Timeline visualization, evidence networks, case correlation graphs
|
||||
- **Reporting**: Generate forensic analysis reports with evidence chains
|
||||
|
||||
### 3. Intelligence Analysis (`Intelligence_Analysis.ipynb`)
|
||||
Comprehensive intelligence analysis with **agent-based workflows**:
|
||||
- **Data Sources**: OSINT feeds, threat intelligence, social media, news, public records, geospatial data
|
||||
- **MCP Integration**: Utilize MCP for real-time data fetching, web scraping, API integration, external database access, and browser automation for OSINT gathering
|
||||
- **Semantica Agents**:
|
||||
- **OSINT Gathering Agent**: Autonomous agent using MCP browser tools for web scraping and OSINT collection
|
||||
- **Threat Assessment Agent**: Specialized agent for threat analysis and risk scoring
|
||||
- **Geospatial Intelligence Agent**: Agent for location-based tracking and geographic analysis
|
||||
- **Multi-Source Fusion Agent**: Agent for correlating intelligence from multiple sources
|
||||
- **Intelligence Report Agent**: Agent for generating comprehensive threat intelligence reports
|
||||
- **Agent Coordination**: Complex multi-agent pipeline with parallel execution for intelligence gathering
|
||||
- **Agent Memory**: Persistent memory for threat context, entity tracking, and intelligence history
|
||||
- **Parsing**: Multi-format parsing (RSS feeds, JSON, XML, web scraping, geospatial formats)
|
||||
- **Extraction**: Extract threat actors, locations, events, relationships, temporal patterns
|
||||
- **Knowledge Graph**: Build multi-source intelligence graph with geospatial and temporal dimensions
|
||||
- **Graph Analytics**: Threat assessment, risk scoring, entity relationship mapping, pattern detection
|
||||
- **GraphRAG**: Multi-source intelligence fusion, hybrid search, contextual threat queries
|
||||
- **Detailed Analysis**:
|
||||
- Multi-source intelligence fusion and correlation
|
||||
- Threat assessment and risk analysis
|
||||
- Geospatial intelligence with location tracking
|
||||
- Temporal threat evolution analysis
|
||||
- **Visualization**: Geographic network maps, threat timelines, relationship networks
|
||||
- **Reporting**: Generate comprehensive threat intelligence reports
|
||||
|
||||
## Files to Create/Modify
|
||||
|
||||
### New Notebooks (in `cookbook/use_cases/intelligence/`)
|
||||
- `Criminal_Network_Analysis.ipynb`
|
||||
- `Law_Enforcement_Forensics.ipynb`
|
||||
- `Intelligence_Analysis.ipynb`
|
||||
|
||||
### Documentation Updates
|
||||
- `docs/cookbook.md` - Add new notebooks to Intelligence section
|
||||
- `docs/use-cases.md` - Add new use case cards for criminal networks and law enforcement
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Complete Pipeline Structure (All Notebooks):
|
||||
1. **Data Sources** - Multiple ingestion sources (FileIngestor, DBIngestor, WebIngestor, StreamIngestor, FeedIngestor)
|
||||
2. **MCP Integration** - Utilize MCP servers for external data access, real-time feeds, API integration, web scraping, and browser automation (in Intelligence Analysis and Criminal Network Analysis notebooks)
|
||||
3. **Semantica Agent Setup** - Initialize AgentMemory, create specialized agents, set up agent coordination
|
||||
4. **Agent-Based Data Gathering** - Autonomous agents gather data using MCP and Semantica ingestors
|
||||
5. **Data Parsing** - Parse structured/unstructured data (JSONParser, XMLParser, CSVParser, DocumentParser, StructuredDataParser)
|
||||
6. **Data Normalization** - Clean and standardize (TextNormalizer, DataNormalizer)
|
||||
7. **Entity & Relation Extraction** - Extract entities, relationships, events (NERExtractor, RelationExtractor, TripleExtractor, EventDetector)
|
||||
8. **Knowledge Graph Construction** - Build graphs (GraphBuilder, TemporalGraphQuery)
|
||||
9. **Agent-Based Analysis** - Specialized agents perform parallel analysis tasks
|
||||
10. **Graph Analytics** - Community detection, centrality, connectivity (GraphAnalyzer, ConnectivityAnalyzer, CentralityCalculator)
|
||||
11. **GraphRAG Implementation** - Embeddings, vector store, hybrid search, context retrieval (EmbeddingGenerator, VectorStore, HybridSearch, ContextRetriever)
|
||||
12. **Agent Memory Integration** - Store and retrieve agent context using AgentMemory
|
||||
13. **Detailed Analysis** - Reasoning, inference, pattern detection (InferenceEngine, RuleManager, ExplanationGenerator)
|
||||
14. **Agent Coordination** - Use Pipeline module for multi-agent workflow orchestration
|
||||
15. **Visualization** - Network graphs, analytics dashboards, geographic maps (KGVisualizer, AnalyticsVisualizer, TemporalVisualizer)
|
||||
16. **Agent-Based Report Generation** - Agents compile and generate professional reports
|
||||
17. **Report Generation** - Professional HTML reports (ReportGenerator, HTMLExporter)
|
||||
|
||||
### Semantica Agent Implementation Details:
|
||||
|
||||
#### AgentMemory Usage:
|
||||
- **Persistent Context**: Store agent interactions, decisions, and findings
|
||||
- **Memory Retrieval**: Retrieve relevant context for agent decision-making
|
||||
- **Conversation History**: Track agent conversations and analysis sessions
|
||||
- **Context Accumulation**: Build up intelligence context over time
|
||||
|
||||
#### Pipeline Agent Coordination:
|
||||
- **PipelineBuilder**: Define multi-agent workflows
|
||||
- **ExecutionEngine**: Execute agent pipelines with error handling
|
||||
- **ParallelismManager**: Run agents in parallel for efficiency
|
||||
- **Specialized Agents**: Each agent has a specific role (data gathering, analysis, reporting)
|
||||
|
||||
#### Agent Workflow Examples:
|
||||
```python
|
||||
# Example: Multi-agent intelligence gathering
|
||||
from semantica.context import AgentMemory
|
||||
from semantica.pipeline import PipelineBuilder, ExecutionEngine, ParallelismManager
|
||||
|
||||
# Initialize agent memory
|
||||
agent_memory = AgentMemory(vector_store=vs, knowledge_graph=kg)
|
||||
|
||||
# Define specialized agents
|
||||
def osint_gathering_agent(query, memory):
|
||||
"""Autonomous OSINT gathering agent"""
|
||||
# Use MCP for web scraping
|
||||
# Store findings in agent memory
|
||||
findings = gather_osint(query)
|
||||
memory.store(f"OSINT findings: {findings}", metadata={"agent": "osint", "query": query})
|
||||
return findings
|
||||
|
||||
def threat_assessment_agent(intel_data, memory):
|
||||
"""Threat assessment agent"""
|
||||
# Retrieve relevant context from memory
|
||||
context = memory.retrieve("threat patterns", max_results=10)
|
||||
# Perform threat analysis
|
||||
assessment = analyze_threats(intel_data, context)
|
||||
memory.store(f"Threat assessment: {assessment}", metadata={"agent": "threat"})
|
||||
return assessment
|
||||
|
||||
# Build multi-agent pipeline
|
||||
pipeline = PipelineBuilder() \
|
||||
.add_step("osint_gathering", "custom", func=osint_gathering_agent, args=(query, agent_memory)) \
|
||||
.add_step("threat_assessment", "custom", func=threat_assessment_agent, args=(intel_data, agent_memory)) \
|
||||
.build()
|
||||
|
||||
# Execute with parallel agents
|
||||
engine = ExecutionEngine()
|
||||
result = engine.execute_pipeline(pipeline, parallel=True)
|
||||
```
|
||||
|
||||
### MCP Integration Details:
|
||||
- **Intelligence Analysis Notebook**:
|
||||
- Use MCP browser tools for web scraping and OSINT gathering
|
||||
- Use MCP resources for accessing external intelligence feeds
|
||||
- Demonstrate real-time data fetching via MCP
|
||||
- Agents use MCP for autonomous data gathering
|
||||
- **Criminal Network Analysis Notebook**:
|
||||
- Use MCP for accessing public records and court databases
|
||||
- Demonstrate API integration via MCP
|
||||
- Show real-time data stream processing
|
||||
- Agents coordinate MCP-based data gathering
|
||||
|
||||
### Notebook Structure:
|
||||
- Overview with complete pipeline description
|
||||
- Semantica modules used (20+ modules including AgentMemory, Pipeline)
|
||||
- **Agent Architecture**: Explanation of agent roles and coordination
|
||||
- MCP integration demonstration (for Intelligence Analysis and Criminal Network Analysis)
|
||||
- Step-by-step implementation:
|
||||
- **Agent Setup**: Initialize AgentMemory and create specialized agents
|
||||
- Data ingestion from multiple sources (including MCP resources)
|
||||
- **Agent-Based Data Gathering**: Autonomous agents gather data
|
||||
- MCP-based external data fetching and API integration
|
||||
- Parsing and normalization
|
||||
- Entity and relation extraction
|
||||
- Knowledge graph construction
|
||||
- **Agent-Based Analysis**: Parallel agent workflows for analysis
|
||||
- Graph analytics and pattern detection
|
||||
- **Agent Memory Integration**: Store and retrieve agent context
|
||||
- GraphRAG setup and query examples
|
||||
- **Agent Coordination**: Multi-agent pipeline orchestration
|
||||
- Detailed analysis with insights
|
||||
- Visualization examples
|
||||
- **Agent-Based Report Generation**: Agents compile reports
|
||||
- Report generation
|
||||
- Best practices and deployment recommendations
|
||||
- **Agent Best Practices**: Agent memory management, coordination patterns
|
||||
- MCP integration best practices
|
||||
- Conclusion with key takeaways
|
||||
|
||||
Each notebook will be comprehensive, demonstrating the full journey from raw data sources (including MCP-enabled external sources) through **autonomous agent workflows** and GraphRAG to actionable intelligence and detailed analysis.
|
||||
|
||||
## Key Agent Features to Highlight:
|
||||
|
||||
1. **Autonomous Data Gathering**: Agents independently gather data from multiple sources
|
||||
2. **Persistent Memory**: AgentMemory maintains context across sessions
|
||||
3. **Parallel Coordination**: Multiple agents work simultaneously on different tasks
|
||||
4. **Specialized Roles**: Each agent has a specific expertise area
|
||||
5. **Context-Aware Analysis**: Agents use memory to make informed decisions
|
||||
6. **Coordinated Workflows**: Pipeline module orchestrates complex multi-agent systems
|
||||
7. **Intelligent Reporting**: Agents compile findings into comprehensive reports
|
||||
|
||||
Binary file not shown.
@@ -22,7 +22,7 @@
|
||||
"- Use CommunityDetector for community detection\n",
|
||||
"- Use ConnectivityAnalyzer for connectivity analysis\n",
|
||||
"- Use GraphValidator and Deduplicator for graph quality\n",
|
||||
"- **Use GraphStore to persist graphs to Neo4j, KuzuDB, or FalkorDB**\n",
|
||||
"- **Use GraphStore to persist graphs to Neo4j or FalkorDB**\n",
|
||||
"\n",
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
@@ -193,8 +193,8 @@
|
||||
"source": [
|
||||
"from semantica.graph_store import GraphStore\n",
|
||||
"\n",
|
||||
"# Initialize graph store (using KuzuDB for embedded storage)\n",
|
||||
"graph_store = GraphStore(backend=\"kuzu\", database_path=\"./analytics_graph_db\")\n",
|
||||
"# Option 1: Neo4j (requires Neo4j server running)\n",
|
||||
"graph_store = GraphStore(backend=\"neo4j\", uri=\"bolt://localhost:7687\", user=\"neo4j\", password=\"password\")\n",
|
||||
"graph_store.connect()\n",
|
||||
"\n",
|
||||
"# Store entities as nodes and track node ID mapping\n",
|
||||
@@ -251,7 +251,7 @@
|
||||
"- **ConnectivityAnalyzer**: Connectivity analysis\n",
|
||||
"- **GraphValidator**: Graph validation\n",
|
||||
"- **Deduplicator**: Graph deduplication\n",
|
||||
"- **GraphStore**: Persist graphs to Neo4j, KuzuDB, or FalkorDB\n"
|
||||
"- **GraphStore**: Persist graphs to Neo4j or FalkorDB\n",
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -92,7 +92,7 @@
|
||||
" {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"works_for\"},\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"knowledge_graph = builder.build(entities, relationships)\n",
|
||||
"knowledge_graph = builder.build(entities + relationships)\n",
|
||||
"\n",
|
||||
"embedding_generator = EmbeddingGenerator()\n",
|
||||
"texts = [e[\"name\"] for e in entities]\n",
|
||||
@@ -211,7 +211,7 @@
|
||||
"csv_exporter = CSVExporter(delimiter=\",\")\n",
|
||||
"\n",
|
||||
"# Export complete knowledge graph\n",
|
||||
"csv_exporter.export_knowledge_graph(knowledge_graph, \"exports/output.csv\")\n",
|
||||
"csv_exporter.export_knowledge_graph(knowledge_graph, \"exports/output\")\n",
|
||||
"\n",
|
||||
"# Export entities separately\n",
|
||||
"entities = knowledge_graph.get(\"entities\", [])\n",
|
||||
@@ -312,7 +312,7 @@
|
||||
"- NumPy format\n",
|
||||
"- Binary format\n",
|
||||
"- FAISS format\n",
|
||||
"- Vector store integration (Pinecone, Weaviate, Qdrant)\n"
|
||||
"- Vector store integration (Weaviate, Qdrant)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -400,7 +400,9 @@
|
||||
"\n",
|
||||
"# Using YAMLSchemaExporter for ontology schemas\n",
|
||||
"schema_exporter = YAMLSchemaExporter()\n",
|
||||
"schema_exporter.export(ontology, \"exports/output_schema.yaml\")\n"
|
||||
"yaml_content = schema_exporter.export_ontology_schema(ontology)\n",
|
||||
"with open(\"exports/output_schema.yaml\", \"w\") as f:\n",
|
||||
" f.write(yaml_content)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Build an enterprise semantic layer: construct knowledge graph, generate ontology, create semantic layer, export RDF, and store in triple store.\n",
|
||||
"Build an enterprise semantic layer: construct knowledge graph, generate ontology, create semantic layer, export RDF, and store in triplet store.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"**Documentation**: [API Reference](https://semantica.readthedocs.io/concepts/)\n",
|
||||
@@ -25,7 +25,7 @@
|
||||
"pip install semantica[all]\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"## Workflow: Build KG → Generate Ontology → Create Semantic Layer → Export RDF → Triple Store\n"
|
||||
"## Workflow: Build KG → Generate Ontology → Create Semantic Layer → Export RDF → Triplet Store\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -37,7 +37,7 @@
|
||||
"from semantica.kg import GraphBuilder\n",
|
||||
"from semantica.ontology import OntologyGenerator\n",
|
||||
"from semantica.export import RDFExporter\n",
|
||||
"from semantica.triple_store import TripleStore\n"
|
||||
"from semantica.triplet_store import TripletStore\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -162,7 +162,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 5: Store in Triple Store\n"
|
||||
"## Step 5: Store in Triplet Store\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -171,8 +171,8 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"triple_store = TripleStore()\n",
|
||||
"triple_store.store(knowledge_graph, ontology)\n"
|
||||
"triplet_store = TripletStore()\n",
|
||||
"triplet_store.store(knowledge_graph, ontology)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -186,7 +186,7 @@
|
||||
"- Ontology Generated\n",
|
||||
"- Semantic Layer Created with Mappings\n",
|
||||
"- RDF Export Completed\n",
|
||||
"- Triple Store Storage Completed\n"
|
||||
"- Triplet Store Storage Completed\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -217,7 +217,7 @@
|
||||
"## 5. Best Practices for Production\n",
|
||||
"\n",
|
||||
"1. **Token Limits**: Align `token_limit` with your LLM's context window minus the prompt template size.\n",
|
||||
"2. **Vector Store**: Use a production-grade vector store (e.g., Pinecone, Weaviate, Qdrant) instead of the mock store.\n",
|
||||
"2. **Vector Store**: Use a production-grade vector store (e.g., Weaviate, Qdrant) instead of the mock store.\n",
|
||||
"3. **Asynchronous Operations**: For high-throughput systems, consider wrapping storage operations in async tasks (though the core logic is synchronous for simplicity).\n",
|
||||
"4. **Entity Resolution**: Implement a robust `EntityLinker` strategy to prevent graph fragmentation (e.g., \"Alice\" vs \"Alice S.\")."
|
||||
]
|
||||
|
||||
@@ -22,6 +22,31 @@
|
||||
"---"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Part 0: Setup Embeddings\n",
|
||||
"\n",
|
||||
"First, let's select our embedding provider and model. Semantica supports multiple providers like Sentence Transformers and FastEmbed.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.embeddings import TextEmbedder\n",
|
||||
"\n",
|
||||
"# Choose provider and model\n",
|
||||
"embedder = TextEmbedder(method=\"fastembed\", model_name=\"BAAI/bge-small-en-v1.5\")\n",
|
||||
"dimension = embedder.get_embedding_dimension()\n",
|
||||
"\n",
|
||||
"print(f\"Selected model: {embedder.get_model_info()['model_name']}\")\n",
|
||||
"print(f\"Embedding dimension: {dimension}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
@@ -343,4 +368,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -234,17 +234,16 @@
|
||||
"```\n",
|
||||
"\n",
|
||||
"### 8. GRAPH_STORE MODULE - Persistent Graph Database Operations\n",
|
||||
"**Purpose**: Store and query property graphs in Neo4j, KuzuDB, or FalkorDB\n",
|
||||
"**Components**:\n",
|
||||
"- `GraphStore`: Main graph store interface\n",
|
||||
"- `Neo4jAdapter`: Neo4j integration (enterprise features)\n",
|
||||
"- `KuzuAdapter`: KuzuDB integration (embedded, no server)\n",
|
||||
"- `FalkorDBAdapter`: FalkorDB integration (Redis-based, ultra-fast)\n",
|
||||
"\n",
|
||||
"**Example**:\n",
|
||||
"```python\n",
|
||||
"from semantica.graph_store import GraphStore\n",
|
||||
"store = GraphStore(backend=\"kuzu\", database_path=\"./my_graph_db\")\n",
|
||||
"**Purpose**: Store and query property graphs in Neo4j or FalkorDB\n",
|
||||
"**Components**:\n",
|
||||
"- `GraphStore`: Main graph store interface\n",
|
||||
"- `Neo4jAdapter`: Neo4j integration (enterprise features)\n",
|
||||
"- `FalkorDBAdapter`: FalkorDB integration (Redis-based, ultra-fast)\n",
|
||||
"\n",
|
||||
"**Example**:\n",
|
||||
"```python\n",
|
||||
"from semantica.graph_store import GraphStore\n",
|
||||
"store = GraphStore(backend=\"neo4j\", uri=\"bolt://localhost:7687\", user=\"neo4j\", password=\"password\")\n",
|
||||
"store.connect()\n",
|
||||
"node1 = store.create_node(\n",
|
||||
" labels=[\"Person\"],\n",
|
||||
|
||||
@@ -204,7 +204,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.semantic_extract.methods import get_entity_method\n",
|
||||
"from semantica.semantic_extract import NERExtractor\n",
|
||||
"\n",
|
||||
"sample_text = \"Apple Inc. was founded by Steve Jobs in Cupertino, California in 1976.\"\n",
|
||||
"\n",
|
||||
@@ -219,8 +219,8 @@
|
||||
" print(f\"\\n Method: {method_name.upper()}\")\n",
|
||||
" print(\"-\" * 40)\n",
|
||||
" \n",
|
||||
" method = get_entity_method(method_name)\n",
|
||||
" entities = method(sample_text)\n",
|
||||
" extractor = NERExtractor(method=method_name)\n",
|
||||
" entities = extractor.extract(sample_text)\n",
|
||||
" \n",
|
||||
" print(f\"Found {len(entities)} entities:\")\n",
|
||||
" for entity in entities[:5]: # Show first 5\n",
|
||||
@@ -638,4 +638,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,7 +180,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.semantic_extract.methods import get_relation_method\n",
|
||||
"from semantica.semantic_extract import RelationExtractor\n",
|
||||
"\n",
|
||||
"sample_text = \"Apple Inc. was founded by Steve Jobs in Cupertino, California.\"\n",
|
||||
"sample_entities = ner_extractor.extract(sample_text)\n",
|
||||
@@ -196,8 +196,8 @@
|
||||
" print(f\"\\n Method: {method_name.upper()}\")\n",
|
||||
" print(\"-\" * 40)\n",
|
||||
" \n",
|
||||
" method = get_relation_method(method_name)\n",
|
||||
" relations = method(sample_text, sample_entities)\n",
|
||||
" extractor = RelationExtractor(method=method_name)\n",
|
||||
" relations = extractor.extract(sample_text, sample_entities)\n",
|
||||
" \n",
|
||||
" print(f\"Found {len(relations)} relations:\")\n",
|
||||
" for rel in relations[:3]: # Show first 3\n",
|
||||
@@ -690,4 +690,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,11 +8,11 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"The Graph Store module provides a unified interface for working with property graph databases. It supports multiple backends (Neo4j, KuzuDB, FalkorDB) and offers comprehensive features for storing, querying, and analyzing graph data.\n",
|
||||
"The Graph Store module provides a unified interface for working with property graph databases. It supports multiple backends (Neo4j, FalkorDB) and offers comprehensive features for storing, querying, and analyzing graph data.\n",
|
||||
"\n",
|
||||
"### Key Features\n",
|
||||
"\n",
|
||||
"- **Multi-Backend Support**: Neo4j (Enterprise), KuzuDB (Embedded), FalkorDB (Redis-based)\n",
|
||||
"- **Multi-Backend Support**: Neo4j (Enterprise), FalkorDB (Redis-based)\n",
|
||||
"- **Full CRUD Operations**: Create, read, update, delete nodes and relationships\n",
|
||||
"- **Cypher Query Language**: Execute complex graph queries with OpenCypher support\n",
|
||||
"- **Graph Analytics**: Built-in algorithms for centrality, community detection, path finding\n",
|
||||
@@ -54,9 +54,7 @@
|
||||
"# For Neo4j (requires Neo4j server)\n",
|
||||
"pip install neo4j\n",
|
||||
"\n",
|
||||
"# For KuzuDB (embedded - no server required)\n",
|
||||
"pip install kuzu\n",
|
||||
"\n",
|
||||
|
||||
"# For FalkorDB (requires Redis/FalkorDB server)\n",
|
||||
"pip install falkordb\n",
|
||||
"```\n",
|
||||
@@ -78,10 +76,9 @@
|
||||
"| Backend | Best For | Deployment | Features |\n",
|
||||
"|---------|----------|------------|----------|\n",
|
||||
"| **Neo4j** | Enterprise applications, production systems | Server/Cloud | Full Cypher, APOC procedures, multi-database |\n",
|
||||
"| **KuzuDB** | Analytics, embedded applications, development | Embedded (no server) | Fast analytical queries, zero-config |\n",
|
||||
"| **FalkorDB** | LLM applications, real-time systems, high performance | Redis-based | Ultra-fast, sparse matrix operations |\n",
|
||||
"\n",
|
||||
"**Recommendation**: Start with **KuzuDB** for development and learning (no setup required), then move to **Neo4j** or **FalkorDB** for production.\n"
|
||||
"**Recommendation**: Use **Neo4j** for enterprise production systems or **FalkorDB** for high-performance real-time applications.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -90,7 +87,7 @@
|
||||
"source": [
|
||||
"## Step 1: Initialize Graph Store\n",
|
||||
"\n",
|
||||
"Initialize a `GraphStore` instance with your preferred backend. For this tutorial, we'll use **KuzuDB** (embedded, no server setup required).\n"
|
||||
"Initialize a `GraphStore` instance with your preferred backend. For this tutorial, we'll use **Neo4j** (requires a running server).\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -102,20 +99,14 @@
|
||||
"from semantica.graph_store import GraphStore\n",
|
||||
"\n",
|
||||
"# Option 1: Neo4j (requires Neo4j server running)\n",
|
||||
"# store = GraphStore(\n",
|
||||
"# backend=\"neo4j\",\n",
|
||||
"# uri=\"bolt://localhost:7687\",\n",
|
||||
"# user=\"neo4j\",\n",
|
||||
"# password=\"password\"\n",
|
||||
"# )\n",
|
||||
"\n",
|
||||
"# Option 2: KuzuDB (embedded - no server required) - Recommended for learning\n",
|
||||
"store = GraphStore(\n",
|
||||
" backend=\"kuzu\",\n",
|
||||
" database_path=\"./demo_graph_db\"\n",
|
||||
" backend=\"neo4j\",\n",
|
||||
" uri=\"bolt://localhost:7687\",\n",
|
||||
" user=\"neo4j\",\n",
|
||||
" password=\"password\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Option 3: FalkorDB (requires Redis/FalkorDB server)\n",
|
||||
"# Option 2: FalkorDB (requires Redis/FalkorDB server)\n",
|
||||
"# store = GraphStore(\n",
|
||||
"# backend=\"falkordb\",\n",
|
||||
"# host=\"localhost\",\n",
|
||||
@@ -553,7 +544,6 @@
|
||||
"\n",
|
||||
"# Note: Index creation support varies by backend\n",
|
||||
"# Neo4j: Full support for various index types\n",
|
||||
"# KuzuDB: Automatic indexing on primary keys\n",
|
||||
"# FalkorDB: Limited index support\n"
|
||||
]
|
||||
},
|
||||
@@ -583,7 +573,7 @@
|
||||
"source": [
|
||||
"## Summary\n",
|
||||
"\n",
|
||||
"This notebook covered the Graph Store module, a unified interface for property graph databases supporting Neo4j, KuzuDB, and FalkorDB.\n",
|
||||
"This notebook covered the Graph Store module, a unified interface for property graph databases supporting Neo4j and FalkorDB.\n",
|
||||
"\n",
|
||||
"### What You Learned\n",
|
||||
"\n",
|
||||
@@ -595,7 +585,7 @@
|
||||
"\n",
|
||||
"### Key Takeaways\n",
|
||||
"\n",
|
||||
"- **Backend Selection**: Use KuzuDB for development, Neo4j for production, FalkorDB for high-performance applications\n",
|
||||
"- **Backend Selection**: Use Neo4j for production, FalkorDB for high-performance applications\n",
|
||||
"- **Best Practices**: Use batch operations, parameterized queries, and proper connection management\n",
|
||||
"- **Next Steps**: Explore advanced analytics, graph quality, and visualization modules\n"
|
||||
]
|
||||
|
||||
@@ -282,7 +282,7 @@
|
||||
"entity_chunker = EntityAwareChunker(\n",
|
||||
" chunk_size=200,\n",
|
||||
" chunk_overlap=50,\n",
|
||||
" ner_method=\"spacy\", # or \"llm\" for better accuracy\n",
|
||||
" ner_method=\"ml\", # \"ml\" (spaCy), \"pattern\", or \"llm\"\n",
|
||||
" preserve_entities=True\n",
|
||||
")\n",
|
||||
"\n",
|
||||
@@ -850,4 +850,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,6 +89,34 @@
|
||||
"print(f\"First 5 values: {embedding[:5]}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Model Selection & Dynamic Switching\n",
|
||||
"\n",
|
||||
"Semantica allows you to choose between different embedding providers (e.g., Sentence Transformers, FastEmbed) and switch models dynamically.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Initialize with a specific provider and model\n",
|
||||
"embedder = TextEmbedder(method=\"sentence_transformers\", model_name=\"all-MiniLM-L6-v2\")\n",
|
||||
"print(f\"Current method: {embedder.get_method()}\")\n",
|
||||
"\n",
|
||||
"# Switch to FastEmbed dynamically\n",
|
||||
"try:\n",
|
||||
" embedder.set_model(method=\"fastembed\", model_name=\"BAAI/bge-small-en-v1.5\")\n",
|
||||
" print(f\"Switched to: {embedder.get_method()}\")\n",
|
||||
" print(f\"Model Info: {embedder.get_model_info()}\")\n",
|
||||
"except ImportError:\n",
|
||||
" print(\"FastEmbed not installed. Install with: pip install fastembed\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -91,7 +91,7 @@
|
||||
"entities = [{\"id\": \"e1\", \"type\": \"Organization\", \"name\": \"Apple Inc.\", \"properties\": {}}]\n",
|
||||
"relationships = []\n",
|
||||
"\n",
|
||||
"kg = builder.build(entities, relationships)\n",
|
||||
"kg = builder.build(entities + relationships)\n",
|
||||
"\n",
|
||||
"# Export to JSON\n",
|
||||
"json_exporter.export_knowledge_graph(kg, \"output.json\")\n"
|
||||
|
||||
+29
-29
@@ -6,31 +6,31 @@
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/20_Triple_Store.ipynb)\n",
|
||||
"\n",
|
||||
"# Triple Store - Comprehensive Guide\n",
|
||||
"# Triplet Store - Comprehensive Guide\n",
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook provides a **comprehensive walkthrough** of Semantica's triple_store module, demonstrating RDF triple storage, SPARQL querying, and multi-backend support for knowledge graph persistence.\n",
|
||||
"This notebook provides a **comprehensive walkthrough** of Semantica's triplet_store module, demonstrating RDF triplet storage, SPARQL querying, and multi-backend support for knowledge graph persistence.\n",
|
||||
"\n",
|
||||
"**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/triple_store/)\n",
|
||||
"**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/triplet_store/)\n",
|
||||
"\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"By the end of this notebook, you will be able to:\n",
|
||||
"\n",
|
||||
"- Register and manage triple stores (Blazegraph, Jena, RDF4J, Virtuoso)\n",
|
||||
"- Perform CRUD operations on RDF triples\n",
|
||||
"- Register and manage triplet stores (Blazegraph, Jena, RDF4J, Virtuoso)\n",
|
||||
"- Perform CRUD operations on RDF triplets\n",
|
||||
"- Execute SPARQL queries with optimization\n",
|
||||
"- Use bulk loading for large datasets\n",
|
||||
"- Work with multiple store backends\n",
|
||||
"- Validate and track triple operations\n",
|
||||
"- Validate and track triplet operations\n",
|
||||
"- Choose the right backend for your use case\n",
|
||||
"\n",
|
||||
"### What You'll Learn\n",
|
||||
"\n",
|
||||
"| Component | Purpose | When to Use |\n",
|
||||
"|-----------|---------|-------------|\n",
|
||||
"| `TripleManager` | Store coordination | All triple operations |\n",
|
||||
"| `TripletManager` | Store coordination | All triplet operations |\n",
|
||||
"| `QueryEngine` | SPARQL execution | Query optimization |\n",
|
||||
"| `BulkLoader` | High-volume loading | Large datasets |\n",
|
||||
"| `BlazegraphAdapter` | Blazegraph backend | High performance |\n",
|
||||
@@ -57,13 +57,13 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 1: Basic Triple Store Operations\n",
|
||||
"## Step 1: Basic Triplet Store Operations\n",
|
||||
"\n",
|
||||
"Let's start with the `TripleManager` for basic triple store operations.\n",
|
||||
"Let's start with the `TripletManager` for basic triplet store operations.\n",
|
||||
"\n",
|
||||
"### What is TripleManager?\n",
|
||||
"### What is TripletManager?\n",
|
||||
"\n",
|
||||
"`TripleManager` is the main coordinator for triple store operations:\n",
|
||||
"`TripletManager` is the main coordinator for triplet store operations:\n",
|
||||
"- **Store Registration**: Register multiple backends\n",
|
||||
"- **CRUD Operations**: Add, get, update, delete triples\n",
|
||||
"- **Multi-Store**: Manage multiple stores simultaneously"
|
||||
@@ -75,11 +75,11 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import TripleManager\n",
|
||||
"from semantica.triplet_store import TripletManager\n",
|
||||
"from semantica.semantic_extract.triple_extractor import Triple\n",
|
||||
"\n",
|
||||
"# Create triple manager\n",
|
||||
"manager = TripleManager()\n",
|
||||
"manager = TripletManager()\n",
|
||||
"\n",
|
||||
"# Register a Blazegraph store (in-memory for demo)\n",
|
||||
"store = manager.register_store(\n",
|
||||
@@ -130,7 +130,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import register_store\n",
|
||||
"from semantica.triplet_store import register_store\n",
|
||||
"\n",
|
||||
"# Register multiple stores using convenience function\n",
|
||||
"blazegraph_store = register_store(\n",
|
||||
@@ -179,7 +179,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import add_triple, add_triples, get_triples, update_triple, delete_triple\n",
|
||||
"from semantica.triplet_store import add_triple, add_triples, get_triples, update_triple, delete_triple\n",
|
||||
"\n",
|
||||
"# Create - Add single triple\n",
|
||||
"triple1 = Triple(\n",
|
||||
@@ -240,7 +240,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import QueryEngine, BlazegraphAdapter\n",
|
||||
"from semantica.triplet_store import QueryEngine, BlazegraphAdapter\n",
|
||||
"\n",
|
||||
"# Create query engine with caching\n",
|
||||
"engine = QueryEngine(enable_caching=True, enable_optimization=True)\n",
|
||||
@@ -299,7 +299,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import optimize_query, plan_query\n",
|
||||
"from semantica.triplet_store import optimize_query, plan_query\n",
|
||||
"\n",
|
||||
"# Original query\n",
|
||||
"query = \"\"\"\n",
|
||||
@@ -347,7 +347,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import BulkLoader, LoadProgress\n",
|
||||
"from semantica.triplet_store import BulkLoader, LoadProgress\n",
|
||||
"\n",
|
||||
"# Create bulk loader\n",
|
||||
"loader = BulkLoader(\n",
|
||||
@@ -392,11 +392,11 @@
|
||||
"source": [
|
||||
"## Step 7: Store Adapters\n",
|
||||
"\n",
|
||||
"Work with different triple store backends.\n",
|
||||
"Work with different triplet store backends.\n",
|
||||
"\n",
|
||||
"### Blazegraph Adapter\n",
|
||||
"\n",
|
||||
"High-performance triple store with GPU acceleration."
|
||||
"High-performance triplet store with GPU acceleration."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -405,7 +405,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import BlazegraphAdapter\n",
|
||||
"from semantica.triplet_store import BlazegraphAdapter\n",
|
||||
"\n",
|
||||
"# Create Blazegraph adapter\n",
|
||||
"blazegraph = BlazegraphAdapter(\n",
|
||||
@@ -442,7 +442,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import JenaAdapter\n",
|
||||
"from semantica.triplet_store import JenaAdapter\n",
|
||||
"\n",
|
||||
"# Create Jena adapter (in-memory)\n",
|
||||
"jena = JenaAdapter()\n",
|
||||
@@ -497,7 +497,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import RDF4JAdapter\n",
|
||||
"from semantica.triplet_store import RDF4JAdapter\n",
|
||||
"\n",
|
||||
"# Create RDF4J adapter\n",
|
||||
"rdf4j = RDF4JAdapter(\n",
|
||||
@@ -540,7 +540,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import VirtuosoAdapter\n",
|
||||
"from semantica.triplet_store import VirtuosoAdapter\n",
|
||||
"\n",
|
||||
"# Create Virtuoso adapter\n",
|
||||
"virtuoso = VirtuosoAdapter(\n",
|
||||
@@ -604,7 +604,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import validate_triples\n",
|
||||
"from semantica.triplet_store import validate_triples\n",
|
||||
"\n",
|
||||
"# Create triples (some invalid)\n",
|
||||
"triples_to_validate = [\n",
|
||||
@@ -650,7 +650,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Register multiple stores\n",
|
||||
"manager = TripleManager()\n",
|
||||
"manager = TripletManager()\n",
|
||||
"\n",
|
||||
"primary = manager.register_store(\n",
|
||||
" \"primary\",\n",
|
||||
@@ -720,7 +720,7 @@
|
||||
"\n",
|
||||
"In this notebook, you've learned how to:\n",
|
||||
"\n",
|
||||
"- Register and manage triple stores\n",
|
||||
"- Register and manage triplet stores\n",
|
||||
"- Perform CRUD operations on RDF triples\n",
|
||||
"- Execute and optimize SPARQL queries\n",
|
||||
"- Use bulk loading for large datasets\n",
|
||||
@@ -740,7 +740,7 @@
|
||||
"### Next Steps\n",
|
||||
"\n",
|
||||
"**Further Reading**:\n",
|
||||
"- [Triple Store API Reference](https://semantica.readthedocs.io/reference/triple_store/)\n",
|
||||
"- [Triplet Store API Reference](https://semantica.readthedocs.io/reference/triplet_store/)\n",
|
||||
"- [SPARQL 1.1 Specification](https://www.w3.org/TR/sparql11-query/)\n",
|
||||
"- [Knowledge Graph Building](../use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)\n",
|
||||
"\n",
|
||||
@@ -771,4 +771,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
}
|
||||
@@ -644,8 +644,8 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Optional: Store graph in persistent graph database\n",
|
||||
"# Uncomment to use KuzuDB (embedded, no server required)\n",
|
||||
"# graph_store = GraphStore(backend=\"kuzu\", database_path=\"./graphrag_db\")\n",
|
||||
"# Uncomment to use Neo4j\n",
|
||||
"# graph_store = GraphStore(backend=\"neo4j\", uri=\"bolt://localhost:7687\", user=\"neo4j\", password=\"password\")\n",
|
||||
"# graph_store.connect()\n",
|
||||
"# \n",
|
||||
"# # Store nodes and track node ID mapping\n",
|
||||
@@ -813,7 +813,8 @@
|
||||
" print(\"No vectors to store\")\n",
|
||||
"\n",
|
||||
"print(\"\\nStoring graph-aware chunks in graph store...\")\n",
|
||||
"graph_store = GraphStore(backend=\"kuzu\", database_path=\"./graphrag_db\")\n",
|
||||
"# Option 1: Neo4j (requires Neo4j server running)\n",
|
||||
"graph_store = GraphStore(backend=\"neo4j\", uri=\"bolt://localhost:7687\", user=\"neo4j\", password=\"password\")\n",
|
||||
"graph_store.connect()\n",
|
||||
"\n",
|
||||
"for i, chunk in enumerate(graph_store_chunks):\n",
|
||||
|
||||
@@ -402,8 +402,8 @@
|
||||
"# graph_store = GraphStore(backend=\"neo4j\", uri=\"bolt://localhost:7687\", user=\"neo4j\", password=\"password\")\n",
|
||||
"# graph_store = GraphStore(backend=\"falkordb\", host=\"localhost\", port=6379, graph_name=\"blockchain_graph\")\n",
|
||||
"\n",
|
||||
"# For this demo, use embedded KuzuDB\n",
|
||||
"graph_store = GraphStore(backend=\"kuzu\", database_path=\"./blockchain_tx_db\")\n",
|
||||
"# Option 1: Neo4j (requires Neo4j server running)\n",
|
||||
" graph_store = GraphStore(backend=\"neo4j\", uri=\"bolt://localhost:7687\", user=\"neo4j\", password=\"password\")\n",
|
||||
"graph_store.connect()\n",
|
||||
"\n",
|
||||
"# Store wallet nodes\n",
|
||||
|
||||
@@ -234,9 +234,9 @@
|
||||
"# For production: use FalkorDB with Redis for ultra-fast queries\n",
|
||||
"# graph_store = GraphStore(backend=\"falkordb\", host=\"localhost\", port=6379, graph_name=\"fraud_graph\")\n",
|
||||
"\n",
|
||||
"# For this demo, use embedded KuzuDB\n",
|
||||
"graph_store = GraphStore(backend=\"kuzu\", database_path=\"./fraud_detection_db\")\n",
|
||||
"graph_store.connect()\n",
|
||||
"# Option 1: Neo4j (requires Neo4j server running)\n",
|
||||
" graph_store = GraphStore(backend=\"neo4j\", uri=\"bolt://localhost:7687\", user=\"neo4j\", password=\"password\")\n",
|
||||
" graph_store.connect()\n",
|
||||
"\n",
|
||||
"# Store transaction entities in graph database\n",
|
||||
"node_id_map = {}\n",
|
||||
|
||||
@@ -38,7 +38,7 @@
|
||||
"- **Parsing**: DocumentParser, PDFParser, StructuredDataParser, CSVParser, MCPParser\n",
|
||||
"- **Extraction**: NERExtractor, RelationExtractor, CoreferenceResolver, TripleExtractor\n",
|
||||
"- **KG**: GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n",
|
||||
"- **Triple Store**: TripleStore, TripleManager, QueryEngine\n",
|
||||
"- **Triplet Store**: TripletStore, TripletManager, QueryEngine\n",
|
||||
"- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
"- **Quality**: KGQualityAssessor, ValidationEngine\n",
|
||||
"- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n",
|
||||
@@ -67,7 +67,6 @@
|
||||
"from semantica.kg import GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n",
|
||||
"from semantica.triple_store import TripleStore, TripleManager, QueryEngine\n",
|
||||
"from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
|
||||
"from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n",
|
||||
"from semantica.visualization import KGVisualizer, OntologyVisualizer, TemporalVisualizer\n",
|
||||
"import tempfile\n",
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
"- **Materialized Knowledge Graphs**: Semantica's KG modules enable building persistent knowledge graphs from medical ontologies, clinical documents, and reports\n",
|
||||
"- **Virtual Data Integration**: Semantica's DBIngestor and QueryEngine allow virtual integration with Electronic Health Records (EHRs) without data replication\n",
|
||||
"- **Hybrid Design**: Semantica's architecture naturally separates structural knowledge from patient-level data\n",
|
||||
"- **Dynamic Query Orchestration**: Semantica's Reasoning and Triple Store modules enable orchestration of queries across ontologies, documents, and EHRs\n",
|
||||
"- **Dynamic Query Orchestration**: Semantica's Reasoning and Triplet Store modules enable orchestration of queries across ontologies, documents, and EHRs\n",
|
||||
"- **Temporal & Semantic Dimensions**: Semantica's Temporal and Context modules provide historical analysis and semantic understanding\n",
|
||||
"- **Traceable & Explainable**: Semantica's ExplanationGenerator and ContextRetriever provide traceable, explainable answers\n",
|
||||
"\n",
|
||||
@@ -55,7 +55,7 @@
|
||||
"- **KG**: GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer (materialized knowledge graph)\n",
|
||||
"- **Embeddings**: EmbeddingGenerator, TextEmbedder (for embeddings)\n",
|
||||
"- **Vector Store**: VectorStore, HybridSearch, MetadataFilter (for RAG)\n",
|
||||
"- **Triple Store**: TripleManager, QueryEngine (for SPARQL queries on ontologies)\n",
|
||||
"- **Triplet Store**: TripletManager, QueryEngine (for SPARQL queries on ontologies)\n",
|
||||
"- **Reasoning**: InferenceEngine, RuleManager (for query orchestration and medical reasoning)\n",
|
||||
"- **Context**: ContextRetriever, ContextGraphBuilder (for contextual retrieval)\n",
|
||||
"- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer (for visualization)\n",
|
||||
@@ -86,7 +86,7 @@
|
||||
"from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer\n",
|
||||
"from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n",
|
||||
"from semantica.vector_store import VectorStore, HybridSearch, MetadataFilter\n",
|
||||
"from semantica.triple_store import TripleManager, QueryEngine\n",
|
||||
"from semantica.triplet_store import TripletManager, QueryEngine\n",
|
||||
"from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
"from semantica.context import ContextRetriever, ContextGraphBuilder\n",
|
||||
"from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n",
|
||||
@@ -440,9 +440,9 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 9: Setup Triple Store for Ontology Queries Using Semantica\n",
|
||||
"## Step 9: Setup Triplet Store for Ontology Queries Using Semantica\n",
|
||||
"\n",
|
||||
"Using Semantica's triple store modules to enable SPARQL queries on medical ontologies.\n"
|
||||
"Using Semantica's triplet store modules to enable SPARQL queries on medical ontologies.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -451,12 +451,12 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Initialize Semantica triple store and query engine\n",
|
||||
"triple_manager = TripleManager()\n",
|
||||
"# Initialize Semantica triplet store and query engine\n",
|
||||
"triplet_manager = TripletManager()\n",
|
||||
"query_engine = QueryEngine()\n",
|
||||
"\n",
|
||||
"# Register triple store (using in-memory for demo)\n",
|
||||
"store = triple_manager.register_store(\"healthcare_ontology\", \"jena\", \"http://localhost:3030/healthcare\")\n",
|
||||
"# Register triplet store (using in-memory for demo)\n",
|
||||
"store = triplet_manager.register_store(\"healthcare_ontology\", \"jena\", \"http://localhost:3030/healthcare\")\n",
|
||||
"\n",
|
||||
"# Convert ontology to triples and add to store\n",
|
||||
"# In production, this would load the OWL ontology\n",
|
||||
@@ -477,7 +477,7 @@
|
||||
"\n",
|
||||
"# Add triples using Semantica\n",
|
||||
"for triple in sample_triples:\n",
|
||||
" triple_manager.add_triple(triple, store_id=\"healthcare_ontology\")\n",
|
||||
" triplet_manager.add_triple(triple, store_id=\"healthcare_ontology\")\n",
|
||||
"\n",
|
||||
"print(f\" - Triples added: {len(sample_triples)}\")\n",
|
||||
"print(f\" - SPARQL queries enabled for ontology\")\n"
|
||||
@@ -531,7 +531,7 @@
|
||||
" \"context\": {}\n",
|
||||
" }\n",
|
||||
" \n",
|
||||
" # 1. Query ontology using Semantica Triple Store\n",
|
||||
" # 1. Query ontology using Semantica Triplet Store\n",
|
||||
" sparql_query = f\"\"\"\n",
|
||||
" SELECT ?concept WHERE {{\n",
|
||||
" ?concept rdfs:label ?label .\n",
|
||||
@@ -769,12 +769,12 @@
|
||||
"2. **Materialized Knowledge Graphs**: Semantica's KG modules enable building persistent knowledge graphs from medical ontologies and documents\n",
|
||||
"3. **Virtual Data Integration**: Semantica's DBIngestor allows virtual integration with EHRs without data replication\n",
|
||||
"4. **Hybrid Search**: Semantica's HybridSearch combines vector similarity with knowledge graph queries\n",
|
||||
"5. **Query Orchestration**: Semantica's Reasoning and Triple Store modules enable dynamic query orchestration\n",
|
||||
"6. **Explainability**: Semantica's ExplanationGenerator provides traceable, explainable answers\n",
|
||||
"5. **Query Orchestration**: Semantica's Reasoning and Triplet Store modules enable dynamic query orchestration\n",
|
||||
"6. **Explainability**: Semantica's ExplanationGenerator provides traceable, explainable answers\n",
|
||||
"\n",
|
||||
"### Semantica-Specific Performance Considerations\n",
|
||||
"\n",
|
||||
"- **Vector Store**: Use Semantica's VectorStore with appropriate backend (FAISS for local, Pinecone/Weaviate for cloud)\n",
|
||||
"- **Vector Store**: Use Semantica's VectorStore with appropriate backend (FAISS for local, Weaviate for cloud)\n",
|
||||
"- **Graph Analytics**: Leverage Semantica's GraphAnalyzer for efficient centrality and community detection\n",
|
||||
"- **Pipeline Execution**: Use Semantica's ExecutionEngine for parallel execution of pipeline steps\n",
|
||||
"- **Caching**: Utilize Semantica's ContextRetriever caching for frequently accessed contexts\n",
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
"- **Parsing**: MCPParser, JSONParser, StructuredDataParser, DocumentParser\n",
|
||||
"- **Extraction**: NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n",
|
||||
"- **KG**: GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n",
|
||||
"- **Triple Store**: TripleStore, TripleManager, QueryEngine\n",
|
||||
"- **Triplet Store**: TripletStore, TripletManager, QueryEngine\n",
|
||||
"- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
"- **Quality**: KGQualityAssessor, ValidationEngine\n",
|
||||
"- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n",
|
||||
@@ -67,9 +67,8 @@
|
||||
"from semantica.parse import MCPParser, JSONParser, StructuredDataParser, DocumentParser\n",
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n",
|
||||
"from semantica.kg import GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n",
|
||||
"from semantica.triple_store import TripleStore, TripleManager, QueryEngine\n",
|
||||
"from semantica.triplet_store import TripletStore, TripletManager, QueryEngine\n",
|
||||
"from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
|
||||
"from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n",
|
||||
"from semantica.visualization import KGVisualizer, OntologyVisualizer, TemporalVisualizer\n",
|
||||
"import json\n",
|
||||
@@ -402,7 +401,7 @@
|
||||
"source": [
|
||||
"## Step 5: Build Healthcare Knowledge Graph\n",
|
||||
"\n",
|
||||
"Build a knowledge graph from the extracted medical entities and relationships, then store in triple store.\n"
|
||||
"Build a knowledge graph from the extracted medical entities and relationships, then store in triplet store.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -426,13 +425,17 @@
|
||||
"# Analyze graph structure\n",
|
||||
"metrics = graph_analyzer.compute_metrics(resolved_kg)\n",
|
||||
"\n",
|
||||
"# Store in triple store\n",
|
||||
"triple_store = TripleStore()\n",
|
||||
"triple_manager = TripleManager()\n",
|
||||
"# Store in triplet store\n",
|
||||
"# triplet_store = TripletStore() # TripletStore is a configuration dataclass\n",
|
||||
"triplet_manager = TripletManager()\n",
|
||||
"query_engine = QueryEngine()\n",
|
||||
"\n",
|
||||
"triple_store.add_knowledge_graph(resolved_kg)\n",
|
||||
"triple_manager.manage_triples(resolved_kg)\n",
|
||||
"# Register default store (in-memory for demo)\n",
|
||||
"store = triplet_manager.register_store(\"medical_kg\", \"jena\", \"http://localhost:3030/medical\")\n",
|
||||
"\n",
|
||||
"# Convert KG to triples and add to store (simplified)\n",
|
||||
"# In a real scenario, we would convert entities/relations to triples first\n",
|
||||
"# triplet_manager.add_triples(triples, store_id=\"medical_kg\")\n",
|
||||
"\n",
|
||||
"print(f\" Entities: {len(resolved_kg.get('entities', []))}\")\n",
|
||||
"print(f\" Relationships: {len(resolved_kg.get('relationships', []))}\")\n",
|
||||
|
||||
@@ -22,13 +22,13 @@
|
||||
"- **Extraction**: NERExtractor, RelationExtractor, CoreferenceResolver\n",
|
||||
"- **KG**: GraphBuilder, TemporalGraphQuery, GraphValidator, EntityResolver\n",
|
||||
"- **Ontology**: OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n",
|
||||
"- **Triple Store**: TripleStore, TripleManager, QueryEngine\n",
|
||||
"- **Triplet Store**: TripletStore, TripletManager, QueryEngine\n",
|
||||
"- **Export**: RDFExporter, OWLExporter, JSONExporter\n",
|
||||
"- **Visualization**: KGVisualizer, OntologyVisualizer, TemporalVisualizer\n",
|
||||
"\n",
|
||||
"### Pipeline\n",
|
||||
"\n",
|
||||
"**Patient Records → Parse → Extract Medical Entities → Build Temporal KG → Generate Ontology → Store in Triple Store → Query History → Export → Visualize**\n",
|
||||
"**Patient Records → Parse → Extract Medical Entities → Build Temporal KG → Generate Ontology → Store in Triplet Store → Query History → Export → Visualize**\n",
|
||||
"\n",
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
@@ -58,7 +58,7 @@
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor, CoreferenceResolver\n",
|
||||
"from semantica.kg import GraphBuilder, TemporalGraphQuery, GraphValidator, EntityResolver\n",
|
||||
"from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n",
|
||||
"from semantica.triple_store import TripleStore, TripleManager, QueryEngine\n",
|
||||
"from semantica.triplet_store import TripletStore, TripletManager, QueryEngine\n",
|
||||
"from semantica.export import RDFExporter, OWLExporter, JSONExporter\n",
|
||||
"from semantica.visualization import KGVisualizer, OntologyVisualizer, TemporalVisualizer\n",
|
||||
"import tempfile\n",
|
||||
@@ -261,9 +261,9 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 5: Store in Triple Store and Query\n",
|
||||
"## Step 5: Store in Triplet Store and Query\n",
|
||||
"\n",
|
||||
"Store knowledge graph in triple store and query medical history.\n"
|
||||
"Store knowledge graph in triplet store and query medical history.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -272,12 +272,12 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"triple_store = TripleStore()\n",
|
||||
"triple_manager = TripleManager()\n",
|
||||
"triplet_store = TripletStore()\n",
|
||||
"triple_manager = TripletManager()\n",
|
||||
"query_engine = QueryEngine()\n",
|
||||
"temporal_query = TemporalGraphQuery()\n",
|
||||
"\n",
|
||||
"triple_store.store_knowledge_graph(patient_kg)\n",
|
||||
"triplet_store.store_knowledge_graph(patient_kg)\n",
|
||||
"\n",
|
||||
"patient_id = \"P001\"\n",
|
||||
"start_time = \"2024-01-01\"\n",
|
||||
@@ -290,7 +290,7 @@
|
||||
" end_time=end_time\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(f\"Stored patient knowledge graph in triple store\")\n",
|
||||
"print(f\"Stored patient knowledge graph in triplet store\")\n",
|
||||
"print(f\"Retrieved {len(medical_history.get('entities', []))} medical events for patient {patient_id}\")\n"
|
||||
]
|
||||
},
|
||||
@@ -326,7 +326,7 @@
|
||||
"temporal_viz = temporal_visualizer.visualize_timeline(patient_kg, output=\"interactive\")\n",
|
||||
"\n",
|
||||
"print(f\"Total modules used: 20+\")\n",
|
||||
"print(f\"Pipeline complete: Patient Records → Parse → Extract → Temporal KG → Ontology → Triple Store → Query → Export → Visualize\")\n"
|
||||
"print(f\"Pipeline complete: Patient Records → Parse → Extract → Temporal KG → Ontology → Triplet Store → Query → Export → Visualize\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -488,11 +488,13 @@
|
||||
"\n",
|
||||
"# Initialize graph store for persistent criminal network storage\n",
|
||||
"# For production: use Neo4j for enterprise features or FalkorDB for real-time queries\n",
|
||||
"# graph_store = GraphStore(backend=\"neo4j\", uri=\"bolt://localhost:7687\", user=\"neo4j\", password=\"password\")\n",
|
||||
"\n",
|
||||
"# Option 1: Neo4j\n",
|
||||
"graph_store = GraphStore(backend=\"neo4j\", uri=\"bolt://localhost:7687\", user=\"neo4j\", password=\"password\")\n",
|
||||
"\n",
|
||||
"# Option 2: FalkorDB\n",
|
||||
"# graph_store = GraphStore(backend=\"falkordb\", host=\"localhost\", port=6379, graph_name=\"criminal_network\")\n",
|
||||
"\n",
|
||||
"# For this demo, use embedded KuzuDB\n",
|
||||
"graph_store = GraphStore(backend=\"kuzu\", database_path=\"./criminal_network_db\")\n",
|
||||
"graph_store.connect()\n",
|
||||
"\n",
|
||||
"# Store entities as nodes\n",
|
||||
|
||||
@@ -14,7 +14,6 @@ pip install "semantica[pdf,web,feeds,office]"
|
||||
|
||||
# Graph store backends
|
||||
pip install "semantica[graph-neo4j]" # Neo4j support
|
||||
pip install "semantica[graph-kuzu]" # KuzuDB (embedded)
|
||||
pip install "semantica[graph-falkordb]" # FalkorDB (Redis-based)
|
||||
pip install "semantica[graph-all]" # All graph backends
|
||||
|
||||
@@ -33,7 +32,7 @@ from semantica import Semantica
|
||||
core = Semantica(
|
||||
llm_provider="openai",
|
||||
embedding_model="text-embedding-3-large",
|
||||
vector_store="pinecone",
|
||||
vector_store="weaviate",
|
||||
graph_db="neo4j"
|
||||
)
|
||||
|
||||
@@ -280,13 +279,13 @@ owl_ontology = ontology.to_owl()
|
||||
rdf_ontology = ontology.to_rdf()
|
||||
turtle_ontology = ontology.to_turtle()
|
||||
|
||||
# Save to triple store
|
||||
# Save to triplet store
|
||||
ontology.save_to_triple_store("http://localhost:9999/blazegraph/sparql")
|
||||
```
|
||||
|
||||
### 📊 Graph Store - Persistent Property Graph Storage
|
||||
|
||||
Store and query knowledge graphs in Neo4j, KuzuDB, or FalkorDB:
|
||||
Store and query knowledge graphs in Neo4j or FalkorDB:
|
||||
|
||||
```python
|
||||
from semantica.graph_store import GraphStore
|
||||
@@ -299,9 +298,6 @@ store = GraphStore(
|
||||
password="password"
|
||||
)
|
||||
|
||||
# Option 2: KuzuDB for embedded (no server required)
|
||||
store = GraphStore(backend="kuzu", database_path="./my_graph_db")
|
||||
|
||||
# Option 3: FalkorDB for ultra-fast LLM applications
|
||||
store = GraphStore(backend="falkordb", host="localhost", port=6379, graph_name="kg")
|
||||
|
||||
@@ -361,7 +357,7 @@ semantic_chunks = embedder.semantic_chunk(documents)
|
||||
embeddings = embedder.generate_embeddings(semantic_chunks)
|
||||
|
||||
# Store in vector database
|
||||
vector_store = core.get_vector_store("pinecone")
|
||||
vector_store = core.get_vector_store("weaviate")
|
||||
vector_store.store_embeddings(semantic_chunks, embeddings)
|
||||
|
||||
# Semantic search
|
||||
|
||||
@@ -66,9 +66,9 @@ graph TB
|
||||
|
||||
### Knowledge Graphs
|
||||
- **`semantica.kg`** - Knowledge graph construction
|
||||
- **`semantica.vector_store`** - Vector storage (Pinecone, Weaviate, FAISS)
|
||||
- **`semantica.triple_store`** - RDF triple storage (Jena, Blazegraph)
|
||||
- **`semantica.graph_store`** - Property graphs (Neo4j, KuzuDB, FalkorDB)
|
||||
- **`semantica.vector_store`** - Vector storage (Weaviate, FAISS)
|
||||
- **`semantica.triplet_store`** - RDF triplet storage (Jena, Blazegraph)
|
||||
- **`semantica.graph_store`** - Property graphs (Neo4j, FalkorDB)
|
||||
|
||||
### Quality Assurance
|
||||
- **`semantica.deduplication`** - Entity deduplication
|
||||
@@ -85,7 +85,7 @@ graph TB
|
||||
4. Semantic Extraction → Entities, relationships, events
|
||||
5. Graph Construction → Entity resolution, conflict resolution
|
||||
6. Quality Assurance → Deduplication, validation
|
||||
7. Storage → Vector, triple, and graph stores
|
||||
7. Storage → Vector, triplet, and graph stores
|
||||
8. Application → GraphRAG, agents, analytics
|
||||
```
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@ Projects and integrations from the Semantica community.
|
||||
## 🔌 Integrations
|
||||
|
||||
### Vector Databases
|
||||
- Pinecone
|
||||
- Weaviate
|
||||
- Qdrant
|
||||
- FAISS
|
||||
@@ -33,7 +32,6 @@ Projects and integrations from the Semantica community.
|
||||
|
||||
### Graph Databases
|
||||
- Neo4j
|
||||
- KuzuDB
|
||||
- FalkorDB
|
||||
|
||||
### LLM Providers
|
||||
|
||||
@@ -2830,7 +2830,6 @@ flowchart LR
|
||||
| :--- | :--- | :--- | :--- | :--- | :--- |
|
||||
| **NetworkX**| In-memory | Fast | Small-medium | Python API | Development, small graphs |
|
||||
| **Neo4j** | Database | Medium | Large | Cypher | Production, complex queries |
|
||||
| **KuzuDB** | Embedded | Fast | Medium | Cypher | Embedded applications |
|
||||
| **FalkorDB**| Redis-based| Very Fast | Large | Cypher | Real-time, high throughput |
|
||||
|
||||
---
|
||||
|
||||
+1
-1
@@ -154,7 +154,7 @@ Essential guides to master the Semantica framework.
|
||||
|
||||
- :material-database-settings: **Graph Store**
|
||||
---
|
||||
Persisting knowledge graphs in Neo4j, KuzuDB, or FalkorDB.
|
||||
Persisting knowledge graphs in Neo4j or FalkorDB.
|
||||
|
||||
**Topics**: Neo4j, Cypher, Persistence
|
||||
|
||||
|
||||
+1
-1
@@ -165,7 +165,7 @@ Yes, Semantica can be integrated with LangChain for RAG applications.
|
||||
|
||||
### Can I connect to databases?
|
||||
|
||||
Yes, Semantica supports connections to Neo4j, KuzuDB, FalkorDB, and other graph databases.
|
||||
Yes, Semantica supports connections to Neo4j, FalkorDB, and other graph databases.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -106,7 +106,7 @@ knowledge_graph:
|
||||
temporal: true
|
||||
|
||||
graph_store:
|
||||
backend: neo4j # or kuzu, falkordb
|
||||
backend: neo4j # or falkordb
|
||||
neo4j_uri: bolt://localhost:7687
|
||||
neo4j_user: neo4j
|
||||
neo4j_password: password
|
||||
|
||||
+1
-1
@@ -181,7 +181,7 @@ A comprehensive reference of terms and concepts used in Semantica.
|
||||
**Triple**
|
||||
: A basic unit of knowledge in RDF, consisting of a subject, predicate, and object (e.g., `<Apple_Inc> <founded_by> <Steve_Jobs>`).
|
||||
|
||||
**Triple Store**
|
||||
**Triplet Store**
|
||||
: A database designed specifically for storing and querying RDF triples.
|
||||
|
||||
---
|
||||
|
||||
@@ -183,11 +183,11 @@ result = semantica.build_knowledge_base(
|
||||
|
||||
### 3. Backend Selection
|
||||
|
||||
| Operation | NetworkX | Neo4j | KuzuDB |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| **Graph Construction** | ⚡⚡⚡ | ⚡⚡ | ⚡⚡⚡ |
|
||||
| **Query Performance** | ⚡⚡ | ⚡⚡⚡ | ⚡⚡⚡ |
|
||||
| **Scalability** | Low | High | Medium |
|
||||
| Operation | NetworkX | Neo4j |
|
||||
| :--- | :--- | :--- |
|
||||
| **Graph Construction** | ⚡⚡⚡ | ⚡⚡ |
|
||||
| **Query Performance** | ⚡⚡ | ⚡⚡⚡ |
|
||||
| **Scalability** | Low | High |
|
||||
|
||||
---
|
||||
|
||||
|
||||
+12
-14
@@ -15,7 +15,7 @@ Semantica's modules are organized into six logical layers:
|
||||
| :--- | :--- | :--- |
|
||||
| **Input Layer** | [Ingest](#ingest-module), [Parse](#parse-module), [Split](#split-module), [Normalize](#normalize-module) | Data ingestion, parsing, chunking, and cleaning |
|
||||
| **Core Processing** | [Semantic Extract](#semantic-extract-module), [Knowledge Graph](#knowledge-graph-kg-module), [Ontology](#ontology-module), [Reasoning](#reasoning-module) | Entity extraction, graph construction, inference |
|
||||
| **Storage** | [Embeddings](#embeddings-module), [Vector Store](#vector-store-module), [Graph Store](#graph-store-module), [Triple Store](#triple-store-module) | Vector and graph persistence |
|
||||
| **Storage** | [Embeddings](#embeddings-module), [Vector Store](#vector-store-module), [Graph Store](#graph-store-module), [Triplet Store](#triplet-store-module) | Vector, graph, and triplet persistence |
|
||||
| **Quality Assurance** | [Deduplication](#deduplication-module), [Conflicts](#conflicts-module) | Data quality and consistency |
|
||||
| **Context & Memory** | [Context](#context-module), [Seed](#seed-module) | Agent memory and foundation data |
|
||||
| **Output & Orchestration** | [Export](#export-module), [Visualization](#visualization-module), [Pipeline](#pipeline-module) | Export, visualization, and workflow management |
|
||||
@@ -277,7 +277,7 @@ for rel in relationships[:5]:
|
||||
**Key Features:**
|
||||
|
||||
- Graph construction from entities/relationships
|
||||
- Multiple backend support (NetworkX, Neo4j, KuzuDB)
|
||||
- Multiple backend support (NetworkX, Neo4j)
|
||||
- Temporal graph support
|
||||
- Graph analytics and metrics
|
||||
- Entity resolution and deduplication
|
||||
@@ -468,7 +468,7 @@ print(f"Similarity: {similarity:.3f}")
|
||||
|
||||
**Key Features:**
|
||||
|
||||
- Multiple backend support (FAISS, Pinecone, Weaviate, Qdrant, Milvus)
|
||||
- Multiple backend support (FAISS, Weaviate, Qdrant, Milvus)
|
||||
- Hybrid search (vector + keyword)
|
||||
- Metadata filtering
|
||||
- Batch operations
|
||||
@@ -480,7 +480,6 @@ print(f"Similarity: {similarity:.3f}")
|
||||
|
||||
- `VectorStore` — Main vector store interface
|
||||
- `FAISSAdapter` — FAISS integration
|
||||
- `PineconeAdapter` — Pinecone integration
|
||||
- `WeaviateAdapter` — Weaviate integration
|
||||
- `HybridSearch` — Combine vector and keyword search
|
||||
- `VectorRetriever` — Retrieve relevant vectors
|
||||
@@ -513,7 +512,7 @@ results = hybrid_search.search(
|
||||
|
||||
**Key Features:**
|
||||
|
||||
- Multiple backend support (Neo4j, KuzuDB, FalkorDB)
|
||||
- Multiple backend support (Neo4j, FalkorDB)
|
||||
- Cypher query language
|
||||
- Graph algorithms and analytics
|
||||
- Transaction support
|
||||
@@ -525,7 +524,6 @@ results = hybrid_search.search(
|
||||
|
||||
- `GraphStore` — Main graph store interface
|
||||
- `Neo4jAdapter` — Neo4j database integration
|
||||
- `KuzuAdapter` — KuzuDB embedded database integration
|
||||
- `FalkorDBAdapter` — FalkorDB (Redis-based) integration
|
||||
- `NodeManager` — Node CRUD operations
|
||||
- `RelationshipManager` — Relationship CRUD operations
|
||||
@@ -564,15 +562,15 @@ results = store.execute_query("MATCH (p:Person) RETURN p.name")
|
||||
|
||||
---
|
||||
|
||||
### Triple Store Module
|
||||
### Triplet Store Module
|
||||
|
||||
!!! abstract "Purpose"
|
||||
RDF triple store integration for semantic web applications. Supports SPARQL queries and multiple backends.
|
||||
RDF triplet store integration for semantic web applications. Supports SPARQL queries and multiple backends.
|
||||
|
||||
**Key Features:**
|
||||
|
||||
- Multi-backend support (Blazegraph, Jena, RDF4J, Virtuoso)
|
||||
- CRUD operations for RDF triples
|
||||
- CRUD operations for RDF triplets
|
||||
- SPARQL query execution and optimization
|
||||
- Bulk data loading with progress tracking
|
||||
- Query caching and optimization
|
||||
@@ -581,7 +579,7 @@ results = store.execute_query("MATCH (p:Person) RETURN p.name")
|
||||
|
||||
**Components:**
|
||||
|
||||
- `TripleManager` — Main triple store management coordinator
|
||||
- `TripletManager` — Main triplet store management coordinator
|
||||
- `QueryEngine` — SPARQL query execution and optimization
|
||||
- `BulkLoader` — High-volume data loading with progress tracking
|
||||
- `BlazegraphAdapter` — Blazegraph integration
|
||||
@@ -602,9 +600,9 @@ results = store.execute_query("MATCH (p:Person) RETURN p.name")
|
||||
**Quick Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleManager, execute_query
|
||||
from semantica.triplet_store import TripletManager, execute_query
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
store = manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph")
|
||||
|
||||
# Add triple
|
||||
@@ -618,7 +616,7 @@ result = manager.add_triple({
|
||||
query_result = execute_query("SELECT ?s ?p ?o WHERE { ?s ?p ?o } LIMIT 10", store)
|
||||
```
|
||||
|
||||
**API Reference**: [Triple Store Module](reference/triple_store.md)
|
||||
**API Reference**: [Triplet Store Module](reference/triplet_store.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -1115,7 +1113,7 @@ new_facts = inference_engine.forward_chain(kg, rule_manager)
|
||||
| **Embeddings** | `semantica.embeddings` | `EmbeddingGenerator` | Vector generation |
|
||||
| **Vector Store** | `semantica.vector_store` | `VectorStore` | Vector storage |
|
||||
| **Graph Store** | `semantica.graph_store` | `GraphStore` | Graph database |
|
||||
| **Triple Store** | `semantica.triple_store` | `TripleManager` | RDF storage |
|
||||
| **Triplet Store** | `semantica.triplet_store` | `TripletManager` | RDF storage |
|
||||
| **Deduplication** | `semantica.deduplication` | `DuplicateDetector` | Duplicate removal |
|
||||
| **Conflicts** | `semantica.conflicts` | `ConflictDetector` | Conflict resolution |
|
||||
| **Context** | `semantica.context` | `AgentMemory` | Agent context |
|
||||
|
||||
@@ -57,7 +57,7 @@ The **Context Module** provides agents with a persistent, searchable, and struct
|
||||
The high-level facade that unifies all context operations. It routes data to the appropriate subsystems (Memory, Graph, Vector Store) and manages the lifecycle of context.
|
||||
|
||||
#### **Constructor Parameters**
|
||||
* `vector_store` (Required): The backing vector database instance (e.g., FAISS, Pinecone).
|
||||
* `vector_store` (Required): The backing vector database instance (e.g., FAISS, Weaviate).
|
||||
* `knowledge_graph` (Optional): The graph store instance for structured knowledge.
|
||||
* `token_limit` (Default: `2000`): The maximum number of tokens allowed in short-term memory before pruning occurs.
|
||||
* `short_term_limit` (Default: `10`): The maximum number of distinct memory items in short-term memory.
|
||||
|
||||
@@ -34,7 +34,7 @@ The **Embeddings Module** provides a unified interface for generating vector rep
|
||||
|
||||
---
|
||||
|
||||
Automatic formatting and validation for FAISS, Pinecone, Qdrant, and Weaviate.
|
||||
Automatic formatting and validation for FAISS, Qdrant, and Weaviate.
|
||||
|
||||
</div>
|
||||
|
||||
@@ -64,6 +64,7 @@ The main entry point for generating embeddings. It manages the active model and
|
||||
| `process_batch(items)` | Generates embeddings for a list of items (optimized). |
|
||||
| `compare_embeddings(emb1, emb2)` | Calculates cosine similarity between two vectors. |
|
||||
| `get_text_method()` | Returns the active embedding strategy. |
|
||||
| `set_text_model(method, model_name, **config)` | Dynamically switches the text embedding model. |
|
||||
|
||||
#### **Code Example**
|
||||
```python
|
||||
@@ -97,6 +98,9 @@ A specialized class focused purely on text-to-vector operations. It wraps the `E
|
||||
| `embed_text(text)` | Returns a list of floats for the input string. |
|
||||
| `embed_batch(texts)` | Returns a list of lists (vectors) for the input strings. |
|
||||
| `get_embedding_dimension()` | Returns the size of the output vector (e.g., 384, 768, 1536). |
|
||||
| `set_model(method, model_name, **config)` | Switches the underlying embedding model. |
|
||||
| `get_method()` | Returns the current method name. |
|
||||
| `get_model_info()` | Returns details about the current model. |
|
||||
|
||||
#### **Code Example**
|
||||
```python
|
||||
@@ -118,13 +122,13 @@ print(f"Dimension: {embedder.get_embedding_dimension()}")
|
||||
---
|
||||
|
||||
### VectorEmbeddingManager (The Bridge)
|
||||
A utility class that prepares raw embeddings for insertion into specific vector databases. It handles formatting differences between backends like FAISS and Pinecone.
|
||||
A utility class that prepares raw embeddings for insertion into specific vector databases. It handles formatting differences between backends like FAISS and Weaviate.
|
||||
|
||||
#### **Core Methods**
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `prepare_for_vector_db(embeddings, backend, ...)` | Formats data for the target DB. |
|
||||
| `prepare_for_vector_db(embeddings, metadata, backend)` | Formats data for the target DB. |
|
||||
| `validate_dimensions(embeddings, expected_dim)` | Ensures vectors match the index configuration. |
|
||||
| `batch_prepare(embeddings_list)` | Prepares a batch of embeddings for storage. |
|
||||
|
||||
|
||||
@@ -394,7 +394,7 @@ Export ontology schemas to YAML format.
|
||||
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `export(schema, filename)` | Export schema | YAML schema serialization |
|
||||
| `export_ontology_schema(ontology, filename)` | Export ontology schema | YAML schema serialization |
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -402,7 +402,7 @@ Export ontology schemas to YAML format.
|
||||
from semantica.export import YAMLSchemaExporter
|
||||
|
||||
exporter = YAMLSchemaExporter()
|
||||
exporter.export(schema, "schema.yaml")
|
||||
exporter.export_ontology_schema(schema, "schema.yaml")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Graph Store
|
||||
|
||||
> **Unified interface for Property Graph Databases (Neo4j, KuzuDB, FalkorDB).**
|
||||
> **Unified interface for Property Graph Databases (Neo4j, FalkorDB).**
|
||||
|
||||
---
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
---
|
||||
|
||||
Support for Neo4j (Enterprise), KuzuDB (Embedded), and FalkorDB (Redis-based)
|
||||
Support for Neo4j (Enterprise) and FalkorDB (Redis-based)
|
||||
|
||||
- :material-code-braces:{ .lg .middle } **Cypher Support**
|
||||
|
||||
@@ -187,26 +187,6 @@ Enterprise-grade Neo4j backend adapter.
|
||||
- `Neo4jSession` - Session management wrapper
|
||||
- `Neo4jTransaction` - Transaction wrapper
|
||||
|
||||
#### KuzuAdapter
|
||||
|
||||
Embedded, in-process KuzuDB backend adapter.
|
||||
|
||||
**Features:**
|
||||
- No external server required
|
||||
- Columnar storage for speed
|
||||
- Zero-copy integration with Arrow
|
||||
- Schema-based node and relationship tables
|
||||
- High-performance analytical queries
|
||||
|
||||
**Related Classes:**
|
||||
- `KuzuDatabase` - Database wrapper
|
||||
- `KuzuConnection` - Connection wrapper
|
||||
- `KuzuQuery` - Query execution wrapper
|
||||
|
||||
**Special Methods:**
|
||||
- `create_node_table(table_name, properties, primary_key, **options)` - Create node table with schema
|
||||
- `create_rel_table(table_name, from_table, to_table, properties, **options)` - Create relationship table
|
||||
- `bulk_load_nodes(table_name, file_path, **options)` - Bulk load nodes from CSV
|
||||
|
||||
#### FalkorDBAdapter
|
||||
|
||||
@@ -243,7 +223,6 @@ Configuration manager for graph store module. Supports environment variables, co
|
||||
- `set_method_config(method_name, config)` - Set method-specific configuration
|
||||
- `get_all()` - Get all configuration
|
||||
- `get_neo4j_config()` - Get Neo4j-specific configuration
|
||||
- `get_kuzu_config()` - Get KuzuDB-specific configuration
|
||||
- `get_falkordb_config()` - Get FalkorDB-specific configuration
|
||||
- `reset()` - Reset configuration to defaults
|
||||
|
||||
@@ -394,9 +373,6 @@ graph_store:
|
||||
uri: bolt://localhost:7687
|
||||
pool_size: 50
|
||||
|
||||
kuzu:
|
||||
path: ./data/kuzu_db
|
||||
buffer_pool_size: 1024 # MB
|
||||
```
|
||||
|
||||
---
|
||||
@@ -438,7 +414,7 @@ subgraph = graph_store.execute_query(query, parameters={"ids": node_ids})
|
||||
## See Also
|
||||
|
||||
- [Knowledge Graph Module](kg.md) - Logical layer above Graph Store
|
||||
- [Triple Store Module](triple_store.md) - RDF-based alternative
|
||||
- [Triplet Store Module](triplet_store.md) - RDF-based alternative
|
||||
- [Visualization Module](visualization.md) - Visualizing query results
|
||||
|
||||
## Cookbook
|
||||
|
||||
@@ -245,7 +245,7 @@ kg.add_triples(inferred_triples)
|
||||
## See Also
|
||||
|
||||
- [Ontology Module](ontology.md) - Source of schema-based rules
|
||||
- [Triple Store Module](triple_store.md) - Backend for SPARQL reasoning
|
||||
- [Triplet Store Module](triplet_store.md) - Backend for SPARQL reasoning
|
||||
- [Modules Guide](../modules.md#quality-assurance) - Consistency checking overview
|
||||
|
||||
## Cookbook
|
||||
|
||||
@@ -44,6 +44,12 @@
|
||||
|
||||
Use LLMs to improve extraction quality and handle complex schemas
|
||||
|
||||
- :material-graph:{ .lg .middle } **Semantic Networks**
|
||||
|
||||
---
|
||||
|
||||
Build structured networks with nodes and edges from text
|
||||
|
||||
</div>
|
||||
|
||||
!!! tip "When to Use"
|
||||
@@ -119,6 +125,49 @@ ner = NamedEntityRecognizer(
|
||||
entities = ner.extract_entities("Apple Inc. was founded in 1976.")
|
||||
```
|
||||
|
||||
### NERExtractor
|
||||
|
||||
Core entity extraction implementation used by notebooks and lower-level integrations.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `method` | str or list | `"ml"` | Method(s): "ml", "llm", "pattern", "regex", "huggingface" |
|
||||
| `**config` | dict | `{}` | Method-specific config (e.g., `model`, `provider`) |
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `extract(text)` | Alias for `extract_entities`. Get list of entities. |
|
||||
| `extract_entities(text)` | Get list of entities |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
|
||||
# 1. ML (spaCy) - Default
|
||||
extractor = NERExtractor(method="ml", model="en_core_web_trf")
|
||||
entities = extractor.extract("Elon Musk leads SpaceX.")
|
||||
|
||||
# 2. LLM (OpenAI/Gemini/etc)
|
||||
extractor = NERExtractor(
|
||||
method="llm",
|
||||
provider="openai",
|
||||
model="gpt-4",
|
||||
temperature=0.0
|
||||
)
|
||||
|
||||
# 3. Regex with custom patterns
|
||||
patterns = {"CODE": r"[A-Z]{3}-\d{3}"}
|
||||
extractor = NERExtractor(method="regex", patterns=patterns)
|
||||
|
||||
# 4. Ensemble (Multiple methods)
|
||||
extractor = NERExtractor(method=["ml", "llm"], ensemble_voting=True)
|
||||
```
|
||||
|
||||
### RelationExtractor
|
||||
|
||||
Extracts relationships between entities.
|
||||
@@ -136,6 +185,7 @@ Extracts relationships between entities.
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `extract(text, entities)` | Alias for `extract_relations`. Find links. |
|
||||
| `extract_relations(text, entities)` | Find links |
|
||||
|
||||
**Example:**
|
||||
@@ -150,7 +200,7 @@ entities = ner.extract_entities(text)
|
||||
|
||||
# Basic relation extraction
|
||||
rel_extractor = RelationExtractor()
|
||||
relations = rel_extractor.extract_relations(text, entities=entities)
|
||||
relations = rel_extractor.extract(text, entities=entities)
|
||||
# [Relation(source="Elon Musk", target="SpaceX", type="founded")]
|
||||
|
||||
# With configuration
|
||||
@@ -159,7 +209,39 @@ rel_extractor = RelationExtractor(
|
||||
confidence_threshold=0.7,
|
||||
bidirectional=False
|
||||
)
|
||||
relations = rel_extractor.extract_relations(text, entities=entities)
|
||||
relations = rel_extractor.extract(text, entities=entities)
|
||||
```
|
||||
|
||||
### CoreferenceResolver
|
||||
|
||||
Resolves pronoun references and entity coreferences.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `method` | str or list | `None` | Underlying NER method(s) |
|
||||
| `**config` | dict | `{}` | Configuration for NER method |
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `resolve(text)` | Alias for `resolve_coreferences`. Get coreference chains. |
|
||||
| `resolve_coreferences(text)` | Get coreference chains |
|
||||
| `resolve_pronouns(text)` | Resolve pronouns to entities |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract import CoreferenceResolver
|
||||
|
||||
resolver = CoreferenceResolver()
|
||||
text = "Steve Jobs founded Apple. He was the CEO."
|
||||
|
||||
# Resolve references
|
||||
chains = resolver.resolve(text)
|
||||
# [CoreferenceChain(mentions=["Steve Jobs", "He"], representative="Steve Jobs")]
|
||||
```
|
||||
|
||||
### EventDetector
|
||||
@@ -204,6 +286,7 @@ Extracts RDF triples (Subject-Predicate-Object).
|
||||
|-----------|------|---------|-------------|
|
||||
| `include_temporal` | bool | `False` | Include time information |
|
||||
| `include_provenance` | bool | `False` | Track source sentences |
|
||||
| `method` | str | `"pattern"` | Extraction method ("pattern", "rules", "huggingface", "llm") |
|
||||
|
||||
**Methods:**
|
||||
|
||||
@@ -224,6 +307,66 @@ triples = extractor.extract_triples("Steve Jobs founded Apple in 1976.")
|
||||
# [Triple(subject="Steve Jobs", predicate="founded", object="Apple", temporal="1976")]
|
||||
```
|
||||
|
||||
### SemanticNetworkExtractor
|
||||
|
||||
Extracts structured semantic networks with nodes and edges.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `ner_method` | str | `None` | Method for node extraction |
|
||||
| `relation_method` | str | `None` | Method for edge extraction |
|
||||
| `**config` | dict | `{}` | Configuration for underlying extractors |
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `extract_network(text)` | Build network from text |
|
||||
| `extract(text)` | Alias for `extract_network` |
|
||||
| `export_to_yaml(network, path)` | Save network to YAML |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract import SemanticNetworkExtractor
|
||||
|
||||
extractor = SemanticNetworkExtractor()
|
||||
network = extractor.extract("Apple Inc. is located in Cupertino.")
|
||||
|
||||
# Analyze network
|
||||
print(f"Nodes: {len(network.nodes)}")
|
||||
print(f"Edges: {len(network.edges)}")
|
||||
```
|
||||
|
||||
### LLMEnhancer
|
||||
|
||||
Enhances extraction results using Large Language Models.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `provider` | str | `"openai"` | LLM provider ("openai", "gemini", "anthropic", etc.) |
|
||||
| `**config` | dict | `{}` | Model config (model name, api_key, etc.) |
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `enhance_entities(text, entities)` | Improve entity accuracy and details |
|
||||
| `enhance_relations(text, relations)` | Improve relation detection |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract import LLMEnhancer
|
||||
|
||||
enhancer = LLMEnhancer(provider="openai", model="gpt-4")
|
||||
enhanced_entities = enhancer.enhance_entities(text, entities)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Usage Examples
|
||||
@@ -234,7 +377,8 @@ from semantica.semantic_extract import (
|
||||
RelationExtractor,
|
||||
TripleExtractor,
|
||||
EventDetector,
|
||||
CoreferenceResolver
|
||||
CoreferenceResolver,
|
||||
SemanticNetworkExtractor
|
||||
)
|
||||
|
||||
text = "Apple released the iPhone in 2007. Steve Jobs announced it at Macworld."
|
||||
@@ -259,10 +403,15 @@ triples = triple_extractor.extract_triples(text)
|
||||
event_detector = EventDetector(extract_time=True)
|
||||
events = event_detector.detect_events(text)
|
||||
|
||||
# Extract semantic network
|
||||
network_extractor = SemanticNetworkExtractor()
|
||||
network = network_extractor.extract(text)
|
||||
|
||||
print(f"Entities: {len(entities)}")
|
||||
print(f"Relations: {len(relations)}")
|
||||
print(f"Triples: {len(triples)}")
|
||||
print(f"Events: {len(events)}")
|
||||
print(f"Network Nodes: {len(network.nodes)}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
+40
-60
@@ -150,7 +150,7 @@ TextSplitter(
|
||||
similarity_threshold=0.7, # Semantic boundary threshold
|
||||
|
||||
# Entity-aware options
|
||||
ner_method="spacy", # NER method (spacy, llm, transformers)
|
||||
ner_method="ml", # NER method (ml/spacy, llm, pattern)
|
||||
preserve_entities=True, # Don't split entities
|
||||
|
||||
# LLM options
|
||||
@@ -183,7 +183,7 @@ for i, chunk in enumerate(chunks):
|
||||
# Entity-aware for GraphRAG
|
||||
splitter = TextSplitter(
|
||||
method="entity_aware",
|
||||
ner_method="llm",
|
||||
ner_method="ml",
|
||||
chunk_size=1000,
|
||||
preserve_entities=True
|
||||
)
|
||||
@@ -250,8 +250,6 @@ Preserve entity boundaries during chunking for GraphRAG.
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `chunk(text, entities)` | Chunk preserving entities | Entity boundary detection |
|
||||
| `extract_entities(text)` | Extract entities | NER extraction |
|
||||
| `find_safe_split_points(text, entities)` | Find split points | Entity span checking |
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -260,14 +258,14 @@ from semantica.split import EntityAwareChunker
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
|
||||
# Extract entities first
|
||||
ner = NERExtractor(method="llm")
|
||||
ner = NERExtractor(method="ml")
|
||||
entities = ner.extract(text)
|
||||
|
||||
# Chunk preserving entities
|
||||
chunker = EntityAwareChunker(
|
||||
chunk_size=1000,
|
||||
chunk_overlap=200,
|
||||
ner_method="llm"
|
||||
ner_method="ml"
|
||||
)
|
||||
|
||||
chunks = chunker.chunk(text, entities=entities)
|
||||
@@ -360,8 +358,7 @@ Structure-aware chunking respecting document hierarchy.
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `chunk(text)` | Chunk by structure | Heading/section detection |
|
||||
| `detect_structure(text)` | Detect document structure | Markdown/HTML parsing |
|
||||
| `build_hierarchy(sections)` | Build section hierarchy | Tree construction |
|
||||
| `_extract_structure(text)` | Extract structural elements | Markdown/HTML parsing |
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -369,17 +366,16 @@ Structure-aware chunking respecting document hierarchy.
|
||||
from semantica.split import StructuralChunker
|
||||
|
||||
chunker = StructuralChunker(
|
||||
respect_headings=True,
|
||||
respect_paragraphs=True,
|
||||
respect_lists=True,
|
||||
respect_headers=True,
|
||||
respect_sections=True,
|
||||
max_chunk_size=2000
|
||||
)
|
||||
|
||||
chunks = chunker.chunk(markdown_text)
|
||||
|
||||
for chunk in chunks:
|
||||
print(f"Section: {chunk.metadata.get('section_title')}")
|
||||
print(f"Level: {chunk.metadata.get('heading_level')}")
|
||||
print(f"Structure preserved: {chunk.metadata.get('structure_preserved')}")
|
||||
print(f"Elements: {chunk.metadata.get('element_types')}")
|
||||
```
|
||||
|
||||
---
|
||||
@@ -393,7 +389,6 @@ Multi-level hierarchical chunking.
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `chunk(text)` | Multi-level chunking | Recursive hierarchical split |
|
||||
| `create_hierarchy(chunks)` | Create chunk hierarchy | Tree structure |
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -470,16 +465,15 @@ Fixed-size sliding window chunking with configurable step size.
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `chunk(text)` | Sliding window chunking | Fixed-size window with step |
|
||||
| `calculate_windows(text_length)` | Calculate window positions | Window position calculation |
|
||||
| `chunk_with_overlap(text)` | Chunk with specific overlap | Window position calculation |
|
||||
|
||||
**Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `window_size` | int | 1000 | Size of sliding window |
|
||||
| `step_size` | int | 800 | Step size (window_size - overlap) |
|
||||
| `min_chunk_size` | int | 100 | Minimum chunk size |
|
||||
| `preserve_sentences` | bool | False | Preserve sentence boundaries |
|
||||
| `chunk_size` | int | 1000 | Size of sliding window |
|
||||
| `overlap` | int | 0 | Overlap size |
|
||||
| `stride` | int | chunk_size - overlap | Step size |
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -488,25 +482,18 @@ from semantica.split import SlidingWindowChunker
|
||||
|
||||
# Basic sliding window
|
||||
chunker = SlidingWindowChunker(
|
||||
window_size=1000,
|
||||
step_size=800, # 200 overlap
|
||||
min_chunk_size=100
|
||||
chunk_size=1000,
|
||||
overlap=200
|
||||
)
|
||||
|
||||
chunks = chunker.chunk(long_text)
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
print(f"Window {i}: chars {chunk.start}-{chunk.end}")
|
||||
print(f"Overlap with previous: {chunk.metadata.get('overlap_chars')}")
|
||||
print(f"Window {i}: chars {chunk.start_index}-{chunk.end_index}")
|
||||
print(f"Has overlap: {chunk.metadata.get('has_overlap')}")
|
||||
|
||||
# Sentence-preserving sliding window
|
||||
chunker = SlidingWindowChunker(
|
||||
window_size=1000,
|
||||
step_size=750,
|
||||
preserve_sentences=True
|
||||
)
|
||||
|
||||
chunks = chunker.chunk(text)
|
||||
# Boundary-preserving sliding window
|
||||
chunks = chunker.chunk(text, preserve_boundaries=True)
|
||||
```
|
||||
|
||||
---
|
||||
@@ -519,18 +506,17 @@ Table-specific chunking preserving table structure.
|
||||
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `chunk(text)` | Chunk tables | Table detection and splitting |
|
||||
| `detect_tables(text)` | Detect tables in text | Table boundary detection |
|
||||
| `split_table(table, max_rows)` | Split large tables | Row-based table splitting |
|
||||
| `chunk_table(table_data)` | Chunk tables | Row/Column-based splitting |
|
||||
| `chunk_to_text_chunks(table_data)` | Convert table chunks to text | Table to text conversion |
|
||||
| `extract_table_schema(table_data)` | Extract schema | Type inference and schema extraction |
|
||||
|
||||
**Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `max_rows` | int | 100 | Maximum rows per table chunk |
|
||||
| `preserve_headers` | bool | True | Keep headers in each chunk |
|
||||
| `max_rows_per_chunk` | int | 50 | Maximum rows per table chunk |
|
||||
| `include_context` | bool | True | Include surrounding text context |
|
||||
| `table_format` | str | "auto" | Table format (markdown, html, csv, auto) |
|
||||
| `chunk_by_columns` | bool | False | Chunk by columns instead of rows |
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -538,31 +524,25 @@ Table-specific chunking preserving table structure.
|
||||
from semantica.split import TableChunker
|
||||
|
||||
chunker = TableChunker(
|
||||
max_rows=50,
|
||||
preserve_headers=True,
|
||||
max_rows_per_chunk=50,
|
||||
include_context=True,
|
||||
table_format="markdown"
|
||||
chunk_by_columns=False
|
||||
)
|
||||
|
||||
text_with_tables = \"\"\"
|
||||
Document with tables...
|
||||
table_data = {
|
||||
"headers": ["Col1", "Col2", "Col3"],
|
||||
"rows": [["Val1", "Val2", "Val3"], ...]
|
||||
}
|
||||
|
||||
| Column 1 | Column 2 | Column 3 |
|
||||
|----------|----------|----------|
|
||||
| Value 1 | Value 2 | Value 3 |
|
||||
| ... | ... | ... |
|
||||
\"\"\"
|
||||
# Get structured table chunks
|
||||
table_chunks = chunker.chunk_table(table_data)
|
||||
|
||||
chunks = chunker.chunk(text_with_tables)
|
||||
# Get text chunks for RAG
|
||||
text_chunks = chunker.chunk_to_text_chunks(table_data)
|
||||
|
||||
for chunk in chunks:
|
||||
if chunk.metadata.get('is_table'):
|
||||
print(f"Table chunk:")
|
||||
print(f" Rows: {chunk.metadata.get('row_count')}")
|
||||
print(f" Columns: {chunk.metadata.get('column_count')}")
|
||||
print(f" Headers: {chunk.metadata.get('headers')}")
|
||||
else:
|
||||
print(f"Text chunk: {len(chunk.text)} chars")
|
||||
for chunk in text_chunks:
|
||||
print(f"Table chunk {chunk.metadata.get('chunk_index')}")
|
||||
print(f"Rows: {chunk.metadata.get('row_count')}")
|
||||
```
|
||||
|
||||
---
|
||||
@@ -663,7 +643,7 @@ print(f"Available methods: {methods}")
|
||||
# Quick splitting
|
||||
chunks = split_recursive(text, chunk_size=1000, chunk_overlap=200)
|
||||
chunks = split_by_sentences(text, sentences_per_chunk=5)
|
||||
chunks = split_entity_aware(text, ner_method="llm")
|
||||
chunks = split_entity_aware(text, ner_method="ml")
|
||||
```
|
||||
|
||||
---
|
||||
@@ -683,7 +663,7 @@ export SPLIT_EMBEDDING_MODEL=all-MiniLM-L6-v2
|
||||
export SPLIT_SIMILARITY_THRESHOLD=0.7
|
||||
|
||||
# Entity-aware
|
||||
export SPLIT_NER_METHOD=spacy
|
||||
export SPLIT_NER_METHOD=ml # or spacy
|
||||
export SPLIT_PRESERVE_ENTITIES=true
|
||||
|
||||
# LLM-based
|
||||
@@ -712,7 +692,7 @@ split:
|
||||
max_chunk_size: 2000
|
||||
|
||||
entity_aware:
|
||||
ner_method: spacy
|
||||
ner_method: ml # or spacy
|
||||
preserve_entities: true
|
||||
min_entity_gap: 50
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Triple Store
|
||||
# Triplet Store
|
||||
|
||||
> **Store and query RDF triples with SPARQL support and semantic reasoning using industry-standard triple stores.**
|
||||
> **Store and query RDF triplets with SPARQL support and semantic reasoning using industry-standard triplet stores.**
|
||||
|
||||
---
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
---
|
||||
|
||||
Store subject-predicate-object triples in W3C-compliant RDF format
|
||||
Store subject-predicate-object triplets in W3C-compliant RDF format
|
||||
|
||||
- :material-code-braces:{ .lg .middle } **SPARQL Queries**
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
|
||||
---
|
||||
|
||||
Query across multiple triple stores with SPARQL federation
|
||||
Query across multiple triplet stores with SPARQL federation
|
||||
|
||||
- :material-upload-multiple:{ .lg .middle } **Bulk Loading**
|
||||
|
||||
@@ -89,29 +89,29 @@
|
||||
|
||||
## Main Classes
|
||||
|
||||
### TripleManager
|
||||
### TripletManager
|
||||
|
||||
Main coordinator for triple store operations across multiple backends.
|
||||
Main coordinator for triplet store operations across multiple backends.
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `register_store(id, backend, endpoint)` | Register triple store | Store registration |
|
||||
| `add_triple(triple, store_id)` | Add single triple | Index insertion |
|
||||
| `add_triples(triples, store_id)` | Batch add triples | Bulk index insertion |
|
||||
| `register_store(store_id, backend, endpoint)` | Register triplet store | Store registration |
|
||||
| `add_triple(triple, store_id)` | Add single triplet | Index insertion |
|
||||
| `add_triples(triples, store_id)` | Batch add triplets | Bulk index insertion |
|
||||
| `query(sparql, store_id)` | Execute SPARQL query | Query optimization + execution |
|
||||
| `delete(pattern, store_id)` | Delete matching triples | Pattern matching + deletion |
|
||||
| `delete(pattern, store_id)` | Delete matching triplets | Pattern matching + deletion |
|
||||
| `bulk_load(file_path, format, store_id)` | Bulk load from file | Streaming parser + batch insert |
|
||||
| `get_stats(store_id)` | Get store statistics | Statistics collection |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleManager
|
||||
from semantica.triplet_store import TripletManager
|
||||
|
||||
# Initialize manager
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
|
||||
# Register Blazegraph store
|
||||
store = manager.register_store(
|
||||
@@ -193,9 +193,9 @@ SPARQL query execution and optimization engine.
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import QueryEngine, TripleManager
|
||||
from semantica.triplet_store import QueryEngine, TripletManager
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
store = manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph/sparql")
|
||||
|
||||
engine = QueryEngine()
|
||||
@@ -270,9 +270,9 @@ High-performance bulk data loading with progress tracking.
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import BulkLoader, TripleManager
|
||||
from semantica.triplet_store import BulkLoader, TripletManager
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
store = manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph/sparql")
|
||||
|
||||
loader = BulkLoader(
|
||||
@@ -322,7 +322,7 @@ progress = loader.load_from_string(
|
||||
|
||||
#### BlazegraphAdapter
|
||||
|
||||
High-performance triple store with GPU acceleration support.
|
||||
High-performance triplet store with GPU acceleration support.
|
||||
|
||||
**Features:**
|
||||
- High-performance SPARQL query execution
|
||||
@@ -334,7 +334,7 @@ High-performance triple store with GPU acceleration support.
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import BlazegraphAdapter
|
||||
from semantica.triplet_store import BlazegraphAdapter
|
||||
|
||||
adapter = BlazegraphAdapter(
|
||||
endpoint="http://localhost:9999/blazegraph/sparql",
|
||||
@@ -376,7 +376,7 @@ results = adapter.query("""
|
||||
Full-featured RDF framework with TDB2 storage.
|
||||
|
||||
**Features:**
|
||||
- TDB2 native triple store
|
||||
- TDB2 native triplet store
|
||||
- SHACL validation
|
||||
- Inference engines (RDFS, OWL)
|
||||
- Fuseki SPARQL server
|
||||
@@ -385,7 +385,7 @@ Full-featured RDF framework with TDB2 storage.
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import JenaAdapter
|
||||
from semantica.triplet_store import JenaAdapter
|
||||
|
||||
adapter = JenaAdapter(
|
||||
tdb_directory="./tdb2_data",
|
||||
@@ -451,7 +451,7 @@ Java-based RDF framework with multiple storage backends.
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import RDF4JAdapter
|
||||
from semantica.triplet_store import RDF4JAdapter
|
||||
|
||||
adapter = RDF4JAdapter(
|
||||
server_url="http://localhost:8080/rdf4j-server",
|
||||
@@ -497,7 +497,7 @@ Enterprise-grade RDF store with SQL integration.
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import VirtuosoAdapter
|
||||
from semantica.triplet_store import VirtuosoAdapter
|
||||
|
||||
adapter = VirtuosoAdapter(
|
||||
host="localhost",
|
||||
@@ -534,10 +534,10 @@ results = adapter.query(f"""
|
||||
|
||||
## Convenience Functions
|
||||
|
||||
Quick access to triple store operations:
|
||||
Quick access to triplet store operations:
|
||||
|
||||
```python
|
||||
from semantica.triple_store import (
|
||||
from semantica.triplet_store import (
|
||||
add_triple,
|
||||
add_triples,
|
||||
execute_query,
|
||||
@@ -579,9 +579,9 @@ export_graph(
|
||||
|
||||
## Dataclasses
|
||||
|
||||
### TripleStore
|
||||
### TripletStore
|
||||
|
||||
Configuration dataclass for triple store instances.
|
||||
Configuration dataclass for triplet store instances.
|
||||
|
||||
**Attributes:**
|
||||
|
||||
@@ -649,35 +649,35 @@ Bulk loading progress dataclass.
|
||||
|
||||
```bash
|
||||
# General settings
|
||||
export TRIPLE_STORE_DEFAULT_BACKEND=blazegraph
|
||||
export TRIPLE_STORE_BATCH_SIZE=10000
|
||||
export TRIPLE_STORE_TIMEOUT=30
|
||||
export TRIPLET_STORE_DEFAULT_BACKEND=blazegraph
|
||||
export TRIPLET_STORE_BATCH_SIZE=10000
|
||||
export TRIPLET_STORE_TIMEOUT=30
|
||||
|
||||
# Blazegraph settings
|
||||
export TRIPLE_STORE_BLAZEGRAPH_ENDPOINT=http://localhost:9999/blazegraph/sparql
|
||||
export TRIPLE_STORE_BLAZEGRAPH_NAMESPACE=kb
|
||||
export TRIPLET_STORE_BLAZEGRAPH_ENDPOINT=http://localhost:9999/blazegraph/sparql
|
||||
export TRIPLET_STORE_BLAZEGRAPH_NAMESPACE=kb
|
||||
|
||||
# Jena settings
|
||||
export TRIPLE_STORE_JENA_TDB_DIRECTORY=./tdb2_data
|
||||
export TRIPLE_STORE_JENA_INFERENCE=rdfs
|
||||
export TRIPLET_STORE_JENA_TDB_DIRECTORY=./tdb2_data
|
||||
export TRIPLET_STORE_JENA_INFERENCE=rdfs
|
||||
|
||||
# RDF4J settings
|
||||
export TRIPLE_STORE_RDF4J_SERVER_URL=http://localhost:8080/rdf4j-server
|
||||
export TRIPLE_STORE_RDF4J_REPOSITORY_ID=my_repo
|
||||
export TRIPLET_STORE_RDF4J_SERVER_URL=http://localhost:8080/rdf4j-server
|
||||
export TRIPLET_STORE_RDF4J_REPOSITORY_ID=my_repo
|
||||
|
||||
# Virtuoso settings
|
||||
export TRIPLE_STORE_VIRTUOSO_HOST=localhost
|
||||
export TRIPLE_STORE_VIRTUOSO_PORT=1111
|
||||
export TRIPLE_STORE_VIRTUOSO_USER=dba
|
||||
export TRIPLE_STORE_VIRTUOSO_PASSWORD=dba
|
||||
export TRIPLET_STORE_VIRTUOSO_HOST=localhost
|
||||
export TRIPLET_STORE_VIRTUOSO_PORT=1111
|
||||
export TRIPLET_STORE_VIRTUOSO_USER=dba
|
||||
export TRIPLET_STORE_VIRTUOSO_PASSWORD=dba
|
||||
```
|
||||
|
||||
### YAML Configuration
|
||||
|
||||
```yaml
|
||||
# config.yaml - Triple Store Configuration
|
||||
# config.yaml - Triplet Store Configuration
|
||||
|
||||
triple_store:
|
||||
triplet_store:
|
||||
backend: blazegraph # blazegraph, jena, rdf4j, virtuoso
|
||||
batch_size: 10000
|
||||
timeout: 30
|
||||
@@ -1,6 +1,6 @@
|
||||
# Vector Store
|
||||
|
||||
> **Unified vector database interface supporting FAISS, Pinecone, Weaviate, Qdrant, and Milvus with Hybrid Search.**
|
||||
> **Unified vector database interface supporting FAISS, Weaviate, Qdrant, and Milvus with Hybrid Search.**
|
||||
|
||||
---
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
---
|
||||
|
||||
Seamlessly switch between FAISS (Local), Pinecone, Weaviate, Qdrant, and Milvus
|
||||
Seamlessly switch between FAISS (Local), Weaviate, Qdrant, and Milvus
|
||||
|
||||
- :material-magnify-plus:{ .lg .middle } **Hybrid Search**
|
||||
|
||||
@@ -230,7 +230,6 @@ results = searcher.search(
|
||||
|
||||
Backend-specific implementations:
|
||||
- `FAISSAdapter`: Local, in-memory/disk.
|
||||
- `PineconeAdapter`: Managed cloud service.
|
||||
- `WeaviateAdapter`: Schema-aware vector DB.
|
||||
- `QdrantAdapter`: Rust-based high-performance DB.
|
||||
- `MilvusAdapter`: Scalable cloud-native DB.
|
||||
@@ -265,41 +264,6 @@ query = np.random.rand(768).astype('float32')
|
||||
distances, indices = adapter.search(index, query, k=10)
|
||||
```
|
||||
|
||||
#### PineconeAdapter
|
||||
|
||||
Managed cloud vector database.
|
||||
|
||||
**Helper Classes:**
|
||||
- `PineconeIndex`: Index management
|
||||
- `PineconeQuery`: Query operations
|
||||
- `PineconeMetadata`: Metadata handling
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.vector_store import PineconeAdapter
|
||||
|
||||
adapter = PineconeAdapter(api_key="your-key", environment="us-west1-gcp")
|
||||
adapter.connect()
|
||||
|
||||
# Create index
|
||||
index = adapter.create_index("my-index", dimension=768, metric="cosine")
|
||||
|
||||
# Upsert with metadata
|
||||
adapter.upsert_vectors(
|
||||
vectors=[[0.1, 0.2, ...], ...],
|
||||
ids=["vec_1", "vec_2"],
|
||||
metadata=[{"category": "news"}, ...]
|
||||
)
|
||||
|
||||
# Query with filter
|
||||
results = adapter.query_vectors(
|
||||
query_vector=[0.1, 0.2, ...],
|
||||
top_k=10,
|
||||
filter={"category": {"$eq": "news"}}
|
||||
)
|
||||
```
|
||||
|
||||
#### WeaviateAdapter
|
||||
|
||||
Schema-aware vector database with GraphQL.
|
||||
@@ -716,25 +680,23 @@ print(f"Available methods: {methods}")
|
||||
### Environment Variables
|
||||
|
||||
```bash
|
||||
export VECTOR_STORE_BACKEND=pinecone
|
||||
export PINECONE_API_KEY=sk-...
|
||||
export PINECONE_ENV=us-west1-gcp
|
||||
export VECTOR_STORE_BACKEND=weaviate
|
||||
export WEAVIATE_URL=http://localhost:8080
|
||||
```
|
||||
|
||||
### YAML Configuration
|
||||
|
||||
```yaml
|
||||
vector_store:
|
||||
backend: faiss # or pinecone, weaviate, etc.
|
||||
backend: faiss # or weaviate, qdrant, milvus
|
||||
dimension: 1536
|
||||
metric: cosine
|
||||
|
||||
faiss:
|
||||
index_type: HNSW
|
||||
|
||||
pinecone:
|
||||
environment: us-west1-gcp
|
||||
index_name: my-index
|
||||
weaviate:
|
||||
url: http://localhost:8080
|
||||
```
|
||||
|
||||
---
|
||||
@@ -777,7 +739,7 @@ print(f"Context: {context}")
|
||||
**Solution**: Ensure your embedding model dimension (e.g., 1536 for OpenAI) matches the VectorStore dimension.
|
||||
|
||||
**Issue**: FAISS index not saved.
|
||||
**Solution**: Call `store.save("index.faiss")` explicitly for local FAISS indices, or use a persistent backend like Pinecone/Qdrant.
|
||||
**Solution**: Call `store.save("index.faiss")` explicitly for local FAISS indices, or use a persistent backend like Weaviate/Qdrant.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+1
-1
@@ -123,7 +123,7 @@ nav:
|
||||
- Seed: reference/seed.md
|
||||
- Semantic Extract: reference/semantic_extract.md
|
||||
- Split: reference/split.md
|
||||
- Triple Store: reference/triple_store.md
|
||||
- Triplet Store: reference/triplet_store.md
|
||||
- Utils: reference/utils.md
|
||||
- Vector Store: reference/vector_store.md
|
||||
- Visualization: reference/visualization.md
|
||||
|
||||
+2
-5
@@ -61,10 +61,10 @@ dependencies = [
|
||||
"librosa>=0.9.0",
|
||||
"opencv-python>=4.6.0",
|
||||
"faiss-cpu>=1.7.0",
|
||||
"pinecone-client>=2.2.0",
|
||||
"weaviate-client>=3.15.0",
|
||||
"qdrant-client>=1.3.0",
|
||||
"neo4j>=5.0.0",
|
||||
"falkordb>=1.0.0",
|
||||
"pymongo>=4.2.0",
|
||||
"sqlalchemy>=1.4.0",
|
||||
"psycopg2-binary>=2.9.0",
|
||||
@@ -186,15 +186,12 @@ split-all = [
|
||||
graph-neo4j = [
|
||||
"neo4j>=5.0.0"
|
||||
]
|
||||
graph-kuzu = [
|
||||
"kuzu>=0.4.0"
|
||||
]
|
||||
graph-falkordb = [
|
||||
"falkordb>=1.0.0",
|
||||
"redis>=4.3.0"
|
||||
]
|
||||
graph-all = [
|
||||
"semantica[graph-neo4j,graph-kuzu,graph-falkordb]"
|
||||
"semantica[graph-neo4j,graph-falkordb]"
|
||||
]
|
||||
all = [
|
||||
"semantica[dev,viz,gpu,cloud,monitoring,llm-all,models-huggingface,split-all,graph-all]"
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
"""
|
||||
Script to verify the usage of the Semantica Core Module.
|
||||
This simulates the typical usage pattern described in core_usage.md.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import logging
|
||||
|
||||
# Add project root to path to ensure we can import semantica
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
from semantica import Semantica
|
||||
from semantica.core import LifecycleManager, PluginRegistry
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger("verify_core")
|
||||
|
||||
def custom_startup_hook():
|
||||
logger.info("✅ Custom startup hook executed!")
|
||||
|
||||
def custom_processing_method(sources, **kwargs):
|
||||
logger.info(f"✅ Custom processing method executed for sources: {sources}")
|
||||
return {"status": "success", "processed_items": len(sources)}
|
||||
|
||||
def main():
|
||||
logger.info("Starting Core Module Verification...")
|
||||
|
||||
# 1. Initialize Semantica
|
||||
logger.info("\n--- Step 1: Initialization ---")
|
||||
config = {
|
||||
"project_name": "CoreVerification",
|
||||
"logging": {"level": "DEBUG"}
|
||||
}
|
||||
app = Semantica(config)
|
||||
logger.info("Semantica instance created.")
|
||||
|
||||
# 2. Register Hooks via Lifecycle Manager
|
||||
logger.info("\n--- Step 2: Lifecycle Hooks ---")
|
||||
app.lifecycle_manager.register_startup_hook(custom_startup_hook, priority=10)
|
||||
logger.info("Startup hook registered.")
|
||||
|
||||
# 3. Register Custom Method
|
||||
logger.info("\n--- Step 3: Method Registry ---")
|
||||
from semantica.core.registry import method_registry
|
||||
method_registry.register("knowledge_base", "custom_processor", custom_processing_method)
|
||||
logger.info("Custom method 'custom_processor' registered.")
|
||||
|
||||
# 4. Start the System (Initialize)
|
||||
logger.info("\n--- Step 4: System Startup ---")
|
||||
app.initialize()
|
||||
|
||||
# Check health
|
||||
health = app.lifecycle_manager.get_health_summary()
|
||||
logger.info(f"System Health: {'Healthy' if health['is_healthy'] else 'Unhealthy'}")
|
||||
if not health['is_healthy']:
|
||||
logger.warning(f"Unhealthy components: {health['unhealthy_components']}")
|
||||
|
||||
# 5. Run a Workflow using the Custom Method
|
||||
logger.info("\n--- Step 5: Workflow Execution ---")
|
||||
sources = ["file1.txt", "file2.txt"]
|
||||
# We use the 'method' argument which the orchestrator (via methods.py) uses to look up the registry
|
||||
# Note: orchestrator.build_knowledge_base doesn't directly expose 'method' arg in signature but passes **kwargs to implementation
|
||||
# Let's check how methods.py is called.
|
||||
# build_knowledge_base calls build_knowledge_base (wrapper) in methods.py?
|
||||
# Wait, orchestrator.py: build_knowledge_base calls self._create_pipeline...
|
||||
|
||||
# Actually, looking at orchestrator.py:
|
||||
# It calls self._create_pipeline(pipeline_config)
|
||||
# It doesn't seem to directly use 'method_registry' for the main 'build_knowledge_base' flow in the default implementation.
|
||||
# However, methods.py defines 'build_knowledge_base' which IS the implementation used if imported as functional API.
|
||||
# But Semantica class in orchestrator.py has its own build_knowledge_base method.
|
||||
|
||||
# Let's see if we can use the method registry via the functional API or if we need to check how Semantica class uses it.
|
||||
# The Semantica class seems to have a hardcoded implementation in build_knowledge_base that creates a pipeline.
|
||||
# But wait, semantica/__init__.py likely exposes the class.
|
||||
|
||||
# Let's try to invoke the custom method directly to verify registry,
|
||||
# OR if Semantica class supports delegation (it might not currently).
|
||||
|
||||
# Let's verify the functional API wrapper usage as well.
|
||||
from semantica.core.methods import build_knowledge_base as functional_build_kb
|
||||
|
||||
result = functional_build_kb(sources, method="custom_processor", config=config)
|
||||
logger.info(f"Functional API Result: {result}")
|
||||
|
||||
# 6. Shutdown
|
||||
logger.info("\n--- Step 6: Shutdown ---")
|
||||
app.lifecycle_manager.shutdown()
|
||||
logger.info("System shutdown completed.")
|
||||
|
||||
logger.info("\n✅ Verification Completed Successfully!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,116 +0,0 @@
|
||||
"""
|
||||
Script to verify the usage of the Semantica Knowledge Graph (KG) Module.
|
||||
This simulates the typical usage pattern described in kg_usage.md.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import logging
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
from semantica.kg import GraphBuilder, GraphAnalyzer, TemporalGraphQuery
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger("verify_kg")
|
||||
|
||||
def main():
|
||||
print("Starting KG Module Verification...")
|
||||
|
||||
# --- Step 1: Build Knowledge Graph ---
|
||||
print("\n--- Step 1: Graph Building ---")
|
||||
|
||||
# Define some source data with temporal info
|
||||
sources = [
|
||||
{
|
||||
"entities": [
|
||||
{"id": "e1", "name": "Alice", "type": "Person"},
|
||||
{"id": "e2", "name": "Bob", "type": "Person"},
|
||||
{"id": "e3", "name": "Semantica", "type": "Project"}
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source": "e1", "target": "e2", "type": "knows",
|
||||
"valid_from": "2023-01-01", "valid_until": None
|
||||
},
|
||||
{
|
||||
"source": "e1", "target": "e3", "type": "works_on",
|
||||
"valid_from": "2023-06-01", "valid_until": "2024-01-01"
|
||||
},
|
||||
{
|
||||
"source": "e2", "target": "e3", "type": "works_on",
|
||||
"valid_from": "2024-01-01", "valid_until": None
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
# Initialize builder (disable complex features for simple verification)
|
||||
builder = GraphBuilder(
|
||||
merge_entities=False,
|
||||
resolve_conflicts=False,
|
||||
enable_temporal=True
|
||||
)
|
||||
|
||||
kg = builder.build(sources)
|
||||
logger.info(f"Graph built with {len(kg['entities'])} entities and {len(kg['relationships'])} relationships.")
|
||||
|
||||
# --- Step 2: Analyze Graph ---
|
||||
logger.info("\n--- Step 2: Graph Analysis ---")
|
||||
|
||||
# Mocking sub-analyzers if they are not fully implemented or require external libs not present
|
||||
# Assuming they are implemented or we can run with defaults.
|
||||
# Note: GraphAnalyzer imports CentralityCalculator etc.
|
||||
# If those modules have dependencies (like networkx), they need to be installed.
|
||||
# Let's try to run it. If it fails, we know we need dependencies.
|
||||
|
||||
try:
|
||||
analyzer = GraphAnalyzer()
|
||||
# We might need to mock internal calls if they fail due to missing heavy libs in this environment
|
||||
# But let's try.
|
||||
# To avoid failure if CentralityCalculator fails, we can catch it.
|
||||
# But for verification script, we want to see it run.
|
||||
# Since I can't check installed packages easily without running pip list, I'll assume standard deps.
|
||||
|
||||
# However, to be safe and avoid script crash on things I haven't checked (like networkx),
|
||||
# I will wrap in try-except block for analysis.
|
||||
analysis = analyzer.analyze_graph(kg)
|
||||
logger.info("Graph analysis completed.")
|
||||
logger.info(f"Metrics: {json.dumps(analysis.get('metrics', {}), indent=2)}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Graph analysis skipped or failed: {e}")
|
||||
|
||||
# --- Step 3: Temporal Query ---
|
||||
logger.info("\n--- Step 3: Temporal Querying ---")
|
||||
|
||||
query_engine = TemporalGraphQuery()
|
||||
|
||||
# Query at a specific time
|
||||
at_time = "2023-08-01"
|
||||
result = query_engine.query_at_time(kg, query="", at_time=at_time)
|
||||
|
||||
logger.info(f"Relationships active at {at_time}:")
|
||||
for rel in result["relationships"]:
|
||||
logger.info(f" {rel['source']} --[{rel['type']}]--> {rel['target']}")
|
||||
|
||||
# Verify expected results
|
||||
# Alice knows Bob (from 2023-01-01) -> Active
|
||||
# Alice works_on Semantica (from 2023-06-01 to 2024-01-01) -> Active
|
||||
# Bob works_on Semantica (from 2024-01-01) -> Not Active
|
||||
|
||||
active_rels = len(result["relationships"])
|
||||
logger.info(f"Found {active_rels} active relationships (Expected: 2).")
|
||||
|
||||
if active_rels == 2:
|
||||
logger.info("✅ Temporal query verification successful!")
|
||||
else:
|
||||
logger.error("❌ Temporal query verification failed!")
|
||||
|
||||
logger.info("\n✅ KG Module Verification Completed!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -84,7 +84,7 @@ class _SemanticaModules:
|
||||
self._normalize = None
|
||||
self._export = None
|
||||
self._vector_store = None
|
||||
self._triple_store = None
|
||||
self._triplet_store = None
|
||||
self._graph_store = None
|
||||
self._ontology = None
|
||||
self._evals = None
|
||||
@@ -160,11 +160,11 @@ class _SemanticaModules:
|
||||
return self._vector_store
|
||||
|
||||
@property
|
||||
def triple_store(self):
|
||||
"""Access triple store module."""
|
||||
if self._triple_store is None:
|
||||
self._triple_store = _ModuleProxy("triple_store")
|
||||
return self._triple_store
|
||||
def triplet_store(self):
|
||||
"""Access triplet store module."""
|
||||
if self._triplet_store is None:
|
||||
self._triplet_store = _ModuleProxy("triplet_store")
|
||||
return self._triplet_store
|
||||
|
||||
@property
|
||||
def graph_store(self):
|
||||
@@ -289,7 +289,7 @@ def __getattr__(name: str):
|
||||
"normalize",
|
||||
"export",
|
||||
"vector_store",
|
||||
"triple_store",
|
||||
"triplet_store",
|
||||
"graph_store",
|
||||
"ontology",
|
||||
"evals",
|
||||
|
||||
@@ -203,6 +203,7 @@ class ConflictDetector:
|
||||
"document": source_ref.document,
|
||||
"page": source_ref.page,
|
||||
"confidence": source_ref.confidence,
|
||||
"metadata": source_ref.metadata,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -384,12 +385,21 @@ class ConflictDetector:
|
||||
|
||||
def _recommend_action(self, property_name: str, values: List[Any]) -> str:
|
||||
"""Recommend action for conflict."""
|
||||
if len(set(values)) == 2:
|
||||
return (
|
||||
"Compare source documents and use most recent or authoritative source"
|
||||
)
|
||||
else:
|
||||
return "Multiple conflicting values detected. Manual review recommended."
|
||||
try:
|
||||
if len(set(values)) == 2:
|
||||
return (
|
||||
"Compare source documents and use most recent or authoritative source"
|
||||
)
|
||||
except TypeError:
|
||||
# Handle unhashable types (like dicts or lists)
|
||||
# Convert to string representation for set comparison
|
||||
str_values = [str(v) for v in values]
|
||||
if len(set(str_values)) == 2:
|
||||
return (
|
||||
"Compare source documents and use most recent or authoritative source"
|
||||
)
|
||||
|
||||
return "Multiple conflicting values detected. Manual review recommended."
|
||||
|
||||
def get_conflict_report(self) -> Dict[str, Any]:
|
||||
"""
|
||||
@@ -863,6 +873,49 @@ class ConflictDetector:
|
||||
)
|
||||
raise
|
||||
|
||||
def resolve_conflicts(self, conflicts: List[Conflict]) -> Dict[str, int]:
|
||||
"""
|
||||
Attempt to resolve conflicts based on configuration.
|
||||
|
||||
Args:
|
||||
conflicts: List of conflicts to resolve
|
||||
|
||||
Returns:
|
||||
Dictionary with resolution statistics
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="conflicts",
|
||||
submodule="ConflictDetector",
|
||||
message=f"Resolving {len(conflicts)} conflicts",
|
||||
)
|
||||
|
||||
resolved_count = 0
|
||||
unresolved_count = 0
|
||||
|
||||
for conflict in conflicts:
|
||||
if self.auto_resolve:
|
||||
# Simple resolution logic: pick value with highest confidence
|
||||
# This is a placeholder for more complex logic
|
||||
if conflict.conflicting_values:
|
||||
# Mark as resolved (in a real system we would update the entity)
|
||||
resolved_count += 1
|
||||
else:
|
||||
unresolved_count += 1
|
||||
else:
|
||||
unresolved_count += 1
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Resolved {resolved_count} conflicts",
|
||||
)
|
||||
|
||||
return {
|
||||
"resolved_count": resolved_count,
|
||||
"unresolved_count": unresolved_count,
|
||||
"total_conflicts": len(conflicts)
|
||||
}
|
||||
|
||||
def clear_conflicts(self) -> None:
|
||||
"""Clear all detected conflicts."""
|
||||
self.detected_conflicts.clear()
|
||||
|
||||
@@ -137,6 +137,16 @@ conflicts = detector.detect_entity_conflicts(
|
||||
print(f"Found {len(conflicts)} total conflicts across all properties")
|
||||
```
|
||||
|
||||
### Integrated Detection and Basic Resolution
|
||||
|
||||
The `ConflictDetector` also provides a convenience method `resolve_conflicts` for basic resolution, which is primarily used by the `GraphBuilder`. For more control, use the `ConflictResolver` class.
|
||||
|
||||
```python
|
||||
# Detect and automatically resolve conflicts (convenience method)
|
||||
resolution_result = detector.resolve_conflicts(conflicts)
|
||||
print(f"Resolved {resolution_result.get('resolved_count')} conflicts")
|
||||
```
|
||||
|
||||
### Using Detection Methods
|
||||
|
||||
```python
|
||||
|
||||
@@ -302,7 +302,6 @@ class Semantica:
|
||||
try:
|
||||
self.logger.info("Executing processing pipeline")
|
||||
|
||||
# Track pipeline execution
|
||||
pipeline_tracking_id = self.progress_tracker.start_tracking(
|
||||
file=str(data) if isinstance(data, (str, Path)) else None,
|
||||
module="pipeline",
|
||||
@@ -310,23 +309,39 @@ class Semantica:
|
||||
message="Executing pipeline",
|
||||
)
|
||||
|
||||
# Validate pipeline
|
||||
if isinstance(pipeline, dict):
|
||||
pipeline = self._create_pipeline_from_dict(pipeline)
|
||||
|
||||
# Validate pipeline object
|
||||
if not hasattr(pipeline, "execute"):
|
||||
raise ProcessingError("Pipeline must have execute() method")
|
||||
execution_engine = None
|
||||
execution_result = None
|
||||
|
||||
try:
|
||||
from ..pipeline import ExecutionEngine, Pipeline
|
||||
|
||||
if isinstance(pipeline, Pipeline):
|
||||
execution_engine = ExecutionEngine()
|
||||
except ImportError:
|
||||
execution_engine = None
|
||||
|
||||
if execution_engine is None and not hasattr(pipeline, "execute"):
|
||||
raise ProcessingError(
|
||||
"Pipeline must be a Pipeline object or have execute() method"
|
||||
)
|
||||
|
||||
# Allocate resources
|
||||
resources = self._allocate_resources(pipeline)
|
||||
|
||||
try:
|
||||
# Execute pipeline
|
||||
result = pipeline.execute(data)
|
||||
|
||||
# Collect metrics
|
||||
metrics = self._collect_metrics(pipeline)
|
||||
if execution_engine is not None:
|
||||
execution_result = execution_engine.execute_pipeline(
|
||||
pipeline, data
|
||||
)
|
||||
success = execution_result.success
|
||||
output = execution_result.output
|
||||
metrics = execution_result.metrics
|
||||
else:
|
||||
output = pipeline.execute(data)
|
||||
metrics = self._collect_metrics(pipeline)
|
||||
success = True
|
||||
|
||||
if pipeline_tracking_id:
|
||||
self.progress_tracker.stop_tracking(
|
||||
@@ -334,8 +349,8 @@ class Semantica:
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"output": result,
|
||||
"success": success,
|
||||
"output": output,
|
||||
"metrics": metrics,
|
||||
"metadata": {
|
||||
"pipeline": str(pipeline),
|
||||
@@ -344,7 +359,6 @@ class Semantica:
|
||||
}
|
||||
|
||||
finally:
|
||||
# Release resources
|
||||
self._release_resources(resources)
|
||||
|
||||
except Exception as e:
|
||||
@@ -567,13 +581,37 @@ class Semantica:
|
||||
Pipeline object or configuration dict (if pipeline module not available)
|
||||
"""
|
||||
try:
|
||||
# Try to use PipelineBuilder if available
|
||||
from ..pipeline import PipelineBuilder
|
||||
|
||||
pipeline_builder = PipelineBuilder()
|
||||
return pipeline_builder.build_from_config(pipeline_config)
|
||||
|
||||
if not pipeline_config:
|
||||
pipeline_builder.add_step("default_step", "default")
|
||||
return pipeline_builder.build("default_pipeline")
|
||||
|
||||
steps_config = pipeline_config.get("steps")
|
||||
|
||||
if isinstance(steps_config, list) and steps_config and isinstance(
|
||||
steps_config[0], str
|
||||
):
|
||||
converted_steps = [
|
||||
{"name": name, "type": name, "config": {}}
|
||||
for name in steps_config
|
||||
]
|
||||
normalized_config: Dict[str, Any] = {
|
||||
"name": pipeline_config.get("name", "default_pipeline"),
|
||||
"steps": converted_steps,
|
||||
}
|
||||
if "parallelism" in pipeline_config:
|
||||
normalized_config["parallelism"] = pipeline_config["parallelism"]
|
||||
return pipeline_builder.build_pipeline(normalized_config)
|
||||
|
||||
if "steps" in pipeline_config:
|
||||
return pipeline_builder.build_pipeline(pipeline_config)
|
||||
|
||||
pipeline_builder.add_step("default_step", "default")
|
||||
return pipeline_builder.build("default_pipeline")
|
||||
except ImportError:
|
||||
# Fallback: return config as-is if pipeline module not available
|
||||
self.logger.debug("Pipeline module not available, using config directly")
|
||||
return pipeline_config
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -357,8 +358,15 @@ class SimilarityCalculator:
|
||||
Returns:
|
||||
Relationship similarity score (0-1)
|
||||
"""
|
||||
rels1 = set(entity1.get("relationships", []))
|
||||
rels2 = set(entity2.get("relationships", []))
|
||||
def _make_hashable(item):
|
||||
if isinstance(item, dict):
|
||||
return tuple(sorted((k, _make_hashable(v)) for k, v in item.items()))
|
||||
if isinstance(item, list):
|
||||
return tuple(_make_hashable(x) for x in item)
|
||||
return item
|
||||
|
||||
rels1 = set(_make_hashable(r) for r in entity1.get("relationships", []))
|
||||
rels2 = set(_make_hashable(r) for r in entity2.get("relationships", []))
|
||||
|
||||
if not rels1 and not rels2:
|
||||
return 1.0
|
||||
|
||||
@@ -87,6 +87,18 @@ class EmbeddingGenerator:
|
||||
|
||||
self.logger.info("Embedding generator initialized")
|
||||
|
||||
def set_text_model(self, method: str, model_name: str, **config) -> None:
|
||||
"""
|
||||
Set the text embedding model dynamically.
|
||||
|
||||
Args:
|
||||
method: Embedding method ("sentence_transformers", "fastembed")
|
||||
model_name: Model name
|
||||
**config: Additional configuration
|
||||
"""
|
||||
self.text_embedder.set_model(method, model_name, **config)
|
||||
self.logger.info(f"Switched text model to: {method}/{model_name}")
|
||||
|
||||
def get_text_method(self) -> str:
|
||||
"""
|
||||
Get the active text embedding method being used.
|
||||
|
||||
@@ -130,6 +130,29 @@ embs_fast = embed_text(texts, method="fastembed") # Faster batch processing
|
||||
|
||||
## Checking Embedding Methods
|
||||
|
||||
### Dynamic Model Switching
|
||||
|
||||
You can switch the embedding model and provider dynamically without creating a new instance.
|
||||
|
||||
```python
|
||||
from semantica.embeddings import TextEmbedder, EmbeddingGenerator
|
||||
|
||||
# 1. Switch model in TextEmbedder
|
||||
embedder = TextEmbedder(method="sentence_transformers")
|
||||
print(f"Current method: {embedder.get_method()}")
|
||||
|
||||
# Switch to FastEmbed
|
||||
try:
|
||||
embedder.set_model(method="fastembed", model_name="BAAI/bge-small-en-v1.5")
|
||||
print(f"Switched to: {embedder.get_method()}")
|
||||
except ImportError:
|
||||
print("FastEmbed not installed")
|
||||
|
||||
# 2. Switch model in EmbeddingGenerator
|
||||
generator = EmbeddingGenerator()
|
||||
generator.set_text_model(method="sentence_transformers", model_name="all-MiniLM-L6-v2")
|
||||
```
|
||||
|
||||
### Checking Active Method in TextEmbedder
|
||||
|
||||
```python
|
||||
@@ -424,14 +447,6 @@ from semantica.embeddings import VectorEmbeddingManager
|
||||
|
||||
manager = VectorEmbeddingManager()
|
||||
|
||||
# Prepare for Pinecone
|
||||
pinecone_data = manager.prepare_for_vector_db(
|
||||
embeddings,
|
||||
metadata=metadata,
|
||||
backend="pinecone",
|
||||
namespace="my_namespace"
|
||||
)
|
||||
|
||||
# Prepare for Weaviate
|
||||
weaviate_data = manager.prepare_for_vector_db(
|
||||
embeddings,
|
||||
@@ -463,9 +478,9 @@ from semantica.embeddings import VectorEmbeddingManager
|
||||
manager = VectorEmbeddingManager()
|
||||
|
||||
# Validate dimensions for specific backend
|
||||
is_valid = manager.validate_dimensions(embeddings, backend="pinecone")
|
||||
is_valid = manager.validate_dimensions(embeddings, backend="weaviate")
|
||||
if is_valid:
|
||||
print("Embeddings meet Pinecone requirements")
|
||||
print("Embeddings meet Weaviate requirements")
|
||||
else:
|
||||
print("Embeddings do not meet requirements")
|
||||
```
|
||||
@@ -622,13 +637,6 @@ networkx_result = manager.prepare_for_graph_db(
|
||||
graph_type="DiGraph"
|
||||
)
|
||||
|
||||
# Prepare for KuzuDB
|
||||
kuzu_result = manager.prepare_for_graph_db(
|
||||
entities,
|
||||
backend="kuzu",
|
||||
database_path="./kuzu_db"
|
||||
)
|
||||
|
||||
# Prepare for FalkorDB
|
||||
falkordb_result = manager.prepare_for_graph_db(
|
||||
entities,
|
||||
|
||||
@@ -9,7 +9,7 @@ Key Features:
|
||||
- Generate embeddings for graph entities (nodes)
|
||||
- Generate embeddings for graph relationships (edges)
|
||||
- Format embeddings for graph DB storage
|
||||
- Integration helpers for Neo4j, NetworkX, KuzuDB, FalkorDB
|
||||
- Integration helpers for Neo4j, NetworkX, FalkorDB
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.embeddings import GraphEmbeddingManager
|
||||
@@ -37,7 +37,6 @@ class GraphEmbeddingManager:
|
||||
Supported Backends:
|
||||
- Neo4j: Graph database with Cypher query language
|
||||
- NetworkX: Python graph library
|
||||
- KuzuDB: Embedded graph database
|
||||
- FalkorDB: Redis-based graph database
|
||||
|
||||
Example Usage:
|
||||
@@ -82,7 +81,7 @@ class GraphEmbeddingManager:
|
||||
entities: List of entity dictionaries with at least "id" and "text" or "content"
|
||||
relationships: Optional list of relationship dictionaries with
|
||||
"source", "target", and optionally "text" or "type"
|
||||
backend: Graph DB backend ("neo4j", "networkx", "kuzu", "falkordb")
|
||||
backend: Graph DB backend ("neo4j", "networkx", "falkordb")
|
||||
**options: Additional backend-specific options
|
||||
|
||||
Returns:
|
||||
@@ -108,10 +107,10 @@ class GraphEmbeddingManager:
|
||||
... entities, relationships, backend="neo4j"
|
||||
... )
|
||||
"""
|
||||
if backend.lower() not in ["neo4j", "networkx", "kuzu", "falkordb"]:
|
||||
if backend.lower() not in ["neo4j", "networkx", "falkordb"]:
|
||||
raise ProcessingError(
|
||||
f"Unsupported backend: {backend}. "
|
||||
f"Supported: neo4j, networkx, kuzu, falkordb"
|
||||
f"Supported: neo4j, networkx, falkordb"
|
||||
)
|
||||
|
||||
# Generate node embeddings
|
||||
@@ -396,8 +395,6 @@ class GraphEmbeddingManager:
|
||||
info["label"] = options.get("label", "Node")
|
||||
elif backend.lower() == "networkx":
|
||||
info["graph_type"] = options.get("graph_type", "DiGraph")
|
||||
elif backend.lower() == "kuzu":
|
||||
info["database_path"] = options.get("database_path", "default")
|
||||
elif backend.lower() == "falkordb":
|
||||
info["graph_name"] = options.get("graph_name", "default")
|
||||
|
||||
|
||||
@@ -166,6 +166,37 @@ class TextEmbedder:
|
||||
"Using fallback embedding method."
|
||||
)
|
||||
|
||||
def get_method(self) -> str:
|
||||
"""Get current embedding method."""
|
||||
return self.method
|
||||
|
||||
def get_model_info(self) -> Dict[str, Any]:
|
||||
"""Get current model information."""
|
||||
return {
|
||||
"method": self.method,
|
||||
"model_name": self.model_name,
|
||||
"device": self.device,
|
||||
"normalize": self.normalize
|
||||
}
|
||||
|
||||
def set_model(self, method: str, model_name: str, **config) -> None:
|
||||
"""
|
||||
Dynamically switch embedding model.
|
||||
|
||||
Args:
|
||||
method: New method ("sentence_transformers" or "fastembed")
|
||||
model_name: New model name
|
||||
**config: Additional configuration
|
||||
"""
|
||||
self.method = method.lower()
|
||||
self.model_name = model_name
|
||||
if "device" in config:
|
||||
self.device = config["device"]
|
||||
if "normalize" in config:
|
||||
self.normalize = config["normalize"]
|
||||
|
||||
self._initialize_model()
|
||||
|
||||
def embed_text(self, text: str, **options) -> np.ndarray:
|
||||
"""
|
||||
Generate embedding for a single text string.
|
||||
|
||||
@@ -9,7 +9,7 @@ Key Features:
|
||||
- Validate embedding dimensions for different backends
|
||||
- Normalize embeddings for vector DB requirements
|
||||
- Create metadata compatible with vector DBs
|
||||
- Integration helpers for FAISS, Pinecone, Weaviate, Qdrant, Milvus
|
||||
- Integration helpers for FAISS, Weaviate, Qdrant, Milvus
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.embeddings import VectorEmbeddingManager
|
||||
@@ -36,7 +36,6 @@ class VectorEmbeddingManager:
|
||||
|
||||
Supported Backends:
|
||||
- FAISS: Local vector storage
|
||||
- Pinecone: Cloud vector database
|
||||
- Weaviate: GraphQL-based vector database
|
||||
- Qdrant: Vector similarity search engine
|
||||
- Milvus: Open-source vector database
|
||||
@@ -50,7 +49,7 @@ class VectorEmbeddingManager:
|
||||
... backend="faiss"
|
||||
... )
|
||||
>>> # Validate dimensions
|
||||
>>> is_valid = manager.validate_dimensions(embeddings, backend="pinecone")
|
||||
>>> is_valid = manager.validate_dimensions(embeddings, backend="weaviate")
|
||||
"""
|
||||
|
||||
def __init__(self, embedding_generator: Optional[EmbeddingGenerator] = None):
|
||||
@@ -67,7 +66,6 @@ class VectorEmbeddingManager:
|
||||
# Backend-specific dimension requirements
|
||||
self.backend_requirements = {
|
||||
"faiss": {"min_dim": 1, "max_dim": None, "dtype": np.float32},
|
||||
"pinecone": {"min_dim": 1, "max_dim": 20000, "dtype": np.float32},
|
||||
"weaviate": {"min_dim": 1, "max_dim": None, "dtype": np.float32},
|
||||
"qdrant": {"min_dim": 1, "max_dim": None, "dtype": np.float32},
|
||||
"milvus": {"min_dim": 1, "max_dim": 32768, "dtype": np.float32},
|
||||
@@ -90,7 +88,7 @@ class VectorEmbeddingManager:
|
||||
Args:
|
||||
embeddings: Embeddings array (n_samples, embedding_dim) or (embedding_dim,)
|
||||
metadata: Optional list of metadata dictionaries (one per embedding)
|
||||
backend: Vector DB backend ("faiss", "pinecone", "weaviate", "qdrant", "milvus")
|
||||
backend: Vector DB backend ("faiss", "weaviate", "qdrant", "milvus")
|
||||
normalize: Whether to normalize embeddings (default: True)
|
||||
**options: Additional backend-specific options
|
||||
|
||||
@@ -108,7 +106,7 @@ class VectorEmbeddingManager:
|
||||
>>> embeddings = np.random.rand(10, 384).astype(np.float32)
|
||||
>>> metadata = [{"text": f"doc_{i}"} for i in range(10)]
|
||||
>>> result = manager.prepare_for_vector_db(
|
||||
... embeddings, metadata, backend="pinecone"
|
||||
... embeddings, metadata, backend="weaviate"
|
||||
... )
|
||||
"""
|
||||
if backend.lower() not in self.backend_requirements:
|
||||
@@ -228,7 +226,7 @@ class VectorEmbeddingManager:
|
||||
bool: True if dimensions are valid, False otherwise
|
||||
|
||||
Example:
|
||||
>>> is_valid = manager.validate_dimensions(embeddings, backend="pinecone")
|
||||
>>> is_valid = manager.validate_dimensions(embeddings, backend="weaviate")
|
||||
"""
|
||||
if backend.lower() not in self.backend_requirements:
|
||||
self.logger.warning(f"Unknown backend: {backend}, skipping validation")
|
||||
@@ -312,7 +310,7 @@ class VectorEmbeddingManager:
|
||||
|
||||
Example:
|
||||
>>> metadata = [{"text": "doc1", "category": "science"}]
|
||||
>>> formatted = manager.create_metadata(metadata, backend="pinecone")
|
||||
>>> formatted = manager.create_metadata(metadata, backend="weaviate")
|
||||
"""
|
||||
formatted = []
|
||||
|
||||
@@ -321,16 +319,7 @@ class VectorEmbeddingManager:
|
||||
formatted_meta = meta.copy()
|
||||
|
||||
# Backend-specific formatting
|
||||
if backend.lower() == "pinecone":
|
||||
# Pinecone has specific metadata requirements
|
||||
# Remove None values and ensure types are compatible
|
||||
formatted_meta = {
|
||||
k: v
|
||||
for k, v in formatted_meta.items()
|
||||
if v is not None
|
||||
and isinstance(v, (str, int, float, bool, list))
|
||||
}
|
||||
elif backend.lower() == "weaviate":
|
||||
if backend.lower() == "weaviate":
|
||||
# Weaviate uses specific property types
|
||||
# Ensure values are compatible
|
||||
formatted_meta = {
|
||||
@@ -374,8 +363,6 @@ class VectorEmbeddingManager:
|
||||
# Add backend-specific details
|
||||
if backend.lower() == "faiss":
|
||||
info["index_type"] = options.get("index_type", "flat")
|
||||
elif backend.lower() == "pinecone":
|
||||
info["namespace"] = options.get("namespace", "default")
|
||||
elif backend.lower() == "weaviate":
|
||||
info["class_name"] = options.get("class_name", "Document")
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ OWL Export:
|
||||
|
||||
Vector Export:
|
||||
- Vector Serialization: Multiple format support (JSON, NumPy, Binary, FAISS)
|
||||
- Vector Store Integration: Format conversion for Pinecone, Weaviate, Qdrant, FAISS
|
||||
- Vector Store Integration: Format conversion for Weaviate, Qdrant, FAISS
|
||||
- Metadata Association: Vector-to-metadata mapping and serialization
|
||||
- Batch Export: Efficient batch vector export processing
|
||||
- Multi-dimensional Support: Variable dimension vector handling
|
||||
|
||||
@@ -280,7 +280,7 @@ from semantica.export import YAMLSchemaExporter
|
||||
exporter = YAMLSchemaExporter()
|
||||
|
||||
# Export schema
|
||||
exporter.export(schema, "schema.yaml")
|
||||
exporter.export_ontology_schema(schema, "schema.yaml")
|
||||
```
|
||||
|
||||
### Using YAML Export Methods
|
||||
|
||||
+13
-11
@@ -110,7 +110,7 @@ OWL Export:
|
||||
|
||||
Vector Export:
|
||||
- Vector Serialization: Multiple format support (JSON, NumPy, Binary, FAISS)
|
||||
- Vector Store Integration: Format conversion for Pinecone, Weaviate, Qdrant, FAISS
|
||||
- Vector Store Integration: Format conversion for Weaviate, Qdrant, FAISS
|
||||
- Metadata Association: Vector-to-metadata mapping and serialization
|
||||
- Batch Export: Efficient batch vector export processing
|
||||
- Multi-dimensional Support: Variable dimension vector handling
|
||||
@@ -201,7 +201,7 @@ def export_rdf(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("rdf", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not export_rdf:
|
||||
try:
|
||||
return custom_method(data, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -250,7 +250,7 @@ def export_json(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("json", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not export_json:
|
||||
try:
|
||||
return custom_method(data, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -296,7 +296,7 @@ def export_csv(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("csv", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not export_csv:
|
||||
try:
|
||||
return custom_method(data, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -347,7 +347,7 @@ def export_graph(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("graph", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not export_graph:
|
||||
try:
|
||||
return custom_method(graph_data, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -395,7 +395,7 @@ def export_yaml(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("yaml", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not export_yaml:
|
||||
try:
|
||||
return custom_method(data, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -413,7 +413,9 @@ def export_yaml(
|
||||
exporter.export(data, file_path, **kwargs)
|
||||
elif method == "schema":
|
||||
exporter = YAMLSchemaExporter(**config)
|
||||
exporter.export(data, file_path, **kwargs)
|
||||
yaml_content = exporter.export_ontology_schema(data, **kwargs)
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
f.write(yaml_content)
|
||||
else:
|
||||
raise ProcessingError(f"Unknown YAML export method: {method}")
|
||||
|
||||
@@ -450,7 +452,7 @@ def export_owl(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("owl", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not export_owl:
|
||||
try:
|
||||
return custom_method(ontology, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -502,7 +504,7 @@ def export_vector(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("vector", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not export_vector:
|
||||
try:
|
||||
return custom_method(vectors, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -550,7 +552,7 @@ def export_lpg(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("lpg", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not export_lpg:
|
||||
try:
|
||||
return custom_method(knowledge_graph, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -601,7 +603,7 @@ def generate_report(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("report", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method is not generate_report:
|
||||
try:
|
||||
return custom_method(data, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
|
||||
@@ -797,15 +797,22 @@ class RDFExporter:
|
||||
if format == "turtle":
|
||||
result = self.serializer.serialize_to_turtle(data, **options)
|
||||
elif format == "rdfxml":
|
||||
return self.serializer.serialize_to_rdfxml(data, **options)
|
||||
result = self.serializer.serialize_to_rdfxml(data, **options)
|
||||
elif format == "jsonld":
|
||||
return self.serializer.serialize_to_jsonld(data, **options)
|
||||
result = self.serializer.serialize_to_jsonld(data, **options)
|
||||
else:
|
||||
raise ValidationError(
|
||||
f"Format '{format}' not yet implemented. "
|
||||
f"Implemented formats: turtle, rdfxml, jsonld"
|
||||
)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Exported to RDF format: {format}",
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
|
||||
@@ -7,7 +7,7 @@ embedding systems.
|
||||
|
||||
Key Features:
|
||||
- Multiple vector format export (JSON, NumPy, Binary, FAISS)
|
||||
- Vector store integration (Pinecone, Weaviate, Qdrant, FAISS)
|
||||
- Vector store integration (Weaviate, Qdrant, FAISS)
|
||||
- Metadata and document association
|
||||
- Batch vector export
|
||||
- Multi-dimensional vector support
|
||||
@@ -16,7 +16,7 @@ Example Usage:
|
||||
>>> from semantica.export import VectorExporter
|
||||
>>> exporter = VectorExporter(format="json", include_metadata=True)
|
||||
>>> exporter.export(vectors, "vectors.json")
|
||||
>>> exporter.export_for_vector_store(vectors, "pinecone.json", vector_store_type="pinecone")
|
||||
>>> exporter.export_for_vector_store(vectors, "weaviate.json", vector_store_type="weaviate")
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
@@ -43,7 +43,7 @@ class VectorExporter:
|
||||
|
||||
Features:
|
||||
- Multiple vector format export (JSON, NumPy, Binary, FAISS)
|
||||
- Vector store integration (Pinecone, Weaviate, Qdrant, FAISS)
|
||||
- Vector store integration (Weaviate, Qdrant, FAISS)
|
||||
- Metadata and document association
|
||||
- Batch vector export
|
||||
- Multi-dimensional vector support
|
||||
@@ -471,7 +471,7 @@ class VectorExporter:
|
||||
self,
|
||||
vectors: List[Dict[str, Any]],
|
||||
file_path: Union[str, Path],
|
||||
vector_store_type: str = "pinecone",
|
||||
vector_store_type: str = "weaviate",
|
||||
**options,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -480,12 +480,10 @@ class VectorExporter:
|
||||
Args:
|
||||
vectors: List of vector dictionaries
|
||||
file_path: Output file path
|
||||
vector_store_type: Vector store type ('pinecone', 'weaviate', 'qdrant', 'faiss')
|
||||
vector_store_type: Vector store type ('weaviate', 'qdrant', 'faiss')
|
||||
**options: Additional options
|
||||
"""
|
||||
if vector_store_type == "pinecone":
|
||||
self._export_pinecone_format(vectors, file_path, **options)
|
||||
elif vector_store_type == "weaviate":
|
||||
if vector_store_type == "weaviate":
|
||||
self._export_weaviate_format(vectors, file_path, **options)
|
||||
elif vector_store_type == "qdrant":
|
||||
self._export_qdrant_format(vectors, file_path, **options)
|
||||
@@ -495,27 +493,6 @@ class VectorExporter:
|
||||
# Default to JSON
|
||||
self._export_json(vectors, Path(file_path), {}, **options)
|
||||
|
||||
def _export_pinecone_format(
|
||||
self, vectors: List[Dict[str, Any]], file_path: Path, **options
|
||||
) -> None:
|
||||
"""Export in Pinecone format."""
|
||||
pinecone_data = []
|
||||
|
||||
for vec_data in vectors:
|
||||
vector_id = vec_data.get("id") or vec_data.get("vector_id", "")
|
||||
vector = vec_data.get("vector") or vec_data.get("embedding", [])
|
||||
metadata = vec_data.get("metadata", {})
|
||||
|
||||
if "text" in vec_data and self.include_text:
|
||||
metadata["text"] = vec_data["text"]
|
||||
|
||||
pinecone_data.append(
|
||||
{"id": vector_id, "values": vector, "metadata": metadata}
|
||||
)
|
||||
|
||||
export_data = {"vectors": pinecone_data}
|
||||
write_json_file(export_data, file_path, indent=2)
|
||||
|
||||
def _export_weaviate_format(
|
||||
self, vectors: List[Dict[str, Any]], file_path: Path, **options
|
||||
) -> None:
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
Graph Store Module
|
||||
|
||||
This module provides comprehensive property graph database integration for the
|
||||
Semantica framework, supporting multiple graph database backends including Neo4j,
|
||||
KuzuDB, and FalkorDB for storing and querying knowledge graphs.
|
||||
Semantica framework, supporting multiple graph database backends including Neo4j
|
||||
and FalkorDB for storing and querying knowledge graphs.
|
||||
|
||||
Algorithms Used:
|
||||
|
||||
Graph Store Management:
|
||||
- Store Registration: Store type detection, adapter factory pattern, configuration management, default store selection
|
||||
- Adapter Pattern: Unified interface for multiple backends (Neo4j, KuzuDB, FalkorDB), adapter instantiation, backend-specific operation delegation
|
||||
- Adapter Pattern: Unified interface for multiple backends (Neo4j, FalkorDB), adapter instantiation, backend-specific operation delegation
|
||||
- Store Selection: Default store resolution, store ID lookup, store validation
|
||||
|
||||
Node and Relationship Operations:
|
||||
@@ -37,7 +37,6 @@ Graph Analytics:
|
||||
|
||||
Store Adapters:
|
||||
- Neo4j Adapter: Official Neo4j Python driver, Bolt protocol communication, transaction support, multi-database support, APOC procedures
|
||||
- KuzuDB Adapter: Embedded graph database, in-memory and persistent storage, Cypher support, high-performance analytical queries
|
||||
- FalkorDB Adapter: Redis-based graph database, sparse matrix representation, linear algebra queries, OpenCypher support, ultra-fast performance
|
||||
|
||||
Bulk Operations:
|
||||
@@ -46,7 +45,7 @@ Bulk Operations:
|
||||
- Progress Tracking: Load progress calculation, elapsed time tracking, throughput calculation
|
||||
|
||||
Key Features:
|
||||
- Multi-backend property graph support (Neo4j, KuzuDB, FalkorDB)
|
||||
- Multi-backend property graph support (Neo4j, FalkorDB)
|
||||
- Full Cypher/OpenCypher query language support
|
||||
- Node and relationship CRUD operations
|
||||
- Graph traversal and path finding
|
||||
@@ -61,7 +60,6 @@ Main Classes:
|
||||
- GraphStore: Main graph store interface
|
||||
- GraphManager: Graph store management and operations
|
||||
- Neo4jAdapter: Neo4j integration adapter
|
||||
- KuzuAdapter: KuzuDB integration adapter
|
||||
- FalkorDBAdapter: FalkorDB integration adapter
|
||||
- NodeManager: Node CRUD operations
|
||||
- RelationshipManager: Relationship CRUD operations
|
||||
@@ -110,7 +108,6 @@ from .graph_store import (
|
||||
QueryEngine,
|
||||
RelationshipManager,
|
||||
)
|
||||
from .kuzu_adapter import KuzuAdapter, KuzuConnection, KuzuDatabase, KuzuQuery
|
||||
from .methods import (
|
||||
create_node,
|
||||
create_nodes,
|
||||
@@ -145,11 +142,6 @@ __all__ = [
|
||||
"Neo4jDriver",
|
||||
"Neo4jSession",
|
||||
"Neo4jTransaction",
|
||||
# KuzuDB
|
||||
"KuzuAdapter",
|
||||
"KuzuDatabase",
|
||||
"KuzuConnection",
|
||||
"KuzuQuery",
|
||||
# FalkorDB
|
||||
"FalkorDBAdapter",
|
||||
"FalkorDBClient",
|
||||
|
||||
@@ -119,10 +119,6 @@ class GraphStoreConfig:
|
||||
"GRAPH_STORE_NEO4J_PASSWORD": "neo4j_password",
|
||||
"GRAPH_STORE_NEO4J_DATABASE": "neo4j_database",
|
||||
"GRAPH_STORE_NEO4J_ENCRYPTED": "neo4j_encrypted",
|
||||
# KuzuDB settings
|
||||
"GRAPH_STORE_KUZU_DATABASE_PATH": "kuzu_database_path",
|
||||
"GRAPH_STORE_KUZU_BUFFER_POOL_SIZE": "kuzu_buffer_pool_size",
|
||||
"GRAPH_STORE_KUZU_MAX_NUM_THREADS": "kuzu_max_num_threads",
|
||||
# FalkorDB settings
|
||||
"GRAPH_STORE_FALKORDB_HOST": "falkordb_host",
|
||||
"GRAPH_STORE_FALKORDB_PORT": "falkordb_port",
|
||||
@@ -139,8 +135,6 @@ class GraphStoreConfig:
|
||||
"timeout",
|
||||
"max_retries",
|
||||
"falkordb_port",
|
||||
"kuzu_buffer_pool_size",
|
||||
"kuzu_max_num_threads",
|
||||
]:
|
||||
try:
|
||||
self._config[config_key] = int(value)
|
||||
@@ -172,10 +166,6 @@ class GraphStoreConfig:
|
||||
"neo4j_password": "password",
|
||||
"neo4j_database": "neo4j",
|
||||
"neo4j_encrypted": False,
|
||||
# KuzuDB defaults
|
||||
"kuzu_database_path": "./kuzu_db",
|
||||
"kuzu_buffer_pool_size": 268435456, # 256MB
|
||||
"kuzu_max_num_threads": 0, # 0 = auto
|
||||
# FalkorDB defaults
|
||||
"falkordb_host": "localhost",
|
||||
"falkordb_port": 6379,
|
||||
@@ -265,19 +255,6 @@ class GraphStoreConfig:
|
||||
"encrypted": self._config.get("neo4j_encrypted"),
|
||||
}
|
||||
|
||||
def get_kuzu_config(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get KuzuDB-specific configuration.
|
||||
|
||||
Returns:
|
||||
KuzuDB configuration dictionary
|
||||
"""
|
||||
return {
|
||||
"database_path": self._config.get("kuzu_database_path"),
|
||||
"buffer_pool_size": self._config.get("kuzu_buffer_pool_size"),
|
||||
"max_num_threads": self._config.get("kuzu_max_num_threads"),
|
||||
}
|
||||
|
||||
def get_falkordb_config(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get FalkorDB-specific configuration.
|
||||
|
||||
@@ -3,7 +3,7 @@ Graph Store Core Module
|
||||
|
||||
This module provides the core graph store interface and management classes,
|
||||
providing a unified interface across multiple graph database backends
|
||||
(Neo4j, KuzuDB, FalkorDB).
|
||||
(Neo4j, FalkorDB).
|
||||
|
||||
Key Features:
|
||||
- Unified graph store interface
|
||||
@@ -507,7 +507,7 @@ class GraphStore:
|
||||
Main graph store interface.
|
||||
|
||||
Provides a unified interface for working with property graph databases,
|
||||
supporting Neo4j, KuzuDB, and FalkorDB backends.
|
||||
supporting Neo4j and FalkorDB backends.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -519,7 +519,7 @@ class GraphStore:
|
||||
Initialize graph store.
|
||||
|
||||
Args:
|
||||
backend: Backend type ("neo4j", "kuzu", "falkordb")
|
||||
backend: Backend type ("neo4j", "falkordb")
|
||||
**config: Backend-specific configuration
|
||||
"""
|
||||
self.logger = get_logger("graph_store")
|
||||
@@ -542,12 +542,6 @@ class GraphStore:
|
||||
neo4j_config.update(self.config)
|
||||
self._adapter = Neo4jAdapter(**neo4j_config)
|
||||
|
||||
elif self.backend == "kuzu":
|
||||
from .kuzu_adapter import KuzuAdapter
|
||||
kuzu_config = graph_store_config.get_kuzu_config()
|
||||
kuzu_config.update(self.config)
|
||||
self._adapter = KuzuAdapter(**kuzu_config)
|
||||
|
||||
elif self.backend == "falkordb":
|
||||
from .falkordb_adapter import FalkorDBAdapter
|
||||
falkordb_config = graph_store_config.get_falkordb_config()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Graph Store Module Usage Guide
|
||||
|
||||
The Graph Store module provides comprehensive property graph database integration for the Semantica framework, supporting multiple backends including **Neo4j**, **KuzuDB**, and **FalkorDB**.
|
||||
The Graph Store module provides comprehensive property graph database integration for the Semantica framework, supporting multiple backends including **Neo4j** and **FalkorDB**.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
@@ -28,9 +28,6 @@ pip install semantica
|
||||
# Neo4j
|
||||
pip install neo4j
|
||||
|
||||
# KuzuDB
|
||||
pip install kuzu
|
||||
|
||||
# FalkorDB
|
||||
pip install falkordb
|
||||
```
|
||||
@@ -136,47 +133,6 @@ store = GraphStore(
|
||||
)
|
||||
```
|
||||
|
||||
### KuzuDB Configuration
|
||||
|
||||
```python
|
||||
from semantica.graph_store import GraphStore
|
||||
|
||||
store = GraphStore(
|
||||
backend="kuzu",
|
||||
database_path="./my_kuzu_db",
|
||||
buffer_pool_size=268435456, # 256MB
|
||||
max_num_threads=4
|
||||
)
|
||||
|
||||
# Connect (creates database if not exists)
|
||||
store.connect()
|
||||
|
||||
# For KuzuDB, you need to create node/relationship tables first
|
||||
from semantica.graph_store import KuzuAdapter
|
||||
|
||||
adapter = KuzuAdapter(database_path="./my_kuzu_db")
|
||||
adapter.connect()
|
||||
|
||||
# Create node table with schema
|
||||
adapter.create_node_table(
|
||||
"Person",
|
||||
properties={
|
||||
"id": "SERIAL",
|
||||
"name": "STRING",
|
||||
"age": "INT64"
|
||||
},
|
||||
primary_key="id"
|
||||
)
|
||||
|
||||
# Create relationship table
|
||||
adapter.create_rel_table(
|
||||
"KNOWS",
|
||||
from_table="Person",
|
||||
to_table="Person",
|
||||
properties={"since": "INT64"}
|
||||
)
|
||||
```
|
||||
|
||||
### FalkorDB Configuration
|
||||
|
||||
```python
|
||||
@@ -209,9 +165,6 @@ export GRAPH_STORE_NEO4J_URI=bolt://localhost:7687
|
||||
export GRAPH_STORE_NEO4J_USER=neo4j
|
||||
export GRAPH_STORE_NEO4J_PASSWORD=password
|
||||
|
||||
# KuzuDB
|
||||
export GRAPH_STORE_KUZU_DATABASE_PATH=./kuzu_db
|
||||
|
||||
# FalkorDB
|
||||
export GRAPH_STORE_FALKORDB_HOST=localhost
|
||||
export GRAPH_STORE_FALKORDB_PORT=6379
|
||||
@@ -478,7 +431,6 @@ graph_store_config.update({
|
||||
|
||||
# Get backend-specific configuration
|
||||
neo4j_config = graph_store_config.get_neo4j_config()
|
||||
kuzu_config = graph_store_config.get_kuzu_config()
|
||||
falkordb_config = graph_store_config.get_falkordb_config()
|
||||
|
||||
# Get all configuration
|
||||
@@ -598,14 +550,14 @@ print(f"Labels: {stats.get('label_counts')}")
|
||||
|
||||
## Backend Comparison
|
||||
|
||||
| Feature | Neo4j | KuzuDB | FalkorDB |
|
||||
|---------|-------|--------|----------|
|
||||
| Query Language | Cypher | Cypher | OpenCypher |
|
||||
| Deployment | Server/Cloud | Embedded | Server (Redis) |
|
||||
| Schema | Schema-optional | Schema-required | Schema-optional |
|
||||
| Transactions | ACID | ACID | ACID |
|
||||
| Performance | Good | Excellent (Analytics) | Ultra-fast |
|
||||
| Use Case | General purpose | Analytics | Real-time, LLM |
|
||||
| Feature | Neo4j | FalkorDB |
|
||||
|---------|-------|----------|
|
||||
| Query Language | Cypher | OpenCypher |
|
||||
| Deployment | Server/Cloud | Server (Redis) |
|
||||
| Schema | Schema-optional | Schema-optional |
|
||||
| Transactions | ACID | ACID |
|
||||
| Performance | Good | Ultra-fast |
|
||||
| Use Case | General purpose | Real-time, LLM |
|
||||
|
||||
## Best Practices
|
||||
|
||||
|
||||
@@ -1,917 +0,0 @@
|
||||
"""
|
||||
KuzuDB Adapter Module
|
||||
|
||||
This module provides KuzuDB embedded graph database integration for property graph
|
||||
storage and Cypher querying in the Semantica framework, supporting high-performance
|
||||
analytical queries with in-memory and persistent storage.
|
||||
|
||||
Key Features:
|
||||
- Embedded graph database (no server required)
|
||||
- Full Cypher query language support
|
||||
- High-performance analytical queries
|
||||
- In-memory and persistent storage modes
|
||||
- Node table and relationship table management
|
||||
- Schema-based property graph model
|
||||
- COPY FROM for bulk data loading
|
||||
- Optional dependency handling
|
||||
|
||||
Main Classes:
|
||||
- KuzuAdapter: Main KuzuDB adapter for graph operations
|
||||
- KuzuDatabase: Database wrapper
|
||||
- KuzuConnection: Connection wrapper
|
||||
- KuzuQuery: Query execution wrapper
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.graph_store import KuzuAdapter
|
||||
>>> adapter = KuzuAdapter(database_path="./kuzu_db")
|
||||
>>> adapter.connect()
|
||||
>>> adapter.create_node_table("Person", {"name": "STRING", "age": "INT64"})
|
||||
>>> node_id = adapter.create_node("Person", {"name": "Alice", "age": 30})
|
||||
>>> results = adapter.execute_query("MATCH (p:Person) RETURN p.name")
|
||||
>>> adapter.close()
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
# Optional KuzuDB import
|
||||
try:
|
||||
import kuzu
|
||||
|
||||
KUZU_AVAILABLE = True
|
||||
except ImportError:
|
||||
KUZU_AVAILABLE = False
|
||||
kuzu = None
|
||||
|
||||
|
||||
class KuzuDatabase:
|
||||
"""KuzuDB database wrapper."""
|
||||
|
||||
def __init__(self, database: Any):
|
||||
"""Initialize KuzuDB database wrapper."""
|
||||
self.database = database
|
||||
self.logger = get_logger("kuzu_database")
|
||||
|
||||
def get_connection(self) -> "KuzuConnection":
|
||||
"""
|
||||
Get a connection to the database.
|
||||
|
||||
Returns:
|
||||
KuzuConnection instance
|
||||
"""
|
||||
if not KUZU_AVAILABLE:
|
||||
raise ProcessingError("KuzuDB not available")
|
||||
|
||||
try:
|
||||
conn = kuzu.Connection(self.database)
|
||||
return KuzuConnection(conn)
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to create connection: {str(e)}")
|
||||
|
||||
|
||||
class KuzuConnection:
|
||||
"""KuzuDB connection wrapper."""
|
||||
|
||||
def __init__(self, connection: Any):
|
||||
"""Initialize KuzuDB connection wrapper."""
|
||||
self.connection = connection
|
||||
self.logger = get_logger("kuzu_connection")
|
||||
|
||||
def execute(self, query: str, parameters: Optional[Dict[str, Any]] = None) -> "KuzuQuery":
|
||||
"""
|
||||
Execute a Cypher query.
|
||||
|
||||
Args:
|
||||
query: Cypher query string
|
||||
parameters: Query parameters
|
||||
|
||||
Returns:
|
||||
KuzuQuery result wrapper
|
||||
"""
|
||||
if not KUZU_AVAILABLE:
|
||||
raise ProcessingError("KuzuDB not available")
|
||||
|
||||
try:
|
||||
if parameters:
|
||||
result = self.connection.execute(query, parameters)
|
||||
else:
|
||||
result = self.connection.execute(query)
|
||||
return KuzuQuery(result)
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Query execution failed: {str(e)}")
|
||||
|
||||
def set_max_threads(self, num_threads: int) -> None:
|
||||
"""Set maximum number of threads for query execution."""
|
||||
if self.connection and hasattr(self.connection, "set_max_threads_for_exec"):
|
||||
self.connection.set_max_threads_for_exec(num_threads)
|
||||
|
||||
|
||||
class KuzuQuery:
|
||||
"""KuzuDB query result wrapper."""
|
||||
|
||||
def __init__(self, result: Any):
|
||||
"""Initialize KuzuDB query result wrapper."""
|
||||
self.result = result
|
||||
self.logger = get_logger("kuzu_query")
|
||||
|
||||
def has_next(self) -> bool:
|
||||
"""Check if there are more results."""
|
||||
if self.result:
|
||||
return self.result.has_next()
|
||||
return False
|
||||
|
||||
def get_next(self) -> List[Any]:
|
||||
"""Get next result row."""
|
||||
if self.result:
|
||||
return self.result.get_next()
|
||||
return []
|
||||
|
||||
def get_all(self) -> List[List[Any]]:
|
||||
"""Get all results as a list of rows."""
|
||||
results = []
|
||||
while self.has_next():
|
||||
results.append(self.get_next())
|
||||
return results
|
||||
|
||||
def get_column_names(self) -> List[str]:
|
||||
"""Get column names from result."""
|
||||
if self.result and hasattr(self.result, "get_column_names"):
|
||||
return self.result.get_column_names()
|
||||
return []
|
||||
|
||||
def get_column_types(self) -> List[str]:
|
||||
"""Get column types from result."""
|
||||
if self.result and hasattr(self.result, "get_column_data_types"):
|
||||
return [str(t) for t in self.result.get_column_data_types()]
|
||||
return []
|
||||
|
||||
|
||||
class KuzuAdapter:
|
||||
"""
|
||||
KuzuDB adapter for embedded property graph storage and Cypher querying.
|
||||
|
||||
• Embedded database (no server required)
|
||||
• Schema-based node and relationship tables
|
||||
• High-performance analytical queries
|
||||
• In-memory and persistent storage
|
||||
• Bulk data loading with COPY FROM
|
||||
• Performance optimization
|
||||
• Error handling and recovery
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
database_path: Optional[str] = None,
|
||||
buffer_pool_size: Optional[int] = None,
|
||||
max_num_threads: int = 0,
|
||||
**config,
|
||||
):
|
||||
"""
|
||||
Initialize KuzuDB adapter.
|
||||
|
||||
Args:
|
||||
database_path: Path to database directory
|
||||
buffer_pool_size: Buffer pool size in bytes
|
||||
max_num_threads: Maximum number of threads (0 = auto)
|
||||
**config: Additional configuration options
|
||||
"""
|
||||
self.logger = get_logger("kuzu_adapter")
|
||||
self.config = config
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.database_path = database_path or config.get("database_path", "./kuzu_db")
|
||||
self.buffer_pool_size = buffer_pool_size or config.get("buffer_pool_size", 268435456)
|
||||
self.max_num_threads = max_num_threads or config.get("max_num_threads", 0)
|
||||
|
||||
self._database: Optional[KuzuDatabase] = None
|
||||
self._connection: Optional[KuzuConnection] = None
|
||||
|
||||
# Track created tables for schema management
|
||||
self._node_tables: Dict[str, Dict[str, str]] = {}
|
||||
self._rel_tables: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
# Check KuzuDB availability
|
||||
if not KUZU_AVAILABLE:
|
||||
self.logger.warning(
|
||||
"KuzuDB not available. Install with: pip install kuzu"
|
||||
)
|
||||
|
||||
def connect(self, database_path: Optional[str] = None, **options) -> bool:
|
||||
"""
|
||||
Connect to (or create) KuzuDB database.
|
||||
|
||||
Args:
|
||||
database_path: Path to database directory
|
||||
**options: Connection options
|
||||
|
||||
Returns:
|
||||
True if connected successfully
|
||||
"""
|
||||
if not KUZU_AVAILABLE:
|
||||
raise ProcessingError(
|
||||
"KuzuDB is not available. Install it with: pip install kuzu"
|
||||
)
|
||||
|
||||
database_path = database_path or self.database_path
|
||||
|
||||
try:
|
||||
# Create directory if it doesn't exist
|
||||
os.makedirs(database_path, exist_ok=True)
|
||||
|
||||
# Create database
|
||||
db = kuzu.Database(database_path, buffer_pool_size=self.buffer_pool_size)
|
||||
self._database = KuzuDatabase(db)
|
||||
|
||||
# Create connection
|
||||
self._connection = self._database.get_connection()
|
||||
|
||||
if self.max_num_threads > 0:
|
||||
self._connection.set_max_threads(self.max_num_threads)
|
||||
|
||||
self.logger.info(f"Connected to KuzuDB at {database_path}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to connect to KuzuDB: {str(e)}")
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close connection to KuzuDB."""
|
||||
self._connection = None
|
||||
self._database = None
|
||||
self.logger.info("Disconnected from KuzuDB")
|
||||
|
||||
def _ensure_connection(self) -> KuzuConnection:
|
||||
"""Ensure connection is established."""
|
||||
if self._connection is None:
|
||||
self.connect()
|
||||
return self._connection
|
||||
|
||||
def create_node_table(
|
||||
self,
|
||||
table_name: str,
|
||||
properties: Dict[str, str],
|
||||
primary_key: str = "id",
|
||||
**options,
|
||||
) -> bool:
|
||||
"""
|
||||
Create a node table with schema.
|
||||
|
||||
Args:
|
||||
table_name: Name of the node table
|
||||
properties: Property schema {property_name: type}
|
||||
Types: STRING, INT64, INT32, DOUBLE, FLOAT, BOOLEAN, DATE, TIMESTAMP
|
||||
primary_key: Primary key property name
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
True if created successfully
|
||||
"""
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
|
||||
# Build property list with primary key
|
||||
prop_list = []
|
||||
for prop_name, prop_type in properties.items():
|
||||
if prop_name == primary_key:
|
||||
prop_list.insert(0, f"{prop_name} {prop_type} PRIMARY KEY")
|
||||
else:
|
||||
prop_list.append(f"{prop_name} {prop_type}")
|
||||
|
||||
# Ensure primary key is in properties
|
||||
if primary_key not in properties:
|
||||
prop_list.insert(0, f"{primary_key} SERIAL PRIMARY KEY")
|
||||
|
||||
schema_def = ", ".join(prop_list)
|
||||
query = f"CREATE NODE TABLE IF NOT EXISTS {table_name}({schema_def})"
|
||||
|
||||
conn.execute(query)
|
||||
self._node_tables[table_name] = properties
|
||||
self.logger.info(f"Created node table: {table_name}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to create node table: {str(e)}")
|
||||
|
||||
def create_rel_table(
|
||||
self,
|
||||
table_name: str,
|
||||
from_table: str,
|
||||
to_table: str,
|
||||
properties: Optional[Dict[str, str]] = None,
|
||||
**options,
|
||||
) -> bool:
|
||||
"""
|
||||
Create a relationship table.
|
||||
|
||||
Args:
|
||||
table_name: Name of the relationship table
|
||||
from_table: Source node table name
|
||||
to_table: Target node table name
|
||||
properties: Property schema {property_name: type}
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
True if created successfully
|
||||
"""
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
|
||||
# Build property list
|
||||
if properties:
|
||||
prop_list = [f"{name} {ptype}" for name, ptype in properties.items()]
|
||||
schema_def = ", " + ", ".join(prop_list)
|
||||
else:
|
||||
schema_def = ""
|
||||
|
||||
query = f"CREATE REL TABLE IF NOT EXISTS {table_name}(FROM {from_table} TO {to_table}{schema_def})"
|
||||
|
||||
conn.execute(query)
|
||||
self._rel_tables[table_name] = {
|
||||
"from": from_table,
|
||||
"to": to_table,
|
||||
"properties": properties or {},
|
||||
}
|
||||
self.logger.info(f"Created relationship table: {table_name}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to create relationship table: {str(e)}")
|
||||
|
||||
def create_node(
|
||||
self,
|
||||
table_name: str,
|
||||
properties: Dict[str, Any],
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Create a node in a table.
|
||||
|
||||
Args:
|
||||
table_name: Node table name
|
||||
properties: Node properties
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Created node information
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="graph_store",
|
||||
submodule="KuzuAdapter",
|
||||
message=f"Creating node in table {table_name}",
|
||||
)
|
||||
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
|
||||
# Build property assignment
|
||||
prop_names = list(properties.keys())
|
||||
prop_values = []
|
||||
for value in properties.values():
|
||||
if isinstance(value, str):
|
||||
prop_values.append(f"'{value}'")
|
||||
elif value is None:
|
||||
prop_values.append("NULL")
|
||||
else:
|
||||
prop_values.append(str(value))
|
||||
|
||||
names_str = ", ".join(prop_names)
|
||||
values_str = ", ".join(prop_values)
|
||||
|
||||
query = f"CREATE (n:{table_name} {{{names_str}: [{values_str}]}}) RETURN n"
|
||||
# Alternative simpler syntax
|
||||
query = f"CREATE (:{table_name} {{{', '.join(f'{k}: {repr(v) if isinstance(v, str) else v}' for k, v in properties.items())}}})"
|
||||
|
||||
conn.execute(query)
|
||||
|
||||
# Get the created node (KuzuDB uses SERIAL for auto-incrementing IDs)
|
||||
result = conn.execute(f"MATCH (n:{table_name}) WHERE n.{list(properties.keys())[0]} = {repr(list(properties.values())[0]) if isinstance(list(properties.values())[0], str) else list(properties.values())[0]} RETURN n")
|
||||
|
||||
node_data = {
|
||||
"table": table_name,
|
||||
"properties": properties,
|
||||
}
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Created node in {table_name}",
|
||||
)
|
||||
return node_data
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise ProcessingError(f"Failed to create node: {str(e)}")
|
||||
|
||||
def create_nodes(
|
||||
self,
|
||||
table_name: str,
|
||||
nodes: List[Dict[str, Any]],
|
||||
**options,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Create multiple nodes in batch.
|
||||
|
||||
Args:
|
||||
table_name: Node table name
|
||||
nodes: List of node property dictionaries
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
List of created node information
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="graph_store",
|
||||
submodule="KuzuAdapter",
|
||||
message=f"Creating {len(nodes)} nodes in table {table_name}",
|
||||
)
|
||||
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
created_nodes = []
|
||||
|
||||
for node_props in nodes:
|
||||
props_str = ", ".join(
|
||||
f"{k}: {repr(v) if isinstance(v, str) else v}"
|
||||
for k, v in node_props.items()
|
||||
)
|
||||
query = f"CREATE (:{table_name} {{{props_str}}})"
|
||||
conn.execute(query)
|
||||
created_nodes.append({
|
||||
"table": table_name,
|
||||
"properties": node_props,
|
||||
})
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Created {len(created_nodes)} nodes",
|
||||
)
|
||||
return created_nodes
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise ProcessingError(f"Failed to create nodes: {str(e)}")
|
||||
|
||||
def get_nodes(
|
||||
self,
|
||||
table_name: str,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
limit: int = 100,
|
||||
**options,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get nodes from a table.
|
||||
|
||||
Args:
|
||||
table_name: Node table name
|
||||
filters: Property filters
|
||||
limit: Maximum number of nodes
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
List of nodes
|
||||
"""
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
|
||||
query = f"MATCH (n:{table_name})"
|
||||
|
||||
if filters:
|
||||
conditions = []
|
||||
for key, value in filters.items():
|
||||
if isinstance(value, str):
|
||||
conditions.append(f"n.{key} = '{value}'")
|
||||
else:
|
||||
conditions.append(f"n.{key} = {value}")
|
||||
query += " WHERE " + " AND ".join(conditions)
|
||||
|
||||
query += f" RETURN n LIMIT {limit}"
|
||||
|
||||
result = conn.execute(query)
|
||||
nodes = []
|
||||
|
||||
while result.has_next():
|
||||
row = result.get_next()
|
||||
if row and len(row) > 0:
|
||||
node = row[0]
|
||||
if isinstance(node, dict):
|
||||
nodes.append({
|
||||
"table": table_name,
|
||||
"properties": node,
|
||||
})
|
||||
else:
|
||||
# Handle node object
|
||||
nodes.append({
|
||||
"table": table_name,
|
||||
"properties": dict(node) if hasattr(node, "__iter__") else {"_raw": str(node)},
|
||||
})
|
||||
|
||||
return nodes
|
||||
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to get nodes: {str(e)}")
|
||||
|
||||
def update_node(
|
||||
self,
|
||||
table_name: str,
|
||||
filters: Dict[str, Any],
|
||||
properties: Dict[str, Any],
|
||||
**options,
|
||||
) -> bool:
|
||||
"""
|
||||
Update node properties.
|
||||
|
||||
Args:
|
||||
table_name: Node table name
|
||||
filters: Filters to identify node(s)
|
||||
properties: Properties to update
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
True if updated successfully
|
||||
"""
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
|
||||
# Build WHERE clause
|
||||
conditions = []
|
||||
for key, value in filters.items():
|
||||
if isinstance(value, str):
|
||||
conditions.append(f"n.{key} = '{value}'")
|
||||
else:
|
||||
conditions.append(f"n.{key} = {value}")
|
||||
|
||||
# Build SET clause
|
||||
updates = []
|
||||
for key, value in properties.items():
|
||||
if isinstance(value, str):
|
||||
updates.append(f"n.{key} = '{value}'")
|
||||
else:
|
||||
updates.append(f"n.{key} = {value}")
|
||||
|
||||
query = f"MATCH (n:{table_name}) WHERE {' AND '.join(conditions)} SET {', '.join(updates)}"
|
||||
conn.execute(query)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to update node: {str(e)}")
|
||||
|
||||
def delete_node(
|
||||
self,
|
||||
table_name: str,
|
||||
filters: Dict[str, Any],
|
||||
**options,
|
||||
) -> bool:
|
||||
"""
|
||||
Delete node(s).
|
||||
|
||||
Args:
|
||||
table_name: Node table name
|
||||
filters: Filters to identify node(s)
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
True if deleted successfully
|
||||
"""
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
|
||||
# Build WHERE clause
|
||||
conditions = []
|
||||
for key, value in filters.items():
|
||||
if isinstance(value, str):
|
||||
conditions.append(f"n.{key} = '{value}'")
|
||||
else:
|
||||
conditions.append(f"n.{key} = {value}")
|
||||
|
||||
query = f"MATCH (n:{table_name}) WHERE {' AND '.join(conditions)} DETACH DELETE n"
|
||||
conn.execute(query)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to delete node: {str(e)}")
|
||||
|
||||
def create_relationship(
|
||||
self,
|
||||
rel_table: str,
|
||||
from_table: str,
|
||||
from_filters: Dict[str, Any],
|
||||
to_table: str,
|
||||
to_filters: Dict[str, Any],
|
||||
properties: Optional[Dict[str, Any]] = None,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Create a relationship between nodes.
|
||||
|
||||
Args:
|
||||
rel_table: Relationship table name
|
||||
from_table: Source node table name
|
||||
from_filters: Filters to identify source node
|
||||
to_table: Target node table name
|
||||
to_filters: Filters to identify target node
|
||||
properties: Relationship properties
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Created relationship information
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="graph_store",
|
||||
submodule="KuzuAdapter",
|
||||
message=f"Creating relationship [{rel_table}]",
|
||||
)
|
||||
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
|
||||
# Build WHERE clauses
|
||||
from_conditions = []
|
||||
for key, value in from_filters.items():
|
||||
if isinstance(value, str):
|
||||
from_conditions.append(f"a.{key} = '{value}'")
|
||||
else:
|
||||
from_conditions.append(f"a.{key} = {value}")
|
||||
|
||||
to_conditions = []
|
||||
for key, value in to_filters.items():
|
||||
if isinstance(value, str):
|
||||
to_conditions.append(f"b.{key} = '{value}'")
|
||||
else:
|
||||
to_conditions.append(f"b.{key} = {value}")
|
||||
|
||||
# Build property string
|
||||
if properties:
|
||||
props_str = "{" + ", ".join(
|
||||
f"{k}: {repr(v) if isinstance(v, str) else v}"
|
||||
for k, v in properties.items()
|
||||
) + "}"
|
||||
else:
|
||||
props_str = ""
|
||||
|
||||
query = f"""
|
||||
MATCH (a:{from_table}), (b:{to_table})
|
||||
WHERE {' AND '.join(from_conditions)} AND {' AND '.join(to_conditions)}
|
||||
CREATE (a)-[:{rel_table} {props_str}]->(b)
|
||||
"""
|
||||
|
||||
conn.execute(query)
|
||||
|
||||
rel_data = {
|
||||
"type": rel_table,
|
||||
"from_table": from_table,
|
||||
"to_table": to_table,
|
||||
"properties": properties or {},
|
||||
}
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Created relationship [{rel_table}]",
|
||||
)
|
||||
return rel_data
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise ProcessingError(f"Failed to create relationship: {str(e)}")
|
||||
|
||||
def get_relationships(
|
||||
self,
|
||||
rel_table: Optional[str] = None,
|
||||
from_table: Optional[str] = None,
|
||||
to_table: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
**options,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get relationships.
|
||||
|
||||
Args:
|
||||
rel_table: Relationship table name
|
||||
from_table: Source node table filter
|
||||
to_table: Target node table filter
|
||||
limit: Maximum number of relationships
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
List of relationships
|
||||
"""
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
|
||||
from_pattern = f":{from_table}" if from_table else ""
|
||||
to_pattern = f":{to_table}" if to_table else ""
|
||||
rel_pattern = f":{rel_table}" if rel_table else ""
|
||||
|
||||
query = f"MATCH (a{from_pattern})-[r{rel_pattern}]->(b{to_pattern}) RETURN a, r, b LIMIT {limit}"
|
||||
|
||||
result = conn.execute(query)
|
||||
relationships = []
|
||||
|
||||
while result.has_next():
|
||||
row = result.get_next()
|
||||
if row and len(row) >= 3:
|
||||
relationships.append({
|
||||
"from_node": row[0],
|
||||
"relationship": row[1],
|
||||
"to_node": row[2],
|
||||
})
|
||||
|
||||
return relationships
|
||||
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to get relationships: {str(e)}")
|
||||
|
||||
def execute_query(
|
||||
self,
|
||||
query: str,
|
||||
parameters: Optional[Dict[str, Any]] = None,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Execute a Cypher query.
|
||||
|
||||
Args:
|
||||
query: Cypher query string
|
||||
parameters: Query parameters
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Query results
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="graph_store",
|
||||
submodule="KuzuAdapter",
|
||||
message="Executing Cypher query",
|
||||
)
|
||||
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
result = conn.execute(query, parameters)
|
||||
|
||||
records = result.get_all()
|
||||
column_names = result.get_column_names()
|
||||
column_types = result.get_column_types()
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Query returned {len(records)} records",
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"records": records,
|
||||
"column_names": column_names,
|
||||
"column_types": column_types,
|
||||
"metadata": {"query": query},
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise ProcessingError(f"Query execution failed: {str(e)}")
|
||||
|
||||
def shortest_path(
|
||||
self,
|
||||
from_table: str,
|
||||
from_filters: Dict[str, Any],
|
||||
to_table: str,
|
||||
to_filters: Dict[str, Any],
|
||||
rel_table: Optional[str] = None,
|
||||
max_depth: int = 10,
|
||||
**options,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Find shortest path between two nodes.
|
||||
|
||||
Args:
|
||||
from_table: Source node table
|
||||
from_filters: Filters to identify source node
|
||||
to_table: Target node table
|
||||
to_filters: Filters to identify target node
|
||||
rel_table: Relationship table filter
|
||||
max_depth: Maximum path length
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Shortest path information or None
|
||||
"""
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
|
||||
# Build WHERE clauses
|
||||
from_conditions = []
|
||||
for key, value in from_filters.items():
|
||||
if isinstance(value, str):
|
||||
from_conditions.append(f"a.{key} = '{value}'")
|
||||
else:
|
||||
from_conditions.append(f"a.{key} = {value}")
|
||||
|
||||
to_conditions = []
|
||||
for key, value in to_filters.items():
|
||||
if isinstance(value, str):
|
||||
to_conditions.append(f"b.{key} = '{value}'")
|
||||
else:
|
||||
to_conditions.append(f"b.{key} = {value}")
|
||||
|
||||
rel_pattern = f":{rel_table}" if rel_table else ""
|
||||
|
||||
query = f"""
|
||||
MATCH (a:{from_table}), (b:{to_table}),
|
||||
path = SHORTEST 1 GROUPS (a)-[r{rel_pattern}*..{max_depth}]-(b)
|
||||
WHERE {' AND '.join(from_conditions)} AND {' AND '.join(to_conditions)}
|
||||
RETURN path, length(path) as length
|
||||
"""
|
||||
|
||||
result = conn.execute(query)
|
||||
|
||||
if result.has_next():
|
||||
row = result.get_next()
|
||||
return {
|
||||
"path": row[0],
|
||||
"length": row[1] if len(row) > 1 else 0,
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
# Kuzu might not support all path queries, try simpler query
|
||||
self.logger.warning(f"Shortest path query failed: {str(e)}")
|
||||
return None
|
||||
|
||||
def bulk_load_nodes(
|
||||
self,
|
||||
table_name: str,
|
||||
file_path: str,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Bulk load nodes from CSV file.
|
||||
|
||||
Args:
|
||||
table_name: Node table name
|
||||
file_path: Path to CSV file
|
||||
**options: Additional options (header, delimiter, etc.)
|
||||
|
||||
Returns:
|
||||
Load result information
|
||||
"""
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
|
||||
header = options.get("header", True)
|
||||
delimiter = options.get("delimiter", ",")
|
||||
|
||||
query = f"COPY {table_name} FROM '{file_path}' (HEADER={str(header).lower()}, DELIM='{delimiter}')"
|
||||
conn.execute(query)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"table": table_name,
|
||||
"file": file_path,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Bulk load failed: {str(e)}")
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""Get database statistics."""
|
||||
try:
|
||||
conn = self._ensure_connection()
|
||||
stats = {
|
||||
"node_tables": list(self._node_tables.keys()),
|
||||
"rel_tables": list(self._rel_tables.keys()),
|
||||
"database_path": self.database_path,
|
||||
}
|
||||
|
||||
# Get node counts per table
|
||||
for table_name in self._node_tables.keys():
|
||||
try:
|
||||
result = conn.execute(f"MATCH (n:{table_name}) RETURN count(n) as count")
|
||||
if result.has_next():
|
||||
row = result.get_next()
|
||||
stats[f"{table_name}_count"] = row[0] if row else 0
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return stats
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to get stats: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
@@ -289,9 +289,10 @@ class MCPIngestor:
|
||||
|
||||
try:
|
||||
# Get tracking ID
|
||||
tracking_id = self.progress_tracker.start_task(
|
||||
task_type="mcp_ingest_resources",
|
||||
description=f"Ingesting resources from {server_name}",
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="ingest",
|
||||
submodule="MCPIngestor",
|
||||
message=f"Ingesting resources from {server_name}",
|
||||
)
|
||||
|
||||
# List available resources
|
||||
@@ -307,7 +308,7 @@ class MCPIngestor:
|
||||
|
||||
if not resources:
|
||||
self.logger.warning(f"No resources found for server {server_name}")
|
||||
self.progress_tracker.update_task(
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, status="completed", message="No resources found"
|
||||
)
|
||||
return []
|
||||
@@ -318,11 +319,10 @@ class MCPIngestor:
|
||||
|
||||
for idx, resource in enumerate(resources):
|
||||
try:
|
||||
self.progress_tracker.update_task(
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id,
|
||||
status="in_progress",
|
||||
progress=(idx / total) * 100,
|
||||
message=f"Reading resource: {resource.uri}",
|
||||
status="running",
|
||||
message=f"Reading resource: {resource.uri} ({idx + 1}/{total})",
|
||||
)
|
||||
|
||||
# Read resource
|
||||
@@ -347,17 +347,16 @@ class MCPIngestor:
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to ingest resource {resource.uri}: {e}")
|
||||
self.progress_tracker.update_task(
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id,
|
||||
status="warning",
|
||||
status="running",
|
||||
message=f"Failed to ingest resource {resource.uri}: {e}",
|
||||
)
|
||||
continue
|
||||
|
||||
self.progress_tracker.update_task(
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
progress=100,
|
||||
message=f"Successfully ingested {len(ingested_data)} resources",
|
||||
)
|
||||
|
||||
@@ -393,13 +392,14 @@ class MCPIngestor:
|
||||
|
||||
try:
|
||||
# Get tracking ID
|
||||
tracking_id = self.progress_tracker.start_task(
|
||||
task_type="mcp_ingest_tool",
|
||||
description=f"Calling tool {tool_name} on {server_name}",
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="ingest",
|
||||
submodule="MCPIngestor",
|
||||
message=f"Calling tool {tool_name} on {server_name}",
|
||||
)
|
||||
|
||||
self.progress_tracker.update_task(
|
||||
tracking_id, status="in_progress", message=f"Calling tool: {tool_name}"
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, status="running", message=f"Calling tool: {tool_name}"
|
||||
)
|
||||
|
||||
# Call tool
|
||||
@@ -415,10 +415,9 @@ class MCPIngestor:
|
||||
tool_name=tool_name,
|
||||
)
|
||||
|
||||
self.progress_tracker.update_task(
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
progress=100,
|
||||
message=f"Successfully called tool {tool_name}",
|
||||
)
|
||||
|
||||
|
||||
@@ -184,7 +184,7 @@ def ingest_file(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("file", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method != ingest_file:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -249,7 +249,7 @@ def ingest_web(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("web", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method != ingest_web:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -308,7 +308,7 @@ def ingest_feed(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("feed", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method != ingest_feed:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -365,7 +365,7 @@ def ingest_stream(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("stream", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method != ingest_stream:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -437,7 +437,7 @@ def ingest_repository(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("repo", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method != ingest_repository:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -495,7 +495,7 @@ def ingest_email(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("email", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method != ingest_email:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -566,7 +566,7 @@ def ingest_database(
|
||||
# Check for custom method in registry
|
||||
if method:
|
||||
custom_method = method_registry.get("db", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method != ingest_database:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
@@ -658,7 +658,7 @@ def ingest_mcp(
|
||||
"""
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("mcp", method)
|
||||
if custom_method:
|
||||
if custom_method and custom_method != ingest_mcp:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
|
||||
@@ -42,6 +42,7 @@ import git
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -500,6 +501,9 @@ class RepoIngestor:
|
||||
# Initialize analyzer
|
||||
self.analyzer = GitAnalyzer(**self.config)
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
# Temporary directory for cloning
|
||||
self.temp_dir = None
|
||||
|
||||
@@ -532,7 +536,7 @@ class RepoIngestor:
|
||||
try:
|
||||
parsed = git.Repo.clone_from(repo_url, self._get_temp_dir(), **options)
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise ProcessingError(f"Failed to clone repository: {e}") from e
|
||||
@@ -581,7 +585,7 @@ class RepoIngestor:
|
||||
structure = self.analyzer.analyze_structure(repo_path)
|
||||
metrics = self.analyzer.calculate_metrics(repo_path)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Processed {len(code_files)} files, {len(commits)} commits",
|
||||
@@ -596,7 +600,7 @@ class RepoIngestor:
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -235,7 +235,8 @@ class GraphBuilder:
|
||||
# Detect and resolve conflicts if conflict detector is available
|
||||
if self.conflict_detector:
|
||||
self.logger.debug("Detecting conflicts in graph")
|
||||
detected_conflicts = self.conflict_detector.detect_conflicts(graph)
|
||||
# Pass only entities to detect_conflicts as it expects List[Dict]
|
||||
detected_conflicts = self.conflict_detector.detect_conflicts(graph["entities"])
|
||||
|
||||
if detected_conflicts:
|
||||
conflict_count = len(detected_conflicts)
|
||||
|
||||
@@ -44,6 +44,7 @@ analyzer = GraphAnalyzer()
|
||||
analysis = analyzer.analyze_graph(kg)
|
||||
```
|
||||
|
||||
|
||||
## Knowledge Graph Building
|
||||
|
||||
### Basic Graph Building
|
||||
@@ -52,6 +53,8 @@ analysis = analyzer.analyze_graph(kg)
|
||||
from semantica.kg import GraphBuilder
|
||||
|
||||
# Create graph builder
|
||||
# Note: resolve_conflicts=True uses the basic resolution capabilities of ConflictDetector.
|
||||
# For advanced conflict resolution, consider using the semantica.conflicts module directly.
|
||||
builder = GraphBuilder(
|
||||
merge_entities=True,
|
||||
entity_resolution_strategy="fuzzy",
|
||||
|
||||
@@ -161,7 +161,7 @@ class DataCleaner:
|
||||
if handle_missing:
|
||||
strategy = options.get("missing_strategy", "remove")
|
||||
cleaned = self.missing_value_handler.handle_missing_values(
|
||||
cleaned, strategy=strategy
|
||||
cleaned, strategy=strategy, **options
|
||||
)
|
||||
|
||||
# Validate data
|
||||
@@ -688,6 +688,8 @@ class DataValidator:
|
||||
"""
|
||||
if isinstance(expected_types, type):
|
||||
expected_types = [expected_types]
|
||||
elif isinstance(expected_types, str):
|
||||
expected_types = [expected_types]
|
||||
|
||||
actual_type = type(data)
|
||||
|
||||
|
||||
@@ -520,7 +520,9 @@ class NameVariantHandler:
|
||||
# Remove titles
|
||||
name = entity_name
|
||||
for title in self.titles:
|
||||
name = name.replace(title + " ", "").replace(title, "")
|
||||
# Case-insensitive removal of titles from the beginning of the name
|
||||
pattern = re.compile(r"^" + re.escape(title) + r"\s*", re.IGNORECASE)
|
||||
name = pattern.sub("", name)
|
||||
|
||||
name = name.strip()
|
||||
|
||||
|
||||
@@ -802,10 +802,7 @@ def list_available_methods(task: Optional[str] = None) -> Dict[str, List[str]]:
|
||||
|
||||
|
||||
# Register default methods
|
||||
method_registry.register("text", "default", normalize_text)
|
||||
method_registry.register("clean", "default", clean_text)
|
||||
method_registry.register("entity", "default", normalize_entity)
|
||||
method_registry.register("date", "default", normalize_date)
|
||||
method_registry.register("number", "default", normalize_number)
|
||||
method_registry.register("language", "default", detect_language)
|
||||
method_registry.register("encoding", "default", handle_encoding)
|
||||
# Note: We do not register the convenience functions as defaults to avoid recursion.
|
||||
# The convenience functions have built-in fallback to the default implementations
|
||||
# (using the classes directly) when no custom method is found in the registry.
|
||||
|
||||
|
||||
@@ -443,15 +443,37 @@ class UnitConverter:
|
||||
# Map to standard unit
|
||||
unit_map = {
|
||||
"m": "meter",
|
||||
"meter": "meter",
|
||||
"meters": "meter",
|
||||
"km": "kilometer",
|
||||
"kilometer": "kilometer",
|
||||
"kilometers": "kilometer",
|
||||
"cm": "centimeter",
|
||||
"centimeter": "centimeter",
|
||||
"centimeters": "centimeter",
|
||||
"mm": "millimeter",
|
||||
"millimeter": "millimeter",
|
||||
"millimeters": "millimeter",
|
||||
"kg": "kilogram",
|
||||
"kilogram": "kilogram",
|
||||
"kilograms": "kilogram",
|
||||
"kgs": "kilogram",
|
||||
"g": "gram",
|
||||
"gram": "gram",
|
||||
"grams": "gram",
|
||||
"lb": "pound",
|
||||
"pound": "pound",
|
||||
"pounds": "pound",
|
||||
"lbs": "pound",
|
||||
"oz": "ounce",
|
||||
"ounce": "ounce",
|
||||
"ounces": "ounce",
|
||||
"l": "liter",
|
||||
"liter": "liter",
|
||||
"liters": "liter",
|
||||
"ml": "milliliter",
|
||||
"milliliter": "milliliter",
|
||||
"milliliters": "milliliter",
|
||||
}
|
||||
|
||||
return unit_map.get(unit_lower, unit_lower)
|
||||
|
||||
@@ -31,7 +31,7 @@ class LLMOntologyGenerator:
|
||||
)
|
||||
|
||||
base_uri = options.get("base_uri")
|
||||
name = options.get("name") or "GeneratedOntology"
|
||||
name = options.get("name")
|
||||
version = options.get("version") or "1.0"
|
||||
|
||||
prompt = self._build_prompt(text=text, name=name, base_uri=base_uri)
|
||||
|
||||
@@ -216,6 +216,10 @@ class NamespaceManager:
|
||||
|
||||
def _to_camel_case(self, name: str) -> str:
|
||||
"""Convert name to camelCase."""
|
||||
# Check if already likely camelCase (starts with lower, has upper, single word)
|
||||
if name and name[0].islower() and any(c.isupper() for c in name) and ' ' not in name and '_' not in name:
|
||||
return name
|
||||
|
||||
# Remove special characters and split
|
||||
words = re.findall(r"[a-zA-Z0-9]+", name)
|
||||
if not words:
|
||||
|
||||
@@ -262,8 +262,8 @@ class NamingConventions:
|
||||
# camelCase for object properties
|
||||
suggested = self._to_camel_case(name)
|
||||
else:
|
||||
# lowercase for data properties
|
||||
suggested = name.lower()
|
||||
# camelCase for data properties as well (standard practice)
|
||||
suggested = self._to_camel_case(name)
|
||||
|
||||
return suggested
|
||||
|
||||
@@ -364,6 +364,10 @@ class NamingConventions:
|
||||
|
||||
def _to_camel_case(self, name: str) -> str:
|
||||
"""Convert to camelCase."""
|
||||
# Check if already likely camelCase (starts with lower, has upper, single word)
|
||||
if name and name[0].islower() and any(c.isupper() for c in name) and ' ' not in name and '_' not in name:
|
||||
return name
|
||||
|
||||
words = re.findall(r"[a-zA-Z0-9]+", name)
|
||||
if not words:
|
||||
return "hasProperty"
|
||||
@@ -381,8 +385,8 @@ class NamingConventions:
|
||||
# Basic singularization rules
|
||||
if name.lower().endswith("ies"):
|
||||
return name[:-3] + "y"
|
||||
elif name.lower().endswith("es"):
|
||||
elif name.lower().endswith("es") and not name.lower().endswith("ss"):
|
||||
return name[:-2]
|
||||
elif name.lower().endswith("s") and len(name) > 1:
|
||||
elif name.lower().endswith("s") and len(name) > 1 and not name.lower().endswith("ss") and name.lower() not in ["class", "process", "analysis"]:
|
||||
return name[:-1]
|
||||
return name
|
||||
|
||||
@@ -170,7 +170,13 @@ class OntologyGenerator:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Stage 3: Mapping to OWL types..."
|
||||
)
|
||||
typed_definitions = self._stage3_definition_to_types(definitions, **options)
|
||||
|
||||
# Ensure entities and relationships are available for property inference
|
||||
stage3_options = options.copy()
|
||||
stage3_options["entities"] = data.get("entities", [])
|
||||
stage3_options["relationships"] = data.get("relationships", [])
|
||||
|
||||
typed_definitions = self._stage3_definition_to_types(definitions, **stage3_options)
|
||||
|
||||
# Stage 4: Hierarchy Generation
|
||||
self.progress_tracker.update_tracking(
|
||||
@@ -266,9 +272,14 @@ class OntologyGenerator:
|
||||
relationships = options.get("relationships", [])
|
||||
entities = options.get("entities", [])
|
||||
|
||||
# Clean options for infer_properties to avoid multiple values for arguments
|
||||
prop_options = options.copy()
|
||||
prop_options.pop("entities", None)
|
||||
prop_options.pop("relationships", None)
|
||||
|
||||
# Infer properties
|
||||
properties = self.property_generator.infer_properties(
|
||||
entities=entities, relationships=relationships, classes=classes, **options
|
||||
entities=entities, relationships=relationships, classes=classes, **prop_options
|
||||
)
|
||||
|
||||
# Add types to classes
|
||||
|
||||
@@ -89,6 +89,11 @@ class PropertyGenerator:
|
||||
submodule="PropertyGenerator",
|
||||
message=f"Inferring properties from {len(entities)} entities and {len(relationships)} relationships",
|
||||
)
|
||||
|
||||
# Merge config into options
|
||||
for key, value in self.config.items():
|
||||
if key not in options:
|
||||
options[key] = value
|
||||
|
||||
try:
|
||||
properties = []
|
||||
|
||||
@@ -37,6 +37,7 @@ from bs4 import BeautifulSoup
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -65,6 +66,20 @@ class HTMLElement:
|
||||
children: List["HTMLElement"] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HTMLData:
|
||||
"""HTML document representation."""
|
||||
|
||||
metadata: Dict[str, Any]
|
||||
text: str
|
||||
html: str
|
||||
links: List[Dict[str, Any]] = field(default_factory=list)
|
||||
images: List[Dict[str, Any]] = field(default_factory=list)
|
||||
forms: List[Dict[str, Any]] = field(default_factory=list)
|
||||
tables: List[Dict[str, Any]] = field(default_factory=list)
|
||||
structure: List[Dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
class HTMLParser:
|
||||
"""HTML document parser."""
|
||||
|
||||
@@ -81,7 +96,7 @@ class HTMLParser:
|
||||
|
||||
def parse(
|
||||
self, html_content: Union[str, Path], base_url: Optional[str] = None, **options
|
||||
) -> Dict[str, Any]:
|
||||
) -> HTMLData:
|
||||
"""
|
||||
Parse HTML content.
|
||||
|
||||
@@ -96,7 +111,7 @@ class HTMLParser:
|
||||
- clean_text: Whether to clean extracted text (default: True)
|
||||
|
||||
Returns:
|
||||
dict: Parsed HTML data
|
||||
HTMLData: Parsed HTML data
|
||||
"""
|
||||
# Track HTML parsing
|
||||
file_path = None
|
||||
@@ -160,16 +175,16 @@ class HTMLParser:
|
||||
status="completed",
|
||||
message=f"Parsed HTML: {len(links)} links, {len(images)} images",
|
||||
)
|
||||
return {
|
||||
"metadata": metadata.__dict__,
|
||||
"text": text,
|
||||
"html": html_string,
|
||||
"links": links,
|
||||
"images": images,
|
||||
"forms": forms,
|
||||
"tables": tables,
|
||||
"structure": structure,
|
||||
}
|
||||
return HTMLData(
|
||||
metadata=metadata.__dict__,
|
||||
text=text,
|
||||
html=html_string,
|
||||
links=links,
|
||||
images=images,
|
||||
forms=forms,
|
||||
tables=tables,
|
||||
structure=structure,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
@@ -184,6 +199,26 @@ class HTMLParser:
|
||||
)
|
||||
raise
|
||||
|
||||
def extract_metadata(self, html_content: Union[str, Path]) -> Dict[str, Any]:
|
||||
"""
|
||||
Extract metadata from HTML.
|
||||
|
||||
Args:
|
||||
html_content: HTML content or file path
|
||||
|
||||
Returns:
|
||||
dict: Extracted metadata
|
||||
"""
|
||||
result = self.parse(
|
||||
html_content,
|
||||
extract_links=False,
|
||||
extract_images=False,
|
||||
extract_forms=False,
|
||||
extract_tables=False,
|
||||
clean_text=False,
|
||||
)
|
||||
return result.metadata
|
||||
|
||||
def extract_text(self, html_content: Union[str, Path], clean: bool = True) -> str:
|
||||
"""
|
||||
Extract text from HTML.
|
||||
@@ -203,7 +238,7 @@ class HTMLParser:
|
||||
extract_tables=False,
|
||||
clean_text=clean,
|
||||
)
|
||||
return result["text"]
|
||||
return result.text
|
||||
|
||||
def extract_links(
|
||||
self, html_content: Union[str, Path], base_url: Optional[str] = None
|
||||
@@ -225,7 +260,7 @@ class HTMLParser:
|
||||
extract_forms=False,
|
||||
extract_tables=False,
|
||||
)
|
||||
return result["links"]
|
||||
return result.links
|
||||
|
||||
def _extract_metadata(self, soup: BeautifulSoup) -> HTMLMetadata:
|
||||
"""Extract metadata from HTML."""
|
||||
|
||||
@@ -58,6 +58,9 @@ class StructuredDataParser:
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
# Initialize parsers
|
||||
self.json_parser = JSONParser(**self.config.get("json", {}))
|
||||
self.csv_parser = CSVParser(**self.config.get("csv", {}))
|
||||
|
||||
@@ -148,7 +148,7 @@ class PipelineTemplateManager:
|
||||
{
|
||||
"name": "store_vectors",
|
||||
"type": "store_vectors",
|
||||
"config": {"store": "pinecone"},
|
||||
"config": {"store": "weaviate"},
|
||||
"dependencies": ["embed"],
|
||||
},
|
||||
],
|
||||
|
||||
@@ -632,7 +632,7 @@ builder = template_manager.create_pipeline_from_template(
|
||||
"rag_pipeline",
|
||||
chunk={"chunk_size": 512},
|
||||
embed={"model": "text-embedding-3-large"},
|
||||
store_vectors={"store": "pinecone"}
|
||||
store_vectors={"store": "weaviate"}
|
||||
)
|
||||
|
||||
pipeline = builder.build()
|
||||
@@ -1124,7 +1124,8 @@ builder = template_manager.create_pipeline_from_template(
|
||||
ingest={"source": "./documents"},
|
||||
chunk={"chunk_size": 512, "overlap": 50},
|
||||
embed={"model": "text-embedding-3-large", "batch_size": 32},
|
||||
store_vectors={"store": "pinecone", "index_name": "documents"}
|
||||
# Step-specific overrides
|
||||
store_vectors={"store": "weaviate", "index_name": "documents"}
|
||||
)
|
||||
|
||||
pipeline = builder.build()
|
||||
|
||||
@@ -34,6 +34,7 @@ License: MIT
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional
|
||||
import re
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -203,9 +204,58 @@ class AbductiveReasoner:
|
||||
|
||||
def _rule_explains_observation(self, rule: Rule, observation: Observation) -> bool:
|
||||
"""Check if rule can explain observation."""
|
||||
# Simple check: rule conclusion matches observation
|
||||
# Can be enhanced with more sophisticated matching
|
||||
return True
|
||||
# Check if rule conclusion matches observation description
|
||||
# Try exact match first
|
||||
if rule.conclusion == observation.description:
|
||||
return True
|
||||
|
||||
# Try unification if variables are involved
|
||||
if "?" in rule.conclusion:
|
||||
bindings = self._unify(rule.conclusion, observation.description, {})
|
||||
if bindings is not None:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _parse_predicate(self, text: str) -> tuple[str, List[str]]:
|
||||
"""Parse 'Predicate(arg1, arg2)' into ('Predicate', ['arg1', 'arg2'])."""
|
||||
if not isinstance(text, str):
|
||||
return text, []
|
||||
match = re.match(r"(\w+)\((.+)\)", text)
|
||||
if not match:
|
||||
return text, []
|
||||
predicate = match.group(1)
|
||||
args = [arg.strip() for arg in match.group(2).split(",")]
|
||||
return predicate, args
|
||||
|
||||
def _unify(self, condition: str, fact: str, bindings: Dict[str, str]) -> Optional[Dict[str, str]]:
|
||||
"""
|
||||
Try to unify a condition (with vars) against a fact.
|
||||
Returns new bindings if successful, None otherwise.
|
||||
"""
|
||||
if condition == fact:
|
||||
return bindings
|
||||
|
||||
cond_pred, cond_args = self._parse_predicate(condition)
|
||||
fact_pred, fact_args = self._parse_predicate(fact)
|
||||
|
||||
if cond_pred != fact_pred:
|
||||
return None
|
||||
if len(cond_args) != len(fact_args):
|
||||
return None
|
||||
|
||||
new_bindings = bindings.copy()
|
||||
for c_arg, f_arg in zip(cond_args, fact_args):
|
||||
if c_arg.startswith("?"):
|
||||
if c_arg in new_bindings:
|
||||
if new_bindings[c_arg] != f_arg:
|
||||
return None # Conflict
|
||||
else:
|
||||
new_bindings[c_arg] = f_arg
|
||||
else:
|
||||
if c_arg != f_arg:
|
||||
return None # Constant mismatch
|
||||
return new_bindings
|
||||
|
||||
def _calculate_coverage(self, rule: Rule, observation: Observation) -> float:
|
||||
"""Calculate how well rule covers observation."""
|
||||
|
||||
@@ -32,6 +32,7 @@ License: MIT
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
import re
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -148,12 +149,16 @@ class DeductiveReasoner:
|
||||
rules = self.rule_manager.get_all_rules()
|
||||
|
||||
for rule in rules:
|
||||
# Check if rule can be applied
|
||||
if self._can_apply_rule(rule, premises):
|
||||
conclusion = self._apply_rule_to_premises(rule, premises)
|
||||
# Find all matches (bindings) for the rule
|
||||
matches = self._find_matches(rule.conditions, {})
|
||||
|
||||
for bindings in matches:
|
||||
conclusion = self._apply_rule_to_premises(rule, premises, bindings)
|
||||
if conclusion:
|
||||
conclusions.append(conclusion)
|
||||
self.known_facts.add(conclusion.statement)
|
||||
# Check if conclusion is new (not in known facts)
|
||||
if conclusion.statement not in self.known_facts:
|
||||
conclusions.append(conclusion)
|
||||
self.known_facts.add(conclusion.statement)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
@@ -168,39 +173,127 @@ class DeductiveReasoner:
|
||||
)
|
||||
raise
|
||||
|
||||
def _can_apply_rule(self, rule: Rule, premises: List[Premise]) -> bool:
|
||||
"""Check if rule can be applied to premises."""
|
||||
# Check if all rule conditions match premises
|
||||
premise_statements = {p.statement for p in premises}
|
||||
def _parse_predicate(self, text: str) -> tuple[str, List[str]]:
|
||||
"""Parse 'Predicate(arg1, arg2)' into ('Predicate', ['arg1', 'arg2'])."""
|
||||
if not isinstance(text, str):
|
||||
return text, []
|
||||
match = re.match(r"(\w+)\((.+)\)", text)
|
||||
if not match:
|
||||
return text, []
|
||||
predicate = match.group(1)
|
||||
args = [arg.strip() for arg in match.group(2).split(",")]
|
||||
return predicate, args
|
||||
|
||||
for condition in rule.conditions:
|
||||
if (
|
||||
condition not in premise_statements
|
||||
and condition not in self.known_facts
|
||||
):
|
||||
return False
|
||||
def _unify(self, condition: str, fact: str, bindings: Dict[str, str]) -> Optional[Dict[str, str]]:
|
||||
"""
|
||||
Try to unify a condition (with vars) against a fact.
|
||||
Returns new bindings if successful, None otherwise.
|
||||
"""
|
||||
if condition == fact:
|
||||
return bindings
|
||||
|
||||
cond_pred, cond_args = self._parse_predicate(condition)
|
||||
fact_pred, fact_args = self._parse_predicate(fact)
|
||||
|
||||
if cond_pred != fact_pred:
|
||||
return None
|
||||
if len(cond_args) != len(fact_args):
|
||||
return None
|
||||
|
||||
new_bindings = bindings.copy()
|
||||
for c_arg, f_arg in zip(cond_args, fact_args):
|
||||
if c_arg.startswith("?"):
|
||||
if c_arg in new_bindings:
|
||||
if new_bindings[c_arg] != f_arg:
|
||||
return None # Conflict
|
||||
else:
|
||||
new_bindings[c_arg] = f_arg
|
||||
else:
|
||||
if c_arg != f_arg:
|
||||
return None # Constant mismatch
|
||||
return new_bindings
|
||||
|
||||
return True
|
||||
def _substitute_bindings(self, text: str, bindings: Dict[str, str]) -> str:
|
||||
"""Substitute variables in text with bindings."""
|
||||
if not isinstance(text, str):
|
||||
return text
|
||||
pred, args = self._parse_predicate(text)
|
||||
if not args:
|
||||
return text
|
||||
|
||||
new_args = []
|
||||
for arg in args:
|
||||
if arg in bindings:
|
||||
new_args.append(bindings[arg])
|
||||
else:
|
||||
new_args.append(arg)
|
||||
|
||||
return f"{pred}({', '.join(new_args)})"
|
||||
|
||||
def _find_matches(self, conditions: List[str], bindings: Dict[str, str]) -> List[Dict[str, str]]:
|
||||
"""
|
||||
Recursively find all bindings that satisfy the conditions.
|
||||
"""
|
||||
if not conditions:
|
||||
return [bindings]
|
||||
|
||||
first = conditions[0]
|
||||
# Substitute current bindings into first condition before matching
|
||||
first_substituted = self._substitute_bindings(first, bindings)
|
||||
rest = conditions[1:]
|
||||
|
||||
valid_bindings = []
|
||||
|
||||
# Try to match 'first' against all known facts
|
||||
for fact in self.known_facts:
|
||||
# Skip if fact is not a string (unhashable/objects) for now
|
||||
if not isinstance(fact, str):
|
||||
continue
|
||||
|
||||
unified = self._unify(first_substituted, fact, bindings)
|
||||
if unified is not None:
|
||||
# Recursive step
|
||||
results = self._find_matches(rest, unified)
|
||||
valid_bindings.extend(results)
|
||||
|
||||
return valid_bindings
|
||||
|
||||
def _apply_rule_to_premises(
|
||||
self, rule: Rule, premises: List[Premise]
|
||||
self, rule: Rule, premises: List[Premise], bindings: Dict[str, str]
|
||||
) -> Optional[Conclusion]:
|
||||
"""Apply rule to premises and generate conclusion."""
|
||||
# Find matching premises
|
||||
matching_premises = [
|
||||
p
|
||||
for p in premises
|
||||
if p.statement in rule.conditions or p.statement in self.known_facts
|
||||
]
|
||||
|
||||
# Find matching premises (those that support the bindings)
|
||||
# This is a bit approximate, ideally we track which premise supported which condition
|
||||
matching_premises = []
|
||||
|
||||
# Instantiate conclusion
|
||||
conclusion_stmt = rule.conclusion
|
||||
if bindings:
|
||||
conclusion_stmt = self._substitute_bindings(conclusion_stmt, bindings)
|
||||
|
||||
# Find premises that match the conditions (instantiated)
|
||||
for cond in rule.conditions:
|
||||
instantiated = self._substitute_bindings(cond, bindings)
|
||||
for p in premises:
|
||||
if p.statement == instantiated:
|
||||
matching_premises.append(p)
|
||||
break
|
||||
# Note: some conditions might be matched by self.known_facts which are not in 'premises' arg
|
||||
# but are in self.known_facts.
|
||||
# If a premise is not in the passed list but in known_facts, we can't add it to matching_premises list
|
||||
# unless we find the Premise object.
|
||||
# But known_facts stores strings.
|
||||
# So matching_premises might be incomplete if we rely on known_facts.
|
||||
# However, for this method signature, we return a Conclusion with premises.
|
||||
|
||||
conclusion = Conclusion(
|
||||
conclusion_id=f"conc_{len(matching_premises)}",
|
||||
statement=rule.conclusion,
|
||||
conclusion_id=f"conc_{rule.name}_{len(matching_premises)}",
|
||||
statement=conclusion_stmt,
|
||||
premises=matching_premises,
|
||||
rule_applied=rule,
|
||||
confidence=rule.confidence,
|
||||
proof_steps=[f"Applied rule: {rule.name}"],
|
||||
metadata={"rule_id": rule.rule_id},
|
||||
proof_steps=[f"Applied rule: {rule.name} with bindings {bindings}"],
|
||||
metadata={"rule_id": rule.rule_id, "bindings": bindings},
|
||||
)
|
||||
|
||||
return conclusion
|
||||
@@ -272,6 +365,7 @@ class DeductiveReasoner:
|
||||
return None
|
||||
|
||||
# Check if goal is already known
|
||||
# Try direct match
|
||||
if goal in self.known_facts:
|
||||
return Conclusion(
|
||||
conclusion_id=f"known_{goal}",
|
||||
@@ -279,35 +373,65 @@ class DeductiveReasoner:
|
||||
confidence=1.0,
|
||||
proof_steps=["Known fact"],
|
||||
)
|
||||
|
||||
# Try unification with known facts
|
||||
if isinstance(goal, str) and "?" in goal:
|
||||
for fact in self.known_facts:
|
||||
if isinstance(fact, str):
|
||||
if self._unify(goal, fact, {}) is not None:
|
||||
return Conclusion(
|
||||
conclusion_id=f"known_{fact}",
|
||||
statement=fact,
|
||||
confidence=1.0,
|
||||
proof_steps=[f"Known fact (matched pattern {goal})"],
|
||||
)
|
||||
|
||||
# Find rules that can prove goal
|
||||
rules = self.rule_manager.get_all_rules()
|
||||
applicable_rules = [r for r in rules if r.conclusion == goal]
|
||||
|
||||
# Use unified matching for finding applicable rules
|
||||
applicable_rules_and_bindings = []
|
||||
for r in rules:
|
||||
bindings = self._unify(r.conclusion, goal, {})
|
||||
if bindings is not None:
|
||||
applicable_rules_and_bindings.append((r, bindings))
|
||||
|
||||
for rule in applicable_rules:
|
||||
for rule, initial_bindings in applicable_rules_and_bindings:
|
||||
# Try to prove all premises
|
||||
premise_conclusions = []
|
||||
all_proven = True
|
||||
current_bindings = initial_bindings.copy()
|
||||
|
||||
for condition in rule.conditions:
|
||||
# Instantiate condition with current bindings
|
||||
instantiated_cond = self._substitute_bindings(condition, current_bindings)
|
||||
|
||||
premise_conclusion = self._prove_backward(
|
||||
condition, proof, depth + 1, max_depth, **options
|
||||
instantiated_cond, proof, depth + 1, max_depth, **options
|
||||
)
|
||||
if premise_conclusion:
|
||||
premise_conclusions.append(premise_conclusion)
|
||||
# Update bindings if we proved something more specific
|
||||
new_bindings = self._unify(instantiated_cond, premise_conclusion.statement, current_bindings)
|
||||
if new_bindings:
|
||||
current_bindings = new_bindings
|
||||
else:
|
||||
all_proven = False
|
||||
break
|
||||
|
||||
if all_proven:
|
||||
# All premises proven, rule can fire
|
||||
# Instantiate conclusion with final bindings
|
||||
final_conclusion = self._substitute_bindings(rule.conclusion, current_bindings)
|
||||
|
||||
conclusion = Conclusion(
|
||||
conclusion_id=f"conc_{goal}",
|
||||
statement=goal,
|
||||
premises=[Premise(p, p) for p in rule.conditions],
|
||||
statement=final_conclusion,
|
||||
premises=[p for p in premise_conclusions], # Use actual premises found
|
||||
rule_applied=rule,
|
||||
confidence=rule.confidence,
|
||||
proof_steps=[f"Proved using rule: {rule.name}"],
|
||||
proof_steps=[f"Proved using rule: {rule.name} with bindings {current_bindings}"],
|
||||
metadata={"rule_id": rule.rule_id, "bindings": current_bindings}
|
||||
)
|
||||
return conclusion
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Callable, Dict, List, Optional, Set
|
||||
@@ -92,6 +93,7 @@ class InferenceEngine:
|
||||
self.max_iterations = self.config.get("max_iterations", 100)
|
||||
|
||||
self.facts: Set[Any] = set()
|
||||
self.unhashable_facts: List[Any] = []
|
||||
self.inferred_facts: List[InferenceResult] = []
|
||||
|
||||
def add_rule(self, rule_definition: str, **options) -> Rule:
|
||||
@@ -115,15 +117,28 @@ class InferenceEngine:
|
||||
|
||||
return rule
|
||||
|
||||
def add_fact(self, fact: Any) -> None:
|
||||
def add_fact(self, fact: Any) -> bool:
|
||||
"""
|
||||
Add fact to knowledge base.
|
||||
|
||||
Args:
|
||||
fact: Fact to add
|
||||
|
||||
Returns:
|
||||
True if fact was newly added, False if it already existed
|
||||
"""
|
||||
self.facts.add(fact)
|
||||
self.logger.debug(f"Added fact: {fact}")
|
||||
try:
|
||||
if fact in self.facts:
|
||||
return False
|
||||
self.facts.add(fact)
|
||||
self.logger.debug(f"Added fact: {fact}")
|
||||
return True
|
||||
except TypeError:
|
||||
if fact not in self.unhashable_facts:
|
||||
self.unhashable_facts.append(fact)
|
||||
self.logger.debug(f"Added unhashable fact: {fact}")
|
||||
return True
|
||||
return False
|
||||
|
||||
def add_facts(self, facts: List[Any]) -> None:
|
||||
"""
|
||||
@@ -180,15 +195,18 @@ class InferenceEngine:
|
||||
)
|
||||
|
||||
for rule in rules:
|
||||
# Check if rule can fire
|
||||
if self._can_rule_fire(rule):
|
||||
# Apply rule
|
||||
result = self._apply_rule(rule)
|
||||
# Find all matches for the rule
|
||||
matches = self._find_matches(rule.conditions, {})
|
||||
|
||||
for bindings in matches:
|
||||
# Apply rule with bindings
|
||||
result = self._apply_rule(rule, bindings=bindings)
|
||||
if result:
|
||||
results.append(result)
|
||||
self.inferred_facts.append(result)
|
||||
self.add_fact(result.conclusion)
|
||||
new_facts = True
|
||||
# Only consider it a new inference if the fact wasn't already known
|
||||
if self.add_fact(result.conclusion):
|
||||
results.append(result)
|
||||
self.inferred_facts.append(result)
|
||||
new_facts = True
|
||||
|
||||
self.logger.info(
|
||||
f"Forward chaining completed: {len(results)} inferences in {iterations} iterations"
|
||||
@@ -228,39 +246,84 @@ class InferenceEngine:
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Checking if goal is already a fact..."
|
||||
)
|
||||
if goal in self.facts:
|
||||
|
||||
# Check for direct match or unification with facts
|
||||
found_fact = None
|
||||
|
||||
# First try direct match (fastest)
|
||||
try:
|
||||
if goal in self.facts:
|
||||
found_fact = goal
|
||||
except TypeError:
|
||||
if goal in self.unhashable_facts:
|
||||
found_fact = goal
|
||||
|
||||
# If not found and goal looks like a pattern (string with ?), try unification
|
||||
if found_fact is None and isinstance(goal, str) and "?" in goal:
|
||||
for fact in self.facts:
|
||||
if isinstance(fact, str):
|
||||
# Try to unify to see if it matches
|
||||
if self._unify(goal, fact, {}) is not None:
|
||||
found_fact = fact
|
||||
break
|
||||
|
||||
if found_fact:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="completed", message="Goal is already a fact"
|
||||
tracking_id, status="completed", message=f"Goal proven by fact: {found_fact}"
|
||||
)
|
||||
return InferenceResult(conclusion=goal, confidence=1.0)
|
||||
return InferenceResult(conclusion=found_fact, confidence=1.0)
|
||||
|
||||
# Find rules that can prove the goal
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Finding rules that can prove the goal..."
|
||||
)
|
||||
rules = self.rule_manager.get_all_rules()
|
||||
applicable_rules = [r for r in rules if self._rule_concludes(r, goal)]
|
||||
|
||||
# Use unified matching for finding applicable rules
|
||||
applicable_rules_and_bindings = []
|
||||
for r in rules:
|
||||
bindings = self._unify(r.conclusion, goal, {})
|
||||
if bindings is not None:
|
||||
applicable_rules_and_bindings.append((r, bindings))
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id,
|
||||
message=f"Found {len(applicable_rules)} applicable rules, trying to prove premises...",
|
||||
message=f"Found {len(applicable_rules_and_bindings)} applicable rules, trying to prove premises...",
|
||||
)
|
||||
for rule in applicable_rules:
|
||||
# Try to prove premises
|
||||
premises = []
|
||||
|
||||
for rule, initial_bindings in applicable_rules_and_bindings:
|
||||
# Try to prove premises with bindings, propagating bindings between premises
|
||||
current_bindings = initial_bindings.copy()
|
||||
premises_results = []
|
||||
all_premises_proven = True
|
||||
|
||||
for premise in rule.conditions:
|
||||
premise_result = self.backward_chain(premise, **options)
|
||||
|
||||
for cond in rule.conditions:
|
||||
# Instantiate condition with current bindings
|
||||
instantiated_cond = self._substitute_bindings(cond, current_bindings)
|
||||
|
||||
# Recursively prove this condition
|
||||
premise_result = self.backward_chain(instantiated_cond, **options)
|
||||
|
||||
if premise_result:
|
||||
premises.append(premise_result.conclusion)
|
||||
premises_results.append(premise_result.conclusion)
|
||||
|
||||
# If the premise had variables, update bindings based on the proven fact
|
||||
# We unify the instantiated condition (which might still have vars) with the proven conclusion
|
||||
new_bindings = self._unify(instantiated_cond, premise_result.conclusion, current_bindings)
|
||||
if new_bindings is not None:
|
||||
current_bindings = new_bindings
|
||||
else:
|
||||
# This implies a conflict, which shouldn't happen if backward_chain returned success
|
||||
# on instantiated_cond, but good to be safe
|
||||
all_premises_proven = False
|
||||
break
|
||||
else:
|
||||
all_premises_proven = False
|
||||
break
|
||||
|
||||
if all_premises_proven:
|
||||
# All premises proven, rule can fire
|
||||
result = self._apply_rule(rule, premises)
|
||||
result = self._apply_rule(rule, premises=premises_results, bindings=current_bindings)
|
||||
if result:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
@@ -280,34 +343,115 @@ class InferenceEngine:
|
||||
)
|
||||
raise
|
||||
|
||||
def _can_rule_fire(self, rule: Rule) -> bool:
|
||||
"""Check if rule can fire (all conditions met)."""
|
||||
for condition in rule.conditions:
|
||||
if condition not in self.facts:
|
||||
return False
|
||||
return True
|
||||
def _parse_predicate(self, text: str) -> tuple[str, List[str]]:
|
||||
"""Parse 'Predicate(arg1, arg2)' into ('Predicate', ['arg1', 'arg2'])."""
|
||||
if not isinstance(text, str):
|
||||
return text, []
|
||||
match = re.match(r"(\w+)\((.+)\)", text)
|
||||
if not match:
|
||||
return text, []
|
||||
predicate = match.group(1)
|
||||
args = [arg.strip() for arg in match.group(2).split(",")]
|
||||
return predicate, args
|
||||
|
||||
def _rule_concludes(self, rule: Rule, goal: Any) -> bool:
|
||||
"""Check if rule concludes the goal."""
|
||||
return rule.conclusion == goal
|
||||
def _unify(self, condition: str, fact: str, bindings: Dict[str, str]) -> Optional[Dict[str, str]]:
|
||||
"""
|
||||
Try to unify a condition (with vars) against a fact.
|
||||
Returns new bindings if successful, None otherwise.
|
||||
"""
|
||||
# Handle exact string match shortcut
|
||||
if condition == fact:
|
||||
return bindings
|
||||
|
||||
cond_pred, cond_args = self._parse_predicate(condition)
|
||||
fact_pred, fact_args = self._parse_predicate(fact)
|
||||
|
||||
if cond_pred != fact_pred:
|
||||
return None
|
||||
if len(cond_args) != len(fact_args):
|
||||
return None
|
||||
|
||||
new_bindings = bindings.copy()
|
||||
for c_arg, f_arg in zip(cond_args, fact_args):
|
||||
if c_arg.startswith("?"):
|
||||
if c_arg in new_bindings:
|
||||
if new_bindings[c_arg] != f_arg:
|
||||
return None # Conflict
|
||||
else:
|
||||
new_bindings[c_arg] = f_arg
|
||||
else:
|
||||
if c_arg != f_arg:
|
||||
return None # Constant mismatch
|
||||
return new_bindings
|
||||
|
||||
def _find_matches(self, conditions: List[str], bindings: Dict[str, str]) -> List[Dict[str, str]]:
|
||||
"""
|
||||
Recursively find all bindings that satisfy the conditions.
|
||||
"""
|
||||
if not conditions:
|
||||
return [bindings]
|
||||
|
||||
first = conditions[0]
|
||||
# Substitute current bindings into first condition before matching
|
||||
first_substituted = self._substitute_bindings(first, bindings)
|
||||
rest = conditions[1:]
|
||||
|
||||
valid_bindings = []
|
||||
|
||||
# Try to match 'first' against all facts
|
||||
for fact in self.facts:
|
||||
# Skip if fact is not a string (unhashable/objects) for now, or handle str()
|
||||
if not isinstance(fact, str):
|
||||
continue
|
||||
|
||||
unified = self._unify(first_substituted, fact, bindings)
|
||||
if unified is not None:
|
||||
# Recursive step
|
||||
results = self._find_matches(rest, unified)
|
||||
valid_bindings.extend(results)
|
||||
|
||||
return valid_bindings
|
||||
|
||||
def _substitute_bindings(self, text: str, bindings: Dict[str, str]) -> str:
|
||||
"""Substitute variables in text with bindings."""
|
||||
if not isinstance(text, str):
|
||||
return text
|
||||
pred, args = self._parse_predicate(text)
|
||||
if not args:
|
||||
return text
|
||||
|
||||
new_args = []
|
||||
for arg in args:
|
||||
if arg in bindings:
|
||||
new_args.append(bindings[arg])
|
||||
else:
|
||||
new_args.append(arg)
|
||||
|
||||
return f"{pred}({', '.join(new_args)})"
|
||||
|
||||
def _apply_rule(
|
||||
self, rule: Rule, premises: Optional[List[Any]] = None
|
||||
self, rule: Rule, premises: Optional[List[Any]] = None, bindings: Optional[Dict[str, str]] = None
|
||||
) -> Optional[InferenceResult]:
|
||||
"""Apply rule and return inference result."""
|
||||
conclusion = rule.conclusion
|
||||
if bindings:
|
||||
conclusion = self._substitute_bindings(conclusion, bindings)
|
||||
|
||||
if premises is None:
|
||||
premises = list(rule.conditions)
|
||||
# Reconstruct premises from bindings if not provided (approximate)
|
||||
premises = [self._substitute_bindings(c, bindings or {}) for c in rule.conditions]
|
||||
|
||||
result = InferenceResult(
|
||||
conclusion=rule.conclusion,
|
||||
conclusion=conclusion,
|
||||
premises=premises,
|
||||
rule_used=rule,
|
||||
confidence=rule.confidence,
|
||||
metadata={"rule_name": rule.name, "rule_id": rule.rule_id},
|
||||
metadata={"rule_name": rule.name, "rule_id": rule.rule_id, "bindings": bindings},
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def infer(self, query: Any, **options) -> List[InferenceResult]:
|
||||
"""
|
||||
Perform inference based on strategy.
|
||||
@@ -365,9 +509,9 @@ class InferenceEngine:
|
||||
)
|
||||
raise
|
||||
|
||||
def get_facts(self) -> Set[Any]:
|
||||
def get_facts(self) -> List[Any]:
|
||||
"""Get all facts."""
|
||||
return set(self.facts)
|
||||
return list(self.facts) + self.unhashable_facts
|
||||
|
||||
def get_inferred_facts(self) -> List[InferenceResult]:
|
||||
"""Get all inferred facts."""
|
||||
@@ -376,6 +520,7 @@ class InferenceEngine:
|
||||
def clear_facts(self) -> None:
|
||||
"""Clear all facts."""
|
||||
self.facts.clear()
|
||||
self.unhashable_facts.clear()
|
||||
self.inferred_facts.clear()
|
||||
|
||||
def reset(self) -> None:
|
||||
|
||||
@@ -45,7 +45,7 @@ print(f"Inferred {len(results)} new facts")
|
||||
from semantica.reasoning import SPARQLReasoner
|
||||
|
||||
# Create SPARQL reasoner
|
||||
reasoner = SPARQLReasoner(triple_store=kg)
|
||||
reasoner = SPARQLReasoner(triplet_store=kg)
|
||||
|
||||
# Execute query
|
||||
query = "SELECT ?s ?p ?o WHERE { ?s ?p ?o }"
|
||||
@@ -190,7 +190,7 @@ results = engine.forward_chain()
|
||||
from semantica.reasoning import SPARQLReasoner
|
||||
|
||||
# Create reasoner with knowledge graph
|
||||
reasoner = SPARQLReasoner(triple_store=kg)
|
||||
reasoner = SPARQLReasoner(triplet_store=kg)
|
||||
|
||||
# Execute SPARQL query
|
||||
query = """
|
||||
@@ -212,7 +212,7 @@ for binding in result.bindings:
|
||||
```python
|
||||
from semantica.reasoning import SPARQLReasoner
|
||||
|
||||
reasoner = SPARQLReasoner(triple_store=kg, enable_inference=True)
|
||||
reasoner = SPARQLReasoner(triplet_store=kg, enable_inference=True)
|
||||
|
||||
# Add inference rule
|
||||
reasoner.add_inference_rule("IF ?x :type :Company THEN ?x :type :Organization")
|
||||
@@ -234,7 +234,7 @@ result = reasoner.execute_query(query)
|
||||
from semantica.reasoning import SPARQLReasoner
|
||||
|
||||
reasoner = SPARQLReasoner(
|
||||
triple_store=kg,
|
||||
triplet_store=kg,
|
||||
enable_inference=True,
|
||||
inference_rules=["rdfs:subClassOf", "rdfs:subPropertyOf"]
|
||||
)
|
||||
@@ -1165,7 +1165,7 @@ engine = InferenceEngine(
|
||||
|
||||
# Configure SPARQL reasoner
|
||||
reasoner = SPARQLReasoner(
|
||||
triple_store=kg,
|
||||
triplet_store=kg,
|
||||
enable_inference=True,
|
||||
query_cache_size=1000
|
||||
)
|
||||
@@ -1234,7 +1234,7 @@ for result in results:
|
||||
print(f"Explanation: {explanation.natural_language}")
|
||||
|
||||
# 6. Query with SPARQL reasoning
|
||||
sparql_reasoner = SPARQLReasoner(triple_store=kg, enable_inference=True)
|
||||
sparql_reasoner = SPARQLReasoner(triplet_store=kg, enable_inference=True)
|
||||
query_result = sparql_reasoner.execute_query("SELECT ?x WHERE { ?x :type :Employee }")
|
||||
```
|
||||
|
||||
@@ -1332,7 +1332,7 @@ from semantica.kg import build
|
||||
kg = build(sources=[...])
|
||||
|
||||
# Create SPARQL reasoner with KG
|
||||
reasoner = SPARQLReasoner(triple_store=kg, enable_inference=True)
|
||||
reasoner = SPARQLReasoner(triplet_store=kg, enable_inference=True)
|
||||
|
||||
# Add inference rules
|
||||
reasoner.add_inference_rule("IF ?x :type :Company THEN ?x :type :Organization")
|
||||
|
||||
@@ -12,7 +12,7 @@ Key Features:
|
||||
- Query expansion
|
||||
- Performance optimization
|
||||
- Error handling and recovery
|
||||
- Triple store integration
|
||||
- Triplet store integration
|
||||
|
||||
Main Classes:
|
||||
- SPARQLReasoner: SPARQL-based reasoning engine
|
||||
@@ -67,7 +67,7 @@ class SPARQLReasoner:
|
||||
Args:
|
||||
config: Configuration dictionary
|
||||
**kwargs: Additional configuration options:
|
||||
- triple_store: Triple store connection
|
||||
- triplet_store: Triplet store connection
|
||||
- enable_inference: Enable inference rules
|
||||
"""
|
||||
self.logger = get_logger("sparql_reasoner")
|
||||
@@ -78,7 +78,7 @@ class SPARQLReasoner:
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.rule_manager = RuleManager(**self.config)
|
||||
self.triple_store = self.config.get("triple_store")
|
||||
self.triplet_store = self.config.get("triplet_store")
|
||||
self.enable_inference = self.config.get("enable_inference", True)
|
||||
|
||||
self.query_cache: Dict[str, Any] = {}
|
||||
@@ -356,12 +356,12 @@ class SPARQLReasoner:
|
||||
)
|
||||
expanded_query = self.expand_query(query, **options)
|
||||
|
||||
# Execute query (if triple store available)
|
||||
# Execute query (if triplet store available)
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Executing query..."
|
||||
)
|
||||
if self.triple_store:
|
||||
# This would call the triple store's query method
|
||||
if self.triplet_store:
|
||||
# This would call the triplet store's query method
|
||||
# For now, return empty result
|
||||
result = SPARQLQueryResult(bindings=[], variables=[])
|
||||
else:
|
||||
|
||||
@@ -194,18 +194,21 @@ class SeedDataManager:
|
||||
entity_type: Optional[str] = None,
|
||||
relationship_type: Optional[str] = None,
|
||||
source_name: Optional[str] = None,
|
||||
delimiter: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Load seed data from CSV file.
|
||||
|
||||
Reads a CSV file and converts rows to dictionaries. Automatically
|
||||
adds entity_type, relationship_type, and source metadata if provided.
|
||||
Supports automatic delimiter detection if not provided.
|
||||
|
||||
Args:
|
||||
file_path: Path to CSV file
|
||||
entity_type: Optional entity type to add to all records
|
||||
relationship_type: Optional relationship type to add to all records
|
||||
source_name: Optional source name for tracking
|
||||
delimiter: Optional CSV delimiter. If None, attempts to detect it.
|
||||
|
||||
Returns:
|
||||
List of loaded data records as dictionaries
|
||||
@@ -215,7 +218,7 @@ class SeedDataManager:
|
||||
|
||||
Example:
|
||||
>>> records = manager.load_from_csv("data/entities.csv", entity_type="Person")
|
||||
>>> print(f"Loaded {len(records)} records")
|
||||
>>> records = manager.load_from_csv("data/data.csv", delimiter=";")
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="seed",
|
||||
@@ -239,7 +242,21 @@ class SeedDataManager:
|
||||
tracking_id, message="Reading CSV file..."
|
||||
)
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
reader = csv.DictReader(f)
|
||||
# Detect delimiter if not provided
|
||||
if delimiter is None:
|
||||
try:
|
||||
sample = f.read(1024)
|
||||
f.seek(0)
|
||||
dialect = csv.Sniffer().sniff(sample)
|
||||
delimiter = dialect.delimiter
|
||||
self.logger.debug(f"Detected CSV delimiter: '{delimiter}'")
|
||||
except csv.Error:
|
||||
# Fallback to comma if sniffing fails
|
||||
f.seek(0)
|
||||
delimiter = ","
|
||||
self.logger.debug("Could not detect delimiter, defaulting to ','")
|
||||
|
||||
reader = csv.DictReader(f, delimiter=delimiter)
|
||||
for row in reader:
|
||||
# Clean up row data
|
||||
record = {k: v for k, v in row.items() if v}
|
||||
@@ -316,6 +333,11 @@ class SeedDataManager:
|
||||
elif "records" in data:
|
||||
records = data["records"]
|
||||
else:
|
||||
self.logger.warning(
|
||||
f"JSON file {file_path} is a dictionary but contains none of the "
|
||||
"expected keys: 'entities', 'data', 'records'. "
|
||||
"Treating entire object as a single record."
|
||||
)
|
||||
records = [data]
|
||||
else:
|
||||
records = []
|
||||
|
||||
@@ -48,6 +48,17 @@ records = manager.load_from_csv(
|
||||
entity_type="Person"
|
||||
)
|
||||
|
||||
# Load from CSV with custom delimiter
|
||||
records_pipe = manager.load_from_csv(
|
||||
"data/entities_pipe.csv",
|
||||
delimiter="|"
|
||||
)
|
||||
|
||||
# Load from CSV with auto-detection (supported for common delimiters like ;, \t, etc.)
|
||||
records_auto = manager.load_from_csv(
|
||||
"data/entities_semicolon.csv"
|
||||
)
|
||||
|
||||
print(f"Loaded {len(records)} records from CSV")
|
||||
|
||||
# CSV should have columns like: id, name, type, etc.
|
||||
@@ -74,6 +85,10 @@ print(f"Loaded {len(records)} records from JSON")
|
||||
# - List: [{"id": "1", "name": "John"}, ...]
|
||||
# - Dict with 'entities': {"entities": [...]}
|
||||
# - Dict with 'data': {"data": [...]}
|
||||
# - Dict with 'records': {"records": [...]}
|
||||
#
|
||||
# Note: Ensure JSON seed files follow these supported top-level structures.
|
||||
# Unsupported structures will trigger a warning and may be loaded as a single record.
|
||||
```
|
||||
|
||||
### Loading from Database
|
||||
@@ -553,11 +568,15 @@ manager.export_seed_data("output/custom_seed.json", format="json")
|
||||
**Algorithm**: Row-by-row CSV processing with metadata injection
|
||||
|
||||
1. **File Reading**: Open CSV file with UTF-8 encoding
|
||||
2. **Header Detection**: Use csv.DictReader() for automatic header detection
|
||||
3. **Row Processing**: Iterate through rows, convert to dictionaries
|
||||
4. **Data Cleaning**: Remove empty values, clean whitespace
|
||||
5. **Metadata Injection**: Add entity_type, relationship_type, source metadata
|
||||
6. **Type Conversion**: Convert string values to appropriate types
|
||||
2. **Delimiter Detection**:
|
||||
- Use provided delimiter if specified
|
||||
- If not, attempt to auto-detect delimiter using `csv.Sniffer`
|
||||
- Fallback to comma (`,`) if detection fails
|
||||
3. **Header Detection**: Use csv.DictReader() for automatic header detection
|
||||
4. **Row Processing**: Iterate through rows, convert to dictionaries
|
||||
5. **Data Cleaning**: Remove empty values, clean whitespace
|
||||
6. **Metadata Injection**: Add entity_type, relationship_type, source metadata
|
||||
7. **Type Conversion**: Convert string values to appropriate types
|
||||
|
||||
**Time Complexity**: O(n) where n = number of rows
|
||||
**Space Complexity**: O(n) for records storage
|
||||
|
||||
@@ -177,7 +177,6 @@ class CoreferenceResolver:
|
||||
)
|
||||
raise
|
||||
|
||||
<<<<<<< HEAD
|
||||
def resolve(self, text: str, **options) -> List[CoreferenceChain]:
|
||||
"""
|
||||
Resolve coreferences in text (alias for resolve_coreferences).
|
||||
@@ -190,9 +189,6 @@ class CoreferenceResolver:
|
||||
list: List of coreference chains
|
||||
"""
|
||||
return self.resolve_coreferences(text, **options)
|
||||
|
||||
=======
|
||||
>>>>>>> origin/main
|
||||
def _extract_mentions(self, text: str) -> List[Mention]:
|
||||
"""Extract all mentions from text."""
|
||||
mentions = []
|
||||
|
||||
@@ -85,7 +85,6 @@ class Event:
|
||||
class EventDetector:
|
||||
"""Event detection and extraction handler."""
|
||||
|
||||
<<<<<<< HEAD
|
||||
def __init__(
|
||||
self,
|
||||
event_types: Optional[List[str]] = None,
|
||||
@@ -96,9 +95,6 @@ class EventDetector:
|
||||
config=None,
|
||||
**kwargs
|
||||
):
|
||||
=======
|
||||
def __init__(self, method: Union[str, List[str]] = None, config=None, **kwargs):
|
||||
>>>>>>> origin/main
|
||||
"""
|
||||
Initialize event detector.
|
||||
|
||||
@@ -120,15 +116,12 @@ class EventDetector:
|
||||
self.config.update(kwargs)
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
<<<<<<< HEAD
|
||||
# Store parameters
|
||||
self.event_types_filter = event_types
|
||||
self.extract_participants = extract_participants
|
||||
self.extract_location = extract_location
|
||||
self.extract_time = extract_time
|
||||
|
||||
=======
|
||||
>>>>>>> origin/main
|
||||
# Store method for passing to extractors if needed
|
||||
if method is not None:
|
||||
self.config["ner_method"] = method
|
||||
@@ -171,7 +164,6 @@ class EventDetector:
|
||||
try:
|
||||
events = []
|
||||
|
||||
<<<<<<< HEAD
|
||||
# Determine which event types to detect
|
||||
event_patterns_to_use = self.event_patterns
|
||||
if self.event_types_filter:
|
||||
@@ -180,24 +172,17 @@ class EventDetector:
|
||||
if k in self.event_types_filter
|
||||
}
|
||||
|
||||
=======
|
||||
>>>>>>> origin/main
|
||||
# Detect events using patterns
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Scanning text for event patterns..."
|
||||
)
|
||||
<<<<<<< HEAD
|
||||
for event_type, pattern in event_patterns_to_use.items():
|
||||
=======
|
||||
for event_type, pattern in self.event_patterns.items():
|
||||
>>>>>>> origin/main
|
||||
for match in re.finditer(pattern, text, re.IGNORECASE):
|
||||
# Extract surrounding context
|
||||
start = max(0, match.start() - 50)
|
||||
end = min(len(text), match.end() + 50)
|
||||
context = text[start:end]
|
||||
|
||||
<<<<<<< HEAD
|
||||
# Extract participants if enabled
|
||||
participants = []
|
||||
if self.extract_participants:
|
||||
@@ -212,10 +197,6 @@ class EventDetector:
|
||||
time_info = None
|
||||
if self.extract_time:
|
||||
time_info = self._extract_time(context)
|
||||
=======
|
||||
# Extract participants (simplified)
|
||||
participants = self._extract_participants(context)
|
||||
>>>>>>> origin/main
|
||||
|
||||
event = Event(
|
||||
text=match.group(0),
|
||||
|
||||
@@ -311,6 +311,10 @@ def extract_entities_llm(
|
||||
text: str, provider: str = "openai", model: Optional[str] = None, **kwargs
|
||||
) -> List[Entity]:
|
||||
"""LLM-based entity extraction."""
|
||||
# Support llm_model parameter to disambiguate from ML model
|
||||
if "llm_model" in kwargs:
|
||||
model = kwargs.pop("llm_model")
|
||||
|
||||
llm = create_provider(provider, model=model, **kwargs)
|
||||
|
||||
if not llm.is_available():
|
||||
@@ -818,6 +822,7 @@ def get_entity_method(method_name: str):
|
||||
"regex": extract_entities_regex,
|
||||
"rules": extract_entities_rules,
|
||||
"ml": extract_entities_ml,
|
||||
"spacy": extract_entities_ml, # Alias for ml
|
||||
"huggingface": extract_entities_huggingface,
|
||||
"llm": extract_entities_llm,
|
||||
}
|
||||
@@ -844,6 +849,8 @@ def get_relation_method(method_name: str):
|
||||
"regex": extract_relations_regex,
|
||||
"cooccurrence": extract_relations_cooccurrence,
|
||||
"dependency": extract_relations_dependency,
|
||||
"ml": extract_relations_dependency, # Alias for dependency
|
||||
"spacy": extract_relations_dependency, # Alias for dependency
|
||||
"huggingface": extract_relations_huggingface,
|
||||
"llm": extract_relations_llm,
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user