mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-30 04:40:16 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a047ebf74f | ||
|
|
88c12b1867 | ||
|
|
094bb8d82b | ||
|
|
7ff2fd9981 | ||
|
|
0a555145e4 | ||
|
|
994e58a170 | ||
|
|
244144dee3 | ||
|
|
5dfca85500 | ||
|
|
f3dd7a05bd | ||
|
|
d03a237278 | ||
|
|
f3ac9fbffa | ||
|
|
6856580a7a | ||
|
|
4a282628ea | ||
|
|
c73e35a2fe | ||
|
|
a99f18b71b | ||
|
|
84b90b45a2 | ||
|
|
d7d589f64e | ||
|
|
d3366bbcf0 | ||
|
|
95c5486d22 |
+1
-1
@@ -58,7 +58,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Production-ready quality assurance modules
|
||||
- Comprehensive documentation with MkDocs
|
||||
- Cookbook with interactive tutorials
|
||||
- Support for multiple vector stores (Pinecone, Weaviate, Qdrant, FAISS)
|
||||
- Support for multiple vector stores (Weaviate, Qdrant, FAISS)
|
||||
- Support for multiple graph databases (Neo4j, NetworkX, RDFLib)
|
||||
- Temporal knowledge graph support
|
||||
- Conflict detection and resolution
|
||||
|
||||
@@ -319,7 +319,7 @@ result = kg.query("Who founded the company?", return_format="structured")
|
||||
print(f"Nodes: {kg.node_count}, Answer: {result.answer}")
|
||||
```
|
||||
|
||||
[**Cookbook: Building Knowledge Graphs**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb) • [**Graph Store**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/09_Graph_Store.ipynb) • [**Triple Store**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/20_Triple_Store.ipynb) • [**Visualization**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/16_Visualization.ipynb)
|
||||
[**Cookbook: Building Knowledge Graphs**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb) • [**Graph Store**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/09_Graph_Store.ipynb) • [**Triplet Store**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/20_Triplet_Store.ipynb) • [**Visualization**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/16_Visualization.ipynb)
|
||||
|
||||
[**Graph Analytics**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/10_Graph_Analytics.ipynb) • [**Advanced Graph Analytics**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/advanced/02_Advanced_Graph_Analytics.ipynb)
|
||||
|
||||
|
||||
@@ -312,7 +312,7 @@
|
||||
"- NumPy format\n",
|
||||
"- Binary format\n",
|
||||
"- FAISS format\n",
|
||||
"- Vector store integration (Pinecone, Weaviate, Qdrant)\n"
|
||||
"- Vector store integration (Weaviate, Qdrant)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Build an enterprise semantic layer: construct knowledge graph, generate ontology, create semantic layer, export RDF, and store in triple store.\n",
|
||||
"Build an enterprise semantic layer: construct knowledge graph, generate ontology, create semantic layer, export RDF, and store in triplet store.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"**Documentation**: [API Reference](https://semantica.readthedocs.io/concepts/)\n",
|
||||
@@ -25,7 +25,7 @@
|
||||
"pip install semantica[all]\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"## Workflow: Build KG → Generate Ontology → Create Semantic Layer → Export RDF → Triple Store\n"
|
||||
"## Workflow: Build KG → Generate Ontology → Create Semantic Layer → Export RDF → Triplet Store\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -37,7 +37,7 @@
|
||||
"from semantica.kg import GraphBuilder\n",
|
||||
"from semantica.ontology import OntologyGenerator\n",
|
||||
"from semantica.export import RDFExporter\n",
|
||||
"from semantica.triple_store import TripleStore\n"
|
||||
"from semantica.triplet_store import TripletStore\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -162,7 +162,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 5: Store in Triple Store\n"
|
||||
"## Step 5: Store in Triplet Store\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -171,8 +171,8 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"triple_store = TripleStore()\n",
|
||||
"triple_store.store(knowledge_graph, ontology)\n"
|
||||
"triplet_store = TripletStore()\n",
|
||||
"triplet_store.store(knowledge_graph, ontology)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -186,7 +186,7 @@
|
||||
"- Ontology Generated\n",
|
||||
"- Semantic Layer Created with Mappings\n",
|
||||
"- RDF Export Completed\n",
|
||||
"- Triple Store Storage Completed\n"
|
||||
"- Triplet Store Storage Completed\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -217,7 +217,7 @@
|
||||
"## 5. Best Practices for Production\n",
|
||||
"\n",
|
||||
"1. **Token Limits**: Align `token_limit` with your LLM's context window minus the prompt template size.\n",
|
||||
"2. **Vector Store**: Use a production-grade vector store (e.g., Pinecone, Weaviate, Qdrant) instead of the mock store.\n",
|
||||
"2. **Vector Store**: Use a production-grade vector store (e.g., Weaviate, Qdrant) instead of the mock store.\n",
|
||||
"3. **Asynchronous Operations**: For high-throughput systems, consider wrapping storage operations in async tasks (though the core logic is synchronous for simplicity).\n",
|
||||
"4. **Entity Resolution**: Implement a robust `EntityLinker` strategy to prevent graph fragmentation (e.g., \"Alice\" vs \"Alice S.\")."
|
||||
]
|
||||
|
||||
@@ -204,7 +204,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.semantic_extract.methods import get_entity_method\n",
|
||||
"from semantica.semantic_extract import NERExtractor\n",
|
||||
"\n",
|
||||
"sample_text = \"Apple Inc. was founded by Steve Jobs in Cupertino, California in 1976.\"\n",
|
||||
"\n",
|
||||
@@ -219,8 +219,8 @@
|
||||
" print(f\"\\n Method: {method_name.upper()}\")\n",
|
||||
" print(\"-\" * 40)\n",
|
||||
" \n",
|
||||
" method = get_entity_method(method_name)\n",
|
||||
" entities = method(sample_text)\n",
|
||||
" extractor = NERExtractor(method=method_name)\n",
|
||||
" entities = extractor.extract(sample_text)\n",
|
||||
" \n",
|
||||
" print(f\"Found {len(entities)} entities:\")\n",
|
||||
" for entity in entities[:5]: # Show first 5\n",
|
||||
@@ -638,4 +638,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,7 +180,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.semantic_extract.methods import get_relation_method\n",
|
||||
"from semantica.semantic_extract import RelationExtractor\n",
|
||||
"\n",
|
||||
"sample_text = \"Apple Inc. was founded by Steve Jobs in Cupertino, California.\"\n",
|
||||
"sample_entities = ner_extractor.extract(sample_text)\n",
|
||||
@@ -196,8 +196,8 @@
|
||||
" print(f\"\\n Method: {method_name.upper()}\")\n",
|
||||
" print(\"-\" * 40)\n",
|
||||
" \n",
|
||||
" method = get_relation_method(method_name)\n",
|
||||
" relations = method(sample_text, sample_entities)\n",
|
||||
" extractor = RelationExtractor(method=method_name)\n",
|
||||
" relations = extractor.extract(sample_text, sample_entities)\n",
|
||||
" \n",
|
||||
" print(f\"Found {len(relations)} relations:\")\n",
|
||||
" for rel in relations[:3]: # Show first 3\n",
|
||||
@@ -690,4 +690,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -282,7 +282,7 @@
|
||||
"entity_chunker = EntityAwareChunker(\n",
|
||||
" chunk_size=200,\n",
|
||||
" chunk_overlap=50,\n",
|
||||
" ner_method=\"spacy\", # or \"llm\" for better accuracy\n",
|
||||
" ner_method=\"ml\", # \"ml\" (spaCy), \"pattern\", or \"llm\"\n",
|
||||
" preserve_entities=True\n",
|
||||
")\n",
|
||||
"\n",
|
||||
@@ -850,4 +850,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+29
-29
@@ -6,31 +6,31 @@
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/20_Triple_Store.ipynb)\n",
|
||||
"\n",
|
||||
"# Triple Store - Comprehensive Guide\n",
|
||||
"# Triplet Store - Comprehensive Guide\n",
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook provides a **comprehensive walkthrough** of Semantica's triple_store module, demonstrating RDF triple storage, SPARQL querying, and multi-backend support for knowledge graph persistence.\n",
|
||||
"This notebook provides a **comprehensive walkthrough** of Semantica's triplet_store module, demonstrating RDF triplet storage, SPARQL querying, and multi-backend support for knowledge graph persistence.\n",
|
||||
"\n",
|
||||
"**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/triple_store/)\n",
|
||||
"**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/triplet_store/)\n",
|
||||
"\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"By the end of this notebook, you will be able to:\n",
|
||||
"\n",
|
||||
"- Register and manage triple stores (Blazegraph, Jena, RDF4J, Virtuoso)\n",
|
||||
"- Perform CRUD operations on RDF triples\n",
|
||||
"- Register and manage triplet stores (Blazegraph, Jena, RDF4J, Virtuoso)\n",
|
||||
"- Perform CRUD operations on RDF triplets\n",
|
||||
"- Execute SPARQL queries with optimization\n",
|
||||
"- Use bulk loading for large datasets\n",
|
||||
"- Work with multiple store backends\n",
|
||||
"- Validate and track triple operations\n",
|
||||
"- Validate and track triplet operations\n",
|
||||
"- Choose the right backend for your use case\n",
|
||||
"\n",
|
||||
"### What You'll Learn\n",
|
||||
"\n",
|
||||
"| Component | Purpose | When to Use |\n",
|
||||
"|-----------|---------|-------------|\n",
|
||||
"| `TripleManager` | Store coordination | All triple operations |\n",
|
||||
"| `TripletManager` | Store coordination | All triplet operations |\n",
|
||||
"| `QueryEngine` | SPARQL execution | Query optimization |\n",
|
||||
"| `BulkLoader` | High-volume loading | Large datasets |\n",
|
||||
"| `BlazegraphAdapter` | Blazegraph backend | High performance |\n",
|
||||
@@ -57,13 +57,13 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 1: Basic Triple Store Operations\n",
|
||||
"## Step 1: Basic Triplet Store Operations\n",
|
||||
"\n",
|
||||
"Let's start with the `TripleManager` for basic triple store operations.\n",
|
||||
"Let's start with the `TripletManager` for basic triplet store operations.\n",
|
||||
"\n",
|
||||
"### What is TripleManager?\n",
|
||||
"### What is TripletManager?\n",
|
||||
"\n",
|
||||
"`TripleManager` is the main coordinator for triple store operations:\n",
|
||||
"`TripletManager` is the main coordinator for triplet store operations:\n",
|
||||
"- **Store Registration**: Register multiple backends\n",
|
||||
"- **CRUD Operations**: Add, get, update, delete triples\n",
|
||||
"- **Multi-Store**: Manage multiple stores simultaneously"
|
||||
@@ -75,11 +75,11 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import TripleManager\n",
|
||||
"from semantica.triplet_store import TripletManager\n",
|
||||
"from semantica.semantic_extract.triple_extractor import Triple\n",
|
||||
"\n",
|
||||
"# Create triple manager\n",
|
||||
"manager = TripleManager()\n",
|
||||
"manager = TripletManager()\n",
|
||||
"\n",
|
||||
"# Register a Blazegraph store (in-memory for demo)\n",
|
||||
"store = manager.register_store(\n",
|
||||
@@ -130,7 +130,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import register_store\n",
|
||||
"from semantica.triplet_store import register_store\n",
|
||||
"\n",
|
||||
"# Register multiple stores using convenience function\n",
|
||||
"blazegraph_store = register_store(\n",
|
||||
@@ -179,7 +179,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import add_triple, add_triples, get_triples, update_triple, delete_triple\n",
|
||||
"from semantica.triplet_store import add_triple, add_triples, get_triples, update_triple, delete_triple\n",
|
||||
"\n",
|
||||
"# Create - Add single triple\n",
|
||||
"triple1 = Triple(\n",
|
||||
@@ -240,7 +240,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import QueryEngine, BlazegraphAdapter\n",
|
||||
"from semantica.triplet_store import QueryEngine, BlazegraphAdapter\n",
|
||||
"\n",
|
||||
"# Create query engine with caching\n",
|
||||
"engine = QueryEngine(enable_caching=True, enable_optimization=True)\n",
|
||||
@@ -299,7 +299,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import optimize_query, plan_query\n",
|
||||
"from semantica.triplet_store import optimize_query, plan_query\n",
|
||||
"\n",
|
||||
"# Original query\n",
|
||||
"query = \"\"\"\n",
|
||||
@@ -347,7 +347,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import BulkLoader, LoadProgress\n",
|
||||
"from semantica.triplet_store import BulkLoader, LoadProgress\n",
|
||||
"\n",
|
||||
"# Create bulk loader\n",
|
||||
"loader = BulkLoader(\n",
|
||||
@@ -392,11 +392,11 @@
|
||||
"source": [
|
||||
"## Step 7: Store Adapters\n",
|
||||
"\n",
|
||||
"Work with different triple store backends.\n",
|
||||
"Work with different triplet store backends.\n",
|
||||
"\n",
|
||||
"### Blazegraph Adapter\n",
|
||||
"\n",
|
||||
"High-performance triple store with GPU acceleration."
|
||||
"High-performance triplet store with GPU acceleration."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -405,7 +405,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import BlazegraphAdapter\n",
|
||||
"from semantica.triplet_store import BlazegraphAdapter\n",
|
||||
"\n",
|
||||
"# Create Blazegraph adapter\n",
|
||||
"blazegraph = BlazegraphAdapter(\n",
|
||||
@@ -442,7 +442,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import JenaAdapter\n",
|
||||
"from semantica.triplet_store import JenaAdapter\n",
|
||||
"\n",
|
||||
"# Create Jena adapter (in-memory)\n",
|
||||
"jena = JenaAdapter()\n",
|
||||
@@ -497,7 +497,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import RDF4JAdapter\n",
|
||||
"from semantica.triplet_store import RDF4JAdapter\n",
|
||||
"\n",
|
||||
"# Create RDF4J adapter\n",
|
||||
"rdf4j = RDF4JAdapter(\n",
|
||||
@@ -540,7 +540,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import VirtuosoAdapter\n",
|
||||
"from semantica.triplet_store import VirtuosoAdapter\n",
|
||||
"\n",
|
||||
"# Create Virtuoso adapter\n",
|
||||
"virtuoso = VirtuosoAdapter(\n",
|
||||
@@ -604,7 +604,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.triple_store import validate_triples\n",
|
||||
"from semantica.triplet_store import validate_triples\n",
|
||||
"\n",
|
||||
"# Create triples (some invalid)\n",
|
||||
"triples_to_validate = [\n",
|
||||
@@ -650,7 +650,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Register multiple stores\n",
|
||||
"manager = TripleManager()\n",
|
||||
"manager = TripletManager()\n",
|
||||
"\n",
|
||||
"primary = manager.register_store(\n",
|
||||
" \"primary\",\n",
|
||||
@@ -720,7 +720,7 @@
|
||||
"\n",
|
||||
"In this notebook, you've learned how to:\n",
|
||||
"\n",
|
||||
"- Register and manage triple stores\n",
|
||||
"- Register and manage triplet stores\n",
|
||||
"- Perform CRUD operations on RDF triples\n",
|
||||
"- Execute and optimize SPARQL queries\n",
|
||||
"- Use bulk loading for large datasets\n",
|
||||
@@ -740,7 +740,7 @@
|
||||
"### Next Steps\n",
|
||||
"\n",
|
||||
"**Further Reading**:\n",
|
||||
"- [Triple Store API Reference](https://semantica.readthedocs.io/reference/triple_store/)\n",
|
||||
"- [Triplet Store API Reference](https://semantica.readthedocs.io/reference/triplet_store/)\n",
|
||||
"- [SPARQL 1.1 Specification](https://www.w3.org/TR/sparql11-query/)\n",
|
||||
"- [Knowledge Graph Building](../use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)\n",
|
||||
"\n",
|
||||
@@ -771,4 +771,4 @@
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
}
|
||||
@@ -38,7 +38,7 @@
|
||||
"- **Parsing**: DocumentParser, PDFParser, StructuredDataParser, CSVParser, MCPParser\n",
|
||||
"- **Extraction**: NERExtractor, RelationExtractor, CoreferenceResolver, TripleExtractor\n",
|
||||
"- **KG**: GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n",
|
||||
"- **Triple Store**: TripleStore, TripleManager, QueryEngine\n",
|
||||
"- **Triplet Store**: TripletStore, TripletManager, QueryEngine\n",
|
||||
"- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
"- **Quality**: KGQualityAssessor, ValidationEngine\n",
|
||||
"- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n",
|
||||
@@ -67,7 +67,6 @@
|
||||
"from semantica.kg import GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n",
|
||||
"from semantica.triple_store import TripleStore, TripleManager, QueryEngine\n",
|
||||
"from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
|
||||
"from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n",
|
||||
"from semantica.visualization import KGVisualizer, OntologyVisualizer, TemporalVisualizer\n",
|
||||
"import tempfile\n",
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
"- **Materialized Knowledge Graphs**: Semantica's KG modules enable building persistent knowledge graphs from medical ontologies, clinical documents, and reports\n",
|
||||
"- **Virtual Data Integration**: Semantica's DBIngestor and QueryEngine allow virtual integration with Electronic Health Records (EHRs) without data replication\n",
|
||||
"- **Hybrid Design**: Semantica's architecture naturally separates structural knowledge from patient-level data\n",
|
||||
"- **Dynamic Query Orchestration**: Semantica's Reasoning and Triple Store modules enable orchestration of queries across ontologies, documents, and EHRs\n",
|
||||
"- **Dynamic Query Orchestration**: Semantica's Reasoning and Triplet Store modules enable orchestration of queries across ontologies, documents, and EHRs\n",
|
||||
"- **Temporal & Semantic Dimensions**: Semantica's Temporal and Context modules provide historical analysis and semantic understanding\n",
|
||||
"- **Traceable & Explainable**: Semantica's ExplanationGenerator and ContextRetriever provide traceable, explainable answers\n",
|
||||
"\n",
|
||||
@@ -55,7 +55,7 @@
|
||||
"- **KG**: GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer (materialized knowledge graph)\n",
|
||||
"- **Embeddings**: EmbeddingGenerator, TextEmbedder (for embeddings)\n",
|
||||
"- **Vector Store**: VectorStore, HybridSearch, MetadataFilter (for RAG)\n",
|
||||
"- **Triple Store**: TripleManager, QueryEngine (for SPARQL queries on ontologies)\n",
|
||||
"- **Triplet Store**: TripletManager, QueryEngine (for SPARQL queries on ontologies)\n",
|
||||
"- **Reasoning**: InferenceEngine, RuleManager (for query orchestration and medical reasoning)\n",
|
||||
"- **Context**: ContextRetriever, ContextGraphBuilder (for contextual retrieval)\n",
|
||||
"- **Visualization**: KGVisualizer, TemporalVisualizer, AnalyticsVisualizer (for visualization)\n",
|
||||
@@ -86,7 +86,7 @@
|
||||
"from semantica.kg import GraphBuilder, GraphAnalyzer, ConnectivityAnalyzer\n",
|
||||
"from semantica.embeddings import EmbeddingGenerator, TextEmbedder\n",
|
||||
"from semantica.vector_store import VectorStore, HybridSearch, MetadataFilter\n",
|
||||
"from semantica.triple_store import TripleManager, QueryEngine\n",
|
||||
"from semantica.triplet_store import TripletManager, QueryEngine\n",
|
||||
"from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
"from semantica.context import ContextRetriever, ContextGraphBuilder\n",
|
||||
"from semantica.visualization import KGVisualizer, TemporalVisualizer, AnalyticsVisualizer\n",
|
||||
@@ -440,9 +440,9 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 9: Setup Triple Store for Ontology Queries Using Semantica\n",
|
||||
"## Step 9: Setup Triplet Store for Ontology Queries Using Semantica\n",
|
||||
"\n",
|
||||
"Using Semantica's triple store modules to enable SPARQL queries on medical ontologies.\n"
|
||||
"Using Semantica's triplet store modules to enable SPARQL queries on medical ontologies.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -451,12 +451,12 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Initialize Semantica triple store and query engine\n",
|
||||
"triple_manager = TripleManager()\n",
|
||||
"# Initialize Semantica triplet store and query engine\n",
|
||||
"triplet_manager = TripletManager()\n",
|
||||
"query_engine = QueryEngine()\n",
|
||||
"\n",
|
||||
"# Register triple store (using in-memory for demo)\n",
|
||||
"store = triple_manager.register_store(\"healthcare_ontology\", \"jena\", \"http://localhost:3030/healthcare\")\n",
|
||||
"# Register triplet store (using in-memory for demo)\n",
|
||||
"store = triplet_manager.register_store(\"healthcare_ontology\", \"jena\", \"http://localhost:3030/healthcare\")\n",
|
||||
"\n",
|
||||
"# Convert ontology to triples and add to store\n",
|
||||
"# In production, this would load the OWL ontology\n",
|
||||
@@ -477,7 +477,7 @@
|
||||
"\n",
|
||||
"# Add triples using Semantica\n",
|
||||
"for triple in sample_triples:\n",
|
||||
" triple_manager.add_triple(triple, store_id=\"healthcare_ontology\")\n",
|
||||
" triplet_manager.add_triple(triple, store_id=\"healthcare_ontology\")\n",
|
||||
"\n",
|
||||
"print(f\" - Triples added: {len(sample_triples)}\")\n",
|
||||
"print(f\" - SPARQL queries enabled for ontology\")\n"
|
||||
@@ -531,7 +531,7 @@
|
||||
" \"context\": {}\n",
|
||||
" }\n",
|
||||
" \n",
|
||||
" # 1. Query ontology using Semantica Triple Store\n",
|
||||
" # 1. Query ontology using Semantica Triplet Store\n",
|
||||
" sparql_query = f\"\"\"\n",
|
||||
" SELECT ?concept WHERE {{\n",
|
||||
" ?concept rdfs:label ?label .\n",
|
||||
@@ -769,12 +769,12 @@
|
||||
"2. **Materialized Knowledge Graphs**: Semantica's KG modules enable building persistent knowledge graphs from medical ontologies and documents\n",
|
||||
"3. **Virtual Data Integration**: Semantica's DBIngestor allows virtual integration with EHRs without data replication\n",
|
||||
"4. **Hybrid Search**: Semantica's HybridSearch combines vector similarity with knowledge graph queries\n",
|
||||
"5. **Query Orchestration**: Semantica's Reasoning and Triple Store modules enable dynamic query orchestration\n",
|
||||
"6. **Explainability**: Semantica's ExplanationGenerator provides traceable, explainable answers\n",
|
||||
"5. **Query Orchestration**: Semantica's Reasoning and Triplet Store modules enable dynamic query orchestration\n",
|
||||
"6. **Explainability**: Semantica's ExplanationGenerator provides traceable, explainable answers\n",
|
||||
"\n",
|
||||
"### Semantica-Specific Performance Considerations\n",
|
||||
"\n",
|
||||
"- **Vector Store**: Use Semantica's VectorStore with appropriate backend (FAISS for local, Pinecone/Weaviate for cloud)\n",
|
||||
"- **Vector Store**: Use Semantica's VectorStore with appropriate backend (FAISS for local, Weaviate for cloud)\n",
|
||||
"- **Graph Analytics**: Leverage Semantica's GraphAnalyzer for efficient centrality and community detection\n",
|
||||
"- **Pipeline Execution**: Use Semantica's ExecutionEngine for parallel execution of pipeline steps\n",
|
||||
"- **Caching**: Utilize Semantica's ContextRetriever caching for frequently accessed contexts\n",
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
"- **Parsing**: MCPParser, JSONParser, StructuredDataParser, DocumentParser\n",
|
||||
"- **Extraction**: NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n",
|
||||
"- **KG**: GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n",
|
||||
"- **Triple Store**: TripleStore, TripleManager, QueryEngine\n",
|
||||
"- **Triplet Store**: TripletStore, TripletManager, QueryEngine\n",
|
||||
"- **Reasoning**: InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
"- **Quality**: KGQualityAssessor, ValidationEngine\n",
|
||||
"- **Export**: JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n",
|
||||
@@ -67,9 +67,8 @@
|
||||
"from semantica.parse import MCPParser, JSONParser, StructuredDataParser, DocumentParser\n",
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor, TripleExtractor, SemanticAnalyzer\n",
|
||||
"from semantica.kg import GraphBuilder, GraphValidator, EntityResolver, GraphAnalyzer\n",
|
||||
"from semantica.triple_store import TripleStore, TripleManager, QueryEngine\n",
|
||||
"from semantica.triplet_store import TripletStore, TripletManager, QueryEngine\n",
|
||||
"from semantica.reasoning import InferenceEngine, RuleManager, ExplanationGenerator\n",
|
||||
|
||||
"from semantica.export import JSONExporter, RDFExporter, OWLExporter, ReportGenerator\n",
|
||||
"from semantica.visualization import KGVisualizer, OntologyVisualizer, TemporalVisualizer\n",
|
||||
"import json\n",
|
||||
@@ -402,7 +401,7 @@
|
||||
"source": [
|
||||
"## Step 5: Build Healthcare Knowledge Graph\n",
|
||||
"\n",
|
||||
"Build a knowledge graph from the extracted medical entities and relationships, then store in triple store.\n"
|
||||
"Build a knowledge graph from the extracted medical entities and relationships, then store in triplet store.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -426,13 +425,17 @@
|
||||
"# Analyze graph structure\n",
|
||||
"metrics = graph_analyzer.compute_metrics(resolved_kg)\n",
|
||||
"\n",
|
||||
"# Store in triple store\n",
|
||||
"triple_store = TripleStore()\n",
|
||||
"triple_manager = TripleManager()\n",
|
||||
"# Store in triplet store\n",
|
||||
"# triplet_store = TripletStore() # TripletStore is a configuration dataclass\n",
|
||||
"triplet_manager = TripletManager()\n",
|
||||
"query_engine = QueryEngine()\n",
|
||||
"\n",
|
||||
"triple_store.add_knowledge_graph(resolved_kg)\n",
|
||||
"triple_manager.manage_triples(resolved_kg)\n",
|
||||
"# Register default store (in-memory for demo)\n",
|
||||
"store = triplet_manager.register_store(\"medical_kg\", \"jena\", \"http://localhost:3030/medical\")\n",
|
||||
"\n",
|
||||
"# Convert KG to triples and add to store (simplified)\n",
|
||||
"# In a real scenario, we would convert entities/relations to triples first\n",
|
||||
"# triplet_manager.add_triples(triples, store_id=\"medical_kg\")\n",
|
||||
"\n",
|
||||
"print(f\" Entities: {len(resolved_kg.get('entities', []))}\")\n",
|
||||
"print(f\" Relationships: {len(resolved_kg.get('relationships', []))}\")\n",
|
||||
|
||||
@@ -22,13 +22,13 @@
|
||||
"- **Extraction**: NERExtractor, RelationExtractor, CoreferenceResolver\n",
|
||||
"- **KG**: GraphBuilder, TemporalGraphQuery, GraphValidator, EntityResolver\n",
|
||||
"- **Ontology**: OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n",
|
||||
"- **Triple Store**: TripleStore, TripleManager, QueryEngine\n",
|
||||
"- **Triplet Store**: TripletStore, TripletManager, QueryEngine\n",
|
||||
"- **Export**: RDFExporter, OWLExporter, JSONExporter\n",
|
||||
"- **Visualization**: KGVisualizer, OntologyVisualizer, TemporalVisualizer\n",
|
||||
"\n",
|
||||
"### Pipeline\n",
|
||||
"\n",
|
||||
"**Patient Records → Parse → Extract Medical Entities → Build Temporal KG → Generate Ontology → Store in Triple Store → Query History → Export → Visualize**\n",
|
||||
"**Patient Records → Parse → Extract Medical Entities → Build Temporal KG → Generate Ontology → Store in Triplet Store → Query History → Export → Visualize**\n",
|
||||
"\n",
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
@@ -58,7 +58,7 @@
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor, CoreferenceResolver\n",
|
||||
"from semantica.kg import GraphBuilder, TemporalGraphQuery, GraphValidator, EntityResolver\n",
|
||||
"from semantica.ontology import OntologyGenerator, ClassInferrer, PropertyGenerator, OntologyValidator\n",
|
||||
"from semantica.triple_store import TripleStore, TripleManager, QueryEngine\n",
|
||||
"from semantica.triplet_store import TripletStore, TripletManager, QueryEngine\n",
|
||||
"from semantica.export import RDFExporter, OWLExporter, JSONExporter\n",
|
||||
"from semantica.visualization import KGVisualizer, OntologyVisualizer, TemporalVisualizer\n",
|
||||
"import tempfile\n",
|
||||
@@ -261,9 +261,9 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 5: Store in Triple Store and Query\n",
|
||||
"## Step 5: Store in Triplet Store and Query\n",
|
||||
"\n",
|
||||
"Store knowledge graph in triple store and query medical history.\n"
|
||||
"Store knowledge graph in triplet store and query medical history.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -272,12 +272,12 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"triple_store = TripleStore()\n",
|
||||
"triple_manager = TripleManager()\n",
|
||||
"triplet_store = TripletStore()\n",
|
||||
"triple_manager = TripletManager()\n",
|
||||
"query_engine = QueryEngine()\n",
|
||||
"temporal_query = TemporalGraphQuery()\n",
|
||||
"\n",
|
||||
"triple_store.store_knowledge_graph(patient_kg)\n",
|
||||
"triplet_store.store_knowledge_graph(patient_kg)\n",
|
||||
"\n",
|
||||
"patient_id = \"P001\"\n",
|
||||
"start_time = \"2024-01-01\"\n",
|
||||
@@ -290,7 +290,7 @@
|
||||
" end_time=end_time\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(f\"Stored patient knowledge graph in triple store\")\n",
|
||||
"print(f\"Stored patient knowledge graph in triplet store\")\n",
|
||||
"print(f\"Retrieved {len(medical_history.get('entities', []))} medical events for patient {patient_id}\")\n"
|
||||
]
|
||||
},
|
||||
@@ -326,7 +326,7 @@
|
||||
"temporal_viz = temporal_visualizer.visualize_timeline(patient_kg, output=\"interactive\")\n",
|
||||
"\n",
|
||||
"print(f\"Total modules used: 20+\")\n",
|
||||
"print(f\"Pipeline complete: Patient Records → Parse → Extract → Temporal KG → Ontology → Triple Store → Query → Export → Visualize\")\n"
|
||||
"print(f\"Pipeline complete: Patient Records → Parse → Extract → Temporal KG → Ontology → Triplet Store → Query → Export → Visualize\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -32,7 +32,7 @@ from semantica import Semantica
|
||||
core = Semantica(
|
||||
llm_provider="openai",
|
||||
embedding_model="text-embedding-3-large",
|
||||
vector_store="pinecone",
|
||||
vector_store="weaviate",
|
||||
graph_db="neo4j"
|
||||
)
|
||||
|
||||
@@ -279,7 +279,7 @@ owl_ontology = ontology.to_owl()
|
||||
rdf_ontology = ontology.to_rdf()
|
||||
turtle_ontology = ontology.to_turtle()
|
||||
|
||||
# Save to triple store
|
||||
# Save to triplet store
|
||||
ontology.save_to_triple_store("http://localhost:9999/blazegraph/sparql")
|
||||
```
|
||||
|
||||
@@ -357,7 +357,7 @@ semantic_chunks = embedder.semantic_chunk(documents)
|
||||
embeddings = embedder.generate_embeddings(semantic_chunks)
|
||||
|
||||
# Store in vector database
|
||||
vector_store = core.get_vector_store("pinecone")
|
||||
vector_store = core.get_vector_store("weaviate")
|
||||
vector_store.store_embeddings(semantic_chunks, embeddings)
|
||||
|
||||
# Semantic search
|
||||
|
||||
@@ -66,8 +66,8 @@ graph TB
|
||||
|
||||
### Knowledge Graphs
|
||||
- **`semantica.kg`** - Knowledge graph construction
|
||||
- **`semantica.vector_store`** - Vector storage (Pinecone, Weaviate, FAISS)
|
||||
- **`semantica.triple_store`** - RDF triple storage (Jena, Blazegraph)
|
||||
- **`semantica.vector_store`** - Vector storage (Weaviate, FAISS)
|
||||
- **`semantica.triplet_store`** - RDF triplet storage (Jena, Blazegraph)
|
||||
- **`semantica.graph_store`** - Property graphs (Neo4j, FalkorDB)
|
||||
|
||||
### Quality Assurance
|
||||
@@ -85,7 +85,7 @@ graph TB
|
||||
4. Semantic Extraction → Entities, relationships, events
|
||||
5. Graph Construction → Entity resolution, conflict resolution
|
||||
6. Quality Assurance → Deduplication, validation
|
||||
7. Storage → Vector, triple, and graph stores
|
||||
7. Storage → Vector, triplet, and graph stores
|
||||
8. Application → GraphRAG, agents, analytics
|
||||
```
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@ Projects and integrations from the Semantica community.
|
||||
## 🔌 Integrations
|
||||
|
||||
### Vector Databases
|
||||
- Pinecone
|
||||
- Weaviate
|
||||
- Qdrant
|
||||
- FAISS
|
||||
|
||||
+1
-1
@@ -181,7 +181,7 @@ A comprehensive reference of terms and concepts used in Semantica.
|
||||
**Triple**
|
||||
: A basic unit of knowledge in RDF, consisting of a subject, predicate, and object (e.g., `<Apple_Inc> <founded_by> <Steve_Jobs>`).
|
||||
|
||||
**Triple Store**
|
||||
**Triplet Store**
|
||||
: A database designed specifically for storing and querying RDF triples.
|
||||
|
||||
---
|
||||
|
||||
+10
-11
@@ -15,7 +15,7 @@ Semantica's modules are organized into six logical layers:
|
||||
| :--- | :--- | :--- |
|
||||
| **Input Layer** | [Ingest](#ingest-module), [Parse](#parse-module), [Split](#split-module), [Normalize](#normalize-module) | Data ingestion, parsing, chunking, and cleaning |
|
||||
| **Core Processing** | [Semantic Extract](#semantic-extract-module), [Knowledge Graph](#knowledge-graph-kg-module), [Ontology](#ontology-module), [Reasoning](#reasoning-module) | Entity extraction, graph construction, inference |
|
||||
| **Storage** | [Embeddings](#embeddings-module), [Vector Store](#vector-store-module), [Graph Store](#graph-store-module), [Triple Store](#triple-store-module) | Vector and graph persistence |
|
||||
| **Storage** | [Embeddings](#embeddings-module), [Vector Store](#vector-store-module), [Graph Store](#graph-store-module), [Triplet Store](#triplet-store-module) | Vector, graph, and triplet persistence |
|
||||
| **Quality Assurance** | [Deduplication](#deduplication-module), [Conflicts](#conflicts-module) | Data quality and consistency |
|
||||
| **Context & Memory** | [Context](#context-module), [Seed](#seed-module) | Agent memory and foundation data |
|
||||
| **Output & Orchestration** | [Export](#export-module), [Visualization](#visualization-module), [Pipeline](#pipeline-module) | Export, visualization, and workflow management |
|
||||
@@ -468,7 +468,7 @@ print(f"Similarity: {similarity:.3f}")
|
||||
|
||||
**Key Features:**
|
||||
|
||||
- Multiple backend support (FAISS, Pinecone, Weaviate, Qdrant, Milvus)
|
||||
- Multiple backend support (FAISS, Weaviate, Qdrant, Milvus)
|
||||
- Hybrid search (vector + keyword)
|
||||
- Metadata filtering
|
||||
- Batch operations
|
||||
@@ -480,7 +480,6 @@ print(f"Similarity: {similarity:.3f}")
|
||||
|
||||
- `VectorStore` — Main vector store interface
|
||||
- `FAISSAdapter` — FAISS integration
|
||||
- `PineconeAdapter` — Pinecone integration
|
||||
- `WeaviateAdapter` — Weaviate integration
|
||||
- `HybridSearch` — Combine vector and keyword search
|
||||
- `VectorRetriever` — Retrieve relevant vectors
|
||||
@@ -563,15 +562,15 @@ results = store.execute_query("MATCH (p:Person) RETURN p.name")
|
||||
|
||||
---
|
||||
|
||||
### Triple Store Module
|
||||
### Triplet Store Module
|
||||
|
||||
!!! abstract "Purpose"
|
||||
RDF triple store integration for semantic web applications. Supports SPARQL queries and multiple backends.
|
||||
RDF triplet store integration for semantic web applications. Supports SPARQL queries and multiple backends.
|
||||
|
||||
**Key Features:**
|
||||
|
||||
- Multi-backend support (Blazegraph, Jena, RDF4J, Virtuoso)
|
||||
- CRUD operations for RDF triples
|
||||
- CRUD operations for RDF triplets
|
||||
- SPARQL query execution and optimization
|
||||
- Bulk data loading with progress tracking
|
||||
- Query caching and optimization
|
||||
@@ -580,7 +579,7 @@ results = store.execute_query("MATCH (p:Person) RETURN p.name")
|
||||
|
||||
**Components:**
|
||||
|
||||
- `TripleManager` — Main triple store management coordinator
|
||||
- `TripletManager` — Main triplet store management coordinator
|
||||
- `QueryEngine` — SPARQL query execution and optimization
|
||||
- `BulkLoader` — High-volume data loading with progress tracking
|
||||
- `BlazegraphAdapter` — Blazegraph integration
|
||||
@@ -601,9 +600,9 @@ results = store.execute_query("MATCH (p:Person) RETURN p.name")
|
||||
**Quick Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleManager, execute_query
|
||||
from semantica.triplet_store import TripletManager, execute_query
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
store = manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph")
|
||||
|
||||
# Add triple
|
||||
@@ -617,7 +616,7 @@ result = manager.add_triple({
|
||||
query_result = execute_query("SELECT ?s ?p ?o WHERE { ?s ?p ?o } LIMIT 10", store)
|
||||
```
|
||||
|
||||
**API Reference**: [Triple Store Module](reference/triple_store.md)
|
||||
**API Reference**: [Triplet Store Module](reference/triplet_store.md)
|
||||
|
||||
---
|
||||
|
||||
@@ -1114,7 +1113,7 @@ new_facts = inference_engine.forward_chain(kg, rule_manager)
|
||||
| **Embeddings** | `semantica.embeddings` | `EmbeddingGenerator` | Vector generation |
|
||||
| **Vector Store** | `semantica.vector_store` | `VectorStore` | Vector storage |
|
||||
| **Graph Store** | `semantica.graph_store` | `GraphStore` | Graph database |
|
||||
| **Triple Store** | `semantica.triple_store` | `TripleManager` | RDF storage |
|
||||
| **Triplet Store** | `semantica.triplet_store` | `TripletManager` | RDF storage |
|
||||
| **Deduplication** | `semantica.deduplication` | `DuplicateDetector` | Duplicate removal |
|
||||
| **Conflicts** | `semantica.conflicts` | `ConflictDetector` | Conflict resolution |
|
||||
| **Context** | `semantica.context` | `AgentMemory` | Agent context |
|
||||
|
||||
@@ -57,7 +57,7 @@ The **Context Module** provides agents with a persistent, searchable, and struct
|
||||
The high-level facade that unifies all context operations. It routes data to the appropriate subsystems (Memory, Graph, Vector Store) and manages the lifecycle of context.
|
||||
|
||||
#### **Constructor Parameters**
|
||||
* `vector_store` (Required): The backing vector database instance (e.g., FAISS, Pinecone).
|
||||
* `vector_store` (Required): The backing vector database instance (e.g., FAISS, Weaviate).
|
||||
* `knowledge_graph` (Optional): The graph store instance for structured knowledge.
|
||||
* `token_limit` (Default: `2000`): The maximum number of tokens allowed in short-term memory before pruning occurs.
|
||||
* `short_term_limit` (Default: `10`): The maximum number of distinct memory items in short-term memory.
|
||||
|
||||
@@ -34,7 +34,7 @@ The **Embeddings Module** provides a unified interface for generating vector rep
|
||||
|
||||
---
|
||||
|
||||
Automatic formatting and validation for FAISS, Pinecone, Qdrant, and Weaviate.
|
||||
Automatic formatting and validation for FAISS, Qdrant, and Weaviate.
|
||||
|
||||
</div>
|
||||
|
||||
@@ -122,13 +122,13 @@ print(f"Dimension: {embedder.get_embedding_dimension()}")
|
||||
---
|
||||
|
||||
### VectorEmbeddingManager (The Bridge)
|
||||
A utility class that prepares raw embeddings for insertion into specific vector databases. It handles formatting differences between backends like FAISS and Pinecone.
|
||||
A utility class that prepares raw embeddings for insertion into specific vector databases. It handles formatting differences between backends like FAISS and Weaviate.
|
||||
|
||||
#### **Core Methods**
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `prepare_for_vector_db(embeddings, backend, ...)` | Formats data for the target DB. |
|
||||
| `prepare_for_vector_db(embeddings, metadata, backend)` | Formats data for the target DB. |
|
||||
| `validate_dimensions(embeddings, expected_dim)` | Ensures vectors match the index configuration. |
|
||||
| `batch_prepare(embeddings_list)` | Prepares a batch of embeddings for storage. |
|
||||
|
||||
|
||||
@@ -414,7 +414,7 @@ subgraph = graph_store.execute_query(query, parameters={"ids": node_ids})
|
||||
## See Also
|
||||
|
||||
- [Knowledge Graph Module](kg.md) - Logical layer above Graph Store
|
||||
- [Triple Store Module](triple_store.md) - RDF-based alternative
|
||||
- [Triplet Store Module](triplet_store.md) - RDF-based alternative
|
||||
- [Visualization Module](visualization.md) - Visualizing query results
|
||||
|
||||
## Cookbook
|
||||
|
||||
@@ -245,7 +245,7 @@ kg.add_triples(inferred_triples)
|
||||
## See Also
|
||||
|
||||
- [Ontology Module](ontology.md) - Source of schema-based rules
|
||||
- [Triple Store Module](triple_store.md) - Backend for SPARQL reasoning
|
||||
- [Triplet Store Module](triplet_store.md) - Backend for SPARQL reasoning
|
||||
- [Modules Guide](../modules.md#quality-assurance) - Consistency checking overview
|
||||
|
||||
## Cookbook
|
||||
|
||||
@@ -44,6 +44,12 @@
|
||||
|
||||
Use LLMs to improve extraction quality and handle complex schemas
|
||||
|
||||
- :material-graph:{ .lg .middle } **Semantic Networks**
|
||||
|
||||
---
|
||||
|
||||
Build structured networks with nodes and edges from text
|
||||
|
||||
</div>
|
||||
|
||||
!!! tip "When to Use"
|
||||
@@ -119,6 +125,49 @@ ner = NamedEntityRecognizer(
|
||||
entities = ner.extract_entities("Apple Inc. was founded in 1976.")
|
||||
```
|
||||
|
||||
### NERExtractor
|
||||
|
||||
Core entity extraction implementation used by notebooks and lower-level integrations.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `method` | str or list | `"ml"` | Method(s): "ml", "llm", "pattern", "regex", "huggingface" |
|
||||
| `**config` | dict | `{}` | Method-specific config (e.g., `model`, `provider`) |
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `extract(text)` | Alias for `extract_entities`. Get list of entities. |
|
||||
| `extract_entities(text)` | Get list of entities |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
|
||||
# 1. ML (spaCy) - Default
|
||||
extractor = NERExtractor(method="ml", model="en_core_web_trf")
|
||||
entities = extractor.extract("Elon Musk leads SpaceX.")
|
||||
|
||||
# 2. LLM (OpenAI/Gemini/etc)
|
||||
extractor = NERExtractor(
|
||||
method="llm",
|
||||
provider="openai",
|
||||
model="gpt-4",
|
||||
temperature=0.0
|
||||
)
|
||||
|
||||
# 3. Regex with custom patterns
|
||||
patterns = {"CODE": r"[A-Z]{3}-\d{3}"}
|
||||
extractor = NERExtractor(method="regex", patterns=patterns)
|
||||
|
||||
# 4. Ensemble (Multiple methods)
|
||||
extractor = NERExtractor(method=["ml", "llm"], ensemble_voting=True)
|
||||
```
|
||||
|
||||
### RelationExtractor
|
||||
|
||||
Extracts relationships between entities.
|
||||
@@ -136,6 +185,7 @@ Extracts relationships between entities.
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `extract(text, entities)` | Alias for `extract_relations`. Find links. |
|
||||
| `extract_relations(text, entities)` | Find links |
|
||||
|
||||
**Example:**
|
||||
@@ -150,7 +200,7 @@ entities = ner.extract_entities(text)
|
||||
|
||||
# Basic relation extraction
|
||||
rel_extractor = RelationExtractor()
|
||||
relations = rel_extractor.extract_relations(text, entities=entities)
|
||||
relations = rel_extractor.extract(text, entities=entities)
|
||||
# [Relation(source="Elon Musk", target="SpaceX", type="founded")]
|
||||
|
||||
# With configuration
|
||||
@@ -159,7 +209,39 @@ rel_extractor = RelationExtractor(
|
||||
confidence_threshold=0.7,
|
||||
bidirectional=False
|
||||
)
|
||||
relations = rel_extractor.extract_relations(text, entities=entities)
|
||||
relations = rel_extractor.extract(text, entities=entities)
|
||||
```
|
||||
|
||||
### CoreferenceResolver
|
||||
|
||||
Resolves pronoun references and entity coreferences.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `method` | str or list | `None` | Underlying NER method(s) |
|
||||
| `**config` | dict | `{}` | Configuration for NER method |
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `resolve(text)` | Alias for `resolve_coreferences`. Get coreference chains. |
|
||||
| `resolve_coreferences(text)` | Get coreference chains |
|
||||
| `resolve_pronouns(text)` | Resolve pronouns to entities |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract import CoreferenceResolver
|
||||
|
||||
resolver = CoreferenceResolver()
|
||||
text = "Steve Jobs founded Apple. He was the CEO."
|
||||
|
||||
# Resolve references
|
||||
chains = resolver.resolve(text)
|
||||
# [CoreferenceChain(mentions=["Steve Jobs", "He"], representative="Steve Jobs")]
|
||||
```
|
||||
|
||||
### EventDetector
|
||||
@@ -204,6 +286,7 @@ Extracts RDF triples (Subject-Predicate-Object).
|
||||
|-----------|------|---------|-------------|
|
||||
| `include_temporal` | bool | `False` | Include time information |
|
||||
| `include_provenance` | bool | `False` | Track source sentences |
|
||||
| `method` | str | `"pattern"` | Extraction method ("pattern", "rules", "huggingface", "llm") |
|
||||
|
||||
**Methods:**
|
||||
|
||||
@@ -224,6 +307,66 @@ triples = extractor.extract_triples("Steve Jobs founded Apple in 1976.")
|
||||
# [Triple(subject="Steve Jobs", predicate="founded", object="Apple", temporal="1976")]
|
||||
```
|
||||
|
||||
### SemanticNetworkExtractor
|
||||
|
||||
Extracts structured semantic networks with nodes and edges.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `ner_method` | str | `None` | Method for node extraction |
|
||||
| `relation_method` | str | `None` | Method for edge extraction |
|
||||
| `**config` | dict | `{}` | Configuration for underlying extractors |
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `extract_network(text)` | Build network from text |
|
||||
| `extract(text)` | Alias for `extract_network` |
|
||||
| `export_to_yaml(network, path)` | Save network to YAML |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract import SemanticNetworkExtractor
|
||||
|
||||
extractor = SemanticNetworkExtractor()
|
||||
network = extractor.extract("Apple Inc. is located in Cupertino.")
|
||||
|
||||
# Analyze network
|
||||
print(f"Nodes: {len(network.nodes)}")
|
||||
print(f"Edges: {len(network.edges)}")
|
||||
```
|
||||
|
||||
### LLMEnhancer
|
||||
|
||||
Enhances extraction results using Large Language Models.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `provider` | str | `"openai"` | LLM provider ("openai", "gemini", "anthropic", etc.) |
|
||||
| `**config` | dict | `{}` | Model config (model name, api_key, etc.) |
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `enhance_entities(text, entities)` | Improve entity accuracy and details |
|
||||
| `enhance_relations(text, relations)` | Improve relation detection |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract import LLMEnhancer
|
||||
|
||||
enhancer = LLMEnhancer(provider="openai", model="gpt-4")
|
||||
enhanced_entities = enhancer.enhance_entities(text, entities)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Usage Examples
|
||||
@@ -234,7 +377,8 @@ from semantica.semantic_extract import (
|
||||
RelationExtractor,
|
||||
TripleExtractor,
|
||||
EventDetector,
|
||||
CoreferenceResolver
|
||||
CoreferenceResolver,
|
||||
SemanticNetworkExtractor
|
||||
)
|
||||
|
||||
text = "Apple released the iPhone in 2007. Steve Jobs announced it at Macworld."
|
||||
@@ -259,10 +403,15 @@ triples = triple_extractor.extract_triples(text)
|
||||
event_detector = EventDetector(extract_time=True)
|
||||
events = event_detector.detect_events(text)
|
||||
|
||||
# Extract semantic network
|
||||
network_extractor = SemanticNetworkExtractor()
|
||||
network = network_extractor.extract(text)
|
||||
|
||||
print(f"Entities: {len(entities)}")
|
||||
print(f"Relations: {len(relations)}")
|
||||
print(f"Triples: {len(triples)}")
|
||||
print(f"Events: {len(events)}")
|
||||
print(f"Network Nodes: {len(network.nodes)}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
+40
-60
@@ -150,7 +150,7 @@ TextSplitter(
|
||||
similarity_threshold=0.7, # Semantic boundary threshold
|
||||
|
||||
# Entity-aware options
|
||||
ner_method="spacy", # NER method (spacy, llm, transformers)
|
||||
ner_method="ml", # NER method (ml/spacy, llm, pattern)
|
||||
preserve_entities=True, # Don't split entities
|
||||
|
||||
# LLM options
|
||||
@@ -183,7 +183,7 @@ for i, chunk in enumerate(chunks):
|
||||
# Entity-aware for GraphRAG
|
||||
splitter = TextSplitter(
|
||||
method="entity_aware",
|
||||
ner_method="llm",
|
||||
ner_method="ml",
|
||||
chunk_size=1000,
|
||||
preserve_entities=True
|
||||
)
|
||||
@@ -250,8 +250,6 @@ Preserve entity boundaries during chunking for GraphRAG.
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `chunk(text, entities)` | Chunk preserving entities | Entity boundary detection |
|
||||
| `extract_entities(text)` | Extract entities | NER extraction |
|
||||
| `find_safe_split_points(text, entities)` | Find split points | Entity span checking |
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -260,14 +258,14 @@ from semantica.split import EntityAwareChunker
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
|
||||
# Extract entities first
|
||||
ner = NERExtractor(method="llm")
|
||||
ner = NERExtractor(method="ml")
|
||||
entities = ner.extract(text)
|
||||
|
||||
# Chunk preserving entities
|
||||
chunker = EntityAwareChunker(
|
||||
chunk_size=1000,
|
||||
chunk_overlap=200,
|
||||
ner_method="llm"
|
||||
ner_method="ml"
|
||||
)
|
||||
|
||||
chunks = chunker.chunk(text, entities=entities)
|
||||
@@ -360,8 +358,7 @@ Structure-aware chunking respecting document hierarchy.
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `chunk(text)` | Chunk by structure | Heading/section detection |
|
||||
| `detect_structure(text)` | Detect document structure | Markdown/HTML parsing |
|
||||
| `build_hierarchy(sections)` | Build section hierarchy | Tree construction |
|
||||
| `_extract_structure(text)` | Extract structural elements | Markdown/HTML parsing |
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -369,17 +366,16 @@ Structure-aware chunking respecting document hierarchy.
|
||||
from semantica.split import StructuralChunker
|
||||
|
||||
chunker = StructuralChunker(
|
||||
respect_headings=True,
|
||||
respect_paragraphs=True,
|
||||
respect_lists=True,
|
||||
respect_headers=True,
|
||||
respect_sections=True,
|
||||
max_chunk_size=2000
|
||||
)
|
||||
|
||||
chunks = chunker.chunk(markdown_text)
|
||||
|
||||
for chunk in chunks:
|
||||
print(f"Section: {chunk.metadata.get('section_title')}")
|
||||
print(f"Level: {chunk.metadata.get('heading_level')}")
|
||||
print(f"Structure preserved: {chunk.metadata.get('structure_preserved')}")
|
||||
print(f"Elements: {chunk.metadata.get('element_types')}")
|
||||
```
|
||||
|
||||
---
|
||||
@@ -393,7 +389,6 @@ Multi-level hierarchical chunking.
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `chunk(text)` | Multi-level chunking | Recursive hierarchical split |
|
||||
| `create_hierarchy(chunks)` | Create chunk hierarchy | Tree structure |
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -470,16 +465,15 @@ Fixed-size sliding window chunking with configurable step size.
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `chunk(text)` | Sliding window chunking | Fixed-size window with step |
|
||||
| `calculate_windows(text_length)` | Calculate window positions | Window position calculation |
|
||||
| `chunk_with_overlap(text)` | Chunk with specific overlap | Window position calculation |
|
||||
|
||||
**Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `window_size` | int | 1000 | Size of sliding window |
|
||||
| `step_size` | int | 800 | Step size (window_size - overlap) |
|
||||
| `min_chunk_size` | int | 100 | Minimum chunk size |
|
||||
| `preserve_sentences` | bool | False | Preserve sentence boundaries |
|
||||
| `chunk_size` | int | 1000 | Size of sliding window |
|
||||
| `overlap` | int | 0 | Overlap size |
|
||||
| `stride` | int | chunk_size - overlap | Step size |
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -488,25 +482,18 @@ from semantica.split import SlidingWindowChunker
|
||||
|
||||
# Basic sliding window
|
||||
chunker = SlidingWindowChunker(
|
||||
window_size=1000,
|
||||
step_size=800, # 200 overlap
|
||||
min_chunk_size=100
|
||||
chunk_size=1000,
|
||||
overlap=200
|
||||
)
|
||||
|
||||
chunks = chunker.chunk(long_text)
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
print(f"Window {i}: chars {chunk.start}-{chunk.end}")
|
||||
print(f"Overlap with previous: {chunk.metadata.get('overlap_chars')}")
|
||||
print(f"Window {i}: chars {chunk.start_index}-{chunk.end_index}")
|
||||
print(f"Has overlap: {chunk.metadata.get('has_overlap')}")
|
||||
|
||||
# Sentence-preserving sliding window
|
||||
chunker = SlidingWindowChunker(
|
||||
window_size=1000,
|
||||
step_size=750,
|
||||
preserve_sentences=True
|
||||
)
|
||||
|
||||
chunks = chunker.chunk(text)
|
||||
# Boundary-preserving sliding window
|
||||
chunks = chunker.chunk(text, preserve_boundaries=True)
|
||||
```
|
||||
|
||||
---
|
||||
@@ -519,18 +506,17 @@ Table-specific chunking preserving table structure.
|
||||
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `chunk(text)` | Chunk tables | Table detection and splitting |
|
||||
| `detect_tables(text)` | Detect tables in text | Table boundary detection |
|
||||
| `split_table(table, max_rows)` | Split large tables | Row-based table splitting |
|
||||
| `chunk_table(table_data)` | Chunk tables | Row/Column-based splitting |
|
||||
| `chunk_to_text_chunks(table_data)` | Convert table chunks to text | Table to text conversion |
|
||||
| `extract_table_schema(table_data)` | Extract schema | Type inference and schema extraction |
|
||||
|
||||
**Parameters:**
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `max_rows` | int | 100 | Maximum rows per table chunk |
|
||||
| `preserve_headers` | bool | True | Keep headers in each chunk |
|
||||
| `max_rows_per_chunk` | int | 50 | Maximum rows per table chunk |
|
||||
| `include_context` | bool | True | Include surrounding text context |
|
||||
| `table_format` | str | "auto" | Table format (markdown, html, csv, auto) |
|
||||
| `chunk_by_columns` | bool | False | Chunk by columns instead of rows |
|
||||
|
||||
**Example:**
|
||||
|
||||
@@ -538,31 +524,25 @@ Table-specific chunking preserving table structure.
|
||||
from semantica.split import TableChunker
|
||||
|
||||
chunker = TableChunker(
|
||||
max_rows=50,
|
||||
preserve_headers=True,
|
||||
max_rows_per_chunk=50,
|
||||
include_context=True,
|
||||
table_format="markdown"
|
||||
chunk_by_columns=False
|
||||
)
|
||||
|
||||
text_with_tables = \"\"\"
|
||||
Document with tables...
|
||||
table_data = {
|
||||
"headers": ["Col1", "Col2", "Col3"],
|
||||
"rows": [["Val1", "Val2", "Val3"], ...]
|
||||
}
|
||||
|
||||
| Column 1 | Column 2 | Column 3 |
|
||||
|----------|----------|----------|
|
||||
| Value 1 | Value 2 | Value 3 |
|
||||
| ... | ... | ... |
|
||||
\"\"\"
|
||||
# Get structured table chunks
|
||||
table_chunks = chunker.chunk_table(table_data)
|
||||
|
||||
chunks = chunker.chunk(text_with_tables)
|
||||
# Get text chunks for RAG
|
||||
text_chunks = chunker.chunk_to_text_chunks(table_data)
|
||||
|
||||
for chunk in chunks:
|
||||
if chunk.metadata.get('is_table'):
|
||||
print(f"Table chunk:")
|
||||
print(f" Rows: {chunk.metadata.get('row_count')}")
|
||||
print(f" Columns: {chunk.metadata.get('column_count')}")
|
||||
print(f" Headers: {chunk.metadata.get('headers')}")
|
||||
else:
|
||||
print(f"Text chunk: {len(chunk.text)} chars")
|
||||
for chunk in text_chunks:
|
||||
print(f"Table chunk {chunk.metadata.get('chunk_index')}")
|
||||
print(f"Rows: {chunk.metadata.get('row_count')}")
|
||||
```
|
||||
|
||||
---
|
||||
@@ -663,7 +643,7 @@ print(f"Available methods: {methods}")
|
||||
# Quick splitting
|
||||
chunks = split_recursive(text, chunk_size=1000, chunk_overlap=200)
|
||||
chunks = split_by_sentences(text, sentences_per_chunk=5)
|
||||
chunks = split_entity_aware(text, ner_method="llm")
|
||||
chunks = split_entity_aware(text, ner_method="ml")
|
||||
```
|
||||
|
||||
---
|
||||
@@ -683,7 +663,7 @@ export SPLIT_EMBEDDING_MODEL=all-MiniLM-L6-v2
|
||||
export SPLIT_SIMILARITY_THRESHOLD=0.7
|
||||
|
||||
# Entity-aware
|
||||
export SPLIT_NER_METHOD=spacy
|
||||
export SPLIT_NER_METHOD=ml # or spacy
|
||||
export SPLIT_PRESERVE_ENTITIES=true
|
||||
|
||||
# LLM-based
|
||||
@@ -712,7 +692,7 @@ split:
|
||||
max_chunk_size: 2000
|
||||
|
||||
entity_aware:
|
||||
ner_method: spacy
|
||||
ner_method: ml # or spacy
|
||||
preserve_entities: true
|
||||
min_entity_gap: 50
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Triple Store
|
||||
# Triplet Store
|
||||
|
||||
> **Store and query RDF triples with SPARQL support and semantic reasoning using industry-standard triple stores.**
|
||||
> **Store and query RDF triplets with SPARQL support and semantic reasoning using industry-standard triplet stores.**
|
||||
|
||||
---
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
---
|
||||
|
||||
Store subject-predicate-object triples in W3C-compliant RDF format
|
||||
Store subject-predicate-object triplets in W3C-compliant RDF format
|
||||
|
||||
- :material-code-braces:{ .lg .middle } **SPARQL Queries**
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
|
||||
---
|
||||
|
||||
Query across multiple triple stores with SPARQL federation
|
||||
Query across multiple triplet stores with SPARQL federation
|
||||
|
||||
- :material-upload-multiple:{ .lg .middle } **Bulk Loading**
|
||||
|
||||
@@ -89,29 +89,29 @@
|
||||
|
||||
## Main Classes
|
||||
|
||||
### TripleManager
|
||||
### TripletManager
|
||||
|
||||
Main coordinator for triple store operations across multiple backends.
|
||||
Main coordinator for triplet store operations across multiple backends.
|
||||
|
||||
**Methods:**
|
||||
|
||||
| Method | Description | Algorithm |
|
||||
|--------|-------------|-----------|
|
||||
| `register_store(id, backend, endpoint)` | Register triple store | Store registration |
|
||||
| `add_triple(triple, store_id)` | Add single triple | Index insertion |
|
||||
| `add_triples(triples, store_id)` | Batch add triples | Bulk index insertion |
|
||||
| `register_store(store_id, backend, endpoint)` | Register triplet store | Store registration |
|
||||
| `add_triple(triple, store_id)` | Add single triplet | Index insertion |
|
||||
| `add_triples(triples, store_id)` | Batch add triplets | Bulk index insertion |
|
||||
| `query(sparql, store_id)` | Execute SPARQL query | Query optimization + execution |
|
||||
| `delete(pattern, store_id)` | Delete matching triples | Pattern matching + deletion |
|
||||
| `delete(pattern, store_id)` | Delete matching triplets | Pattern matching + deletion |
|
||||
| `bulk_load(file_path, format, store_id)` | Bulk load from file | Streaming parser + batch insert |
|
||||
| `get_stats(store_id)` | Get store statistics | Statistics collection |
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleManager
|
||||
from semantica.triplet_store import TripletManager
|
||||
|
||||
# Initialize manager
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
|
||||
# Register Blazegraph store
|
||||
store = manager.register_store(
|
||||
@@ -193,9 +193,9 @@ SPARQL query execution and optimization engine.
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import QueryEngine, TripleManager
|
||||
from semantica.triplet_store import QueryEngine, TripletManager
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
store = manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph/sparql")
|
||||
|
||||
engine = QueryEngine()
|
||||
@@ -270,9 +270,9 @@ High-performance bulk data loading with progress tracking.
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import BulkLoader, TripleManager
|
||||
from semantica.triplet_store import BulkLoader, TripletManager
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
store = manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph/sparql")
|
||||
|
||||
loader = BulkLoader(
|
||||
@@ -322,7 +322,7 @@ progress = loader.load_from_string(
|
||||
|
||||
#### BlazegraphAdapter
|
||||
|
||||
High-performance triple store with GPU acceleration support.
|
||||
High-performance triplet store with GPU acceleration support.
|
||||
|
||||
**Features:**
|
||||
- High-performance SPARQL query execution
|
||||
@@ -334,7 +334,7 @@ High-performance triple store with GPU acceleration support.
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import BlazegraphAdapter
|
||||
from semantica.triplet_store import BlazegraphAdapter
|
||||
|
||||
adapter = BlazegraphAdapter(
|
||||
endpoint="http://localhost:9999/blazegraph/sparql",
|
||||
@@ -376,7 +376,7 @@ results = adapter.query("""
|
||||
Full-featured RDF framework with TDB2 storage.
|
||||
|
||||
**Features:**
|
||||
- TDB2 native triple store
|
||||
- TDB2 native triplet store
|
||||
- SHACL validation
|
||||
- Inference engines (RDFS, OWL)
|
||||
- Fuseki SPARQL server
|
||||
@@ -385,7 +385,7 @@ Full-featured RDF framework with TDB2 storage.
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import JenaAdapter
|
||||
from semantica.triplet_store import JenaAdapter
|
||||
|
||||
adapter = JenaAdapter(
|
||||
tdb_directory="./tdb2_data",
|
||||
@@ -451,7 +451,7 @@ Java-based RDF framework with multiple storage backends.
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import RDF4JAdapter
|
||||
from semantica.triplet_store import RDF4JAdapter
|
||||
|
||||
adapter = RDF4JAdapter(
|
||||
server_url="http://localhost:8080/rdf4j-server",
|
||||
@@ -497,7 +497,7 @@ Enterprise-grade RDF store with SQL integration.
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.triple_store import VirtuosoAdapter
|
||||
from semantica.triplet_store import VirtuosoAdapter
|
||||
|
||||
adapter = VirtuosoAdapter(
|
||||
host="localhost",
|
||||
@@ -534,10 +534,10 @@ results = adapter.query(f"""
|
||||
|
||||
## Convenience Functions
|
||||
|
||||
Quick access to triple store operations:
|
||||
Quick access to triplet store operations:
|
||||
|
||||
```python
|
||||
from semantica.triple_store import (
|
||||
from semantica.triplet_store import (
|
||||
add_triple,
|
||||
add_triples,
|
||||
execute_query,
|
||||
@@ -579,9 +579,9 @@ export_graph(
|
||||
|
||||
## Dataclasses
|
||||
|
||||
### TripleStore
|
||||
### TripletStore
|
||||
|
||||
Configuration dataclass for triple store instances.
|
||||
Configuration dataclass for triplet store instances.
|
||||
|
||||
**Attributes:**
|
||||
|
||||
@@ -649,35 +649,35 @@ Bulk loading progress dataclass.
|
||||
|
||||
```bash
|
||||
# General settings
|
||||
export TRIPLE_STORE_DEFAULT_BACKEND=blazegraph
|
||||
export TRIPLE_STORE_BATCH_SIZE=10000
|
||||
export TRIPLE_STORE_TIMEOUT=30
|
||||
export TRIPLET_STORE_DEFAULT_BACKEND=blazegraph
|
||||
export TRIPLET_STORE_BATCH_SIZE=10000
|
||||
export TRIPLET_STORE_TIMEOUT=30
|
||||
|
||||
# Blazegraph settings
|
||||
export TRIPLE_STORE_BLAZEGRAPH_ENDPOINT=http://localhost:9999/blazegraph/sparql
|
||||
export TRIPLE_STORE_BLAZEGRAPH_NAMESPACE=kb
|
||||
export TRIPLET_STORE_BLAZEGRAPH_ENDPOINT=http://localhost:9999/blazegraph/sparql
|
||||
export TRIPLET_STORE_BLAZEGRAPH_NAMESPACE=kb
|
||||
|
||||
# Jena settings
|
||||
export TRIPLE_STORE_JENA_TDB_DIRECTORY=./tdb2_data
|
||||
export TRIPLE_STORE_JENA_INFERENCE=rdfs
|
||||
export TRIPLET_STORE_JENA_TDB_DIRECTORY=./tdb2_data
|
||||
export TRIPLET_STORE_JENA_INFERENCE=rdfs
|
||||
|
||||
# RDF4J settings
|
||||
export TRIPLE_STORE_RDF4J_SERVER_URL=http://localhost:8080/rdf4j-server
|
||||
export TRIPLE_STORE_RDF4J_REPOSITORY_ID=my_repo
|
||||
export TRIPLET_STORE_RDF4J_SERVER_URL=http://localhost:8080/rdf4j-server
|
||||
export TRIPLET_STORE_RDF4J_REPOSITORY_ID=my_repo
|
||||
|
||||
# Virtuoso settings
|
||||
export TRIPLE_STORE_VIRTUOSO_HOST=localhost
|
||||
export TRIPLE_STORE_VIRTUOSO_PORT=1111
|
||||
export TRIPLE_STORE_VIRTUOSO_USER=dba
|
||||
export TRIPLE_STORE_VIRTUOSO_PASSWORD=dba
|
||||
export TRIPLET_STORE_VIRTUOSO_HOST=localhost
|
||||
export TRIPLET_STORE_VIRTUOSO_PORT=1111
|
||||
export TRIPLET_STORE_VIRTUOSO_USER=dba
|
||||
export TRIPLET_STORE_VIRTUOSO_PASSWORD=dba
|
||||
```
|
||||
|
||||
### YAML Configuration
|
||||
|
||||
```yaml
|
||||
# config.yaml - Triple Store Configuration
|
||||
# config.yaml - Triplet Store Configuration
|
||||
|
||||
triple_store:
|
||||
triplet_store:
|
||||
backend: blazegraph # blazegraph, jena, rdf4j, virtuoso
|
||||
batch_size: 10000
|
||||
timeout: 30
|
||||
@@ -1,6 +1,6 @@
|
||||
# Vector Store
|
||||
|
||||
> **Unified vector database interface supporting FAISS, Pinecone, Weaviate, Qdrant, and Milvus with Hybrid Search.**
|
||||
> **Unified vector database interface supporting FAISS, Weaviate, Qdrant, and Milvus with Hybrid Search.**
|
||||
|
||||
---
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
---
|
||||
|
||||
Seamlessly switch between FAISS (Local), Pinecone, Weaviate, Qdrant, and Milvus
|
||||
Seamlessly switch between FAISS (Local), Weaviate, Qdrant, and Milvus
|
||||
|
||||
- :material-magnify-plus:{ .lg .middle } **Hybrid Search**
|
||||
|
||||
@@ -230,7 +230,6 @@ results = searcher.search(
|
||||
|
||||
Backend-specific implementations:
|
||||
- `FAISSAdapter`: Local, in-memory/disk.
|
||||
- `PineconeAdapter`: Managed cloud service.
|
||||
- `WeaviateAdapter`: Schema-aware vector DB.
|
||||
- `QdrantAdapter`: Rust-based high-performance DB.
|
||||
- `MilvusAdapter`: Scalable cloud-native DB.
|
||||
@@ -265,41 +264,6 @@ query = np.random.rand(768).astype('float32')
|
||||
distances, indices = adapter.search(index, query, k=10)
|
||||
```
|
||||
|
||||
#### PineconeAdapter
|
||||
|
||||
Managed cloud vector database.
|
||||
|
||||
**Helper Classes:**
|
||||
- `PineconeIndex`: Index management
|
||||
- `PineconeQuery`: Query operations
|
||||
- `PineconeMetadata`: Metadata handling
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
from semantica.vector_store import PineconeAdapter
|
||||
|
||||
adapter = PineconeAdapter(api_key="your-key", environment="us-west1-gcp")
|
||||
adapter.connect()
|
||||
|
||||
# Create index
|
||||
index = adapter.create_index("my-index", dimension=768, metric="cosine")
|
||||
|
||||
# Upsert with metadata
|
||||
adapter.upsert_vectors(
|
||||
vectors=[[0.1, 0.2, ...], ...],
|
||||
ids=["vec_1", "vec_2"],
|
||||
metadata=[{"category": "news"}, ...]
|
||||
)
|
||||
|
||||
# Query with filter
|
||||
results = adapter.query_vectors(
|
||||
query_vector=[0.1, 0.2, ...],
|
||||
top_k=10,
|
||||
filter={"category": {"$eq": "news"}}
|
||||
)
|
||||
```
|
||||
|
||||
#### WeaviateAdapter
|
||||
|
||||
Schema-aware vector database with GraphQL.
|
||||
@@ -716,25 +680,23 @@ print(f"Available methods: {methods}")
|
||||
### Environment Variables
|
||||
|
||||
```bash
|
||||
export VECTOR_STORE_BACKEND=pinecone
|
||||
export PINECONE_API_KEY=sk-...
|
||||
export PINECONE_ENV=us-west1-gcp
|
||||
export VECTOR_STORE_BACKEND=weaviate
|
||||
export WEAVIATE_URL=http://localhost:8080
|
||||
```
|
||||
|
||||
### YAML Configuration
|
||||
|
||||
```yaml
|
||||
vector_store:
|
||||
backend: faiss # or pinecone, weaviate, etc.
|
||||
backend: faiss # or weaviate, qdrant, milvus
|
||||
dimension: 1536
|
||||
metric: cosine
|
||||
|
||||
faiss:
|
||||
index_type: HNSW
|
||||
|
||||
pinecone:
|
||||
environment: us-west1-gcp
|
||||
index_name: my-index
|
||||
weaviate:
|
||||
url: http://localhost:8080
|
||||
```
|
||||
|
||||
---
|
||||
@@ -777,7 +739,7 @@ print(f"Context: {context}")
|
||||
**Solution**: Ensure your embedding model dimension (e.g., 1536 for OpenAI) matches the VectorStore dimension.
|
||||
|
||||
**Issue**: FAISS index not saved.
|
||||
**Solution**: Call `store.save("index.faiss")` explicitly for local FAISS indices, or use a persistent backend like Pinecone/Qdrant.
|
||||
**Solution**: Call `store.save("index.faiss")` explicitly for local FAISS indices, or use a persistent backend like Weaviate/Qdrant.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+1
-1
@@ -123,7 +123,7 @@ nav:
|
||||
- Seed: reference/seed.md
|
||||
- Semantic Extract: reference/semantic_extract.md
|
||||
- Split: reference/split.md
|
||||
- Triple Store: reference/triple_store.md
|
||||
- Triplet Store: reference/triplet_store.md
|
||||
- Utils: reference/utils.md
|
||||
- Vector Store: reference/vector_store.md
|
||||
- Visualization: reference/visualization.md
|
||||
|
||||
@@ -61,7 +61,6 @@ dependencies = [
|
||||
"librosa>=0.9.0",
|
||||
"opencv-python>=4.6.0",
|
||||
"faiss-cpu>=1.7.0",
|
||||
"pinecone-client>=2.2.0",
|
||||
"weaviate-client>=3.15.0",
|
||||
"qdrant-client>=1.3.0",
|
||||
"neo4j>=5.0.0",
|
||||
|
||||
@@ -84,7 +84,7 @@ class _SemanticaModules:
|
||||
self._normalize = None
|
||||
self._export = None
|
||||
self._vector_store = None
|
||||
self._triple_store = None
|
||||
self._triplet_store = None
|
||||
self._graph_store = None
|
||||
self._ontology = None
|
||||
self._evals = None
|
||||
@@ -160,11 +160,11 @@ class _SemanticaModules:
|
||||
return self._vector_store
|
||||
|
||||
@property
|
||||
def triple_store(self):
|
||||
"""Access triple store module."""
|
||||
if self._triple_store is None:
|
||||
self._triple_store = _ModuleProxy("triple_store")
|
||||
return self._triple_store
|
||||
def triplet_store(self):
|
||||
"""Access triplet store module."""
|
||||
if self._triplet_store is None:
|
||||
self._triplet_store = _ModuleProxy("triplet_store")
|
||||
return self._triplet_store
|
||||
|
||||
@property
|
||||
def graph_store(self):
|
||||
@@ -289,7 +289,7 @@ def __getattr__(name: str):
|
||||
"normalize",
|
||||
"export",
|
||||
"vector_store",
|
||||
"triple_store",
|
||||
"triplet_store",
|
||||
"graph_store",
|
||||
"ontology",
|
||||
"evals",
|
||||
|
||||
@@ -302,7 +302,6 @@ class Semantica:
|
||||
try:
|
||||
self.logger.info("Executing processing pipeline")
|
||||
|
||||
# Track pipeline execution
|
||||
pipeline_tracking_id = self.progress_tracker.start_tracking(
|
||||
file=str(data) if isinstance(data, (str, Path)) else None,
|
||||
module="pipeline",
|
||||
@@ -310,23 +309,39 @@ class Semantica:
|
||||
message="Executing pipeline",
|
||||
)
|
||||
|
||||
# Validate pipeline
|
||||
if isinstance(pipeline, dict):
|
||||
pipeline = self._create_pipeline_from_dict(pipeline)
|
||||
|
||||
# Validate pipeline object
|
||||
if not hasattr(pipeline, "execute"):
|
||||
raise ProcessingError("Pipeline must have execute() method")
|
||||
execution_engine = None
|
||||
execution_result = None
|
||||
|
||||
try:
|
||||
from ..pipeline import ExecutionEngine, Pipeline
|
||||
|
||||
if isinstance(pipeline, Pipeline):
|
||||
execution_engine = ExecutionEngine()
|
||||
except ImportError:
|
||||
execution_engine = None
|
||||
|
||||
if execution_engine is None and not hasattr(pipeline, "execute"):
|
||||
raise ProcessingError(
|
||||
"Pipeline must be a Pipeline object or have execute() method"
|
||||
)
|
||||
|
||||
# Allocate resources
|
||||
resources = self._allocate_resources(pipeline)
|
||||
|
||||
try:
|
||||
# Execute pipeline
|
||||
result = pipeline.execute(data)
|
||||
|
||||
# Collect metrics
|
||||
metrics = self._collect_metrics(pipeline)
|
||||
if execution_engine is not None:
|
||||
execution_result = execution_engine.execute_pipeline(
|
||||
pipeline, data
|
||||
)
|
||||
success = execution_result.success
|
||||
output = execution_result.output
|
||||
metrics = execution_result.metrics
|
||||
else:
|
||||
output = pipeline.execute(data)
|
||||
metrics = self._collect_metrics(pipeline)
|
||||
success = True
|
||||
|
||||
if pipeline_tracking_id:
|
||||
self.progress_tracker.stop_tracking(
|
||||
@@ -334,8 +349,8 @@ class Semantica:
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"output": result,
|
||||
"success": success,
|
||||
"output": output,
|
||||
"metrics": metrics,
|
||||
"metadata": {
|
||||
"pipeline": str(pipeline),
|
||||
@@ -344,7 +359,6 @@ class Semantica:
|
||||
}
|
||||
|
||||
finally:
|
||||
# Release resources
|
||||
self._release_resources(resources)
|
||||
|
||||
except Exception as e:
|
||||
@@ -567,13 +581,37 @@ class Semantica:
|
||||
Pipeline object or configuration dict (if pipeline module not available)
|
||||
"""
|
||||
try:
|
||||
# Try to use PipelineBuilder if available
|
||||
from ..pipeline import PipelineBuilder
|
||||
|
||||
pipeline_builder = PipelineBuilder()
|
||||
return pipeline_builder.build_from_config(pipeline_config)
|
||||
|
||||
if not pipeline_config:
|
||||
pipeline_builder.add_step("default_step", "default")
|
||||
return pipeline_builder.build("default_pipeline")
|
||||
|
||||
steps_config = pipeline_config.get("steps")
|
||||
|
||||
if isinstance(steps_config, list) and steps_config and isinstance(
|
||||
steps_config[0], str
|
||||
):
|
||||
converted_steps = [
|
||||
{"name": name, "type": name, "config": {}}
|
||||
for name in steps_config
|
||||
]
|
||||
normalized_config: Dict[str, Any] = {
|
||||
"name": pipeline_config.get("name", "default_pipeline"),
|
||||
"steps": converted_steps,
|
||||
}
|
||||
if "parallelism" in pipeline_config:
|
||||
normalized_config["parallelism"] = pipeline_config["parallelism"]
|
||||
return pipeline_builder.build_pipeline(normalized_config)
|
||||
|
||||
if "steps" in pipeline_config:
|
||||
return pipeline_builder.build_pipeline(pipeline_config)
|
||||
|
||||
pipeline_builder.add_step("default_step", "default")
|
||||
return pipeline_builder.build("default_pipeline")
|
||||
except ImportError:
|
||||
# Fallback: return config as-is if pipeline module not available
|
||||
self.logger.debug("Pipeline module not available, using config directly")
|
||||
return pipeline_config
|
||||
|
||||
|
||||
@@ -447,14 +447,6 @@ from semantica.embeddings import VectorEmbeddingManager
|
||||
|
||||
manager = VectorEmbeddingManager()
|
||||
|
||||
# Prepare for Pinecone
|
||||
pinecone_data = manager.prepare_for_vector_db(
|
||||
embeddings,
|
||||
metadata=metadata,
|
||||
backend="pinecone",
|
||||
namespace="my_namespace"
|
||||
)
|
||||
|
||||
# Prepare for Weaviate
|
||||
weaviate_data = manager.prepare_for_vector_db(
|
||||
embeddings,
|
||||
@@ -486,9 +478,9 @@ from semantica.embeddings import VectorEmbeddingManager
|
||||
manager = VectorEmbeddingManager()
|
||||
|
||||
# Validate dimensions for specific backend
|
||||
is_valid = manager.validate_dimensions(embeddings, backend="pinecone")
|
||||
is_valid = manager.validate_dimensions(embeddings, backend="weaviate")
|
||||
if is_valid:
|
||||
print("Embeddings meet Pinecone requirements")
|
||||
print("Embeddings meet Weaviate requirements")
|
||||
else:
|
||||
print("Embeddings do not meet requirements")
|
||||
```
|
||||
|
||||
@@ -9,7 +9,7 @@ Key Features:
|
||||
- Validate embedding dimensions for different backends
|
||||
- Normalize embeddings for vector DB requirements
|
||||
- Create metadata compatible with vector DBs
|
||||
- Integration helpers for FAISS, Pinecone, Weaviate, Qdrant, Milvus
|
||||
- Integration helpers for FAISS, Weaviate, Qdrant, Milvus
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.embeddings import VectorEmbeddingManager
|
||||
@@ -36,7 +36,6 @@ class VectorEmbeddingManager:
|
||||
|
||||
Supported Backends:
|
||||
- FAISS: Local vector storage
|
||||
- Pinecone: Cloud vector database
|
||||
- Weaviate: GraphQL-based vector database
|
||||
- Qdrant: Vector similarity search engine
|
||||
- Milvus: Open-source vector database
|
||||
@@ -50,7 +49,7 @@ class VectorEmbeddingManager:
|
||||
... backend="faiss"
|
||||
... )
|
||||
>>> # Validate dimensions
|
||||
>>> is_valid = manager.validate_dimensions(embeddings, backend="pinecone")
|
||||
>>> is_valid = manager.validate_dimensions(embeddings, backend="weaviate")
|
||||
"""
|
||||
|
||||
def __init__(self, embedding_generator: Optional[EmbeddingGenerator] = None):
|
||||
@@ -67,7 +66,6 @@ class VectorEmbeddingManager:
|
||||
# Backend-specific dimension requirements
|
||||
self.backend_requirements = {
|
||||
"faiss": {"min_dim": 1, "max_dim": None, "dtype": np.float32},
|
||||
"pinecone": {"min_dim": 1, "max_dim": 20000, "dtype": np.float32},
|
||||
"weaviate": {"min_dim": 1, "max_dim": None, "dtype": np.float32},
|
||||
"qdrant": {"min_dim": 1, "max_dim": None, "dtype": np.float32},
|
||||
"milvus": {"min_dim": 1, "max_dim": 32768, "dtype": np.float32},
|
||||
@@ -90,7 +88,7 @@ class VectorEmbeddingManager:
|
||||
Args:
|
||||
embeddings: Embeddings array (n_samples, embedding_dim) or (embedding_dim,)
|
||||
metadata: Optional list of metadata dictionaries (one per embedding)
|
||||
backend: Vector DB backend ("faiss", "pinecone", "weaviate", "qdrant", "milvus")
|
||||
backend: Vector DB backend ("faiss", "weaviate", "qdrant", "milvus")
|
||||
normalize: Whether to normalize embeddings (default: True)
|
||||
**options: Additional backend-specific options
|
||||
|
||||
@@ -108,7 +106,7 @@ class VectorEmbeddingManager:
|
||||
>>> embeddings = np.random.rand(10, 384).astype(np.float32)
|
||||
>>> metadata = [{"text": f"doc_{i}"} for i in range(10)]
|
||||
>>> result = manager.prepare_for_vector_db(
|
||||
... embeddings, metadata, backend="pinecone"
|
||||
... embeddings, metadata, backend="weaviate"
|
||||
... )
|
||||
"""
|
||||
if backend.lower() not in self.backend_requirements:
|
||||
@@ -228,7 +226,7 @@ class VectorEmbeddingManager:
|
||||
bool: True if dimensions are valid, False otherwise
|
||||
|
||||
Example:
|
||||
>>> is_valid = manager.validate_dimensions(embeddings, backend="pinecone")
|
||||
>>> is_valid = manager.validate_dimensions(embeddings, backend="weaviate")
|
||||
"""
|
||||
if backend.lower() not in self.backend_requirements:
|
||||
self.logger.warning(f"Unknown backend: {backend}, skipping validation")
|
||||
@@ -312,7 +310,7 @@ class VectorEmbeddingManager:
|
||||
|
||||
Example:
|
||||
>>> metadata = [{"text": "doc1", "category": "science"}]
|
||||
>>> formatted = manager.create_metadata(metadata, backend="pinecone")
|
||||
>>> formatted = manager.create_metadata(metadata, backend="weaviate")
|
||||
"""
|
||||
formatted = []
|
||||
|
||||
@@ -321,16 +319,7 @@ class VectorEmbeddingManager:
|
||||
formatted_meta = meta.copy()
|
||||
|
||||
# Backend-specific formatting
|
||||
if backend.lower() == "pinecone":
|
||||
# Pinecone has specific metadata requirements
|
||||
# Remove None values and ensure types are compatible
|
||||
formatted_meta = {
|
||||
k: v
|
||||
for k, v in formatted_meta.items()
|
||||
if v is not None
|
||||
and isinstance(v, (str, int, float, bool, list))
|
||||
}
|
||||
elif backend.lower() == "weaviate":
|
||||
if backend.lower() == "weaviate":
|
||||
# Weaviate uses specific property types
|
||||
# Ensure values are compatible
|
||||
formatted_meta = {
|
||||
@@ -374,8 +363,6 @@ class VectorEmbeddingManager:
|
||||
# Add backend-specific details
|
||||
if backend.lower() == "faiss":
|
||||
info["index_type"] = options.get("index_type", "flat")
|
||||
elif backend.lower() == "pinecone":
|
||||
info["namespace"] = options.get("namespace", "default")
|
||||
elif backend.lower() == "weaviate":
|
||||
info["class_name"] = options.get("class_name", "Document")
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ OWL Export:
|
||||
|
||||
Vector Export:
|
||||
- Vector Serialization: Multiple format support (JSON, NumPy, Binary, FAISS)
|
||||
- Vector Store Integration: Format conversion for Pinecone, Weaviate, Qdrant, FAISS
|
||||
- Vector Store Integration: Format conversion for Weaviate, Qdrant, FAISS
|
||||
- Metadata Association: Vector-to-metadata mapping and serialization
|
||||
- Batch Export: Efficient batch vector export processing
|
||||
- Multi-dimensional Support: Variable dimension vector handling
|
||||
|
||||
@@ -110,7 +110,7 @@ OWL Export:
|
||||
|
||||
Vector Export:
|
||||
- Vector Serialization: Multiple format support (JSON, NumPy, Binary, FAISS)
|
||||
- Vector Store Integration: Format conversion for Pinecone, Weaviate, Qdrant, FAISS
|
||||
- Vector Store Integration: Format conversion for Weaviate, Qdrant, FAISS
|
||||
- Metadata Association: Vector-to-metadata mapping and serialization
|
||||
- Batch Export: Efficient batch vector export processing
|
||||
- Multi-dimensional Support: Variable dimension vector handling
|
||||
|
||||
@@ -7,7 +7,7 @@ embedding systems.
|
||||
|
||||
Key Features:
|
||||
- Multiple vector format export (JSON, NumPy, Binary, FAISS)
|
||||
- Vector store integration (Pinecone, Weaviate, Qdrant, FAISS)
|
||||
- Vector store integration (Weaviate, Qdrant, FAISS)
|
||||
- Metadata and document association
|
||||
- Batch vector export
|
||||
- Multi-dimensional vector support
|
||||
@@ -16,7 +16,7 @@ Example Usage:
|
||||
>>> from semantica.export import VectorExporter
|
||||
>>> exporter = VectorExporter(format="json", include_metadata=True)
|
||||
>>> exporter.export(vectors, "vectors.json")
|
||||
>>> exporter.export_for_vector_store(vectors, "pinecone.json", vector_store_type="pinecone")
|
||||
>>> exporter.export_for_vector_store(vectors, "weaviate.json", vector_store_type="weaviate")
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
@@ -43,7 +43,7 @@ class VectorExporter:
|
||||
|
||||
Features:
|
||||
- Multiple vector format export (JSON, NumPy, Binary, FAISS)
|
||||
- Vector store integration (Pinecone, Weaviate, Qdrant, FAISS)
|
||||
- Vector store integration (Weaviate, Qdrant, FAISS)
|
||||
- Metadata and document association
|
||||
- Batch vector export
|
||||
- Multi-dimensional vector support
|
||||
@@ -471,7 +471,7 @@ class VectorExporter:
|
||||
self,
|
||||
vectors: List[Dict[str, Any]],
|
||||
file_path: Union[str, Path],
|
||||
vector_store_type: str = "pinecone",
|
||||
vector_store_type: str = "weaviate",
|
||||
**options,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -480,12 +480,10 @@ class VectorExporter:
|
||||
Args:
|
||||
vectors: List of vector dictionaries
|
||||
file_path: Output file path
|
||||
vector_store_type: Vector store type ('pinecone', 'weaviate', 'qdrant', 'faiss')
|
||||
vector_store_type: Vector store type ('weaviate', 'qdrant', 'faiss')
|
||||
**options: Additional options
|
||||
"""
|
||||
if vector_store_type == "pinecone":
|
||||
self._export_pinecone_format(vectors, file_path, **options)
|
||||
elif vector_store_type == "weaviate":
|
||||
if vector_store_type == "weaviate":
|
||||
self._export_weaviate_format(vectors, file_path, **options)
|
||||
elif vector_store_type == "qdrant":
|
||||
self._export_qdrant_format(vectors, file_path, **options)
|
||||
@@ -495,27 +493,6 @@ class VectorExporter:
|
||||
# Default to JSON
|
||||
self._export_json(vectors, Path(file_path), {}, **options)
|
||||
|
||||
def _export_pinecone_format(
|
||||
self, vectors: List[Dict[str, Any]], file_path: Path, **options
|
||||
) -> None:
|
||||
"""Export in Pinecone format."""
|
||||
pinecone_data = []
|
||||
|
||||
for vec_data in vectors:
|
||||
vector_id = vec_data.get("id") or vec_data.get("vector_id", "")
|
||||
vector = vec_data.get("vector") or vec_data.get("embedding", [])
|
||||
metadata = vec_data.get("metadata", {})
|
||||
|
||||
if "text" in vec_data and self.include_text:
|
||||
metadata["text"] = vec_data["text"]
|
||||
|
||||
pinecone_data.append(
|
||||
{"id": vector_id, "values": vector, "metadata": metadata}
|
||||
)
|
||||
|
||||
export_data = {"vectors": pinecone_data}
|
||||
write_json_file(export_data, file_path, indent=2)
|
||||
|
||||
def _export_weaviate_format(
|
||||
self, vectors: List[Dict[str, Any]], file_path: Path, **options
|
||||
) -> None:
|
||||
|
||||
@@ -148,7 +148,7 @@ class PipelineTemplateManager:
|
||||
{
|
||||
"name": "store_vectors",
|
||||
"type": "store_vectors",
|
||||
"config": {"store": "pinecone"},
|
||||
"config": {"store": "weaviate"},
|
||||
"dependencies": ["embed"],
|
||||
},
|
||||
],
|
||||
|
||||
@@ -632,7 +632,7 @@ builder = template_manager.create_pipeline_from_template(
|
||||
"rag_pipeline",
|
||||
chunk={"chunk_size": 512},
|
||||
embed={"model": "text-embedding-3-large"},
|
||||
store_vectors={"store": "pinecone"}
|
||||
store_vectors={"store": "weaviate"}
|
||||
)
|
||||
|
||||
pipeline = builder.build()
|
||||
@@ -1124,7 +1124,8 @@ builder = template_manager.create_pipeline_from_template(
|
||||
ingest={"source": "./documents"},
|
||||
chunk={"chunk_size": 512, "overlap": 50},
|
||||
embed={"model": "text-embedding-3-large", "batch_size": 32},
|
||||
store_vectors={"store": "pinecone", "index_name": "documents"}
|
||||
# Step-specific overrides
|
||||
store_vectors={"store": "weaviate", "index_name": "documents"}
|
||||
)
|
||||
|
||||
pipeline = builder.build()
|
||||
|
||||
@@ -45,7 +45,7 @@ print(f"Inferred {len(results)} new facts")
|
||||
from semantica.reasoning import SPARQLReasoner
|
||||
|
||||
# Create SPARQL reasoner
|
||||
reasoner = SPARQLReasoner(triple_store=kg)
|
||||
reasoner = SPARQLReasoner(triplet_store=kg)
|
||||
|
||||
# Execute query
|
||||
query = "SELECT ?s ?p ?o WHERE { ?s ?p ?o }"
|
||||
@@ -190,7 +190,7 @@ results = engine.forward_chain()
|
||||
from semantica.reasoning import SPARQLReasoner
|
||||
|
||||
# Create reasoner with knowledge graph
|
||||
reasoner = SPARQLReasoner(triple_store=kg)
|
||||
reasoner = SPARQLReasoner(triplet_store=kg)
|
||||
|
||||
# Execute SPARQL query
|
||||
query = """
|
||||
@@ -212,7 +212,7 @@ for binding in result.bindings:
|
||||
```python
|
||||
from semantica.reasoning import SPARQLReasoner
|
||||
|
||||
reasoner = SPARQLReasoner(triple_store=kg, enable_inference=True)
|
||||
reasoner = SPARQLReasoner(triplet_store=kg, enable_inference=True)
|
||||
|
||||
# Add inference rule
|
||||
reasoner.add_inference_rule("IF ?x :type :Company THEN ?x :type :Organization")
|
||||
@@ -234,7 +234,7 @@ result = reasoner.execute_query(query)
|
||||
from semantica.reasoning import SPARQLReasoner
|
||||
|
||||
reasoner = SPARQLReasoner(
|
||||
triple_store=kg,
|
||||
triplet_store=kg,
|
||||
enable_inference=True,
|
||||
inference_rules=["rdfs:subClassOf", "rdfs:subPropertyOf"]
|
||||
)
|
||||
@@ -1165,7 +1165,7 @@ engine = InferenceEngine(
|
||||
|
||||
# Configure SPARQL reasoner
|
||||
reasoner = SPARQLReasoner(
|
||||
triple_store=kg,
|
||||
triplet_store=kg,
|
||||
enable_inference=True,
|
||||
query_cache_size=1000
|
||||
)
|
||||
@@ -1234,7 +1234,7 @@ for result in results:
|
||||
print(f"Explanation: {explanation.natural_language}")
|
||||
|
||||
# 6. Query with SPARQL reasoning
|
||||
sparql_reasoner = SPARQLReasoner(triple_store=kg, enable_inference=True)
|
||||
sparql_reasoner = SPARQLReasoner(triplet_store=kg, enable_inference=True)
|
||||
query_result = sparql_reasoner.execute_query("SELECT ?x WHERE { ?x :type :Employee }")
|
||||
```
|
||||
|
||||
@@ -1332,7 +1332,7 @@ from semantica.kg import build
|
||||
kg = build(sources=[...])
|
||||
|
||||
# Create SPARQL reasoner with KG
|
||||
reasoner = SPARQLReasoner(triple_store=kg, enable_inference=True)
|
||||
reasoner = SPARQLReasoner(triplet_store=kg, enable_inference=True)
|
||||
|
||||
# Add inference rules
|
||||
reasoner.add_inference_rule("IF ?x :type :Company THEN ?x :type :Organization")
|
||||
|
||||
@@ -12,7 +12,7 @@ Key Features:
|
||||
- Query expansion
|
||||
- Performance optimization
|
||||
- Error handling and recovery
|
||||
- Triple store integration
|
||||
- Triplet store integration
|
||||
|
||||
Main Classes:
|
||||
- SPARQLReasoner: SPARQL-based reasoning engine
|
||||
@@ -67,7 +67,7 @@ class SPARQLReasoner:
|
||||
Args:
|
||||
config: Configuration dictionary
|
||||
**kwargs: Additional configuration options:
|
||||
- triple_store: Triple store connection
|
||||
- triplet_store: Triplet store connection
|
||||
- enable_inference: Enable inference rules
|
||||
"""
|
||||
self.logger = get_logger("sparql_reasoner")
|
||||
@@ -78,7 +78,7 @@ class SPARQLReasoner:
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.rule_manager = RuleManager(**self.config)
|
||||
self.triple_store = self.config.get("triple_store")
|
||||
self.triplet_store = self.config.get("triplet_store")
|
||||
self.enable_inference = self.config.get("enable_inference", True)
|
||||
|
||||
self.query_cache: Dict[str, Any] = {}
|
||||
@@ -356,12 +356,12 @@ class SPARQLReasoner:
|
||||
)
|
||||
expanded_query = self.expand_query(query, **options)
|
||||
|
||||
# Execute query (if triple store available)
|
||||
# Execute query (if triplet store available)
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Executing query..."
|
||||
)
|
||||
if self.triple_store:
|
||||
# This would call the triple store's query method
|
||||
if self.triplet_store:
|
||||
# This would call the triplet store's query method
|
||||
# For now, return empty result
|
||||
result = SPARQLQueryResult(bindings=[], variables=[])
|
||||
else:
|
||||
|
||||
@@ -311,6 +311,10 @@ def extract_entities_llm(
|
||||
text: str, provider: str = "openai", model: Optional[str] = None, **kwargs
|
||||
) -> List[Entity]:
|
||||
"""LLM-based entity extraction."""
|
||||
# Support llm_model parameter to disambiguate from ML model
|
||||
if "llm_model" in kwargs:
|
||||
model = kwargs.pop("llm_model")
|
||||
|
||||
llm = create_provider(provider, model=model, **kwargs)
|
||||
|
||||
if not llm.is_available():
|
||||
@@ -818,6 +822,7 @@ def get_entity_method(method_name: str):
|
||||
"regex": extract_entities_regex,
|
||||
"rules": extract_entities_rules,
|
||||
"ml": extract_entities_ml,
|
||||
"spacy": extract_entities_ml, # Alias for ml
|
||||
"huggingface": extract_entities_huggingface,
|
||||
"llm": extract_entities_llm,
|
||||
}
|
||||
@@ -844,6 +849,8 @@ def get_relation_method(method_name: str):
|
||||
"regex": extract_relations_regex,
|
||||
"cooccurrence": extract_relations_cooccurrence,
|
||||
"dependency": extract_relations_dependency,
|
||||
"ml": extract_relations_dependency, # Alias for dependency
|
||||
"spacy": extract_relations_dependency, # Alias for dependency
|
||||
"huggingface": extract_relations_huggingface,
|
||||
"llm": extract_relations_llm,
|
||||
}
|
||||
|
||||
@@ -101,6 +101,7 @@ class NamedEntityRecognizer:
|
||||
# Use NERExtractor for actual extraction
|
||||
ner_config = self.config.get("ner", {})
|
||||
ner_config["confidence_threshold"] = confidence_threshold
|
||||
ner_config["min_confidence"] = confidence_threshold
|
||||
ner_config["merge_overlapping"] = merge_overlapping
|
||||
if method is not None:
|
||||
ner_config["method"] = method
|
||||
|
||||
@@ -142,6 +142,19 @@ class NERExtractor:
|
||||
f"spaCy model {self.model_name} not found. ML method will fallback."
|
||||
)
|
||||
|
||||
def extract(self, text: str, **kwargs) -> List[Entity]:
|
||||
"""
|
||||
Alias for extract_entities.
|
||||
|
||||
Args:
|
||||
text: Input text
|
||||
**kwargs: Extraction options
|
||||
|
||||
Returns:
|
||||
list: List of extracted entities
|
||||
"""
|
||||
return self.extract_entities(text, **kwargs)
|
||||
|
||||
def extract_entities(self, text: str, **options) -> List[Entity]:
|
||||
"""
|
||||
Extract named entities from text.
|
||||
|
||||
@@ -155,6 +155,20 @@ class RelationExtractor:
|
||||
}
|
||||
|
||||
|
||||
def extract(self, text: str, entities: List[Entity], **kwargs) -> List[Relation]:
|
||||
"""
|
||||
Alias for extract_relations.
|
||||
|
||||
Args:
|
||||
text: Input text
|
||||
entities: List of entities in the text
|
||||
**kwargs: Extraction options
|
||||
|
||||
Returns:
|
||||
list: List of extracted relations
|
||||
"""
|
||||
return self.extract_relations(text, entities, **kwargs)
|
||||
|
||||
def extract_relations(
|
||||
self, text: str, entities: List[Entity], **options
|
||||
) -> List[Relation]:
|
||||
|
||||
@@ -12,10 +12,9 @@ This comprehensive guide demonstrates how to use the semantic extraction module
|
||||
6. [Coreference Resolution](#coreference-resolution)
|
||||
7. [Semantic Analysis](#semantic-analysis)
|
||||
8. [Semantic Networks](#semantic-networks)
|
||||
9. [Using Methods](#using-methods)
|
||||
10. [Using Registry](#using-registry)
|
||||
11. [Configuration](#configuration)
|
||||
12. [Advanced Examples](#advanced-examples)
|
||||
9. [Using Registry](#using-registry)
|
||||
10. [Configuration](#configuration)
|
||||
11. [Advanced Examples](#advanced-examples)
|
||||
|
||||
## Basic Usage
|
||||
|
||||
@@ -31,7 +30,7 @@ print(f"Entities: {entities}")
|
||||
|
||||
# Extract relations
|
||||
rel_extractor = RelationExtractor()
|
||||
relations = rel_extractor.extract_relations(text, entities=entities)
|
||||
relations = rel_extractor.extract(text, entities=entities)
|
||||
print(f"Relations: {relations}")
|
||||
|
||||
print(f"Extracted {len(entities)} entities and {len(relations)} relations")
|
||||
@@ -55,33 +54,33 @@ for entity in entities:
|
||||
### Different Entity Extraction Methods
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract.methods import get_entity_method
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
|
||||
text = "Apple Inc. was founded by Steve Jobs in 1976."
|
||||
|
||||
# Pattern-based extraction
|
||||
pattern_method = get_entity_method("pattern")
|
||||
entities = pattern_method(text)
|
||||
extractor = NERExtractor(method="pattern")
|
||||
entities = extractor.extract(text)
|
||||
print(f"Pattern method: {len(entities)} entities")
|
||||
|
||||
# Regex-based extraction
|
||||
regex_method = get_entity_method("regex")
|
||||
entities = regex_method(text)
|
||||
extractor = NERExtractor(method="regex")
|
||||
entities = extractor.extract(text)
|
||||
print(f"Regex method: {len(entities)} entities")
|
||||
|
||||
# ML-based extraction (spaCy)
|
||||
ml_method = get_entity_method("ml")
|
||||
entities = ml_method(text)
|
||||
extractor = NERExtractor(method="ml")
|
||||
entities = extractor.extract(text)
|
||||
print(f"ML method: {len(entities)} entities")
|
||||
|
||||
# HuggingFace model extraction
|
||||
hf_method = get_entity_method("huggingface")
|
||||
entities = hf_method(text, model="dslim/bert-base-NER")
|
||||
extractor = NERExtractor(method="huggingface")
|
||||
entities = extractor.extract(text, model="dslim/bert-base-NER")
|
||||
print(f"HuggingFace method: {len(entities)} entities")
|
||||
|
||||
# LLM-based extraction
|
||||
llm_method = get_entity_method("llm")
|
||||
entities = llm_method(text, provider="openai", model="gpt-4")
|
||||
extractor = NERExtractor(method="llm")
|
||||
entities = extractor.extract(text, provider="openai", model="gpt-4")
|
||||
print(f"LLM method: {len(entities)} entities")
|
||||
```
|
||||
|
||||
@@ -90,13 +89,29 @@ print(f"LLM method: {len(entities)} entities")
|
||||
```python
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
|
||||
extractor = NERExtractor(method="ml")
|
||||
# 1. Standard ML (spaCy)
|
||||
extractor = NERExtractor(method="ml", model="en_core_web_trf")
|
||||
entities = extractor.extract(text)
|
||||
|
||||
# 2. LLM-based extraction
|
||||
extractor = NERExtractor(
|
||||
method="llm",
|
||||
provider="openai",
|
||||
model="gpt-4",
|
||||
temperature=0.0
|
||||
)
|
||||
entities = extractor.extract(text)
|
||||
|
||||
# 3. Regex with custom patterns
|
||||
extractor = NERExtractor(
|
||||
method="regex",
|
||||
patterns={"CODE": r"[A-Z]{3}-\d{3}"}
|
||||
)
|
||||
entities = extractor.extract(text)
|
||||
|
||||
for entity in entities:
|
||||
print(f"Entity: {entity.text}")
|
||||
print(f" Type: {entity.type}")
|
||||
print(f" Start: {entity.start}, End: {entity.end}")
|
||||
print(f" Type: {entity.label}")
|
||||
print(f" Confidence: {entity.confidence}")
|
||||
```
|
||||
|
||||
@@ -129,7 +144,7 @@ from semantica.semantic_extract import RelationExtractor
|
||||
extractor = RelationExtractor()
|
||||
text = "Steve Jobs founded Apple Inc. in 1976."
|
||||
|
||||
relations = extractor.extract_relations(text, entities=entities)
|
||||
relations = extractor.extract(text, entities=entities)
|
||||
|
||||
for relation in relations:
|
||||
print(f"{relation.subject} --[{relation.predicate}]--> {relation.object}")
|
||||
@@ -139,29 +154,29 @@ for relation in relations:
|
||||
### Different Relation Extraction Methods
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract.methods import get_relation_method
|
||||
from semantica.semantic_extract import RelationExtractor
|
||||
|
||||
text = "Steve Jobs founded Apple Inc."
|
||||
|
||||
# Pattern-based extraction
|
||||
pattern_method = get_relation_method("pattern")
|
||||
relations = pattern_method(text, entities=entities)
|
||||
extractor = RelationExtractor(method="pattern")
|
||||
relations = extractor.extract(text, entities=entities)
|
||||
|
||||
# Dependency parsing-based
|
||||
dependency_method = get_relation_method("dependency")
|
||||
relations = dependency_method(text, entities=entities)
|
||||
extractor = RelationExtractor(method="dependency")
|
||||
relations = extractor.extract(text, entities=entities)
|
||||
|
||||
# Co-occurrence based
|
||||
cooccurrence_method = get_relation_method("cooccurrence")
|
||||
relations = cooccurrence_method(text, entities=entities)
|
||||
extractor = RelationExtractor(method="cooccurrence")
|
||||
relations = extractor.extract(text, entities=entities)
|
||||
|
||||
# HuggingFace model
|
||||
hf_method = get_relation_method("huggingface")
|
||||
relations = hf_method(text, entities=entities, model="microsoft/DialoGPT-medium")
|
||||
extractor = RelationExtractor(method="huggingface")
|
||||
relations = extractor.extract(text, entities=entities, model="microsoft/DialoGPT-medium")
|
||||
|
||||
# LLM-based
|
||||
llm_method = get_relation_method("llm")
|
||||
relations = llm_method(text, entities=entities, provider="openai")
|
||||
extractor = RelationExtractor(method="llm")
|
||||
relations = extractor.extract(text, entities=entities, provider="openai")
|
||||
```
|
||||
|
||||
### Relation Types
|
||||
@@ -201,25 +216,25 @@ for triple in triples:
|
||||
### Different Triple Extraction Methods
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract.methods import get_triple_method
|
||||
from semantica.semantic_extract import TripleExtractor
|
||||
|
||||
text = "Apple Inc. was founded by Steve Jobs in 1976."
|
||||
|
||||
# Pattern-based
|
||||
pattern_method = get_triple_method("pattern")
|
||||
triples = pattern_method(text)
|
||||
extractor = TripleExtractor(method="pattern")
|
||||
triples = extractor.extract_triples(text)
|
||||
|
||||
# Rules-based
|
||||
rules_method = get_triple_method("rules")
|
||||
triples = rules_method(text)
|
||||
extractor = TripleExtractor(method="rules")
|
||||
triples = extractor.extract_triples(text)
|
||||
|
||||
# HuggingFace model
|
||||
hf_method = get_triple_method("huggingface")
|
||||
triples = hf_method(text, model="t5-base")
|
||||
extractor = TripleExtractor(method="huggingface")
|
||||
triples = extractor.extract_triples(text, model="t5-base")
|
||||
|
||||
# LLM-based
|
||||
llm_method = get_triple_method("llm")
|
||||
triples = llm_method(text, provider="openai", model="gpt-4")
|
||||
extractor = TripleExtractor(method="llm")
|
||||
triples = extractor.extract_triples(text, provider="openai", model="gpt-4")
|
||||
```
|
||||
|
||||
### RDF Serialization
|
||||
@@ -462,29 +477,6 @@ print(f"Node: {node.label}")
|
||||
print(f"Edge: {edge.source} --[{edge.relation}]--> {edge.target}")
|
||||
```
|
||||
|
||||
## Using Methods
|
||||
|
||||
### Getting Available Methods
|
||||
|
||||
```python
|
||||
from semantica.semantic_extract.methods import (
|
||||
get_entity_method,
|
||||
get_relation_method,
|
||||
get_triple_method
|
||||
)
|
||||
|
||||
# Get entity extraction method
|
||||
entity_method = get_entity_method("llm")
|
||||
entities = entity_method(text, provider="openai")
|
||||
|
||||
# Get relation extraction method
|
||||
relation_method = get_relation_method("dependency")
|
||||
relations = relation_method(text, entities=entities)
|
||||
|
||||
# Get triple extraction method
|
||||
triple_method = get_triple_method("pattern")
|
||||
triples = triple_method(text)
|
||||
```
|
||||
|
||||
## Using Registry
|
||||
|
||||
@@ -504,9 +496,9 @@ def custom_entity_extraction(text, **kwargs):
|
||||
method_registry.register("entity", "custom_method", custom_entity_extraction)
|
||||
|
||||
# Use custom method
|
||||
from semantica.semantic_extract.methods import get_entity_method
|
||||
custom_method = get_entity_method("custom_method")
|
||||
entities = custom_method(text)
|
||||
from semantica.semantic_extract import NERExtractor
|
||||
extractor = NERExtractor(method="custom_method")
|
||||
entities = extractor.extract(text)
|
||||
```
|
||||
|
||||
### Listing Registered Methods
|
||||
|
||||
+115
-3
@@ -160,6 +160,15 @@ try:
|
||||
except ImportError:
|
||||
SEMANTIC_EXTRACT_AVAILABLE = False
|
||||
|
||||
# Import specialized chunkers
|
||||
try:
|
||||
from .structural_chunker import StructuralChunker
|
||||
from .sliding_window_chunker import SlidingWindowChunker
|
||||
|
||||
SPECIALIZED_CHUNKERS_AVAILABLE = True
|
||||
except ImportError:
|
||||
SPECIALIZED_CHUNKERS_AVAILABLE = False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Standard Splitting Methods
|
||||
@@ -1012,9 +1021,14 @@ def split_relation_aware(
|
||||
return split_recursive(text, chunk_size=chunk_size, **kwargs)
|
||||
|
||||
try:
|
||||
# Extract entities first (required for relation extraction)
|
||||
ner_method = kwargs.get("ner_method", "ml")
|
||||
ner_extractor = NERExtractor(method=ner_method, **kwargs)
|
||||
entities = ner_extractor.extract(text)
|
||||
|
||||
# Extract relations/triples
|
||||
relation_extractor = RelationExtractor(method=relation_method, **kwargs)
|
||||
relations = relation_extractor.extract(text)
|
||||
relations = relation_extractor.extract(text, entities)
|
||||
|
||||
# Create triple boundaries (subject, relation, object must be in same chunk)
|
||||
triple_boundaries = []
|
||||
@@ -1412,13 +1426,23 @@ def split_hierarchical(
|
||||
|
||||
# Fall back to paragraph level
|
||||
if "paragraph" in levels:
|
||||
# Remove chunk_size from kwargs to avoid multiple values error
|
||||
para_kwargs = kwargs.copy()
|
||||
if "chunk_size" in para_kwargs:
|
||||
del para_kwargs["chunk_size"]
|
||||
|
||||
return split_by_paragraphs(
|
||||
text, chunk_size=chunk_sizes[0] if chunk_sizes else 1000, **kwargs
|
||||
text, chunk_size=chunk_sizes[0] if chunk_sizes else 1000, **para_kwargs
|
||||
)
|
||||
|
||||
# Fall back to sentence level
|
||||
# Remove chunk_size from kwargs to avoid multiple values error
|
||||
sent_kwargs = kwargs.copy()
|
||||
if "chunk_size" in sent_kwargs:
|
||||
del sent_kwargs["chunk_size"]
|
||||
|
||||
return split_by_sentences(
|
||||
text, chunk_size=chunk_sizes[0] if chunk_sizes else 1000, **kwargs
|
||||
text, chunk_size=chunk_sizes[0] if chunk_sizes else 1000, **sent_kwargs
|
||||
)
|
||||
|
||||
|
||||
@@ -1515,6 +1539,91 @@ def split_topic_based(
|
||||
return split_semantic_transformer(text, chunk_size=chunk_size, **kwargs)
|
||||
|
||||
|
||||
def split_structural(
|
||||
text: str,
|
||||
max_chunk_size: int = 2000,
|
||||
respect_headers: bool = True,
|
||||
respect_sections: bool = True,
|
||||
**kwargs,
|
||||
) -> List[Chunk]:
|
||||
"""
|
||||
Structure-aware chunking respecting document hierarchy.
|
||||
|
||||
Args:
|
||||
text: Input text
|
||||
max_chunk_size: Maximum chunk size
|
||||
respect_headers: Whether to respect heading hierarchy
|
||||
respect_sections: Whether to respect section boundaries
|
||||
**kwargs: Additional options
|
||||
|
||||
Returns:
|
||||
List of chunks
|
||||
"""
|
||||
if not SPECIALIZED_CHUNKERS_AVAILABLE:
|
||||
logger.warning(
|
||||
"StructuralChunker not available, falling back to recursive splitting"
|
||||
)
|
||||
return split_recursive(text, chunk_size=max_chunk_size, **kwargs)
|
||||
|
||||
try:
|
||||
chunker = StructuralChunker(
|
||||
max_chunk_size=max_chunk_size,
|
||||
respect_headers=respect_headers,
|
||||
respect_sections=respect_sections,
|
||||
**kwargs,
|
||||
)
|
||||
return chunker.chunk(text, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Error in structural splitting: {e}, falling back to recursive splitting"
|
||||
)
|
||||
return split_recursive(text, chunk_size=max_chunk_size, **kwargs)
|
||||
|
||||
|
||||
def split_sliding_window(
|
||||
text: str,
|
||||
chunk_size: int = 1000,
|
||||
overlap: int = 200,
|
||||
stride: Optional[int] = None,
|
||||
preserve_boundaries: bool = True,
|
||||
**kwargs,
|
||||
) -> List[Chunk]:
|
||||
"""
|
||||
Sliding window chunking with optional boundary preservation.
|
||||
|
||||
Args:
|
||||
text: Input text
|
||||
chunk_size: Chunk size in characters
|
||||
overlap: Overlap size in characters
|
||||
stride: Stride size (default: chunk_size - overlap)
|
||||
preserve_boundaries: Whether to preserve word/sentence boundaries
|
||||
**kwargs: Additional options
|
||||
|
||||
Returns:
|
||||
List of chunks
|
||||
"""
|
||||
if not SPECIALIZED_CHUNKERS_AVAILABLE:
|
||||
logger.warning(
|
||||
"SlidingWindowChunker not available, falling back to recursive splitting"
|
||||
)
|
||||
return split_recursive(
|
||||
text, chunk_size=chunk_size, chunk_overlap=overlap, **kwargs
|
||||
)
|
||||
|
||||
try:
|
||||
chunker = SlidingWindowChunker(
|
||||
chunk_size=chunk_size, overlap=overlap, stride=stride, **kwargs
|
||||
)
|
||||
return chunker.chunk(text, preserve_boundaries=preserve_boundaries, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Error in sliding window splitting: {e}, falling back to recursive splitting"
|
||||
)
|
||||
return split_recursive(
|
||||
text, chunk_size=chunk_size, chunk_overlap=overlap, **kwargs
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Method Dispatcher
|
||||
# ============================================================================
|
||||
@@ -1542,6 +1651,9 @@ _SPLIT_METHODS = {
|
||||
"centrality_based": split_centrality_based,
|
||||
"subgraph": split_subgraph,
|
||||
"topic_based": split_topic_based,
|
||||
# Specialized methods
|
||||
"structural": split_structural,
|
||||
"sliding_window": split_sliding_window,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -209,7 +209,7 @@ chunks = split_entity_aware(
|
||||
text,
|
||||
chunk_size=1000,
|
||||
chunk_overlap=200,
|
||||
ner_method="llm", # or "spacy", "huggingface"
|
||||
ner_method="ml", # "ml" (spaCy), "llm", or "pattern"
|
||||
preserve_entities=True
|
||||
)
|
||||
|
||||
@@ -324,7 +324,7 @@ chunks = table_chunker.chunk(text_with_tables)
|
||||
entity_chunker = EntityAwareChunker(
|
||||
chunk_size=1000,
|
||||
chunk_overlap=200,
|
||||
ner_method="llm",
|
||||
ner_method="ml",
|
||||
preserve_entities=True
|
||||
)
|
||||
chunks = entity_chunker.chunk(text)
|
||||
@@ -408,7 +408,7 @@ chunks6 = split_by_words(text, chunk_size=500, chunk_overlap=50)
|
||||
# Advanced methods
|
||||
chunks7 = split_semantic_transformer(text, chunk_size=1000, chunk_overlap=200)
|
||||
chunks8 = split_llm(text, chunk_size=1000, provider="openai")
|
||||
chunks9 = split_entity_aware(text, chunk_size=1000, ner_method="llm")
|
||||
chunks9 = split_entity_aware(text, chunk_size=1000, ner_method="ml")
|
||||
chunks10 = split_relation_aware(text, chunk_size=1000)
|
||||
chunks11 = split_graph_based(text, chunk_size=1000)
|
||||
chunks12 = split_ontology_aware(text, chunk_size=1000)
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
"""
|
||||
Triple Store Module
|
||||
Triplet Store Module
|
||||
|
||||
This module provides comprehensive triple store integration and management
|
||||
for RDF data storage and querying, supporting multiple triple store backends
|
||||
This module provides comprehensive triplet store integration and management
|
||||
for RDF data storage and querying, supporting multiple triplet store backends
|
||||
with unified interfaces.
|
||||
|
||||
Algorithms Used:
|
||||
|
||||
Triple Store Management:
|
||||
Triplet Store Management:
|
||||
- Store Registration: Store type detection, adapter factory pattern, configuration management, default store selection
|
||||
- Adapter Pattern: Unified interface for multiple backends (Blazegraph, Jena, RDF4J, Virtuoso), adapter instantiation, backend-specific operation delegation
|
||||
- Store Selection: Default store resolution, store ID lookup, store validation
|
||||
|
||||
CRUD Operations:
|
||||
- Triple Addition: Single triple insertion, batch triple insertion, triple validation (subject/predicate/object checking, confidence validation), adapter delegation
|
||||
- Triple Retrieval: Pattern matching (subject/predicate/object filtering), SPARQL query construction, result binding extraction, triple reconstruction
|
||||
- Triple Deletion: Triple matching, deletion operation delegation, result verification
|
||||
- Triple Update: Delete-then-add pattern, atomic update operations, conflict detection
|
||||
- Triplet Addition: Single triplet insertion, batch triplet insertion, triplet validation (subject/predicate/object checking, confidence validation), adapter delegation
|
||||
- Triplet Retrieval: Pattern matching (subject/predicate/object filtering), SPARQL query construction, result binding extraction, triplet reconstruction
|
||||
- Triplet Deletion: Triplet matching, deletion operation delegation, result verification
|
||||
- Triplet Update: Delete-then-add pattern, atomic update operations, conflict detection
|
||||
|
||||
Bulk Loading:
|
||||
- Batch Processing: Chunking algorithm (fixed-size batch creation), batch size optimization, memory management for large datasets
|
||||
@@ -46,7 +46,7 @@ Store Adapters:
|
||||
Data Validation:
|
||||
- Triple Validation: Required field checking (subject, predicate, object), confidence range validation (0-1), URI format validation
|
||||
- Pre-load Validation: Empty component detection, URI format checking, confidence threshold checking, error/warning categorization
|
||||
|
||||
|
||||
Performance Optimization:
|
||||
- Batch Size Optimization: Configurable batch size, memory-aware batching, throughput-based optimization
|
||||
- Connection Pooling: Adapter-level connection management, connection reuse, connection lifecycle management
|
||||
@@ -55,7 +55,7 @@ Performance Optimization:
|
||||
|
||||
Key Features:
|
||||
- Multi-backend support (Blazegraph, Jena, RDF4J, Virtuoso)
|
||||
- CRUD operations for RDF triples
|
||||
- CRUD operations for RDF triplets
|
||||
- SPARQL query execution and optimization
|
||||
- Bulk data loading with progress tracking
|
||||
- Query caching and optimization
|
||||
@@ -65,41 +65,41 @@ Key Features:
|
||||
- Configuration management with environment variables and config files
|
||||
|
||||
Main Classes:
|
||||
- TripleManager: Main triple store management coordinator
|
||||
- TripletManager: Main triplet store management coordinator
|
||||
- QueryEngine: SPARQL query execution and optimization
|
||||
- BulkLoader: High-volume data loading
|
||||
- BlazegraphAdapter: Blazegraph integration adapter
|
||||
- JenaAdapter: Apache Jena integration adapter
|
||||
- RDF4JAdapter: Eclipse RDF4J integration adapter
|
||||
- VirtuosoAdapter: Virtuoso RDF store integration adapter
|
||||
- TripleStore: Triple store configuration dataclass
|
||||
- TripletStore: Triplet store configuration dataclass
|
||||
- QueryResult: Query result representation dataclass
|
||||
- QueryPlan: Query execution plan dataclass
|
||||
- LoadProgress: Bulk loading progress dataclass
|
||||
|
||||
Convenience Functions:
|
||||
- register_store: Register triple store wrapper
|
||||
- add_triple: Add single triple wrapper
|
||||
- add_triples: Add multiple triples wrapper
|
||||
- get_triples: Get triples matching pattern wrapper
|
||||
- delete_triple: Delete triple wrapper
|
||||
- register_store: Register triplet store wrapper
|
||||
- add_triple: Add single triplet wrapper
|
||||
- add_triples: Add multiple triplets wrapper
|
||||
- get_triples: Get triplets matching pattern wrapper
|
||||
- delete_triple: Delete triplet wrapper
|
||||
- execute_query: Execute SPARQL query wrapper
|
||||
- optimize_query: Optimize SPARQL query wrapper
|
||||
- bulk_load: Bulk load triples wrapper
|
||||
- get_triple_store_method: Get triple store method by task and name
|
||||
- list_available_methods: List registered triple store methods
|
||||
- bulk_load: Bulk load triplets wrapper
|
||||
- get_triplet_store_method: Get triplet store method by task and name
|
||||
- list_available_methods: List registered triplet store methods
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.triple_store import TripleManager, register_store, add_triple, execute_query
|
||||
>>> from semantica.triplet_store import TripletManager, register_store, add_triple, execute_query
|
||||
>>> # Using convenience functions
|
||||
>>> store = register_store("main", "blazegraph", "http://localhost:9999/blazegraph")
|
||||
>>> result = add_triple(triple, store_id="main")
|
||||
>>> query_result = execute_query(sparql_query, store_adapter)
|
||||
>>> # Using classes directly
|
||||
>>> manager = TripleManager()
|
||||
>>> manager = TripletManager()
|
||||
>>> store = manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph")
|
||||
>>> result = manager.add_triple(triple, store_id="main")
|
||||
>>> from semantica.triple_store import QueryEngine
|
||||
>>> from semantica.triplet_store import QueryEngine
|
||||
>>> engine = QueryEngine()
|
||||
>>> query_result = engine.execute_query(sparql_query, store_adapter)
|
||||
|
||||
@@ -109,7 +109,7 @@ License: MIT
|
||||
|
||||
from .blazegraph_adapter import BlazegraphAdapter
|
||||
from .bulk_loader import BulkLoader, LoadProgress
|
||||
from .config import TripleStoreConfig, triple_store_config
|
||||
from .config import TripletStoreConfig, triplet_store_config
|
||||
from .jena_adapter import JenaAdapter
|
||||
from .methods import (
|
||||
add_triple,
|
||||
@@ -117,7 +117,7 @@ from .methods import (
|
||||
bulk_load,
|
||||
delete_triple,
|
||||
execute_query,
|
||||
get_triple_store_method,
|
||||
get_triplet_store_method,
|
||||
get_triples,
|
||||
list_available_methods,
|
||||
optimize_query,
|
||||
@@ -129,13 +129,13 @@ from .methods import (
|
||||
from .query_engine import QueryEngine, QueryPlan, QueryResult
|
||||
from .rdf4j_adapter import RDF4JAdapter
|
||||
from .registry import MethodRegistry, method_registry
|
||||
from .triple_manager import TripleManager, TripleStore
|
||||
from .triplet_manager import TripletManager, TripletStore
|
||||
from .virtuoso_adapter import VirtuosoAdapter
|
||||
|
||||
__all__ = [
|
||||
# Triple management
|
||||
"TripleManager",
|
||||
"TripleStore",
|
||||
"TripletManager",
|
||||
"TripletStore",
|
||||
# Store adapters
|
||||
"BlazegraphAdapter",
|
||||
"JenaAdapter",
|
||||
@@ -160,11 +160,11 @@ __all__ = [
|
||||
"plan_query",
|
||||
"bulk_load",
|
||||
"validate_triples",
|
||||
"get_triple_store_method",
|
||||
"get_triplet_store_method",
|
||||
"list_available_methods",
|
||||
# Configuration and registry
|
||||
"TripleStoreConfig",
|
||||
"triple_store_config",
|
||||
"TripletStoreConfig",
|
||||
"triplet_store_config",
|
||||
"MethodRegistry",
|
||||
"method_registry",
|
||||
]
|
||||
+3
-3
@@ -17,7 +17,7 @@ Main Classes:
|
||||
- BlazegraphAdapter: Main Blazegraph integration adapter
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.triple_store import BlazegraphAdapter
|
||||
>>> from semantica.triplet_store import BlazegraphAdapter
|
||||
>>> adapter = BlazegraphAdapter(endpoint="http://localhost:9999/blazegraph", namespace="kb")
|
||||
>>> result = adapter.execute_sparql(sparql_query)
|
||||
>>> load_result = adapter.bulk_load(triples)
|
||||
@@ -40,7 +40,7 @@ from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
class BlazegraphAdapter:
|
||||
"""
|
||||
Blazegraph triple store adapter.
|
||||
Blazegraph triplet store adapter.
|
||||
|
||||
• Blazegraph connection and authentication
|
||||
• SPARQL query execution
|
||||
@@ -119,7 +119,7 @@ class BlazegraphAdapter:
|
||||
Query results
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="triple_store",
|
||||
module="triplet_store",
|
||||
submodule="BlazegraphAdapter",
|
||||
message="Executing SPARQL query on Blazegraph",
|
||||
)
|
||||
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
Bulk Loader Module
|
||||
|
||||
This module provides high-volume data loading capabilities for triple stores,
|
||||
This module provides high-volume data loading capabilities for triplet stores,
|
||||
enabling efficient batch processing with progress tracking and error recovery.
|
||||
|
||||
Key Features:
|
||||
@@ -18,7 +18,7 @@ Main Classes:
|
||||
- LoadProgress: Bulk loading progress representation dataclass
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.triple_store import BulkLoader
|
||||
>>> from semantica.triplet_store import BulkLoader
|
||||
>>> loader = BulkLoader(batch_size=1000, max_retries=3)
|
||||
>>> progress = loader.load_triples(triples, store_adapter)
|
||||
>>> print(f"Loaded {progress.loaded_triples}/{progress.total_triples} triples")
|
||||
@@ -56,7 +56,7 @@ class LoadProgress:
|
||||
|
||||
class BulkLoader:
|
||||
"""
|
||||
High-volume data loading system for triple stores.
|
||||
High-volume data loading system for triplet stores.
|
||||
|
||||
• High-volume data loading strategies
|
||||
• Batch processing and chunking
|
||||
@@ -104,7 +104,7 @@ class BulkLoader:
|
||||
Load progress information
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="triple_store",
|
||||
module="triplet_store",
|
||||
submodule="BulkLoader",
|
||||
message=f"Loading {len(triples)} triples in bulk",
|
||||
)
|
||||
@@ -1,14 +1,14 @@
|
||||
"""
|
||||
Configuration Management Module for Triple Store
|
||||
Configuration Management Module for Triplet Store
|
||||
|
||||
This module provides centralized configuration management for triple store operations,
|
||||
This module provides centralized configuration management for triplet store operations,
|
||||
supporting multiple configuration sources including environment variables, config files,
|
||||
and programmatic configuration.
|
||||
|
||||
Supported Configuration Sources:
|
||||
- Environment variables: TRIPLE_STORE_DEFAULT_STORE, TRIPLE_STORE_BATCH_SIZE, TRIPLE_STORE_ENABLE_CACHING, etc.
|
||||
- Environment variables: TRIPLET_STORE_DEFAULT_STORE, TRIPLET_STORE_BATCH_SIZE, TRIPLET_STORE_ENABLE_CACHING, etc.
|
||||
- Config files: YAML, JSON, TOML formats
|
||||
- Programmatic: Python API for setting triple store configurations
|
||||
- Programmatic: Python API for setting triplet store configurations
|
||||
|
||||
Algorithms Used:
|
||||
- Environment Variable Parsing: OS-level environment variable access
|
||||
@@ -19,7 +19,7 @@ Algorithms Used:
|
||||
- Dictionary Merging: Deep merge algorithms for configuration updates
|
||||
|
||||
Key Features:
|
||||
- Environment variable support for triple store parameters
|
||||
- Environment variable support for triplet store parameters
|
||||
- Config file support (YAML, JSON, TOML formats)
|
||||
- Programmatic configuration via Python API
|
||||
- Method-specific configuration management
|
||||
@@ -27,13 +27,13 @@ Key Features:
|
||||
- Global config instance for easy access
|
||||
|
||||
Main Classes:
|
||||
- TripleStoreConfig: Main configuration manager class for triple store module
|
||||
- TripletStoreConfig: Main configuration manager class for triplet store module
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.triple_store.config import triple_store_config
|
||||
>>> default_store = triple_store_config.get("default_store", default="main")
|
||||
>>> triple_store_config.set("default_store", "main")
|
||||
>>> method_config = triple_store_config.get_method_config("add_triple")
|
||||
>>> from semantica.triplet_store.config import triplet_store_config
|
||||
>>> default_store = triplet_store_config.get("default_store", default="main")
|
||||
>>> triplet_store_config.set("default_store", "main")
|
||||
>>> method_config = triplet_store_config.get_method_config("add_triple")
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -43,8 +43,8 @@ from typing import Any, Dict, Optional
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
|
||||
class TripleStoreConfig:
|
||||
"""Configuration manager for triple store module - supports .env files, environment variables, and programmatic config."""
|
||||
class TripletStoreConfig:
|
||||
"""Configuration manager for triplet store module - supports .env files, environment variables, and programmatic config."""
|
||||
|
||||
def __init__(self, config_file: Optional[str] = None):
|
||||
"""
|
||||
@@ -53,7 +53,7 @@ class TripleStoreConfig:
|
||||
Args:
|
||||
config_file: Optional path to configuration file (YAML, JSON, or TOML)
|
||||
"""
|
||||
self.logger = get_logger("triple_store_config")
|
||||
self.logger = get_logger("triplet_store_config")
|
||||
self.config_file = config_file
|
||||
self._config: Dict[str, Any] = {}
|
||||
self._method_configs: Dict[str, Dict[str, Any]] = {}
|
||||
@@ -86,40 +86,40 @@ class TripleStoreConfig:
|
||||
|
||||
with open(file_path, "r") as f:
|
||||
config_data = yaml.safe_load(f)
|
||||
if config_data and "triple_store" in config_data:
|
||||
self._config.update(config_data["triple_store"])
|
||||
if config_data and "triplet_store" in config_data:
|
||||
self._config.update(config_data["triplet_store"])
|
||||
elif file_path.suffix == ".json":
|
||||
import json
|
||||
|
||||
with open(file_path, "r") as f:
|
||||
config_data = json.load(f)
|
||||
if config_data and "triple_store" in config_data:
|
||||
self._config.update(config_data["triple_store"])
|
||||
if config_data and "triplet_store" in config_data:
|
||||
self._config.update(config_data["triplet_store"])
|
||||
elif file_path.suffix == ".toml":
|
||||
import tomli
|
||||
|
||||
with open(file_path, "rb") as f:
|
||||
config_data = tomli.load(f)
|
||||
if config_data and "triple_store" in config_data:
|
||||
self._config.update(config_data["triple_store"])
|
||||
if config_data and "triplet_store" in config_data:
|
||||
self._config.update(config_data["triplet_store"])
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to load config file: {e}")
|
||||
|
||||
def _load_from_env(self) -> None:
|
||||
"""Load configuration from environment variables."""
|
||||
env_mappings = {
|
||||
"TRIPLE_STORE_DEFAULT_STORE": "default_store",
|
||||
"TRIPLE_STORE_BATCH_SIZE": "batch_size",
|
||||
"TRIPLE_STORE_ENABLE_CACHING": "enable_caching",
|
||||
"TRIPLE_STORE_CACHE_SIZE": "cache_size",
|
||||
"TRIPLE_STORE_ENABLE_OPTIMIZATION": "enable_optimization",
|
||||
"TRIPLE_STORE_MAX_RETRIES": "max_retries",
|
||||
"TRIPLE_STORE_RETRY_DELAY": "retry_delay",
|
||||
"TRIPLE_STORE_TIMEOUT": "timeout",
|
||||
"TRIPLE_STORE_BLAZEGRAPH_ENDPOINT": "blazegraph_endpoint",
|
||||
"TRIPLE_STORE_JENA_ENDPOINT": "jena_endpoint",
|
||||
"TRIPLE_STORE_RDF4J_ENDPOINT": "rdf4j_endpoint",
|
||||
"TRIPLE_STORE_VIRTUOSO_ENDPOINT": "virtuoso_endpoint",
|
||||
"TRIPLET_STORE_DEFAULT_STORE": "default_store",
|
||||
"TRIPLET_STORE_BATCH_SIZE": "batch_size",
|
||||
"TRIPLET_STORE_ENABLE_CACHING": "enable_caching",
|
||||
"TRIPLET_STORE_CACHE_SIZE": "cache_size",
|
||||
"TRIPLET_STORE_ENABLE_OPTIMIZATION": "enable_optimization",
|
||||
"TRIPLET_STORE_MAX_RETRIES": "max_retries",
|
||||
"TRIPLET_STORE_RETRY_DELAY": "retry_delay",
|
||||
"TRIPLET_STORE_TIMEOUT": "timeout",
|
||||
"TRIPLET_STORE_BLAZEGRAPH_ENDPOINT": "blazegraph_endpoint",
|
||||
"TRIPLET_STORE_JENA_ENDPOINT": "jena_endpoint",
|
||||
"TRIPLET_STORE_RDF4J_ENDPOINT": "rdf4j_endpoint",
|
||||
"TRIPLET_STORE_VIRTUOSO_ENDPOINT": "virtuoso_endpoint",
|
||||
}
|
||||
|
||||
for env_var, config_key in env_mappings.items():
|
||||
@@ -238,4 +238,4 @@ class TripleStoreConfig:
|
||||
|
||||
|
||||
# Global configuration instance
|
||||
triple_store_config = TripleStoreConfig()
|
||||
triplet_store_config = TripletStoreConfig()
|
||||
@@ -16,7 +16,7 @@ Main Classes:
|
||||
- JenaAdapter: Main Jena integration adapter
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.triple_store import JenaAdapter
|
||||
>>> from semantica.triplet_store import JenaAdapter
|
||||
>>> adapter = JenaAdapter(endpoint="http://localhost:3030/ds", dataset="default")
|
||||
>>> result = adapter.add_triples(triples)
|
||||
>>> query_result = adapter.execute_sparql(sparql_query)
|
||||
@@ -47,7 +47,7 @@ except ImportError:
|
||||
|
||||
class JenaAdapter:
|
||||
"""
|
||||
Apache Jena adapter for triple store operations.
|
||||
Apache Jena adapter for triplet store operations.
|
||||
|
||||
• Jena connection and configuration
|
||||
• SPARQL query execution
|
||||
@@ -129,7 +129,7 @@ class JenaAdapter:
|
||||
Operation status
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="triple_store",
|
||||
module="triplet_store",
|
||||
submodule="JenaAdapter",
|
||||
message=f"Adding {len(triples)} triples to Jena model",
|
||||
)
|
||||
@@ -1,14 +1,14 @@
|
||||
"""
|
||||
Triple Store Methods Module
|
||||
Triplet Store Methods Module
|
||||
|
||||
This module provides all triple store methods as simple, reusable functions for
|
||||
registering stores, adding triples, querying, and managing triple stores. It supports
|
||||
This module provides all triplet store methods as simple, reusable functions for
|
||||
registering stores, adding triples, querying, and managing triplet stores. It supports
|
||||
multiple approaches and integrates with the method registry for extensibility.
|
||||
|
||||
Supported Methods:
|
||||
|
||||
Store Registration:
|
||||
- "default": Default store registration using TripleManager
|
||||
- "default": Default store registration using TripletManager
|
||||
- "blazegraph": Blazegraph-specific registration
|
||||
- "jena": Jena-specific registration
|
||||
- "rdf4j": RDF4J-specific registration
|
||||
@@ -78,7 +78,7 @@ Bulk Loading:
|
||||
- Stream Processing: Iterator-based processing, incremental batch collection
|
||||
|
||||
Key Features:
|
||||
- Multiple triple store operation methods
|
||||
- Multiple triplet store operation methods
|
||||
- Store registration with method dispatch
|
||||
- Method dispatchers with registry support
|
||||
- Custom method registration capability
|
||||
@@ -95,11 +95,11 @@ Main Functions:
|
||||
- optimize_query: Query optimization wrapper
|
||||
- bulk_load: Bulk loading wrapper
|
||||
- validate_triples: Triple validation wrapper
|
||||
- get_triple_store_method: Get triple store method by task and name
|
||||
- get_triplet_store_method: Get triplet store method by task and name
|
||||
- list_available_methods: List registered methods
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.triple_store.methods import register_store, add_triple, execute_query
|
||||
>>> from semantica.triplet_store.methods import register_store, add_triple, execute_query
|
||||
>>> store = register_store("main", "blazegraph", "http://localhost:9999/blazegraph", method="default")
|
||||
>>> result = add_triple(triple, store_id="main", method="default")
|
||||
>>> query_result = execute_query(sparql_query, store_adapter, method="default")
|
||||
@@ -109,23 +109,23 @@ from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..semantic_extract.triple_extractor import Triple
|
||||
from .bulk_loader import BulkLoader, LoadProgress
|
||||
from .config import triple_store_config
|
||||
from .config import triplet_store_config
|
||||
from .query_engine import QueryEngine, QueryPlan, QueryResult
|
||||
from .registry import method_registry
|
||||
from .triple_manager import TripleManager, TripleStore
|
||||
from .triplet_manager import TripletManager, TripletStore
|
||||
|
||||
# Global manager instances
|
||||
_global_manager: Optional[TripleManager] = None
|
||||
_global_manager: Optional[TripletManager] = None
|
||||
_global_query_engine: Optional[QueryEngine] = None
|
||||
_global_bulk_loader: Optional[BulkLoader] = None
|
||||
|
||||
|
||||
def _get_manager() -> TripleManager:
|
||||
"""Get or create global TripleManager instance."""
|
||||
def _get_manager() -> TripletManager:
|
||||
"""Get or create global TripletManager instance."""
|
||||
global _global_manager
|
||||
if _global_manager is None:
|
||||
config = triple_store_config.get_all()
|
||||
_global_manager = TripleManager(config=config)
|
||||
config = triplet_store_config.get_all()
|
||||
_global_manager = TripletManager(config=config)
|
||||
return _global_manager
|
||||
|
||||
|
||||
@@ -133,7 +133,7 @@ def _get_query_engine() -> QueryEngine:
|
||||
"""Get or create global QueryEngine instance."""
|
||||
global _global_query_engine
|
||||
if _global_query_engine is None:
|
||||
config = triple_store_config.get_all()
|
||||
config = triplet_store_config.get_all()
|
||||
_global_query_engine = QueryEngine(config=config)
|
||||
return _global_query_engine
|
||||
|
||||
@@ -142,16 +142,16 @@ def _get_bulk_loader() -> BulkLoader:
|
||||
"""Get or create global BulkLoader instance."""
|
||||
global _global_bulk_loader
|
||||
if _global_bulk_loader is None:
|
||||
config = triple_store_config.get_all()
|
||||
config = triplet_store_config.get_all()
|
||||
_global_bulk_loader = BulkLoader(config=config)
|
||||
return _global_bulk_loader
|
||||
|
||||
|
||||
def register_store(
|
||||
store_id: str, store_type: str, endpoint: str, method: str = "default", **options
|
||||
) -> TripleStore:
|
||||
) -> TripletStore:
|
||||
"""
|
||||
Register a triple store.
|
||||
Register a triplet store.
|
||||
|
||||
Args:
|
||||
store_id: Store identifier
|
||||
@@ -321,7 +321,7 @@ def execute_query(
|
||||
|
||||
Args:
|
||||
query: SPARQL query string
|
||||
store_adapter: Triple store adapter instance
|
||||
store_adapter: Triplet store adapter instance
|
||||
method: Query method name (default: "default")
|
||||
**options: Additional options
|
||||
|
||||
@@ -383,7 +383,7 @@ def bulk_load(
|
||||
|
||||
Args:
|
||||
triples: List of triples to load
|
||||
store_adapter: Triple store adapter instance
|
||||
store_adapter: Triplet store adapter instance
|
||||
method: Loading method name (default: "default")
|
||||
**options: Additional options
|
||||
|
||||
@@ -424,9 +424,9 @@ def validate_triples(
|
||||
return loader.validate_before_load(triples, **options)
|
||||
|
||||
|
||||
def get_triple_store_method(task: str, method_name: str) -> Optional[Any]:
|
||||
def get_triplet_store_method(task: str, method_name: str) -> Optional[Any]:
|
||||
"""
|
||||
Get triple store method by task and name.
|
||||
Get triplet store method by task and name.
|
||||
|
||||
Args:
|
||||
task: Task type (register, add, get, delete, update, query, optimize, bulk_load, validate)
|
||||
@@ -440,7 +440,7 @@ def get_triple_store_method(task: str, method_name: str) -> Optional[Any]:
|
||||
|
||||
def list_available_methods(task: Optional[str] = None) -> Dict[str, List[str]]:
|
||||
"""
|
||||
List all available triple store methods.
|
||||
List all available triplet store methods.
|
||||
|
||||
Args:
|
||||
task: Optional task type to filter by
|
||||
@@ -2,7 +2,7 @@
|
||||
Query Engine Module
|
||||
|
||||
This module provides comprehensive SPARQL query execution and optimization
|
||||
for triple store operations, including query planning, caching, and performance
|
||||
for triplet store operations, including query planning, caching, and performance
|
||||
monitoring.
|
||||
|
||||
Key Features:
|
||||
@@ -20,7 +20,7 @@ Main Classes:
|
||||
- QueryPlan: Query execution plan representation dataclass
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.triple_store import QueryEngine
|
||||
>>> from semantica.triplet_store import QueryEngine
|
||||
>>> engine = QueryEngine(enable_caching=True, enable_optimization=True)
|
||||
>>> result = engine.execute_query(sparql_query, store_adapter)
|
||||
>>> plan = engine.plan_query(sparql_query)
|
||||
@@ -109,7 +109,7 @@ class QueryEngine:
|
||||
Query result
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="triple_store",
|
||||
module="triplet_store",
|
||||
submodule="QueryEngine",
|
||||
message="Executing SPARQL query",
|
||||
)
|
||||
@@ -16,7 +16,7 @@ Main Classes:
|
||||
- RDF4JAdapter: Main RDF4J integration adapter
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.triple_store import RDF4JAdapter
|
||||
>>> from semantica.triplet_store import RDF4JAdapter
|
||||
>>> adapter = RDF4JAdapter(endpoint="http://localhost:8080/rdf4j-server", repository_id="repo1")
|
||||
>>> result = adapter.execute_sparql(sparql_query)
|
||||
>>> tx_id = adapter.begin_transaction()
|
||||
@@ -38,7 +38,7 @@ from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
class RDF4JAdapter:
|
||||
"""
|
||||
Eclipse RDF4J adapter for triple store operations.
|
||||
Eclipse RDF4J adapter for triplet store operations.
|
||||
|
||||
• RDF4J connection and repository management
|
||||
• SPARQL query execution
|
||||
@@ -184,7 +184,7 @@ class RDF4JAdapter:
|
||||
Query results
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="triple_store",
|
||||
module="triplet_store",
|
||||
submodule="RDF4JAdapter",
|
||||
message="Executing SPARQL query on RDF4J",
|
||||
)
|
||||
@@ -250,7 +250,7 @@ class RDF4JAdapter:
|
||||
Operation status
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="triple_store",
|
||||
module="triplet_store",
|
||||
submodule="RDF4JAdapter",
|
||||
message=f"Adding {len(triples)} triples to RDF4J repository",
|
||||
)
|
||||
@@ -1,11 +1,11 @@
|
||||
"""
|
||||
Method Registry Module for Triple Store
|
||||
Method Registry Module for Triplet Store
|
||||
|
||||
This module provides a method registry system for registering custom triple store methods,
|
||||
enabling extensibility and community contributions to the triple store toolkit.
|
||||
This module provides a method registry system for registering custom triplet store methods,
|
||||
enabling extensibility and community contributions to the triplet store toolkit.
|
||||
|
||||
Supported Registration Types:
|
||||
- Method Registry: Register custom triple store methods for:
|
||||
- Method Registry: Register custom triplet store methods for:
|
||||
* "register": Store registration methods
|
||||
* "add": Triple addition methods
|
||||
* "get": Triple retrieval methods
|
||||
@@ -24,20 +24,20 @@ Algorithms Used:
|
||||
- Task-based Organization: Hierarchical organization by task type
|
||||
|
||||
Key Features:
|
||||
- Method registry for custom triple store methods
|
||||
- Method registry for custom triplet store methods
|
||||
- Task-based method organization (register, add, get, delete, update, query, optimize, bulk_load, validate)
|
||||
- Dynamic registration and unregistration
|
||||
- Easy discovery of available methods
|
||||
- Support for community-contributed extensions
|
||||
|
||||
Main Classes:
|
||||
- MethodRegistry: Registry for custom triple store methods
|
||||
- MethodRegistry: Registry for custom triplet store methods
|
||||
|
||||
Global Instances:
|
||||
- method_registry: Global method registry instance
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.triple_store.registry import method_registry
|
||||
>>> from semantica.triplet_store.registry import method_registry
|
||||
>>> method_registry.register("add", "custom_method", custom_add_function)
|
||||
>>> available = method_registry.list_all("add")
|
||||
"""
|
||||
@@ -46,7 +46,7 @@ from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
|
||||
class MethodRegistry:
|
||||
"""Registry for custom triple store methods."""
|
||||
"""Registry for custom triplet store methods."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize method registry."""
|
||||
+26
-26
@@ -1,25 +1,25 @@
|
||||
"""
|
||||
Triple Manager Module
|
||||
Triplet Manager Module
|
||||
|
||||
This module provides comprehensive CRUD operations for RDF triples and triple
|
||||
store management, enabling unified access to multiple triple store backends
|
||||
This module provides comprehensive CRUD operations for RDF triplets and triplet
|
||||
store management, enabling unified access to multiple triplet store backends
|
||||
through a common interface.
|
||||
|
||||
Key Features:
|
||||
- CRUD operations for RDF triples
|
||||
- CRUD operations for RDF triplets
|
||||
- Multi-store management and registration
|
||||
- Batch operations and bulk loading
|
||||
- Triple validation and consistency
|
||||
- Triplet validation and consistency
|
||||
- Store adapter pattern
|
||||
- Error handling and recovery
|
||||
|
||||
Main Classes:
|
||||
- TripleManager: Main triple store management coordinator
|
||||
- TripleStore: Triple store configuration dataclass
|
||||
- TripletManager: Main triplet store management coordinator
|
||||
- TripletStore: Triplet store configuration dataclass
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.triple_store import TripleManager
|
||||
>>> manager = TripleManager()
|
||||
>>> from semantica.triplet_store import TripletManager
|
||||
>>> manager = TripletManager()
|
||||
>>> store = manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph")
|
||||
>>> result = manager.add_triple(triple, store_id="main")
|
||||
>>> triples = manager.get_triple(subject="http://example.org/entity1")
|
||||
@@ -39,8 +39,8 @@ from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
@dataclass
|
||||
class TripleStore:
|
||||
"""Triple store configuration."""
|
||||
class TripletStore:
|
||||
"""Triplet store configuration."""
|
||||
|
||||
store_id: str
|
||||
store_type: str # "blazegraph", "jena", "rdf4j", "virtuoso"
|
||||
@@ -49,9 +49,9 @@ class TripleStore:
|
||||
connected: bool = False
|
||||
|
||||
|
||||
class TripleManager:
|
||||
class TripletManager:
|
||||
"""
|
||||
Triple store management system.
|
||||
Triplet store management system.
|
||||
|
||||
• CRUD operations for RDF triples
|
||||
• Batch operations and bulk loading
|
||||
@@ -63,26 +63,26 @@ class TripleManager:
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None, **kwargs):
|
||||
"""
|
||||
Initialize triple manager.
|
||||
Initialize triplet manager.
|
||||
|
||||
Args:
|
||||
config: Configuration dictionary
|
||||
**kwargs: Additional configuration options:
|
||||
- default_store: Default triple store to use
|
||||
- default_store: Default triplet store to use
|
||||
"""
|
||||
self.logger = get_logger("triple_manager")
|
||||
self.logger = get_logger("triplet_manager")
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.stores: Dict[str, TripleStore] = {}
|
||||
self.stores: Dict[str, TripletStore] = {}
|
||||
self.default_store_id = self.config.get("default_store")
|
||||
|
||||
def register_store(
|
||||
self, store_id: str, store_type: str, endpoint: str, **config
|
||||
) -> TripleStore:
|
||||
) -> TripletStore:
|
||||
"""
|
||||
Register a triple store.
|
||||
Register a triplet store.
|
||||
|
||||
Args:
|
||||
store_id: Store identifier
|
||||
@@ -93,7 +93,7 @@ class TripleManager:
|
||||
Returns:
|
||||
Registered store
|
||||
"""
|
||||
store = TripleStore(
|
||||
store = TripletStore(
|
||||
store_id=store_id, store_type=store_type, endpoint=endpoint, config=config
|
||||
)
|
||||
|
||||
@@ -102,7 +102,7 @@ class TripleManager:
|
||||
if not self.default_store_id:
|
||||
self.default_store_id = store_id
|
||||
|
||||
self.logger.info(f"Registered triple store: {store_id} ({store_type})")
|
||||
self.logger.info(f"Registered triplet store: {store_id} ({store_type})")
|
||||
|
||||
return store
|
||||
|
||||
@@ -158,8 +158,8 @@ class TripleManager:
|
||||
Operation status
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="triple_store",
|
||||
submodule="TripleManager",
|
||||
module="triplet_store",
|
||||
submodule="TripletManager",
|
||||
message=f"Adding {len(triples)} triples to store",
|
||||
)
|
||||
|
||||
@@ -301,7 +301,7 @@ class TripleManager:
|
||||
|
||||
return True
|
||||
|
||||
def _get_store(self, store_id: Optional[str] = None) -> TripleStore:
|
||||
def _get_store(self, store_id: Optional[str] = None) -> TripletStore:
|
||||
"""Get store by ID."""
|
||||
store_id = store_id or self.default_store_id
|
||||
|
||||
@@ -313,7 +313,7 @@ class TripleManager:
|
||||
|
||||
return self.stores[store_id]
|
||||
|
||||
def _get_adapter(self, store: TripleStore) -> Any:
|
||||
def _get_adapter(self, store: TripletStore) -> Any:
|
||||
"""Get adapter for store type."""
|
||||
store_type = store.store_type.lower()
|
||||
|
||||
@@ -336,7 +336,7 @@ class TripleManager:
|
||||
else:
|
||||
raise ValidationError(f"Unsupported store type: {store_type}")
|
||||
|
||||
def get_store(self, store_id: str) -> Optional[TripleStore]:
|
||||
def get_store(self, store_id: str) -> Optional[TripletStore]:
|
||||
"""Get store by ID."""
|
||||
return self.stores.get(store_id)
|
||||
|
||||
+77
-77
@@ -1,6 +1,6 @@
|
||||
# Triple Store Module Usage Guide
|
||||
# Triplet Store Module Usage Guide
|
||||
|
||||
This comprehensive guide demonstrates how to use the triple store module for RDF data storage and querying, supporting multiple triple store backends (Blazegraph, Jena, RDF4J, Virtuoso) with unified interfaces, SPARQL query execution, bulk loading, and query optimization.
|
||||
This comprehensive guide demonstrates how to use the triplet store module for RDF data storage and querying, supporting multiple triplet store backends (Blazegraph, Jena, RDF4J, Virtuoso) with unified interfaces, SPARQL query execution, bulk loading, and query optimization.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
@@ -17,14 +17,14 @@ This comprehensive guide demonstrates how to use the triple store module for RDF
|
||||
|
||||
## Basic Usage
|
||||
|
||||
### Using TripleManager
|
||||
### Using TripletManager
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleManager
|
||||
from semantica.triplet_store import TripletManager
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
# Create triple manager
|
||||
manager = TripleManager()
|
||||
# Create triplet manager
|
||||
manager = TripletManager()
|
||||
|
||||
# Register a store
|
||||
store = manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph")
|
||||
@@ -44,7 +44,7 @@ print(f"Triple added: {result['success']}")
|
||||
### Using Convenience Functions
|
||||
|
||||
```python
|
||||
from semantica.triple_store import register_store, add_triple, get_triples, execute_query
|
||||
from semantica.triplet_store import register_store, add_triple, get_triples, execute_query
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
# Register store
|
||||
@@ -66,7 +66,7 @@ print(f"Found {len(triples)} triples")
|
||||
### Using QueryEngine
|
||||
|
||||
```python
|
||||
from semantica.triple_store import QueryEngine, BlazegraphAdapter
|
||||
from semantica.triplet_store import QueryEngine, BlazegraphAdapter
|
||||
|
||||
# Create query engine
|
||||
engine = QueryEngine(enable_caching=True, enable_optimization=True)
|
||||
@@ -87,9 +87,9 @@ print(f"Execution time: {result.execution_time:.2f}s")
|
||||
### Registering a Store
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleManager
|
||||
from semantica.triplet_store import TripletManager
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
|
||||
# Register Blazegraph store
|
||||
blazegraph_store = manager.register_store(
|
||||
@@ -126,7 +126,7 @@ virtuoso_store = manager.register_store(
|
||||
### Using Convenience Function
|
||||
|
||||
```python
|
||||
from semantica.triple_store import register_store
|
||||
from semantica.triplet_store import register_store
|
||||
|
||||
# Register store using convenience function
|
||||
store = register_store(
|
||||
@@ -143,9 +143,9 @@ print(f"Store type: {store.store_type}")
|
||||
### Multiple Stores
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleManager
|
||||
from semantica.triplet_store import TripletManager
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
|
||||
# Register multiple stores
|
||||
manager.register_store("primary", "blazegraph", "http://localhost:9999/blazegraph")
|
||||
@@ -165,10 +165,10 @@ print(f"Store endpoint: {store.endpoint}")
|
||||
### Adding Triples
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleManager
|
||||
from semantica.triplet_store import TripletManager
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph")
|
||||
|
||||
# Add single triple
|
||||
@@ -194,9 +194,9 @@ print(f"Added {result['total_triples']} triples in {result['batches']} batches")
|
||||
### Retrieving Triples
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleManager
|
||||
from semantica.triplet_store import TripletManager
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph")
|
||||
|
||||
# Get all triples for a subject
|
||||
@@ -225,10 +225,10 @@ triples = manager.get_triple(
|
||||
### Deleting Triples
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleManager
|
||||
from semantica.triplet_store import TripletManager
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph")
|
||||
|
||||
# Delete triple
|
||||
@@ -244,10 +244,10 @@ print(f"Deleted: {result['success']}")
|
||||
### Updating Triples
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleManager
|
||||
from semantica.triplet_store import TripletManager
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
manager.register_store("main", "blazegraph", "http://localhost:9999/blazegraph")
|
||||
|
||||
# Update triple (delete old, add new)
|
||||
@@ -270,7 +270,7 @@ print(f"Updated: {result['success']}")
|
||||
### Basic Query Execution
|
||||
|
||||
```python
|
||||
from semantica.triple_store import QueryEngine, BlazegraphAdapter
|
||||
from semantica.triplet_store import QueryEngine, BlazegraphAdapter
|
||||
|
||||
# Create query engine
|
||||
engine = QueryEngine(enable_caching=True)
|
||||
@@ -298,7 +298,7 @@ for binding in result.bindings[:5]:
|
||||
### Using Convenience Function
|
||||
|
||||
```python
|
||||
from semantica.triple_store import execute_query, BlazegraphAdapter
|
||||
from semantica.triplet_store import execute_query, BlazegraphAdapter
|
||||
|
||||
adapter = BlazegraphAdapter(endpoint="http://localhost:9999/blazegraph")
|
||||
|
||||
@@ -311,7 +311,7 @@ print(f"Found {len(result.bindings)} results")
|
||||
### Query Result Processing
|
||||
|
||||
```python
|
||||
from semantica.triple_store import QueryEngine, BlazegraphAdapter
|
||||
from semantica.triplet_store import QueryEngine, BlazegraphAdapter
|
||||
|
||||
engine = QueryEngine()
|
||||
adapter = BlazegraphAdapter(endpoint="http://localhost:9999/blazegraph")
|
||||
@@ -332,7 +332,7 @@ print(f"Metadata: {result.metadata}")
|
||||
### Query Caching
|
||||
|
||||
```python
|
||||
from semantica.triple_store import QueryEngine, BlazegraphAdapter
|
||||
from semantica.triplet_store import QueryEngine, BlazegraphAdapter
|
||||
|
||||
# Enable caching
|
||||
engine = QueryEngine(enable_caching=True, cache_size=1000)
|
||||
@@ -357,7 +357,7 @@ engine.clear_cache()
|
||||
### Basic Query Optimization
|
||||
|
||||
```python
|
||||
from semantica.triple_store import QueryEngine
|
||||
from semantica.triplet_store import QueryEngine
|
||||
|
||||
engine = QueryEngine(enable_optimization=True)
|
||||
|
||||
@@ -377,7 +377,7 @@ print(f"Optimized query:\n{optimized}")
|
||||
### Query Planning
|
||||
|
||||
```python
|
||||
from semantica.triple_store import QueryEngine
|
||||
from semantica.triplet_store import QueryEngine
|
||||
|
||||
engine = QueryEngine(enable_optimization=True)
|
||||
|
||||
@@ -403,7 +403,7 @@ print(f"Execution steps: {plan.execution_steps}")
|
||||
### Query Statistics
|
||||
|
||||
```python
|
||||
from semantica.triple_store import QueryEngine, BlazegraphAdapter
|
||||
from semantica.triplet_store import QueryEngine, BlazegraphAdapter
|
||||
|
||||
engine = QueryEngine(enable_caching=True)
|
||||
adapter = BlazegraphAdapter(endpoint="http://localhost:9999/blazegraph")
|
||||
@@ -427,7 +427,7 @@ print(f"Cache size: {stats['cache_size']}")
|
||||
### Basic Bulk Loading
|
||||
|
||||
```python
|
||||
from semantica.triple_store import BulkLoader, BlazegraphAdapter
|
||||
from semantica.triplet_store import BulkLoader, BlazegraphAdapter
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
# Create bulk loader
|
||||
@@ -455,7 +455,7 @@ print(f"Throughput: {progress.metadata.get('throughput', 0):.0f} triples/sec")
|
||||
### Progress Tracking
|
||||
|
||||
```python
|
||||
from semantica.triple_store import BulkLoader, BlazegraphAdapter, LoadProgress
|
||||
from semantica.triplet_store import BulkLoader, BlazegraphAdapter, LoadProgress
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
loader = BulkLoader(batch_size=1000)
|
||||
@@ -476,7 +476,7 @@ progress = loader.load_triples(triples, adapter, progress_callback=progress_call
|
||||
### Pre-load Validation
|
||||
|
||||
```python
|
||||
from semantica.triple_store import BulkLoader
|
||||
from semantica.triplet_store import BulkLoader
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
loader = BulkLoader()
|
||||
@@ -501,7 +501,7 @@ print(f"Valid triples: {validation['valid_triples']}/{validation['total_triples'
|
||||
### Stream-based Loading
|
||||
|
||||
```python
|
||||
from semantica.triple_store import BulkLoader, BlazegraphAdapter
|
||||
from semantica.triplet_store import BulkLoader, BlazegraphAdapter
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
loader = BulkLoader(batch_size=1000)
|
||||
@@ -522,7 +522,7 @@ print(f"Loaded {progress.loaded_triples} triples from stream")
|
||||
### Blazegraph Adapter
|
||||
|
||||
```python
|
||||
from semantica.triple_store import BlazegraphAdapter
|
||||
from semantica.triplet_store import BlazegraphAdapter
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
# Create Blazegraph adapter
|
||||
@@ -548,7 +548,7 @@ print(f"Found {len(result['bindings'])} results")
|
||||
### Jena Adapter
|
||||
|
||||
```python
|
||||
from semantica.triple_store import JenaAdapter
|
||||
from semantica.triplet_store import JenaAdapter
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
# Create Jena adapter (in-memory)
|
||||
@@ -575,7 +575,7 @@ print(turtle)
|
||||
### RDF4J Adapter
|
||||
|
||||
```python
|
||||
from semantica.triple_store import RDF4JAdapter
|
||||
from semantica.triplet_store import RDF4JAdapter
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
# Create RDF4J adapter
|
||||
@@ -594,7 +594,7 @@ result = adapter.add_triples(triples)
|
||||
### Virtuoso Adapter
|
||||
|
||||
```python
|
||||
from semantica.triple_store import VirtuosoAdapter
|
||||
from semantica.triplet_store import VirtuosoAdapter
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
# Create Virtuoso adapter
|
||||
@@ -613,7 +613,7 @@ result = adapter.add_triples(triples)
|
||||
|
||||
## Algorithms and Methods
|
||||
|
||||
### Triple Store Management Algorithms
|
||||
### Triplet Store Management Algorithms
|
||||
|
||||
#### Store Registration
|
||||
**Algorithm**: Store type detection and adapter factory pattern
|
||||
@@ -783,9 +783,9 @@ cost = engine._estimate_query_cost(query)
|
||||
|
||||
### Methods
|
||||
|
||||
#### TripleManager Methods
|
||||
#### TripletManager Methods
|
||||
|
||||
- `register_store(store_id, store_type, endpoint, **config)`: Register triple store
|
||||
- `register_store(store_id, store_type, endpoint, **config)`: Register triplet store
|
||||
- `add_triple(triple, store_id, **options)`: Add single triple
|
||||
- `add_triples(triples, store_id, **options)`: Add multiple triples
|
||||
- `get_triple(subject, predicate, object, store_id, **options)`: Get triples matching pattern
|
||||
@@ -824,14 +824,14 @@ cost = engine._estimate_query_cost(query)
|
||||
|
||||
## Dataclasses
|
||||
|
||||
### TripleStore
|
||||
### TripletStore
|
||||
|
||||
Configuration dataclass for triple store instances.
|
||||
Configuration dataclass for triplet store instances.
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleStore
|
||||
from semantica.triplet_store import TripletStore
|
||||
|
||||
store = TripleStore(
|
||||
store = TripletStore(
|
||||
store_id="main",
|
||||
store_type="blazegraph",
|
||||
endpoint="http://localhost:9999/blazegraph/sparql",
|
||||
@@ -856,7 +856,7 @@ print(f"Type: {store.store_type}")
|
||||
Query execution result dataclass.
|
||||
|
||||
```python
|
||||
from semantica.triple_store import QueryEngine, QueryResult
|
||||
from semantica.triplet_store import QueryEngine, QueryResult
|
||||
|
||||
engine = QueryEngine()
|
||||
result: QueryResult = engine.execute_query(query, adapter)
|
||||
@@ -877,7 +877,7 @@ print(f"Execution time: {result.execution_time:.2f}s")
|
||||
Query execution plan dataclass.
|
||||
|
||||
```python
|
||||
from semantica.triple_store import QueryEngine, QueryPlan
|
||||
from semantica.triplet_store import QueryEngine, QueryPlan
|
||||
|
||||
engine = QueryEngine(enable_optimization=True)
|
||||
plan: QueryPlan = engine.plan_query(query)
|
||||
@@ -897,38 +897,38 @@ print(f"Execution steps: {plan.execution_steps}")
|
||||
### Environment Variables
|
||||
|
||||
```bash
|
||||
# Triple store configuration
|
||||
export TRIPLE_STORE_DEFAULT_STORE=main
|
||||
export TRIPLE_STORE_BATCH_SIZE=1000
|
||||
export TRIPLE_STORE_ENABLE_CACHING=true
|
||||
export TRIPLE_STORE_CACHE_SIZE=1000
|
||||
export TRIPLE_STORE_ENABLE_OPTIMIZATION=true
|
||||
export TRIPLE_STORE_MAX_RETRIES=3
|
||||
export TRIPLE_STORE_RETRY_DELAY=1.0
|
||||
export TRIPLE_STORE_TIMEOUT=30
|
||||
# Triplet store configuration
|
||||
export TRIPLET_STORE_DEFAULT_STORE=main
|
||||
export TRIPLET_STORE_BATCH_SIZE=1000
|
||||
export TRIPLET_STORE_ENABLE_CACHING=true
|
||||
export TRIPLET_STORE_CACHE_SIZE=1000
|
||||
export TRIPLET_STORE_ENABLE_OPTIMIZATION=true
|
||||
export TRIPLET_STORE_MAX_RETRIES=3
|
||||
export TRIPLET_STORE_RETRY_DELAY=1.0
|
||||
export TRIPLET_STORE_TIMEOUT=30
|
||||
|
||||
# Store endpoints
|
||||
export TRIPLE_STORE_BLAZEGRAPH_ENDPOINT=http://localhost:9999/blazegraph
|
||||
export TRIPLE_STORE_JENA_ENDPOINT=http://localhost:3030/ds
|
||||
export TRIPLE_STORE_RDF4J_ENDPOINT=http://localhost:8080/rdf4j-server
|
||||
export TRIPLE_STORE_VIRTUOSO_ENDPOINT=http://localhost:8890/sparql
|
||||
export TRIPLET_STORE_BLAZEGRAPH_ENDPOINT=http://localhost:9999/blazegraph
|
||||
export TRIPLET_STORE_JENA_ENDPOINT=http://localhost:3030/ds
|
||||
export TRIPLET_STORE_RDF4J_ENDPOINT=http://localhost:8080/rdf4j-server
|
||||
export TRIPLET_STORE_VIRTUOSO_ENDPOINT=http://localhost:8890/sparql
|
||||
```
|
||||
|
||||
### Programmatic Configuration
|
||||
|
||||
```python
|
||||
from semantica.triple_store.config import triple_store_config
|
||||
from semantica.triplet_store.config import triplet_store_config
|
||||
|
||||
# Get configuration
|
||||
batch_size = triple_store_config.get("batch_size", default=1000)
|
||||
enable_caching = triple_store_config.get("enable_caching", default=True)
|
||||
batch_size = triplet_store_config.get("batch_size", default=1000)
|
||||
enable_caching = triplet_store_config.get("enable_caching", default=True)
|
||||
|
||||
# Set configuration
|
||||
triple_store_config.set("batch_size", 2000)
|
||||
triple_store_config.set("enable_caching", False)
|
||||
triplet_store_config.set("batch_size", 2000)
|
||||
triplet_store_config.set("enable_caching", False)
|
||||
|
||||
# Update with dictionary
|
||||
triple_store_config.update({
|
||||
triplet_store_config.update({
|
||||
"batch_size": 2000,
|
||||
"enable_caching": True,
|
||||
"cache_size": 2000
|
||||
@@ -939,7 +939,7 @@ triple_store_config.update({
|
||||
|
||||
```yaml
|
||||
# config.yaml
|
||||
triple_store:
|
||||
triplet_store:
|
||||
default_store: main
|
||||
batch_size: 1000
|
||||
enable_caching: true
|
||||
@@ -956,11 +956,11 @@ triple_store:
|
||||
|
||||
## Advanced Examples
|
||||
|
||||
### Complete Triple Store Pipeline
|
||||
### Complete Triplet Store Pipeline
|
||||
|
||||
```python
|
||||
from semantica.triple_store import (
|
||||
TripleManager,
|
||||
from semantica.triplet_store import (
|
||||
TripletManager,
|
||||
QueryEngine,
|
||||
BulkLoader,
|
||||
register_store,
|
||||
@@ -980,7 +980,7 @@ triples = [
|
||||
result = add_triples(triples, store_id="main", batch_size=100)
|
||||
|
||||
# 3. Execute queries
|
||||
from semantica.triple_store import BlazegraphAdapter
|
||||
from semantica.triplet_store import BlazegraphAdapter
|
||||
adapter = BlazegraphAdapter(endpoint="http://localhost:9999/blazegraph")
|
||||
query = "SELECT ?s ?p ?o WHERE { ?s ?p ?o } LIMIT 10"
|
||||
query_result = execute_query(query, adapter)
|
||||
@@ -992,10 +992,10 @@ print(f"Query returned {len(query_result.bindings)} results")
|
||||
### Multi-Store Operations
|
||||
|
||||
```python
|
||||
from semantica.triple_store import TripleManager
|
||||
from semantica.triplet_store import TripletManager
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
|
||||
# Register multiple stores
|
||||
manager.register_store("primary", "blazegraph", "http://localhost:9999/blazegraph")
|
||||
@@ -1012,7 +1012,7 @@ manager.add_triple(triple, store_id="backup")
|
||||
### Query Optimization Workflow
|
||||
|
||||
```python
|
||||
from semantica.triple_store import QueryEngine, BlazegraphAdapter
|
||||
from semantica.triplet_store import QueryEngine, BlazegraphAdapter
|
||||
|
||||
engine = QueryEngine(enable_optimization=True, enable_caching=True)
|
||||
adapter = BlazegraphAdapter(endpoint="http://localhost:9999/blazegraph")
|
||||
@@ -1040,7 +1040,7 @@ print(f"Optimized: {result.metadata.get('optimized', False)}")
|
||||
### Bulk Loading with Validation
|
||||
|
||||
```python
|
||||
from semantica.triple_store import BulkLoader, BlazegraphAdapter
|
||||
from semantica.triplet_store import BulkLoader, BlazegraphAdapter
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
loader = BulkLoader(batch_size=1000, max_retries=3)
|
||||
@@ -1066,22 +1066,22 @@ else:
|
||||
### Custom Method Registration
|
||||
|
||||
```python
|
||||
from semantica.triple_store.registry import method_registry
|
||||
from semantica.triple_store import add_triple
|
||||
from semantica.triplet_store.registry import method_registry
|
||||
from semantica.triplet_store import add_triple
|
||||
|
||||
# Register custom add method
|
||||
def custom_add_triple(triple, store_id=None, **options):
|
||||
# Custom logic
|
||||
print(f"Custom add: {triple.subject}")
|
||||
# Call default implementation
|
||||
from semantica.triple_store.methods import _get_manager
|
||||
from semantica.triplet_store.methods import _get_manager
|
||||
manager = _get_manager()
|
||||
return manager.add_triple(triple, store_id=store_id, **options)
|
||||
|
||||
method_registry.register("add", "custom", custom_add_triple)
|
||||
|
||||
# Use custom method
|
||||
from semantica.triple_store.methods import add_triple
|
||||
from semantica.triplet_store.methods import add_triple
|
||||
result = add_triple(triple, store_id="main", method="custom")
|
||||
```
|
||||
|
||||
+3
-3
@@ -16,7 +16,7 @@ Main Classes:
|
||||
- VirtuosoAdapter: Main Virtuoso integration adapter
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.triple_store import VirtuosoAdapter
|
||||
>>> from semantica.triplet_store import VirtuosoAdapter
|
||||
>>> adapter = VirtuosoAdapter(endpoint="http://localhost:8890/sparql", username="dba", password="dba")
|
||||
>>> result = adapter.execute_sparql(sparql_query)
|
||||
>>> load_result = adapter.bulk_load(triples, graph="http://example.org/graph")
|
||||
@@ -147,7 +147,7 @@ class VirtuosoAdapter:
|
||||
Query results
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="triple_store",
|
||||
module="triplet_store",
|
||||
submodule="VirtuosoAdapter",
|
||||
message="Executing SPARQL query on Virtuoso",
|
||||
)
|
||||
@@ -240,7 +240,7 @@ class VirtuosoAdapter:
|
||||
Load status
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="triple_store",
|
||||
module="triplet_store",
|
||||
submodule="VirtuosoAdapter",
|
||||
message=f"Bulk loading {len(triples)} triples to Virtuoso",
|
||||
)
|
||||
@@ -71,7 +71,6 @@ SUPPORTED_RDF_FORMATS = ["turtle", "rdfxml", "jsonld", "n3", "ntriples"]
|
||||
# Supported Vector Store Backends
|
||||
SUPPORTED_VECTOR_STORES = [
|
||||
"faiss",
|
||||
"pinecone",
|
||||
"weaviate",
|
||||
"qdrant",
|
||||
"milvus",
|
||||
|
||||
@@ -121,7 +121,7 @@ class ConsoleProgressDisplay(ProgressDisplay):
|
||||
"semantic_extract": "🎯",
|
||||
"seed": "🌱",
|
||||
"split": "✂️",
|
||||
"triple_store": "🗄️",
|
||||
"triplet_store": "🗄️",
|
||||
"vector_store": "📊",
|
||||
"export": "💾",
|
||||
"reasoning": "🤔",
|
||||
@@ -176,7 +176,7 @@ class ConsoleProgressDisplay(ProgressDisplay):
|
||||
"semantic_extract": "is extracting",
|
||||
"seed": "is seeding",
|
||||
"split": "is splitting",
|
||||
"triple_store": "is storing",
|
||||
"triplet_store": "is storing",
|
||||
"vector_store": "is indexing",
|
||||
"export": "is exporting",
|
||||
"reasoning": "is reasoning",
|
||||
@@ -352,7 +352,7 @@ class JupyterProgressDisplay(ProgressDisplay):
|
||||
"semantic_extract": "🎯",
|
||||
"seed": "🌱",
|
||||
"split": "✂️",
|
||||
"triple_store": "🗄️",
|
||||
"triplet_store": "🗄️",
|
||||
"vector_store": "📊",
|
||||
"export": "💾",
|
||||
"reasoning": "🤔",
|
||||
@@ -405,7 +405,7 @@ class JupyterProgressDisplay(ProgressDisplay):
|
||||
"semantic_extract": "is extracting",
|
||||
"seed": "is seeding",
|
||||
"split": "is splitting",
|
||||
"triple_store": "is storing",
|
||||
"triplet_store": "is storing",
|
||||
"vector_store": "is indexing",
|
||||
"export": "is exporting",
|
||||
"reasoning": "is reasoning",
|
||||
@@ -901,7 +901,7 @@ class ProgressTracker:
|
||||
"semantic_extract": "🎯",
|
||||
"seed": "🌱",
|
||||
"split": "✂️",
|
||||
"triple_store": "🗄️",
|
||||
"triplet_store": "🗄️",
|
||||
"vector_store": "📊",
|
||||
"export": "💾",
|
||||
"reasoning": "🤔",
|
||||
|
||||
@@ -3,7 +3,7 @@ Vector Store Management Module
|
||||
|
||||
This module provides comprehensive vector storage and retrieval capabilities for the
|
||||
Semantica framework, including support for multiple vector store backends (FAISS,
|
||||
Pinecone, Weaviate, Qdrant, Milvus), hybrid search combining vector similarity and
|
||||
Weaviate, Qdrant, Milvus), hybrid search combining vector similarity and
|
||||
metadata filtering, metadata management, and namespace isolation.
|
||||
|
||||
Algorithms Used:
|
||||
@@ -50,25 +50,30 @@ Namespace Management:
|
||||
|
||||
Adapter Pattern:
|
||||
- FAISS Adapter: Local vector storage, FAISS index management, index persistence (save/load), batch operations, multiple index types support
|
||||
- Pinecone Adapter: Cloud vector database integration, HTTP API communication, index management, upsert operations, query operations, metadata filtering
|
||||
- Weaviate Adapter: GraphQL-based queries, schema management, object-oriented storage, rich metadata support, batch operations
|
||||
- Qdrant Adapter: REST API communication, collection management, vector operations, payload (metadata) filtering, batch operations
|
||||
- Milvus Adapter: gRPC communication, collection management, vector operations, metadata filtering, batch operations
|
||||
- Unified Interface: Common interface for all adapters, backend-specific operation delegation, adapter factory pattern, connection management
|
||||
- Weaviate Adapter: Schema-aware storage, GraphQL query support, object-oriented data model, batch operations, schema management
|
||||
- Qdrant Adapter: Point-based storage, payload filtering, collection management, optimized search, batch operations
|
||||
- Milvus Adapter: Scalable vector database, collection management, partitioning, complex querying, index building
|
||||
|
||||
Batch Operations:
|
||||
- Batch Vector Operations: Chunking algorithm (fixed-size batch creation), batch processing, progress tracking, error handling per batch, retry mechanism
|
||||
- Batch Indexing: Batch vector addition to index, incremental index updates, batch index training, batch index optimization
|
||||
- Batch Search: Batch query processing, parallel search execution (when supported), result aggregation, batch result formatting
|
||||
Supported Backends:
|
||||
- FAISS: In-memory/local disk (Facebook AI Similarity Search)
|
||||
- Weaviate: Cloud/Self-hosted (Schema-aware vector database)
|
||||
- Qdrant: Cloud/Self-hosted (Vector database for the next generation of AI)
|
||||
- Milvus: Cloud/Self-hosted (Highly scalable vector database)
|
||||
- InMemory: Simple list-based storage for testing/small datasets
|
||||
|
||||
Performance Optimization:
|
||||
- Vector Normalization: L2 normalization for cosine similarity, normalization caching, batch normalization
|
||||
- Index Optimization: Index parameter tuning, index rebuilding for better performance, memory optimization, search speed optimization
|
||||
- Caching: Query result caching, vector caching, metadata caching, cache invalidation strategies
|
||||
- Parallel Processing: Batch-level parallelization, multi-threaded search (when supported), concurrent index operations
|
||||
Configuration:
|
||||
- Environment variables (SEMANTICA_VECTOR_STORE_*)
|
||||
- Configuration files (yaml/json)
|
||||
- Runtime configuration via VectorStoreConfig
|
||||
|
||||
Dependencies:
|
||||
- faiss-cpu (or faiss-gpu)
|
||||
- weaviate-client
|
||||
- qdrant-client
|
||||
- pymilvus
|
||||
|
||||
Key Features:
|
||||
- Multi-backend vector store support (FAISS, Pinecone, Weaviate, Qdrant, Milvus)
|
||||
- Multi-backend vector store support (FAISS, Weaviate, Qdrant, Milvus)
|
||||
- Vector indexing and similarity search
|
||||
- Metadata indexing and filtering
|
||||
- Hybrid search combining vector and metadata queries
|
||||
@@ -84,7 +89,6 @@ Main Classes:
|
||||
- VectorRetriever: Vector retrieval and similarity search
|
||||
- VectorManager: Vector store management and operations
|
||||
- FAISSAdapter: FAISS integration for local vector storage
|
||||
- PineconeAdapter: Pinecone cloud vector database integration
|
||||
- WeaviateAdapter: Weaviate vector database integration
|
||||
- QdrantAdapter: Qdrant vector database integration
|
||||
- MilvusAdapter: Milvus vector database integration
|
||||
@@ -141,12 +145,6 @@ from .methods import (
|
||||
)
|
||||
from .milvus_adapter import MilvusAdapter, MilvusClient, MilvusCollection, MilvusSearch
|
||||
from .namespace_manager import Namespace, NamespaceManager
|
||||
from .pinecone_adapter import (
|
||||
PineconeAdapter,
|
||||
PineconeIndex,
|
||||
PineconeMetadata,
|
||||
PineconeQuery,
|
||||
)
|
||||
from .qdrant_adapter import QdrantAdapter, QdrantClient, QdrantCollection, QdrantSearch
|
||||
from .registry import MethodRegistry, method_registry
|
||||
from .vector_store import VectorIndexer, VectorManager, VectorRetriever, VectorStore
|
||||
@@ -168,11 +166,6 @@ __all__ = [
|
||||
"FAISSIndex",
|
||||
"FAISSSearch",
|
||||
"FAISSIndexBuilder",
|
||||
# Pinecone
|
||||
"PineconeAdapter",
|
||||
"PineconeIndex",
|
||||
"PineconeQuery",
|
||||
"PineconeMetadata",
|
||||
# Weaviate
|
||||
"WeaviateAdapter",
|
||||
"WeaviateClient",
|
||||
|
||||
@@ -116,8 +116,6 @@ class VectorStoreConfig:
|
||||
"VECTOR_STORE_ENABLE_HYBRID_SEARCH": "enable_hybrid_search",
|
||||
"VECTOR_STORE_NAMESPACE": "default_namespace",
|
||||
"VECTOR_STORE_FAISS_INDEX_TYPE": "faiss_index_type",
|
||||
"VECTOR_STORE_PINECONE_API_KEY": "pinecone_api_key",
|
||||
"VECTOR_STORE_PINECONE_ENVIRONMENT": "pinecone_environment",
|
||||
"VECTOR_STORE_WEAVIATE_URL": "weaviate_url",
|
||||
"VECTOR_STORE_QDRANT_URL": "qdrant_url",
|
||||
"VECTOR_STORE_MILVUS_HOST": "milvus_host",
|
||||
|
||||
@@ -1,510 +0,0 @@
|
||||
"""
|
||||
Pinecone Adapter Module
|
||||
|
||||
This module provides Pinecone cloud vector database integration for vector storage
|
||||
and similarity search in the Semantica framework, supporting serverless and pod-based
|
||||
deployments with namespace isolation and metadata filtering.
|
||||
|
||||
Key Features:
|
||||
- Cloud-based vector storage and retrieval
|
||||
- Serverless and pod-based index specifications
|
||||
- Namespace isolation for multi-tenant support
|
||||
- Metadata filtering and querying
|
||||
- Batch upsert and query operations
|
||||
- Index statistics and monitoring
|
||||
- Optional dependency handling
|
||||
|
||||
Main Classes:
|
||||
- PineconeAdapter: Main Pinecone adapter for cloud vector operations
|
||||
- PineconeIndex: Pinecone index wrapper with operations
|
||||
- PineconeQuery: Pinecone query builder and executor
|
||||
- PineconeMetadata: Metadata validation and sanitization
|
||||
|
||||
Example Usage:
|
||||
>>> from semantica.vector_store import PineconeAdapter
|
||||
>>> adapter = PineconeAdapter(api_key="your-api-key")
|
||||
>>> adapter.connect()
|
||||
>>> index = adapter.create_index("my-index", dimension=768, metric="cosine")
|
||||
>>> adapter.upsert_vectors(vectors, ids, metadata, namespace="docs")
|
||||
>>> results = adapter.query_vectors(query_vector, top_k=10, namespace="docs")
|
||||
>>> stats = adapter.get_stats()
|
||||
|
||||
Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
# Optional Pinecone import
|
||||
try:
|
||||
import pinecone
|
||||
from pinecone import Pinecone, PodSpec, ServerlessSpec
|
||||
|
||||
PINECONE_AVAILABLE = True
|
||||
except ImportError:
|
||||
PINECONE_AVAILABLE = False
|
||||
pinecone = None
|
||||
Pinecone = None
|
||||
ServerlessSpec = None
|
||||
PodSpec = None
|
||||
|
||||
|
||||
class PineconeIndex:
|
||||
"""Pinecone index wrapper."""
|
||||
|
||||
def __init__(self, index: Any, index_name: str):
|
||||
"""Initialize Pinecone index wrapper."""
|
||||
self.index = index
|
||||
self.index_name = index_name
|
||||
self.logger = get_logger("pinecone_index")
|
||||
|
||||
def upsert_vectors(
|
||||
self, vectors: List[Dict[str, Any]], namespace: Optional[str] = None, **options
|
||||
) -> Dict[str, Any]:
|
||||
"""Upsert vectors to index."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
response = self.index.upsert(
|
||||
vectors=vectors, namespace=namespace, **options
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to upsert vectors: {str(e)}")
|
||||
|
||||
def query_vectors(
|
||||
self,
|
||||
query_vector: np.ndarray,
|
||||
top_k: int = 10,
|
||||
namespace: Optional[str] = None,
|
||||
filter: Optional[Dict[str, Any]] = None,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""Query similar vectors."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
response = self.index.query(
|
||||
vector=query_vector.tolist(),
|
||||
top_k=top_k,
|
||||
namespace=namespace,
|
||||
filter=filter,
|
||||
include_metadata=True,
|
||||
**options,
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to query vectors: {str(e)}")
|
||||
|
||||
def delete_vectors(
|
||||
self, ids: List[str], namespace: Optional[str] = None, **options
|
||||
) -> Dict[str, Any]:
|
||||
"""Delete vectors from index."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
response = self.index.delete(ids=ids, namespace=namespace, **options)
|
||||
return response
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to delete vectors: {str(e)}")
|
||||
|
||||
def fetch_vectors(
|
||||
self, ids: List[str], namespace: Optional[str] = None, **options
|
||||
) -> Dict[str, Any]:
|
||||
"""Fetch vectors by IDs."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
response = self.index.fetch(ids=ids, namespace=namespace, **options)
|
||||
return response
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to fetch vectors: {str(e)}")
|
||||
|
||||
def describe_index_stats(self, namespace: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Get index statistics."""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
stats = self.index.describe_index_stats(namespace=namespace)
|
||||
return stats
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to get index stats: {str(e)}")
|
||||
|
||||
|
||||
class PineconeQuery:
|
||||
"""Pinecone query builder."""
|
||||
|
||||
def __init__(self, index: PineconeIndex):
|
||||
"""Initialize Pinecone query builder."""
|
||||
self.index = index
|
||||
self.logger = get_logger("pinecone_query")
|
||||
|
||||
def build_query(
|
||||
self,
|
||||
query_vector: np.ndarray,
|
||||
top_k: int = 10,
|
||||
namespace: Optional[str] = None,
|
||||
filter: Optional[Dict[str, Any]] = None,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build query parameters."""
|
||||
return {
|
||||
"vector": query_vector.tolist(),
|
||||
"top_k": top_k,
|
||||
"namespace": namespace,
|
||||
"filter": filter,
|
||||
**options,
|
||||
}
|
||||
|
||||
def execute(self, query_params: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Execute query and format results."""
|
||||
response = self.index.query_vectors(**query_params)
|
||||
|
||||
results = []
|
||||
for match in response.get("matches", []):
|
||||
results.append(
|
||||
{
|
||||
"id": match.get("id"),
|
||||
"score": match.get("score", 0.0),
|
||||
"metadata": match.get("metadata", {}),
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
class PineconeMetadata:
|
||||
"""Pinecone metadata handler."""
|
||||
|
||||
@staticmethod
|
||||
def validate_metadata(metadata: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Validate and sanitize metadata."""
|
||||
# Pinecone metadata restrictions
|
||||
validated = {}
|
||||
|
||||
for key, value in metadata.items():
|
||||
# Convert to allowed types
|
||||
if isinstance(value, (str, int, float, bool, list)):
|
||||
validated[key] = value
|
||||
elif isinstance(value, dict):
|
||||
# Nested dicts not directly supported
|
||||
validated[key] = str(value)
|
||||
else:
|
||||
validated[key] = str(value)
|
||||
|
||||
return validated
|
||||
|
||||
|
||||
class PineconeAdapter:
|
||||
"""
|
||||
Pinecone adapter for vector storage and similarity search.
|
||||
|
||||
• Pinecone connection and authentication
|
||||
• Vector storage and retrieval
|
||||
• Similarity search and filtering
|
||||
• Namespace and index management
|
||||
• Performance optimization
|
||||
• Error handling and recovery
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, api_key: Optional[str] = None, environment: Optional[str] = None, **config
|
||||
):
|
||||
"""Initialize Pinecone adapter."""
|
||||
self.logger = get_logger("pinecone_adapter")
|
||||
self.config = config
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
self.api_key = api_key or config.get("api_key")
|
||||
self.environment = environment or config.get("environment")
|
||||
|
||||
self.client: Optional[Any] = None
|
||||
self.index: Optional[PineconeIndex] = None
|
||||
self.query_builder: Optional[PineconeQuery] = None
|
||||
|
||||
# Check Pinecone availability
|
||||
if not PINECONE_AVAILABLE:
|
||||
self.logger.warning(
|
||||
"Pinecone not available. Install with: pip install pinecone-client"
|
||||
)
|
||||
|
||||
def connect(self, api_key: Optional[str] = None, **options) -> bool:
|
||||
"""
|
||||
Connect to Pinecone service.
|
||||
|
||||
Args:
|
||||
api_key: Pinecone API key
|
||||
**options: Connection options
|
||||
|
||||
Returns:
|
||||
True if connected successfully
|
||||
"""
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError(
|
||||
"Pinecone is not available. Install it with: pip install pinecone-client"
|
||||
)
|
||||
|
||||
api_key = api_key or self.api_key
|
||||
if not api_key:
|
||||
raise ValidationError("Pinecone API key is required")
|
||||
|
||||
try:
|
||||
self.client = Pinecone(api_key=api_key)
|
||||
self.logger.info("Connected to Pinecone")
|
||||
return True
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to connect to Pinecone: {str(e)}")
|
||||
|
||||
def create_index(
|
||||
self,
|
||||
index_name: str,
|
||||
dimension: int,
|
||||
metric: str = "cosine",
|
||||
spec: Optional[Dict[str, Any]] = None,
|
||||
**options,
|
||||
) -> PineconeIndex:
|
||||
"""
|
||||
Create new vector index.
|
||||
|
||||
Args:
|
||||
index_name: Name of the index
|
||||
dimension: Vector dimension
|
||||
metric: Distance metric ("cosine", "euclidean", "dotproduct")
|
||||
spec: Index specification (serverless or pod)
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
PineconeIndex instance
|
||||
"""
|
||||
if self.client is None:
|
||||
self.connect()
|
||||
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
# Check if index exists
|
||||
existing_indexes = [idx.name for idx in self.client.list_indexes()]
|
||||
if index_name in existing_indexes:
|
||||
self.logger.info(f"Index {index_name} already exists")
|
||||
return self.get_index(index_name)
|
||||
|
||||
# Create index specification
|
||||
if spec is None:
|
||||
spec = ServerlessSpec(cloud="aws", region="us-east-1")
|
||||
|
||||
# Create index
|
||||
self.client.create_index(
|
||||
name=index_name,
|
||||
dimension=dimension,
|
||||
metric=metric,
|
||||
spec=spec,
|
||||
**options,
|
||||
)
|
||||
|
||||
self.logger.info(f"Created Pinecone index: {index_name}")
|
||||
return self.get_index(index_name)
|
||||
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to create index: {str(e)}")
|
||||
|
||||
def get_index(self, index_name: str) -> PineconeIndex:
|
||||
"""
|
||||
Get existing index.
|
||||
|
||||
Args:
|
||||
index_name: Name of the index
|
||||
|
||||
Returns:
|
||||
PineconeIndex instance
|
||||
"""
|
||||
if self.client is None:
|
||||
self.connect()
|
||||
|
||||
if not PINECONE_AVAILABLE:
|
||||
raise ProcessingError("Pinecone not available")
|
||||
|
||||
try:
|
||||
index = self.client.Index(index_name)
|
||||
self.index = PineconeIndex(index, index_name)
|
||||
self.query_builder = PineconeQuery(self.index)
|
||||
return self.index
|
||||
except Exception as e:
|
||||
raise ProcessingError(f"Failed to get index: {str(e)}")
|
||||
|
||||
def upsert_vectors(
|
||||
self,
|
||||
vectors: List[Union[np.ndarray, List[float]]],
|
||||
ids: List[str],
|
||||
metadata: Optional[List[Dict[str, Any]]] = None,
|
||||
namespace: Optional[str] = None,
|
||||
**options,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Insert or update vectors.
|
||||
|
||||
Args:
|
||||
vectors: List of vectors
|
||||
ids: Vector IDs
|
||||
metadata: Vector metadata
|
||||
namespace: Namespace name
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Upsert response
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="vector_store",
|
||||
submodule="PineconeAdapter",
|
||||
message=f"Upserting {len(vectors)} vectors to Pinecone",
|
||||
)
|
||||
|
||||
try:
|
||||
if self.index is None:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message="Index not initialized"
|
||||
)
|
||||
raise ProcessingError(
|
||||
"Index not initialized. Call create_index() or get_index() first."
|
||||
)
|
||||
|
||||
# Format vectors
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Formatting vectors..."
|
||||
)
|
||||
formatted_vectors = []
|
||||
for i, vector in enumerate(vectors):
|
||||
if isinstance(vector, np.ndarray):
|
||||
vector = vector.tolist()
|
||||
|
||||
vector_data = {"id": ids[i], "values": vector}
|
||||
|
||||
if metadata and i < len(metadata):
|
||||
vector_data["metadata"] = PineconeMetadata.validate_metadata(
|
||||
metadata[i]
|
||||
)
|
||||
|
||||
formatted_vectors.append(vector_data)
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Upserting vectors to Pinecone..."
|
||||
)
|
||||
result = self.index.upsert_vectors(formatted_vectors, namespace, **options)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Upserted {len(vectors)} vectors",
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise
|
||||
|
||||
def query_vectors(
|
||||
self,
|
||||
query_vector: np.ndarray,
|
||||
top_k: int = 10,
|
||||
namespace: Optional[str] = None,
|
||||
filter: Optional[Dict[str, Any]] = None,
|
||||
**options,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Query similar vectors.
|
||||
|
||||
Args:
|
||||
query_vector: Query vector
|
||||
top_k: Number of results
|
||||
namespace: Namespace name
|
||||
filter: Metadata filter
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
List of search results
|
||||
"""
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="vector_store",
|
||||
submodule="PineconeAdapter",
|
||||
message=f"Querying {top_k} similar vectors from Pinecone",
|
||||
)
|
||||
|
||||
try:
|
||||
if self.query_builder is None:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message="Index not initialized"
|
||||
)
|
||||
raise ProcessingError(
|
||||
"Index not initialized. Call create_index() or get_index() first."
|
||||
)
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Building query..."
|
||||
)
|
||||
query_params = self.query_builder.build_query(
|
||||
query_vector, top_k, namespace, filter, **options
|
||||
)
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Executing query..."
|
||||
)
|
||||
results = self.query_builder.execute(query_params)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id,
|
||||
status="completed",
|
||||
message=f"Query completed: {len(results) if isinstance(results, list) else 'N/A'} results",
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(
|
||||
tracking_id, status="failed", message=str(e)
|
||||
)
|
||||
raise
|
||||
|
||||
def delete_vectors(
|
||||
self, ids: List[str], namespace: Optional[str] = None, **options
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Delete vectors from index.
|
||||
|
||||
Args:
|
||||
ids: Vector IDs to delete
|
||||
namespace: Namespace name
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Delete response
|
||||
"""
|
||||
if self.index is None:
|
||||
raise ProcessingError(
|
||||
"Index not initialized. Call create_index() or get_index() first."
|
||||
)
|
||||
|
||||
return self.index.delete_vectors(ids, namespace, **options)
|
||||
|
||||
def get_stats(self, namespace: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Get index statistics."""
|
||||
if self.index is None:
|
||||
raise ProcessingError(
|
||||
"Index not initialized. Call create_index() or get_index() first."
|
||||
)
|
||||
|
||||
stats = self.index.describe_index_stats(namespace)
|
||||
return {
|
||||
"total_vector_count": stats.get("total_vector_count", 0),
|
||||
"dimension": stats.get("dimension", 0),
|
||||
"index_fullness": stats.get("index_fullness", 0.0),
|
||||
"namespaces": stats.get("namespaces", {}),
|
||||
}
|
||||
@@ -58,8 +58,16 @@ class VectorStore:
|
||||
• Provides vector store operations
|
||||
"""
|
||||
|
||||
SUPPORTED_BACKENDS = {"faiss", "weaviate", "qdrant", "milvus", "inmemory"}
|
||||
|
||||
def __init__(self, backend="faiss", config=None, **kwargs):
|
||||
"""Initialize vector store."""
|
||||
if backend.lower() not in self.SUPPORTED_BACKENDS:
|
||||
raise ValueError(
|
||||
f"Unsupported backend: {backend}. "
|
||||
f"Supported backends are: {', '.join(sorted(self.SUPPORTED_BACKENDS))}"
|
||||
)
|
||||
|
||||
self.logger = get_logger("vector_store")
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Vector Store Module Usage Guide
|
||||
|
||||
This comprehensive guide demonstrates how to use the vector store module for vector storage and retrieval, supporting multiple vector store backends (FAISS, Pinecone, Weaviate, Qdrant, Milvus), hybrid search combining vector similarity and metadata filtering, metadata management, and namespace isolation.
|
||||
This comprehensive guide demonstrates how to use the vector store module for vector storage and retrieval, supporting multiple vector store backends (FAISS, Weaviate, Qdrant, Milvus), hybrid search combining vector similarity and metadata filtering, metadata management, and namespace isolation.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
@@ -687,34 +687,6 @@ distances, indices = adapter.search(index, query_vector, k=10)
|
||||
print(f"Found {len(indices)} similar vectors")
|
||||
```
|
||||
|
||||
### Pinecone Adapter
|
||||
|
||||
```python
|
||||
from semantica.vector_store import PineconeAdapter
|
||||
import numpy as np
|
||||
|
||||
# Create Pinecone adapter
|
||||
adapter = PineconeAdapter(api_key="your-api-key", environment="us-west1-gcp")
|
||||
|
||||
# Connect
|
||||
adapter.connect()
|
||||
|
||||
# Create index
|
||||
index = adapter.create_index("my-index", dimension=768, metric="cosine")
|
||||
|
||||
# Upsert vectors
|
||||
vectors = [np.random.rand(768).tolist() for _ in range(100)]
|
||||
ids = [f"vec_{i}" for i in range(100)]
|
||||
metadata = [{"category": "science"} for _ in range(100)]
|
||||
adapter.upsert_vectors(vectors, ids, metadata)
|
||||
|
||||
# Query
|
||||
query_vector = np.random.rand(768).tolist()
|
||||
results = adapter.query_vectors(query_vector, top_k=10, include_metadata=True)
|
||||
|
||||
print(f"Found {len(results)} results")
|
||||
```
|
||||
|
||||
### Weaviate Adapter
|
||||
|
||||
```python
|
||||
@@ -1104,10 +1076,6 @@ export VECTOR_STORE_NAMESPACE=default
|
||||
# FAISS configuration
|
||||
export VECTOR_STORE_FAISS_INDEX_TYPE=flat
|
||||
|
||||
# Pinecone configuration
|
||||
export VECTOR_STORE_PINECONE_API_KEY=your-api-key
|
||||
export VECTOR_STORE_PINECONE_ENVIRONMENT=us-west1-gcp
|
||||
|
||||
# Weaviate configuration
|
||||
export VECTOR_STORE_WEAVIATE_URL=http://localhost:8080
|
||||
|
||||
@@ -1153,8 +1121,6 @@ vector_store:
|
||||
enable_hybrid_search: true
|
||||
default_namespace: default
|
||||
faiss_index_type: flat
|
||||
pinecone_api_key: your-api-key
|
||||
pinecone_environment: us-west1-gcp
|
||||
weaviate_url: http://localhost:8080
|
||||
qdrant_url: http://localhost:6333
|
||||
milvus_host: localhost
|
||||
@@ -1204,7 +1170,7 @@ print(f"Found {len(results)} hybrid search results")
|
||||
### Multi-Backend Vector Store
|
||||
|
||||
```python
|
||||
from semantica.vector_store import FAISSAdapter, PineconeAdapter
|
||||
from semantica.vector_store import FAISSAdapter, WeaviateAdapter
|
||||
import numpy as np
|
||||
|
||||
# Local FAISS store
|
||||
@@ -1213,17 +1179,17 @@ faiss_index = faiss_adapter.create_index(index_type="flat", metric="L2")
|
||||
faiss_vectors = np.random.rand(1000, 768).astype('float32')
|
||||
faiss_adapter.add_vectors(faiss_index, faiss_vectors, ids=[f"faiss_{i}" for i in range(1000)])
|
||||
|
||||
# Cloud Pinecone store
|
||||
pinecone_adapter = PineconeAdapter(api_key="your-key")
|
||||
pinecone_adapter.connect()
|
||||
pinecone_index = pinecone_adapter.create_index("my-index", dimension=768)
|
||||
pinecone_vectors = [np.random.rand(768).tolist() for _ in range(1000)]
|
||||
pinecone_adapter.upsert_vectors(pinecone_vectors, [f"pinecone_{i}" for i in range(1000)])
|
||||
# Self-hosted Weaviate store
|
||||
weaviate_adapter = WeaviateAdapter(url="http://localhost:8080")
|
||||
weaviate_adapter.connect()
|
||||
weaviate_index = weaviate_adapter.create_index("my-index", dimension=768)
|
||||
weaviate_vectors = [np.random.rand(768).tolist() for _ in range(1000)]
|
||||
weaviate_adapter.upsert_vectors(weaviate_vectors, [f"weaviate_{i}" for i in range(1000)])
|
||||
|
||||
# Search both
|
||||
query_vector = np.random.rand(768)
|
||||
faiss_results = faiss_adapter.search(faiss_index, query_vector, k=10)
|
||||
pinecone_results = pinecone_adapter.query_vectors(query_vector, top_k=10)
|
||||
weaviate_results = weaviate_adapter.query_vectors(query_vector, top_k=10)
|
||||
```
|
||||
|
||||
### Hybrid Search with Custom Ranking
|
||||
|
||||
@@ -34,10 +34,19 @@ License: MIT
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
try:
|
||||
import numpy as np
|
||||
except ImportError:
|
||||
np = None
|
||||
|
||||
try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
except ImportError:
|
||||
px = None
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -68,6 +77,23 @@ class AnalyticsVisualizer:
|
||||
except (KeyError, AttributeError):
|
||||
self.color_scheme = ColorScheme.DEFAULT
|
||||
|
||||
def _check_dependencies(self):
|
||||
"""Check if dependencies are available."""
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for analytics visualization. "
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
if np is None:
|
||||
raise ProcessingError(
|
||||
"NumPy is required for analytics visualization. "
|
||||
"Install with: pip install numpy"
|
||||
)
|
||||
|
||||
def visualize_centrality(self, *args, **kwargs):
|
||||
"""Alias for visualize_centrality_rankings."""
|
||||
return self.visualize_centrality_rankings(*args, **kwargs)
|
||||
|
||||
def visualize_centrality_rankings(
|
||||
self,
|
||||
centrality: Dict[str, Any],
|
||||
@@ -91,6 +117,7 @@ class AnalyticsVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="visualization",
|
||||
submodule="AnalyticsVisualizer",
|
||||
@@ -188,6 +215,7 @@ class AnalyticsVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing community structure")
|
||||
|
||||
# Use KG visualizer for community visualization
|
||||
@@ -217,6 +245,7 @@ class AnalyticsVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing connectivity analysis")
|
||||
|
||||
# Extract metrics
|
||||
@@ -291,6 +320,7 @@ class AnalyticsVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing degree distribution")
|
||||
|
||||
# Calculate degrees
|
||||
@@ -360,6 +390,7 @@ class AnalyticsVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing graph metrics dashboard")
|
||||
|
||||
# Extract key metrics
|
||||
@@ -498,6 +529,7 @@ class AnalyticsVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing centrality comparison")
|
||||
|
||||
# Extract top nodes for each centrality type
|
||||
|
||||
@@ -35,10 +35,16 @@ from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
import seaborn as sns
|
||||
from plotly.subplots import make_subplots
|
||||
|
||||
try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
except ImportError:
|
||||
px = None
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
from sklearn.decomposition import PCA
|
||||
from sklearn.manifold import TSNE
|
||||
|
||||
@@ -85,6 +91,14 @@ class EmbeddingVisualizer:
|
||||
self.color_scheme = ColorScheme.DEFAULT
|
||||
self.point_size = config.get("point_size", 5)
|
||||
|
||||
def _check_dependencies(self):
|
||||
"""Check if dependencies are available."""
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for embedding visualization. "
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_2d_projection(
|
||||
self,
|
||||
embeddings: np.ndarray,
|
||||
@@ -92,17 +106,30 @@ class EmbeddingVisualizer:
|
||||
method: str = "umap",
|
||||
output: str = "interactive",
|
||||
file_path: Optional[Union[str, Path]] = None,
|
||||
color_by: Optional[List[Any]] = None,
|
||||
size_by: Optional[List[float]] = None,
|
||||
hover_data: Optional[List[Dict[str, Any]]] = None,
|
||||
**options,
|
||||
) -> Optional[Any]:
|
||||
"""
|
||||
Visualize embeddings in 2D using dimensionality reduction.
|
||||
|
||||
Implements the 5-step visualization process:
|
||||
1. Problem setting: Dimensionality reduction choice
|
||||
2. Data analysis: Logs embedding statistics
|
||||
3. Layout: 2D Projection (UMAP/t-SNE/PCA)
|
||||
4. Styling: Configurable color and size mapping
|
||||
5. Interaction: Rich hover data
|
||||
|
||||
Args:
|
||||
embeddings: Embedding matrix (n_samples, n_features)
|
||||
labels: Optional labels for coloring points
|
||||
labels: Optional labels for points (used as default color_by if provided)
|
||||
method: Reduction method ("umap", "tsne", "pca")
|
||||
output: Output type ("interactive", "html", "png", "svg")
|
||||
file_path: Output file path
|
||||
color_by: List of values to map to color (overrides labels)
|
||||
size_by: List of values to map to point size
|
||||
hover_data: List of dictionaries containing metadata for each point
|
||||
**options: Additional options:
|
||||
- n_components: Number of components (default: 2)
|
||||
- perplexity: Perplexity for t-SNE
|
||||
@@ -111,6 +138,7 @@ class EmbeddingVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="visualization",
|
||||
submodule="EmbeddingVisualizer",
|
||||
@@ -119,6 +147,10 @@ class EmbeddingVisualizer:
|
||||
|
||||
try:
|
||||
self.logger.info(f"Visualizing 2D projection using {method}")
|
||||
|
||||
# Step 2: Data Analysis
|
||||
n_samples, n_features = embeddings.shape
|
||||
self.logger.info(f"Embedding Analysis: {n_samples} samples, {n_features} dimensions")
|
||||
|
||||
if embeddings.shape[1] <= 2:
|
||||
# Already 2D or less, use directly
|
||||
@@ -138,7 +170,14 @@ class EmbeddingVisualizer:
|
||||
tracking_id, message="Generating visualization..."
|
||||
)
|
||||
result = self._visualize_2d_plotly(
|
||||
projected, labels, output, file_path, **options
|
||||
projected,
|
||||
labels,
|
||||
output,
|
||||
file_path,
|
||||
color_by=color_by,
|
||||
size_by=size_by,
|
||||
hover_data=hover_data,
|
||||
**options
|
||||
)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
@@ -176,6 +215,7 @@ class EmbeddingVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="visualization",
|
||||
submodule="EmbeddingVisualizer",
|
||||
@@ -238,6 +278,7 @@ class EmbeddingVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="visualization",
|
||||
submodule="EmbeddingVisualizer",
|
||||
@@ -340,6 +381,7 @@ class EmbeddingVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="visualization",
|
||||
submodule="EmbeddingVisualizer",
|
||||
@@ -444,6 +486,7 @@ class EmbeddingVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="visualization",
|
||||
submodule="EmbeddingVisualizer",
|
||||
|
||||
@@ -33,12 +33,23 @@ License: MIT
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
|
||||
try:
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
mpatches = None
|
||||
plt = None
|
||||
|
||||
try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
except ImportError:
|
||||
px = None
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -96,25 +107,47 @@ class KGVisualizer:
|
||||
self.hierarchical_layout = HierarchicalLayout(**config)
|
||||
self.circular_layout = CircularLayout(**config)
|
||||
|
||||
def _check_dependencies(self):
|
||||
"""Check if dependencies are available."""
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for KG visualization. "
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_network(
|
||||
self,
|
||||
graph: Dict[str, Any],
|
||||
output: str = "interactive",
|
||||
file_path: Optional[Union[str, Path]] = None,
|
||||
node_color_by: str = "type",
|
||||
node_size_by: Optional[str] = None,
|
||||
hover_data: Optional[List[str]] = None,
|
||||
**options,
|
||||
) -> Optional[Any]:
|
||||
"""
|
||||
Visualize knowledge graph as interactive network.
|
||||
|
||||
Implements the 5-step visualization process:
|
||||
1. Problem setting: implicit in graph selection
|
||||
2. Data analysis: logs graph statistics
|
||||
3. Layout: configurable via options
|
||||
4. Styling: configurable node color/size mappings
|
||||
5. Interaction: rich hover data and zoom capabilities
|
||||
|
||||
Args:
|
||||
graph: Knowledge graph dictionary with entities and relationships
|
||||
output: Output type ("interactive", "html", "png", "svg")
|
||||
file_path: Output file path (required for non-interactive)
|
||||
node_color_by: Property to map to node color (default: "type")
|
||||
node_size_by: Property to map to node size (default: fixed)
|
||||
hover_data: List of properties to show in hover tooltip
|
||||
**options: Additional visualization options
|
||||
|
||||
Returns:
|
||||
Plotly figure (if interactive) or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="visualization",
|
||||
submodule="KGVisualizer",
|
||||
@@ -137,18 +170,34 @@ class KGVisualizer:
|
||||
)
|
||||
raise ProcessingError("No entities found in graph")
|
||||
|
||||
# Step 2: Data Analysis - Understand data structure
|
||||
nodes = self._extract_nodes(entities)
|
||||
edges = self._extract_edges(relationships, entities)
|
||||
|
||||
num_nodes = len(nodes)
|
||||
num_edges = len(edges)
|
||||
entity_types = set(n.get("type", "unknown") for n in nodes)
|
||||
|
||||
self.logger.info(f"Graph Structure Analysis: {num_nodes} nodes, {num_edges} edges")
|
||||
self.logger.info(f"Entity Types: {', '.join(sorted(entity_types))}")
|
||||
|
||||
# Build node and edge lists
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Building node and edge lists..."
|
||||
)
|
||||
nodes = self._extract_nodes(entities)
|
||||
edges = self._extract_edges(relationships, entities)
|
||||
|
||||
self.progress_tracker.update_tracking(
|
||||
tracking_id, message="Generating visualization..."
|
||||
)
|
||||
result = self._visualize_network_plotly(
|
||||
nodes, edges, output, file_path, **options
|
||||
nodes,
|
||||
edges,
|
||||
output,
|
||||
file_path,
|
||||
node_color_by=node_color_by,
|
||||
node_size_by=node_size_by,
|
||||
hover_data=hover_data,
|
||||
**options
|
||||
)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
@@ -184,6 +233,7 @@ class KGVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing knowledge graph communities")
|
||||
|
||||
entities = graph.get("entities", [])
|
||||
@@ -242,6 +292,7 @@ class KGVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info(
|
||||
f"Visualizing knowledge graph with {centrality_type} centrality"
|
||||
)
|
||||
@@ -295,6 +346,7 @@ class KGVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing entity type distribution")
|
||||
|
||||
entities = graph.get("entities", [])
|
||||
@@ -339,6 +391,7 @@ class KGVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing relationship matrix")
|
||||
|
||||
entities = graph.get("entities", [])
|
||||
@@ -442,6 +495,9 @@ class KGVisualizer:
|
||||
edges: List[Dict[str, Any]],
|
||||
output: str,
|
||||
file_path: Optional[Path],
|
||||
node_color_by: str = "type",
|
||||
node_size_by: Optional[str] = None,
|
||||
hover_data: Optional[List[str]] = None,
|
||||
**options,
|
||||
) -> Optional[Any]:
|
||||
"""Create Plotly network visualization."""
|
||||
@@ -458,11 +514,91 @@ class KGVisualizer:
|
||||
else:
|
||||
pos = self.force_layout.compute_layout(node_ids, edge_tuples, **options)
|
||||
|
||||
# Get entity type colors
|
||||
entity_types = list(set(n.get("type", "entity") for n in nodes))
|
||||
type_colors = ColorPalette.get_entity_type_colors(
|
||||
entity_types, self.color_scheme
|
||||
)
|
||||
# Step 4: Styling - Node Colors
|
||||
# Priority 1: Explicit color set in node (e.g. from visualize_communities)
|
||||
# Priority 2: Mapped property via node_color_by
|
||||
|
||||
node_colors = []
|
||||
if any("color" in n for n in nodes):
|
||||
node_colors = [n.get("color", "#888") for n in nodes if n["id"] in pos]
|
||||
else:
|
||||
if node_color_by == "type":
|
||||
entity_types = list(set(n.get("type", "entity") for n in nodes))
|
||||
type_colors = ColorPalette.get_entity_type_colors(
|
||||
entity_types, self.color_scheme
|
||||
)
|
||||
node_colors = [
|
||||
type_colors.get(n.get("type", "entity"), "#888")
|
||||
for n in nodes
|
||||
if n["id"] in pos
|
||||
]
|
||||
else:
|
||||
# Custom property mapping
|
||||
values = []
|
||||
for n in nodes:
|
||||
if n["id"] not in pos: continue
|
||||
val = n.get(node_color_by) or n.get("metadata", {}).get(node_color_by, "Unknown")
|
||||
values.append(str(val))
|
||||
|
||||
unique_vals = sorted(list(set(values)))
|
||||
colors = ColorPalette.get_colors(self.color_scheme, len(unique_vals))
|
||||
val_map = dict(zip(unique_vals, colors))
|
||||
|
||||
node_colors = []
|
||||
for n in nodes:
|
||||
if n["id"] not in pos: continue
|
||||
val = str(n.get(node_color_by) or n.get("metadata", {}).get(node_color_by, "Unknown"))
|
||||
node_colors.append(val_map.get(val, "#888"))
|
||||
|
||||
# Step 4: Styling - Node Sizes
|
||||
# Priority 1: Explicit size set in node (e.g. from visualize_centrality)
|
||||
# Priority 2: Mapped property via node_size_by
|
||||
|
||||
node_sizes = []
|
||||
if any("size" in n for n in nodes) and not node_size_by:
|
||||
node_sizes = [n.get("size", self.node_size) for n in nodes if n["id"] in pos]
|
||||
elif node_size_by:
|
||||
raw_sizes = []
|
||||
valid_indices = []
|
||||
for i, n in enumerate(nodes):
|
||||
if n["id"] not in pos: continue
|
||||
val = n.get(node_size_by) or n.get("metadata", {}).get(node_size_by, 0)
|
||||
try:
|
||||
s = float(val)
|
||||
except (ValueError, TypeError):
|
||||
s = 0
|
||||
raw_sizes.append(s)
|
||||
valid_indices.append(i)
|
||||
|
||||
# Normalize to range [10, 50]
|
||||
if raw_sizes and max(raw_sizes) > min(raw_sizes):
|
||||
min_s, max_s = min(raw_sizes), max(raw_sizes)
|
||||
node_sizes = [10 + 40 * ((s - min_s) / (max_s - min_s)) for s in raw_sizes]
|
||||
else:
|
||||
node_sizes = [self.node_size] * len(raw_sizes)
|
||||
else:
|
||||
node_sizes = [self.node_size for n in nodes if n["id"] in pos]
|
||||
|
||||
# Step 5: Interaction - Rich Hover
|
||||
node_text = []
|
||||
for n in nodes:
|
||||
if n["id"] not in pos: continue
|
||||
|
||||
# Basic info
|
||||
text = f"<b>{n['label']}</b><br>Type: {n.get('type', 'entity')}"
|
||||
|
||||
# Additional hover data
|
||||
if hover_data:
|
||||
for field in hover_data:
|
||||
val = n.get(field) or n.get("metadata", {}).get(field, "N/A")
|
||||
text += f"<br>{field}: {val}"
|
||||
|
||||
# Add dynamic styling info if relevant
|
||||
if node_size_by:
|
||||
val = n.get(node_size_by) or n.get("metadata", {}).get(node_size_by, "N/A")
|
||||
text += f"<br>{node_size_by}: {val}"
|
||||
|
||||
node_text.append(text)
|
||||
|
||||
# Prepare edge traces
|
||||
edge_x = []
|
||||
@@ -485,13 +621,6 @@ class KGVisualizer:
|
||||
# Prepare node traces
|
||||
node_x = [pos[n["id"]][0] for n in nodes if n["id"] in pos]
|
||||
node_y = [pos[n["id"]][1] for n in nodes if n["id"] in pos]
|
||||
node_text = [n["label"] for n in nodes if n["id"] in pos]
|
||||
node_colors = [
|
||||
type_colors.get(n.get("type", "entity"), "#888")
|
||||
for n in nodes
|
||||
if n["id"] in pos
|
||||
]
|
||||
node_sizes = [n.get("size", self.node_size) for n in nodes if n["id"] in pos]
|
||||
|
||||
node_trace = go.Scatter(
|
||||
x=node_x,
|
||||
@@ -499,9 +628,12 @@ class KGVisualizer:
|
||||
mode="markers+text",
|
||||
hoverinfo="text",
|
||||
text=node_text,
|
||||
textposition="middle center",
|
||||
textposition="top center",
|
||||
marker=dict(
|
||||
size=node_sizes, color=node_colors, line=dict(width=2, color="white")
|
||||
size=node_sizes,
|
||||
color=node_colors,
|
||||
line=dict(width=2, color="white"),
|
||||
opacity=0.9
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -37,10 +37,17 @@ from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import matplotlib.patches as mpatches
|
||||
import matplotlib.pyplot as plt
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
|
||||
try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
except ImportError:
|
||||
px = None
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
from matplotlib.patches import FancyBboxPatch
|
||||
from plotly.subplots import make_subplots
|
||||
|
||||
try:
|
||||
import graphviz
|
||||
@@ -90,26 +97,60 @@ class OntologyVisualizer:
|
||||
self.color_scheme = ColorScheme.DEFAULT
|
||||
self.node_size = config.get("node_size", 15)
|
||||
|
||||
def _check_dependencies(self, require_graphviz: bool = False):
|
||||
"""Check if dependencies are available."""
|
||||
if require_graphviz:
|
||||
if graphviz is None:
|
||||
raise ProcessingError(
|
||||
"Graphviz is required for DOT export. "
|
||||
"Install with: pip install graphviz"
|
||||
)
|
||||
else:
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for ontology visualization. "
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_hierarchy(
|
||||
self,
|
||||
ontology: Dict[str, Any],
|
||||
output: str = "interactive",
|
||||
file_path: Optional[Union[str, Path]] = None,
|
||||
node_color_by: str = "level",
|
||||
node_size_by: str = "instances",
|
||||
hover_data: Optional[List[str]] = None,
|
||||
**options,
|
||||
) -> Optional[Any]:
|
||||
"""
|
||||
Visualize class hierarchy as tree.
|
||||
|
||||
Implements the 5-step visualization process:
|
||||
1. Problem setting: Implicit in ontology selection
|
||||
2. Data analysis: Logs ontology statistics
|
||||
3. Layout: Hierarchical tree layout
|
||||
4. Styling: Configurable node color (e.g. by level) and size (e.g. by instances)
|
||||
5. Interaction: Rich hover data
|
||||
|
||||
Args:
|
||||
ontology: Ontology dictionary with classes, or SemanticNetwork object,
|
||||
or ontology generator result
|
||||
output: Output type ("interactive", "html", "png", "svg", "dot")
|
||||
file_path: Output file path
|
||||
node_color_by: Property to map to node color (default: "level")
|
||||
node_size_by: Property to map to node size (default: "instances")
|
||||
hover_data: List of properties to show in hover tooltip
|
||||
**options: Additional options
|
||||
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
# Check dependencies
|
||||
if output == "dot":
|
||||
self._check_dependencies(require_graphviz=True)
|
||||
else:
|
||||
self._check_dependencies()
|
||||
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="visualization",
|
||||
submodule="OntologyVisualizer",
|
||||
@@ -150,6 +191,15 @@ class OntologyVisualizer:
|
||||
"No classes found in ontology. Please provide classes or a semantic network."
|
||||
)
|
||||
|
||||
# Step 2: Data Analysis
|
||||
num_classes = len(classes)
|
||||
max_depth = 0
|
||||
for cls in classes:
|
||||
depth = self._calculate_class_depth(cls, classes)
|
||||
max_depth = max(max_depth, depth)
|
||||
|
||||
self.logger.info(f"Ontology Analysis: {num_classes} classes, max depth {max_depth}")
|
||||
|
||||
# If output is dot and graphviz is available, use it
|
||||
if output == "dot" and graphviz is not None and file_path:
|
||||
self.progress_tracker.update_tracking(
|
||||
@@ -175,7 +225,14 @@ class OntologyVisualizer:
|
||||
tracking_id, message="Generating visualization..."
|
||||
)
|
||||
result = self._visualize_hierarchy_plotly(
|
||||
hierarchy, classes, output, file_path, **options
|
||||
hierarchy,
|
||||
classes,
|
||||
output,
|
||||
file_path,
|
||||
node_color_by=node_color_by,
|
||||
node_size_by=node_size_by,
|
||||
hover_data=hover_data,
|
||||
**options
|
||||
)
|
||||
|
||||
self.progress_tracker.stop_tracking(
|
||||
@@ -209,6 +266,7 @@ class OntologyVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing ontology properties")
|
||||
|
||||
# Handle different input formats
|
||||
@@ -258,6 +316,7 @@ class OntologyVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing ontology structure")
|
||||
|
||||
classes = ontology.get("classes", [])
|
||||
@@ -342,6 +401,7 @@ class OntologyVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing class-property matrix")
|
||||
|
||||
classes = ontology.get("classes", [])
|
||||
@@ -411,6 +471,7 @@ class OntologyVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing ontology metrics")
|
||||
|
||||
classes = ontology.get("classes", [])
|
||||
@@ -603,6 +664,7 @@ class OntologyVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing semantic model")
|
||||
|
||||
# Handle OntologyGenerator result
|
||||
@@ -651,6 +713,9 @@ class OntologyVisualizer:
|
||||
classes: List[Dict[str, Any]],
|
||||
output: str,
|
||||
file_path: Optional[Path],
|
||||
node_color_by: str = "level",
|
||||
node_size_by: str = "instances",
|
||||
hover_data: Optional[List[str]] = None,
|
||||
**options,
|
||||
) -> Optional[Any]:
|
||||
"""Create Plotly hierarchy visualization."""
|
||||
@@ -676,7 +741,16 @@ class OntologyVisualizer:
|
||||
edges = []
|
||||
|
||||
def add_node_and_children(cls_name, level=0, x_offset=0):
|
||||
nodes.append({"name": cls_name, "level": level, "x": x_offset, "y": -level})
|
||||
# Find class data
|
||||
cls_data = all_class_names.get(cls_name, {})
|
||||
|
||||
nodes.append({
|
||||
"name": cls_name,
|
||||
"level": level,
|
||||
"x": x_offset,
|
||||
"y": -level,
|
||||
"data": cls_data
|
||||
})
|
||||
|
||||
children = hierarchy.get(cls_name, [])
|
||||
child_width = 1.0 / max(len(children), 1)
|
||||
@@ -692,6 +766,63 @@ class OntologyVisualizer:
|
||||
root_x = (i + 0.5) * root_width
|
||||
add_node_and_children(root, 0, root_x)
|
||||
|
||||
# Step 4: Styling - Node Colors
|
||||
# Default to coloring by level
|
||||
node_colors = []
|
||||
if node_color_by == "level":
|
||||
node_colors = [n["level"] for n in nodes]
|
||||
else:
|
||||
# Map custom property
|
||||
values = []
|
||||
for n in nodes:
|
||||
val = str(n["data"].get(node_color_by, "Unknown"))
|
||||
values.append(val)
|
||||
|
||||
unique_vals = sorted(list(set(values)))
|
||||
colors = ColorPalette.get_colors(self.color_scheme, len(unique_vals))
|
||||
val_map = dict(zip(unique_vals, colors))
|
||||
node_colors = [val_map.get(str(n["data"].get(node_color_by, "Unknown")), "#888") for n in nodes]
|
||||
|
||||
# Step 4: Styling - Node Sizes
|
||||
# Default to sizing by instances (if available) or fixed size
|
||||
node_sizes = []
|
||||
if node_size_by:
|
||||
raw_sizes = []
|
||||
for n in nodes:
|
||||
val = n["data"].get(node_size_by, 0)
|
||||
try:
|
||||
s = float(val)
|
||||
except (ValueError, TypeError):
|
||||
s = 0
|
||||
raw_sizes.append(s)
|
||||
|
||||
if raw_sizes and max(raw_sizes) > min(raw_sizes):
|
||||
min_s, max_s = min(raw_sizes), max(raw_sizes)
|
||||
# Scale between 10 and 40
|
||||
node_sizes = [10 + 30 * ((s - min_s) / (max_s - min_s)) for s in raw_sizes]
|
||||
else:
|
||||
node_sizes = [self.node_size] * len(nodes)
|
||||
else:
|
||||
node_sizes = [self.node_size] * len(nodes)
|
||||
|
||||
# Step 5: Interaction - Rich Hover
|
||||
node_text = []
|
||||
for n in nodes:
|
||||
cls_data = n["data"]
|
||||
text = f"<b>{n['name']}</b><br>Level: {n['level']}"
|
||||
|
||||
# Add instances if available
|
||||
if "instances" in cls_data:
|
||||
text += f"<br>Instances: {cls_data['instances']}"
|
||||
|
||||
# Additional hover data
|
||||
if hover_data:
|
||||
for field in hover_data:
|
||||
val = cls_data.get(field, "N/A")
|
||||
text += f"<br>{field}: {val}"
|
||||
|
||||
node_text.append(text)
|
||||
|
||||
# Create visualization
|
||||
edge_x = []
|
||||
edge_y = []
|
||||
@@ -714,18 +845,21 @@ class OntologyVisualizer:
|
||||
|
||||
node_x = [n["x"] for n in nodes]
|
||||
node_y = [n["y"] for n in nodes]
|
||||
node_text = [n["name"] for n in nodes]
|
||||
|
||||
node_trace = go.Scatter(
|
||||
x=node_x,
|
||||
y=node_y,
|
||||
mode="markers+text",
|
||||
text=node_text,
|
||||
textposition="middle center",
|
||||
text=[n["name"] for n in nodes], # Keep label on node simple
|
||||
hovertext=node_text, # Rich hover text
|
||||
hoverinfo="text",
|
||||
textposition="top center",
|
||||
marker=dict(
|
||||
size=self.node_size * 10,
|
||||
color="lightblue",
|
||||
line=dict(width=2, color="darkblue"),
|
||||
size=node_sizes,
|
||||
color=node_colors,
|
||||
colorscale="Viridis" if node_color_by == "level" else None,
|
||||
line=dict(width=2, color="white"),
|
||||
showscale=True if node_color_by == "level" else False
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -33,9 +33,19 @@ License: MIT
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
try:
|
||||
import numpy as np
|
||||
except ImportError:
|
||||
np = None
|
||||
|
||||
try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
except ImportError:
|
||||
px = None
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -66,6 +76,14 @@ class QualityVisualizer:
|
||||
except (KeyError, AttributeError):
|
||||
self.color_scheme = ColorScheme.DEFAULT
|
||||
|
||||
def _check_dependencies(self):
|
||||
"""Check if dependencies are available."""
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for quality visualization. "
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_dashboard(
|
||||
self,
|
||||
quality_report: Any,
|
||||
@@ -85,6 +103,7 @@ class QualityVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="visualization",
|
||||
submodule="QualityVisualizer",
|
||||
@@ -266,6 +285,7 @@ class QualityVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing quality score distribution")
|
||||
|
||||
fig = go.Figure(
|
||||
@@ -315,6 +335,7 @@ class QualityVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing quality issues")
|
||||
|
||||
# Extract issues
|
||||
@@ -405,6 +426,7 @@ class QualityVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing completeness metrics")
|
||||
|
||||
# Extract metrics
|
||||
@@ -468,6 +490,13 @@ class QualityVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
if np is None:
|
||||
raise ProcessingError(
|
||||
"NumPy is required for consistency heatmap visualization. "
|
||||
"Install with: pip install numpy"
|
||||
)
|
||||
|
||||
self.logger.info("Visualizing consistency heatmap")
|
||||
|
||||
# Extract consistency matrix
|
||||
@@ -477,8 +506,6 @@ class QualityVisualizer:
|
||||
if not matrix:
|
||||
raise ProcessingError("No consistency matrix found")
|
||||
|
||||
import numpy as np
|
||||
|
||||
matrix = np.array(matrix)
|
||||
|
||||
fig = go.Figure(
|
||||
|
||||
@@ -33,8 +33,12 @@ License: MIT
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
except ImportError:
|
||||
px = None
|
||||
go = None
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -62,6 +66,14 @@ class SemanticNetworkVisualizer:
|
||||
except (KeyError, AttributeError):
|
||||
self.color_scheme = ColorScheme.DEFAULT
|
||||
|
||||
def _check_dependencies(self):
|
||||
"""Check if dependencies are available."""
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for semantic network visualization. "
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_network(
|
||||
self,
|
||||
semantic_network: Any,
|
||||
@@ -87,6 +99,7 @@ class SemanticNetworkVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="visualization",
|
||||
submodule="SemanticNetworkVisualizer",
|
||||
@@ -274,6 +287,7 @@ class SemanticNetworkVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing semantic network node types")
|
||||
|
||||
# Extract nodes
|
||||
@@ -325,6 +339,7 @@ class SemanticNetworkVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing semantic network edge types")
|
||||
|
||||
# Extract edges
|
||||
|
||||
@@ -33,9 +33,14 @@ License: MIT
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
try:
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
from plotly.subplots import make_subplots
|
||||
except ImportError:
|
||||
px = None
|
||||
go = None
|
||||
make_subplots = None
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
@@ -66,6 +71,14 @@ class TemporalVisualizer:
|
||||
except (KeyError, AttributeError):
|
||||
self.color_scheme = ColorScheme.DEFAULT
|
||||
|
||||
def _check_dependencies(self):
|
||||
"""Check if dependencies are available."""
|
||||
if px is None or go is None:
|
||||
raise ProcessingError(
|
||||
"Plotly is required for temporal visualization. "
|
||||
"Install with: pip install plotly"
|
||||
)
|
||||
|
||||
def visualize_timeline(
|
||||
self,
|
||||
temporal_data: Dict[str, Any],
|
||||
@@ -85,6 +98,7 @@ class TemporalVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="visualization",
|
||||
submodule="TemporalVisualizer",
|
||||
@@ -208,6 +222,7 @@ class TemporalVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing temporal patterns")
|
||||
|
||||
if not patterns:
|
||||
@@ -279,6 +294,7 @@ class TemporalVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing snapshot comparison")
|
||||
|
||||
timestamps = sorted(snapshots.keys())
|
||||
@@ -371,6 +387,7 @@ class TemporalVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing version history")
|
||||
|
||||
# Build tree structure
|
||||
@@ -438,6 +455,7 @@ class TemporalVisualizer:
|
||||
Returns:
|
||||
Visualization figure or None
|
||||
"""
|
||||
self._check_dependencies()
|
||||
self.logger.info("Visualizing metrics evolution")
|
||||
|
||||
fig = go.Figure()
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ modules = {
|
||||
"kg": "semantica.kg",
|
||||
"embeddings": "semantica.embeddings",
|
||||
"vector_store": "semantica.vector_store",
|
||||
"triple_store": "semantica.triple_store",
|
||||
"triplet_store": "semantica.triplet_store",
|
||||
"ontology": "semantica.ontology",
|
||||
"reasoning": "semantica.reasoning",
|
||||
"pipeline": "semantica.pipeline",
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import pytest
|
||||
|
||||
from semantica import build as module_build
|
||||
from semantica.core import Semantica
|
||||
from semantica.core.methods import (
|
||||
initialize_framework,
|
||||
get_status,
|
||||
run_pipeline,
|
||||
build_knowledge_base,
|
||||
)
|
||||
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
class DummyPipeline:
|
||||
def __init__(self):
|
||||
self.executed_with = None
|
||||
|
||||
def execute(self, data):
|
||||
self.executed_with = data
|
||||
return {"value": data}
|
||||
|
||||
|
||||
def test_initialize_and_get_status_integration():
|
||||
framework = initialize_framework()
|
||||
status = get_status(framework=framework, method="summary")
|
||||
assert status["state"] in {"ready", "running", "initializing"}
|
||||
assert "health" in status
|
||||
framework.shutdown(graceful=True)
|
||||
|
||||
|
||||
def test_semantica_run_pipeline_with_dummy_pipeline():
|
||||
pipeline = DummyPipeline()
|
||||
framework = Semantica()
|
||||
framework.initialize()
|
||||
data = {"input": "value"}
|
||||
result = framework.run_pipeline(pipeline, data)
|
||||
assert result["success"] is True
|
||||
assert result["output"] == {"value": data}
|
||||
assert pipeline.executed_with == data
|
||||
framework.shutdown(graceful=True)
|
||||
|
||||
|
||||
def test_core_methods_run_pipeline_with_dummy_pipeline():
|
||||
pipeline = DummyPipeline()
|
||||
data = "sample"
|
||||
result = run_pipeline(pipeline, data)
|
||||
assert result["success"] is True
|
||||
assert result["output"] == {"value": data}
|
||||
|
||||
|
||||
def test_framework_build_knowledge_base_end_to_end(tmp_path):
|
||||
source_path = tmp_path / "sample_e2e_framework.txt"
|
||||
source_path.write_text("Apple Inc. is a technology company.")
|
||||
framework = Semantica()
|
||||
result = framework.build_knowledge_base(
|
||||
sources=[str(source_path)],
|
||||
embeddings=False,
|
||||
graph=False,
|
||||
pipeline={
|
||||
"name": "e2e_pipeline",
|
||||
"steps": [
|
||||
{"name": "step1", "type": "default", "config": {}},
|
||||
],
|
||||
},
|
||||
)
|
||||
stats = result["statistics"]
|
||||
assert stats["sources_processed"] == 1
|
||||
assert len(result["results"]) == 1
|
||||
assert result["results"][0]["success"] is True
|
||||
framework.shutdown(graceful=True)
|
||||
|
||||
|
||||
def test_core_methods_build_knowledge_base_end_to_end(tmp_path):
|
||||
source_path = tmp_path / "sample_e2e_core_methods.txt"
|
||||
source_path.write_text("Tim Cook leads Apple.")
|
||||
result = build_knowledge_base(
|
||||
sources=str(source_path),
|
||||
method="minimal",
|
||||
embeddings=False,
|
||||
graph=False,
|
||||
pipeline={"steps": ["step1", "step2"]},
|
||||
)
|
||||
stats = result["statistics"]
|
||||
assert stats["sources_processed"] == 1
|
||||
assert len(result["results"]) == 1
|
||||
assert result["results"][0]["success"] is True
|
||||
|
||||
|
||||
def test_module_build_end_to_end_default_pipeline(tmp_path):
|
||||
source_path = tmp_path / "sample_e2e_module_build.txt"
|
||||
source_path.write_text("Sample data for end-to-end test.")
|
||||
result = module_build(str(source_path), embeddings=False, graph=False)
|
||||
stats = result["statistics"]
|
||||
assert stats["sources_processed"] == 1
|
||||
assert len(result["results"]) == 1
|
||||
assert result["results"][0]["success"] is True
|
||||
@@ -4,6 +4,8 @@ from unittest.mock import MagicMock, patch
|
||||
from semantica.ingest import MCPIngestor, ingest_mcp, DBIngestor, FileIngestor
|
||||
from semantica.ingest.mcp_ingestor import MCPData
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestCookbookIntegration:
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -18,6 +18,8 @@ from semantica.ingest import (
|
||||
MCPIngestor, IngestConfig, ingest_config
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestNotebook02DataIngestion:
|
||||
|
||||
def setup_method(self):
|
||||
|
||||
@@ -7,6 +7,8 @@ from semantica.ingest import FileIngestor, WebIngestor, DBIngestor, StreamIngest
|
||||
from semantica.kg import GraphBuilder, EntityResolver, ProvenanceTracker
|
||||
from semantica.conflicts import ConflictDetector
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestNotebook06MultiSourceIntegration:
|
||||
|
||||
def setup_method(self):
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
import unittest
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.normalize import methods
|
||||
from semantica.normalize.config import normalize_config
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestNormalizeIntegration(unittest.TestCase):
|
||||
def test_normalize_text_integration(self):
|
||||
text = "Hello World"
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.ontology import (
|
||||
OntologyEngine,
|
||||
ClassInferrer,
|
||||
@@ -11,6 +14,8 @@ from semantica.ontology import (
|
||||
)
|
||||
from semantica.visualization import OntologyVisualizer
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestNotebook14(unittest.TestCase):
|
||||
"""
|
||||
Tests mirroring the steps in cookbook/introduction/14_Ontology.ipynb
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
from collections import defaultdict
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.ontology.class_inferrer import ClassInferrer
|
||||
from semantica.ontology.property_generator import PropertyGenerator
|
||||
from semantica.ontology.naming_conventions import NamingConventions
|
||||
@@ -10,6 +12,8 @@ from semantica.ontology.ontology_validator import OntologyValidator, ValidationR
|
||||
from semantica.ontology.namespace_manager import NamespaceManager
|
||||
from semantica.ontology.module_manager import ModuleManager
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestOntologyComprehensive(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
|
||||
import unittest
|
||||
import os
|
||||
import tempfile
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.parse import DocumentParser, CSVParser, JSONParser, XMLParser, HTMLParser, StructuredDataParser
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestNotebook03(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
|
||||
@@ -6,6 +6,8 @@ import json
|
||||
import csv
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.parse.document_parser import DocumentParser, PDFParser, DOCXParser, HTMLParser
|
||||
from semantica.parse.pptx_parser import PPTXParser
|
||||
from semantica.parse.excel_parser import ExcelParser
|
||||
@@ -17,6 +19,8 @@ from semantica.parse.web_parser import WebParser
|
||||
from semantica.parse.registry import MethodRegistry
|
||||
from semantica.parse.config import ParseConfig
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestParseComprehensive(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.pipeline import (
|
||||
PipelineBuilder,
|
||||
ExecutionEngine,
|
||||
@@ -10,6 +13,8 @@ from semantica.pipeline import (
|
||||
RetryStrategy
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestNotebook07(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
||||
@@ -3,6 +3,8 @@ from unittest.mock import MagicMock, patch
|
||||
import time
|
||||
from typing import Dict, Any
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.pipeline.pipeline_builder import PipelineBuilder, StepStatus, Pipeline
|
||||
from semantica.pipeline.execution_engine import ExecutionEngine, PipelineStatus
|
||||
from semantica.pipeline.failure_handler import (
|
||||
@@ -11,6 +13,8 @@ from semantica.pipeline.failure_handler import (
|
||||
from semantica.pipeline.parallelism_manager import ParallelismManager, Task
|
||||
from semantica.pipeline.pipeline_validator import PipelineValidator
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestPipelineComprehensive(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
|
||||
import unittest
|
||||
import sys
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
# Add project root to path
|
||||
import pytest
|
||||
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from semantica.embeddings import EmbeddingGenerator, TextEmbedder
|
||||
@@ -13,6 +13,8 @@ from semantica.vector_store import (
|
||||
SearchRanker, NamespaceManager
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestSemanticaFeatures(unittest.TestCase):
|
||||
|
||||
def test_01_embedding_generation(self):
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
|
||||
import unittest
|
||||
import sys
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
from dataclasses import asdict
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from semantica.semantic_extract.ner_extractor import NERExtractor, Entity
|
||||
from semantica.semantic_extract.named_entity_recognizer import NamedEntityRecognizer
|
||||
from semantica.semantic_extract.methods import get_entity_method
|
||||
|
||||
class TestNERConfigurations(unittest.TestCase):
|
||||
"""
|
||||
Test suite to verify NER with different configurations:
|
||||
- LLM
|
||||
- ML (spaCy)
|
||||
- Regex
|
||||
- Pattern
|
||||
- Fallbacks and Ensemble
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.text = "Apple Inc. was founded by Steve Jobs."
|
||||
|
||||
@patch('semantica.semantic_extract.methods.create_provider')
|
||||
def test_ner_llm_config(self, mock_create_provider):
|
||||
"""Test NER with LLM configuration"""
|
||||
print("\nTesting NER with LLM configuration...")
|
||||
|
||||
# Mock LLM provider
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.is_available.return_value = True
|
||||
mock_provider.generate_structured.return_value = [
|
||||
{"text": "Apple Inc.", "label": "ORG", "start": 0, "end": 10, "confidence": 0.95},
|
||||
{"text": "Steve Jobs", "label": "PERSON", "start": 26, "end": 36, "confidence": 0.98}
|
||||
]
|
||||
mock_create_provider.return_value = mock_provider
|
||||
|
||||
# Initialize extractor with LLM method
|
||||
extractor = NERExtractor(
|
||||
method="llm",
|
||||
provider="openai",
|
||||
model="gpt-4",
|
||||
temperature=0.1
|
||||
)
|
||||
|
||||
entities = extractor.extract_entities(self.text)
|
||||
|
||||
# Verify provider creation args
|
||||
mock_create_provider.assert_called_with("openai", model="gpt-4", temperature=0.1)
|
||||
|
||||
# Verify extraction
|
||||
self.assertEqual(len(entities), 2)
|
||||
self.assertEqual(entities[0].text, "Apple Inc.")
|
||||
self.assertEqual(entities[0].label, "ORG")
|
||||
self.assertEqual(entities[0].metadata["extraction_method"], "llm")
|
||||
self.assertEqual(entities[0].metadata["model"], "gpt-4")
|
||||
|
||||
@patch('semantica.semantic_extract.methods.spacy')
|
||||
def test_ner_ml_config_spacy_available(self, mock_spacy):
|
||||
"""Test NER with ML (spaCy) configuration when spaCy is available"""
|
||||
print("\nTesting NER with ML (spaCy) configuration...")
|
||||
|
||||
# Mock spaCy nlp model
|
||||
mock_nlp = MagicMock()
|
||||
mock_doc = MagicMock()
|
||||
|
||||
# Mock entities
|
||||
ent1 = MagicMock()
|
||||
ent1.text = "Apple Inc."
|
||||
ent1.label_ = "ORG"
|
||||
ent1.start_char = 0
|
||||
ent1.end_char = 10
|
||||
ent1.confidence = 1.0 # Optional attribute
|
||||
|
||||
ent2 = MagicMock()
|
||||
ent2.text = "Steve Jobs"
|
||||
ent2.label_ = "PERSON"
|
||||
ent2.start_char = 26
|
||||
ent2.end_char = 36
|
||||
ent2.confidence = 0.99
|
||||
|
||||
mock_doc.ents = [ent1, ent2]
|
||||
mock_nlp.return_value = mock_doc
|
||||
mock_spacy.load.return_value = mock_nlp
|
||||
|
||||
# Patch SPACY_AVAILABLE in methods module
|
||||
with patch('semantica.semantic_extract.methods.SPACY_AVAILABLE', True):
|
||||
extractor = NERExtractor(method="ml", model="en_core_web_trf")
|
||||
entities = extractor.extract_entities(self.text)
|
||||
|
||||
# Verify spacy load called with correct model
|
||||
mock_spacy.load.assert_called_with("en_core_web_trf")
|
||||
|
||||
self.assertEqual(len(entities), 2)
|
||||
self.assertEqual(entities[0].text, "Apple Inc.")
|
||||
self.assertEqual(entities[0].label, "ORG")
|
||||
self.assertEqual(entities[0].metadata["extraction_method"], "ml")
|
||||
self.assertEqual(entities[0].metadata["model"], "en_core_web_trf")
|
||||
|
||||
def test_ner_regex_config(self):
|
||||
"""Test NER with Regex configuration"""
|
||||
print("\nTesting NER with Regex configuration...")
|
||||
|
||||
custom_patterns = {
|
||||
"COMPANY": r"Apple Inc\.",
|
||||
"FOUNDER": r"Steve Jobs"
|
||||
}
|
||||
|
||||
extractor = NERExtractor(method="regex", patterns=custom_patterns)
|
||||
entities = extractor.extract_entities(self.text)
|
||||
|
||||
self.assertEqual(len(entities), 2)
|
||||
|
||||
# Check if labels match custom keys
|
||||
labels = sorted([e.label for e in entities])
|
||||
self.assertEqual(labels, ["COMPANY", "FOUNDER"])
|
||||
|
||||
# Check metadata
|
||||
self.assertEqual(entities[0].metadata["extraction_method"], "regex")
|
||||
|
||||
def test_ner_pattern_config(self):
|
||||
"""Test NER with default Pattern configuration"""
|
||||
print("\nTesting NER with Pattern configuration...")
|
||||
|
||||
# Default patterns in methods.py match "Apple Inc" (ORG) and "Steve Jobs" (PERSON)
|
||||
# Note: The pattern for ORG in methods.py expects "Inc|Corp..."
|
||||
|
||||
extractor = NERExtractor(method="pattern")
|
||||
entities = extractor.extract_entities(self.text)
|
||||
|
||||
self.assertTrue(len(entities) >= 2)
|
||||
texts = [e.text for e in entities]
|
||||
self.assertIn("Apple Inc", texts) # Regex pattern does not capture the trailing dot
|
||||
# Actually methods.py regex: r"\b([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s+(?:Inc|Corp|LLC|Ltd|Company))\b"
|
||||
# "Apple Inc." -> "Apple Inc" (dot is outside \b if not matched?)
|
||||
# Let's check the result strictly
|
||||
|
||||
@patch('semantica.semantic_extract.methods.create_provider')
|
||||
@patch('semantica.semantic_extract.methods.spacy')
|
||||
def test_ner_ensemble_config(self, mock_spacy, mock_create_provider):
|
||||
"""Test NER with Ensemble (Multiple Methods)"""
|
||||
print("\nTesting NER with Ensemble configuration...")
|
||||
|
||||
# Setup mocks
|
||||
# LLM returns 1 entity
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.is_available.return_value = True
|
||||
mock_provider.generate_structured.return_value = [
|
||||
{"text": "Apple Inc.", "label": "ORG", "start": 0, "end": 10, "confidence": 0.95}
|
||||
]
|
||||
mock_create_provider.return_value = mock_provider
|
||||
|
||||
# ML returns 2 entities
|
||||
mock_nlp = MagicMock()
|
||||
mock_doc = MagicMock()
|
||||
ent1 = MagicMock()
|
||||
ent1.text = "Apple Inc."
|
||||
ent1.label_ = "ORG"
|
||||
ent1.start_char = 0
|
||||
ent1.end_char = 10
|
||||
ent1.confidence = 0.95
|
||||
ent2 = MagicMock()
|
||||
ent2.text = "Steve Jobs"
|
||||
ent2.label_ = "PERSON"
|
||||
ent2.start_char = 26
|
||||
ent2.end_char = 36
|
||||
ent2.confidence = 0.99
|
||||
mock_doc.ents = [ent1, ent2]
|
||||
mock_nlp.return_value = mock_doc
|
||||
mock_spacy.load.return_value = mock_nlp
|
||||
|
||||
with patch('semantica.semantic_extract.methods.SPACY_AVAILABLE', True):
|
||||
# Init extractor with list of methods
|
||||
extractor = NERExtractor(method=["llm", "ml"], ensemble_voting=True)
|
||||
entities = extractor.extract_entities(self.text)
|
||||
|
||||
# Since ensemble_voting=True (implied merge), we expect unique entities
|
||||
# Apple Inc (from both) + Steve Jobs (from ML)
|
||||
|
||||
texts = [e.text for e in entities]
|
||||
self.assertIn("Apple Inc.", texts)
|
||||
self.assertIn("Steve Jobs", texts)
|
||||
|
||||
@patch('semantica.semantic_extract.methods.HuggingFaceModelLoader')
|
||||
def test_ner_huggingface_config(self, mock_loader_cls):
|
||||
"""Test NER with HuggingFace configuration"""
|
||||
print("\nTesting NER with HuggingFace configuration...")
|
||||
|
||||
mock_loader = MagicMock()
|
||||
mock_loader_cls.return_value = mock_loader
|
||||
|
||||
# Mock extract_entities return
|
||||
# HuggingFace loader typically returns list of dicts or objects
|
||||
mock_loader.extract_entities.return_value = [
|
||||
{"word": "Apple Inc.", "entity_group": "ORG", "score": 0.99, "start": 0, "end": 10}
|
||||
]
|
||||
|
||||
extractor = NERExtractor(
|
||||
method="huggingface",
|
||||
huggingface_model="dslim/bert-base-NER",
|
||||
device="cpu"
|
||||
)
|
||||
entities = extractor.extract_entities(self.text)
|
||||
|
||||
mock_loader.load_ner_model.assert_called_with("dslim/bert-base-NER")
|
||||
self.assertEqual(len(entities), 1)
|
||||
self.assertEqual(entities[0].text, "Apple Inc.")
|
||||
self.assertEqual(entities[0].label, "ORG")
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -4,6 +4,8 @@ import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.kg import GraphBuilder
|
||||
from semantica.export import (
|
||||
JSONExporter,
|
||||
@@ -12,6 +14,8 @@ from semantica.export import (
|
||||
GraphExporter,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestNotebook15Export(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
|
||||
@@ -2,9 +2,12 @@ import sys
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
# Add project root to path
|
||||
import pytest
|
||||
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
def log(msg):
|
||||
print(msg)
|
||||
with open("test_progress.log", "a") as f:
|
||||
|
||||
@@ -3,9 +3,12 @@ import sys
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
# Add project root to path
|
||||
import pytest
|
||||
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestNotebooks(unittest.TestCase):
|
||||
|
||||
def test_12_embedding_generation(self):
|
||||
|
||||
@@ -6,6 +6,8 @@ import numpy as np
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.kg import GraphBuilder
|
||||
from semantica.export import (
|
||||
JSONExporter,
|
||||
@@ -24,6 +26,8 @@ from semantica.export import (
|
||||
export_config
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestNotebooks(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
import unittest
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from semantica.semantic_extract import (
|
||||
NERExtractor,
|
||||
NamedEntityRecognizer,
|
||||
RelationExtractor,
|
||||
TripleExtractor,
|
||||
Entity,
|
||||
Relation
|
||||
)
|
||||
from semantica.semantic_extract.methods import get_entity_method, get_relation_method
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestNotebooksVerification(unittest.TestCase):
|
||||
"""
|
||||
Test suite to verify the code snippets from the notebooks:
|
||||
- 05_Entity_Extraction.ipynb
|
||||
- 06_Relation_Extraction.ipynb
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.ner_extractor = NERExtractor()
|
||||
self.relation_extractor = RelationExtractor()
|
||||
|
||||
def test_05_entity_extraction_notebook_flow(self):
|
||||
"""Verify the flow demonstrated in 05_Entity_Extraction.ipynb"""
|
||||
print("\nTesting 05_Entity_Extraction.ipynb flow...")
|
||||
|
||||
# --- Step 1: Basic Entity Extraction ---
|
||||
text = """
|
||||
Apple Inc. is a technology company founded by Steve Jobs, Steve Wozniak, and Ronald Wayne
|
||||
in Cupertino, California on April 1, 1976. The company's current CEO is Tim Cook, who took
|
||||
over from Steve Jobs in August 2011. Apple is headquartered at One Apple Park Way in Cupertino.
|
||||
"""
|
||||
|
||||
entities = self.ner_extractor.extract(text)
|
||||
self.assertIsInstance(entities, list)
|
||||
if len(entities) > 0:
|
||||
first_entity = entities[0]
|
||||
# Notebook handles dict or object, let's verify what we get
|
||||
is_dict = isinstance(first_entity, dict)
|
||||
is_object = hasattr(first_entity, 'text')
|
||||
self.assertTrue(is_dict or is_object, "Entity must be dict or object")
|
||||
|
||||
if is_object:
|
||||
print(f"NERExtractor returned objects: {first_entity.text} ({first_entity.label})")
|
||||
else:
|
||||
print(f"NERExtractor returned dicts: {first_entity.get('text')} ({first_entity.get('label')})")
|
||||
|
||||
# --- Step 3: Different Extraction Methods ---
|
||||
methods_to_try = ["pattern", "regex"] # Skipping 'ml' as it might require spaCy which might be missing/mocked
|
||||
|
||||
sample_text = "Apple Inc. was founded by Steve Jobs in Cupertino, California in 1976."
|
||||
|
||||
for method_name in methods_to_try:
|
||||
try:
|
||||
method = get_entity_method(method_name)
|
||||
method_entities = method(sample_text)
|
||||
self.assertIsInstance(method_entities, list)
|
||||
print(f"Method '{method_name}' returned {len(method_entities)} entities")
|
||||
except Exception as e:
|
||||
print(f"Method '{method_name}' failed as expected/unexpected: {e}")
|
||||
|
||||
# --- Step 4: Advanced Entity Recognition ---
|
||||
# Note: We use patterns/regex here to avoid spaCy dependency issues in CI/Test env
|
||||
# but the notebook uses 'spacy'. We'll adapt for robustness.
|
||||
ner = NamedEntityRecognizer(
|
||||
methods=["pattern", "regex"],
|
||||
confidence_threshold=0.5,
|
||||
merge_overlapping=True,
|
||||
include_standard_types=True
|
||||
)
|
||||
|
||||
texts = [
|
||||
"Tim Cook is the CEO of Apple Inc., based in Cupertino.",
|
||||
"Microsoft Corporation, founded by Bill Gates, is headquartered in Redmond, Washington."
|
||||
]
|
||||
|
||||
for text in texts:
|
||||
entities = ner.extract_entities(text)
|
||||
self.assertIsInstance(entities, list)
|
||||
|
||||
def test_06_relation_extraction_notebook_flow(self):
|
||||
"""Verify the flow demonstrated in 06_Relation_Extraction.ipynb"""
|
||||
print("\nTesting 06_Relation_Extraction.ipynb flow...")
|
||||
|
||||
# --- Step 1: Basic Relation Extraction ---
|
||||
text = """
|
||||
Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976.
|
||||
The company is headquartered in Cupertino, California. Tim Cook is the current CEO
|
||||
of Apple Inc. and took over from Steve Jobs in August 2011.
|
||||
"""
|
||||
|
||||
# First extract entities
|
||||
entities = self.ner_extractor.extract(text)
|
||||
|
||||
# Then extract relationships
|
||||
# Note: RelationExtractor might default to 'dependency' which needs spaCy.
|
||||
# We should check if it falls back or if we need to specify a method.
|
||||
# The notebook calls `relation_extractor.extract(text, entities)` directly.
|
||||
|
||||
relationships = self.relation_extractor.extract(text, entities)
|
||||
self.assertIsInstance(relationships, list)
|
||||
|
||||
if len(relationships) > 0:
|
||||
first_rel = relationships[0]
|
||||
is_dict = isinstance(first_rel, dict)
|
||||
is_object = hasattr(first_rel, 'subject')
|
||||
self.assertTrue(is_dict or is_object, "Relation must be dict or object")
|
||||
|
||||
if is_object:
|
||||
print(f"RelationExtractor returned objects: {first_rel.subject} --[{first_rel.predicate}]--> {first_rel.object}")
|
||||
else:
|
||||
print(f"RelationExtractor returned dicts: {first_rel.get('subject')} --[{first_rel.get('predicate')}]--> {first_rel.get('object')}")
|
||||
|
||||
# --- Step 2: Different Extraction Methods ---
|
||||
methods_to_try = ["pattern", "cooccurrence"] # Skipping 'dependency' to be safe
|
||||
|
||||
sample_text = "Apple Inc. was founded by Steve Jobs in Cupertino, California."
|
||||
sample_entities = self.ner_extractor.extract(sample_text)
|
||||
|
||||
for method_name in methods_to_try:
|
||||
try:
|
||||
method = get_relation_method(method_name)
|
||||
# Some methods might need specific args, but notebook shows standard call signature
|
||||
if method_name == "cooccurrence":
|
||||
# cooccurrence might return empty if window is small or entities far apart
|
||||
# but interface should hold
|
||||
rels = method(sample_text, sample_entities)
|
||||
else:
|
||||
rels = method(sample_text, sample_entities)
|
||||
|
||||
self.assertIsInstance(rels, list)
|
||||
print(f"Method '{method_name}' returned {len(rels)} relations")
|
||||
except Exception as e:
|
||||
print(f"Method '{method_name}' failed: {e}")
|
||||
|
||||
# --- Step 3: Advanced Relation Extraction ---
|
||||
advanced_extractor = RelationExtractor(
|
||||
relation_types=["founded_by", "located_in", "works_for"],
|
||||
confidence_threshold=0.1, # Low threshold to ensure we catch something
|
||||
bidirectional=False,
|
||||
max_distance=50
|
||||
)
|
||||
|
||||
texts = [
|
||||
"Microsoft was founded by Bill Gates and Paul Allen in Albuquerque, New Mexico.",
|
||||
"Satya Nadella works for Microsoft as the CEO."
|
||||
]
|
||||
|
||||
for text in texts:
|
||||
ents = self.ner_extractor.extract(text)
|
||||
rels = advanced_extractor.extract(text, ents)
|
||||
self.assertIsInstance(rels, list)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -21,6 +21,8 @@ from semantica.pipeline import (
|
||||
from semantica.pipeline.pipeline_builder import Pipeline, PipelineSerializer
|
||||
from semantica.pipeline.execution_engine import ExecutionResult
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
# --- Fixtures ---
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
import unittest
|
||||
import sys
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
|
||||
|
||||
from semantica.semantic_extract.ner_extractor import NERExtractor, Entity
|
||||
from semantica.semantic_extract.named_entity_recognizer import (
|
||||
NamedEntityRecognizer,
|
||||
EntityClassifier,
|
||||
EntityConfidenceScorer,
|
||||
CustomEntityDetector
|
||||
)
|
||||
from semantica.semantic_extract.relation_extractor import RelationExtractor, Relation
|
||||
from semantica.semantic_extract.triple_extractor import (
|
||||
TripleExtractor,
|
||||
TripleValidator,
|
||||
TripleQualityChecker,
|
||||
RDFSerializer,
|
||||
Triple
|
||||
)
|
||||
from semantica.semantic_extract.methods import get_entity_method, get_relation_method
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestSemanticExtractDeepDive(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.text = "Apple Inc. was founded by Steve Jobs in Cupertino. Tim Cook is the CEO."
|
||||
self.entities = [
|
||||
Entity(text="Apple Inc.", label="ORG", start_char=0, end_char=10, confidence=0.9),
|
||||
Entity(text="Steve Jobs", label="PERSON", start_char=26, end_char=36, confidence=0.95),
|
||||
Entity(text="Cupertino", label="GPE", start_char=40, end_char=49, confidence=0.8),
|
||||
Entity(text="Tim Cook", label="PERSON", start_char=51, end_char=59, confidence=0.9),
|
||||
Entity(text="CEO", label="TITLE", start_char=67, end_char=70, confidence=0.7)
|
||||
]
|
||||
self.relations = [
|
||||
Relation(subject=self.entities[0], predicate="founded_by", object=self.entities[1], confidence=0.85),
|
||||
Relation(subject=self.entities[3], predicate="works_for", object=self.entities[0], confidence=0.8)
|
||||
]
|
||||
|
||||
# --- NER Tests ---
|
||||
|
||||
def test_ner_extractor_pattern(self):
|
||||
"""Test NERExtractor with pattern method"""
|
||||
extractor = NERExtractor(method="pattern")
|
||||
# Using a text that matches the hardcoded patterns in methods.py
|
||||
text = "Steve Jobs worked at Apple Inc. in New York City on 12/12/2023."
|
||||
entities = extractor.extract_entities(text)
|
||||
|
||||
# Verify entities are extracted
|
||||
texts = [e.text for e in entities]
|
||||
labels = [e.label for e in entities]
|
||||
|
||||
# Note: Patterns in methods.py might be specific, let's verify if they match
|
||||
# PERSON: \b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b -> "Steve Jobs" should match
|
||||
# ORG: \b([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s+(?:Inc|Corp|LLC|Ltd|Company))\b -> "Apple Inc." should match
|
||||
|
||||
self.assertIn("Steve Jobs", texts)
|
||||
self.assertIn("Apple Inc", texts)
|
||||
self.assertIn("PERSON", labels)
|
||||
self.assertIn("ORG", labels)
|
||||
|
||||
def test_named_entity_recognizer_flow(self):
|
||||
"""Test NamedEntityRecognizer with mocked method"""
|
||||
# We mock the internal extraction to avoid dependency on models
|
||||
with patch('semantica.semantic_extract.methods.get_entity_method') as mock_get:
|
||||
mock_method = MagicMock()
|
||||
mock_method.return_value = self.entities
|
||||
mock_get.return_value = mock_method
|
||||
|
||||
ner = NamedEntityRecognizer(confidence_threshold=0.8)
|
||||
extracted = ner.extract_entities(self.text)
|
||||
|
||||
# Should filter out CEO (conf 0.7)
|
||||
self.assertEqual(len(extracted), 4)
|
||||
self.assertNotIn("CEO", [e.text for e in extracted])
|
||||
|
||||
def test_entity_classifier(self):
|
||||
"""Test EntityClassifier"""
|
||||
classifier = EntityClassifier()
|
||||
classified = classifier.classify_entities(self.entities)
|
||||
|
||||
self.assertIn("PERSON", classified)
|
||||
self.assertIn("ORG", classified)
|
||||
self.assertEqual(len(classified["PERSON"]), 2) # Steve Jobs, Tim Cook
|
||||
self.assertEqual(len(classified["ORG"]), 1) # Apple Inc.
|
||||
|
||||
def test_entity_confidence_scorer(self):
|
||||
"""Test EntityConfidenceScorer"""
|
||||
scorer = EntityConfidenceScorer()
|
||||
scored = scorer.score_entities(self.entities)
|
||||
|
||||
# Ensure confidence scores are preserved or modified correctly
|
||||
for entity in scored:
|
||||
self.assertTrue(0 <= entity.confidence <= 1.0)
|
||||
|
||||
def test_custom_entity_detector(self):
|
||||
"""Test CustomEntityDetector"""
|
||||
patterns = {"EMAIL": r"[\w\.-]+@[\w\.-]+"}
|
||||
detector = CustomEntityDetector(patterns=patterns)
|
||||
text = "Contact us at test@example.com"
|
||||
|
||||
entities = detector.detect_custom_entities(text, "EMAIL")
|
||||
self.assertEqual(len(entities), 1)
|
||||
self.assertEqual(entities[0].text, "test@example.com")
|
||||
self.assertEqual(entities[0].label, "EMAIL")
|
||||
|
||||
# --- Relation Tests ---
|
||||
|
||||
def test_relation_extractor_pattern(self):
|
||||
"""Test RelationExtractor with pattern method"""
|
||||
extractor = RelationExtractor(method="pattern")
|
||||
# Text matching "founded by" pattern
|
||||
text = "Apple was founded by Steve"
|
||||
|
||||
# We need entities for relation extraction
|
||||
entities = [
|
||||
Entity(text="Apple", label="ORG", start_char=0, end_char=5),
|
||||
Entity(text="Steve", label="PERSON", start_char=21, end_char=26)
|
||||
]
|
||||
|
||||
relations = extractor.extract_relations(text, entities)
|
||||
|
||||
self.assertTrue(len(relations) > 0)
|
||||
self.assertEqual(relations[0].predicate, "founded_by")
|
||||
self.assertEqual(relations[0].subject.text, "Apple")
|
||||
self.assertEqual(relations[0].object.text, "Steve")
|
||||
|
||||
def test_relation_extractor_cooccurrence(self):
|
||||
"""Test RelationExtractor with cooccurrence method"""
|
||||
# Set low confidence threshold because cooccurrence yields 0.5 confidence
|
||||
extractor = RelationExtractor(method="cooccurrence", confidence_threshold=0.4)
|
||||
# Entities close to each other
|
||||
text = "Apple Inc. CEO Tim Cook announced..."
|
||||
entities = [
|
||||
Entity(text="Apple Inc.", label="ORG", start_char=0, end_char=10),
|
||||
Entity(text="Tim Cook", label="PERSON", start_char=15, end_char=23)
|
||||
]
|
||||
|
||||
relations = extractor.extract_relations(text, entities)
|
||||
self.assertTrue(len(relations) > 0)
|
||||
self.assertEqual(relations[0].predicate, "related_to")
|
||||
|
||||
# --- Triple Tests ---
|
||||
|
||||
def test_triple_extractor(self):
|
||||
"""Test TripleExtractor"""
|
||||
# Mocking get_triple_method to return a simple extraction function
|
||||
with patch('semantica.semantic_extract.methods.get_triple_method') as mock_get:
|
||||
def mock_extract(text, entities, relations, **kwargs):
|
||||
triples = []
|
||||
for rel in relations:
|
||||
triples.append(Triple(
|
||||
subject=rel.subject.text,
|
||||
predicate=rel.predicate,
|
||||
object=rel.object.text,
|
||||
confidence=rel.confidence
|
||||
))
|
||||
return triples
|
||||
|
||||
mock_get.return_value = mock_extract
|
||||
|
||||
extractor = TripleExtractor()
|
||||
triples = extractor.extract_triples(self.text, self.entities, self.relations)
|
||||
|
||||
self.assertEqual(len(triples), 2)
|
||||
self.assertEqual(triples[0].subject, "Apple Inc.")
|
||||
self.assertEqual(triples[0].predicate, "founded_by")
|
||||
self.assertEqual(triples[0].object, "Steve Jobs")
|
||||
|
||||
def test_triple_validator(self):
|
||||
"""Test TripleValidator"""
|
||||
validator = TripleValidator()
|
||||
|
||||
# Create a valid and invalid triple
|
||||
valid_triple = Triple(subject="S", predicate="P", object="O", confidence=0.9)
|
||||
invalid_triple = Triple(subject="", predicate="P", object="O", confidence=0.9) # Empty subject
|
||||
low_conf_triple = Triple(subject="S", predicate="P", object="O", confidence=0.2)
|
||||
|
||||
triples = [valid_triple, invalid_triple, low_conf_triple]
|
||||
|
||||
validated = validator.validate_triples(triples, min_confidence=0.5)
|
||||
|
||||
self.assertEqual(len(validated), 1)
|
||||
self.assertEqual(validated[0], valid_triple)
|
||||
|
||||
def test_rdf_serializer(self):
|
||||
"""Test RDFSerializer"""
|
||||
serializer = RDFSerializer()
|
||||
triple = Triple(subject="Apple_Inc", predicate="founded_by", object="Steve_Jobs")
|
||||
|
||||
# Test N-Triples format
|
||||
rdf_output = serializer.serialize_to_rdf([triple], format="ntriples")
|
||||
self.assertIsInstance(rdf_output, str)
|
||||
# Check if basic components are in the output (format might vary slightly)
|
||||
# N-Triples: <subject> <predicate> <object> .
|
||||
# The serializer might handle URIs, let's just check non-empty
|
||||
self.assertTrue(len(rdf_output) > 0)
|
||||
|
||||
def test_triple_quality_checker(self):
|
||||
"""Test TripleQualityChecker"""
|
||||
checker = TripleQualityChecker()
|
||||
triples = [
|
||||
Triple(subject="Apple", predicate="founded", object="Jobs", confidence=0.9),
|
||||
Triple(subject="Apple", predicate="located", object="US", confidence=0.8)
|
||||
]
|
||||
|
||||
scores = checker.calculate_quality_scores(triples)
|
||||
|
||||
self.assertIn("average_score", scores)
|
||||
self.assertAlmostEqual(scores["average_score"], 0.85)
|
||||
# triple_count is not returned by calculate_quality_scores
|
||||
# self.assertIn("triple_count", scores)
|
||||
# self.assertEqual(scores["triple_count"], 2)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,235 @@
|
||||
import unittest
|
||||
import sys
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from semantica.semantic_extract.named_entity_recognizer import (
|
||||
NamedEntityRecognizer, EntityClassifier, EntityConfidenceScorer, CustomEntityDetector
|
||||
)
|
||||
from semantica.semantic_extract.ner_extractor import NERExtractor, Entity
|
||||
from semantica.semantic_extract.relation_extractor import RelationExtractor, Relation
|
||||
from semantica.semantic_extract.triple_extractor import TripleExtractor, Triple
|
||||
from semantica.semantic_extract.event_detector import EventDetector, Event
|
||||
from semantica.semantic_extract.semantic_analyzer import SemanticAnalyzer, SemanticRole
|
||||
from semantica.semantic_extract.methods import (
|
||||
extract_entities_regex, extract_entities_rules,
|
||||
extract_relations_regex, extract_relations_dependency,
|
||||
extract_triples_rules
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestSemanticExtractDeepDivePart2(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.text = "Steve Jobs founded Apple Inc. in 1976."
|
||||
self.entities = [
|
||||
Entity(text="Steve Jobs", label="PERSON", start_char=0, end_char=10),
|
||||
Entity(text="Apple Inc.", label="ORG", start_char=19, end_char=29),
|
||||
Entity(text="1976", label="DATE", start_char=33, end_char=37)
|
||||
]
|
||||
|
||||
# --- Entity Classifier Tests ---
|
||||
|
||||
def test_entity_classifier(self):
|
||||
"""Test EntityClassifier type classification"""
|
||||
classifier = EntityClassifier()
|
||||
|
||||
# Test type normalization
|
||||
e1 = Entity(text="Steve", label="PER", start_char=0, end_char=5)
|
||||
type1 = classifier.classify_entity_type(e1)
|
||||
self.assertEqual(type1, "PERSON")
|
||||
|
||||
e2 = Entity(text="Apple", label="ORGANIZATION", start_char=0, end_char=5)
|
||||
type2 = classifier.classify_entity_type(e2)
|
||||
self.assertEqual(type2, "ORG")
|
||||
|
||||
e3 = Entity(text="Unknown", label="CUSTOM", start_char=0, end_char=7)
|
||||
type3 = classifier.classify_entity_type(e3)
|
||||
self.assertEqual(type3, "CUSTOM")
|
||||
|
||||
def test_entity_classifier_disambiguation(self):
|
||||
"""Test EntityClassifier disambiguation"""
|
||||
classifier = EntityClassifier()
|
||||
|
||||
target = Entity(text="Apple", label="ORG", start_char=0, end_char=5)
|
||||
candidates = [
|
||||
Entity(text="Apple", label="FRUIT", start_char=0, end_char=5, confidence=0.6),
|
||||
Entity(text="Apple", label="ORG", start_char=0, end_char=5, confidence=0.9),
|
||||
Entity(text="Apple", label="ORG", start_char=0, end_char=5, confidence=0.5)
|
||||
]
|
||||
|
||||
best = classifier.disambiguate_entity(target, candidates)
|
||||
self.assertIsNotNone(best)
|
||||
self.assertEqual(best.label, "ORG")
|
||||
self.assertEqual(best.confidence, 0.9)
|
||||
|
||||
# --- Entity Confidence Scorer Tests ---
|
||||
|
||||
def test_entity_confidence_scorer(self):
|
||||
"""Test EntityConfidenceScorer"""
|
||||
scorer = EntityConfidenceScorer()
|
||||
|
||||
# Test scoring adjustments
|
||||
e1 = Entity(text="s", label="PERSON", start_char=0, end_char=1) # Too short
|
||||
scored_e1 = scorer.score_entities([e1])[0]
|
||||
self.assertLess(scored_e1.confidence, 1.0)
|
||||
|
||||
e2 = Entity(text="steve jobs", label="PERSON", start_char=0, end_char=10) # Lowercase person
|
||||
scored_e2 = scorer.score_entities([e2])[0]
|
||||
self.assertLess(scored_e2.confidence, 1.0)
|
||||
|
||||
e3 = Entity(text="1999", label="DATE", start_char=0, end_char=4) # Digit date
|
||||
# Should be boosted (capped at 1.0)
|
||||
scored_e3 = scorer.score_entities([e3])[0]
|
||||
self.assertLessEqual(scored_e3.confidence, 1.0)
|
||||
|
||||
# --- Custom Entity Detector Tests ---
|
||||
|
||||
def test_custom_entity_detector(self):
|
||||
"""Test CustomEntityDetector"""
|
||||
config = {
|
||||
"patterns": {
|
||||
"PROJECT": r"Project\s+[A-Z]\w+"
|
||||
}
|
||||
}
|
||||
detector = CustomEntityDetector(**config)
|
||||
text = "We are working on Project Apollo and Project Gemini."
|
||||
|
||||
entities = detector.detect_custom_entities(text, "PROJECT")
|
||||
self.assertEqual(len(entities), 2)
|
||||
self.assertEqual(entities[0].text, "Project Apollo")
|
||||
self.assertEqual(entities[0].label, "PROJECT")
|
||||
self.assertEqual(entities[1].text, "Project Gemini")
|
||||
|
||||
# --- Method Implementation Tests ---
|
||||
|
||||
def test_extract_entities_regex(self):
|
||||
"""Test regex-based entity extraction"""
|
||||
text = "Contact support@example.com or admin@test.org"
|
||||
patterns = {"EMAIL": r"[\w\.-]+@[\w\.-]+"}
|
||||
|
||||
entities = extract_entities_regex(text, patterns=patterns)
|
||||
self.assertEqual(len(entities), 2)
|
||||
self.assertEqual(entities[0].label, "EMAIL")
|
||||
self.assertEqual(entities[0].text, "support@example.com")
|
||||
|
||||
def test_extract_entities_rules(self):
|
||||
"""Test rule-based entity extraction (sentence start rule)"""
|
||||
text = "Alice went to the park. Bob stayed home."
|
||||
# Assuming rule: Capitalized word at start of sentence is PERSON
|
||||
entities = extract_entities_rules(text)
|
||||
|
||||
# This depends on exact implementation details in methods.py
|
||||
# Current impl: Checks first word of sentence
|
||||
names = [e.text for e in entities]
|
||||
self.assertIn("Alice", names)
|
||||
self.assertIn("Bob", names)
|
||||
|
||||
def test_extract_relations_regex(self):
|
||||
"""Test regex-based relation extraction"""
|
||||
text = "London is located in UK"
|
||||
entities = [
|
||||
Entity(text="London", label="GPE", start_char=0, end_char=6),
|
||||
Entity(text="UK", label="GPE", start_char=21, end_char=23)
|
||||
]
|
||||
|
||||
relations = extract_relations_regex(text, entities)
|
||||
self.assertTrue(len(relations) > 0)
|
||||
self.assertEqual(relations[0].predicate, "located_in")
|
||||
|
||||
@patch("semantica.semantic_extract.methods.SPACY_AVAILABLE", False)
|
||||
@patch("semantica.semantic_extract.methods.extract_relations_pattern")
|
||||
def test_extract_relations_dependency_fallback(self, mock_pattern):
|
||||
"""Test dependency extraction fallback when spaCy is missing"""
|
||||
mock_pattern.return_value = []
|
||||
extract_relations_dependency("text", [])
|
||||
mock_pattern.assert_called_once()
|
||||
|
||||
def test_extract_triples_rules(self):
|
||||
"""Test rule-based triple extraction"""
|
||||
text = "Steve founded Apple"
|
||||
entities = [
|
||||
Entity(text="Steve", label="PERSON", start_char=0, end_char=5),
|
||||
Entity(text="Apple", label="ORG", start_char=14, end_char=19)
|
||||
]
|
||||
|
||||
triples = extract_triples_rules(text, entities)
|
||||
self.assertTrue(len(triples) > 0)
|
||||
self.assertEqual(triples[0].predicate, "founded")
|
||||
self.assertEqual(triples[0].subject, "Steve")
|
||||
self.assertEqual(triples[0].object, "Apple")
|
||||
|
||||
# --- Event Detector Tests ---
|
||||
|
||||
def test_event_detector_basic(self):
|
||||
"""Test EventDetector basic flow"""
|
||||
# EventDetector uses internal patterns, so we test with text matching those patterns
|
||||
# Patterns include: founded, acquired, launched, etc.
|
||||
text = "Apple was founded by Steve Jobs in 1976."
|
||||
|
||||
# Mock _extract_participants to avoid complex logic and potential flake
|
||||
# or just let it run if it's simple. It looks simple in the code.
|
||||
# But we must be careful.
|
||||
|
||||
detector = EventDetector()
|
||||
events = detector.detect_events(text)
|
||||
|
||||
self.assertTrue(len(events) > 0)
|
||||
self.assertEqual(events[0].event_type, "founded")
|
||||
# Check if participants were extracted (simple capitalization rule)
|
||||
# "Steve" and "Jobs" should be captured.
|
||||
# The logic captures capitalized words > 2 chars.
|
||||
# "Apple" (if in context), "Steve", "Jobs" might be captured.
|
||||
|
||||
# We'll check if "Steve" or "Jobs" is in participants list
|
||||
participants = events[0].participants
|
||||
self.assertTrue(any("Steve" in p for p in participants) or any("Jobs" in p for p in participants))
|
||||
|
||||
# --- Semantic Analyzer Tests ---
|
||||
|
||||
def test_semantic_analyzer_similarity(self):
|
||||
"""Test SemanticAnalyzer similarity"""
|
||||
analyzer = SemanticAnalyzer()
|
||||
# Jaccard similarity
|
||||
s1 = "apple banana"
|
||||
s2 = "apple orange"
|
||||
score = analyzer.calculate_similarity(s1, s2, method="jaccard")
|
||||
# intersection: apple (1), union: apple, banana, orange (3) -> 1/3 ~ 0.33
|
||||
self.assertAlmostEqual(score, 1/3)
|
||||
|
||||
# --- Coreference Resolver Tests ---
|
||||
|
||||
def test_coreference_resolver_pronouns(self):
|
||||
"""Test CoreferenceResolver pronoun resolution"""
|
||||
from semantica.semantic_extract.coreference_resolver import CoreferenceResolver, Mention
|
||||
|
||||
resolver = CoreferenceResolver()
|
||||
|
||||
# "Steve Jobs founded Apple. He was the CEO."
|
||||
# We need to manually construct mentions because we are testing the resolver logic
|
||||
# independent of the entity extractor for this unit test
|
||||
|
||||
mentions = [
|
||||
Mention(text="Steve Jobs", start_char=0, end_char=10, mention_type="entity", entity_id="e1"),
|
||||
Mention(text="Apple", start_char=19, end_char=24, mention_type="entity", entity_id="e2"),
|
||||
Mention(text="He", start_char=26, end_char=28, mention_type="pronoun")
|
||||
]
|
||||
|
||||
text = "Steve Jobs founded Apple. He was the CEO."
|
||||
|
||||
# Use the pronoun resolver directly or via main resolver
|
||||
resolutions = resolver.pronoun_resolver.resolve_pronouns(text, mentions)
|
||||
|
||||
self.assertTrue(len(resolutions) > 0)
|
||||
# Should resolve "He" to "Steve Jobs" (closest preceding entity)
|
||||
self.assertEqual(resolutions[0][0], "He")
|
||||
self.assertEqual(resolutions[0][1], "Steve Jobs")
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,145 @@
|
||||
|
||||
import unittest
|
||||
import sys
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from semantica.semantic_extract.triple_extractor import (
|
||||
TripleExtractor, Triple, TripleValidator, RDFSerializer, TripleQualityChecker
|
||||
)
|
||||
from semantica.semantic_extract.ner_extractor import Entity
|
||||
from semantica.semantic_extract.relation_extractor import Relation
|
||||
|
||||
class TestSemanticExtractTriples(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.entities = [
|
||||
Entity(text="Steve Jobs", label="PERSON", start_char=0, end_char=10),
|
||||
Entity(text="Apple", label="ORG", start_char=19, end_char=24)
|
||||
]
|
||||
self.relations = [
|
||||
Relation(
|
||||
subject=self.entities[0],
|
||||
predicate="founded",
|
||||
object=self.entities[1],
|
||||
confidence=0.9,
|
||||
context="Steve Jobs founded Apple."
|
||||
)
|
||||
]
|
||||
self.triples = [
|
||||
Triple(subject="Steve_Jobs", predicate="founded", object="Apple", confidence=0.9),
|
||||
Triple(subject="Apple", predicate="located_in", object="Cupertino", confidence=0.8)
|
||||
]
|
||||
|
||||
# --- Triple Extractor Tests ---
|
||||
|
||||
def test_triple_extractor_init(self):
|
||||
"""Test TripleExtractor initialization"""
|
||||
extractor = TripleExtractor()
|
||||
self.assertIsNotNone(extractor.triple_validator)
|
||||
self.assertIsNotNone(extractor.rdf_serializer)
|
||||
self.assertIsNotNone(extractor.quality_checker)
|
||||
|
||||
def test_triple_extractor_extract_from_relations(self):
|
||||
"""Test extracting triples by converting relations (fallback/default)"""
|
||||
extractor = TripleExtractor(method=[]) # No specific method, force fallback
|
||||
|
||||
# Mocking progress tracker to avoid console clutter/errors
|
||||
extractor.progress_tracker = MagicMock()
|
||||
|
||||
triples = extractor.extract_triples(
|
||||
text="Steve Jobs founded Apple.",
|
||||
entities=self.entities,
|
||||
relationships=self.relations
|
||||
)
|
||||
|
||||
self.assertEqual(len(triples), 1)
|
||||
# Predicate is formatted as URI
|
||||
self.assertTrue(triples[0].predicate.endswith("founded") or triples[0].predicate == "founded")
|
||||
# Check URI formatting (simple implementation in _format_uri)
|
||||
# "Steve Jobs" -> "Steve_Jobs", prepended with http://example.org/ if not http
|
||||
self.assertIn("Steve_Jobs", triples[0].subject)
|
||||
|
||||
# --- Triple Validator Tests ---
|
||||
|
||||
def test_triple_validator_valid(self):
|
||||
"""Test TripleValidator with valid triple"""
|
||||
validator = TripleValidator()
|
||||
triple = Triple(subject="S", predicate="P", object="O", confidence=0.9)
|
||||
self.assertTrue(validator.validate_triple(triple))
|
||||
|
||||
def test_triple_validator_invalid_structure(self):
|
||||
"""Test TripleValidator with missing fields"""
|
||||
validator = TripleValidator()
|
||||
triple = Triple(subject="", predicate="P", object="O") # Empty subject
|
||||
self.assertFalse(validator.validate_triple(triple))
|
||||
|
||||
def test_triple_validator_low_confidence(self):
|
||||
"""Test TripleValidator confidence threshold"""
|
||||
validator = TripleValidator()
|
||||
triple = Triple(subject="S", predicate="P", object="O", confidence=0.4)
|
||||
self.assertFalse(validator.validate_triple(triple, min_confidence=0.5))
|
||||
|
||||
# --- RDF Serializer Tests ---
|
||||
|
||||
def test_rdf_serializer_turtle(self):
|
||||
"""Test RDF serialization to Turtle"""
|
||||
serializer = RDFSerializer()
|
||||
output = serializer.serialize_to_rdf(self.triples, format="turtle")
|
||||
self.assertIn("@prefix", output)
|
||||
self.assertIn("Steve_Jobs", output)
|
||||
self.assertIn("founded", output)
|
||||
self.assertIn("Apple", output)
|
||||
self.assertTrue(output.strip().endswith("."))
|
||||
|
||||
def test_rdf_serializer_ntriples(self):
|
||||
"""Test RDF serialization to N-Triples"""
|
||||
serializer = RDFSerializer()
|
||||
output = serializer.serialize_to_rdf(self.triples, format="ntriples")
|
||||
self.assertNotIn("@prefix", output)
|
||||
self.assertIn("<Steve_Jobs>", output)
|
||||
self.assertIn("<founded>", output)
|
||||
|
||||
def test_rdf_serializer_jsonld(self):
|
||||
"""Test RDF serialization to JSON-LD"""
|
||||
serializer = RDFSerializer()
|
||||
output = serializer.serialize_to_rdf(self.triples, format="jsonld")
|
||||
import json
|
||||
data = json.loads(output)
|
||||
self.assertIn("@graph", data)
|
||||
self.assertEqual(len(data["@graph"]), 2)
|
||||
|
||||
def test_rdf_serializer_xml(self):
|
||||
"""Test RDF serialization to XML"""
|
||||
serializer = RDFSerializer()
|
||||
output = serializer.serialize_to_rdf(self.triples, format="xml")
|
||||
self.assertIn("rdf:RDF", output)
|
||||
self.assertIn("rdf:Description", output)
|
||||
|
||||
# --- Triple Quality Checker Tests ---
|
||||
|
||||
def test_triple_quality_checker_assess(self):
|
||||
"""Test TripleQualityChecker assessment"""
|
||||
checker = TripleQualityChecker()
|
||||
triple = Triple(subject="S", predicate="P", object="O", confidence=0.85)
|
||||
assessment = checker.assess_triple_quality(triple)
|
||||
|
||||
self.assertEqual(assessment["confidence"], 0.85)
|
||||
self.assertEqual(assessment["completeness"], 1.0)
|
||||
self.assertEqual(assessment["quality_score"], 0.85)
|
||||
|
||||
def test_triple_quality_checker_stats(self):
|
||||
"""Test TripleQualityChecker statistics"""
|
||||
checker = TripleQualityChecker()
|
||||
stats = checker.calculate_quality_scores(self.triples)
|
||||
|
||||
# Implementation returns average_score, min_score, max_score, high_quality, medium_quality, low_quality
|
||||
self.assertIn("average_score", stats)
|
||||
self.assertIn("high_quality", stats) # 0.9 and 0.8 are >= 0.8
|
||||
self.assertEqual(stats["high_quality"], 2)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
+21
-21
@@ -1,19 +1,19 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
from semantica.triple_store.triple_manager import TripleManager, TripleStore
|
||||
from semantica.triple_store.query_engine import QueryEngine, QueryResult
|
||||
from semantica.triplet_store.triplet_manager import TripletManager, TripletStore
|
||||
from semantica.triplet_store.query_engine import QueryEngine, QueryResult
|
||||
from semantica.semantic_extract.triple_extractor import Triple
|
||||
|
||||
class TestTripleStore(unittest.TestCase):
|
||||
class TestTripletStore(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.mock_logger = MagicMock()
|
||||
self.mock_tracker = MagicMock()
|
||||
|
||||
self.logger_patcher = patch('semantica.triple_store.triple_manager.get_logger', return_value=self.mock_logger)
|
||||
self.tracker_patcher = patch('semantica.triple_store.triple_manager.get_progress_tracker', return_value=self.mock_tracker)
|
||||
self.logger_patcher_qe = patch('semantica.triple_store.query_engine.get_logger', return_value=self.mock_logger)
|
||||
self.tracker_patcher_qe = patch('semantica.triple_store.query_engine.get_progress_tracker', return_value=self.mock_tracker)
|
||||
self.logger_patcher = patch('semantica.triplet_store.triplet_manager.get_logger', return_value=self.mock_logger)
|
||||
self.tracker_patcher = patch('semantica.triplet_store.triplet_manager.get_progress_tracker', return_value=self.mock_tracker)
|
||||
self.logger_patcher_qe = patch('semantica.triplet_store.query_engine.get_logger', return_value=self.mock_logger)
|
||||
self.tracker_patcher_qe = patch('semantica.triplet_store.query_engine.get_progress_tracker', return_value=self.mock_tracker)
|
||||
|
||||
self.logger_patcher.start()
|
||||
self.tracker_patcher.start()
|
||||
@@ -26,23 +26,23 @@ class TestTripleStore(unittest.TestCase):
|
||||
self.logger_patcher_qe.stop()
|
||||
self.tracker_patcher_qe.stop()
|
||||
|
||||
def test_triple_manager_init(self):
|
||||
manager = TripleManager(default_store="main")
|
||||
def test_triplet_manager_init(self):
|
||||
manager = TripletManager(default_store="main")
|
||||
self.assertEqual(manager.default_store_id, "main")
|
||||
self.assertEqual(manager.stores, {})
|
||||
|
||||
def test_register_store(self):
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
store = manager.register_store("main", "blazegraph", "http://localhost:9999")
|
||||
self.assertIsInstance(store, TripleStore)
|
||||
self.assertIsInstance(store, TripletStore)
|
||||
self.assertEqual(store.store_id, "main")
|
||||
self.assertEqual(store.store_type, "blazegraph")
|
||||
self.assertEqual(store.endpoint, "http://localhost:9999")
|
||||
self.assertIn("main", manager.stores)
|
||||
|
||||
@patch('semantica.triple_store.triple_manager.TripleManager._get_adapter')
|
||||
@patch('semantica.triplet_store.triplet_manager.TripletManager._get_adapter')
|
||||
def test_add_triple(self, mock_get_adapter):
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
manager.register_store("main", "blazegraph", "http://localhost:9999")
|
||||
|
||||
mock_adapter = MagicMock()
|
||||
@@ -56,9 +56,9 @@ class TestTripleStore(unittest.TestCase):
|
||||
self.assertEqual(result["store_id"], "main")
|
||||
mock_adapter.add_triple.assert_called_once_with(triple)
|
||||
|
||||
@patch('semantica.triple_store.triple_manager.TripleManager._get_adapter')
|
||||
@patch('semantica.triplet_store.triplet_manager.TripletManager._get_adapter')
|
||||
def test_add_triples(self, mock_get_adapter):
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
manager.register_store("main", "blazegraph", "http://localhost:9999")
|
||||
|
||||
mock_adapter = MagicMock()
|
||||
@@ -76,9 +76,9 @@ class TestTripleStore(unittest.TestCase):
|
||||
self.assertEqual(result["total_triples"], 2)
|
||||
mock_adapter.add_triples.assert_called()
|
||||
|
||||
@patch('semantica.triple_store.triple_manager.TripleManager._get_adapter')
|
||||
@patch('semantica.triplet_store.triplet_manager.TripletManager._get_adapter')
|
||||
def test_get_triple(self, mock_get_adapter):
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
manager.register_store("main", "blazegraph", "http://localhost:9999")
|
||||
|
||||
mock_adapter = MagicMock()
|
||||
@@ -91,9 +91,9 @@ class TestTripleStore(unittest.TestCase):
|
||||
self.assertEqual(result, expected_triples)
|
||||
mock_adapter.get_triples.assert_called_once_with("s", None, None)
|
||||
|
||||
@patch('semantica.triple_store.triple_manager.TripleManager._get_adapter')
|
||||
@patch('semantica.triplet_store.triplet_manager.TripletManager._get_adapter')
|
||||
def test_delete_triple(self, mock_get_adapter):
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
manager.register_store("main", "blazegraph", "http://localhost:9999")
|
||||
|
||||
mock_adapter = MagicMock()
|
||||
@@ -106,9 +106,9 @@ class TestTripleStore(unittest.TestCase):
|
||||
self.assertTrue(result["success"])
|
||||
mock_adapter.delete_triple.assert_called_once_with(triple)
|
||||
|
||||
@patch('semantica.triple_store.triple_manager.TripleManager._get_adapter')
|
||||
@patch('semantica.triplet_store.triplet_manager.TripletManager._get_adapter')
|
||||
def test_update_triple(self, mock_get_adapter):
|
||||
manager = TripleManager()
|
||||
manager = TripletManager()
|
||||
manager.register_store("main", "blazegraph", "http://localhost:9999")
|
||||
|
||||
mock_adapter = MagicMock()
|
||||
@@ -0,0 +1,62 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Ensure semantica is in path
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')))
|
||||
|
||||
from semantica.vector_store.vector_store import VectorStore
|
||||
from semantica.vector_store.registry import method_registry
|
||||
from semantica.vector_store.config import vector_store_config
|
||||
|
||||
class TestPineconeRemoval(unittest.TestCase):
|
||||
"""Verify that Pinecone has been completely removed from the system."""
|
||||
|
||||
def test_pinecone_backend_rejected(self):
|
||||
"""Test that initializing VectorStore with backend='pinecone' raises an error."""
|
||||
with self.assertRaises(ValueError) as context:
|
||||
VectorStore(backend="pinecone")
|
||||
|
||||
# The error message might be generic "Unknown backend" or specific.
|
||||
# We just want to ensure it fails.
|
||||
self.assertTrue("pinecone" in str(context.exception).lower() or "unknown" in str(context.exception).lower())
|
||||
|
||||
def test_registry_clean(self):
|
||||
"""Test that no Pinecone methods are registered."""
|
||||
# Check all task types
|
||||
task_types = ["store", "search", "index", "hybrid_search", "metadata", "namespace"]
|
||||
|
||||
for task in task_types:
|
||||
methods = method_registry.list_all(task)
|
||||
# Flatten if it's a dict
|
||||
if isinstance(methods, dict):
|
||||
method_names = methods.get(task, [])
|
||||
else:
|
||||
method_names = methods
|
||||
|
||||
for name in method_names:
|
||||
self.assertNotIn("pinecone", name.lower(), f"Found pinecone reference in registry task {task}: {name}")
|
||||
|
||||
def test_config_clean(self):
|
||||
"""Test that configuration does not contain Pinecone keys."""
|
||||
config = vector_store_config.get_all()
|
||||
|
||||
for key in config.keys():
|
||||
self.assertNotIn("pinecone", key.lower(), f"Found pinecone key in config: {key}")
|
||||
|
||||
def test_adapters_existence(self):
|
||||
"""Verify that other adapters exist but PineconeAdapter does not."""
|
||||
try:
|
||||
from semantica.vector_store import faiss_adapter
|
||||
from semantica.vector_store import weaviate_adapter
|
||||
from semantica.vector_store import qdrant_adapter
|
||||
from semantica.vector_store import milvus_adapter
|
||||
except ImportError as e:
|
||||
self.fail(f"Failed to import a required adapter: {e}")
|
||||
|
||||
with self.assertRaises(ImportError):
|
||||
from semantica.vector_store import pinecone_adapter
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,375 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch, ANY
|
||||
import numpy as np
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.append(str(Path(__file__).parent.parent.parent))
|
||||
|
||||
from semantica.vector_store.vector_store import VectorStore, VectorIndexer, VectorRetriever, VectorManager
|
||||
from semantica.vector_store.registry import MethodRegistry, method_registry
|
||||
from semantica.vector_store.faiss_adapter import FAISSAdapter, FAISSIndex, FAISSIndexBuilder, FAISSSearch
|
||||
from semantica.vector_store.milvus_adapter import MilvusAdapter, MilvusClient, MilvusCollection, MilvusSearch
|
||||
from semantica.vector_store.qdrant_adapter import QdrantAdapter
|
||||
from semantica.vector_store.weaviate_adapter import WeaviateAdapter
|
||||
from semantica.vector_store.hybrid_search import HybridSearch, MetadataFilter, SearchRanker
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
class TestVectorStoreDeepDive(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.vectors = [np.array([1.0, 0.0]), np.array([0.0, 1.0])]
|
||||
self.ids = ["vec_1", "vec_2"]
|
||||
self.metadata = [{"type": "a"}, {"type": "b"}]
|
||||
|
||||
def test_vector_store_in_memory(self):
|
||||
"""Test the default in-memory VectorStore implementation."""
|
||||
store = VectorStore(backend="inmemory", dimension=2)
|
||||
|
||||
# Test storing vectors
|
||||
ids = store.store_vectors(self.vectors, self.metadata)
|
||||
self.assertEqual(len(ids), 2)
|
||||
self.assertEqual(store.vectors[ids[0]].tolist(), self.vectors[0].tolist())
|
||||
|
||||
# Test searching vectors (exact match)
|
||||
results = store.search_vectors(np.array([1.0, 0.0]), k=1)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], ids[0])
|
||||
# Score should be close to 1.0 (cosine similarity of identical vectors)
|
||||
self.assertAlmostEqual(results[0]["score"], 1.0)
|
||||
|
||||
# Test searching vectors (orthogonal)
|
||||
results = store.search_vectors(np.array([0.0, 1.0]), k=1)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], ids[1])
|
||||
|
||||
# Test updating vectors
|
||||
new_vec = np.array([0.5, 0.5])
|
||||
store.update_vectors([ids[0]], [new_vec])
|
||||
self.assertTrue(np.array_equal(store.get_vector(ids[0]), new_vec))
|
||||
|
||||
# Test deleting vectors
|
||||
store.delete_vectors([ids[0]])
|
||||
self.assertIsNone(store.get_vector(ids[0]))
|
||||
self.assertEqual(len(store.vectors), 1)
|
||||
|
||||
def test_vector_indexer_retriever(self):
|
||||
"""Test VectorIndexer and VectorRetriever directly."""
|
||||
indexer = VectorIndexer(backend="inmemory", dimension=2)
|
||||
index = indexer.create_index(self.vectors, self.ids)
|
||||
self.assertIsNotNone(index)
|
||||
self.assertEqual(len(index["vectors"]), 2)
|
||||
|
||||
retriever = VectorRetriever(backend="inmemory")
|
||||
results = retriever.search_similar(
|
||||
np.array([1.0, 0.0]),
|
||||
self.vectors,
|
||||
self.ids,
|
||||
k=1
|
||||
)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "vec_1")
|
||||
|
||||
# Test hybrid search (metadata filter)
|
||||
results = retriever.search_hybrid(
|
||||
np.array([1.0, 0.0]),
|
||||
{"type": "b"}, # Filter for vec_2
|
||||
self.vectors,
|
||||
self.metadata,
|
||||
k=1
|
||||
)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["vector"].tolist(), self.vectors[1].tolist())
|
||||
|
||||
def test_method_registry(self):
|
||||
"""Test the MethodRegistry."""
|
||||
registry = MethodRegistry()
|
||||
|
||||
def custom_store(): return "stored"
|
||||
|
||||
# Register
|
||||
registry.register("store", "custom", custom_store, version="1.0")
|
||||
self.assertTrue(registry.has("store", "custom"))
|
||||
|
||||
# Get
|
||||
func = registry.get("store", "custom")
|
||||
self.assertEqual(func(), "stored")
|
||||
|
||||
# Metadata
|
||||
meta = registry.get_metadata("store", "custom")
|
||||
self.assertEqual(meta["version"], "1.0")
|
||||
|
||||
# List
|
||||
all_methods = registry.list_all("store")
|
||||
self.assertEqual(all_methods["store"], ["custom"])
|
||||
|
||||
# Unregister
|
||||
registry.unregister("store", "custom")
|
||||
self.assertFalse(registry.has("store", "custom"))
|
||||
|
||||
@patch('semantica.vector_store.faiss_adapter.faiss')
|
||||
@patch('semantica.vector_store.faiss_adapter.FAISS_AVAILABLE', True)
|
||||
def test_faiss_adapter(self, mock_faiss):
|
||||
"""Test FAISSAdapter with mocked faiss."""
|
||||
# Setup mock
|
||||
mock_index = MagicMock()
|
||||
mock_faiss.IndexFlatL2.return_value = mock_index
|
||||
mock_faiss.read_index.return_value = mock_index
|
||||
|
||||
# Mock search return
|
||||
# distances, indices
|
||||
mock_index.search.return_value = (np.array([[0.0, 0.1]]), np.array([[0, 1]]))
|
||||
mock_index.ntotal = 2
|
||||
|
||||
# Test Init
|
||||
adapter = FAISSAdapter(dimension=2)
|
||||
|
||||
# Test Create Index
|
||||
adapter.create_index(index_type="flat")
|
||||
mock_faiss.IndexFlatL2.assert_called_with(2)
|
||||
|
||||
# Test Add Vectors
|
||||
adapter.add_vectors(self.vectors, self.ids, self.metadata)
|
||||
mock_index.add.assert_called()
|
||||
self.assertEqual(len(adapter.index.vector_ids), 2)
|
||||
|
||||
# Test Search
|
||||
results = adapter.search_similar(np.array([1.0, 0.0]), k=2)
|
||||
self.assertEqual(len(results), 2)
|
||||
self.assertEqual(results[0]["id"], "vec_1")
|
||||
|
||||
# Test Save
|
||||
adapter.save_index("test.index")
|
||||
mock_faiss.write_index.assert_called()
|
||||
|
||||
# Test Load
|
||||
adapter.load_index("test.index")
|
||||
mock_faiss.read_index.assert_called()
|
||||
|
||||
@patch('semantica.vector_store.milvus_adapter.connections')
|
||||
@patch('semantica.vector_store.milvus_adapter.Collection')
|
||||
@patch('semantica.vector_store.milvus_adapter.utility')
|
||||
@patch('semantica.vector_store.milvus_adapter.DataType')
|
||||
@patch('semantica.vector_store.milvus_adapter.FieldSchema')
|
||||
@patch('semantica.vector_store.milvus_adapter.CollectionSchema')
|
||||
@patch('semantica.vector_store.milvus_adapter.MILVUS_AVAILABLE', True)
|
||||
def test_milvus_adapter(self, mock_collection_schema, mock_field_schema, mock_data_type, mock_utility, mock_collection_cls, mock_connections):
|
||||
"""Test MilvusAdapter with mocked pymilvus."""
|
||||
# Setup mocks
|
||||
mock_data_type.INT64 = 1
|
||||
mock_data_type.FLOAT_VECTOR = 2
|
||||
# Setup mocks
|
||||
mock_utility.has_collection.return_value = False
|
||||
mock_collection_instance = MagicMock()
|
||||
mock_collection_cls.return_value = mock_collection_instance
|
||||
|
||||
# Mock search results
|
||||
mock_hit = MagicMock()
|
||||
mock_hit.id = 1
|
||||
mock_hit.distance = 0.1
|
||||
mock_collection_instance.search.return_value = [[mock_hit]]
|
||||
|
||||
# Test Init
|
||||
adapter = MilvusAdapter(host="localhost")
|
||||
|
||||
# Test Connect
|
||||
adapter.connect()
|
||||
mock_connections.connect.assert_called_with(
|
||||
alias="default", host="localhost", port=19530, user=None, password=None
|
||||
)
|
||||
|
||||
# Test Create Collection
|
||||
adapter.create_collection("test_coll", dimension=2)
|
||||
mock_collection_cls.assert_called()
|
||||
mock_collection_instance.create_index.assert_called()
|
||||
|
||||
# Test Insert
|
||||
adapter.insert_vectors(self.vectors)
|
||||
mock_collection_instance.insert.assert_called()
|
||||
|
||||
# Test Search
|
||||
results = adapter.search_vectors(np.array([1.0, 0.0]), limit=1)
|
||||
mock_collection_instance.search.assert_called()
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], 1)
|
||||
|
||||
@patch('semantica.vector_store.qdrant_adapter.QdrantClientLib')
|
||||
@patch('semantica.vector_store.qdrant_adapter.VectorParams')
|
||||
@patch('semantica.vector_store.qdrant_adapter.Distance')
|
||||
@patch('semantica.vector_store.qdrant_adapter.PointStruct')
|
||||
@patch('semantica.vector_store.qdrant_adapter.QDRANT_AVAILABLE', True)
|
||||
def test_qdrant_adapter(self, mock_point_struct, mock_distance, mock_vector_params, mock_qdrant_cls):
|
||||
"""Test QdrantAdapter with mocked qdrant_client."""
|
||||
mock_client = MagicMock()
|
||||
mock_qdrant_cls.return_value = mock_client
|
||||
|
||||
# Mock search response
|
||||
mock_hit = MagicMock()
|
||||
mock_hit.id = "vec_1"
|
||||
mock_hit.score = 0.9
|
||||
mock_hit.payload = {"type": "a"}
|
||||
mock_client.search.return_value = [mock_hit]
|
||||
|
||||
adapter = QdrantAdapter(url="http://localhost:6333")
|
||||
|
||||
# Connect
|
||||
adapter.connect()
|
||||
mock_qdrant_cls.assert_called()
|
||||
|
||||
# Create Collection
|
||||
adapter.create_collection("test-collection", vector_size=2)
|
||||
mock_client.create_collection.assert_called()
|
||||
|
||||
# Insert
|
||||
adapter.insert_vectors(self.vectors, self.ids, payloads=self.metadata)
|
||||
mock_client.upsert.assert_called()
|
||||
|
||||
# Search
|
||||
results = adapter.search_vectors(np.array([1.0, 0.0]), limit=1)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "vec_1")
|
||||
|
||||
@patch('semantica.vector_store.weaviate_adapter.weaviate')
|
||||
@patch('semantica.vector_store.weaviate_adapter.MetadataQuery')
|
||||
@patch('semantica.vector_store.weaviate_adapter.WEAVIATE_AVAILABLE', True)
|
||||
def test_weaviate_adapter(self, mock_metadata_query, mock_weaviate):
|
||||
"""Test WeaviateAdapter with mocked weaviate."""
|
||||
mock_client = MagicMock()
|
||||
mock_weaviate.connect_to_local.return_value = mock_client
|
||||
|
||||
mock_collection = MagicMock()
|
||||
mock_client.collections.get.return_value = mock_collection
|
||||
|
||||
# Mock search response
|
||||
mock_obj = MagicMock()
|
||||
mock_obj.uuid = "uuid-1"
|
||||
mock_obj.properties = {"text": "hello"}
|
||||
mock_obj.metadata.distance = 0.1
|
||||
|
||||
mock_query_response = MagicMock()
|
||||
mock_query_response.objects = [mock_obj]
|
||||
|
||||
mock_collection.query.near_vector.return_value = mock_query_response
|
||||
|
||||
adapter = WeaviateAdapter(url="http://localhost:8080")
|
||||
|
||||
# Connect
|
||||
adapter.connect()
|
||||
mock_weaviate.connect_to_local.assert_called()
|
||||
|
||||
# Create Schema
|
||||
adapter.create_schema("TestClass", properties=[])
|
||||
mock_client.collections.create.assert_called()
|
||||
|
||||
# Add Objects
|
||||
# Need to mock batch context manager
|
||||
mock_batch = MagicMock()
|
||||
mock_collection.batch.dynamic.return_value.__enter__.return_value = mock_batch
|
||||
|
||||
adapter.get_collection("TestClass")
|
||||
adapter.add_objects([{"text": "hello"}], vectors=self.vectors)
|
||||
mock_batch.add_object.assert_called()
|
||||
|
||||
# Query
|
||||
results = adapter.query_vectors(np.array([1.0, 0.0]), limit=1)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "uuid-1")
|
||||
|
||||
def test_hybrid_search(self):
|
||||
"""Test HybridSearch, MetadataFilter and SearchRanker."""
|
||||
search = HybridSearch()
|
||||
|
||||
# Test MetadataFilter
|
||||
meta_filter = MetadataFilter().eq("type", "a")
|
||||
self.assertTrue(meta_filter.matches({"type": "a"}))
|
||||
self.assertFalse(meta_filter.matches({"type": "b"}))
|
||||
|
||||
meta_filter = MetadataFilter().gt("val", 10)
|
||||
self.assertTrue(meta_filter.matches({"val": 20}))
|
||||
self.assertFalse(meta_filter.matches({"val": 5}))
|
||||
|
||||
# Test Search
|
||||
results = search.search(
|
||||
query_vector=np.array([1.0, 0.0]),
|
||||
vectors=self.vectors,
|
||||
metadata=self.metadata,
|
||||
vector_ids=self.ids,
|
||||
k=2
|
||||
)
|
||||
self.assertEqual(len(results), 2)
|
||||
self.assertEqual(results[0]["id"], "vec_1")
|
||||
|
||||
# Test Filtered Search
|
||||
results = search.search(
|
||||
query_vector=np.array([1.0, 0.0]),
|
||||
vectors=self.vectors,
|
||||
metadata=self.metadata,
|
||||
vector_ids=self.ids,
|
||||
k=2,
|
||||
metadata_filter=MetadataFilter().eq("type", "b")
|
||||
)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["id"], "vec_2")
|
||||
|
||||
# Test Ranker
|
||||
ranker = SearchRanker(strategy="reciprocal_rank_fusion")
|
||||
res1 = [{"id": "1", "score": 0.9}, {"id": "2", "score": 0.8}]
|
||||
res2 = [{"id": "2", "score": 0.85}, {"id": "1", "score": 0.7}]
|
||||
|
||||
fused = ranker.rank([res1, res2])
|
||||
self.assertEqual(len(fused), 2)
|
||||
# ID 2 should be top because it's high in both? Or ID 1?
|
||||
# RRF: 1/(k+1) + 1/(k+2) vs 1/(k+2) + 1/(k+1). They are equal rank-wise (1st and 2nd).
|
||||
|
||||
# Multi-source search
|
||||
sources = [
|
||||
{"vectors": [self.vectors[0]], "metadata": [self.metadata[0]], "ids": ["vec_1"]},
|
||||
{"vectors": [self.vectors[1]], "metadata": [self.metadata[1]], "ids": ["vec_2"]}
|
||||
]
|
||||
multi_res = search.multi_source_search(np.array([1.0, 0.0]), sources, k=2)
|
||||
self.assertEqual(len(multi_res), 2)
|
||||
|
||||
def test_vector_manager(self):
|
||||
"""Test VectorManager."""
|
||||
manager = VectorManager()
|
||||
store = VectorStore(backend="inmemory")
|
||||
store.store_vectors(self.vectors, self.metadata)
|
||||
|
||||
# Test statistics
|
||||
stats = manager.collect_statistics(store)
|
||||
self.assertEqual(stats["total_vectors"], 2)
|
||||
self.assertEqual(stats["backend"], "inmemory")
|
||||
|
||||
# Test maintenance
|
||||
health = manager.maintain_store(store)
|
||||
self.assertTrue(health["healthy"])
|
||||
|
||||
# Test manage_store wrapper
|
||||
results = manager.manage_store(store, statistics=True, optimize=True)
|
||||
self.assertIn("statistics", results)
|
||||
self.assertIn("optimize", results)
|
||||
|
||||
def test_config(self):
|
||||
"""Test VectorStoreConfig."""
|
||||
from semantica.vector_store.config import vector_store_config
|
||||
|
||||
# Test get default
|
||||
self.assertEqual(vector_store_config.get("default_backend"), "faiss")
|
||||
|
||||
# Test set
|
||||
vector_store_config.set("test_key", "test_value")
|
||||
self.assertEqual(vector_store_config.get("test_key"), "test_value")
|
||||
|
||||
# Test update
|
||||
vector_store_config.update({"test_key_2": "val2"})
|
||||
self.assertEqual(vector_store_config.get("test_key_2"), "val2")
|
||||
|
||||
# Test method config
|
||||
vector_store_config.set_method_config("test_method", {"param": 1})
|
||||
self.assertEqual(vector_store_config.get_method_config("test_method")["param"], 1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -5,7 +5,8 @@ import traceback
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
# Configure logging
|
||||
import pytest
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
logger = logging.getLogger("verify_backends")
|
||||
|
||||
@@ -15,6 +16,8 @@ except ImportError:
|
||||
logger.error("Failed to import semantica. Make sure you are in the project root or semantica is installed.")
|
||||
exit(1)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
def verify_backend(backend_name: str, config: Dict[str, Any]) -> bool:
|
||||
logger.info(f"\n{'='*20} Verifying {backend_name.upper()} {'='*20}")
|
||||
store = None
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user