diff --git a/docs/CodeExamples.md b/docs/CodeExamples.md index 006fb201..829e28a1 100644 --- a/docs/CodeExamples.md +++ b/docs/CodeExamples.md @@ -26,12 +26,19 @@ pip install -e ".[dev]" ### ⚡ 30-Second Demo: From Any Format to Knowledge ```python -from semantica.core import Semantica +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder +from semantica.embeddings import TextEmbedder -# Initialize with preferred providers -core = Semantica( - llm_provider="openai", - embedding_model="text-embedding-3-large", +# Use individual modules with preferred providers +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor(method="llm", provider="openai") +rel_extractor = RelationExtractor() +builder = GraphBuilder() +embedder = TextEmbedder(method="openai", model="text-embedding-3-large") vector_store="weaviate", graph_db="neo4j" ) diff --git a/docs/LIBS_README.md b/docs/LIBS_README.md index df47e6a4..dadf4607 100644 --- a/docs/LIBS_README.md +++ b/docs/LIBS_README.md @@ -82,40 +82,82 @@ pip install -e ".[dev]" #### API Usage Patterns -**Pattern 1: Using Semantica class (Recommended)** +**Pattern 1: Using Individual Modules (Recommended)** +```python +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder +from semantica.embeddings import TextEmbedder + +# Use individual modules for full control +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() +rel_extractor = RelationExtractor() +builder = GraphBuilder(merge_entities=True) +embedder = TextEmbedder() + +# Build knowledge base step by step +docs = ingestor.ingest_file("doc1.pdf") +parsed = parser.parse_document("doc1.pdf") +text = parsed.get("full_text", "") +entities = ner.extract_entities(text) +relationships = rel_extractor.extract_relations(text, entities=entities) +kg = builder.build_graph(entities=entities, relationships=relationships) +embeddings = embedder.embed_batch([e.text for e in entities]) +``` + +**Pattern 2: Using Semantica Class (Orchestration)** ```python from semantica.core import Semantica -# Initialize and build knowledge base -semantica = Semantica() -result = semantica.build_knowledge_base(["doc1.pdf", "doc2.docx"], embeddings=True, graph=True) +# Use Semantica class for orchestration of complex workflows +# For orchestration, use Semantica class +from semantica.core import Semantica +framework = Semantica() +framework.initialize() +framework.initialize() +result = framework.build_knowledge_base(["doc1.pdf", "doc2.docx"], embeddings=True, graph=True) +framework.shutdown() ``` -**Pattern 2: Direct class usage (Fine-grained control)** -```python -from semantica.kg import GraphBuilder -from semantica.embeddings import EmbeddingGenerator -from semantica.ingest import FileIngestor - -# Use classes directly -builder = GraphBuilder(merge_entities=True) -generator = EmbeddingGenerator() -ingestor = FileIngestor() -``` +!!! tip "Which Pattern to Use?" + - **Use Individual Modules** (Pattern 1) for most use cases - gives you full control and transparency + - **Use Semantica Class** (Pattern 2) for complex workflows that need lifecycle management and orchestration ### 1. Basic Document Processing ```python -from semantica.core import Semantica +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder +from semantica.embeddings import TextEmbedder -# Build knowledge base from documents (auto-initializes) -semantica = Semantica() +# Use individual modules documents = ["document1.pdf", "document2.docx", "document3.txt"] -result = semantica.build_knowledge_base( - documents, - embeddings=True, - graph=True, - normalize=True -) +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() +rel_extractor = RelationExtractor() +builder = GraphBuilder(merge_entities=True) +embedder = TextEmbedder() + +# Process each document +all_entities = [] +all_relationships = [] +for doc_path in documents: + doc = ingestor.ingest_file(doc_path) + parsed = parser.parse_document(doc_path) + text = parsed.get("full_text", "") + entities = ner.extract_entities(text) + relationships = rel_extractor.extract_relations(text, entities=entities) + all_entities.extend(entities) + all_relationships.extend(relationships) + +# Build knowledge graph and generate embeddings +kg = builder.build_graph(entities=all_entities, relationships=all_relationships) +embeddings = embedder.embed_batch([e.text for e in all_entities]) # Access results knowledge_graph = result["knowledge_graph"] @@ -159,21 +201,56 @@ sitemap_url = "https://example.com/sitemap.xml" pages = web_ingestor.crawl_sitemap(sitemap_url) # Build knowledge base from web content -sources = [web_content.url for web_content in pages] -semantica_instance = Semantica() -result = semantica_instance.build_knowledge_base(sources) +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder + +parser = DocumentParser() +ner = NERExtractor() +rel_extractor = RelationExtractor() +builder = GraphBuilder() + +all_entities = [] +all_relationships = [] +for web_content in pages: + parsed = parser.parse_document(web_content.url) + text = parsed.get("full_text", "") + entities = ner.extract_entities(text) + relationships = rel_extractor.extract_relations(text, entities=entities) + all_entities.extend(entities) + all_relationships.extend(relationships) + +kg = builder.build_graph(entities=all_entities, relationships=all_relationships) ``` ### 3. Knowledge Graph Analytics ```python -from semantica.core import Semantica +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor from semantica.kg import GraphBuilder, GraphAnalyzer, CentralityCalculator, CommunityDetector -# Build knowledge graph using Semantica class +# Build knowledge graph using individual modules sources = ["document1.pdf", "document2.pdf"] -semantica = Semantica() -result = semantica.build_knowledge_base(sources, graph=True) -kg_data = result["knowledge_graph"] +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() +rel_extractor = RelationExtractor() +builder = GraphBuilder(merge_entities=True, entity_resolution_strategy="fuzzy") + +# Process documents +all_entities = [] +all_relationships = [] +for source in sources: + doc = ingestor.ingest_file(source) + parsed = parser.parse_document(source) + text = parsed.get("full_text", "") + entities = ner.extract_entities(text) + relationships = rel_extractor.extract_relations(text, entities=entities) + all_entities.extend(entities) + all_relationships.extend(relationships) + +kg = builder.build_graph(entities=all_entities, relationships=all_relationships) # Build graph object from extracted entities and relationships graph_builder = GraphBuilder( @@ -1197,13 +1274,29 @@ temporal_viz.visualize_metrics_evolution(metrics_history, timestamps, #### Quick Visualization Example ```python -from semantica.core import Semantica +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder +from semantica.embeddings import TextEmbedder from semantica.visualization import KGVisualizer, EmbeddingVisualizer import numpy as np -# Build knowledge graph -semantica = Semantica() -result = semantica.build_knowledge_base(["document.pdf"], graph=True, embeddings=True) +# Build knowledge graph using individual modules +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() +rel_extractor = RelationExtractor() +builder = GraphBuilder() +embedder = TextEmbedder() + +doc = ingestor.ingest_file("document.pdf") +parsed = parser.parse_document("document.pdf") +text = parsed.get("full_text", "") +entities = ner.extract_entities(text) +relationships = rel_extractor.extract_relations(text, entities=entities) +kg = builder.build_graph(entities=entities, relationships=relationships) +embeddings = embedder.embed_batch([e.text for e in entities]) # Visualize knowledge graph kg_viz = KGVisualizer(layout="force", color_scheme="vibrant") @@ -1229,39 +1322,39 @@ if "embeddings" in result: ### Basic Configuration ```python -from semantica.core import Semantica, Config +from semantica.semantic_extract import NERExtractor +from semantica.kg import GraphBuilder -# Create configuration -config = Config({ - "processing": { - "batch_size": 100, - "max_workers": 4 - }, - "quality": { - "min_confidence": 0.7, - "validation_enabled": True - }, - "security": { - "encryption_enabled": True, - "access_control_enabled": True - } -}) +# Configure modules individually +ner = NERExtractor( + method="llm", + provider="openai", + model="gpt-4", + confidence_threshold=0.7 +) -# Use configuration with Semantica -semantica_instance = Semantica(config=config) -result = semantica_instance.build_knowledge_base(["document.pdf"]) +builder = GraphBuilder( + merge_entities=True, + merge_threshold=0.9 +) ``` ### Advanced Configuration ```python -from semantica.core import Semantica, Config +from semantica.core import Config, ConfigManager +from semantica.semantic_extract import NERExtractor +from semantica.kg import GraphBuilder -# Advanced configuration -config = Config({ - "llm_provider": { - "name": "openai", - "api_key": "your-api-key", - "model": "gpt-4" +# Load configuration from file +config_manager = ConfigManager() +config = config_manager.load_from_file("config.yaml") + +# Use configuration with modules +ner = NERExtractor( + method="llm", + provider=config.get("llm_provider.name"), + model=config.get("llm_provider.model"), + api_key=config.get("llm_provider.api_key") }, "embedding_model": { "name": "sentence-transformers", diff --git a/docs/concepts.md b/docs/concepts.md index f72f4e94..e95c31cb 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -59,87 +59,37 @@ graph LR ``` -**Practical Examples**: +**Learn by Doing:** -=== "Basic Usage" - Build a knowledge graph from a document with just a few lines: - - ```python - from semantica.core import Semantica +Knowledge graphs are best understood through hands-on practice. The following cookbooks provide step-by-step tutorials: - # Initialize Semantica with default settings - semantica = Semantica() +- **[Your First Knowledge Graph](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Build a knowledge graph from a document + - **Topics**: Entity extraction, relationship extraction, graph construction, visualization + - **Difficulty**: Beginner + - **Time**: 20-30 minutes + - **Use Cases**: Learning the basics, understanding graph structure - # Build knowledge graph from a PDF document - result = semantica.build_knowledge_base( - sources=["company_report.pdf"], - embeddings=True, # Generate vector embeddings - graph=True # Build knowledge graph - ) +- **[Building Knowledge Graphs](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb)**: Advanced graph construction techniques + - **Topics**: Graph building, entity merging, conflict resolution, temporal graphs + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Production graph construction, multi-source integration - kg = result["knowledge_graph"] +- **[Multi-Source Data Integration](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb)**: Merge knowledge from multiple sources + - **Topics**: Multi-source integration, entity resolution, conflict handling + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Building unified knowledge graphs from diverse sources + parsed = parser.parse_document(doc) if isinstance(doc, str) else doc + text = parsed.get("full_text", "") if isinstance(parsed, dict) else str(parsed) + entities = ner.extract_entities(text) + relationships = rel_extractor.extract_relations(text, entities=entities) + all_entities.extend(entities) + all_relationships.extend(relationships) + return builder.build_graph(entities=all_entities, relationships=all_relationships) - print(f"Extracted {len(kg['entities'])} entities") - print(f"Found {len(kg['relationships'])} relationships") - - for entity in kg['entities'][:5]: - print(f" {entity['text']} ({entity['type']})") - ``` -=== "Direct KG Module Usage" - Use the `kg` module directly for more control: - - ```python - from semantica.kg import GraphBuilder - - # Option 1: Using the convenience function (semantica.kg.build) - kg = build( - sources=[ - { - "entities": [ - {"id": "e1", "text": "Apple Inc.", "type": "Organization"}, - {"id": "e2", "text": "Tim Cook", "type": "Person"}, - {"id": "e3", "text": "Cupertino", "type": "Location"} - ], - "relationships": [ - {"source": "e2", "target": "e1", "type": "CEO_OF"}, - {"source": "e1", "target": "e3", "type": "LOCATED_IN"} - ] - } - ], - merge_entities=True, - resolve_conflicts=True - ) - - print(f"Graph built: {kg['metadata']['num_entities']} entities, " - f"{kg['metadata']['num_relationships']} relationships") - - # Option 2: Using GraphBuilder for advanced configuration - builder = GraphBuilder( - merge_entities=True, - entity_resolution_strategy="fuzzy", # "exact", "fuzzy", or "semantic" - resolve_conflicts=True, - enable_temporal=True, # Enable time-aware edges - temporal_granularity="day" - ) - kg = builder.build(sources) - ``` -=== "Multi-Source Integration" - Merge knowledge from multiple data sources: - - ```python - from semantica.core import Semantica - - semantica = Semantica() - - # Build graphs from different sources - kg_news = semantica.build_knowledge_base( - sources=["news_articles/"], - normalize=True - )["knowledge_graph"] - - kg_reports = semantica.build_knowledge_base( - sources=["financial_reports/"], - normalize=True + kg_news = build_kg_from_source("news_articles/") + kg_reports = build_kg_from_source("financial_reports/") )["knowledge_graph"] # Merge into a unified knowledge graph @@ -231,82 +181,27 @@ graph LR !!! tip "Custom Entities" Semantica allows you to define custom entity types via the [`Ontology`](reference/ontology.md) module. You aren't limited to the standard set! -**Extraction Methods**: +**Extraction Methods:** +Semantica supports multiple extraction methods: +- **Machine Learning Models**: spaCy, transformers (BERT, RoBERTa) +- **Rule-Based**: Pattern matching for specific formats +- **LLM-Based**: Zero-shot extraction using large language models +- **Hybrid**: Combine multiple methods for better accuracy -=== "Quick Start" - Extract entities from text using the convenience function: +**Learn by Doing:** - - ```python - from semantica.semantic_extract import NamedEntityRecognizer - +- **[Entity Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)**: Learn different NER methods and configurations + - **Topics**: Named entity recognition, entity types, confidence scores, extraction methods + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Understanding entity extraction options, choosing the right method - text = """ - Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne - in Cupertino, California on April 1, 1976. The company is now worth - over $3 trillion and employs more than 160,000 people worldwide. - """ - - # Initialize the NER extractor - ner = NamedEntityRecognizer() - - # Extract entities - entities = ner.extract_entities(text) - - print(f"Extracted {len(entities)} entities:") - for entity in entities: - print(f" {entity['text']:20} | Type: {entity['type']:15} | " - f"Confidence: {entity.get('confidence', 0.0):.2f}") - ``` -=== "Named Entity Recognizer" - Use the `NamedEntityRecognizer` class for advanced control: - - ```python - from semantica.semantic_extract import NamedEntityRecognizer - - # Initialize with custom configuration - ner = NamedEntityRecognizer( - methods=["spacy", "rule-based"], # Use multiple methods - confidence_threshold=0.7, - merge_overlapping=True - ) - - text = "Elon Musk announced that Tesla will invest $10B in Texas." - - # Extract entities with detailed output - entities = ner.extract_entities(text) - - for entity in entities: - print(f""" - Entity: {entity['text']} - Type: {entity['type']} - Start: {entity['start_char']}, End: {entity['end_char']} - Confidence: {entity['confidence']:.2f} - Method: {entity.get('extraction_method', 'N/A')} - """) - ``` -=== "Custom Entity Types" - Define and extract custom entity types for your domain: - - ```python - from semantica.semantic_extract import ( - NamedEntityRecognizer, - CustomEntityDetector - ) - - # Define custom entity patterns for a medical domain - custom_detector = CustomEntityDetector( - patterns={ - "Drug": [ - r"\b(aspirin|ibuprofen|acetaminophen)\b", - r"\b\w+(?:mycin|cillin|phen)\b" # Common drug suffixes - ], - "Dosage": [ - r"\d+\s*(?:mg|ml|g|mcg)\b", - r"\b(?:once|twice|three times)\s+(?:daily|weekly)\b" - ], - "Condition": [ +- **[Advanced Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb)**: Advanced extraction patterns and custom entity types + - **Topics**: Custom entity types, domain-specific extraction, hybrid methods + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Domain-specific extraction, custom entity definitions r"\b(?:diabetes|hypertension|arthritis)\b", r"\b\w+itis\b" # Inflammation conditions ] @@ -339,80 +234,7 @@ graph LR # Dosage: 400mg, twice daily # Condition: arthritis ``` -=== "LLM-Enhanced Extraction" - Use LLMs for context-aware, high-accuracy extraction: - - ```python - from semantica.semantic_extract import ( - NamedEntityRecognizer, - LLMExtraction, - create_provider - ) - - # Create LLM provider (supports OpenAI, Anthropic, Groq, Ollama) - provider = create_provider( - provider_type="openai", - model="gpt-4o", - api_key="your-api-key" # Or use OPENAI_API_KEY env var - ) - - # Initialize LLM-enhanced NER - enhancer = LLMEnhancer(provider=provider) - ner = NamedEntityRecognizer( - llm_enhancer=enhancer, - use_llm_verification=True # Verify with LLM - ) - - # Extract with LLM enhancement for ambiguous cases - text = """ - Apple reported record revenue. Tim said the new Apple Watch - and the partnership with Goldman Sachs exceeded expectations. - """ - - entities = ner.extract_entities( - text, - context="Financial earnings report" # Provide context for better accuracy - ) - - # LLM helps distinguish: - # - "Apple" (company) vs "apple" (fruit) - # - "Tim" (person - Tim Cook) from context - # - "Goldman Sachs" (organization) - # - "Apple Watch" (product vs organization) - ``` -=== "Batch Processing" - Process multiple documents efficiently: - - ```python - from semantica.semantic_extract import NamedEntityRecognizer - - ner = NamedEntityRecognizer(batch_size=32) - - documents = [ - "Microsoft acquired Activision for $69 billion.", - "Google announced Gemini AI at their Mountain View headquarters.", - "Amazon's Andy Jassy unveiled new AWS services in Seattle.", - # ... hundreds more documents - ] - - # Process in batches for efficiency - all_entities = [] - for i, doc in enumerate(documents): - entities = ner.extract_entities(doc) - all_entities.extend([ - {**e, "doc_id": i} for e in entities - ]) - - if (i + 1) % 100 == 0: - print(f"Processed {i + 1}/{len(documents)} documents") - - from collections import Counter - entity_types = Counter(e['type'] for e in all_entities) - print(f"Entity Distribution:") - for etype, count in entity_types.most_common(): - print(f" {etype}: {count}") - - ``` +**For advanced extraction patterns including LLM-enhanced extraction and batch processing, see:** **Related Modules**: - [`semantic_extract` Module](reference/semantic_extract.md) - Entity and relationship extraction @@ -486,177 +308,19 @@ graph LR **Practical Examples**: -=== "Basic Relation Extraction" - Extract relationships between entities: - - ```python - from semantica.semantic_extract import RelationExtractor, NamedEntityRecognizer - - text = """ - Tim Cook became CEO of Apple Inc. in 2011, succeeding Steve Jobs. - Apple is headquartered in Cupertino, California. The company - acquired Beats Electronics in 2014 for $3 billion. - """ - - # First extract entities - ner = NamedEntityRecognizer() - entities = ner.extract_entities(text) - - # Then extract relationships - rel_extractor = RelationExtractor() - relations = rel_extractor.extract_relations(text, entities=entities) - - print("Extracted Relationships:") - for rel in relations: - print(f" ({rel['source']}) --[{rel['type']}]--> ({rel['target']})") - print(f" Confidence: {rel.get('confidence', 0.0):.2f}") - ``` -=== "RelationExtractor Class" - Use the `RelationExtractor` for advanced control: - - ```python - from semantica.semantic_extract import ( - RelationExtractor, - NamedEntityRecognizer - ) - - # First extract entities - ner = NamedEntityRecognizer() - text = """ - Dr. Sarah Chen published her research on quantum computing at MIT. - Her work was funded by DARPA and received the ACM Award in 2023. - """ - entities = ner.extract_entities(text) - - # Then extract relationships - rel_extractor = RelationExtractor( - relation_types=["works_at", "funded_by", "received", "published"], - bidirectional=False, - confidence_threshold=0.6 - ) - - relations = rel_extractor.extract_relations(text, entities=entities) - - print("Knowledge Graph Edges:") - for rel in relations: - arrow = "<->" if rel.get('bidirectional') else "->" - print(f" {rel['source_text']} {arrow} {rel['target_text']}") - print(f" Relation: {rel['type']}") - ``` -=== "Triplet Extraction (RDF)" - Extract subject-predicate-object triplets for RDF/semantic web: - - ```python - from semantica.semantic_extract import ( - TripletExtractor, - RDFSerializer, - TripletValidator - ) - - text = """ - Albert Einstein was born in Ulm, Germany in 1879. He developed - the theory of relativity and won the Nobel Prize in Physics in 1921. - Einstein worked at Princeton University until his death in 1955. - """ - - # Extract RDF-style triplets - extractor = TripletExtractor( - include_temporal=True, # Include time information - include_provenance=True # Track source sentences - ) - - triplets = extractor.extract_triplets(text) - - print("Extracted Triplets (Subject-Predicate-Object):") - for triplet in triplets: - print(f" Subject: {triplet['subject']}") - print(f" Predicate: {triplet['predicate']}") - print(f" Object: {triplet['object']}") - if triplet.get('temporal'): - print(f" When: {triplet['temporal']}") - print() - - validator = TripletValidator() - validation = validator.validate(triplets) - print(f"Valid triplets: {validation['valid_count']}/{len(triplets)}") - - serializer = RDFSerializer(format="turtle") - turtle_output = serializer.serialize( - triplets, - base_uri="https://example.org/knowledge/" - ) - print("Turtle Output:") - print(turtle_output) - ``` -=== "Event Detection" - Extract events with temporal information: - - ```python - from semantica.semantic_extract import ( - EventDetector, - TemporalEventProcessor - ) - - news_text = """ - On March 15, 2024, SpaceX successfully launched Starship from - Boca Chica, Texas. The rocket reached orbit before splashing down - in the Indian Ocean. CEO Elon Musk announced plans for a Mars - mission by 2026. - """ - - # Detect events - detector = EventDetector( - event_types=["launch", "announcement", "achievement"], - extract_participants=True, - extract_location=True, - extract_time=True - ) - - events = detector.detect_events(news_text) - - print("Detected Events:") - for event in events: - print(f" Event: {event['description']}") - print(f" Type: {event['type']}") - print(f" When: {event.get('datetime', 'Unknown')}") - print(f" Where: {event.get('location', 'Unknown')}") - print(f" Who: {', '.join(event.get('participants', []))}") - print() - - # Process temporal relationships between events - temporal = TemporalEventProcessor() - timeline = temporal.build_timeline(events) - - print("Event Timeline:") - for i, evt in enumerate(timeline, 1): - print(f" {i}. {evt['datetime']}: {evt['description']}") - ``` -=== "Coreference Resolution" - Resolve pronouns and entity references: - - ```python - from semantica.semantic_extract import CoreferenceResolver - - text = """ - Apple Inc. announced their new iPhone. The company said it would - be available in September. Tim Cook presented the device at their - headquarters. He emphasized its improved camera capabilities. - """ - - resolver = CoreferenceResolver() - result = resolver.resolve(text) - - print("Coreference Chains:") - for chain in result['chains']: - print(f" Entity: {chain['canonical']}") - print(f" Mentions: {', '.join(chain['mentions'])}") - print() - - resolved_text = resolver.get_resolved_text(text) - print("Resolved Text:") - print(resolved_text) - ``` +**Learn by Doing:** +- **[Relation Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/06_Relation_Extraction.ipynb)**: Learn to extract relationships between entities + - **Topics**: Relationship extraction, dependency parsing, semantic role labeling, triplet extraction + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Building rich knowledge graphs with relationships + +- **[Advanced Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb)**: Advanced extraction patterns including event detection and coreference resolution + - **Topics**: Event detection, coreference resolution, temporal relationships, RDF triplets + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Complex extraction scenarios, temporal analysis **Related Modules**: - [`semantic_extract` Module](reference/semantic_extract.md) - Relationship extraction @@ -676,16 +340,9 @@ graph LR Embeddings convert text into numerical vectors that capture semantic meaning. Similar texts have similar vectors, enabling semantic search and similarity calculations. -**Example**: +**How It Works:** -```python -Text: "machine learning" -Embedding: [0.123, -0.456, 0.789, ..., 0.234] # (vector of 1536 dimensions) - -# Similar texts will have similar vectors -"artificial intelligence" → [0.145, -0.432, 0.801, ..., 0.221] # Close in vector space -"cooking recipes" → [-0.234, 0.567, -0.123, ..., -0.456] # Far in vector space -``` +Embeddings convert text into numerical vectors that capture semantic meaning. Similar texts have similar vectors, enabling semantic search and similarity calculations. The vectors are typically 384-3072 dimensions depending on the model used. **Embedding Providers**: @@ -916,6 +573,14 @@ Embedding: [0.123, -0.456, 0.789, ..., 0.234] # (vector of 1536 dimensions) ``` +**Learn by Doing:** + +- **[Embeddings Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/09_Embeddings.ipynb)**: Learn to generate and use embeddings + - **Topics**: Embedding generation, similarity search, vector operations, pooling strategies + - **Difficulty**: Beginner + - **Time**: 20-30 minutes + - **Use Cases**: Understanding embeddings, semantic search setup + **Related Modules**: - [`embeddings` Module](reference/embeddings.md) - Embedding generation - [`vector_store` Module](reference/vector_store.md) - Vector storage and search @@ -1216,56 +881,19 @@ flowchart TD **Practical Examples**: -=== "Basic GraphRAG" - Build a knowledge base and query with GraphRAG: - - ```python - from semantica.core import Semantica - from semantica.vector_store import VectorStore, store_vectors, search_vectors - from semantica.semantic_extract import NamedEntityRecognizer - from semantica.embeddings import embed_text - - # Initialize Semantica - semantica = Semantica() - - # Build knowledge base from documents - result = semantica.build_knowledge_base( - sources=["company_docs/", "research_papers/"], - embeddings=True, - graph=True - ) - - kg = result["knowledge_graph"] - embeddings_data = result["embeddings"] - - # Initialize vector store - vector_store = VectorStore(backend="faiss", dimension=768) - - # Store embeddings - vector_ids = vector_store.store_vectors( - vectors=embeddings_data["vectors"], - metadata=embeddings_data["metadata"] - ) - - # Initialize NER for query processing - ner = NamedEntityRecognizer() - - # Process a query - query = "Who is the CEO of Apple?" - - # Get query embedding and search - query_embedding = embed_text(query, method="sentence_transformers") - vector_results = vector_store.search(query_embedding, k=5) - - # Extract entities from query - query_entities = ner.extract_entities(query) - - # Get graph context for query entities - graph_context = [] - for entity in query_entities: - for rel in kg["relationships"]: - if entity["text"].lower() in rel.get("source_text", "").lower(): - graph_context.append(rel) +**Learn by Doing:** + +- **[GraphRAG Complete Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)**: Build a production-ready GraphRAG system + - **Topics**: GraphRAG, hybrid retrieval, graph traversal, LLM integration + - **Difficulty**: Advanced + - **Time**: 1-2 hours + - **Use Cases**: Production GraphRAG systems, enhanced RAG applications + +- **[RAG vs. GraphRAG Comparison](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/02_RAG_vs_GraphRAG_Comparison.ipynb)**: Side-by-side comparison + - **Topics**: RAG comparison, reasoning gap, inference engines + - **Difficulty**: Intermediate + - **Time**: 45-60 minutes + - **Use Cases**: Understanding GraphRAG advantages, choosing the right approach print(f"Vector results: {len(vector_results)}") print(f"Graph context: {len(graph_context)} relevant relationships") @@ -1423,13 +1051,14 @@ flowchart TD Integrate with LLM providers for answer generation: ```python - from semantica.core import Semantica - from semantica.semantic_extract import create_provider, OpenAIProvider + from semantica.semantic_extract import NERExtractor from semantica.context import ContextRetriever from semantica.vector_store import VectorStore + from semantica.llms import LLMProvider # Initialize components vector_store = VectorStore(backend="faiss", dimension=768) + llm_provider = LLMProvider(provider="openai", model="gpt-4o") # Create LLM provider provider = create_provider( @@ -2684,14 +2313,28 @@ semantica = Semantica(config=config) - Gracefully handle API failures ```python -from semantica.core import Semantica +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder import logging logging.basicConfig(level=logging.INFO) -semantica = Semantica() + +# Use individual modules +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() +rel_extractor = RelationExtractor() +builder = GraphBuilder() try: - result = semantica.build_knowledge_base(["doc.pdf"]) + doc = ingestor.ingest_file("doc.pdf") + parsed = parser.parse_document("doc.pdf") + text = parsed.get("full_text", "") + entities = ner.extract_entities(text) + relationships = rel_extractor.extract_relations(text, entities=entities) + kg = builder.build_graph(entities=entities, relationships=relationships) except Exception as e: logging.error(f"Error building KG: {e}") # Handle error appropriately @@ -2741,4 +2384,3 @@ Now that you understand the core concepts: !!! info "Contribute" Found an issue or want to improve this guide? [Contribute on GitHub](https://github.com/Hawksight-AI/semantica) -**Last Updated**: 2024 diff --git a/docs/examples.md b/docs/examples.md index e5b5647b..5e52862a 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -45,60 +45,63 @@ Real-world examples and use cases for Semantica. **Difficulty**: Beginner -Build a knowledge graph from a single document. +Build a knowledge graph from a single document using Semantica's modular approach. This example demonstrates the complete workflow from document ingestion to graph construction. -```python -from semantica.core import Semantica +**What it demonstrates:** +- Document ingestion and parsing +- Entity and relationship extraction +- Knowledge graph construction -semantica = Semantica() - -# Build KG from PDF -result = semantica.build_knowledge_base( - sources=["research_paper.pdf"], - embeddings=True, - graph=True -) - -kg = result["knowledge_graph"] -print(f"Entities: {len(kg['entities'])}") -print(f"Relationships: {len(kg['relationships'])}") -``` +**For complete step-by-step examples, see:** +- **[Your First Knowledge Graph Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Complete walkthrough + - **Topics**: Ingestion, parsing, extraction, graph building + - **Difficulty**: Beginner + - **Time**: 20-30 minutes + - **Use Cases**: Learning the complete workflow ### Example 2: Entity Extraction **Difficulty**: Beginner -Extract entities from text using Named Entity Recognition. +Extract entities from text using Named Entity Recognition. This example shows how to identify and classify named entities in text. -```python -from semantica.core import Semantica +**What it demonstrates:** +- Named Entity Recognition (NER) +- Entity type classification +- Confidence scoring -semantica = Semantica() -text = "Apple Inc. is a technology company founded by Steve Jobs." +**For complete examples, see:** +- **[Entity Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)**: Learn entity extraction + - **Topics**: NER methods, entity types, extraction techniques + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Understanding entity extraction + print(f"{entity.text}: {entity.label}") +``` -entities = semantica.semantic_extract.extract_entities(text) -for entity in entities["entities"]: - print(f"{entity['text']}: {entity['type']}") +**Expected Output:** +``` +Apple Inc.: ORGANIZATION +Steve Jobs: PERSON ``` ### Example 3: Multi-Source Integration **Difficulty**: Beginner -Combine data from multiple sources into a unified knowledge graph. +Combine data from multiple sources into a unified knowledge graph. This example demonstrates integrating data from diverse sources. -```python -from semantica.core import Semantica +**What it demonstrates:** +- Multi-source data ingestion +- Entity merging and resolution +- Unified graph construction -semantica = Semantica() -sources = [ - "documents/finance_report.pdf", - "https://example.com/news-article" -] - -result = semantica.build_knowledge_base(sources) -print(f"Unified graph: {len(result['knowledge_graph']['entities'])} entities") -``` +**For complete examples, see:** +- **[Multi-Source Data Integration Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb)**: Advanced integration patterns + - **Topics**: Multi-source integration, entity resolution, conflict handling + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Building unified knowledge graphs from diverse sources --- @@ -108,60 +111,110 @@ print(f"Unified graph: {len(result['knowledge_graph']['entities'])} entities") **Difficulty**: Intermediate -Resolve conflicts in data from multiple sources. +Resolve conflicts in data from multiple sources. This example shows how to identify and resolve conflicting information. -```python -from semantica.core import Semantica -from semantica.conflicts import ConflictDetector, ConflictResolver +**What it demonstrates:** +- Conflict detection +- Conflict resolution strategies +- Data quality assurance -semantica = Semantica() -result = semantica.build_knowledge_base(["source1.pdf", "source2.pdf"]) +**For complete examples, see:** +- **[Multi-Source Data Integration Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb)**: Conflict resolution patterns + - **Topics**: Conflict detection, resolution strategies, data quality + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Data integration, quality assurance +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() + +all_entities = [] +for source in ["source1.pdf", "source2.pdf"]: + doc = ingestor.ingest_file(source) + parsed = parser.parse_document(source) + text = parsed.get("full_text", "") + entities = ner.extract_entities(text) + all_entities.extend(entities) # Detect and resolve conflicts -kg = result["knowledge_graph"] detector = ConflictDetector() -conflicts = detector.detect_conflicts(kg["entities"]) +conflicts = detector.detect_conflicts(all_entities) + resolver = ConflictResolver(default_strategy="voting") resolved = resolver.resolve_conflicts(conflicts) + +print(f"Detected {len(conflicts)} conflicts") +print(f"Resolved {len(resolved)} conflicts") ``` -### Example 5: Custom Configuration +### Example 5: Custom Entity Extraction Configuration **Difficulty**: Intermediate -Use custom configuration for specific use cases. +Use custom configuration for entity extraction with specific models and thresholds. ```python -from semantica.core import Semantica, Config +from semantica.semantic_extract import NERExtractor +from semantica.kg import GraphBuilder -config = Config( - embeddings=True, - graph=True, - normalize=True, - conflict_resolution="highest_confidence" +# Use LLM-based extraction with custom configuration +ner = NERExtractor( + method="llm", + provider="openai", + model="gpt-4", + confidence_threshold=0.8, + temperature=0.0 ) -semantica = Semantica(config=config) -result = semantica.build_knowledge_base(["document.pdf"]) +text = "Your document text here..." +entities = ner.extract_entities(text) + +# Build graph with custom merge settings +builder = GraphBuilder( + merge_entities=True, + merge_threshold=0.9 +) +kg = builder.build_graph(entities=entities, relationships=[]) ``` ### Example 6: Incremental Graph Building **Difficulty**: Intermediate -Build knowledge graph incrementally. +Build knowledge graph incrementally from multiple sources. ```python -from semantica.core import Semantica +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder, GraphMerger -semantica = Semantica() +def build_kg_from_source(source_path): + """Helper function to build a knowledge graph from a single source.""" + ingestor = FileIngestor() + parser = DocumentParser() + ner = NERExtractor() + rel_extractor = RelationExtractor() + + doc = ingestor.ingest_file(source_path) + parsed = parser.parse_document(source_path) + text = parsed.get("full_text", "") + + entities = ner.extract_entities(text) + relationships = rel_extractor.extract_relations(text, entities=entities) + + builder = GraphBuilder() + return builder.build_graph(entities=entities, relationships=relationships) # Build graphs separately -kg1 = semantica.kg.build_graph(["source1.pdf"]) -kg2 = semantica.kg.build_graph(["source2.pdf"]) +kg1 = build_kg_from_source("source1.pdf") +kg2 = build_kg_from_source("source2.pdf") # Merge into unified graph -merged_kg = semantica.kg.merge([kg1, kg2]) +merger = GraphMerger() +merged_kg = merger.merge([kg1, kg2]) + +print(f"Merged graph: {len(merged_kg.nodes)} nodes, {len(merged_kg.edges)} edges") ``` --- @@ -175,15 +228,32 @@ merged_kg = semantica.kg.merge([kg1, kg2]) Visualize your knowledge graph to understand entity relationships. ```python -import semantica -from semantica.visualization import GraphVisualizer +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder +from semantica.visualization import KGVisualizer # Build a small graph -kg = semantica.kg.build_graph(["semantica_intro.pdf"]) +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() +rel_extractor = RelationExtractor() + +doc = ingestor.ingest_file("semantica_intro.pdf") +parsed = parser.parse_document("semantica_intro.pdf") +text = parsed.get("full_text", "") + +entities = ner.extract_entities(text) +relationships = rel_extractor.extract_relations(text, entities=entities) + +builder = GraphBuilder() +kg = builder.build_graph(entities=entities, relationships=relationships) # Visualize -viz = GraphVisualizer() -viz.plot(kg, title="Semantica Knowledge Map") +viz = KGVisualizer() +viz.visualize_network(kg, output="html", file_path="semantica_knowledge_map.html") +print("Visualization saved to semantica_knowledge_map.html") ``` --- @@ -306,18 +376,31 @@ Process data streams in real-time. ```python from semantica.ingest import StreamIngestor -from semantica.core import Semantica +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder -semantica = Semantica() stream_ingestor = StreamIngestor(stream_uri="kafka://localhost:9092/topic") +parser = DocumentParser() +ner = NERExtractor() +rel_extractor = RelationExtractor() +builder = GraphBuilder() for batch in stream_ingestor.stream(batch_size=100): - result = semantica.build_knowledge_base( - sources=batch, - embeddings=True, - graph=True - ) + all_entities = [] + all_relationships = [] + + for item in batch: + text = str(item) # Convert stream item to text + entities = ner.extract_entities(text) + relationships = rel_extractor.extract_relations(text, entities=entities) + all_entities.extend(entities) + all_relationships.extend(relationships) + + # Build graph from batch + kg = builder.build_graph(entities=all_entities, relationships=all_relationships) # Process results + print(f"Processed batch: {len(kg.nodes)} nodes") ``` ### Example 13: Batch Processing Large Datasets @@ -327,16 +410,42 @@ for batch in stream_ingestor.stream(batch_size=100): Process large datasets efficiently with batching. ```python -from semantica.core import Semantica +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder + +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() +rel_extractor = RelationExtractor() +builder = GraphBuilder() -semantica = Semantica() sources = [f"data/doc_{i}.pdf" for i in range(1000)] batch_size = 50 for i in range(0, len(sources), batch_size): batch = sources[i:i+batch_size] - result = semantica.build_knowledge_base(batch) + + all_entities = [] + all_relationships = [] + + for source in batch: + doc = ingestor.ingest_file(source) + parsed = parser.parse_document(source) + text = parsed.get("full_text", "") + + entities = ner.extract_entities(text) + relationships = rel_extractor.extract_relations(text, entities=entities) + + all_entities.extend(entities) + all_relationships.extend(relationships) + + # Build graph from batch + kg = builder.build_graph(entities=all_entities, relationships=all_relationships) + # Save intermediate results + print(f"Processed batch {i//batch_size + 1}: {len(kg.nodes)} nodes") ``` --- @@ -348,9 +457,34 @@ for i in range(0, len(sources), batch_size): - **[Cookbook](cookbook.md)** - Interactive Jupyter notebooks - **[Use Cases](use-cases.md)** - Real-world applications +### 🍳 Recommended Cookbook Tutorials + +- **[Welcome to Semantica](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Comprehensive introduction to all modules + - **Topics**: Framework overview, all modules, architecture, configuration + - **Difficulty**: Beginner + - **Time**: 30-45 minutes + - **Use Cases**: First-time users, understanding the framework + +- **[Your First Knowledge Graph](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Build your first knowledge graph + - **Topics**: Entity extraction, relationship extraction, graph construction, visualization + - **Difficulty**: Beginner + - **Time**: 20-30 minutes + - **Use Cases**: Learning the basics, quick start + +- **[GraphRAG Complete](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)**: Production-ready GraphRAG system + - **Topics**: GraphRAG, hybrid retrieval, vector search, graph traversal, LLM integration + - **Difficulty**: Advanced + - **Time**: 1-2 hours + - **Use Cases**: Production RAG applications + +- **[RAG vs. GraphRAG Comparison](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/02_RAG_vs_GraphRAG_Comparison.ipynb)**: Benchmark standard RAG vs GraphRAG + - **Topics**: RAG, GraphRAG, benchmarking, visualization, reasoning gap + - **Difficulty**: Intermediate + - **Time**: 45-60 minutes + - **Use Cases**: Understanding GraphRAG advantages, choosing the right approach + --- !!! info "Contribute" Have an example to share? [Contribute on GitHub](https://github.com/Hawksight-AI/semantica) -**Last Updated**: 2024 diff --git a/docs/faq.md b/docs/faq.md index 0168836c..9978ff78 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -70,11 +70,26 @@ A structured representation where entities (nodes) are connected by relationship ### How do I build a knowledge graph? ```python -from semantica.core import Semantica +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder -semantica = Semantica() -result = semantica.build_knowledge_base(["document.pdf"]) -kg = result["knowledge_graph"] +# Use individual modules +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() +rel_extractor = RelationExtractor() + +doc = ingestor.ingest_file("document.pdf") +parsed = parser.parse_document("document.pdf") +text = parsed.get("full_text", "") + +entities = ner.extract_entities(text) +relationships = rel_extractor.extract_relations(text, entities=entities) + +builder = GraphBuilder() +kg = builder.build_graph(entities=entities, relationships=relationships) ``` ### Can I merge multiple knowledge graphs? @@ -102,11 +117,11 @@ Yes! Semantica supports PDF, DOCX, HTML, JSON, CSV, and many other formats. ### How do I extract entities from text? ```python -from semantica.core import Semantica +from semantica.semantic_extract import NERExtractor -semantica = Semantica() -result = semantica.semantic_extract.extract_entities("Your text") -entities = result["entities"] +# Use NER extractor directly +ner = NERExtractor() +entities = ner.extract_entities("Your text") ``` ### Can I use my own models? diff --git a/docs/getting-started.md b/docs/getting-started.md index 29fa5d05..33c7922e 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -77,48 +77,138 @@ print(semantica.__version__) --- +## 🏗️ Understanding Semantica's Architecture + +Semantica uses a **modular architecture** where each module handles a specific aspect of semantic processing. This design gives you flexibility and control over your pipeline. + +### Primary Approach: Individual Modules + +The recommended approach is to use individual modules directly. Each module can be imported and used independently: + +- **`semantica.ingest`**: Data ingestion from files, web, databases +- **`semantica.parse`**: Document parsing and text extraction +- **`semantica.semantic_extract`**: Entity and relationship extraction +- **`semantica.kg`**: Knowledge graph construction +- **`semantica.embeddings`**: Vector embedding generation +- **`semantica.vector_store`**: Vector database operations + +**Benefits of the modular approach:** +- **Full control**: Customize each step of your pipeline +- **Flexibility**: Mix and match modules as needed +- **Transparency**: Clear understanding of what each step does +- **Easy debugging**: Isolate issues to specific modules + +**Quick Example:** +```python +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder + +# Each module is used independently +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() +builder = GraphBuilder() +``` + +**For detailed examples, see:** +- **[Welcome to Semantica Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Comprehensive introduction to all modules and architecture + - **Topics**: Framework overview, all modules, architecture, configuration + - **Difficulty**: Beginner + - **Time**: 30-45 minutes + - **Use Cases**: First-time users, understanding the framework structure + +### Alternative Approach: Orchestration Class + +For complex workflows, you can use the `` `Semantica` `` class for orchestration. This class coordinates multiple modules and provides lifecycle management. + +**When to use orchestration:** +- Complex multi-step workflows spanning multiple modules +- Need lifecycle management (initialization, shutdown) +- Want centralized configuration +- Building applications with multiple components + +!!! tip "Getting Started" + For beginners, start with individual modules to understand how each component works. As you build more complex applications, consider using the orchestration class for workflow management. See the [Core Module Reference](reference/core.md) for orchestration details. + ## ⚙️ Configuration -Semantica can be configured using environment variables or a configuration file. +Semantica modules can be configured individually or through environment variables. Configuration options vary by module, allowing you to customize behavior for your specific needs. ### Environment Variables +Common configuration via environment variables: + ```bash -export SEMANTICA_API_KEY=your_openai_key -export SEMANTICA_EMBEDDING_PROVIDER=openai -export SEMANTICA_MODEL_NAME=gpt-4 +export OPENAI_API_KEY=your_openai_key +export EMBEDDING_MODEL=all-MiniLM-L6-v2 +export EMBEDDING_DEVICE=cuda ``` +### Module-Specific Configuration + +Each module accepts configuration parameters when instantiated. For example, the NER extractor can be configured with different methods, providers, and thresholds. + ### Config File (`config.yaml`) +For centralized configuration, you can use a YAML config file to manage settings across multiple modules: + ```yaml api_keys: openai: your_key_here - anthropic: your_key_here embedding: provider: openai model: text-embedding-3-large - dimensions: 3072 knowledge_graph: - backend: networkx # or neo4j, arangodb + backend: networkx temporal: true - -graph_store: - backend: neo4j # or falkordb - neo4j_uri: bolt://localhost:7687 - neo4j_user: neo4j - neo4j_password: password ``` +**For detailed configuration examples, see:** +- **[Welcome to Semantica Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Configuration examples for all modules +- **[Core Module Reference](reference/core.md)**: Complete configuration documentation + --- ## ⏭️ Next Steps Now that you understand the basics, here are recommended next steps: -1. **[Your First Knowledge Graph](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Build your first knowledge graph from a document. -2. **[Welcome to Semantica](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Learn the framework basics and configuration. -3. **[Core Workflows](cookbook.md#core-tutorials)**: Learn common patterns and workflows. -4. **[Use Cases](cookbook.md#industry-use-cases)**: Explore domain-specific applications. +### 🍳 Interactive Tutorials (Cookbook) + +Get hands-on experience with these interactive Jupyter notebooks: + +1. **[Welcome to Semantica](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Comprehensive introduction to all Semantica modules + - **Topics**: Framework overview, all modules, architecture, configuration + - **Difficulty**: Beginner + - **Time**: 30-45 minutes + - **Use Cases**: First-time users, understanding the framework structure + +2. **[Your First Knowledge Graph](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Build your first knowledge graph from a document + - **Topics**: Entity extraction, relationship extraction, graph construction, visualization + - **Difficulty**: Beginner + - **Time**: 20-30 minutes + - **Use Cases**: Learning the basics, quick start + +3. **[Data Ingestion](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)**: Learn to ingest from multiple sources + - **Topics**: File, web, feed, stream, database ingestion + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Loading data from various sources + +4. **[Document Parsing](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/03_Document_Parsing.ipynb)**: Parse various document formats + - **Topics**: PDF, DOCX, HTML, JSON parsing + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Extracting text from different file formats + +### 📚 Documentation + +- **[Quick Start Guide](quickstart.md)**: Step-by-step tutorial to build your first knowledge graph +- **[Core Concepts](concepts.md)**: Deep dive into knowledge graphs, ontologies, and semantic reasoning +- **[API Reference](reference/core.md)**: Complete technical documentation for all modules +- **[Examples](examples.md)**: Real-world examples and use cases +- **[Cookbook](cookbook.md)**: Full list of interactive Jupyter notebooks diff --git a/docs/index.md b/docs/index.md index 0481dbba..8b885775 100644 --- a/docs/index.md +++ b/docs/index.md @@ -374,24 +374,45 @@ Power GraphRAG applications with: ## 🚦 Quick Example +Semantica uses a modular architecture. You can use individual modules directly for maximum flexibility: + ```python -from semantica.core import Semantica +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder -# Initialize -core = Semantica() +# 1. Ingest documents +ingestor = FileIngestor() +documents = ingestor.ingest_directory("documents/", recursive=True) -# Ingest documents -docs = core.ingest.load("documents/", recursive=True) +# 2. Parse documents +parser = DocumentParser() +parsed_docs = [parser.parse_document(doc) for doc in documents] -# Build knowledge graph -kg = core.kg.build_graph(docs, merge_entities=True) +# 3. Extract entities and relationships +ner = NERExtractor() +rel_extractor = RelationExtractor() -# Query -result = kg.query("Who founded Apple Inc.?") -print(result.answer) # Steve Jobs, Steve Wozniak, Ronald Wayne -print(result.confidence) # 0.98 +entities = [] +relationships = [] +for doc in parsed_docs: + text = doc.get("full_text", "") + doc_entities = ner.extract_entities(text) + doc_rels = rel_extractor.extract_relations(text, entities=doc_entities) + entities.extend(doc_entities) + relationships.extend(doc_rels) + +# 4. Build knowledge graph +builder = GraphBuilder(merge_entities=True) +kg = builder.build_graph(entities=entities, relationships=relationships) + +print(f"Created graph with {len(kg.nodes)} nodes and {len(kg.edges)} edges") ``` +!!! tip "Orchestration Option" + For complex workflows, you can also use the `Semantica` class for orchestration. See the [Core Module](reference/core.md) documentation for details. + --- ## 🎯 Why Semantica? @@ -446,6 +467,25 @@ print(result.confidence) # 0.98 - [Cookbook](cookbook.md) - Real-world examples and **14 domain-specific cookbooks** - [API Reference](reference/core.md) - Complete technical documentation +### 🍳 Recommended Cookbook Tutorials + +Get hands-on with interactive Jupyter notebooks: + +- **[Welcome to Semantica](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Comprehensive introduction to all Semantica modules + - **Topics**: Framework overview, all modules, architecture + - **Difficulty**: Beginner + - **Use Cases**: First-time users, understanding the framework + +- **[Your First Knowledge Graph](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Build your first knowledge graph from scratch + - **Topics**: Entity extraction, relationship extraction, graph construction + - **Difficulty**: Beginner + - **Use Cases**: Learning the basics, quick start + +- **[GraphRAG Complete](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)**: Production-ready Graph Retrieval Augmented Generation + - **Topics**: GraphRAG, hybrid retrieval, vector search, graph traversal + - **Difficulty**: Advanced + - **Use Cases**: Building AI applications with knowledge graphs + ---
diff --git a/docs/installation.md b/docs/installation.md index 17e6ac23..f5ef45d2 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -225,14 +225,49 @@ It's recommended to use a virtual environment: | Disk Space | Sufficient for data | Generous storage | | OS | Windows/Linux/Mac | Linux/Mac | +## After Installation + +Once Semantica is installed, verify your setup and get started: + +### Verify Your Installation + +Test that everything works correctly: + +```bash +python -c "import semantica; print(semantica.__version__)" +``` + +**For detailed setup verification and first steps, see:** +- **[Welcome to Semantica Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Verify installation and explore all modules + - **Topics**: Framework overview, installation verification, module exploration + - **Difficulty**: Beginner + - **Time**: 30-45 minutes + - **Use Cases**: First-time setup, understanding the framework + ## Next Steps Now that Semantica is installed: -1. **[Quick Start Guide](quickstart.md)** - Build your first knowledge graph -2. **[Examples](examples.md)** - See real-world use cases -3. **[API Reference](reference/core.md) - Explore the full API -4. **[Cookbook](cookbook.md)** - Interactive tutorials +1. **[Quick Start Guide](quickstart.md)** - Build your first knowledge graph in 5 minutes +2. **[Getting Started Guide](getting-started.md)** - Learn the fundamentals +3. **[Examples](examples.md)** - See real-world use cases +4. **[Cookbook](cookbook.md)** - Interactive Jupyter notebook tutorials + +### 🍳 Recommended First Cookbooks + +Start with these interactive tutorials: + +- **[Welcome to Semantica](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Comprehensive introduction + - **Topics**: Framework overview, all modules, architecture, configuration + - **Difficulty**: Beginner + - **Time**: 30-45 minutes + - **Use Cases**: First-time users, understanding the framework + +- **[Your First Knowledge Graph](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Build your first graph + - **Topics**: Entity extraction, relationship extraction, graph construction + - **Difficulty**: Beginner + - **Time**: 20-30 minutes + - **Use Cases**: Hands-on practice, quick start ## Getting Help @@ -240,4 +275,8 @@ If you encounter issues: - Check the [troubleshooting section](#troubleshooting) above - Review [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) -- Ask questions in discussions +- Ask questions in [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions) + +**For installation and setup help:** +- **[Welcome to Semantica Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Includes setup verification steps +- **[Installation Troubleshooting Guide](getting-started.md#installation--setup)**: Additional troubleshooting tips diff --git a/docs/learning-more.md b/docs/learning-more.md index 00328124..def221ea 100644 --- a/docs/learning-more.md +++ b/docs/learning-more.md @@ -43,19 +43,36 @@ Additional resources, tutorials, and advanced learning materials for Semantica. 1. **Installation & Setup** (15 min) - [Installation Guide](installation.md) - - [Welcome to Semantica](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb) + - **[Welcome to Semantica Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Comprehensive introduction + - **Topics**: Framework overview, all modules, architecture, configuration + - **Difficulty**: Beginner + - **Time**: 30-45 minutes + - **Use Cases**: First-time users, understanding the framework 2. **Core Concepts** (30 min) - [Core Concepts](concepts.md) - [Getting Started Guide](getting-started.md) + - **[Data Ingestion Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)**: Learn to ingest from multiple sources + - **Topics**: File, web, feed, stream, database ingestion + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Loading data from various sources 3. **First Knowledge Graph** (30 min) - [Quickstart Tutorial](quickstart.md) - - [Your First Knowledge Graph](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb) + - **[Your First Knowledge Graph Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Build your first knowledge graph + - **Topics**: Entity extraction, relationship extraction, graph construction, visualization + - **Difficulty**: Beginner + - **Time**: 20-30 minutes + - **Use Cases**: Learning the basics, quick start 4. **Basic Operations** (30 min) - [Examples](examples.md) - - Extract entities and relationships + - **[Entity Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)**: Learn entity extraction + - **Topics**: Named entity recognition, entity types, extraction methods + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Understanding entity extraction --- @@ -63,18 +80,41 @@ Additional resources, tutorials, and advanced learning materials for Semantica. 1. **Advanced Concepts** (1 hour) - [Modules Guide](modules.md) - - Understand: Embeddings, GraphRAG, Ontologies + - **[Building Knowledge Graphs Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb)**: Advanced graph construction + - **Topics**: Graph building, entity merging, conflict resolution, temporal graphs + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Production graph construction + - **[Embeddings Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/09_Embeddings.ipynb)**: Learn embeddings + - **Topics**: Embedding generation, similarity search, vector operations + - **Difficulty**: Beginner + - **Time**: 20-30 minutes + - **Use Cases**: Understanding embeddings, semantic search 2. **Use Cases** (1 hour) - [Use Cases Guide](use-cases.md) - - Implement a complete use case + - **[GraphRAG Complete Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)**: Build production GraphRAG + - **Topics**: GraphRAG, hybrid retrieval, graph traversal, LLM integration + - **Difficulty**: Advanced + - **Time**: 1-2 hours + - **Use Cases**: Production GraphRAG systems 3. **Advanced Examples** (1 hour) - - [Examples](examples.md) - Conflict resolution, custom config + - [Examples](examples.md) + - **[Advanced Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb)**: Advanced extraction patterns + - **Topics**: Custom entity types, domain-specific extraction, hybrid methods + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Domain-specific extraction 4. **Quality & Optimization** (1 hour) - [Quality Assurance](concepts.md#8-quality-assurance) - [Performance Optimization](#performance-optimization) + - **[Multi-Source Data Integration Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb)**: Integrate multiple sources + - **Topics**: Multi-source integration, entity resolution, conflict handling + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Building unified knowledge graphs --- @@ -82,15 +122,36 @@ Additional resources, tutorials, and advanced learning materials for Semantica. 1. **Advanced Architecture** (2 hours) - [Architecture Guide](architecture.md) - - Plugin development + - **[Temporal Graphs Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/04_Temporal_Graphs.ipynb)**: Build temporal graphs + - **Topics**: Time-stamped entities, temporal relationships, historical queries + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Time-aware knowledge graphs + - **[Ontology Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb)**: Generate ontologies + - **Topics**: Ontology generation, OWL, schema design + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Formal knowledge representation 2. **Production Deployment** (2 hours) - [Security Best Practices](#security-best-practices) - - Scalability patterns + - **[GraphRAG Complete Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)**: Production GraphRAG + - **Topics**: Production deployment, scalability, optimization + - **Difficulty**: Advanced + - **Time**: 1-2 hours + - **Use Cases**: Production systems 3. **Customization** (2 hours) - - Custom extractors and exporters - - API extensions + - **[Complete Visualization Suite Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/03_Complete_Visualization_Suite.ipynb)**: Advanced visualization + - **Topics**: Custom layouts, filtering, styling, multiple graph types + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Production visualizations + - **[Multi-Format Export Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/05_Multi_Format_Export.ipynb)**: Advanced export patterns + - **Topics**: Batch export, custom formats, format conversion + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Production exports --- @@ -98,23 +159,17 @@ Additional resources, tutorials, and advanced learning materials for Semantica. ### Common Operations -```python -from semantica.core import Semantica -semantica = Semantica() +The typical workflow involves these steps: -# Build Knowledge Graph -result = semantica.build_knowledge_base( - sources=["doc.pdf"], - embeddings=True, - graph=True -) +1. **Ingest** documents using `` `FileIngestor` `` +2. **Parse** documents using `` `DocumentParser` `` +3. **Extract** entities and relationships using `` `NERExtractor` `` and `` `RelationExtractor` `` +4. **Build** knowledge graph using `` `GraphBuilder` `` +5. **Generate** embeddings using `` `TextEmbedder` `` -# Extract Entities -entities = semantica.semantic_extract.extract_entities(text) - -# Query Graph -results = semantica.kg.query("MATCH (n) RETURN n LIMIT 10") -``` +**For complete examples, see:** +- **[Your First Knowledge Graph Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Complete workflow example +- **[Welcome to Semantica Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: All modules overview ### Configuration Reference @@ -162,24 +217,15 @@ results = semantica.kg.query("MATCH (n) RETURN n LIMIT 10") ### 1. Batch Processing -Process multiple documents together for better throughput. +Process multiple documents together for better throughput. Use batch processing when working with large document collections. -```python -sources = ["doc1.pdf", "doc2.pdf", ..., "doc100.pdf"] -result = semantica.build_knowledge_base(sources, batch_size=10) -``` +**For examples, see:** +- **[Data Ingestion Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)**: Batch ingestion patterns +- **[Multi-Source Data Integration Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb)**: Advanced integration ### 2. Parallel Execution -Use parallel processing for independent operations. - -```python -result = semantica.build_knowledge_base( - sources=sources, - parallel=True, - max_workers=8 -) -``` +Use parallel processing for independent operations to improve performance on multi-core systems. ### 3. Backend Selection @@ -220,13 +266,29 @@ A: Yes, it is designed for production with proper configuration. ## Next Steps -- **[Deep Dive](deep-dive.md)** - Advanced architecture +Continue your learning journey: + +- **[Cookbook](cookbook.md)** - Interactive Jupyter notebook tutorials - **[API Reference](reference/core.md)** - Complete API documentation -- **[Cookbook](cookbook.md)** - Interactive tutorials +- **[Use Cases](use-cases.md)** - Real-world applications +- **[Examples](examples.md)** - Code examples and patterns + +### 🍳 Recommended Next Cookbooks + +- **[GraphRAG Complete](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)**: Production GraphRAG system + - **Topics**: GraphRAG, hybrid retrieval, LLM integration + - **Difficulty**: Advanced + - **Time**: 1-2 hours + - **Use Cases**: Production RAG applications + +- **[RAG vs. GraphRAG Comparison](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/02_RAG_vs_GraphRAG_Comparison.ipynb)**: Understand the differences + - **Topics**: RAG comparison, reasoning gap, inference engines + - **Difficulty**: Intermediate + - **Time**: 45-60 minutes + - **Use Cases**: Choosing the right approach --- !!! info "Contribute" Have questions? [Open an issue](https://github.com/Hawksight-AI/semantica/issues) or [start a discussion](https://github.com/Hawksight-AI/semantica/discussions)! -**Last Updated**: 2024 diff --git a/docs/modules.md b/docs/modules.md index 581222d1..def7dc54 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -54,19 +54,13 @@ These modules handle data ingestion, parsing, chunking, and preparation. - `RepoIngestor` — Git repository analysis - `MCPIngestor` — Connect to MCP servers for resource and tool-based ingestion -**Quick Example:** +**Try It:** -```python -from semantica.ingest import FileIngestor, WebIngestor - -# Ingest local files -file_ingestor = FileIngestor() -documents = file_ingestor.ingest("data/", recursive=True) - -# Ingest web content -web_ingestor = WebIngestor() -web_docs = web_ingestor.ingest("https://example.com") -``` +- **[Data Ingestion Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)**: Learn to ingest from multiple sources + - **Topics**: File, web, feed, stream, database ingestion + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Loading data from various sources **API Reference**: [Ingest Module](reference/ingest.md) @@ -98,18 +92,13 @@ web_docs = web_ingestor.ingest("https://example.com") - `ImageParser` — OCR and image analysis - `CodeParser` — Parse source code files -**Quick Example:** +**Try It:** -```python -from semantica.parse import DocumentParser - -parser = DocumentParser(ocr_enabled=True) -parsed_docs = parser.parse(documents) - -for doc in parsed_docs: - print(f"Content: {doc.content[:100]}...") - print(f"Tables found: {len(doc.tables)}") -``` +- **[Document Parsing Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/03_Document_Parsing.ipynb)**: Learn to parse various document formats + - **Topics**: PDF, DOCX, HTML, JSON parsing, OCR, table extraction + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Extracting text from different file formats **API Reference**: [Parse Module](reference/parse.md) @@ -156,19 +145,13 @@ for doc in parsed_docs: | **Standard** | recursive, token, sentence, paragraph, character, word, semantic_transformer, llm | | **KG/Ontology** | entity_aware, relation_aware, graph_based, ontology_aware, hierarchical, community_detection, centrality_based | -**Quick Example:** +**Try It:** -```python -from semantica.split import TextSplitter - -# Standard recursive splitting -splitter = TextSplitter(method="recursive", chunk_size=1000, chunk_overlap=200) -chunks = splitter.split(text) - -# Entity-aware for GraphRAG -splitter = TextSplitter(method="entity_aware", ner_method="llm", chunk_size=1000) -chunks = splitter.split(text) -``` +- **[Text Splitting Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/04_Data_Normalization.ipynb)**: Learn different splitting methods + - **Topics**: Recursive, token, sentence splitting, entity-aware chunking + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Document chunking for processing --- @@ -198,21 +181,13 @@ chunks = splitter.split(text) - `LanguageDetector` — Detect document language - `EncodingHandler` — Handle character encoding -**Quick Example:** +**Try It:** -```python -from semantica.normalize import TextNormalizer - -normalizer = TextNormalizer( - normalize_entities=True, - normalize_dates=True, - detect_language=True -) -normalized = normalizer.normalize(parsed_docs) - -for doc in normalized: - print(f"Language: {doc.language}") -``` +- **[Data Normalization Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/04_Data_Normalization.ipynb)**: Learn text normalization + - **Topics**: Text cleaning, encoding normalization, entity standardization + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Preparing text for processing **API Reference**: [Normalize Module](reference/normalize.md) @@ -248,8 +223,21 @@ These modules form the intelligence core—extracting meaning, building relation - `EventExtractor` — Extract events from text - `CoreferenceResolver` — Resolve entity coreferences -**Quick Example:** +**Try It:** +- **[Entity Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)**: Learn entity extraction + - **Topics**: Named entity recognition, entity types, extraction methods + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Understanding entity extraction + +- **[Relation Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/06_Relation_Extraction.ipynb)**: Learn relationship extraction + - **Topics**: Relationship extraction, dependency parsing, semantic role labeling + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Building rich knowledge graphs + +**Quick Example:** ```python from semantica.semantic_extract import NERExtractor, RelationExtractor @@ -1099,92 +1087,47 @@ result = pipeline.execute(sources=["data/"], parallel=True) ### Pattern 1: Complete Knowledge Graph Pipeline -```python -from semantica.core import Semantica +Build a complete knowledge graph from documents using the full pipeline. -semantica = Semantica() -result = semantica.build_knowledge_base( - sources=["documents/"], - embeddings=True, - graph=True, - normalize=True -) -``` +**For complete examples, see:** +- **[Your First Knowledge Graph Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Complete pipeline walkthrough + - **Topics**: Ingestion, parsing, extraction, graph building, embeddings + - **Difficulty**: Beginner + - **Time**: 20-30 minutes + - **Use Cases**: Learning the complete workflow ### Pattern 2: Custom Pipeline with Module Selection -```python -from semantica.ingest import FileIngestor -from semantica.parse import DocumentParser -from semantica.split import TextSplitter -from semantica.normalize import TextNormalizer -from semantica.semantic_extract import NERExtractor, RelationExtractor -from semantica.kg import GraphBuilder -from semantica.deduplication import DuplicateDetector, EntityMerger, MergeStrategy +Build custom pipelines with specific module selections and quality assurance. -# Ingest and parse -documents = FileIngestor().ingest("data/") -parsed = DocumentParser().parse(documents) - -# Split and normalize -chunks = TextSplitter(method="entity_aware").split(parsed) -normalized = TextNormalizer().normalize(chunks) - -# Extract and build -entities = NERExtractor().extract(normalized) -relationships = RelationExtractor().extract(normalized, entities) -kg = GraphBuilder().build(entities, relationships) - -# Quality assurance - deduplicate entities -detector = DuplicateDetector(similarity_threshold=0.8) -duplicate_groups = detector.detect_duplicate_groups(entities) -merger = EntityMerger() -merge_operations = merger.merge_duplicates(entities, strategy=MergeStrategy.KEEP_MOST_COMPLETE) -deduplicated = [op.merged_entity for op in merge_operations] -``` +**For examples, see:** +- **[Building Knowledge Graphs Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb)**: Advanced graph construction + - **Topics**: Custom pipelines, entity merging, conflict resolution + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Production graph construction ### Pattern 3: GraphRAG with Hybrid Search -```python -from semantica.core import Semantica -from semantica.vector_store import VectorStore, HybridSearch -from semantica.context import AgentMemory +Build GraphRAG systems with hybrid search combining vector and graph retrieval. -semantica = Semantica() -result = semantica.build_knowledge_base(["documents/"]) - -vector_store = VectorStore() -vector_store.store(result["embeddings"], result["documents"]) - -# Agent memory with RAG -memory = AgentMemory(vector_store=vector_store, knowledge_graph=result["knowledge_graph"]) -memory.store("User query about AI", metadata={"type": "query"}) - -# Hybrid search -hybrid_search = HybridSearch(vector_store) -results = hybrid_search.search( - query="What is the relationship between X and Y?", - graph=result["knowledge_graph"], - top_k=10 -) -``` +**For complete examples, see:** +- **[GraphRAG Complete Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)**: Production GraphRAG system + - **Topics**: GraphRAG, hybrid retrieval, graph traversal, LLM integration + - **Difficulty**: Advanced + - **Time**: 1-2 hours + - **Use Cases**: Production RAG applications ### Pattern 4: Temporal Graph with Reasoning -```python -from semantica.kg import GraphBuilder -from semantica.reasoning import Reasoner +Build temporal graphs with logical reasoning capabilities. -# Build temporal graph -builder = GraphBuilder(temporal=True) -kg = builder.build(entities, relationships) - -# Add reasoning -reasoner = Reasoner() -reasoner.add_rule("IF A THEN B") - -new_facts = reasoner.infer_facts(kg) -``` +**For examples, see:** +- **[Temporal Graphs Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/04_Temporal_Graphs.ipynb)**: Temporal graph construction + - **Topics**: Time-stamped entities, temporal relationships, historical queries + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Time-aware knowledge graphs --- @@ -1220,11 +1163,25 @@ new_facts = reasoner.infer_facts(kg) - **[Core Concepts](concepts.md)** — Understand the fundamental concepts - **[Use Cases](use-cases.md)** — See real-world applications - **[Examples](examples.md)** — Practical code examples +- **[Cookbook](cookbook.md)** — Interactive Jupyter notebook tutorials - **[API Reference](reference/core.md)** — Detailed API documentation +### 🍳 Recommended Cookbooks + +- **[Welcome to Semantica](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Comprehensive introduction to all modules + - **Topics**: Framework overview, all modules, architecture + - **Difficulty**: Beginner + - **Time**: 30-45 minutes + - **Use Cases**: Understanding the complete framework + +- **[Your First Knowledge Graph](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Build your first knowledge graph + - **Topics**: Complete pipeline from ingestion to graph construction + - **Difficulty**: Beginner + - **Time**: 20-30 minutes + - **Use Cases**: Hands-on practice with all modules + --- !!! info "Contribute" Found an issue or want to improve this guide? [Contribute on GitHub](https://github.com/Hawksight-AI/semantica) -**Last Updated**: 2024 diff --git a/docs/quickstart.md b/docs/quickstart.md index 5087298f..723f76b8 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -34,215 +34,190 @@ See the [Installation Guide](installation.md) for detailed instructions. ## Step 2: Your First Knowledge Graph -Let's build a knowledge graph from a document: +Building a knowledge graph involves these key steps: +1. **Ingest** your documents using `` `FileIngestor` `` +2. **Parse** documents to extract text using `` `DocumentParser` `` +3. **Extract** entities and relationships using `` `NERExtractor` `` and `` `RelationExtractor` `` +4. **Build** the graph using `` `GraphBuilder` `` +5. **Generate** embeddings (optional) using `` `TextEmbedder` `` + +**Quick Example:** ```python -from semantica.core import Semantica +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder -# Initialize Semantica -semantica = Semantica() +# Build your first knowledge graph +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() +rel_extractor = RelationExtractor() +builder = GraphBuilder() -# Build knowledge graph from a document -result = semantica.build_knowledge_base( - sources=["document.pdf"], - embeddings=True, - graph=True -) - -# Access results -kg = result["knowledge_graph"] -embeddings = result["embeddings"] -statistics = result["statistics"] - -print(f"Extracted {len(kg['entities'])} entities") -print(f"Created {len(kg['relationships'])} relationships") -print(f"Generated {len(embeddings)} embeddings") +# Process document and build graph +doc = ingestor.ingest_file("document.pdf") +parsed = parser.parse_document("document.pdf") +entities = ner.extract_entities(parsed.get("full_text", "")) +relationships = rel_extractor.extract_relations(parsed.get("full_text", ""), entities=entities) +kg = builder.build_graph(entities=entities, relationships=relationships) ``` -**Expected Output:** -``` -Extracted 45 entities -Created 32 relationships -Generated 45 embeddings -``` +**For complete step-by-step examples with detailed explanations, see:** +- **[Your First Knowledge Graph Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Full tutorial with detailed explanations and expected outputs + - **Topics**: Entity extraction, relationship extraction, graph construction, visualization + - **Difficulty**: Beginner + - **Time**: 20-30 minutes + - **Use Cases**: Learning the basics, quick start ## Step 3: Extract Entities and Relationships -Extract structured information from text: +The semantic extraction step identifies named entities (people, organizations, locations) and relationships between them from your text. -```python -from semantica.semantic_extract import NamedEntityRecognizer, RelationExtractor +**What gets extracted:** +- **Entities**: People, organizations, locations, dates, and other named entities +- **Relationships**: Connections between entities (e.g., `founded_by`, `located_in`, `has_ceo`) -# Sample text -text = """ -Apple Inc. was founded by Steve Jobs in Cupertino, California in 1976. -The company designs and manufactures consumer electronics and software. -Tim Cook is the current CEO of Apple. -""" +**For detailed examples and different extraction methods, see:** +- **[Entity Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)**: Learn different NER methods and configurations + - **Topics**: Named entity recognition, entity types, confidence scores + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Understanding entity extraction options -# Extract entities -ner = NamedEntityRecognizer() -entities = ner.extract_entities(text) - -print("Extracted Entities:") -for entity in entities: - print(f" - {entity.text} ({entity.label})") - -# Extract relationships -rel_extractor = RelationExtractor() -relationships = rel_extractor.extract_relations(text, entities=entities) - -print("\nExtracted Relationships:") -for rel in relationships: - print(f" - {rel.subject.text} --[{rel.predicate}]--> {rel.object.text}") -``` - -**Expected Output:** -``` -Extracted Entities: - - Apple Inc. (ORGANIZATION) - - Steve Jobs (PERSON) - - Cupertino (LOCATION) - - California (LOCATION) - - Tim Cook (PERSON) - -Extracted Relationships: - - Apple Inc. --[founded_by]--> Steve Jobs - - Apple Inc. --[located_in]--> Cupertino - - Apple Inc. --[has_ceo]--> Tim Cook -``` +- **[Relation Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/06_Relation_Extraction.ipynb)**: Learn to extract relationships between entities + - **Topics**: Relationship extraction, dependency parsing, semantic role labeling + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Building rich knowledge graphs with relationships ## Step 4: Build Knowledge Graph from Multiple Sources -Combine data from multiple sources: +You can combine data from multiple sources (files, web, databases) to build a unified knowledge graph. The process involves: -```python -from semantica.core import Semantica +1. **Ingest** from multiple sources using different ingestors +2. **Parse** all documents to extract text +3. **Extract** entities and relationships from each source +4. **Build** a unified graph with entity merging enabled -semantica = Semantica() +**For complete examples with multiple sources, see:** +- **[Data Ingestion Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)**: Learn to ingest from files, web, feeds, streams, and databases + - **Topics**: File, web, feed, stream, database ingestion + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Loading data from various sources -# Multiple data sources -sources = [ - "documents/research_paper.pdf", - "documents/company_report.docx", - "https://example.com/news-article" -] - -# Build unified knowledge graph -result = semantica.build_knowledge_base( - sources=sources, - embeddings=True, - graph=True, - normalize=True -) - -kg = result["knowledge_graph"] - -# Analyze the graph -print(f"Total entities: {len(kg['entities'])}") -print(f"Total relationships: {len(kg['relationships'])}") -print(f"Sources processed: {len(result['metadata']['sources'])}") -``` +- **[Multi-Source Data Integration Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb)**: Advanced patterns for integrating multiple data sources + - **Topics**: Multi-source integration, entity resolution, conflict handling + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Building knowledge graphs from diverse data sources ## Step 5: Visualize Your Knowledge Graph -Visualize the knowledge graph you created: +Visualization helps you understand and explore your knowledge graph structure. Semantica supports multiple visualization formats including interactive HTML, static images, and export formats. -```python -from semantica.core import Semantica -from semantica.visualization import KGVisualizer +**For detailed visualization examples, see:** +- **[Visualization Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/16_Visualization.ipynb)**: Learn to create interactive and static visualizations + - **Topics**: Network graphs, interactive HTML, static images, export formats + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Exploring graph structure, presentations, analysis -semantica = Semantica() - -# Build graph -result = semantica.build_knowledge_base(["document.pdf"]) -kg = result["knowledge_graph"] - -# Visualize -visualizer = KGVisualizer() -visualizer.visualize_network(kg, output="html", file_path="graph.html") -print("Graph visualization saved to graph.html") -``` - -Open `graph.html` in your browser to see an interactive visualization. +- **[Complete Visualization Suite Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/03_Complete_Visualization_Suite.ipynb)**: Advanced visualization techniques + - **Topics**: Custom layouts, filtering, styling, multiple graph types + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Production visualizations, custom dashboards ## Step 6: Export Your Knowledge Graph -Export your knowledge graph in various formats: +Export your knowledge graph to various formats for integration with other systems or tools. Semantica supports RDF, JSON, CSV, OWL, GraphML, and more. -```python -from semantica.core import Semantica -from semantica.export import export_rdf, export_json, export_csv, export_owl +**Supported export formats:** +- **RDF**: Turtle, RDF/XML, JSON-LD, N-Triples +- **JSON**: Standard JSON, JSON-LD, Cytoscape.js format +- **CSV**: Node and edge lists for spreadsheet tools +- **OWL**: OWL/XML and Turtle for ontologies +- **Graph Formats**: GraphML, GEXF, DOT for visualization tools -semantica = Semantica() +**For detailed export examples, see:** +- **[Export Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/15_Export.ipynb)**: Learn to export to all supported formats + - **Topics**: RDF, JSON, CSV, OWL, GraphML export + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Data integration, sharing knowledge graphs -# Build graph -result = semantica.build_knowledge_base(["data.pdf"]) -kg = result["knowledge_graph"] - -# Export to different formats -export_rdf(kg, "output.rdf") # RDF/XML format -export_json(kg, "output.json") # JSON format -export_csv(kg, "output.csv") # CSV format -export_owl(kg, "output.owl") # OWL ontology format - -print("Exported knowledge graph to multiple formats") -``` +- **[Multi-Format Export Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/05_Multi_Format_Export.ipynb)**: Advanced export patterns + - **Topics**: Batch export, custom formats, format conversion + - **Difficulty**: Intermediate + - **Time**: 30-45 minutes + - **Use Cases**: Production exports, format migration ## Common Patterns ### Pattern 1: Process Text Directly -```python -from semantica.core import Semantica +You can process text directly without file ingestion. This is useful when you already have text content in memory. -semantica = Semantica() +**For examples, see:** +- **[Entity Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)**: Processing text directly +- **[Building Knowledge Graphs Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb)**: Graph construction from text -text = "Your text content here..." -result = semantica.process_document(text) -``` +### Pattern 2: Custom Entity Extraction -### Pattern 2: Custom Configuration +Configure entity extraction with different methods (ML models, LLMs) and parameters for your specific needs. -```python -from semantica.core import Semantica, Config - -# Create custom configuration -config = Config( - embeddings=True, - graph=True, - normalize=True, - conflict_resolution="voting" -) - -semantica = Semantica(config=config) -result = semantica.build_knowledge_base(["document.pdf"]) -``` +**For examples, see:** +- **[Entity Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)**: Different extraction methods and configurations +- **[Advanced Extraction Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb)**: Advanced extraction patterns ### Pattern 3: Incremental Building -```python -from semantica.core import Semantica +Build knowledge graphs incrementally from multiple sources and merge them together. -semantica = Semantica() - -# Build incrementally -kg1 = semantica.kg.build_graph(["source1.pdf"]) -kg2 = semantica.kg.build_graph(["source2.pdf"]) - -# Merge knowledge graphs -merged_kg = semantica.kg.merge([kg1, kg2]) -``` +**For examples, see:** +- **[Building Knowledge Graphs Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb)**: Graph construction and merging +- **[Multi-Source Data Integration Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb)**: Advanced integration patterns ## Next Steps Now that you've built your first knowledge graph: 1. **[Explore Examples](examples.md)** - See more advanced use cases -2. **[API Reference](reference/core.md) - Learn about all available methods +2. **[API Reference](reference/core.md)** - Learn about all available methods 3. **[Cookbook](cookbook.md)** - Interactive Jupyter notebooks 4. **[Full Documentation](https://github.com/Hawksight-AI/semantica/blob/main/README.md)** - Comprehensive guide +### 🍳 Recommended Cookbook Tutorials + +Continue learning with these interactive tutorials: + +- **[Welcome to Semantica](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Comprehensive introduction to all modules + - **Topics**: Framework overview, all modules, architecture, configuration + - **Difficulty**: Beginner + - **Time**: 30-45 minutes + - **Use Cases**: Understanding the complete framework + +- **[Your First Knowledge Graph](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Build your first knowledge graph + - **Topics**: Entity extraction, relationship extraction, graph construction, visualization + - **Difficulty**: Beginner + - **Time**: 20-30 minutes + - **Use Cases**: Hands-on practice with complete workflow + +- **[Data Ingestion](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)**: Learn to ingest from multiple sources + - **Topics**: File, web, feed, stream, database ingestion + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Loading data from various sources + +- **[Document Parsing](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/03_Document_Parsing.ipynb)**: Parse various document formats + - **Topics**: PDF, DOCX, HTML, JSON parsing + - **Difficulty**: Beginner + - **Time**: 15-20 minutes + - **Use Cases**: Extracting text from different file formats + ## Troubleshooting ### Common Issues diff --git a/docs/reference/conflicts.md b/docs/reference/conflicts.md index a1f09284..8a9d051d 100644 --- a/docs/reference/conflicts.md +++ b/docs/reference/conflicts.md @@ -56,6 +56,9 @@ ## ⚙️ Algorithms Used ### Conflict Detection + +The conflict detection system identifies discrepancies using: + - **Value Comparison**: Equality checking with type normalization - **Type Mismatch**: Entity type hierarchy validation - **Temporal Analysis**: Timestamp comparison for time-based conflicts @@ -66,11 +69,14 @@ - Number of conflicting sources ### Conflict Resolution -- **Voting (Majority Rule)**: `max(frequency(values))` using Counter -- **Credibility Weighted**: `Σ(value_i * source_credibility_i) / Σ(source_credibility)` -- **Temporal Selection**: Select value with latest timestamp (`max(timestamp)`) + +The module provides multiple resolution strategies: + +- **Voting (Majority Rule)**: `` `max(frequency(values))` `` using Counter +- **Credibility Weighted**: `` `Σ(value_i * source_credibility_i) / Σ(source_credibility)` `` +- **Temporal Selection**: Select value with latest timestamp (`` `max(timestamp)` ``) - **Confidence Selection**: Select value with highest extraction confidence -- **Hybrid Resolution**: Waterfall approach (e.g., Voting -> Credibility -> Recency) +- **Hybrid Resolution**: Waterfall approach (e.g., Voting → Credibility → Recency) ### Analysis & Tracking - **Pattern Identification**: Frequency analysis of conflict types @@ -255,13 +261,27 @@ conflicts: ```python from semantica.conflicts import ConflictDetector, ConflictResolver -from semantica.core import Semantica +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor +from semantica.kg import GraphBuilder -# 1. Build knowledge base from multiple sources -semantica = Semantica() -result = semantica.build_knowledge_base(["source1.pdf", "source2.html"]) -kg = result["knowledge_graph"] -entities = kg.get("entities", []) +# 1. Build knowledge base from multiple sources using individual modules +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() +builder = GraphBuilder() + +all_entities = [] +for source in ["source1.pdf", "source2.html"]: + doc = ingestor.ingest_file(source) + parsed = parser.parse_document(source) + text = parsed.get("full_text", "") + entities = ner.extract_entities(text) + all_entities.extend(entities) + +kg = builder.build_graph(entities=all_entities, relationships=[]) +entities = all_entities # 3. Detect conflicts detector = ConflictDetector() @@ -320,4 +340,9 @@ tracker.set_source_credibility("bad_source", 0.1) ## Cookbook -- [Conflict Detection & Resolution](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/17_Conflict_Detection_and_Resolution.ipynb) +Interactive tutorials to learn conflict detection and resolution: + +- **[Conflict Detection & Resolution](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/17_Conflict_Detection_and_Resolution.ipynb)**: Strategies for handling contradictory information from multiple sources + - **Topics**: Truth discovery, voting, confidence scoring, conflict resolution strategies + - **Difficulty**: Advanced + - **Use Cases**: Multi-source data integration, quality assurance diff --git a/docs/reference/context.md b/docs/reference/context.md index 0a02353c..4e1f0827 100644 --- a/docs/reference/context.md +++ b/docs/reference/context.md @@ -57,12 +57,13 @@ The **Context Module** provides agents with a persistent, searchable, and struct The high-level facade that unifies all context operations. It routes data to the appropriate subsystems (Memory, Graph, Vector Store) and manages the lifecycle of context. #### **Constructor Parameters** -* `vector_store` (Required): The backing vector database instance (e.g., FAISS, Weaviate). -* `knowledge_graph` (Optional): The graph store instance for structured knowledge. -* `token_limit` (Default: `2000`): The maximum number of tokens allowed in short-term memory before pruning occurs. -* `short_term_limit` (Default: `10`): The maximum number of distinct memory items in short-term memory. -* `hybrid_alpha` (Default: `0.5`): The weighting factor for retrieval (0.0 = Pure Vector, 1.0 = Pure Graph). -* `use_graph_expansion` (Default: `True`): Whether to fetch neighbors of retrieved nodes from the graph. + +- `` `vector_store` `` (Required): The backing vector database instance (e.g., FAISS, Weaviate) +- `` `knowledge_graph` `` (Optional): The graph store instance for structured knowledge +- `` `token_limit` `` (Default: `` `2000` ``): The maximum number of tokens allowed in short-term memory before pruning occurs +- `` `short_term_limit` `` (Default: `` `10` ``): The maximum number of distinct memory items in short-term memory +- `` `hybrid_alpha` `` (Default: `` `0.5` ``): The weighting factor for retrieval (`` `0.0` `` = Pure Vector, `` `1.0` `` = Pure Graph) +- `` `use_graph_expansion` `` (Default: `` `True` ``): Whether to fetch neighbors of retrieved nodes from the graph #### **Core Methods** @@ -507,5 +508,15 @@ context_config.set("retention_days", 60) - [Reasoning](reasoning.md) - Uses context for logic ## Cookbook -- [Context Module](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/19_Context_Module.ipynb) -- [Advanced Context Engineering](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/11_Advanced_Context_Engineering.ipynb) + +Interactive tutorials to learn context management and GraphRAG: + +- **[Context Module](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/19_Context_Module.ipynb)**: Practical guide to the context module for AI agents + - **Topics**: Agent memory, context graph, hybrid retrieval, entity linking + - **Difficulty**: Intermediate + - **Use Cases**: Building stateful AI agents, persistent memory systems + +- **[Advanced Context Engineering](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/11_Advanced_Context_Engineering.ipynb)**: Build a production-grade memory system for AI agents + - **Topics**: Agent memory, GraphRAG, entity injection, lifecycle management, persistent stores + - **Difficulty**: Advanced + - **Use Cases**: Production agent systems, advanced memory management diff --git a/docs/reference/core.md b/docs/reference/core.md index b4587a93..067e8699 100644 --- a/docs/reference/core.md +++ b/docs/reference/core.md @@ -6,13 +6,37 @@ ## 🎯 Overview +The **Core Module** provides framework infrastructure, lifecycle management, configuration, and orchestration capabilities. It's the foundation that enables coordination between all Semantica modules. + +### What is the Core Module? + +The Core module provides: + +- **Orchestration**: The `` `Semantica` `` class coordinates multiple modules for complex workflows +- **Lifecycle Management**: Handles initialization, startup, shutdown, and state transitions +- **Configuration**: Unified configuration management via YAML and environment variables +- **Plugin System**: Extensible plugin registry for custom modules and capabilities +- **Method Registry**: Registry for custom orchestration methods + +### When to Use the Core Module + +!!! tip "Primary Approach: Individual Modules" + For most use cases, **use individual modules directly** (e.g., `semantica.ingest`, `semantica.kg`). This gives you full control and transparency. + +!!! note "When to Use Orchestration" + Use the `Semantica` orchestration class when you need: + - **Complex Workflows**: Multi-step pipelines that span multiple modules + - **Lifecycle Management**: Application-level initialization and shutdown + - **Centralized Configuration**: Global settings that affect multiple modules + - **Plugin Integration**: Custom plugins that need framework coordination +
- :material-cogs:{ .lg .middle } **Semantica** --- - Main framework class coordinating all components and workflows + Orchestration class for coordinating complex workflows across modules - :material-lifecycle:{ .lg .middle } **Lifecycle Management** @@ -40,60 +64,107 @@
-!!! tip "When to Use" - - **Application Startup**: Initializing the Semantica framework in your app - - **Configuration**: Tuning global settings - - **Extension**: Developing custom plugins or modules - - **Orchestration**: Coordinating complex workflows across multiple modules - --- ## ⚙️ Algorithms Used ### Lifecycle Management + +**What is Lifecycle Management?** + +Lifecycle management handles the initialization, startup, running, and shutdown phases of the Semantica framework. It ensures that all components are properly initialized, resources are managed correctly, and cleanup happens gracefully. + +**How it works:** - **State Machine**: `UNINITIALIZED` -> `INITIALIZING` -> `READY` -> `RUNNING` -> `STOPPING` -> `STOPPED` - **Priority-based Hooks**: Startup and shutdown hooks executed in priority order (lower = earlier) - **Graceful Shutdown**: Ensuring all resources (DB connections, thread pools) are closed properly +**Why it matters:** +- Prevents resource leaks (database connections, file handles) +- Ensures proper initialization order (dependencies are ready before use) +- Enables clean application shutdown +- Supports health monitoring and status tracking + ### Configuration + +**What is Configuration Management?** + +Configuration management provides a unified way to configure all Semantica modules. It supports multiple configuration sources with a clear priority order, ensuring consistent settings across your application. + +**How it works:** - **Layered Loading**: Defaults -> Config File -> Environment Variables (Priority order) - **Schema Validation**: Validating config structure against defined schemas - **Nested Access**: Dot notation for accessing nested configuration values +**Why it matters:** +- Centralized configuration for all modules +- Environment-specific settings (dev, staging, production) +- Secure credential management (via environment variables) +- Validation prevents configuration errors + ### Plugin System + +**What is the Plugin System?** + +The plugin system allows you to extend Semantica with custom modules and capabilities. Plugins can add new functionality, modify existing behavior, or integrate with external systems. + +**How it works:** - **Discovery**: Auto-discovery of plugins via directory scanning - **Registration**: Dynamic registration of classes and functions - **Dependency Resolution**: Automatic loading of plugin dependencies +**Why it matters:** +- Extend Semantica with custom functionality +- Integrate with external systems and APIs +- Modify or enhance existing modules +- Share custom functionality across projects + --- ## Main Classes ### Semantica -The main framework class that coordinates all components. +The **Semantica** class is an orchestration class that coordinates multiple modules for complex workflows. It's designed for applications that need lifecycle management, centralized configuration, and multi-step pipeline coordination. + +!!! important "Not a Convenience Wrapper" + The `Semantica` class is **not** a convenience wrapper. It's an orchestration tool for complex workflows. For most use cases, use individual modules directly for better control and transparency. + +**What it does:** +- Coordinates multiple modules (ingest, parse, extract, kg, etc.) +- Manages application lifecycle (initialization, shutdown) +- Provides centralized configuration +- Enables plugin integration +- Handles complex multi-step workflows + +**When to use it:** +- Building applications with multiple components +- Need lifecycle management (startup/shutdown hooks) +- Complex workflows spanning multiple modules +- Want centralized configuration +- Integrating custom plugins **Methods:** | Method | Description | |--------|-------------| -| `__init__(config=None, **kwargs)` | Initialize framework with optional configuration | -| `initialize()` | Initialize all framework components | -| `build_knowledge_base(sources, **kwargs)` | Build knowledge base from data sources | -| `run_pipeline(pipeline, data)` | Execute a processing pipeline | -| `get_status()` | Get system health and status | -| `shutdown(graceful=True)` | Shutdown the framework gracefully | +| `` `__init__(config=None, **kwargs)` `` | Initialize framework with optional configuration | +| `` `initialize()` `` | Initialize all framework components and modules | +| `` `build_knowledge_base(sources, **kwargs)` `` | Orchestrate building a knowledge base from data sources | +| `` `run_pipeline(pipeline, data)` `` | Execute a processing pipeline | +| `` `get_status()` `` | Get system health and status | +| `` `shutdown(graceful=True)` `` | Shutdown the framework gracefully | -**Example:** +**Example - Orchestration for Complex Workflow:** ```python from semantica.core import Semantica -# Initialize framework +# Initialize framework for orchestration framework = Semantica() framework.initialize() -# Build knowledge base +# Build knowledge base (orchestrates multiple modules) result = framework.build_knowledge_base( sources=["doc1.pdf", "doc2.docx"], embeddings=True, @@ -104,10 +175,36 @@ result = framework.build_knowledge_base( status = framework.get_status() print(f"System state: {status['state']}") -# Shutdown +# Shutdown gracefully framework.shutdown() ``` +**Alternative - Using Individual Modules (Recommended):** + +```python +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor, RelationExtractor +from semantica.kg import GraphBuilder +from semantica.embeddings import TextEmbedder + +# Use modules directly for full control +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() +rel_extractor = RelationExtractor() +builder = GraphBuilder() +embedder = TextEmbedder() + +# Build your pipeline step by step +docs = ingestor.ingest_file("doc1.pdf") +parsed = parser.parse_document("doc1.pdf") +entities = ner.extract_entities(parsed.get("full_text", "")) +relationships = rel_extractor.extract_relations(parsed.get("full_text", ""), entities=entities) +kg = builder.build_graph(entities=entities, relationships=relationships) +embeddings = embedder.embed_batch([e.text for e in entities]) +``` + ### ConfigManager Manages global configuration loading, validation, and merging. @@ -116,9 +213,9 @@ Manages global configuration loading, validation, and merging. | Method | Description | |--------|-------------| -| `load_from_file(file_path, validate=True)` | Load config from YAML or JSON file | -| `load_from_dict(config_dict, validate=True)` | Load config from dictionary | -| `merge_configs(*configs, validate=True)` | Merge multiple configurations | +| `` `load_from_file(file_path, validate=True)` `` | Load config from YAML or JSON file | +| `` `load_from_dict(config_dict, validate=True)` `` | Load config from dictionary | +| `` `merge_configs(*configs, validate=True)` `` | Merge multiple configurations | | `get_config()` | Get current configuration | | `set_config(config, validate=True)` | Set current configuration | | `reload(file_path=None)` | Reload configuration from file | @@ -550,9 +647,20 @@ result = build_knowledge_base(sources=["doc.pdf"], method="fast") --- ## See Also -- [Core Usage Guide](core.md) - Comprehensive usage guide with detailed examples -- [Pipeline Module](pipeline.md) - Executed by the Semantica framework +- [Pipeline Module](pipeline.md) - Pipeline execution and orchestration - [Utils Module](utils.md) - Shared utilities used by Core +- [Getting Started Guide](../getting-started.md) - Learn the basics ## Cookbook -- [Welcome to Semantica](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb) + +Interactive tutorials to learn orchestration and lifecycle management: + +- **[Welcome to Semantica](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: Comprehensive introduction to all Semantica modules and architecture + - **Topics**: Framework overview, all modules, architecture, configuration, lifecycle + - **Difficulty**: Beginner + - **Use Cases**: Understanding the framework structure, first-time users + +- **[Pipeline Orchestration](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/07_Pipeline_Orchestration.ipynb)**: Build robust, automated data processing pipelines + - **Topics**: Workflows, automation, error handling, pipeline orchestration + - **Difficulty**: Advanced + - **Use Cases**: Complex multi-step workflows, production pipelines diff --git a/docs/reference/deduplication.md b/docs/reference/deduplication.md index cb4b6be4..dd01e1e0 100644 --- a/docs/reference/deduplication.md +++ b/docs/reference/deduplication.md @@ -56,27 +56,37 @@ ## ⚙️ Algorithms Used ### Similarity Calculation + +The deduplication system uses multiple similarity metrics: + - **Levenshtein Distance**: Edit distance for string difference - **Jaro-Winkler**: String similarity with prefix weighting (Default for strings, optimized for entity names) - **Cosine Similarity**: Vector similarity for embeddings - **Jaccard Similarity**: Set overlap for properties/relationships -- **Property Matching**: Handles disjoint properties with neutral scoring (0.5) to prevent false negatives +- **Property Matching**: Handles disjoint properties with neutral scoring (`` `0.5` ``) to prevent false negatives - **Multi-factor Aggregation**: Weighted sum of multiple metrics ### Default Configuration + The deduplication module uses the following default weights to prioritize name matching while considering other factors: -- **String Similarity**: 0.6 (Primary factor, using Jaro-Winkler) -- **Property Similarity**: 0.2 (Handles missing values neutrally) -- **Relationship Similarity**: 0.2 -- **Embedding Similarity**: 0.0 (Optional, enabled if embeddings are present) + +- **String Similarity**: `` `0.6` `` (Primary factor, using Jaro-Winkler) +- **Property Similarity**: `` `0.2` `` (Handles missing values neutrally) +- **Relationship Similarity**: `` `0.2` `` +- **Embedding Similarity**: `` `0.0` `` (Optional, enabled if embeddings are present) ### Duplicate Detection -- **Pairwise Comparison**: O(n²) comparison (for small sets) + +The system uses efficient detection algorithms: + +- **Pairwise Comparison**: `` `O(n²)` `` comparison (for small sets) - **Blocking/Indexing**: Reduce search space for large sets - **Union-Find**: Disjoint set data structure for grouping duplicates -- **Confidence Scoring**: `0.0 - 1.0` probability score for duplicates +- **Confidence Scoring**: `` `0.0 - 1.0` `` probability score for duplicates ### Clustering + +The module provides clustering algorithms for grouping similar entities: - **Hierarchical Clustering**: Agglomerative bottom-up clustering - **Connected Components**: Graph-based cluster detection - **Cluster Quality**: Cohesion and separation metrics @@ -686,13 +696,25 @@ Configuration is loaded in the following priority order: ```python from semantica.core import Semantica +from semantica.ingest import FileIngestor +from semantica.parse import DocumentParser +from semantica.semantic_extract import NERExtractor from semantica.deduplication import DuplicateDetector, EntityMerger, MergeStrategy -# 1. Build knowledge base -semantica = Semantica() -result = semantica.build_knowledge_base(files) -kg = result["knowledge_graph"] -raw_entities = kg.get("entities", []) +# 1. Build knowledge base using individual modules +ingestor = FileIngestor() +parser = DocumentParser() +ner = NERExtractor() + +all_entities = [] +for file_path in files: + doc = ingestor.ingest_file(file_path) + parsed = parser.parse_document(file_path) + text = parsed.get("full_text", "") + entities = ner.extract_entities(text) + all_entities.extend(entities) + +raw_entities = all_entities # 2. Deduplicate detector = DuplicateDetector(similarity_threshold=0.85) @@ -744,4 +766,9 @@ detector = DuplicateDetector( ## Cookbook -- [Deduplication](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/18_Deduplication.ipynb) +Interactive tutorials to learn deduplication: + +- **[Deduplication](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/18_Deduplication.ipynb)**: Advanced deduplication techniques for entity resolution + - **Topics**: Entity deduplication, fuzzy matching, similarity thresholds, merge strategies + - **Difficulty**: Intermediate + - **Use Cases**: Entity resolution, data cleaning, multi-source integration diff --git a/docs/reference/embeddings.md b/docs/reference/embeddings.md index 51268661..675ce733 100644 --- a/docs/reference/embeddings.md +++ b/docs/reference/embeddings.md @@ -8,6 +8,27 @@ The **Embeddings Module** provides a unified interface for generating vector representations of text. It abstracts away the complexity of different providers (OpenAI, HuggingFace, FastEmbed) and ensures consistent formatting for vector databases. +### What are Embeddings? + +**Embeddings** are numerical representations of text that capture semantic meaning. They convert words, sentences, or documents into dense vectors (arrays of numbers) in a high-dimensional space. Similar texts have similar vectors, enabling semantic search, similarity matching, and machine learning applications. + +### Why Use the Embeddings Module? + +- **Unified Interface**: Switch between different embedding providers without changing your code +- **Consistent Formatting**: All embeddings are normalized and formatted consistently for vector databases +- **Performance**: Optimized batch processing for high-throughput embedding generation +- **Flexibility**: Support for local models (FastEmbed, Sentence Transformers) and API-based models (OpenAI, Anthropic) +- **Vector DB Ready**: Automatic formatting for FAISS, Qdrant, Weaviate, and Milvus + +### How It Works + +The embeddings module uses a provider-based architecture: + +1. **EmbeddingGenerator**: Main orchestrator that manages the active model +2. **Provider Stores**: Backend implementations for each provider (FastEmbed, OpenAI, etc.) +3. **TextEmbedder**: Simplified interface focused on text-to-vector operations +4. **VectorEmbeddingManager**: Prepares embeddings for specific vector database formats + ### Key Capabilities
@@ -60,10 +81,10 @@ The main entry point for generating embeddings. It manages the active model and | Method | Description | |--------|-------------| -| `generate_embeddings(data, data_type="text")` | Generates an embedding for a single item. | -| `process_batch(items)` | Generates embeddings for a list of items (optimized). | -| `compare_embeddings(emb1, emb2)` | Calculates cosine similarity between two vectors. | -| `get_text_method()` | Returns the active embedding strategy. | +| `` `generate_embeddings(data, data_type="text")` `` | Generates an embedding for a single item | +| `` `process_batch(items)` `` | Generates embeddings for a list of items (optimized) | +| `` `compare_embeddings(emb1, emb2)` `` | Calculates cosine similarity between two vectors | +| `` `get_text_method()` `` | Returns the active embedding strategy | | `set_text_model(method, model_name, **config)` | Dynamically switches the text embedding model. | #### **Code Example** @@ -95,12 +116,12 @@ A specialized class focused purely on text-to-vector operations. It wraps the `E | Method | Description | |--------|-------------| -| `embed_text(text)` | Returns a list of floats for the input string. | -| `embed_batch(texts)` | Returns a list of lists (vectors) for the input strings. | -| `get_embedding_dimension()` | Returns the size of the output vector (e.g., 384, 768, 1536). | -| `set_model(method, model_name, **config)` | Switches the underlying embedding model. | -| `get_method()` | Returns the current method name. | -| `get_model_info()` | Returns details about the current model. | +| `` `embed_text(text)` `` | Returns a list of floats for the input string | +| `` `embed_batch(texts)` `` | Returns a list of lists (vectors) for the input strings | +| `` `get_embedding_dimension()` `` | Returns the size of the output vector (e.g., 384, 768, 1536) | +| `` `set_model(method, model_name, **config)` `` | Switches the underlying embedding model | +| `` `get_method()` `` | Returns the current method name | +| `` `get_model_info()` `` | Returns details about the current model | #### **Code Example** ```python @@ -199,11 +220,37 @@ providers = check_available_providers() if providers["fastembed"]: print("FastEmbed is ready!") if providers["openai"]: + print("OpenAI embeddings are available!") +if providers["sentence_transformers"]: + print("Sentence Transformers is installed!") + +# Check all providers +for provider, available in providers.items(): + status = "✓" if available else "✗" + print(f"{status} {provider}: {'Available' if available else 'Not installed'}") +``` + +This is useful for checking which embedding providers are available before initializing an embedder, especially when working in different environments. ## See Also - [Vector Store](vector_store.md) - Stores the generated embeddings - [Ingest](ingest.md) - Uses embeddings during processing ## Cookbook -- [Embedding Generation](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/12_Embedding_Generation.ipynb) -- [Vector Store](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb) + +Interactive tutorials to learn embeddings in practice: + +- **[Embedding Generation](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/12_Embedding_Generation.ipynb)**: Learn how to generate embeddings using different providers + - **Topics**: FastEmbed, OpenAI, Sentence Transformers, batch processing, normalization + - **Difficulty**: Intermediate + - **Use Cases**: Understanding embedding generation, choosing the right provider + +- **[Vector Store](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb)**: Set up and use vector stores for similarity search + - **Topics**: FAISS, Weaviate, Qdrant, hybrid search, metadata filtering + - **Difficulty**: Intermediate + - **Use Cases**: Storing and searching embeddings, building RAG systems + +- **[Advanced Vector Store and Search](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb)**: Advanced vector store operations and optimization + - **Topics**: Index optimization, hybrid search, performance tuning, namespace management + - **Difficulty**: Advanced + - **Use Cases**: Production deployments, performance optimization diff --git a/docs/reference/export.md b/docs/reference/export.md index 38648080..61b944ab 100644 --- a/docs/reference/export.md +++ b/docs/reference/export.md @@ -57,17 +57,51 @@ ## ⚙️ Algorithms Used ### Serialization Algorithms -- **RDF/XML Serialization**: W3C RDF/XML specification -- **Turtle Serialization**: Compact RDF format with prefix compression -- **JSON-LD Serialization**: JSON-based linked data with context -- **GraphML Generation**: XML-based graph format -- **Cypher Query Generation**: Neo4j query language generation + +**RDF/XML Serialization**: +- **Standard**: W3C RDF/XML specification +- **Format**: XML-based RDF representation +- **Use case**: Interoperability with XML-based systems + +**Turtle Serialization**: +- **Standard**: W3C Turtle specification +- **Format**: Compact RDF format with prefix compression +- **Use case**: Human-readable RDF representation + +**JSON-LD Serialization**: +- **Standard**: W3C JSON-LD specification +- **Format**: JSON-based linked data with context +- **Use case**: Web APIs and JavaScript applications + +**GraphML Generation**: +- **Format**: XML-based graph format +- **Use case**: Graph visualization tools (Gephi, Cytoscape) + +**Cypher Query Generation**: +- **Format**: Neo4j query language +- **Use case**: Direct import into Neo4j graph database ### Export Optimization -- **Streaming Export**: Memory-efficient export for large graphs -- **Batch Processing**: Chunked export with configurable batch sizes -- **Compression**: GZIP compression for large exports -- **Incremental Export**: Export only changed data + +**Streaming Export**: +- **Purpose**: Memory-efficient export for large graphs +- **How it works**: Processes data in chunks without loading entire graph into memory +- **Use case**: Exporting graphs with millions of nodes/edges + +**Batch Processing**: +- **Purpose**: Efficient large-scale data export +- **How it works**: Chunked export with configurable batch sizes +- **Use case**: Exporting multiple graphs or large datasets + +**Compression**: +- **Purpose**: Reduce file size for large exports +- **How it works**: GZIP compression for large exports +- **Use case**: Network transfer and storage optimization + +**Incremental Export**: +- **Purpose**: Export only changed data +- **How it works**: Tracks changes and exports only modified entities/relationships +- **Use case**: Regular updates and synchronization --- @@ -81,10 +115,10 @@ Export knowledge graphs to RDF formats (Turtle, RDF/XML, JSON-LD, N-Triples). | Method | Description | Algorithm | |--------|-------------|-----------| -| `export(graph, filename, format)` | Export to RDF format | RDF serialization with format-specific encoding | -| `export_knowledge_graph(kg, filename, format)` | Export knowledge graph | Knowledge graph to RDF conversion | -| `serialize(graph, format)` | Serialize to string | In-memory RDF generation | -| `validate_rdf(rdf_data)` | Validate RDF syntax | RDF schema validation | +| `` `export(graph, filename, format)` `` | Export to RDF format | RDF serialization with format-specific encoding | +| `` `export_knowledge_graph(kg, filename, format)` `` | Export knowledge graph | Knowledge graph to RDF conversion | +| `` `serialize(graph, format)` `` | Serialize to string | In-memory RDF generation | +| `` `validate_rdf(rdf_data)` `` | Validate RDF syntax | RDF schema validation | ### RDFSerializer @@ -94,9 +128,9 @@ RDF serialization engine for format conversion. | Method | Description | Algorithm | |--------|-------------|-----------| -| `serialize_to_turtle(rdf_data)` | Serialize to Turtle | Compact RDF format with prefix compression | -| `serialize_to_rdfxml(rdf_data)` | Serialize to RDF/XML | XML-based RDF format | -| `serialize_to_jsonld(rdf_data)` | Serialize to JSON-LD | JSON-based linked data format | +| `` `serialize_to_turtle(rdf_data)` `` | Serialize to Turtle | Compact RDF format with prefix compression | +| `` `serialize_to_rdfxml(rdf_data)` `` | Serialize to RDF/XML | XML-based RDF format | +| `` `serialize_to_jsonld(rdf_data)` `` | Serialize to JSON-LD | JSON-based linked data format | ### RDFValidator @@ -106,8 +140,8 @@ RDF validation engine for syntax and consistency checking. | Method | Description | Algorithm | |--------|-------------|-----------| -| `validate_rdf_syntax(rdf_data, format)` | Validate RDF syntax | Format-specific syntax validation | -| `check_rdf_consistency(rdf_data)` | Check consistency | Entity reference and structure validation | +| `` `validate_rdf_syntax(rdf_data, format)` `` | Validate RDF syntax | Format-specific syntax validation | +| `` `check_rdf_consistency(rdf_data)` `` | Check consistency | Entity reference and structure validation | ### NamespaceManager @@ -117,9 +151,9 @@ RDF namespace management and conflict resolution. | Method | Description | Algorithm | |--------|-------------|-----------| -| `extract_namespaces(rdf_data)` | Extract namespaces | Namespace discovery from RDF data | -| `generate_namespace_declarations(namespaces, format)` | Generate declarations | Format-specific namespace declaration | -| `resolve_conflicts(namespaces)` | Resolve conflicts | Prefix conflict resolution | +| `` `extract_namespaces(rdf_data)` `` | Extract namespaces | Namespace discovery from RDF data | +| `` `generate_namespace_declarations(namespaces, format)` `` | Generate declarations | Format-specific namespace declaration | +| `` `resolve_conflicts(namespaces)` `` | Resolve conflicts | Prefix conflict resolution | **Supported RDF Formats:** @@ -169,10 +203,10 @@ Export knowledge graphs to JSON formats including JSON-LD, Cytoscape.js, and D3. | Method | Description | Algorithm | |--------|-------------|-----------| -| `export(graph, filename, format)` | Export to JSON | JSON serialization with schema | -| `export_nodes(graph)` | Export nodes only | Node extraction and serialization | -| `export_edges(graph)` | Export edges only | Edge extraction and serialization | -| `export_cytoscape(graph)` | Export Cytoscape.js format | Cytoscape JSON generation | +| `` `export(graph, filename, format)` `` | Export to JSON | JSON serialization with schema | +| `` `export_nodes(graph)` `` | Export nodes only | Node extraction and serialization | +| `` `export_edges(graph)` `` | Export edges only | Edge extraction and serialization | +| `` `export_cytoscape(graph)` `` | Export Cytoscape.js format | Cytoscape JSON generation | | `export_d3(graph)` | Export D3.js format | D3 force-directed graph format | **JSON Formats:** @@ -210,11 +244,11 @@ Export to graph visualization formats (GraphML, GEXF, DOT, Pajek). | Method | Description | Algorithm | |--------|-------------|-----------| -| `export(graph, filename, format)` | Export to graph format | Format-specific serialization | -| `to_graphml(graph, filename)` | Export to GraphML | XML-based graph format | -| `to_gexf(graph, filename)` | Export to GEXF | Gephi exchange format | -| `to_dot(graph, filename)` | Export to DOT | Graphviz format | -| `to_pajek(graph, filename)` | Export to Pajek | Pajek network format | +| `` `export(graph, filename, format)` `` | Export to graph format | Format-specific serialization | +| `` `to_graphml(graph, filename)` `` | Export to GraphML | XML-based graph format | +| `` `to_gexf(graph, filename)` `` | Export to GEXF | Gephi exchange format | +| `` `to_dot(graph, filename)` `` | Export to DOT | Graphviz format | +| `` `to_pajek(graph, filename)` `` | Export to Pajek | Pajek network format | **Graph Formats:** @@ -250,9 +284,9 @@ Export to LPG (Labeled Property Graph) format for Neo4j, Memgraph, and similar d | Method | Description | Algorithm | |--------|-------------|-----------| -| `export(knowledge_graph, file_path)` | Export to LPG format | Cypher query generation | -| `export_knowledge_graph(kg, file_path)` | Export knowledge graph | Knowledge graph to Cypher conversion | -| `generate_cypher(kg)` | Generate Cypher queries | CREATE/MERGE statement generation | +| `` `export(knowledge_graph, file_path)` `` | Export to LPG format | Cypher query generation | +| `` `export_knowledge_graph(kg, file_path)` `` | Export knowledge graph | Knowledge graph to Cypher conversion | +| `` `generate_cypher(kg)` `` | Generate Cypher queries | CREATE/MERGE statement generation | **Cypher Generation:** ```cypher @@ -289,10 +323,10 @@ Export to CSV format for spreadsheets and database imports. | Method | Description | Algorithm | |--------|-------------|-----------| -| `export_nodes(graph, filename)` | Export nodes to CSV | Node flattening and CSV writing | -| `export_edges(graph, filename)` | Export edges to CSV | Edge list CSV generation | -| `export_combined(graph, prefix)` | Export nodes + edges | Separate CSV files | -| `flatten_properties(properties)` | Flatten nested properties | Recursive property flattening | +| `` `export_nodes(graph, filename)` `` | Export nodes to CSV | Node flattening and CSV writing | +| `` `export_edges(graph, filename)` `` | Export edges to CSV | Edge list CSV generation | +| `` `export_combined(graph, prefix)` `` | Export nodes + edges | Separate CSV files | +| `` `flatten_properties(properties)` `` | Flatten nested properties | Recursive property flattening | **CSV Formats:** - **Nodes CSV**: id, label, properties (flattened) @@ -325,10 +359,10 @@ Export vector embeddings to various formats. | Method | Description | Algorithm | |--------|-------------|-----------| -| `export(vectors, filename, format)` | Export vectors | Format-specific vector serialization | -| `export_numpy(vectors, filename)` | Export to NumPy | .npy format | -| `export_hdf5(vectors, filename)` | Export to HDF5 | Hierarchical data format | -| `export_parquet(vectors, filename)` | Export to Parquet | Columnar storage format | +| `` `export(vectors, filename, format)` `` | Export vectors | Format-specific vector serialization | +| `` `export_numpy(vectors, filename)` `` | Export to NumPy | .npy format | +| `` `export_hdf5(vectors, filename)` `` | Export to HDF5 | Hierarchical data format | +| `` `export_parquet(vectors, filename)` `` | Export to Parquet | Columnar storage format | **Example:** @@ -349,9 +383,9 @@ Export ontologies to OWL format (OWL/XML, Turtle). | Method | Description | Algorithm | |--------|-------------|-----------| -| `export(ontology, filename, format)` | Export ontology | OWL serialization | -| `export_classes(classes, filename)` | Export classes | Class definition export | -| `export_properties(properties, filename)` | Export properties | Property definition export | +| `` `export(ontology, filename, format)` `` | Export ontology | OWL serialization | +| `` `export_classes(classes, filename)` `` | Export classes | Class definition export | +| `` `export_properties(properties, filename)` `` | Export properties | Property definition export | **Example:** @@ -372,8 +406,8 @@ Export semantic networks to YAML format. | Method | Description | Algorithm | |--------|-------------|-----------| -| `export(semantic_network, filename)` | Export semantic network | YAML serialization | -| `export_semantic_network(semantic_network)` | Export to string | In-memory YAML generation | +| `` `export(semantic_network, filename)` `` | Export semantic network | YAML serialization | +| `` `export_semantic_network(semantic_network)` `` | Export to string | In-memory YAML generation | **Example:** @@ -394,7 +428,7 @@ Export ontology schemas to YAML format. | Method | Description | Algorithm | |--------|-------------|-----------| -| `export_ontology_schema(ontology, filename)` | Export ontology schema | YAML schema serialization | +| `` `export_ontology_schema(ontology, filename)` `` | Export ontology schema | YAML schema serialization | **Example:** @@ -415,8 +449,8 @@ Generate reports in multiple formats (HTML, Markdown, JSON, Text). | Method | Description | Algorithm | |--------|-------------|-----------| -| `generate_report(data, filename, format)` | Generate report | Template-based report generation | -| `generate_quality_report(metrics, filename, format)` | Generate quality report | Quality metrics aggregation | +| `` `generate_report(data, filename, format)` `` | Generate report | Template-based report generation | +| `` `generate_quality_report(metrics, filename, format)` `` | Generate quality report | Quality metrics aggregation | **Example:** @@ -437,11 +471,11 @@ Registry for custom export methods. | Method | Description | Algorithm | |--------|-------------|-----------| -| `register(task, name, method_func)` | Register method | Dictionary-based registration | -| `get(task, name)` | Get method | Hash-based lookup | -| `list_all(task)` | List methods | Method discovery | -| `unregister(task, name)` | Unregister method | Method removal | -| `clear(task)` | Clear methods | Registry cleanup | +| `` `register(task, name, method_func)` `` | Register method | Dictionary-based registration | +| `` `get(task, name)` `` | Get method | Hash-based lookup | +| `` `list_all(task)` `` | List methods | Method discovery | +| `` `unregister(task, name)` `` | Unregister method | Method removal | +| `` `clear(task)` `` | Clear methods | Registry cleanup | **Global Instance:** - `method_registry`: Global method registry instance @@ -466,10 +500,10 @@ Configuration manager for export module. | Method | Description | Algorithm | |--------|-------------|-----------| -| `set(key, value)` | Set configuration | Configuration storage | -| `get(key, default)` | Get configuration | Configuration retrieval | -| `set_method_config(task, **config)` | Set method config | Method-specific configuration | -| `get_method_config(task)` | Get method config | Method configuration retrieval | +| `` `set(key, value)` `` | Set configuration | Configuration storage | +| `` `get(key, default)` `` | Get configuration | Configuration retrieval | +| `` `set_method_config(task, **config)` `` | Set method config | Method-specific configuration | +| `` `get_method_config(task)` `` | Get method config | Method configuration retrieval | **Global Instance:** - `export_config`: Global export configuration instance @@ -495,22 +529,22 @@ config = ExportConfig(config_file="config.yaml") | Function | Description | Format | |----------|-------------|--------| -| `export_rdf(data, file_path, format)` | Export to RDF | turtle, rdfxml, jsonld, ntriples, n3 | -| `export_json(data, file_path, format)` | Export to JSON | json, json-ld | -| `export_csv(data, file_path)` | Export to CSV | csv | -| `export_graph(graph_data, file_path, format)` | Export to graph format | graphml, gexf, dot | -| `export_yaml(data, file_path, method)` | Export to YAML | semantic_network, schema | -| `export_owl(ontology, file_path, format)` | Export to OWL | owl-xml, turtle | -| `export_vector(vectors, file_path, format)` | Export vectors | json, numpy, binary, faiss | -| `export_lpg(kg, file_path, method)` | Export to LPG | cypher, lpg | -| `generate_report(data, file_path, format)` | Generate report | html, markdown, json, text | +| `` `export_rdf(data, file_path, format)` `` | Export to RDF | turtle, rdfxml, jsonld, ntriples, n3 | +| `` `export_json(data, file_path, format)` `` | Export to JSON | json, json-ld | +| `` `export_csv(data, file_path)` `` | Export to CSV | csv | +| `` `export_graph(graph_data, file_path, format)` `` | Export to graph format | graphml, gexf, dot | +| `` `export_yaml(data, file_path, method)` `` | Export to YAML | semantic_network, schema | +| `` `export_owl(ontology, file_path, format)` `` | Export to OWL | owl-xml, turtle | +| `` `export_vector(vectors, file_path, format)` `` | Export vectors | json, numpy, binary, faiss | +| `` `export_lpg(kg, file_path, method)` `` | Export to LPG | cypher, lpg | +| `` `generate_report(data, file_path, format)` `` | Generate report | html, markdown, json, text | ### Registry Functions | Function | Description | |----------|-------------| -| `get_export_method(task, name)` | Get registered export method | -| `list_available_methods(task)` | List all available methods | +| `` `get_export_method(task, name)` `` | Get registered export method | +| `` `list_available_methods(task)` `` | List all available methods | **Example:** @@ -596,17 +630,17 @@ export: ## Cookbook -- [Export](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/15_Export.ipynb) -- [Multi-Format Export](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/05_Multi_Format_Export.ipynb) +Interactive tutorials to learn export capabilities: +- **[Export](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/15_Export.ipynb)**: Export knowledge graphs to various formats + - **Topics**: RDF, JSON, CSV, OWL export, format conversion + - **Difficulty**: Intermediate + - **Use Cases**: Data export, format conversion, interoperability -## Overview - -- **Multi-Format Export**: Support for 10+ export formats -- **Graph Database Export**: Direct export to Neo4j, ArangoDB, Memgraph -- **RDF Serialization**: W3C-compliant RDF formats -- **Custom Serializers**: Extensible serialization framework -- **Batch Export**: Efficient large-scale data export +- **[Multi-Format Export](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/05_Multi_Format_Export.ipynb)**: Exporting to RDF, OWL, JSON-LD, and NetworkX formats + - **Topics**: Serialization, interoperability, multiple formats, batch export + - **Difficulty**: Intermediate + - **Use Cases**: Multi-format export, data interoperability --- @@ -636,11 +670,11 @@ export: | Method | Description | Algorithm | |--------|-------------|-----------| -| `export(graph, filename, format)` | Export to RDF format | RDF serialization with format-specific encoding | -| `serialize(graph, format)` | Serialize to string | In-memory RDF generation | -| `validate(rdf_data)` | Validate RDF syntax | RDF schema validation | -| `add_namespace(prefix, uri)` | Add namespace | Prefix registration | -| `set_base_uri(uri)` | Set base URI | Base URI configuration | +| `` `export(graph, filename, format)` `` | Export to RDF format | RDF serialization with format-specific encoding | +| `` `serialize(graph, format)` `` | Serialize to string | In-memory RDF generation | +| `` `validate(rdf_data)` `` | Validate RDF syntax | RDF schema validation | +| `` `add_namespace(prefix, uri)` `` | Add namespace | Prefix registration | +| `` `set_base_uri(uri)` `` | Set base URI | Base URI configuration | **Supported RDF Formats:** @@ -689,10 +723,10 @@ exporter.export( | Method | Description | Algorithm | |--------|-------------|-----------| -| `export(graph, filename, format)` | Export to JSON | JSON serialization with schema | -| `export_nodes(graph)` | Export nodes only | Node extraction and serialization | -| `export_edges(graph)` | Export edges only | Edge extraction and serialization | -| `export_cytoscape(graph)` | Export Cytoscape.js format | Cytoscape JSON generation | +| `` `export(graph, filename, format)` `` | Export to JSON | JSON serialization with schema | +| `` `export_nodes(graph)` `` | Export nodes only | Node extraction and serialization | +| `` `export_edges(graph)` `` | Export edges only | Edge extraction and serialization | +| `` `export_cytoscape(graph)` `` | Export Cytoscape.js format | Cytoscape JSON generation | | `export_d3(graph)` | Export D3.js format | D3 force-directed graph format | **JSON Formats:** @@ -729,11 +763,11 @@ exporter.export_cytoscape(kg, "cytoscape.json") | Method | Description | Algorithm | |--------|-------------|-----------| -| `export(graph, filename, format)` | Export to graph format | Format-specific serialization | -| `to_graphml(graph, filename)` | Export to GraphML | XML-based graph format | -| `to_gexf(graph, filename)` | Export to GEXF | Gephi exchange format | -| `to_dot(graph, filename)` | Export to DOT | Graphviz format | -| `to_pajek(graph, filename)` | Export to Pajek | Pajek network format | +| `` `export(graph, filename, format)` `` | Export to graph format | Format-specific serialization | +| `` `to_graphml(graph, filename)` `` | Export to GraphML | XML-based graph format | +| `` `to_gexf(graph, filename)` `` | Export to GEXF | Gephi exchange format | +| `` `to_dot(graph, filename)` `` | Export to DOT | Graphviz format | +| `` `to_pajek(graph, filename)` `` | Export to Pajek | Pajek network format | **Graph Formats:** @@ -768,11 +802,11 @@ exporter.to_dot(kg, "graph.dot") | Method | Description | Algorithm | |--------|-------------|-----------| -| `export(graph, uri, username, password)` | Export to Neo4j | Cypher query execution | -| `generate_cypher(graph)` | Generate Cypher queries | CREATE/MERGE statement generation | -| `batch_import(graph, batch_size)` | Batch import | Chunked Cypher execution | -| `create_indexes(properties)` | Create indexes | Index creation for performance | -| `create_constraints(constraints)` | Create constraints | Uniqueness constraint creation | +| `` `export(graph, uri, username, password)` `` | Export to Neo4j | Cypher query execution | +| `` `generate_cypher(graph)` `` | Generate Cypher queries | CREATE/MERGE statement generation | +| `` `batch_import(graph, batch_size)` `` | Batch import | Chunked Cypher execution | +| `` `create_indexes(properties)` `` | Create indexes | Index creation for performance | +| `` `create_constraints(constraints)` `` | Create constraints | Uniqueness constraint creation | **Cypher Generation:** ```cypher @@ -813,10 +847,10 @@ print(cypher_queries) | Method | Description | Algorithm | |--------|-------------|-----------| -| `export_nodes(graph, filename)` | Export nodes to CSV | Node flattening and CSV writing | -| `export_edges(graph, filename)` | Export edges to CSV | Edge list CSV generation | -| `export_combined(graph, prefix)` | Export nodes + edges | Separate CSV files | -| `flatten_properties(properties)` | Flatten nested properties | Recursive property flattening | +| `` `export_nodes(graph, filename)` `` | Export nodes to CSV | Node flattening and CSV writing | +| `` `export_edges(graph, filename)` `` | Export edges to CSV | Edge list CSV generation | +| `` `export_combined(graph, prefix)` `` | Export nodes + edges | Separate CSV files | +| `` `flatten_properties(properties)` `` | Flatten nested properties | Recursive property flattening | **CSV Formats:** - **Nodes CSV**: id, label, properties (flattened) @@ -848,10 +882,10 @@ exporter.export_combined(kg, prefix="graph") | Method | Description | Algorithm | |--------|-------------|-----------| -| `export(embeddings, filename, format)` | Export vectors | Format-specific vector serialization | -| `export_numpy(embeddings, filename)` | Export to NumPy | .npy format | -| `export_hdf5(embeddings, filename)` | Export to HDF5 | Hierarchical data format | -| `export_parquet(embeddings, filename)` | Export to Parquet | Columnar storage format | +| `` `export(embeddings, filename, format)` `` | Export vectors | Format-specific vector serialization | +| `` `export_numpy(embeddings, filename)` `` | Export to NumPy | .npy format | +| `` `export_hdf5(embeddings, filename)` `` | Export to HDF5 | Hierarchical data format | +| `` `export_parquet(embeddings, filename)` `` | Export to Parquet | Columnar storage format | --- diff --git a/docs/reference/graph_store.md b/docs/reference/graph_store.md index 85e424a0..c436ca55 100644 --- a/docs/reference/graph_store.md +++ b/docs/reference/graph_store.md @@ -57,18 +57,27 @@ ## ⚙️ Algorithms Used ### Query Execution -- **Cypher Translation**: Adapting queries for specific backend nuances (though most support OpenCypher). -- **Query Optimization**: Index utilization and execution plan analysis. + +The module provides efficient query execution: + +- **Cypher Translation**: Adapting queries for specific backend nuances (though most support OpenCypher) +- **Query Optimization**: Index utilization and execution plan analysis ### Graph Analytics -- **PageRank**: Measuring node importance based on incoming links. -- **Louvain Modularity**: Detecting communities by optimizing modularity. -- **Shortest Path**: Dijkstra/A* for finding optimal routes. -- **Jaccard Similarity**: Measuring node similarity based on shared neighbors. + +Built-in graph analytics algorithms include: + +- **PageRank**: Measuring node importance based on incoming links +- **Louvain Modularity**: Detecting communities by optimizing modularity +- **Shortest Path**: Dijkstra/A* for finding optimal routes +- **Jaccard Similarity**: Measuring node similarity based on shared neighbors ### Bulk Operations -- **Chunking**: Splitting large datasets into optimal batch sizes (e.g., 5000 records) to prevent memory overflow. -- **Parallel Loading**: Concurrent batch insertion (backend dependent). + +Efficient bulk loading capabilities: + +- **Chunking**: Splitting large datasets into optimal batch sizes (e.g., `` `5000` `` records) to prevent memory overflow +- **Parallel Loading**: Concurrent batch insertion (backend dependent) --- @@ -419,4 +428,9 @@ subgraph = graph_store.execute_query(query, parameters={"ids": node_ids}) ## Cookbook -- [Graph Store](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/09_Graph_Store.ipynb) +Interactive tutorials to learn graph storage: + +- **[Graph Store](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/09_Graph_Store.ipynb)**: Persist knowledge graphs in Neo4j or FalkorDB + - **Topics**: Neo4j, FalkorDB, Cypher, persistence, graph databases + - **Difficulty**: Intermediate + - **Use Cases**: Persistent storage, production deployments, graph database integration diff --git a/docs/reference/ingest.md b/docs/reference/ingest.md index 392cdad8..55c1cb98 100644 --- a/docs/reference/ingest.md +++ b/docs/reference/ingest.md @@ -6,6 +6,36 @@ ## 🎯 Overview +The **Ingest Module** is the entry point for loading data into Semantica. It provides universal data ingestion from files, web content, feeds, streams, repositories, emails, databases, and more. + +### What is Data Ingestion? + +**Data ingestion** is the process of loading data from various sources into Semantica for processing. The ingest module handles: +- **File Systems**: Local files, cloud storage (S3, GCS, Azure) +- **Web Content**: Websites, RSS feeds, APIs +- **Streams**: Real-time data from Kafka, RabbitMQ, etc. +- **Databases**: SQL and NoSQL databases +- **Repositories**: Git repositories (GitHub, GitLab) +- **Email**: IMAP, POP3 servers +- **MCP**: Model Context Protocol servers + +### Why Use the Ingest Module? + +- **Universal Support**: Handle multiple data formats and sources +- **Automatic Detection**: Automatically detect file types and content +- **Streaming Support**: Process real-time data streams +- **Cloud Integration**: Direct support for cloud storage +- **Rate Limiting**: Built-in rate limiting for web crawling +- **Error Handling**: Robust error handling and retry logic + +### How It Works + +1. **Source Detection**: Automatically detect the type of data source +2. **Connection**: Establish connection to the source (file system, web, database, etc.) +3. **Loading**: Load data from the source +4. **Format Detection**: Detect the format of the loaded data +5. **Output**: Return data in a standardized format for processing +
- :material-file-document-multiple:{ .lg .middle } **File Ingestion** @@ -247,5 +277,14 @@ ingestor.monitor( ## Cookbook -- [Data Ingestion](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb) -- [Multi-Source Data Integration](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb) +Interactive tutorials to learn data ingestion: + +- **[Data Ingestion](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/02_Data_Ingestion.ipynb)**: Comprehensive guide to data ingestion from multiple sources + - **Topics**: File ingestion, web scraping, database integration, streams, feeds, repositories, email, MCP + - **Difficulty**: Beginner + - **Use Cases**: Loading data from various sources, understanding ingestion capabilities + +- **[Multi-Source Data Integration](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/06_Multi_Source_Data_Integration.ipynb)**: Merge data from disparate sources into a unified graph + - **Topics**: Entity resolution, merging, fusion, multi-source integration + - **Difficulty**: Advanced + - **Use Cases**: Combining data from multiple sources, data fusion diff --git a/docs/reference/kg.md b/docs/reference/kg.md index 2c1a05c9..675e795e 100644 --- a/docs/reference/kg.md +++ b/docs/reference/kg.md @@ -6,6 +6,33 @@ ## 🎯 Overview +The **Knowledge Graph (KG) Module** is the core module for building, managing, and analyzing knowledge graphs. It transforms extracted entities and relationships into structured, queryable knowledge graphs. + +### What is a Knowledge Graph? + +A **knowledge graph** is a structured representation of information where: +- **Nodes** represent entities (people, organizations, concepts, etc.) +- **Edges** represent relationships between entities +- **Properties** store additional information about nodes and edges + +Knowledge graphs enable semantic queries, relationship traversal, and complex reasoning that traditional databases cannot handle. + +### Why Use the KG Module? + +- **Structured Knowledge**: Transform unstructured data into structured, queryable graphs +- **Entity Resolution**: Automatically merge duplicate entities using fuzzy matching +- **Temporal Support**: Track how knowledge changes over time +- **Graph Analytics**: Analyze graph structure, importance, and communities +- **Provenance Tracking**: Know where every piece of information came from + +### How It Works + +1. **Input**: Entities and relationships from semantic extraction +2. **Entity Resolution**: Merge similar entities to avoid duplicates +3. **Graph Construction**: Build nodes and edges from entities and relationships +4. **Enrichment**: Add temporal information, provenance, and metadata +5. **Analysis**: Perform graph analytics (centrality, communities, etc.) +
- :material-graph-outline:{ .lg .middle } **KG Construction** @@ -80,8 +107,8 @@ Constructs the KG from raw data. | Method | Description | |--------|-------------| -| `build(sources)` | Build graph from inputs | -| `merge_entities()` | Merge duplicate entities during building | +| `` `build(sources)` `` | Build graph from inputs | +| `` `merge_entities()` `` | Merge duplicate entities during building | **Example:** @@ -100,8 +127,8 @@ Runs analytical algorithms. | Method | Description | |--------|-------------| -| `centrality(method)` | Calculate importance | -| `communities(method)` | Find clusters | +| `` `centrality(method)` `` | Calculate importance | +| `` `communities(method)` `` | Find clusters | ### TemporalGraphQuery @@ -111,8 +138,8 @@ Queries time-aware graphs. | Method | Description | |--------|-------------| -| `at_time(timestamp)` | Graph state at T | -| `during(start, end)` | Graph state in interval | +| `` `at_time(timestamp)` `` | Graph state at T | +| `` `during(start, end)` `` | Graph state in interval | --- @@ -200,9 +227,31 @@ print(f"New nodes since 2020: {len(diff.nodes)}") ## Cookbook -- [Building Knowledge Graphs](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb) -- [Your First Knowledge Graph](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb) -- [Graph Analytics](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/10_Graph_Analytics.ipynb) -- [Advanced Graph Analytics](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/02_Advanced_Graph_Analytics.ipynb) -- [Temporal Knowledge Graphs](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb) -- [Deduplication Module](deduplication.md) - Advanced deduplication +Interactive tutorials to learn knowledge graph construction and analysis: + +- **[Building Knowledge Graphs](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/07_Building_Knowledge_Graphs.ipynb)**: Learn the fundamentals of building knowledge graphs + - **Topics**: Graph construction, entity resolution, relationship mapping + - **Difficulty**: Beginner + - **Use Cases**: Understanding graph construction basics + +- **[Your First Knowledge Graph](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/08_Your_First_Knowledge_Graph.ipynb)**: Build your first knowledge graph from scratch + - **Topics**: Entity extraction, relationship extraction, graph construction, visualization + - **Difficulty**: Beginner + - **Use Cases**: First-time users, quick start + +- **[Graph Analytics](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/10_Graph_Analytics.ipynb)**: Analyze knowledge graphs with centrality and community detection + - **Topics**: Centrality measures, community detection, graph metrics + - **Difficulty**: Intermediate + - **Use Cases**: Understanding graph structure, finding important nodes + +- **[Advanced Graph Analytics](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/02_Advanced_Graph_Analytics.ipynb)**: Advanced graph analysis techniques + - **Topics**: PageRank, Louvain algorithm, shortest path, graph mining + - **Difficulty**: Advanced + - **Use Cases**: Complex graph analysis, research applications + +- **[Temporal Knowledge Graphs](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/10_Temporal_Knowledge_Graphs.ipynb)**: Model and query data that changes over time + - **Topics**: Time series, temporal logic, temporal queries, graph evolution + - **Difficulty**: Advanced + - **Use Cases**: Tracking changes over time, temporal reasoning + +- **[Deduplication Module](deduplication.md)**: Advanced deduplication techniques for entity resolution diff --git a/docs/reference/llms.md b/docs/reference/llms.md index b2387a24..ce0a3a17 100644 --- a/docs/reference/llms.md +++ b/docs/reference/llms.md @@ -4,7 +4,34 @@ The `semantica.llms` module provides a unified interface for LLM providers, supp ## Overview -The LLM Providers module abstracts away provider-specific details, providing a consistent interface for text generation across multiple LLM providers. This enables easy switching between providers and integration with GraphRAG reasoning features. +The **LLM Providers Module** provides a unified interface for Large Language Model (LLM) providers. It abstracts away provider-specific details, enabling you to switch between different LLM providers without changing your code. + +### What is the LLM Providers Module? + +The LLM Providers module provides: + +- **Unified LLM APIs**: Single interface for Groq, OpenAI, HuggingFace, and LiteLLM (100+ models) +- **Easy Provider Switching**: Change providers without code changes +- **Multiple Model Support**: Access to 100+ LLMs through LiteLLM +- **GraphRAG Integration**: Seamless integration with GraphRAG reasoning features +- **Structured Output**: Generate structured data from LLM responses + +### Why Use the LLM Providers Module? + +- **Flexibility**: Switch between providers based on cost, speed, or capability +- **Consistency**: Same API regardless of provider +- **Local Models**: Support for local HuggingFace models +- **Fast Inference**: Groq for ultra-fast inference +- **Enterprise Models**: Access to OpenAI, Anthropic, and other enterprise providers + +### How It Works + +The LLM Providers module follows a simple workflow: + +1. **Provider Selection**: Choose a provider (Groq, OpenAI, HuggingFace, LiteLLM) +2. **Model Configuration**: Configure model name, API keys, and parameters +3. **Text Generation**: Generate text using a consistent API +4. **Structured Output**: Optionally generate structured data (JSON, entities, etc.) ## Quick Start @@ -269,6 +296,20 @@ pip install transformers torch pip install litellm ``` +## Cookbook + +Interactive tutorials that use LLM providers: + +- **[Advanced Extraction](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb)**: Custom extractors and LLM-based extraction + - **Topics**: LLM extraction, custom models, complex pattern matching + - **Difficulty**: Advanced + - **Use Cases**: Domain-specific extraction, complex schemas + +- **[GraphRAG Complete](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)**: Production-ready GraphRAG system using LLMs + - **Topics**: GraphRAG, LLM integration, hybrid retrieval + - **Difficulty**: Advanced + - **Use Cases**: Building AI applications with knowledge graphs + ## See Also - [Context Module](context.md) - GraphRAG with multi-hop reasoning diff --git a/docs/reference/normalize.md b/docs/reference/normalize.md index de6a9c31..8b8084ad 100644 --- a/docs/reference/normalize.md +++ b/docs/reference/normalize.md @@ -59,34 +59,59 @@ ## ⚙️ Algorithms Used ### Text Normalization + +**Purpose**: Clean and standardize text for consistent processing. + +**How it works**: + - **Unicode Normalization**: NFC, NFD, NFKC, NFKD forms using Unicode standard -- **Whitespace Normalization**: Regex-based cleanup (`\s+` → single space) +- **Whitespace Normalization**: Regex-based cleanup (`` `\s+` `` → single space) - **Case Folding**: Locale-aware case normalization (Unicode case folding) - **Diacritic Removal**: Unicode decomposition and combining character removal - **Punctuation Handling**: Smart punctuation normalization preserving sentence structure ### Entity Normalization -- **Fuzzy Matching**: Levenshtein distance with configurable threshold (default: 0.85) + +**Purpose**: Standardize entity names and resolve variations to canonical forms. + +**How it works**: + +- **Fuzzy Matching**: Levenshtein distance with configurable threshold (default: `` `0.85` ``) - **Phonetic Matching**: Soundex and Metaphone algorithms for name variants - **Abbreviation Expansion**: Dictionary-based expansion with context awareness - **Canonical Form Selection**: Frequency-based or confidence-based selection - **Entity Linking**: Hash-based entity ID generation for cross-document linking ### Date/Time Normalization + +**Purpose**: Parse and standardize date/time formats to ISO 8601. + +**How it works**: + - **Parsing**: dateutil parser with 100+ format support - **Timezone Handling**: pytz for timezone conversion and DST handling -- **Standardization**: ISO 8601 format output (YYYY-MM-DDTHH:MM:SSZ) +- **Standardization**: ISO 8601 format output (`` `YYYY-MM-DDTHH:MM:SSZ` ``) - **Relative Date Resolution**: Convert "yesterday", "last week" to absolute dates - **Fuzzy Date Parsing**: Handle incomplete dates (e.g., "March 2024") ### Number Normalization -- **Numeric Parsing**: Handle various formats (1,000.00, 1.000,00, 1 000.00) + +**Purpose**: Standardize numeric values, units, and measurements. + +**How it works**: + +- **Numeric Parsing**: Handle various formats (`` `1,000.00` ``, `` `1.000,00` ``, `` `1 000.00` ``) - **Unit Conversion**: Standardize units (km → meters, lbs → kg) - **Scientific Notation**: Parse and normalize scientific notation - **Percentage Handling**: Normalize percentage representations - **Currency Normalization**: Standardize currency symbols and amounts ### Language Detection + +**Purpose**: Automatically detect document language with confidence scoring. + +**How it works**: + - **N-gram Analysis**: Character and word n-gram frequency analysis - **Statistical Models**: Language-specific statistical models - **Confidence Scoring**: Probability-based confidence scores @@ -104,10 +129,10 @@ Main text normalization orchestrator with comprehensive cleaning capabilities. | Method | Description | |--------|-------------| -| `normalize_text(text, ...)` | Normalize single text using full pipeline | -| `clean_text(text, ...)` | Clean text (HTML removal, sanitization) | -| `standardize_format(text, format_type)` | Standardize formatting (standard/compact/preserve) | -| `process_batch(texts, ...)` | Batch normalize multiple texts | +| `` `normalize_text(text, ...)` `` | Normalize single text using full pipeline | +| `` `clean_text(text, ...)` `` | Clean text (HTML removal, sanitization) | +| `` `standardize_format(text, format_type)` `` | Standardize formatting (standard/compact/preserve) | +| `` `process_batch(texts, ...)` `` | Batch normalize multiple texts | **Example:** @@ -137,10 +162,10 @@ Standardize entity names and resolve variations to canonical forms. | Method | Description | |--------|-------------| -| `normalize_entity(name, ...)` | Normalize entity name to canonical form | -| `resolve_aliases(name, ...)` | Resolve aliases via alias map | -| `disambiguate_entity(name, ...)` | Disambiguate using context and candidates | -| `link_entities(names, ...)` | Link a list of names to canonical forms | +| `` `normalize_entity(name, ...)` `` | Normalize entity name to canonical form | +| `` `resolve_aliases(name, ...)` `` | Resolve aliases via alias map | +| `` `disambiguate_entity(name, ...)` `` | Disambiguate using context and candidates | +| `` `link_entities(names, ...)` `` | Link a list of names to canonical forms | **Configuration Options:** @@ -180,9 +205,9 @@ Parse and standardize date/time formats to ISO 8601. | Method | Description | |--------|-------------| -| `normalize_date(date_str, ...)` | Parse and normalize date | -| `normalize_time(time_str, ...)` | Normalize time-only strings | -| `parse_temporal_expression(expr)` | Parse date ranges and temporal phrases | +| `` `normalize_date(date_str, ...)` `` | Parse and normalize date | +| `` `normalize_time(time_str, ...)` `` | Normalize time-only strings | +| `` `parse_temporal_expression(expr)` `` | Parse date ranges and temporal phrases | **Configuration Options:** @@ -220,10 +245,10 @@ Standardize numeric values, units, and measurements. | Method | Description | |--------|-------------| -| `normalize_number(input, ...)` | Parse and normalize number | -| `normalize_quantity(quantity, ...)` | Parse value with unit | -| `convert_units(value, from_unit, to_unit)` | Convert units | -| `process_currency(text, ...)` | Parse currency amount and code | +| `` `normalize_number(input, ...)` `` | Parse and normalize number | +| `` `normalize_quantity(quantity, ...)` `` | Parse value with unit | +| `` `convert_units(value, from_unit, to_unit)` `` | Convert units | +| `` `process_currency(text, ...)` `` | Parse currency amount and code | **Example:** @@ -250,10 +275,10 @@ Detect document language with confidence scoring. | Method | Description | |--------|-------------| -| `detect(text)` | Detect language | -| `detect_with_confidence(text)` | Detect with confidence score | -| `detect_multiple(text, top_n)` | List top-N candidate languages | -| `detect_batch(texts)` | Batch language detection | +| `` `detect(text)` `` | Detect language | +| `` `detect_with_confidence(text)` `` | Detect with confidence score | +| `` `detect_multiple(text, top_n)` `` | List top-N candidate languages | +| `` `detect_batch(texts)` `` | Batch language detection | **Example:** @@ -568,4 +593,9 @@ normalized_docs = normalizer.process_batch(documents) ## Cookbook -- [Data Normalization](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/04_Data_Normalization.ipynb) +Interactive tutorials to learn data normalization: + +- **[Data Normalization](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/04_Data_Normalization.ipynb)**: Pipelines for cleaning, normalizing, and preparing text + - **Topics**: Text cleaning, Unicode, formatting, language detection, entity normalization + - **Difficulty**: Beginner + - **Use Cases**: Data preprocessing, text cleaning, standardization diff --git a/docs/reference/ontology.md b/docs/reference/ontology.md index 60ad6b4d..f0598903 100644 --- a/docs/reference/ontology.md +++ b/docs/reference/ontology.md @@ -44,16 +44,22 @@ ## ⚙️ Algorithms Used ### 6-Stage Generation Pipeline -1. **Semantic Network Parsing**: Extract concepts and patterns from raw entity/relationship data. -2. **YAML-to-Definition**: Transform patterns into intermediate class definitions. -3. **Definition-to-Types**: Map definitions to OWL types (`owl:Class`, `owl:ObjectProperty`). -4. **Hierarchy Generation**: Build taxonomy trees using transitive closure and cycle detection. -5. **TTL Generation**: Serialize to Turtle format using `rdflib`. + +The ontology generation process follows these stages: + +1. **Semantic Network Parsing**: Extract concepts and patterns from raw entity/relationship data +2. **YAML-to-Definition**: Transform patterns into intermediate class definitions +3. **Definition-to-Types**: Map definitions to OWL types (`` `owl:Class` ``, `` `owl:ObjectProperty` ``) +4. **Hierarchy Generation**: Build taxonomy trees using transitive closure and cycle detection +5. **TTL Generation**: Serialize to Turtle format using `` `rdflib` `` ### Inference Algorithms -- **Class Inference**: Clustering entities by type and attribute similarity. -- **Property Inference**: Determining domain/range based on connected entity types. -- **Hierarchy Inference**: `A is_a B` detection based on subset relationships. + +The module uses several inference algorithms: + +- **Class Inference**: Clustering entities by type and attribute similarity +- **Property Inference**: Determining domain/range based on connected entity types +- **Hierarchy Inference**: `` `A is_a B` `` detection based on subset relationships --- @@ -248,5 +254,14 @@ kg.add_entities(full_dataset) # Will raise error if violates schema ## Cookbook -- [Ontology](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb) -- [Unstructured to Ontology](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/12_Unstructured_to_Ontology.ipynb) +Interactive tutorials to learn ontology generation and management: + +- **[Ontology](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/14_Ontology.ipynb)**: Define domain schemas and ontologies to structure your data + - **Topics**: OWL, RDF, schema design, ontology generation + - **Difficulty**: Intermediate + - **Use Cases**: Structuring domain knowledge, schema definition + +- **[Unstructured to Ontology](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/12_Unstructured_to_Ontology.ipynb)**: Generate ontologies automatically from unstructured data + - **Topics**: Automatic ontology generation, 6-stage pipeline, OWL validation + - **Difficulty**: Advanced + - **Use Cases**: Domain modeling, automatic schema generation diff --git a/docs/reference/parse.md b/docs/reference/parse.md index c11b04e3..25812d58 100644 --- a/docs/reference/parse.md +++ b/docs/reference/parse.md @@ -6,6 +6,35 @@ ## 🎯 Overview +The **Parse Module** extracts structured content from raw files and data sources. It converts various file formats (PDF, DOCX, HTML, JSON, etc.) into usable text and structured data for semantic processing. + +### What is Document Parsing? + +**Document parsing** is the process of extracting text, structure, and metadata from files. The parse module handles: +- **Documents**: PDF, DOCX, PPTX, Excel, TXT, RTF +- **Web Content**: HTML, XML, JavaScript-rendered pages +- **Structured Data**: JSON, CSV, XML, YAML +- **Email**: MIME messages with attachments +- **Code**: Source code parsing into ASTs +- **Media**: OCR for images, metadata for audio/video + +### Why Use the Parse Module? + +- **Universal Format Support**: Handle dozens of file formats +- **Structure Preservation**: Maintain document structure (tables, headings, etc.) +- **OCR Support**: Extract text from scanned documents and images +- **Metadata Extraction**: Extract document metadata (author, date, etc.) +- **Code Analysis**: Parse source code for structure and dependencies +- **Error Handling**: Robust handling of corrupted or malformed files + +### How It Works + +1. **Format Detection**: Automatically detect file format +2. **Parser Selection**: Choose appropriate parser for the format +3. **Content Extraction**: Extract text, structure, and metadata +4. **Normalization**: Normalize extracted content +5. **Output**: Return structured data ready for semantic processing +
- :material-file-document:{ .lg .middle } **Document Parsing** @@ -47,7 +76,7 @@
!!! tip "When to Use" - - **Ingestion**: The first step after loading raw files to convert them into usable text/data + - **After Ingestion**: The first step after loading raw files to convert them into usable text/data - **Data Extraction**: Pulling specific fields from structured files (JSON/CSV) - **Content Analysis**: Analyzing codebases or email archives - **OCR**: Extracting text from scanned documents or images @@ -83,10 +112,10 @@ Unified interface for document formats. | Method | Description | |--------|-------------| -| `parse_document(path)` | Auto-detect format and parse | -| `extract_text(path)` | Extract text from PDF/DOCX/HTML/TXT | -| `extract_metadata(path)` | Extract document metadata | -| `parse_batch(paths)` | Parse multiple documents | +| `` `parse_document(path)` `` | Auto-detect format and parse | +| `` `extract_text(path)` `` | Extract text from PDF/DOCX/HTML/TXT | +| `` `extract_metadata(path)` `` | Extract document metadata | +| `` `parse_batch(paths)` `` | Parse multiple documents | **Example:** @@ -107,10 +136,10 @@ Parses web content. | Method | Description | |--------|-------------| -| `parse_web_content(content, content_type)` | Parse HTML/XML | -| `extract_text(content)` | Clean text from HTML | -| `extract_links(content)` | Extract hyperlinks | -| `render_javascript(url)` | Render JS for dynamic pages | +| `` `parse_web_content(content, content_type)` `` | Parse HTML/XML | +| `` `extract_text(content)` `` | Clean text from HTML | +| `` `extract_links(content)` `` | Extract hyperlinks | +| `` `render_javascript(url)` `` | Render JS for dynamic pages | ### StructuredDataParser @@ -120,7 +149,7 @@ Parses data files. | Method | Description | |--------|-------------| -| `parse_data(path, data_format)` | Parse JSON/CSV/XML/YAML | +| `` `parse_data(path, data_format)` `` | Parse JSON/CSV/XML/YAML | **Example:** @@ -140,7 +169,7 @@ Parses source code. | Method | Description | |--------|-------------| -| `parse_code(path)` | Parse code file; returns structure, comments, dependencies | +| `` `parse_code(path)` `` | Parse code file; returns structure, comments, dependencies | **Example:** @@ -161,10 +190,10 @@ Parses email messages. | Method | Description | |--------|-------------| -| `parse_email(path)` | Parse full email (headers/body/attachments) | -| `parse_headers(path)` | Extract headers only | -| `extract_body(path)` | Extract text/HTML body | -| `analyze_thread(path)` | Thread reconstruction | +| `` `parse_email(path)` `` | Parse full email (headers/body/attachments) | +| `` `parse_headers(path)` `` | Extract headers only | +| `` `extract_body(path)` `` | Extract text/HTML body | +| `` `analyze_thread(path)` `` | Thread reconstruction | **Example:** @@ -185,7 +214,7 @@ Parses media files. | Method | Description | |--------|-------------| -| `parse_media(path, media_type)` | Parse image/audio/video | +| `` `parse_media(path, media_type)` `` | Parse image/audio/video | **Example:** @@ -311,4 +340,9 @@ Use parser classes directly in pipelines and services. Avoid convenience functio ## Cookbook -- [Document Parsing](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/03_Document_Parsing.ipynb) +Interactive tutorials to learn document parsing: + +- **[Document Parsing](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/03_Document_Parsing.ipynb)**: Extract clean text from complex formats + - **Topics**: OCR, PDF parsing, text extraction, format detection + - **Difficulty**: Beginner + - **Use Cases**: Processing documents, extracting text from various formats diff --git a/docs/reference/pipeline.md b/docs/reference/pipeline.md index 5df06913..74eea1dd 100644 --- a/docs/reference/pipeline.md +++ b/docs/reference/pipeline.md @@ -6,6 +6,35 @@ ## 🎯 Overview +The **Pipeline Module** provides a robust orchestration engine for building, executing, and managing complex data processing workflows. It enables you to create reusable, scalable pipelines with error handling, parallel execution, and resource management. + +### What is Pipeline Orchestration? + +**Pipeline orchestration** is the process of coordinating multiple processing steps into a workflow. The Pipeline module enables: +- **DAG Construction**: Build directed acyclic graphs (DAGs) of processing steps +- **Parallel Execution**: Run independent steps simultaneously +- **Error Handling**: Retry, fallback, and recovery strategies +- **Resource Management**: CPU and memory allocation +- **Progress Tracking**: Monitor pipeline execution + +### Why Use the Pipeline Module? + +- **Complex Workflows**: Coordinate multi-step data processing +- **Reusability**: Create reusable pipeline templates +- **Reliability**: Built-in error handling and retry logic +- **Performance**: Parallel execution for faster processing +- **Monitoring**: Track progress and performance +- **Scalability**: Handle large-scale data processing + +### How It Works + +1. **Pipeline Definition**: Define steps and their dependencies +2. **Validation**: Validate pipeline structure (no cycles, valid dependencies) +3. **Execution**: Execute steps in dependency order +4. **Parallelization**: Run independent steps in parallel +5. **Error Handling**: Retry failed steps, apply fallbacks +6. **Monitoring**: Track progress and resource usage +
- :material-pipe:{ .lg .middle } **Pipeline Builder** @@ -56,21 +85,41 @@ ## ⚙️ Algorithms Used ### Execution Management -- **DAG Topological Sort**: Determines execution order of steps -- **State Management**: Tracks `PENDING`, `RUNNING`, `COMPLETED`, `FAILED` states + +**Purpose**: Manage pipeline execution order and state tracking. + +**How it works**: + +- **DAG Topological Sort**: Determines execution order of steps based on dependencies +- **State Management**: Tracks `` `PENDING` ``, `` `RUNNING` ``, `` `COMPLETED` ``, `` `FAILED` `` states - **Checkpointing**: Saves intermediate results to allow resuming failed pipelines ### Parallelism + +**Purpose**: Execute independent steps concurrently for maximum performance. + +**How it works**: + - **ThreadPoolExecutor**: For I/O-bound tasks (network requests, DB writes) - **ProcessPoolExecutor**: For CPU-bound tasks (parsing, embedding generation) - **Dependency Resolution**: Identifies steps that can run concurrently ### Error Handling -- **Exponential Backoff**: `wait = base * (factor ^ attempt)` + +**Purpose**: Robust error recovery with configurable retry policies. + +**How it works**: + +- **Exponential Backoff**: `` `wait = base * (factor ^ attempt)` `` - **Jitter**: Randomization to prevent thundering herd problem - **Circuit Breaker**: Stops execution after threshold failures to prevent cascading issues ### Resource Scheduling + +**Purpose**: Manage CPU/Memory allocation for resource-intensive tasks. + +**How it works**: + - **Token Bucket**: Rate limiting for API calls - **Semaphore**: Concurrency limiting for resource constraints - **Priority Queue**: Scheduling critical tasks first @@ -375,6 +424,15 @@ result = engine.execute_pipeline(pipeline, data={"path": "document.pdf"}) --- +## Cookbook + +Interactive tutorials to learn pipeline orchestration: + +- **[Pipeline Orchestration](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/07_Pipeline_Orchestration.ipynb)**: Build robust, automated data processing pipelines + - **Topics**: Workflows, automation, error handling, pipeline orchestration, DAG construction + - **Difficulty**: Advanced + - **Use Cases**: Complex multi-step workflows, production pipelines, ETL processes + ## See Also - [Ingest Module](ingest.md) - Common first step diff --git a/docs/reference/reasoning.md b/docs/reference/reasoning.md index 188aad10..af448e01 100644 --- a/docs/reference/reasoning.md +++ b/docs/reference/reasoning.md @@ -6,6 +6,31 @@ ## 🎯 Overview +The **Reasoning Module** provides logical inference capabilities for deriving new knowledge from existing facts. It supports rule-based inference, SPARQL-based reasoning, and high-performance pattern matching. + +### What is Reasoning? + +**Reasoning** is the process of deriving new facts from existing knowledge using logical rules. For example: +- **Given**: `` `Parent(Alice, Bob)` `` and `` `Parent(Bob, Charlie)` `` +- **Rule**: `` `IF Parent(?x, ?y) AND Parent(?y, ?z) THEN Grandparent(?x, ?z)` `` +- **Inferred**: `` `Grandparent(Alice, Charlie)` `` + +### Why Use the Reasoning Module? + +- **Knowledge Discovery**: Find implicit relationships not explicitly stored +- **Query Expansion**: Answer queries that require inference +- **Validation**: Check logical consistency of knowledge graphs +- **Explanation**: Understand how facts were derived +- **Rule-Based Logic**: Define domain-specific inference rules + +### How It Works + +1. **Rule Definition**: Define inference rules (IF-THEN patterns) +2. **Fact Matching**: Match facts against rule conditions +3. **Variable Binding**: Bind variables in rules to actual entities +4. **Inference**: Derive new facts from matched rules +5. **Explanation**: Generate explanations for inferred facts +
- :material-brain:{ .lg .middle } **Rule-based Inference** @@ -35,7 +60,7 @@
!!! tip "When to Use" - - **Inference**: Deriving new facts from existing data (e.g., `Parent(A,B) & Parent(B,C) -> Grandparent(A,C)`) + - **Inference**: Deriving new facts from existing data (e.g., `` `Parent(A,B) & Parent(B,C) -> Grandparent(A,C)` ``) - **Query Expansion**: Finding results that aren't explicitly stored but implied - **Explanation**: Understanding the reasoning path for any derived fact - **Validation**: Checking logical consistency of the knowledge graph @@ -45,15 +70,39 @@ ## ⚙️ Algorithms Used ### Forward Chaining -- **Variable Substitution**: Supports patterns like `Person(?x)` to match facts and bind variables. -- **Recursive Inference**: Continues deriving facts until no new information can be found. -- **Priority-based Execution**: Rules can be prioritized to control the inference flow. + +**Purpose**: Derive new facts from existing knowledge using logical rules. + +**How it works**: + +- **Variable Substitution**: Supports patterns like `` `Person(?x)` `` to match facts and bind variables +- **Recursive Inference**: Continues deriving facts until no new information can be found +- **Priority-based Execution**: Rules can be prioritized to control the inference flow + +**Complexity**: `` `O(n * m)` `` where n is the number of facts and m is the number of rules + +**Example**: + +```python +# Forward chaining implementation +reasoner = Reasoner() +rules = ["IF Person(?x) THEN Human(?x)"] +facts = ["Person(John)"] +new_facts = reasoner.infer_facts(facts, rules) +``` ### Rete Algorithm -- **Alpha Nodes**: Filter facts by single attributes (e.g., `type=Person`). -- **Beta Nodes**: Join results from Alpha nodes (e.g., `Person.id == Parent.child_id`). -- **Memory**: Stores partial matches to avoid re-computation. -- **Efficiency**: Optimal for scenarios with many rules and frequent fact updates. + +**Purpose**: High-performance pattern matching for large rule sets with frequent fact updates. + +**How it works**: + +- **Alpha Nodes**: Filter facts by single attributes (e.g., `` `type=Person` ``) +- **Beta Nodes**: Join results from Alpha nodes (e.g., `` `Person.id == Parent.child_id` ``) +- **Memory**: Stores partial matches to avoid re-computation +- **Efficiency**: Optimal for scenarios with many rules and frequent fact updates + +**Complexity**: `` `O(n + m)` `` where n is the number of facts and m is the number of rules (amortized) --- @@ -67,11 +116,11 @@ The high-level interface for the reasoning module. | Method | Description | |--------|-------------| -| `infer_facts(facts, rules)` | Derive new facts from initial state | -| `backward_chain(goal)` | Prove a goal using backward chaining | -| `add_rule(rule)` | Add a new inference rule | -| `add_fact(fact)` | Add a fact to working memory | -| `clear()` | Reset the reasoner state | +| `` `infer_facts(facts, rules)` `` | Derive new facts from initial state | +| `` `backward_chain(goal)` `` | Prove a goal using backward chaining | +| `` `add_rule(rule)` `` | Add a new inference rule | +| `` `add_fact(fact)` `` | Add a fact to working memory | +| `` `clear()` `` | Reset the reasoner state | ### ReteEngine @@ -81,9 +130,9 @@ High-performance pattern matching engine. | Method | Description | |--------|-------------| -| `build_network(rules)` | Compile rules into a Rete network | -| `add_fact(fact)` | Propagate fact through the network | -| `match_patterns()` | Get triggered rules | +| `` `build_network(rules)` `` | Compile rules into a Rete network | +| `` `add_fact(fact)` `` | Propagate fact through the network | +| `` `match_patterns()` `` | Get triggered rules | ### ExplanationGenerator @@ -93,7 +142,7 @@ Explains *why* a fact was inferred. | Method | Description | |--------|-------------| -| `generate_explanation(result)` | Generate reasoning trace for an InferenceResult | +| `` `generate_explanation(result)` `` | Generate reasoning trace for an InferenceResult | --- @@ -164,13 +213,22 @@ for fact_str in inferred: ## Best Practices -1. **Limit Recursion**: Be careful with recursive rules (e.g., `A(x,y) -> A(y,x)`) which can cause infinite loops in naive implementations. -2. **Use Rete for Scale**: For >100 rules or >10k facts, always use the Rete engine. -3. **Materialize vs. Query**: Materialize (pre-compute) for read-heavy workloads; Query-rewrite for write-heavy workloads. -4. **Validate Rules**: Ensure rules are logically consistent to avoid exploding the fact space. +1. **Limit Recursion**: Be careful with recursive rules (e.g., `` `A(x,y) -> A(y,x)` ``) which can cause infinite loops in naive implementations +2. **Use Rete for Scale**: For >100 rules or >10k facts, always use the Rete engine +3. **Materialize vs. Query**: Materialize (pre-compute) for read-heavy workloads; Query-rewrite for write-heavy workloads +4. **Validate Rules**: Ensure rules are logically consistent to avoid exploding the fact space --- +## Cookbook + +Interactive tutorials to learn reasoning and inference: + +- **[Reasoning and Inference](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)**: Use logical reasoning to infer new knowledge from existing facts + - **Topics**: Logic rules, inference engines, forward chaining, SPARQL reasoning, Rete algorithm + - **Difficulty**: Advanced + - **Use Cases**: Deriving new facts, query expansion, logical validation + ## See Also - [Ontology Module](ontology.md) - Source of schema-based rules diff --git a/docs/reference/seed.md b/docs/reference/seed.md index e652ebe9..882fd4f1 100644 --- a/docs/reference/seed.md +++ b/docs/reference/seed.md @@ -6,6 +6,33 @@ ## 🎯 Overview +The **Seed Module** provides a system for initializing Knowledge Graphs with verified, structured data from trusted sources. It enables bootstrapping knowledge graphs with reference data, taxonomies, and verified entities. + +### What is Seed Data? + +**Seed data** is verified, structured data used to bootstrap or enhance knowledge graphs. Examples include: + +- **Taxonomies**: Hierarchical classifications (product categories, organizational structures) +- **Reference Data**: Immutable reference information (countries, codes, standards) +- **Verified Entities**: Pre-validated entities from authoritative sources +- **Foundation Graphs**: Initial graph structures to build upon + +### Why Use the Seed Module? + +- **Bootstrap KGs**: Start with verified data instead of empty graphs +- **Quality Assurance**: Use trusted, validated data sources +- **Faster Development**: Skip initial data extraction for known entities +- **Data Integration**: Merge seed data with extracted data +- **Versioning**: Manage different versions of seed data + +### How It Works + +1. **Load Seed Data**: Load from CSV, JSON, databases, or APIs +2. **Validate**: Validate data quality and schema compliance +3. **Transform**: Convert to knowledge graph format +4. **Merge**: Integrate with extracted data using configurable strategies +5. **Version**: Track versions of seed data sources +
- :material-database-import:{ .lg .middle } **Multi-Source Loading** @@ -56,17 +83,32 @@ ## ⚙️ Algorithms Used ### Data Loading + +**Purpose**: Load seed data from various formats efficiently. + +**How it works**: + - **Format Detection**: Auto-detection of CSV delimiters, JSON structure - **Streaming**: Row-by-row processing for large files - **Normalization**: Type conversion and encoding handling ### Integration & Merging + +**Purpose**: Merge seed data with extracted data using configurable strategies. + +**How it works**: + - **Seed-First Strategy**: Seed data overrides extracted data (Trust Seed) - **Extracted-First Strategy**: Extracted data overrides seed (Trust Extraction) - **Smart Merge**: Property-level merging with conflict resolution - **ID Matching**: Entity resolution between seed and extracted entities ### Validation + +**Purpose**: Validate seed data quality and schema compliance. + +**How it works**: + - **Schema Validation**: Template-based structure checking - **Constraint Checking**: Required field and type validation - **Consistency Check**: Reference integrity (relationships point to existing entities) @@ -177,6 +219,20 @@ final_kg = seed_manager.integrate_seed_extracted( --- +## Cookbook + +Interactive tutorials that use seed data: + +- **[Financial Data Integration MCP](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/finance/01_Financial_Data_Integration_MCP.ipynb)**: Merging financial data with seed data integration + - **Topics**: Finance, data fusion, MCP integration, seed data + - **Difficulty**: Intermediate + - **Use Cases**: Integrating structured seed data with extracted data + +- **[Energy Market Analysis](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/renewable_energy/01_Energy_Market_Analysis.ipynb)**: Analyzing trends with seed data integration + - **Topics**: Energy, time series, temporal analysis, seed data + - **Difficulty**: Intermediate + - **Use Cases**: Bootstrapping knowledge graphs with verified data + ## See Also - [Ingest Module](ingest.md) - Loading unstructured data diff --git a/docs/reference/semantic_extract.md b/docs/reference/semantic_extract.md index 06238390..a9982a48 100644 --- a/docs/reference/semantic_extract.md +++ b/docs/reference/semantic_extract.md @@ -6,6 +6,36 @@ ## 🎯 Overview +The **Semantic Extract Module** extracts structured information from unstructured text. It identifies entities, relationships, events, and semantic structures that form the foundation of knowledge graphs. + +### What is Semantic Extraction? + +**Semantic extraction** is the process of identifying meaningful information from text: +- **Named Entities**: People, organizations, locations, dates, etc. +- **Relationships**: Connections between entities (e.g., "founded_by", "located_in") +- **Events**: Actions with temporal information and participants +- **Triplets**: Subject-Predicate-Object structures for knowledge graphs +- **Semantic Networks**: Structured networks of nodes and edges + +### Why Use the Semantic Extract Module? + +- **Multiple Methods**: Support for ML models, LLMs, and rule-based extraction +- **High Accuracy**: LLM-based extraction for complex schemas +- **Flexible Configuration**: Customize extraction for your domain +- **Confidence Scores**: Get confidence scores for all extractions +- **Batch Processing**: Efficient batch processing for large datasets +- **Coreference Resolution**: Resolve pronouns to their entity references + +### How It Works + +1. **Text Input**: Receive parsed text from the parse module +2. **Entity Extraction**: Identify named entities using NER +3. **Coreference Resolution**: Resolve pronouns to entities (optional) +4. **Relationship Extraction**: Identify relationships between entities +5. **Event Detection**: Detect events with temporal information +6. **Triplet Generation**: Generate RDF triplets for knowledge graphs +7. **Output**: Return structured entities, relationships, and triplets +
- :material-account-search:{ .lg .middle } **NER** @@ -39,7 +69,7 @@ Extract Subject-Predicate-Object triplets for Knowledge Graphs - :material-robot:{ .lg .middle } **LLM Extraction** - + --- Use LLMs to improve extraction quality and handle complex schemas @@ -63,23 +93,43 @@ ## ⚙️ Algorithms Used ### Named Entity Recognition (NER) -- **Transformer Models**: BERT/RoBERTa for token classification. -- **Regex Patterns**: Pattern matching for specific formats (Emails, IDs). -- **LLM Prompting**: Zero-shot extraction for custom entity types. + +**Purpose**: Identify and classify named entities in text. + +**How it works**: + +- **Transformer Models**: BERT/RoBERTa for token classification +- **Regex Patterns**: Pattern matching for specific formats (Emails, IDs) +- **LLM Prompting**: Zero-shot extraction for custom entity types ### Relation Extraction -- **Dependency Parsing**: Analyzing grammatical structure to find subject-verb-object paths. -- **Joint Extraction**: Extracting entities and relations simultaneously. -- **Semantic Role Labeling**: Identifying "Who did What to Whom". + +**Purpose**: Identify relationships between entities. + +**How it works**: + +- **Dependency Parsing**: Analyzing grammatical structure to find subject-verb-object paths +- **Joint Extraction**: Extracting entities and relations simultaneously +- **Semantic Role Labeling**: Identifying "Who did What to Whom" ### Coreference Resolution -- **Mention Detection**: Finding all potential references (nouns, pronouns). -- **Clustering**: Grouping mentions that refer to the same real-world entity. -- **Pronoun Resolution**: Mapping pronouns to the most likely antecedent. + +**Purpose**: Resolve pronouns and references to their entity references. + +**How it works**: + +- **Mention Detection**: Finding all potential references (nouns, pronouns) +- **Clustering**: Grouping mentions that refer to the same real-world entity +- **Pronoun Resolution**: Mapping pronouns to the most likely antecedent ### Triplet Extraction -- **OpenIE**: Open Information Extraction for arbitrary relation strings. -- **Schema-Based**: Mapping extracted relations to a predefined ontology. + +**Purpose**: Extract Subject-Predicate-Object triplets for Knowledge Graphs. + +**How it works**: + +- **OpenIE**: Open Information Extraction for arbitrary relation strings +- **Schema-Based**: Mapping extracted relations to a predefined ontology - **Reification**: Handling complex relations (time, location) by creating event nodes. --- @@ -487,6 +537,19 @@ kg = builder.build(sources) ## Cookbook -- [Entity Extraction](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb) -- [Relation Extraction](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/06_Relation_Extraction.ipynb) -- [Advanced Extraction](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb) +Interactive tutorials to learn semantic extraction: + +- **[Entity Extraction](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/05_Entity_Extraction.ipynb)**: Extract named entities from text using NER + - **Topics**: NER, Spacy, LLM extraction, entity types, confidence scores + - **Difficulty**: Beginner + - **Use Cases**: Identifying entities in text, building entity lists + +- **[Relation Extraction](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/06_Relation_Extraction.ipynb)**: Discover and classify relationships between entities + - **Topics**: Relation classification, dependency parsing, relationship types + - **Difficulty**: Beginner + - **Use Cases**: Finding relationships, building knowledge graphs + +- **[Advanced Extraction](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/01_Advanced_Extraction.ipynb)**: Custom extractors, LLM-based extraction, and complex pattern matching + - **Topics**: Custom models, regex, LLMs, ensemble methods, domain-specific extraction + - **Difficulty**: Advanced + - **Use Cases**: Custom extraction schemas, domain-specific entities diff --git a/docs/reference/split.md b/docs/reference/split.md index c5e7ba7b..bb97054d 100644 --- a/docs/reference/split.md +++ b/docs/reference/split.md @@ -58,7 +58,12 @@ ## ⚙️ Algorithms Used ### Standard Splitting Algorithms -- **Recursive Splitting**: Separator hierarchy (`\n\n`, `\n`, ` `, ``) with greedy splitting + +**Purpose**: Split documents into chunks using various strategies. + +**How it works**: + +- **Recursive Splitting**: Separator hierarchy (`` `\n\n` ``, `` `\n` ``, `` ` ` ``, ``) with greedy splitting - **Token Counting**: BPE tokenization using tiktoken or transformers - **Sentence Segmentation**: NLTK punkt, spaCy sentencizer, or regex-based - **Paragraph Detection**: Double newline detection with whitespace normalization @@ -66,16 +71,26 @@ - **Word Splitting**: Whitespace tokenization with word boundary preservation ### Semantic Chunking Algorithms -- **Semantic Boundary Detection**: + +**Purpose**: Intelligent boundary detection using embeddings and NLP. + +**How it works**: + +- **Semantic Boundary Detection**: - Sentence transformer embeddings (384-1024 dim) - Cosine similarity between consecutive sentences - - Threshold-based boundary detection (default: 0.7) -- **LLM-based Splitting**: + - Threshold-based boundary detection (default: `` `0.7` ``) +- **LLM-based Splitting**: - Prompt engineering for optimal split point detection - Context window management - Coherence scoring ### KG/Ontology Chunking Algorithms + +**Purpose**: Preserve entities, relationships, and graph structure for GraphRAG workflows. + +**How it works**: + - **Entity Boundary Detection**: - NER-based entity extraction (spaCy, LLM) - Entity span tracking @@ -85,9 +100,9 @@ - Subject-predicate-object span tracking - Relationship boundary preservation - **Graph Centrality Analysis**: - - Degree centrality: `C_D(v) = deg(v) / (n-1)` - - Betweenness centrality: `C_B(v) = Σ(σ_st(v) / σ_st)` - - Closeness centrality: `C_C(v) = (n-1) / Σd(v,u)` + - Degree centrality: `` `C_D(v) = deg(v) / (n-1)` `` + - Betweenness centrality: `` `C_B(v) = Σ(σ_st(v) / σ_st)` `` + - Closeness centrality: `` `C_C(v) = (n-1) / Σd(v,u)` `` - Eigenvector centrality: Power iteration method - **Community Detection**: - Louvain algorithm: Modularity optimization O(n log n) @@ -939,4 +954,10 @@ splitter = TextSplitter( - [Vector Store Module](vector_store.md) - Vector storage ## Cookbook -- [Chunking and Splitting](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/11_Chunking_and_Splitting.ipynb) + +Interactive tutorials to learn text chunking and splitting: + +- **[Chunking and Splitting](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/11_Chunking_and_Splitting.ipynb)**: Split documents for RAG and processing + - **Topics**: Recursive character splitting, semantic splitting, token-based splitting, chunking strategies + - **Difficulty**: Beginner + - **Use Cases**: Preparing text for RAG, document chunking diff --git a/docs/reference/triplet_store.md b/docs/reference/triplet_store.md index 2d9de590..321af524 100644 --- a/docs/reference/triplet_store.md +++ b/docs/reference/triplet_store.md @@ -6,6 +6,35 @@ ## 🎯 Overview +The **Triplet Store Module** provides storage and querying for RDF (Resource Description Framework) triplets. It supports industry-standard triplet stores with SPARQL querying and semantic reasoning capabilities. + +### What is a Triplet Store? + +A **triplet store** (also called an RDF store) is a database designed to store and query RDF triplets. RDF triplets are statements in the form: + +- **Subject**: The entity being described +- **Predicate**: The relationship or property +- **Object**: The value or related entity + +**Example**: `` `(Apple Inc., foundedBy, Steve Jobs)` `` + +### Why Use the Triplet Store Module? + +- **W3C Standards**: Full support for RDF and SPARQL standards +- **Semantic Reasoning**: RDFS and OWL reasoning for inference +- **Multiple Backends**: Support for Blazegraph, Apache Jena, RDF4J +- **SPARQL Queries**: Powerful SPARQL 1.1 query language +- **Federation**: Query across multiple stores +- **Bulk Loading**: High-performance data loading + +### How It Works + +1. **Store Selection**: Choose a backend (Blazegraph, Jena, RDF4J) +2. **Triplet Storage**: Store subject-predicate-object triplets +3. **SPARQL Queries**: Query using SPARQL 1.1 +4. **Reasoning**: Apply RDFS/OWL reasoning for inference +5. **Federation**: Query across multiple stores if needed +
- :material-graph-outline:{ .lg .middle } **RDF Storage** @@ -126,6 +155,15 @@ SPARQL query execution and optimization engine. --- +## Cookbook + +Interactive tutorials that use triplet stores: + +- **[Reasoning and Inference](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/08_Reasoning_and_Inference.ipynb)**: Use logical reasoning with SPARQL and triplet stores + - **Topics**: SPARQL reasoning, RDF stores, inference engines + - **Difficulty**: Advanced + - **Use Cases**: Semantic reasoning, SPARQL queries, RDF-based knowledge graphs + ## 🚀 Usage ### Initialization diff --git a/docs/reference/utils.md b/docs/reference/utils.md index 21fc4fc6..5baa16e9 100644 --- a/docs/reference/utils.md +++ b/docs/reference/utils.md @@ -6,6 +6,37 @@ ## 🎯 Overview +The **Utils Module** provides shared utilities used throughout all Semantica modules. It includes logging, error handling, validation, progress tracking, and common helper functions. + +### What is the Utils Module? + +The Utils module provides: + +- **Logging**: Structured logging with performance tracking +- **Error Handling**: Custom exception hierarchy and error formatting +- **Validation**: Data validation for entities, relationships, and configuration +- **Progress Tracking**: Track long-running operations +- **Helpers**: Common functions for text cleaning, hashing, file I/O +- **Type Definitions**: Shared TypedDicts and Enums for type safety + +### Why Use the Utils Module? + +- **Consistency**: Shared utilities ensure consistent behavior across modules +- **Error Handling**: Standardized error handling and reporting +- **Logging**: Unified logging across all modules +- **Validation**: Reusable validation functions +- **Type Safety**: Shared type definitions for better IDE support + +### How It Works + +The Utils module is used internally by all Semantica modules. You typically don't use it directly, but it provides: + +- **Logging Functions**: `` `get_logger()` ``, `` `setup_logging()` `` +- **Validation Functions**: `` `validate_entity()` ``, `` `validate_relationship()` `` +- **Error Classes**: Custom exceptions for different error types +- **Progress Tracking**: `` `ProgressTracker` `` for long operations +- **Helper Functions**: Text cleaning, hashing, file operations +
- :material-console-line:{ .lg .middle } **Logging** @@ -181,6 +212,15 @@ export SEMANTICA_PROGRESS_BAR=true --- +## Cookbook + +The Utils module is used throughout all Semantica modules. See any cookbook tutorial for examples of logging, validation, and error handling in practice. + +- **[Welcome to Semantica](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/01_Welcome_to_Semantica.ipynb)**: See utils in action across all modules + - **Topics**: Framework overview, all modules, utilities + - **Difficulty**: Beginner + - **Use Cases**: Understanding utility functions used throughout Semantica + ## See Also - [Core Module](core.md) - Uses Utils for infrastructure diff --git a/docs/reference/vector_store.md b/docs/reference/vector_store.md index 241ed80d..4be87a00 100644 --- a/docs/reference/vector_store.md +++ b/docs/reference/vector_store.md @@ -6,6 +6,33 @@ ## 🎯 Overview +The **Vector Store Module** provides a unified interface for storing and searching vector embeddings. It supports multiple backends (FAISS, Weaviate, Qdrant, Milvus) and enables semantic search, RAG, and similarity matching. + +### What is a Vector Store? + +A **vector store** is a database optimized for storing and searching high-dimensional vectors (embeddings). It enables: +- **Semantic Search**: Find documents similar to a query based on meaning +- **Similarity Matching**: Compare vectors to find similar items +- **Hybrid Search**: Combine vector search with keyword/metadata filtering +- **Scalable Storage**: Handle millions of vectors efficiently + +### Why Use the Vector Store Module? + +- **Multiple Backends**: Switch between FAISS (local), Weaviate, Qdrant, and Milvus +- **Unified Interface**: Same API regardless of backend +- **Hybrid Search**: Combine vector similarity with metadata filtering +- **Performance**: Optimized for high-throughput search operations +- **Namespace Support**: Multi-tenant isolation via namespaces +- **Metadata Filtering**: Rich filtering capabilities for precise queries + +### How It Works + +1. **Index Creation**: Create an index optimized for your vector dimensions +2. **Vector Storage**: Store embeddings with associated metadata +3. **Query Processing**: Convert query text to embedding and search +4. **Hybrid Search**: Combine vector similarity with metadata filters +5. **Result Ranking**: Rank results by relevance and return top-k +
- :material-database:{ .lg .middle } **Multi-Backend Support** @@ -749,5 +776,15 @@ print(f"Context: {context}") - [Ingest Module](ingest.md) - Source of data ## Cookbook -- [Vector Store](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb) -- [Advanced Vector Store and Search](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb) + +Interactive tutorials to learn vector stores: + +- **[Vector Store](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/13_Vector_Store.ipynb)**: Set up and use vector stores for similarity search + - **Topics**: FAISS, Weaviate, Qdrant, hybrid search, metadata filtering + - **Difficulty**: Intermediate + - **Use Cases**: Storing and searching embeddings, building RAG systems + +- **[Advanced Vector Store and Search](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/Advanced_Vector_Store_and_Search.ipynb)**: Advanced vector store operations and optimization + - **Topics**: Index optimization, hybrid search, performance tuning, namespace management + - **Difficulty**: Advanced + - **Use Cases**: Production deployments, performance optimization diff --git a/docs/reference/visualization.md b/docs/reference/visualization.md index 4e26ce4b..2de774f9 100644 --- a/docs/reference/visualization.md +++ b/docs/reference/visualization.md @@ -57,15 +57,21 @@ ## ⚙️ Algorithms Used ### Layout Algorithms -- **Force-Directed**: Simulates physical forces (repulsion between nodes, springs for edges) to find equilibrium. -- **Hierarchical**: Tree-based layout for taxonomies and directed acyclic graphs (DAGs). -- **Circular**: Arranges nodes in a circle, useful for analyzing interconnectivity. -- **Community-Based**: Groups nodes by community (Louvain/Leiden) and separates clusters. + +The visualization module uses various layout algorithms: + +- **Force-Directed**: Simulates physical forces (repulsion between nodes, springs for edges) to find equilibrium +- **Hierarchical**: Tree-based layout for taxonomies and directed acyclic graphs (DAGs) +- **Circular**: Arranges nodes in a circle, useful for analyzing interconnectivity +- **Community-Based**: Groups nodes by community (Louvain/Leiden) and separates clusters ### Dimensionality Reduction -- **UMAP**: Uniform Manifold Approximation and Projection. Preserves global structure better than t-SNE. -- **t-SNE**: t-Distributed Stochastic Neighbor Embedding. Good for local clustering. -- **PCA**: Principal Component Analysis. Linear projection for variance maximization. + +The module supports multiple dimensionality reduction techniques: + +- **UMAP**: Uniform Manifold Approximation and Projection - Preserves global structure better than t-SNE +- **t-SNE**: t-Distributed Stochastic Neighbor Embedding - Good for local clustering +- **PCA**: Principal Component Analysis - Linear projection for variance maximization ### Analytics Visualization - **Centrality Sizing**: Node size proportional to Degree/Betweenness/PageRank. @@ -282,5 +288,15 @@ analytics_viz.visualize_degree_distribution(kg, output="degree_dist.png") - [Ontology Module](ontology.md) - Source for hierarchy visualizations ## Cookbook -- [Visualization](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/16_Visualization.ipynb) -- [Complete Visualization Suite](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/03_Complete_Visualization_Suite.ipynb) + +Interactive tutorials to learn graph visualization: + +- **[Visualization](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/introduction/16_Visualization.ipynb)**: Basic graph visualization techniques + - **Topics**: Graph visualization, network diagrams, basic plotting + - **Difficulty**: Beginner + - **Use Cases**: Visualizing knowledge graphs, understanding graph structure + +- **[Complete Visualization Suite](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/advanced/03_Complete_Visualization_Suite.ipynb)**: Creating interactive, publication-ready visualizations + - **Topics**: PyVis, NetworkX, D3.js, interactive visualizations, publication-ready graphics + - **Difficulty**: Intermediate + - **Use Cases**: Advanced visualizations, presentations, publications diff --git a/docs/use-cases.md b/docs/use-cases.md index d63da96e..9ad0013c 100644 --- a/docs/use-cases.md +++ b/docs/use-cases.md @@ -9,21 +9,17 @@ Semantica is designed to solve complex data challenges across various domains. T ## Use Case Comparison -| Use Case | Difficulty | Time | Domain | Key Features | -| :-------------------------------- | :------------ | :---------- | :---------- | :---------------------------------------------- | -| **Research Paper Analysis** | Beginner | 30 min | Research | Citation networks, concept extraction | -| **Biomedical Knowledge Graphs** | Intermediate | 1-2 hours | Healthcare | Gene-protein-disease relationships | -| **Financial Market Intelligence** | Intermediate | 1 hour | Finance | Sentiment analysis, trend detection | -| **Algorithmic Trading** | Advanced | 2-3 hours | Finance | Multi-source integration, signal generation | -| **Blockchain Analytics** | Intermediate | 1-2 hours | Finance | Transaction tracing, fraud detection | -| **Medical Record Analysis** | Intermediate | 1 hour | Healthcare | Patient history, temporal tracking | -| **Cybersecurity Threat Intelligence**| Advanced | 2-3 hours | Security | Threat mapping, pattern detection | -| **OSINT** | Intermediate | 1-2 hours | Security | Multi-source intelligence | -| **Supply Chain Optimization** | Intermediate | 1-2 hours | Industry | Route optimization, risk management | -| **GraphRAG** | Intermediate | 1 hour | AI | Enhanced RAG with knowledge graphs | -| **Legal Document Analysis** | Intermediate | 1-2 hours | Legal | Contract analysis, clause extraction | -| **Social Media Analysis** | Beginner | 30 min | Social | Sentiment, trend analysis | -| **Customer Support KB** | Beginner | 30 min | Support | FAQ generation, knowledge base | +| Use Case | Difficulty | Time | Domain | Key Features | Cookbook | +| :-------------------------------- | :------------ | :---------- | :---------- | :---------------------------------------------- | :------------------------------------------ | +| **Biomedical Knowledge Graphs** | Intermediate | 1-2 hours | Healthcare | Gene-protein-disease relationships | Drug Discovery, Genomic Variant Analysis | +| **Financial Data Integration** | Intermediate | 1-2 hours | Finance | MCP integration, real-time data | Financial Data Integration MCP | +| **Fraud Detection** | Advanced | 2-3 hours | Finance | Temporal graphs, pattern detection | Fraud Detection | +| **Blockchain Analytics** | Intermediate | 1-2 hours | Finance | Transaction tracing, DeFi intelligence | DeFi Protocol Intelligence, Transaction Network | +| **Cybersecurity Threat Intelligence**| Advanced | 2-3 hours | Security | Threat mapping, anomaly detection | Real-Time Anomaly Detection, Threat Intelligence | +| **Intelligence Analysis** | Intermediate | 1-2 hours | Security | Criminal networks, OSINT analysis | Criminal Network Analysis, Intelligence Orchestrator | +| **Supply Chain Optimization** | Intermediate | 1-2 hours | Industry | Data integration, route optimization | Supply Chain Data Integration | +| **Renewable Energy Management** | Intermediate | 1-2 hours | Energy | Energy market analysis, optimization | Energy Market Analysis | +| **GraphRAG** | Advanced | 1-2 hours | AI | Enhanced RAG with knowledge graphs | GraphRAG Complete, RAG vs GraphRAG | **Difficulty Levels**: - **Beginner**: Basic Semantica knowledge required @@ -36,14 +32,6 @@ Semantica is designed to solve complex data challenges across various domains. T
-- :material-school: **Research Paper Analysis** - --- - Extract structured knowledge from academic papers to discover trends, relationships, and key concepts. - - **Goal**: Ingest PDFs, extract entities (Authors, Concepts, Methods), and build a citation network. - - **Difficulty**: Beginner - - :material-dna: **Biomedical Knowledge Graphs** --- Accelerate drug discovery and understand disease pathways by connecting genes, proteins, drugs, and diseases. @@ -51,69 +39,32 @@ Semantica is designed to solve complex data challenges across various domains. T **Goal**: Connect genes, proteins, drugs, and diseases from scientific literature and databases. **Difficulty**: Intermediate + + [:material-arrow-right: Drug Discovery Pipeline](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/biomedical/01_Drug_Discovery_Pipeline.ipynb) + + [:material-arrow-right: Genomic Variant Analysis](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/biomedical/02_Genomic_Variant_Analysis.ipynb)
-### Research Paper Analysis Implementation - -**Prerequisites**: -- Semantica installed -- Sample research papers (PDF format) - -**Code Example**: - -```python -from semantica.core import Semantica -from semantica.visualization import KGVisualizer - -# Initialize -semantica = Semantica() - -# Build knowledge graph from research papers -result = semantica.build_knowledge_base( - sources=[ - "papers/machine_learning_survey.pdf", - "papers/deep_learning_review.pdf" - ], - embeddings=True, - graph=True, - normalize=True -) - -# Visualize citation network -kg = result["knowledge_graph"] -visualizer = KGVisualizer() -visualizer.visualize(kg, output_path="citation_network.html") -``` - ### Biomedical Knowledge Graphs Implementation **Prerequisites**: - Domain knowledge of biomedical concepts - Access to biomedical literature/databases -**Code Example**: +**Implementation Guides:** -```python -from semantica.core import Semantica -from semantica.ontology import OntologyGenerator +- **[Drug Discovery Pipeline Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/biomedical/01_Drug_Discovery_Pipeline.ipynb)**: Build knowledge graphs from PubMed RSS feeds + - **Topics**: PubMed RSS ingestion, entity-aware chunking, GraphRAG, vector similarity search + - **Difficulty**: Intermediate + - **Time**: 1-2 hours + - **Use Cases**: Drug discovery, biomedical research -semantica = Semantica() -custom_entities = ["Gene", "Protein", "Drug", "Disease", "Pathway"] - -# Build knowledge graph -result = semantica.build_knowledge_base( - sources=["literature/cancer_research.pdf"], - embeddings=True, - graph=True, - custom_entity_types=custom_entities -) - -# Generate ontology -kg = result["knowledge_graph"] -ontology_gen = OntologyGenerator(base_uri="https://biomed.example.org/ontology/") -ontology = ontology_gen.generate_from_graph(kg) -``` +- **[Genomic Variant Analysis Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/biomedical/02_Genomic_Variant_Analysis.ipynb)**: Analyze genomic variants using temporal knowledge graphs + - **Topics**: bioRxiv RSS, temporal KGs, deduplication, pathway analysis + - **Difficulty**: Intermediate + - **Time**: 1-2 hours + - **Use Cases**: Genomic research, variant analysis --- @@ -121,37 +72,36 @@ ontology = ontology_gen.generate_from_graph(kg)
-- :material-finance: **Financial Market Intelligence** +- :material-finance: **Financial Data Integration** --- - Analyze market trends and sentiment from news and reports. + Integrate financial data from multiple sources using MCP servers and real-time ingestion. - **Goal**: Ingest earnings call transcripts, news articles, and analyst reports to gauge market sentiment. + **Goal**: Connect Alpha Vantage API, MCP servers, seed data, and real-time ingestion for comprehensive financial analysis. [:material-arrow-right: View Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/finance/01_Financial_Data_Integration_MCP.ipynb) +- :material-shield-alert: **Fraud Detection** + --- + Detect complex fraud rings using temporal knowledge graphs and pattern detection. + + **Goal**: Build a graph of Users, Devices, IP Addresses, and Transactions to find cycles and detect fraud patterns. + + [:material-arrow-right: View Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/finance/02_Fraud_Detection.ipynb) + - :material-bitcoin: **Blockchain Analytics** --- - Trace funds and identify illicit activity. + Analyze DeFi protocols and transaction networks for intelligence and fraud detection. - **Goal**: Map transaction flows between wallets and exchanges to detect money laundering or fraud. + **Goal**: Map transaction flows between wallets and exchanges, analyze DeFi protocols, and detect illicit activity. - [:material-arrow-right: View Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/blockchain/01_DeFi_Protocol_Intelligence.ipynb) + [:material-arrow-right: DeFi Protocol Intelligence](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/blockchain/01_DeFi_Protocol_Intelligence.ipynb) + + [:material-arrow-right: Transaction Network Analysis](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/blockchain/02_Transaction_Network_Analysis.ipynb)
--- -## Healthcare & Life Sciences - -
- -- :material-account-heart: **Patient Journey Mapping** - --- - Visualize and analyze the complete patient experience. - - **Goal**: Connect clinical encounters, lab results, and patient feedback to improve care delivery. - -
--- @@ -161,11 +111,13 @@ ontology = ontology_gen.generate_from_graph(kg) - :material-shield-lock: **Cybersecurity Threat Intelligence** --- - Proactively identify and mitigate cyber threats. + Proactively identify and mitigate cyber threats using real-time anomaly detection and threat intelligence. - **Goal**: Ingest threat feeds (STIX/TAXII), CVE databases, and system logs to map attack vectors. + **Goal**: Ingest threat feeds (CVE databases, security RSS), detect anomalies in streaming data, and build threat intelligence knowledge graphs. - [:material-arrow-right: View Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/cybersecurity/01_Real_Time_Anomaly_Detection.ipynb) + [:material-arrow-right: Real-Time Anomaly Detection](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/cybersecurity/01_Real_Time_Anomaly_Detection.ipynb) + + [:material-arrow-right: Threat Intelligence Hybrid RAG](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/cybersecurity/02_Threat_Intelligence_Hybrid_RAG.ipynb) - :material-account-network: **Criminal Network Analysis** --- @@ -183,14 +135,6 @@ ontology = ontology_gen.generate_from_graph(kg) [:material-arrow-right: View Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/intelligence/02_Intelligence_Analysis_Orchestrator_Worker.ipynb) -- :material-incognito: **Fraud Detection** - --- - Detect complex fraud rings using temporal knowledge graphs and pattern detection. - - **Goal**: Build a graph of Users, Devices, IP Addresses, and Transactions to find cycles. - - [:material-arrow-right: View Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/finance/02_Fraud_Detection.ipynb) -
--- @@ -225,141 +169,30 @@ ontology = ontology_gen.generate_from_graph(kg) - :material-robot: **Graph-Augmented Generation (GraphRAG)** --- - Enhance LLM responses with structured ground truth. + Enhance LLM responses with structured ground truth using knowledge graphs. - **Goal**: Use the knowledge graph to retrieve precise context for RAG applications. + **Goal**: Use the knowledge graph to retrieve precise context for RAG applications with hybrid retrieval and logical inference. - [:material-arrow-right: View Cookbook](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb) - [:material-scale-balance: RAG vs GraphRAG Benchmark](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/02_RAG_vs_GraphRAG_Comparison.ipynb) - -- :material-domain: **Corporate Intelligence** - --- - Unify internal documents into a single semantic layer. + [:material-arrow-right: GraphRAG Complete](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/01_GraphRAG_Complete.ipynb) - **Goal**: Connect People, Projects, and Decisions across the organization. - -- :material-gavel: **Legal Document Review** - --- - Analyze contracts and legal texts. - - **Goal**: Parse contracts, extract clauses, and identify relationships like "supersedes". + [:material-scale-balance: RAG vs GraphRAG Comparison](https://github.com/Hawksight-AI/semantica/blob/main/cookbook/use_cases/advanced_rag/02_RAG_vs_GraphRAG_Comparison.ipynb)
--- -## New Use Cases - -### Legal Document Analysis - -!!! abstract "Use Case" - Analyze contracts and legal texts to extract clauses, identify relationships, and understand document structure. - -**Difficulty**: Intermediate| **Domain**: Legal - -**Prerequisites**: -- Legal document samples (contracts, agreements) -- LLM API access (recommended) - -**Code Example**: - -```python -from semantica.core import Semantica - -semantica = Semantica() -legal_entities = ["Party", "Clause", "Section", "Contract", "Term"] - -# Build knowledge graph from contracts -result = semantica.build_knowledge_base( - sources=["contracts/agreement1.pdf"], - custom_entity_types=legal_entities, - graph=True, - temporal=True -) - -kg = result["knowledge_graph"] -clause_rels = [r for r in kg['relationships'] - if r.get('predicate') in ['supersedes', 'amends']] -print(f"Found {len(clause_rels)} clause relationships") -``` - -### Social Media Analysis - -!!! abstract "Use Case" - Analyze social media content to extract sentiment, trends, and relationships between users and topics. - -**Difficulty**: Beginner| **Domain**: Social Media - -**Prerequisites**: -- Social media data (JSON, CSV) - -**Code Example**: - -```python -from semantica.core import Semantica -from semantica.ingest import FileIngestor - -semantica = Semantica() -ingestor = FileIngestor() -posts = ingestor.ingest("social_media/posts.json") - -# Build knowledge graph -result = semantica.build_knowledge_base( - sources=posts, - embeddings=True, - graph=True -) - -kg = result["knowledge_graph"] -hashtags = [e for e in kg['entities'] if e.get('text', '').startswith('#')] -print(f"Hashtags: {len(hashtags)}") -``` - -### Customer Support Knowledge Base - -!!! abstract "Use Case" - Build a knowledge base from support tickets, documentation, and FAQs to improve customer service. - -**Difficulty**: Beginner| **Domain**: Customer Support - -**Prerequisites**: -- Support tickets or documentation - -**Code Example**: - -```python -from semantica.core import Semantica -from semantica.vector_store import VectorStore, HybridSearch - -semantica = Semantica() - -# Build knowledge base -result = semantica.build_knowledge_base( - sources=["support/tickets/", "support/faqs/"], - embeddings=True, - graph=True -) - -# Search -vector_store = VectorStore() -vector_store.store(result["embeddings"], result["documents"]) -hybrid_search = HybridSearch(vector_store) -results = hybrid_search.search(query="How do I reset my password?", top_k=5) -``` --- ## Summary -This guide covered use cases across multiple domains: +This guide covered use cases across multiple domains with corresponding cookbooks: -- **Research & Science**: Academic paper analysis, biomedical knowledge graphs -- **Finance & Trading**: Market intelligence, trading signals, blockchain analytics -- **Healthcare**: Medical records, patient journey mapping -- **Security**: Threat intelligence, OSINT, fraud detection -- **Industry**: Supply chain, energy management -- **AI Applications**: GraphRAG, corporate intelligence -- **New Use Cases**: Legal analysis, social media, customer support +- **Research & Science**: Biomedical knowledge graphs (Drug Discovery, Genomic Variant Analysis) +- **Finance & Trading**: Financial data integration, fraud detection, blockchain analytics +- **Security & Intelligence**: Cybersecurity threat intelligence, criminal network analysis, intelligence orchestration +- **Industry**: Supply chain optimization, renewable energy management +- **AI Applications**: GraphRAG (Complete implementation and comparison) --- @@ -375,4 +208,3 @@ This guide covered use cases across multiple domains: !!! info "Contribute" Have a use case to add? [Contribute on GitHub](https://github.com/Hawksight-AI/semantica) -**Last Updated**: 2024