diff --git a/cookbook/advanced/11_Text_Chunking_Strategies.ipynb b/cookbook/advanced/11_Text_Chunking_Strategies.ipynb deleted file mode 100644 index d076be91..00000000 --- a/cookbook/advanced/11_Text_Chunking_Strategies.ipynb +++ /dev/null @@ -1,265 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/advanced/11_Text_Chunking_Strategies.ipynb)\n", - "\n", - "# Text Chunking Strategies\n", - "\n", - "## Overview\n", - "\n", - "Explore different text chunking strategies: semantic, structural, sliding window, and table chunking for optimal document processing.\n", - "\n", - "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/split/)\n", - "\n", - "## Installation\n", - "\n", - "Install Semantica from PyPI:\n", - "\n", - "```bash\n", - "pip install semantica\n", - "# Or with all optional dependencies:\n", - "pip install semantica[all]\n", - "```\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.parse import TextSplitter\n", - "import re\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Prepare Sample Document\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "document = \"\"\"\n", - "# Introduction to Knowledge Graphs\n", - "\n", - "Knowledge graphs are powerful data structures that represent information as entities and their relationships. \n", - "They enable semantic understanding and reasoning over complex data.\n", - "\n", - "## What are Knowledge Graphs?\n", - "\n", - "A knowledge graph is a graph-based data model used to represent knowledge. It consists of nodes (entities) \n", - "and edges (relationships) that connect these entities. Knowledge graphs are widely used in search engines, \n", - "recommendation systems, and AI applications.\n", - "\n", - "## Applications\n", - "\n", - "Knowledge graphs have numerous applications:\n", - "- Search engines use them to understand user queries\n", - "- Recommendation systems leverage them for personalized suggestions\n", - "- AI systems use them for reasoning and inference\n", - "\n", - "## Conclusion\n", - "\n", - "In summary, knowledge graphs provide a flexible and powerful way to represent and reason about complex information.\n", - "\"\"\"\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: Semantic Chunking\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class SemanticChunker:\n", - " def chunk(self, document, chunk_size=500):\n", - " paragraphs = [p.strip() for p in document.split('\\n\\n') if p.strip()]\n", - " \n", - " chunks = []\n", - " current_chunk = \"\"\n", - " \n", - " for para in paragraphs:\n", - " if len(current_chunk) + len(para) <= chunk_size:\n", - " current_chunk += para + \"\\n\\n\"\n", - " else:\n", - " if current_chunk:\n", - " chunks.append(current_chunk.strip())\n", - " current_chunk = para + \"\\n\\n\"\n", - " \n", - " if current_chunk:\n", - " chunks.append(current_chunk.strip())\n", - " \n", - " return chunks\n", - "\n", - "semantic_chunker = SemanticChunker()\n", - "semantic_chunks = semantic_chunker.chunk(document, chunk_size=500)\n", - "\n", - "for i, chunk in enumerate(semantic_chunks, 1):\n", - " print(f\"Chunk {i}: {len(chunk)} characters\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Structural Chunking\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class StructuralChunker:\n", - " def chunk(self, document):\n", - " chunks = []\n", - " current_section = \"\"\n", - " current_header = \"\"\n", - " \n", - " lines = document.split('\\n')\n", - " \n", - " for line in lines:\n", - " if line.startswith('#'):\n", - " if current_section:\n", - " chunks.append({\n", - " 'header': current_header,\n", - " 'content': current_section.strip()\n", - " })\n", - " current_header = line.strip()\n", - " current_section = \"\"\n", - " else:\n", - " current_section += line + \"\\n\"\n", - " \n", - " if current_section:\n", - " chunks.append({\n", - " 'header': current_header,\n", - " 'content': current_section.strip()\n", - " })\n", - " \n", - " return chunks\n", - "\n", - "structural_chunker = StructuralChunker()\n", - "structural_chunks = structural_chunker.chunk(document)\n", - "\n", - "for i, chunk in enumerate(structural_chunks, 1):\n", - " header = chunk['header'][:50] if chunk['header'] else \"No header\"\n", - " print(f\"Chunk {i}: {header}... ({len(chunk['content'])} chars)\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 4: Sliding Window Chunking\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class SlidingWindowChunker:\n", - " def chunk(self, document, window_size=200, overlap=50):\n", - " words = document.split()\n", - " chunks = []\n", - " \n", - " start = 0\n", - " while start < len(words):\n", - " end = min(start + window_size, len(words))\n", - " chunk_words = words[start:end]\n", - " chunks.append(' '.join(chunk_words))\n", - " \n", - " start += window_size - overlap\n", - " \n", - " return chunks\n", - "\n", - "sliding_chunker = SlidingWindowChunker()\n", - "sliding_chunks = sliding_chunker.chunk(document, window_size=200, overlap=50)\n", - "\n", - "for i, chunk in enumerate(sliding_chunks[:3], 1):\n", - " print(f\"Chunk {i}: {len(chunk)} characters\")\n", - "if len(sliding_chunks) > 3:\n", - " print(f\"... and {len(sliding_chunks) - 3} more chunks\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Table Chunking\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class TableChunker:\n", - " def chunk(self, table_data):\n", - " if isinstance(table_data, str):\n", - " rows = [row.strip() for row in table_data.split('\\n') if row.strip()]\n", - " chunks = []\n", - " for row in rows:\n", - " if '|' in row:\n", - " chunks.append(row)\n", - " return chunks\n", - " elif isinstance(table_data, list):\n", - " return [str(row) for row in table_data]\n", - " else:\n", - " return [str(table_data)]\n", - "\n", - "table_data = \"\"\"\n", - "| Name | Age | Role |\n", - "|------|-----|------|\n", - "| Alice | 30 | Engineer |\n", - "| Bob | 35 | Manager |\n", - "| Charlie | 28 | Developer |\n", - "\"\"\"\n", - "\n", - "table_chunker = TableChunker()\n", - "table_chunks = table_chunker.chunk(table_data)\n", - "\n", - "for i, chunk in enumerate(table_chunks, 1):\n", - " print(f\"Chunk {i}: {chunk[:50]}...\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "Chunking strategies:\n", - "- Semantic Chunking (by meaning/paragraphs)\n", - "- Structural Chunking (by document structure)\n", - "- Sliding Window Chunking (with overlap)\n", - "- Table Chunking (for structured data)\n" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/introduction/11_Chunking_and_Splitting.ipynb b/cookbook/introduction/11_Chunking_and_Splitting.ipynb new file mode 100644 index 00000000..5f5d10fc --- /dev/null +++ b/cookbook/introduction/11_Chunking_and_Splitting.ipynb @@ -0,0 +1,853 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/introduction/11_Chunking_and_Splitting.ipynb)\n", + "\n", + "# Chunking and Splitting - Comprehensive Guide\n", + "\n", + "## Overview\n", + "\n", + "This notebook provides a **comprehensive walkthrough** of Semantica's split module, demonstrating all chunking strategies and methods for optimal document processing. You'll learn to use 15+ splitting methods including standard, semantic, and knowledge graph-aware approaches.\n", + "\n", + "**Documentation**: [API Reference](https://semantica.readthedocs.io/reference/split/)\n", + "\n", + "### Learning Objectives\n", + "\n", + "By the end of this notebook, you will be able to:\n", + "\n", + "- Use `TextSplitter` with multiple methods\n", + "- Apply standard splitting methods (recursive, token, sentence, paragraph)\n", + "- Use semantic chunking for topic coherence\n", + "- Apply KG-aware chunking (entity-aware, relation-aware, graph-based)\n", + "- Use specialized chunkers (structural, sliding window, table, hierarchical)\n", + "- Validate chunk quality with `ChunkValidator`\n", + "- Track provenance with `ProvenanceTracker`\n", + "- Choose the right method for your use case\n", + "\n", + "### What You'll Learn\n", + "\n", + "| Component | Purpose | When to Use |\n", + "|-----------|---------|-------------|\n", + "| `TextSplitter` | Unified splitter | All chunking needs |\n", + "| `SemanticChunker` | Semantic boundaries | Topic-based chunks |\n", + "| `EntityAwareChunker` | Preserve entities | GraphRAG workflows |\n", + "| `RelationAwareChunker` | Preserve triples | KG construction |\n", + "| `StructuralChunker` | Document structure | Formatted documents |\n", + "| `HierarchicalChunker` | Multi-level chunks | Large documents |\n", + "\n", + "---\n", + "\n", + "## Installation\n", + "\n", + "Install Semantica from PyPI:\n", + "\n", + "```bash\n", + "pip install semantica\n", + "# Or with all optional dependencies:\n", + "pip install semantica[all]\n", + "```\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Basic Chunking with TextSplitter\n", + "\n", + "Let's start with the unified `TextSplitter` interface, which provides access to all chunking methods.\n", + "\n", + "### What is TextSplitter?\n", + "\n", + "`TextSplitter` is a unified interface that supports 15+ chunking methods:\n", + "- **Standard**: recursive, token, sentence, paragraph, character, word\n", + "- **Semantic**: semantic_transformer, llm, huggingface, nltk\n", + "- **KG/Ontology**: entity_aware, relation_aware, graph_based, ontology_aware\n", + "- **Advanced**: hierarchical, structural, sliding_window, table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import TextSplitter\n", + "\n", + "# Sample long text\n", + "text = \"\"\"\n", + "Apple Inc. is a technology company founded by Steve Jobs, Steve Wozniak, and Ronald Wayne \n", + "in Cupertino, California on April 1, 1976. The company's current CEO is Tim Cook, who took \n", + "over from Steve Jobs in August 2011. Apple is headquartered at One Apple Park Way in Cupertino.\n", + "\n", + "Apple develops and sells consumer electronics, computer software, and online services. The company's \n", + "hardware products include the iPhone smartphone, the iPad tablet computer, the Mac personal computer, \n", + "the iPod portable media player, the Apple Watch smartwatch, the Apple TV digital media player, and the \n", + "HomePod smart speaker.\n", + "\n", + "Apple's software includes the macOS and iOS operating systems, the iTunes media player, the Safari web \n", + "browser, and the iLife and iWork creativity and productivity suites. Its online services include the \n", + "iTunes Store, the iOS App Store and Mac App Store, Apple Music, and iCloud.\n", + "\"\"\"\n", + "\n", + "# Basic recursive splitting\n", + "splitter = TextSplitter(\n", + " method=\"recursive\",\n", + " chunk_size=200,\n", + " chunk_overlap=50\n", + ")\n", + "\n", + "chunks = splitter.split(text)\n", + "\n", + "print(f\"Split into {len(chunks)} chunks using recursive method\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Length: {len(chunk.text)} characters\")\n", + " print(f\" Start: {chunk.start}, End: {chunk.end}\")\n", + " print(f\" Text: {chunk.text[:100]}...\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Standard Splitting Methods\n", + "\n", + "Let's compare different standard splitting methods.\n", + "\n", + "### Method Comparison\n", + "\n", + "| Method | Best For | Speed | Accuracy |\n", + "|--------|----------|-------|----------|\n", + "| **recursive** | General text | Fast | Good |\n", + "| **sentence** | Coherent chunks | Medium | Very Good |\n", + "| **token** | LLM context | Medium | Excellent |\n", + "| **paragraph** | Natural breaks | Fast | Good |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Compare different methods\n", + "methods = [\"recursive\", \"sentence\", \"paragraph\"]\n", + "\n", + "print(\"Comparing Standard Splitting Methods:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for method in methods:\n", + " splitter = TextSplitter(\n", + " method=method,\n", + " chunk_size=200,\n", + " chunk_overlap=50\n", + " )\n", + " \n", + " chunks = splitter.split(text)\n", + " \n", + " print(f\"\\nMethod: {method.upper()}\")\n", + " print(\"-\" * 40)\n", + " print(f\" Chunks created: {len(chunks)}\")\n", + " print(f\" Avg chunk size: {sum(len(c.text) for c in chunks) / len(chunks):.0f} chars\")\n", + " print(f\" First chunk: {chunks[0].text[:80]}...\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Token-Based Splitting\n", + "\n", + "Token-based splitting is crucial for LLM applications where you need to respect token limits.\n", + "\n", + "### Why Token-Based?\n", + "\n", + "- **LLM Context Windows**: GPT-4 has 8K/32K token limits\n", + "- **Accurate Counting**: Character count ≠ token count\n", + "- **Cost Optimization**: Tokens determine API costs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import split_by_tokens\n", + "\n", + "# Token-based splitting\n", + "chunks = split_by_tokens(\n", + " text,\n", + " chunk_size=100, # 100 tokens\n", + " chunk_overlap=20,\n", + " tokenizer=\"tiktoken\",\n", + " model=\"gpt-4\"\n", + ")\n", + "\n", + "print(\"Token-Based Splitting Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " token_count = chunk.metadata.get('token_count', 'N/A')\n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Tokens: {token_count}\")\n", + " print(f\" Characters: {len(chunk.text)}\")\n", + " print(f\" Ratio: {len(chunk.text)/token_count if token_count != 'N/A' else 'N/A':.2f} chars/token\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Semantic Chunking\n", + "\n", + "Semantic chunking creates chunks based on semantic boundaries using embeddings.\n", + "\n", + "### How It Works\n", + "\n", + "1. Split text into sentences\n", + "2. Generate embeddings for each sentence\n", + "3. Calculate similarity between consecutive sentences\n", + "4. Create boundaries where similarity drops below threshold" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import SemanticChunker\n", + "\n", + "# Semantic chunking\n", + "semantic_chunker = SemanticChunker(\n", + " chunk_size=200,\n", + " chunk_overlap=50,\n", + " embedding_model=\"all-MiniLM-L6-v2\",\n", + " similarity_threshold=0.7\n", + ")\n", + "\n", + "chunks = semantic_chunker.chunk(text)\n", + "\n", + "print(\"Semantic Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " coherence = chunk.metadata.get('coherence_score', 'N/A')\n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Length: {len(chunk.text)} chars\")\n", + " print(f\" Coherence: {coherence}\")\n", + " print(f\" Text: {chunk.text[:100]}...\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Entity-Aware Chunking for GraphRAG\n", + "\n", + "Entity-aware chunking preserves entity boundaries, crucial for GraphRAG workflows.\n", + "\n", + "### Why Entity-Aware?\n", + "\n", + "- **Preserve Entities**: Don't split \"Steve Jobs\" across chunks\n", + "- **Better Extraction**: Complete entities improve NER accuracy\n", + "- **GraphRAG**: Essential for knowledge graph construction" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import EntityAwareChunker\n", + "\n", + "# Entity-aware chunking\n", + "entity_chunker = EntityAwareChunker(\n", + " chunk_size=200,\n", + " chunk_overlap=50,\n", + " ner_method=\"spacy\", # or \"llm\" for better accuracy\n", + " preserve_entities=True\n", + ")\n", + "\n", + "chunks = entity_chunker.chunk(text)\n", + "\n", + "print(\"Entity-Aware Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " entities = chunk.metadata.get('entities', [])\n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Length: {len(chunk.text)} chars\")\n", + " print(f\" Entities: {len(entities)}\")\n", + " \n", + " if entities:\n", + " entity_texts = [e.get('text', e.get('entity', '')) if isinstance(e, dict) else str(e) for e in entities[:3]]\n", + " print(f\" Sample entities: {entity_texts}\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Relation-Aware Chunking\n", + "\n", + "Relation-aware chunking preserves relationship triples within chunks.\n", + "\n", + "### Why Relation-Aware?\n", + "\n", + "- **Preserve Triples**: Keep (subject, predicate, object) together\n", + "- **KG Construction**: Better for building knowledge graphs\n", + "- **Context**: Relationships need complete context" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import RelationAwareChunker\n", + "\n", + "# Relation-aware chunking\n", + "relation_chunker = RelationAwareChunker(\n", + " chunk_size=200,\n", + " chunk_overlap=50,\n", + " preserve_triples=True\n", + ")\n", + "\n", + "chunks = relation_chunker.chunk(text)\n", + "\n", + "print(\"Relation-Aware Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " triples = chunk.metadata.get('triples', [])\n", + " relationships = chunk.metadata.get('relationships', [])\n", + " \n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Length: {len(chunk.text)} chars\")\n", + " print(f\" Triples: {len(triples)}\")\n", + " print(f\" Relationships: {len(relationships)}\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Structural Chunking\n", + "\n", + "Structural chunking respects document structure like headings, paragraphs, and lists.\n", + "\n", + "### When to Use?\n", + "\n", + "- **Formatted Documents**: Markdown, HTML, structured text\n", + "- **Preserve Hierarchy**: Keep sections together\n", + "- **Better Context**: Headings provide context" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import StructuralChunker\n", + "\n", + "# Markdown text with structure\n", + "markdown_text = \"\"\"\n", + "# Apple Inc.\n", + "\n", + "## History\n", + "\n", + "Apple Inc. was founded by Steve Jobs, Steve Wozniak, and Ronald Wayne in 1976.\n", + "\n", + "## Products\n", + "\n", + "### Hardware\n", + "- iPhone\n", + "- iPad\n", + "- Mac\n", + "\n", + "### Software\n", + "- macOS\n", + "- iOS\n", + "- Safari\n", + "\"\"\"\n", + "\n", + "# Structural chunking\n", + "structural_chunker = StructuralChunker(\n", + " respect_headings=True,\n", + " respect_paragraphs=True,\n", + " respect_lists=True,\n", + " max_chunk_size=500\n", + ")\n", + "\n", + "chunks = structural_chunker.chunk(markdown_text)\n", + "\n", + "print(\"Structural Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " section = chunk.metadata.get('section_title', 'N/A')\n", + " level = chunk.metadata.get('heading_level', 'N/A')\n", + " \n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Section: {section}\")\n", + " print(f\" Level: {level}\")\n", + " print(f\" Text: {chunk.text[:80]}...\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 8: Hierarchical Chunking\n", + "\n", + "Hierarchical chunking creates multi-level chunks for large documents.\n", + "\n", + "### Benefits\n", + "\n", + "- **Multiple Granularities**: Document → Section → Paragraph\n", + "- **Better Navigation**: Parent-child relationships\n", + "- **Flexible Retrieval**: Query at different levels" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import HierarchicalChunker\n", + "\n", + "# Hierarchical chunking\n", + "hierarchical_chunker = HierarchicalChunker(\n", + " chunk_sizes=[400, 200, 100], # 3 levels\n", + " chunk_overlaps=[80, 40, 20],\n", + " create_parent_chunks=True\n", + ")\n", + "\n", + "chunks = hierarchical_chunker.chunk(text)\n", + "\n", + "print(\"Hierarchical Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " level = chunk.metadata.get('level', 'N/A')\n", + " parent_id = chunk.metadata.get('parent_id', None)\n", + " child_ids = chunk.metadata.get('child_ids', [])\n", + " \n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Level: {level}\")\n", + " print(f\" Length: {len(chunk.text)} chars\")\n", + " print(f\" Parent: {parent_id if parent_id else 'None (root)'}\")\n", + " print(f\" Children: {len(child_ids)}\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 9: Sliding Window Chunking\n", + "\n", + "Sliding window creates overlapping fixed-size chunks.\n", + "\n", + "### Use Cases\n", + "\n", + "- **Dense Retrieval**: Ensure no information is missed\n", + "- **Fixed Context**: Consistent chunk sizes\n", + "- **Overlap Control**: Precise overlap management" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import SlidingWindowChunker\n", + "\n", + "# Sliding window chunking\n", + "sliding_chunker = SlidingWindowChunker(\n", + " window_size=150,\n", + " step_size=100, # 50 char overlap\n", + " min_chunk_size=50\n", + ")\n", + "\n", + "chunks = sliding_chunker.chunk(text)\n", + "\n", + "print(\"Sliding Window Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " overlap = chunk.metadata.get('overlap_chars', 0)\n", + " \n", + " print(f\"\\nWindow {i}:\")\n", + " print(f\" Position: {chunk.start}-{chunk.end}\")\n", + " print(f\" Length: {len(chunk.text)} chars\")\n", + " print(f\" Overlap with previous: {overlap} chars\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 10: Table Chunking\n", + "\n", + "Table chunking preserves table structure while splitting large tables.\n", + "\n", + "### Features\n", + "\n", + "- **Preserve Headers**: Keep column headers in each chunk\n", + "- **Row-Based Splitting**: Split by rows, not characters\n", + "- **Context Inclusion**: Include surrounding text" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import TableChunker\n", + "\n", + "# Text with table\n", + "text_with_table = \"\"\"\n", + "Apple's product lineup includes:\n", + "\n", + "| Product | Category | Release Year |\n", + "|---------|----------|-------------|\n", + "| iPhone | Smartphone | 2007 |\n", + "| iPad | Tablet | 2010 |\n", + "| Mac | Computer | 1984 |\n", + "| Apple Watch | Wearable | 2015 |\n", + "| AirPods | Audio | 2016 |\n", + "\n", + "These products have revolutionized their respective categories.\n", + "\"\"\"\n", + "\n", + "# Table chunking\n", + "table_chunker = TableChunker(\n", + " preserve_headers=True,\n", + " max_rows_per_chunk=3,\n", + " include_context=True,\n", + " table_format=\"markdown\"\n", + ")\n", + "\n", + "chunks = table_chunker.chunk(text_with_table)\n", + "\n", + "print(\"Table Chunking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "for i, chunk in enumerate(chunks, 1):\n", + " is_table = chunk.metadata.get('is_table', False)\n", + " \n", + " print(f\"\\nChunk {i}:\")\n", + " print(f\" Type: {'Table' if is_table else 'Text'}\")\n", + " \n", + " if is_table:\n", + " rows = chunk.metadata.get('row_count', 'N/A')\n", + " cols = chunk.metadata.get('column_count', 'N/A')\n", + " print(f\" Rows: {rows}, Columns: {cols}\")\n", + " \n", + " print(f\" Content: {chunk.text[:100]}...\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 11: Chunk Validation\n", + "\n", + "Validate chunk quality to ensure optimal processing.\n", + "\n", + "### Validation Checks\n", + "\n", + "- **Size Constraints**: Min/max chunk size\n", + "- **Overlap**: Appropriate overlap percentage\n", + "- **Completeness**: Full text coverage\n", + "- **Quality Score**: Overall quality metric" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import ChunkValidator\n", + "\n", + "# Create chunks\n", + "splitter = TextSplitter(method=\"recursive\", chunk_size=200, chunk_overlap=50)\n", + "chunks = splitter.split(text)\n", + "\n", + "# Validate chunks\n", + "validator = ChunkValidator(\n", + " min_chunk_size=50,\n", + " max_chunk_size=300,\n", + " min_overlap=20,\n", + " max_overlap=100\n", + ")\n", + "\n", + "validation_result = validator.validate(chunks)\n", + "\n", + "print(\"Chunk Validation Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "print(f\"\\nOverall Valid: {validation_result.get('valid', False)}\")\n", + "print(f\"Quality Score: {validation_result.get('quality_score', 0):.2f}\")\n", + "\n", + "issues = validation_result.get('issues', [])\n", + "if issues:\n", + " print(f\"\\nIssues Found: {len(issues)}\")\n", + " for issue in issues[:3]:\n", + " print(f\" - {issue}\")\n", + "else:\n", + " print(\"\\nNo issues found!\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 12: Provenance Tracking\n", + "\n", + "Track chunk origins for data lineage and debugging.\n", + "\n", + "### Why Track Provenance?\n", + "\n", + "- **Data Lineage**: Know where chunks came from\n", + "- **Debugging**: Trace issues back to source\n", + "- **Compliance**: Required for some use cases" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from semantica.split import ProvenanceTracker\n", + "\n", + "# Create chunks\n", + "splitter = TextSplitter(method=\"recursive\", chunk_size=200, chunk_overlap=50)\n", + "chunks = splitter.split(text)\n", + "\n", + "# Track provenance\n", + "tracker = ProvenanceTracker()\n", + "\n", + "for chunk in chunks:\n", + " tracker.track(\n", + " chunk=chunk,\n", + " source={\n", + " \"document_id\": \"apple_doc_001\",\n", + " \"file_path\": \"data/apple.txt\",\n", + " \"timestamp\": \"2024-01-01T00:00:00Z\",\n", + " \"method\": \"recursive\"\n", + " }\n", + " )\n", + "\n", + "print(\"Provenance Tracking Results:\\n\")\n", + "print(\"=\" * 80)\n", + "\n", + "# Get lineage for first chunk\n", + "if chunks:\n", + " lineage = tracker.get_lineage(chunks[0].id)\n", + " \n", + " print(f\"\\nLineage for Chunk 1:\")\n", + " print(f\" Source Document: {lineage.get('source', {}).get('document_id')}\")\n", + " print(f\" File Path: {lineage.get('source', {}).get('file_path')}\")\n", + " print(f\" Method: {lineage.get('source', {}).get('method')}\")\n", + " print(f\" Timestamp: {lineage.get('source', {}).get('timestamp')}\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 13: Method Comparison\n", + "\n", + "Let's compare all methods side-by-side to help you choose the right one.\n", + "\n", + "### Comparison Criteria\n", + "\n", + "- **Chunk Count**: Number of chunks created\n", + "- **Average Size**: Average chunk size\n", + "- **Processing Time**: Speed of chunking" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "# Methods to compare\n", + "methods_to_compare = [\n", + " (\"recursive\", {}),\n", + " (\"sentence\", {}),\n", + " (\"paragraph\", {}),\n", + " (\"token\", {\"tokenizer\": \"tiktoken\"}),\n", + "]\n", + "\n", + "print(\"Method Comparison:\\n\")\n", + "print(\"=\" * 80)\n", + "print(f\"{'Method':<15} {'Chunks':<10} {'Avg Size':<12} {'Time (ms)':<12}\")\n", + "print(\"-\" * 80)\n", + "\n", + "for method, kwargs in methods_to_compare:\n", + " try:\n", + " start_time = time.time()\n", + " \n", + " splitter = TextSplitter(\n", + " method=method,\n", + " chunk_size=200,\n", + " chunk_overlap=50,\n", + " **kwargs\n", + " )\n", + " \n", + " chunks = splitter.split(text)\n", + " \n", + " elapsed = (time.time() - start_time) * 1000\n", + " avg_size = sum(len(c.text) for c in chunks) / len(chunks) if chunks else 0\n", + " \n", + " print(f\"{method:<15} {len(chunks):<10} {avg_size:<12.0f} {elapsed:<12.2f}\")\n", + " \n", + " except Exception as e:\n", + " print(f\"{method:<15} Error: {str(e)[:40]}\")\n", + "\n", + "print(\"=\" * 80)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 14: Best Practices\n", + "\n", + "### Choosing the Right Method\n", + "\n", + "1. **General Documents**: Use `recursive` for speed and simplicity\n", + "2. **LLM Applications**: Use `token` to respect context windows\n", + "3. **Semantic Search**: Use `semantic_transformer` for topic coherence\n", + "4. **GraphRAG**: Use `entity_aware` or `relation_aware`\n", + "5. **Structured Docs**: Use `structural` for formatted documents\n", + "6. **Large Documents**: Use `hierarchical` for multi-level access\n", + "\n", + "### Chunk Size Guidelines\n", + "\n", + "| Use Case | Recommended Size | Overlap |\n", + "|----------|------------------|----------|\n", + "| Semantic Search | 512-1024 chars | 20% |\n", + "| LLM Context | 2000-4000 chars | 10-20% |\n", + "| Entity Extraction | 500-1500 chars | 15-25% |\n", + "| Question Answering | 1000-2000 chars | 20% |\n", + "\n", + "### Overlap Recommendations\n", + "\n", + "- **10-15%**: Fast processing, less redundancy\n", + "- **20-25%**: Balanced (recommended)\n", + "- **30-40%**: Maximum context preservation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "### What You've Learned\n", + "\n", + "In this notebook, you've learned how to:\n", + "\n", + "- Use `TextSplitter` with multiple methods\n", + "- Apply standard splitting (recursive, token, sentence, paragraph)\n", + "- Use semantic chunking for topic coherence\n", + "- Apply KG-aware chunking (entity-aware, relation-aware)\n", + "- Use specialized chunkers (structural, hierarchical, sliding window, table)\n", + "- Validate chunk quality\n", + "- Track provenance\n", + "- Choose the right method for your use case\n", + "\n", + "### Key Takeaways\n", + "\n", + "1. **Method Selection Matters**: Different methods for different needs\n", + "2. **Chunk Size is Critical**: Balance between context and processing\n", + "3. **Overlap Helps**: 20% overlap is a good default\n", + "4. **Validate Quality**: Always validate chunks before use\n", + "5. **Track Provenance**: Important for debugging and compliance\n", + "6. **KG-Aware for GraphRAG**: Use entity/relation-aware for knowledge graphs\n", + "\n", + "### Next Steps\n", + "\n", + "**Next Notebook**: [12_Embedding_Generation.ipynb](./12_Embedding_Generation.ipynb) \n", + "Learn how to generate embeddings for your chunks!\n", + "\n", + "**Further Reading**:\n", + "- [Split Module API Reference](https://semantica.readthedocs.io/reference/split/)\n", + "- [Advanced Chunking Strategies](../advanced/11_Text_Chunking_Strategies.ipynb)\n", + "- [GraphRAG Pipeline](../use_cases/advanced_rag/01_GraphRAG_Complete.ipynb)\n", + "\n", + "---\n", + "\n", + "**Questions or Issues?** Check out our [GitHub repository](https://github.com/Hawksight-AI/semantica) or [documentation](https://semantica.readthedocs.io)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/docs/reference/split.md b/docs/reference/split.md index 90bcfc9d..a83d5629 100644 --- a/docs/reference/split.md +++ b/docs/reference/split.md @@ -416,6 +416,157 @@ for chunk in chunks: --- +### OntologyAwareChunker + +Chunk based on ontology concepts and relationships. + +**Methods:** + +| Method | Description | Algorithm | +|--------|-------------|-----------| +| `chunk(text, ontology)` | Chunk by ontology concepts | Concept boundary detection | +| `extract_concepts(text)` | Extract ontology concepts | Concept extraction | +| `find_concept_boundaries(text, concepts)` | Find concept boundaries | Concept span checking | + +**Parameters:** + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `chunk_size` | int | 1000 | Target chunk size | +| `chunk_overlap` | int | 200 | Overlap between chunks | +| `ontology_path` | str | None | Path to ontology file (.owl, .rdf) | +| `preserve_concepts` | bool | True | Don't split ontology concepts | +| `concept_extraction_method` | str | "llm" | Method for concept extraction | + +**Example:** + +```python +from semantica.split import OntologyAwareChunker + +chunker = OntologyAwareChunker( + chunk_size=1000, + chunk_overlap=200, + ontology_path="domain_ontology.owl", + preserve_concepts=True, + concept_extraction_method="llm" +) + +chunks = chunker.chunk(text) + +for chunk in chunks: + concepts = chunk.metadata.get('concepts', []) + print(f"Concepts in chunk: {[c['label'] for c in concepts]}") + print(f"Concept types: {[c['type'] for c in concepts]}") +``` + +--- + +### SlidingWindowChunker + +Fixed-size sliding window chunking with configurable step size. + +**Methods:** + +| Method | Description | Algorithm | +|--------|-------------|-----------| +| `chunk(text)` | Sliding window chunking | Fixed-size window with step | +| `calculate_windows(text_length)` | Calculate window positions | Window position calculation | + +**Parameters:** + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `window_size` | int | 1000 | Size of sliding window | +| `step_size` | int | 800 | Step size (window_size - overlap) | +| `min_chunk_size` | int | 100 | Minimum chunk size | +| `preserve_sentences` | bool | False | Preserve sentence boundaries | + +**Example:** + +```python +from semantica.split import SlidingWindowChunker + +# Basic sliding window +chunker = SlidingWindowChunker( + window_size=1000, + step_size=800, # 200 overlap + min_chunk_size=100 +) + +chunks = chunker.chunk(long_text) + +for i, chunk in enumerate(chunks): + print(f"Window {i}: chars {chunk.start}-{chunk.end}") + print(f"Overlap with previous: {chunk.metadata.get('overlap_chars')}") + +# Sentence-preserving sliding window +chunker = SlidingWindowChunker( + window_size=1000, + step_size=750, + preserve_sentences=True +) + +chunks = chunker.chunk(text) +``` + +--- + +### TableChunker + +Table-specific chunking preserving table structure. + +**Methods:** + +| Method | Description | Algorithm | +|--------|-------------|-----------| +| `chunk(text)` | Chunk tables | Table detection and splitting | +| `detect_tables(text)` | Detect tables in text | Table boundary detection | +| `split_table(table, max_rows)` | Split large tables | Row-based table splitting | + +**Parameters:** + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `preserve_headers` | bool | True | Keep headers in each chunk | +| `max_rows_per_chunk` | int | 50 | Maximum rows per table chunk | +| `include_context` | bool | True | Include surrounding text context | +| `table_format` | str | "auto" | Table format (markdown, html, csv, auto) | + +**Example:** + +```python +from semantica.split import TableChunker + +chunker = TableChunker( + preserve_headers=True, + max_rows_per_chunk=50, + include_context=True, + table_format="markdown" +) + +text_with_tables = \"\"\" +Document with tables... + +| Column 1 | Column 2 | Column 3 | +|----------|----------|----------| +| Value 1 | Value 2 | Value 3 | +| ... | ... | ... | +\"\"\" + +chunks = chunker.chunk(text_with_tables) + +for chunk in chunks: + if chunk.metadata.get('is_table'): + print(f"Table chunk:") + print(f" Rows: {chunk.metadata.get('row_count')}") + print(f" Columns: {chunk.metadata.get('column_count')}") + print(f" Headers: {chunk.metadata.get('headers')}") + else: + print(f"Text chunk: {len(chunk.text)} chars") +``` + +--- + ### ChunkValidator Validate chunk quality and completeness. diff --git a/semantica/split/split_usage.md b/semantica/split/split_usage.md index 9c34ea52..dae4b869 100644 --- a/semantica/split/split_usage.md +++ b/semantica/split/split_usage.md @@ -285,7 +285,12 @@ from semantica.split import ( SemanticChunker, StructuralChunker, SlidingWindowChunker, - EntityAwareChunker + TableChunker, + EntityAwareChunker, + RelationAwareChunker, + GraphBasedChunker, + OntologyAwareChunker, + HierarchicalChunker ) # Semantic chunking @@ -301,17 +306,60 @@ chunks = structural_chunker.chunk(text) # Sliding window chunking sliding_chunker = SlidingWindowChunker( - chunk_size=1000, - chunk_overlap=200 + window_size=1000, + step_size=800, # 200 overlap + min_chunk_size=100 ) chunks = sliding_chunker.chunk(text) +# Table-specific chunking +table_chunker = TableChunker( + preserve_headers=True, + max_rows_per_chunk=50, + include_context=True +) +chunks = table_chunker.chunk(text_with_tables) + # Entity-aware chunking entity_chunker = EntityAwareChunker( chunk_size=1000, - ner_method="llm" + chunk_overlap=200, + ner_method="llm", + preserve_entities=True ) chunks = entity_chunker.chunk(text) + +# Relation-aware chunking +relation_chunker = RelationAwareChunker( + chunk_size=1000, + preserve_triples=True +) +chunks = relation_chunker.chunk(text) + +# Graph-based chunking +graph_chunker = GraphBasedChunker( + chunk_size=1000, + centrality_method="betweenness", + community_algorithm="louvain" +) +chunks = graph_chunker.chunk(text, graph=knowledge_graph) + +# Ontology-aware chunking +ontology_chunker = OntologyAwareChunker( + chunk_size=1000, + chunk_overlap=200, + ontology_path="domain_ontology.owl", + preserve_concepts=True +) +chunks = ontology_chunker.chunk(text) + +# Hierarchical chunking +hierarchical_chunker = HierarchicalChunker( + chunk_sizes=[2000, 1000, 500], + chunk_overlaps=[400, 200, 100], + create_parent_chunks=True +) +chunks = hierarchical_chunker.chunk(text) ``` ## Using Methods diff --git a/update_notebook.py b/update_notebook.py deleted file mode 100644 index e136fba6..00000000 --- a/update_notebook.py +++ /dev/null @@ -1,43 +0,0 @@ -import json - -# Read the notebook -with open(r'c:\Users\Mohd Kaif\semantica\cookbook\use_cases\advanced_rag\01_GraphRAG_Complete.ipynb', 'r', encoding='utf-8') as f: - data = json.load(f) - -# Find and update the cell that uses build function (cell 14, lines 522-546) -# Replace the build function usage with class-based approach -data['cells'][14]['source'] = [ - "from semantica.semantic_extract import NamedEntityRecognizer, RelationExtractor, TripleExtractor\n", - "\n", - "print(\"Extracting entities, relationships, and triples...\")\n", - "\n", - "ner = NamedEntityRecognizer()\n", - "rel_extractor = RelationExtractor()\n", - "triple_extractor = TripleExtractor()\n", - "\n", - "flat_entities = []\n", - "flat_relationships = []\n", - "flat_triples = []\n", - "\n", - "for doc in normalized_documents:\n", - " text = str(doc.content) if hasattr(doc, 'content') else str(doc)\n", - " \n", - " entities = ner.extract_entities(text)\n", - " flat_entities.extend(entities if isinstance(entities, list) else [entities])\n", - " \n", - " relations = rel_extractor.extract_relations(text, entities=entities)\n", - " flat_relationships.extend(relations if isinstance(relations, list) else [relations])\n", - " \n", - " triples = triple_extractor.extract_triples(text, entities=entities, relationships=relations)\n", - " flat_triples.extend(triples if isinstance(triples, list) else [triples])\n", - "\n", - "print(f\"Extracted {len(flat_entities)} entities\")\n", - "print(f\"Extracted {len(flat_relationships)} relationships\")\n", - "print(f\"Extracted {len(flat_triples)} triples\")\n" -] - -# Write back the notebook -with open(r'c:\Users\Mohd Kaif\semantica\cookbook\use_cases\advanced_rag\01_GraphRAG_Complete.ipynb', 'w', encoding='utf-8') as f: - json.dump(data, f, indent=1, ensure_ascii=False) - -print("Updated notebook successfully!")