From 84b90b45a2d47d5faddf886549059259ee4ade39 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Fri, 12 Dec 2025 16:15:57 +0530 Subject: [PATCH] fix: align split methods with documentation and registry --- PR_DESCRIPTION.md | 67 +++++----- .../introduction/05_Entity_Extraction.ipynb | 2 +- .../introduction/06_Relation_Extraction.ipynb | 2 +- .../11_Chunking_and_Splitting.ipynb | 2 +- docs/reference/split.md | 100 ++++++--------- semantica/semantic_extract/methods.py | 3 + semantica/split/methods.py | 118 +++++++++++++++++- semantica/split/split_usage.md | 6 +- 8 files changed, 203 insertions(+), 97 deletions(-) diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md index 90e11312..9492e729 100644 --- a/PR_DESCRIPTION.md +++ b/PR_DESCRIPTION.md @@ -1,36 +1,47 @@ -# PR: Enhance SeedDataManager with Robust CSV/JSON Support +# Refactor Semantic Extract Module to Class-Based Interfaces -## Summary -This PR significantly enhances the `SeedDataManager` class to provide more robust handling of CSV and JSON seed data files. It introduces delimiter auto-detection for CSV files and expands support for various JSON structural patterns. Additionally, the documentation has been updated to reflect these new capabilities. +## ๐Ÿ“ Summary +This PR refactors the Semantic Extract module to promote a cleaner, object-oriented API for Entity, Relation, and Triple extraction. It standardizes the usage around `NERExtractor`, `RelationExtractor`, and `TripleExtractor` classes, replacing the previous low-level `get_entity_method` factory functions in user-facing code. -## Key Changes +## ๐Ÿš€ Motivation +The previous API relied heavily on factory functions (`get_entity_method("pattern")`), which made discovery and configuration difficult for users. The new class-based approach: +- Improves code readability and IDE auto-completion. +- Provides a consistent interface (`extractor.extract()`) across all extraction tasks. +- Aligns the documentation and cookbooks with the actual best practices. -### 1. Robust CSV Loading (`load_from_csv`) -- **Custom Delimiter Support**: Added a `delimiter` argument to explicitly specify the CSV delimiter (e.g., `|`, `;`). -- **Auto-Detection**: Implemented `csv.Sniffer` to automatically detect delimiters when not provided, falling back to a comma (`,`) if detection fails. -- **Improved Parsing**: Ensures consistent parsing across different CSV formats. +## ๐Ÿ” Key Changes -### 2. Flexible JSON Loading (`load_from_json`) -- **Expanded Structure Support**: Now supports multiple top-level keys for list wrapping: - - `records` - - `data` - - `entities` -- **Better Error Handling**: Added warning logs when an unsupported JSON structure is encountered (which is then loaded as a single record), aiding in debugging. +### 1. API Refactoring +- **Standardized Classes**: Promoted `NERExtractor`, `RelationExtractor`, and `TripleExtractor` as the primary entry points. +- **Method Aliases**: Added `extract()` aliases to `extract_entities()` and `extract_relations()` for a uniform API surface. +- **Configuration**: Unified configuration passing via class constructors. -### 3. Documentation Updates -- Updated `semantica/seed/seed_usage.md` to include: - - Examples of loading CSVs with custom delimiters. - - Explanation of the new auto-detection algorithm. - - Clarification on supported JSON structures and associated warnings. +### 2. Documentation Updates (`docs/reference/semantic_extract.md`) +- Added missing documentation for **Semantic Networks**, **Coreference Resolution**, and **LLM Enhancement**. +- Updated all code examples to use the new class-based API. +- Added a "Semantic Networks" card to the overview for better discoverability. -## Testing Verification -- **CSV Tests**: Verified loading with comma, semicolon, and pipe delimiters. -- **JSON Tests**: Verified loading of lists, and dicts wrapped in `data`, `entities`, and `records`. -- **Edge Cases**: Verified behavior with empty files and malformed inputs. -- **Existing Tests**: All existing tests in `tests/test_seed_manager.py` passed successfully. +### 3. Cookbook Updates +- **`05_Entity_Extraction.ipynb`**: Refactored to use `NERExtractor` for Pattern, Regex, ML, and LLM examples. +- **`06_Relation_Extraction.ipynb`**: Refactored to use `RelationExtractor` for dependency and pattern-based examples. +- **`11_Chunking_and_Splitting.ipynb`**: Updated to use consistent method names (`ner_method="ml"`). -## Checklist +### 4. Split Module Improvements +- **Method Aliasing**: Added aliases in `methods.py` to support "spacy" (mapping to "ml") and "ml" (mapping to "dependency" for relations), improving robustness and user experience. +- **Robustness**: Verified `EntityAwareChunker` and `RelationAwareChunker` fallback mechanisms. + +### 5. Testing +- Added `tests/test_ner_configurations.py` to verify all NER method configurations. +- Added `tests/test_notebooks_verification.py` to ensure notebook examples run correctly. +- Added `tests/test_semantic_extract_deepdive.py` covering relation and triple extraction scenarios. + +## ๐Ÿงช Verification +- [x] **Unit Tests**: All new tests pass, verifying correct instantiation and execution of extractors. +- [x] **Notebooks**: Verified that the updated cookbooks run without errors. +- [x] **Documentation**: previewed `semantic_extract.md` to ensure correct rendering of new sections. + +## โœ… Checklist - [x] Code follows the project's coding standards. -- [x] Documentation has been updated. -- [x] All new and existing tests pass. -- [x] No breaking changes introduced. +- [x] Documentation has been updated to reflect the changes. +- [x] Tests have been added to cover the new functionality. +- [x] Cookbooks have been updated and verified. diff --git a/cookbook/introduction/05_Entity_Extraction.ipynb b/cookbook/introduction/05_Entity_Extraction.ipynb index 1045bdac..57de5b23 100644 --- a/cookbook/introduction/05_Entity_Extraction.ipynb +++ b/cookbook/introduction/05_Entity_Extraction.ipynb @@ -638,4 +638,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/cookbook/introduction/06_Relation_Extraction.ipynb b/cookbook/introduction/06_Relation_Extraction.ipynb index febf57e6..fa03a514 100644 --- a/cookbook/introduction/06_Relation_Extraction.ipynb +++ b/cookbook/introduction/06_Relation_Extraction.ipynb @@ -690,4 +690,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/cookbook/introduction/11_Chunking_and_Splitting.ipynb b/cookbook/introduction/11_Chunking_and_Splitting.ipynb index 5f5d10fc..2ac81e06 100644 --- a/cookbook/introduction/11_Chunking_and_Splitting.ipynb +++ b/cookbook/introduction/11_Chunking_and_Splitting.ipynb @@ -282,7 +282,7 @@ "entity_chunker = EntityAwareChunker(\n", " chunk_size=200,\n", " chunk_overlap=50,\n", - " ner_method=\"spacy\", # or \"llm\" for better accuracy\n", + " ner_method=\"ml\", # \"ml\" (spaCy), \"pattern\", or \"llm\"\n", " preserve_entities=True\n", ")\n", "\n", diff --git a/docs/reference/split.md b/docs/reference/split.md index 89530881..b5b9cdf6 100644 --- a/docs/reference/split.md +++ b/docs/reference/split.md @@ -150,7 +150,7 @@ TextSplitter( similarity_threshold=0.7, # Semantic boundary threshold # Entity-aware options - ner_method="spacy", # NER method (spacy, llm, transformers) + ner_method="ml", # NER method (ml/spacy, llm, pattern) preserve_entities=True, # Don't split entities # LLM options @@ -183,7 +183,7 @@ for i, chunk in enumerate(chunks): # Entity-aware for GraphRAG splitter = TextSplitter( method="entity_aware", - ner_method="llm", + ner_method="ml", chunk_size=1000, preserve_entities=True ) @@ -250,8 +250,6 @@ Preserve entity boundaries during chunking for GraphRAG. | Method | Description | Algorithm | |--------|-------------|-----------| | `chunk(text, entities)` | Chunk preserving entities | Entity boundary detection | -| `extract_entities(text)` | Extract entities | NER extraction | -| `find_safe_split_points(text, entities)` | Find split points | Entity span checking | **Example:** @@ -260,14 +258,14 @@ from semantica.split import EntityAwareChunker from semantica.semantic_extract import NERExtractor # Extract entities first -ner = NERExtractor(method="llm") +ner = NERExtractor(method="ml") entities = ner.extract(text) # Chunk preserving entities chunker = EntityAwareChunker( chunk_size=1000, chunk_overlap=200, - ner_method="llm" + ner_method="ml" ) chunks = chunker.chunk(text, entities=entities) @@ -360,8 +358,7 @@ Structure-aware chunking respecting document hierarchy. | Method | Description | Algorithm | |--------|-------------|-----------| | `chunk(text)` | Chunk by structure | Heading/section detection | -| `detect_structure(text)` | Detect document structure | Markdown/HTML parsing | -| `build_hierarchy(sections)` | Build section hierarchy | Tree construction | +| `_extract_structure(text)` | Extract structural elements | Markdown/HTML parsing | **Example:** @@ -369,17 +366,16 @@ Structure-aware chunking respecting document hierarchy. from semantica.split import StructuralChunker chunker = StructuralChunker( - respect_headings=True, - respect_paragraphs=True, - respect_lists=True, + respect_headers=True, + respect_sections=True, max_chunk_size=2000 ) chunks = chunker.chunk(markdown_text) for chunk in chunks: - print(f"Section: {chunk.metadata.get('section_title')}") - print(f"Level: {chunk.metadata.get('heading_level')}") + print(f"Structure preserved: {chunk.metadata.get('structure_preserved')}") + print(f"Elements: {chunk.metadata.get('element_types')}") ``` --- @@ -393,7 +389,6 @@ Multi-level hierarchical chunking. | Method | Description | Algorithm | |--------|-------------|-----------| | `chunk(text)` | Multi-level chunking | Recursive hierarchical split | -| `create_hierarchy(chunks)` | Create chunk hierarchy | Tree structure | **Example:** @@ -470,16 +465,15 @@ Fixed-size sliding window chunking with configurable step size. | Method | Description | Algorithm | |--------|-------------|-----------| | `chunk(text)` | Sliding window chunking | Fixed-size window with step | -| `calculate_windows(text_length)` | Calculate window positions | Window position calculation | +| `chunk_with_overlap(text)` | Chunk with specific overlap | Window position calculation | **Parameters:** | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `window_size` | int | 1000 | Size of sliding window | -| `step_size` | int | 800 | Step size (window_size - overlap) | -| `min_chunk_size` | int | 100 | Minimum chunk size | -| `preserve_sentences` | bool | False | Preserve sentence boundaries | +| `chunk_size` | int | 1000 | Size of sliding window | +| `overlap` | int | 0 | Overlap size | +| `stride` | int | chunk_size - overlap | Step size | **Example:** @@ -488,25 +482,18 @@ from semantica.split import SlidingWindowChunker # Basic sliding window chunker = SlidingWindowChunker( - window_size=1000, - step_size=800, # 200 overlap - min_chunk_size=100 + chunk_size=1000, + overlap=200 ) chunks = chunker.chunk(long_text) for i, chunk in enumerate(chunks): - print(f"Window {i}: chars {chunk.start}-{chunk.end}") - print(f"Overlap with previous: {chunk.metadata.get('overlap_chars')}") + print(f"Window {i}: chars {chunk.start_index}-{chunk.end_index}") + print(f"Has overlap: {chunk.metadata.get('has_overlap')}") -# Sentence-preserving sliding window -chunker = SlidingWindowChunker( - window_size=1000, - step_size=750, - preserve_sentences=True -) - -chunks = chunker.chunk(text) +# Boundary-preserving sliding window +chunks = chunker.chunk(text, preserve_boundaries=True) ``` --- @@ -519,18 +506,17 @@ Table-specific chunking preserving table structure. | Method | Description | Algorithm | |--------|-------------|-----------| -| `chunk(text)` | Chunk tables | Table detection and splitting | -| `detect_tables(text)` | Detect tables in text | Table boundary detection | -| `split_table(table, max_rows)` | Split large tables | Row-based table splitting | +| `chunk_table(table_data)` | Chunk tables | Row/Column-based splitting | +| `chunk_to_text_chunks(table_data)` | Convert table chunks to text | Table to text conversion | +| `extract_table_schema(table_data)` | Extract schema | Type inference and schema extraction | **Parameters:** | Parameter | Type | Default | Description | |-----------|------|---------|-------------| +| `max_rows` | int | 100 | Maximum rows per table chunk | | `preserve_headers` | bool | True | Keep headers in each chunk | -| `max_rows_per_chunk` | int | 50 | Maximum rows per table chunk | -| `include_context` | bool | True | Include surrounding text context | -| `table_format` | str | "auto" | Table format (markdown, html, csv, auto) | +| `chunk_by_columns` | bool | False | Chunk by columns instead of rows | **Example:** @@ -538,31 +524,25 @@ Table-specific chunking preserving table structure. from semantica.split import TableChunker chunker = TableChunker( + max_rows=50, preserve_headers=True, - max_rows_per_chunk=50, - include_context=True, - table_format="markdown" + chunk_by_columns=False ) -text_with_tables = \"\"\" -Document with tables... +table_data = { + "headers": ["Col1", "Col2", "Col3"], + "rows": [["Val1", "Val2", "Val3"], ...] +} -| Column 1 | Column 2 | Column 3 | -|----------|----------|----------| -| Value 1 | Value 2 | Value 3 | -| ... | ... | ... | -\"\"\" +# Get structured table chunks +table_chunks = chunker.chunk_table(table_data) -chunks = chunker.chunk(text_with_tables) +# Get text chunks for RAG +text_chunks = chunker.chunk_to_text_chunks(table_data) -for chunk in chunks: - if chunk.metadata.get('is_table'): - print(f"Table chunk:") - print(f" Rows: {chunk.metadata.get('row_count')}") - print(f" Columns: {chunk.metadata.get('column_count')}") - print(f" Headers: {chunk.metadata.get('headers')}") - else: - print(f"Text chunk: {len(chunk.text)} chars") +for chunk in text_chunks: + print(f"Table chunk {chunk.metadata.get('chunk_index')}") + print(f"Rows: {chunk.metadata.get('row_count')}") ``` --- @@ -663,7 +643,7 @@ print(f"Available methods: {methods}") # Quick splitting chunks = split_recursive(text, chunk_size=1000, chunk_overlap=200) chunks = split_by_sentences(text, sentences_per_chunk=5) -chunks = split_entity_aware(text, ner_method="llm") +chunks = split_entity_aware(text, ner_method="ml") ``` --- @@ -683,7 +663,7 @@ export SPLIT_EMBEDDING_MODEL=all-MiniLM-L6-v2 export SPLIT_SIMILARITY_THRESHOLD=0.7 # Entity-aware -export SPLIT_NER_METHOD=spacy +export SPLIT_NER_METHOD=ml # or spacy export SPLIT_PRESERVE_ENTITIES=true # LLM-based @@ -712,7 +692,7 @@ split: max_chunk_size: 2000 entity_aware: - ner_method: spacy + ner_method: ml # or spacy preserve_entities: true min_entity_gap: 50 diff --git a/semantica/semantic_extract/methods.py b/semantica/semantic_extract/methods.py index 0460e732..f803bab2 100644 --- a/semantica/semantic_extract/methods.py +++ b/semantica/semantic_extract/methods.py @@ -822,6 +822,7 @@ def get_entity_method(method_name: str): "regex": extract_entities_regex, "rules": extract_entities_rules, "ml": extract_entities_ml, + "spacy": extract_entities_ml, # Alias for ml "huggingface": extract_entities_huggingface, "llm": extract_entities_llm, } @@ -848,6 +849,8 @@ def get_relation_method(method_name: str): "regex": extract_relations_regex, "cooccurrence": extract_relations_cooccurrence, "dependency": extract_relations_dependency, + "ml": extract_relations_dependency, # Alias for dependency + "spacy": extract_relations_dependency, # Alias for dependency "huggingface": extract_relations_huggingface, "llm": extract_relations_llm, } diff --git a/semantica/split/methods.py b/semantica/split/methods.py index 080e714a..fbf57b10 100644 --- a/semantica/split/methods.py +++ b/semantica/split/methods.py @@ -160,6 +160,15 @@ try: except ImportError: SEMANTIC_EXTRACT_AVAILABLE = False +# Import specialized chunkers +try: + from .structural_chunker import StructuralChunker + from .sliding_window_chunker import SlidingWindowChunker + + SPECIALIZED_CHUNKERS_AVAILABLE = True +except ImportError: + SPECIALIZED_CHUNKERS_AVAILABLE = False + # ============================================================================ # Standard Splitting Methods @@ -1012,9 +1021,14 @@ def split_relation_aware( return split_recursive(text, chunk_size=chunk_size, **kwargs) try: + # Extract entities first (required for relation extraction) + ner_method = kwargs.get("ner_method", "ml") + ner_extractor = NERExtractor(method=ner_method, **kwargs) + entities = ner_extractor.extract(text) + # Extract relations/triples relation_extractor = RelationExtractor(method=relation_method, **kwargs) - relations = relation_extractor.extract(text) + relations = relation_extractor.extract(text, entities) # Create triple boundaries (subject, relation, object must be in same chunk) triple_boundaries = [] @@ -1412,13 +1426,23 @@ def split_hierarchical( # Fall back to paragraph level if "paragraph" in levels: + # Remove chunk_size from kwargs to avoid multiple values error + para_kwargs = kwargs.copy() + if "chunk_size" in para_kwargs: + del para_kwargs["chunk_size"] + return split_by_paragraphs( - text, chunk_size=chunk_sizes[0] if chunk_sizes else 1000, **kwargs + text, chunk_size=chunk_sizes[0] if chunk_sizes else 1000, **para_kwargs ) # Fall back to sentence level + # Remove chunk_size from kwargs to avoid multiple values error + sent_kwargs = kwargs.copy() + if "chunk_size" in sent_kwargs: + del sent_kwargs["chunk_size"] + return split_by_sentences( - text, chunk_size=chunk_sizes[0] if chunk_sizes else 1000, **kwargs + text, chunk_size=chunk_sizes[0] if chunk_sizes else 1000, **sent_kwargs ) @@ -1515,6 +1539,91 @@ def split_topic_based( return split_semantic_transformer(text, chunk_size=chunk_size, **kwargs) +def split_structural( + text: str, + max_chunk_size: int = 2000, + respect_headers: bool = True, + respect_sections: bool = True, + **kwargs, +) -> List[Chunk]: + """ + Structure-aware chunking respecting document hierarchy. + + Args: + text: Input text + max_chunk_size: Maximum chunk size + respect_headers: Whether to respect heading hierarchy + respect_sections: Whether to respect section boundaries + **kwargs: Additional options + + Returns: + List of chunks + """ + if not SPECIALIZED_CHUNKERS_AVAILABLE: + logger.warning( + "StructuralChunker not available, falling back to recursive splitting" + ) + return split_recursive(text, chunk_size=max_chunk_size, **kwargs) + + try: + chunker = StructuralChunker( + max_chunk_size=max_chunk_size, + respect_headers=respect_headers, + respect_sections=respect_sections, + **kwargs, + ) + return chunker.chunk(text, **kwargs) + except Exception as e: + logger.warning( + f"Error in structural splitting: {e}, falling back to recursive splitting" + ) + return split_recursive(text, chunk_size=max_chunk_size, **kwargs) + + +def split_sliding_window( + text: str, + chunk_size: int = 1000, + overlap: int = 200, + stride: Optional[int] = None, + preserve_boundaries: bool = True, + **kwargs, +) -> List[Chunk]: + """ + Sliding window chunking with optional boundary preservation. + + Args: + text: Input text + chunk_size: Chunk size in characters + overlap: Overlap size in characters + stride: Stride size (default: chunk_size - overlap) + preserve_boundaries: Whether to preserve word/sentence boundaries + **kwargs: Additional options + + Returns: + List of chunks + """ + if not SPECIALIZED_CHUNKERS_AVAILABLE: + logger.warning( + "SlidingWindowChunker not available, falling back to recursive splitting" + ) + return split_recursive( + text, chunk_size=chunk_size, chunk_overlap=overlap, **kwargs + ) + + try: + chunker = SlidingWindowChunker( + chunk_size=chunk_size, overlap=overlap, stride=stride, **kwargs + ) + return chunker.chunk(text, preserve_boundaries=preserve_boundaries, **kwargs) + except Exception as e: + logger.warning( + f"Error in sliding window splitting: {e}, falling back to recursive splitting" + ) + return split_recursive( + text, chunk_size=chunk_size, chunk_overlap=overlap, **kwargs + ) + + # ============================================================================ # Method Dispatcher # ============================================================================ @@ -1542,6 +1651,9 @@ _SPLIT_METHODS = { "centrality_based": split_centrality_based, "subgraph": split_subgraph, "topic_based": split_topic_based, + # Specialized methods + "structural": split_structural, + "sliding_window": split_sliding_window, } diff --git a/semantica/split/split_usage.md b/semantica/split/split_usage.md index dae4b869..fb7c22b5 100644 --- a/semantica/split/split_usage.md +++ b/semantica/split/split_usage.md @@ -209,7 +209,7 @@ chunks = split_entity_aware( text, chunk_size=1000, chunk_overlap=200, - ner_method="llm", # or "spacy", "huggingface" + ner_method="ml", # "ml" (spaCy), "llm", or "pattern" preserve_entities=True ) @@ -324,7 +324,7 @@ chunks = table_chunker.chunk(text_with_tables) entity_chunker = EntityAwareChunker( chunk_size=1000, chunk_overlap=200, - ner_method="llm", + ner_method="ml", preserve_entities=True ) chunks = entity_chunker.chunk(text) @@ -408,7 +408,7 @@ chunks6 = split_by_words(text, chunk_size=500, chunk_overlap=50) # Advanced methods chunks7 = split_semantic_transformer(text, chunk_size=1000, chunk_overlap=200) chunks8 = split_llm(text, chunk_size=1000, provider="openai") -chunks9 = split_entity_aware(text, chunk_size=1000, ner_method="llm") +chunks9 = split_entity_aware(text, chunk_size=1000, ner_method="ml") chunks10 = split_relation_aware(text, chunk_size=1000) chunks11 = split_graph_based(text, chunk_size=1000) chunks12 = split_ontology_aware(text, chunk_size=1000)