From 05caff91fa1900cd512fb5e646cff1b04e443de6 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Wed, 12 Nov 2025 17:56:45 +0530 Subject: [PATCH] Add progress tracking to seed, semantic_extract, and split modules - Added progress tracking to seed module (seed_manager.py) - Added progress tracking to all semantic_extract files (10 files): * triple_extractor.py * semantic_network_extractor.py * relation_extractor.py * semantic_analyzer.py * ner_extractor.py * named_entity_recognizer.py * llm_enhancer.py * extraction_validator.py * event_detector.py * coreference_resolver.py - Added progress tracking to all split files (6 files): * semantic_chunker.py * sliding_window_chunker.py * structural_chunker.py * table_chunker.py * chunk_validator.py * provenance_tracker.py - All methods now use start_tracking, update_tracking, and stop_tracking - Consistent error handling with try-except blocks across all modules --- semantica/seed/seed_manager.py | 209 +++++++++++------- .../semantic_extract/coreference_resolver.py | 43 +++- semantica/semantic_extract/event_detector.py | 64 ++++-- .../semantic_extract/extraction_validator.py | 109 +++++---- semantica/semantic_extract/llm_enhancer.py | 27 ++- .../named_entity_recognizer.py | 17 +- semantica/semantic_extract/ner_extractor.py | 38 +++- .../semantic_extract/relation_extractor.py | 40 +++- .../semantic_extract/semantic_analyzer.py | 46 ++-- .../semantic_network_extractor.py | 50 +++-- .../semantic_extract/triple_extractor.py | 83 ++++--- semantica/split/chunk_validator.py | 102 +++++---- semantica/split/provenance_tracker.py | 44 ++-- semantica/split/semantic_chunker.py | 34 ++- semantica/split/sliding_window_chunker.py | 36 ++- semantica/split/structural_chunker.py | 36 ++- semantica/split/table_chunker.py | 57 +++-- 17 files changed, 685 insertions(+), 350 deletions(-) diff --git a/semantica/seed/seed_manager.py b/semantica/seed/seed_manager.py index e7264c0a..71569de0 100644 --- a/semantica/seed/seed_manager.py +++ b/semantica/seed/seed_manager.py @@ -42,6 +42,7 @@ from ..utils.exceptions import ValidationError, ProcessingError from ..utils.logging import get_logger from ..utils.helpers import read_json_file, write_json_file from ..utils.types import EntityDict, RelationshipDict +from ..utils.progress_tracker import get_progress_tracker @dataclass @@ -122,6 +123,7 @@ class SeedDataManager: self.logger = get_logger("seed_manager") self.config = config or {} self.config.update(kwargs) + self.progress_tracker = get_progress_tracker() self.sources: Dict[str, SeedDataSource] = {} self.seed_data: SeedData = SeedData() @@ -213,13 +215,21 @@ class SeedDataManager: >>> records = manager.load_from_csv("data/entities.csv", entity_type="Person") >>> print(f"Loaded {len(records)} records") """ - file_path = Path(file_path) - if not file_path.exists(): - raise ProcessingError(f"CSV file not found: {file_path}") - - records = [] + tracking_id = self.progress_tracker.start_tracking( + module="seed", + submodule="SeedDataManager", + message=f"Loading seed data from CSV: {file_path}" + ) try: + file_path = Path(file_path) + if not file_path.exists(): + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=f"CSV file not found: {file_path}") + raise ProcessingError(f"CSV file not found: {file_path}") + + records = [] + + self.progress_tracker.update_tracking(tracking_id, message="Reading CSV file...") with open(file_path, 'r', encoding='utf-8') as f: reader = csv.DictReader(f) for row in reader: @@ -236,9 +246,12 @@ class SeedDataManager: records.append(record) self.logger.info(f"Loaded {len(records)} records from CSV: {file_path}") + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Loaded {len(records)} records from CSV") return records except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) raise ProcessingError(f"Failed to load CSV: {e}") from e def load_from_json( @@ -565,47 +578,62 @@ class SeedDataManager: >>> foundation = manager.create_foundation_graph() >>> print(f"Created graph with {len(foundation['entities'])} entities") """ - foundation = { - "entities": [], - "relationships": [], - "metadata": { - "created_at": datetime.now().isoformat(), - "source_count": len(self.sources), - "verified": True - } - } - - # Load data from all sources - for source_name in self.sources: - try: - records = self.load_source(source_name) - - for record in records: - # Extract entities - if 'entity_type' in record or 'id' in record: - entity = self._record_to_entity(record) - if entity: - foundation["entities"].append(entity) - - # Extract relationships - if 'relationship_type' in record or ('source_id' in record and 'target_id' in record): - relationship = self._record_to_relationship(record) - if relationship: - foundation["relationships"].append(relationship) - - except Exception as e: - self.logger.warning(f"Failed to load source '{source_name}': {e}") - - # Validate against schema template if provided - if schema_template: - foundation = self._validate_against_template(foundation, schema_template) - - self.logger.info( - f"Created foundation graph: {len(foundation['entities'])} entities, " - f"{len(foundation['relationships'])} relationships" + tracking_id = self.progress_tracker.start_tracking( + module="seed", + submodule="SeedDataManager", + message="Creating foundation graph from seed data" ) - return foundation + try: + foundation = { + "entities": [], + "relationships": [], + "metadata": { + "created_at": datetime.now().isoformat(), + "source_count": len(self.sources), + "verified": True + } + } + + # Load data from all sources + self.progress_tracker.update_tracking(tracking_id, message=f"Loading data from {len(self.sources)} sources...") + for source_name in self.sources: + try: + self.progress_tracker.update_tracking(tracking_id, message=f"Loading source: {source_name}") + records = self.load_source(source_name) + + for record in records: + # Extract entities + if 'entity_type' in record or 'id' in record: + entity = self._record_to_entity(record) + if entity: + foundation["entities"].append(entity) + + # Extract relationships + if 'relationship_type' in record or ('source_id' in record and 'target_id' in record): + relationship = self._record_to_relationship(record) + if relationship: + foundation["relationships"].append(relationship) + + except Exception as e: + self.logger.warning(f"Failed to load source '{source_name}': {e}") + + # Validate against schema template if provided + if schema_template: + self.progress_tracker.update_tracking(tracking_id, message="Validating against schema template...") + foundation = self._validate_against_template(foundation, schema_template) + + self.logger.info( + f"Created foundation graph: {len(foundation['entities'])} entities, " + f"{len(foundation['relationships'])} relationships" + ) + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Created foundation graph: {len(foundation['entities'])} entities, {len(foundation['relationships'])} relationships") + return foundation + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def integrate_with_extracted( self, @@ -733,48 +761,63 @@ class SeedDataManager: >>> if not validation["valid"]: ... print(f"Found {len(validation['errors'])} errors") """ - results = { - "valid": True, - "errors": [], - "warnings": [], - "metrics": {} - } + tracking_id = self.progress_tracker.start_tracking( + module="seed", + submodule="SeedDataManager", + message="Validating seed data quality" + ) - entities = seed_data.get("entities", []) - relationships = seed_data.get("relationships", []) - - # Check entities - entity_ids = [] - for entity in entities: - if "id" not in entity: - results["errors"].append("Entity missing 'id' field") - results["valid"] = False - else: - if entity["id"] in entity_ids: - results["warnings"].append(f"Duplicate entity ID: {entity['id']}") - entity_ids.append(entity["id"]) + try: + results = { + "valid": True, + "errors": [], + "warnings": [], + "metrics": {} + } - if "type" not in entity: - results["warnings"].append(f"Entity {entity.get('id')} missing 'type' field") - - # Check relationships - for rel in relationships: - if "source_id" not in rel or "target_id" not in rel: - results["errors"].append("Relationship missing source_id or target_id") - results["valid"] = False + entities = seed_data.get("entities", []) + relationships = seed_data.get("relationships", []) - if "type" not in rel: - results["warnings"].append("Relationship missing 'type' field") - - # Calculate metrics - results["metrics"] = { - "entity_count": len(entities), - "relationship_count": len(relationships), - "unique_entity_ids": len(set(entity_ids)), - "duplicate_entities": len(entities) - len(set(entity_ids)) - } - - return results + # Check entities + self.progress_tracker.update_tracking(tracking_id, message=f"Validating {len(entities)} entities...") + entity_ids = [] + for entity in entities: + if "id" not in entity: + results["errors"].append("Entity missing 'id' field") + results["valid"] = False + else: + if entity["id"] in entity_ids: + results["warnings"].append(f"Duplicate entity ID: {entity['id']}") + entity_ids.append(entity["id"]) + + if "type" not in entity: + results["warnings"].append(f"Entity {entity.get('id')} missing 'type' field") + + # Check relationships + self.progress_tracker.update_tracking(tracking_id, message=f"Validating {len(relationships)} relationships...") + for rel in relationships: + if "source_id" not in rel or "target_id" not in rel: + results["errors"].append("Relationship missing source_id or target_id") + results["valid"] = False + + if "type" not in rel: + results["warnings"].append("Relationship missing 'type' field") + + # Calculate metrics + results["metrics"] = { + "entity_count": len(entities), + "relationship_count": len(relationships), + "unique_entity_ids": len(set(entity_ids)), + "duplicate_entities": len(entities) - len(set(entity_ids)) + } + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Validation complete: {len(results['errors'])} errors, {len(results['warnings'])} warnings") + return results + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def _record_to_entity(self, record: Dict[str, Any]) -> Optional[EntityDict]: """ diff --git a/semantica/semantic_extract/coreference_resolver.py b/semantica/semantic_extract/coreference_resolver.py index 4c53f4b8..83dc647d 100644 --- a/semantica/semantic_extract/coreference_resolver.py +++ b/semantica/semantic_extract/coreference_resolver.py @@ -37,6 +37,7 @@ from typing import Any, Dict, List, Optional, Tuple from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .ner_extractor import Entity @@ -69,6 +70,7 @@ class CoreferenceResolver: self.logger = get_logger("coreference_resolver") self.config = config or {} self.config.update(kwargs) + self.progress_tracker = get_progress_tracker() self.pronoun_resolver = PronounResolver(**self.config.get("pronoun", {})) self.entity_detector = EntityCoreferenceDetector(**self.config.get("entity", {})) @@ -85,19 +87,36 @@ class CoreferenceResolver: Returns: list: List of coreference chains """ - # Extract mentions - mentions = self._extract_mentions(text) + tracking_id = self.progress_tracker.start_tracking( + module="semantic_extract", + submodule="CoreferenceResolver", + message="Resolving coreferences in text" + ) - # Resolve pronouns - pronoun_resolutions = self.pronoun_resolver.resolve_pronouns(text, mentions, **options) - - # Detect entity coreferences - entity_corefs = self.entity_detector.detect_entity_coreferences(text, mentions, **options) - - # Build chains - chains = self.chain_builder.build_coreference_chains(mentions, **options) - - return chains + try: + # Extract mentions + self.progress_tracker.update_tracking(tracking_id, message="Extracting mentions...") + mentions = self._extract_mentions(text) + + # Resolve pronouns + self.progress_tracker.update_tracking(tracking_id, message="Resolving pronouns...") + pronoun_resolutions = self.pronoun_resolver.resolve_pronouns(text, mentions, **options) + + # Detect entity coreferences + self.progress_tracker.update_tracking(tracking_id, message="Detecting entity coreferences...") + entity_corefs = self.entity_detector.detect_entity_coreferences(text, mentions, **options) + + # Build chains + self.progress_tracker.update_tracking(tracking_id, message="Building coreference chains...") + chains = self.chain_builder.build_coreference_chains(mentions, **options) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Resolved {len(chains)} coreference chains") + return chains + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def _extract_mentions(self, text: str) -> List[Mention]: """Extract all mentions from text.""" diff --git a/semantica/semantic_extract/event_detector.py b/semantica/semantic_extract/event_detector.py index df19be94..de904020 100644 --- a/semantica/semantic_extract/event_detector.py +++ b/semantica/semantic_extract/event_detector.py @@ -36,6 +36,7 @@ from typing import Any, Dict, List, Optional from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .ner_extractor import Entity @@ -62,6 +63,7 @@ class EventDetector: self.logger = get_logger("event_detector") self.config = config or {} self.config.update(kwargs) + self.progress_tracker = get_progress_tracker() self.event_classifier = EventClassifier(**self.config.get("classifier", {})) self.temporal_processor = TemporalEventProcessor(**self.config.get("temporal", {})) @@ -87,31 +89,45 @@ class EventDetector: Returns: list: List of detected events """ - events = [] + tracking_id = self.progress_tracker.start_tracking( + module="semantic_extract", + submodule="EventDetector", + message="Detecting events in text" + ) - # Detect events using patterns - for event_type, pattern in self.event_patterns.items(): - for match in re.finditer(pattern, text, re.IGNORECASE): - # Extract surrounding context - start = max(0, match.start() - 50) - end = min(len(text), match.end() + 50) - context = text[start:end] - - # Extract participants (simplified) - participants = self._extract_participants(context) - - event = Event( - text=match.group(0), - event_type=event_type, - start_char=match.start(), - end_char=match.end(), - participants=participants, - confidence=0.7, - metadata={"context": context} - ) - events.append(event) - - return events + try: + events = [] + + # Detect events using patterns + self.progress_tracker.update_tracking(tracking_id, message="Scanning text for event patterns...") + for event_type, pattern in self.event_patterns.items(): + for match in re.finditer(pattern, text, re.IGNORECASE): + # Extract surrounding context + start = max(0, match.start() - 50) + end = min(len(text), match.end() + 50) + context = text[start:end] + + # Extract participants (simplified) + participants = self._extract_participants(context) + + event = Event( + text=match.group(0), + event_type=event_type, + start_char=match.start(), + end_char=match.end(), + participants=participants, + confidence=0.7, + metadata={"context": context} + ) + events.append(event) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Detected {len(events)} events") + return events + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def _extract_participants(self, context: str) -> List[str]: """Extract event participants from context.""" diff --git a/semantica/semantic_extract/extraction_validator.py b/semantica/semantic_extract/extraction_validator.py index 5b627ebf..61edab85 100644 --- a/semantica/semantic_extract/extraction_validator.py +++ b/semantica/semantic_extract/extraction_validator.py @@ -33,6 +33,7 @@ from typing import Any, Dict, List, Optional from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .ner_extractor import Entity from .relation_extractor import Relation @@ -62,6 +63,7 @@ class ExtractionValidator: """ self.logger = get_logger("extraction_validator") self.config = config + self.progress_tracker = get_progress_tracker() self.min_confidence = config.get("min_confidence", 0.5) self.validate_consistency = config.get("validate_consistency", True) @@ -77,52 +79,69 @@ class ExtractionValidator: Returns: ValidationResult: Validation result """ - errors = [] - warnings = [] - metrics = {} - - min_confidence = options.get("min_confidence", self.min_confidence) - - # Check confidence scores - low_confidence = [e for e in entities if e.confidence < min_confidence] - if low_confidence: - warnings.append(f"{len(low_confidence)} entities below confidence threshold") - - # Check for duplicates - entity_texts = [e.text.lower() for e in entities] - duplicates = len(entity_texts) - len(set(entity_texts)) - if duplicates > 0: - warnings.append(f"{duplicates} duplicate entities found") - - # Check for empty entities - empty_entities = [e for e in entities if not e.text.strip()] - if empty_entities: - errors.append(f"{len(empty_entities)} empty entities found") - - # Calculate metrics - metrics = { - "total_entities": len(entities), - "high_confidence": len([e for e in entities if e.confidence >= 0.8]), - "medium_confidence": len([e for e in entities if min_confidence <= e.confidence < 0.8]), - "low_confidence": len(low_confidence), - "unique_entities": len(set(entity_texts)), - "duplicates": duplicates, - "entity_types": len(set(e.label for e in entities)), - "average_confidence": sum(e.confidence for e in entities) / len(entities) if entities else 0.0 - } - - # Calculate score - score = self._calculate_entity_score(entities, metrics) - - valid = len(errors) == 0 - - return ValidationResult( - valid=valid, - score=score, - errors=errors, - warnings=warnings, - metrics=metrics + tracking_id = self.progress_tracker.start_tracking( + module="semantic_extract", + submodule="ExtractionValidator", + message=f"Validating {len(entities)} entities" ) + + try: + errors = [] + warnings = [] + metrics = {} + + min_confidence = options.get("min_confidence", self.min_confidence) + + # Check confidence scores + self.progress_tracker.update_tracking(tracking_id, message="Checking confidence scores...") + low_confidence = [e for e in entities if e.confidence < min_confidence] + if low_confidence: + warnings.append(f"{len(low_confidence)} entities below confidence threshold") + + # Check for duplicates + self.progress_tracker.update_tracking(tracking_id, message="Checking for duplicates...") + entity_texts = [e.text.lower() for e in entities] + duplicates = len(entity_texts) - len(set(entity_texts)) + if duplicates > 0: + warnings.append(f"{duplicates} duplicate entities found") + + # Check for empty entities + empty_entities = [e for e in entities if not e.text.strip()] + if empty_entities: + errors.append(f"{len(empty_entities)} empty entities found") + + # Calculate metrics + metrics = { + "total_entities": len(entities), + "high_confidence": len([e for e in entities if e.confidence >= 0.8]), + "medium_confidence": len([e for e in entities if min_confidence <= e.confidence < 0.8]), + "low_confidence": len(low_confidence), + "unique_entities": len(set(entity_texts)), + "duplicates": duplicates, + "entity_types": len(set(e.label for e in entities)), + "average_confidence": sum(e.confidence for e in entities) / len(entities) if entities else 0.0 + } + + # Calculate score + score = self._calculate_entity_score(entities, metrics) + + valid = len(errors) == 0 + + result = ValidationResult( + valid=valid, + score=score, + errors=errors, + warnings=warnings, + metrics=metrics + ) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Validation complete: {len(errors)} errors, {len(warnings)} warnings") + return result + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def validate_relations(self, relations: List[Relation], **options) -> ValidationResult: """ diff --git a/semantica/semantic_extract/llm_enhancer.py b/semantica/semantic_extract/llm_enhancer.py index b2c610a1..3fea516c 100644 --- a/semantica/semantic_extract/llm_enhancer.py +++ b/semantica/semantic_extract/llm_enhancer.py @@ -31,6 +31,7 @@ from typing import Any, Dict, List, Optional from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .ner_extractor import Entity from .relation_extractor import Relation @@ -61,6 +62,7 @@ class LLMEnhancer: """ self.logger = get_logger("llm_enhancer") self.config = config + self.progress_tracker = get_progress_tracker() self.provider = config.get("provider", "openai") self.model = config.get("model", "gpt-3.5-turbo") @@ -106,18 +108,33 @@ class LLMEnhancer: Returns: list: Enhanced entities """ - if not self.client: - self.logger.warning("LLM client not available. Returning original entities.") - return entities - - prompt = self._build_entity_prompt(text, entities) + tracking_id = self.progress_tracker.start_tracking( + module="semantic_extract", + submodule="LLMEnhancer", + message="Enhancing entities using LLM" + ) try: + if not self.client: + self.logger.warning("LLM client not available. Returning original entities.") + self.progress_tracker.stop_tracking(tracking_id, status="completed", message="LLM client not available") + return entities + + self.progress_tracker.update_tracking(tracking_id, message="Building prompt...") + prompt = self._build_entity_prompt(text, entities) + + self.progress_tracker.update_tracking(tracking_id, message="Calling LLM API...") response = self._call_llm(prompt, **options) + + self.progress_tracker.update_tracking(tracking_id, message="Parsing LLM response...") enhanced_entities = self._parse_entity_response(response, entities) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Enhanced {len(enhanced_entities)} entities") return enhanced_entities except Exception as e: self.logger.error(f"Failed to enhance entities with LLM: {e}") + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) return entities def enhance_relations(self, text: str, relations: List[Relation], **options) -> List[Relation]: diff --git a/semantica/semantic_extract/named_entity_recognizer.py b/semantica/semantic_extract/named_entity_recognizer.py index 43a996d6..c5eac53a 100644 --- a/semantica/semantic_extract/named_entity_recognizer.py +++ b/semantica/semantic_extract/named_entity_recognizer.py @@ -35,6 +35,7 @@ from typing import Any, Dict, List, Optional from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .ner_extractor import Entity, NERExtractor @@ -55,6 +56,7 @@ class NamedEntityRecognizer: self.logger = get_logger("named_entity_recognizer") self.config = config or {} self.config.update(kwargs) + self.progress_tracker = get_progress_tracker() # Use NERExtractor for actual extraction self.ner_extractor = NERExtractor(**self.config.get("ner", {})) @@ -72,7 +74,20 @@ class NamedEntityRecognizer: Returns: list: List of extracted entities """ - return self.ner_extractor.extract_entities(text, **options) + tracking_id = self.progress_tracker.start_tracking( + module="semantic_extract", + submodule="NamedEntityRecognizer", + message="Extracting named entities" + ) + + try: + entities = self.ner_extractor.extract_entities(text, **options) + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Extracted {len(entities)} entities") + return entities + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def classify_entities(self, entities: List[Entity], **context) -> Dict[str, List[Entity]]: """ diff --git a/semantica/semantic_extract/ner_extractor.py b/semantica/semantic_extract/ner_extractor.py index cfac8520..c66fc564 100644 --- a/semantica/semantic_extract/ner_extractor.py +++ b/semantica/semantic_extract/ner_extractor.py @@ -31,6 +31,7 @@ from typing import Any, Dict, List, Optional, Tuple from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker try: import spacy @@ -70,6 +71,7 @@ class NERExtractor: self.model_name = config.get("model", "en_core_web_sm") self.language = config.get("language", "en") self.min_confidence = config.get("min_confidence", 0.5) + self.progress_tracker = get_progress_tracker() # Initialize spaCy model if available self.nlp = None @@ -92,16 +94,34 @@ class NERExtractor: Returns: list: List of extracted entities """ - if not text: - return [] + tracking_id = self.progress_tracker.start_tracking( + module="semantic_extract", + submodule="NERExtractor", + message="Extracting named entities from text" + ) - min_confidence = options.get("min_confidence", self.min_confidence) - entity_types = options.get("entity_types") - - if self.nlp: - return self._extract_with_spacy(text, min_confidence, entity_types) - else: - return self._extract_fallback(text) + try: + if not text: + self.progress_tracker.stop_tracking(tracking_id, status="completed", message="No text provided") + return [] + + min_confidence = options.get("min_confidence", self.min_confidence) + entity_types = options.get("entity_types") + + if self.nlp: + self.progress_tracker.update_tracking(tracking_id, message="Extracting entities using spaCy...") + entities = self._extract_with_spacy(text, min_confidence, entity_types) + else: + self.progress_tracker.update_tracking(tracking_id, message="Extracting entities using fallback method...") + entities = self._extract_fallback(text) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Extracted {len(entities)} entities") + return entities + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def _extract_with_spacy(self, text: str, min_confidence: float, entity_types: Optional[List[str]]) -> List[Entity]: """Extract entities using spaCy.""" diff --git a/semantica/semantic_extract/relation_extractor.py b/semantica/semantic_extract/relation_extractor.py index 21aebe18..ee66eb72 100644 --- a/semantica/semantic_extract/relation_extractor.py +++ b/semantica/semantic_extract/relation_extractor.py @@ -33,6 +33,7 @@ from typing import Any, Dict, List, Optional, Tuple from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .ner_extractor import Entity @@ -62,6 +63,7 @@ class RelationExtractor: """ self.logger = get_logger("relation_extractor") self.config = config + self.progress_tracker = get_progress_tracker() self.min_confidence = config.get("min_confidence", 0.5) self.use_llm = config.get("use_llm", False) @@ -98,18 +100,34 @@ class RelationExtractor: Returns: list: List of extracted relations """ - if not text or not entities: - return [] + tracking_id = self.progress_tracker.start_tracking( + module="semantic_extract", + submodule="RelationExtractor", + message=f"Extracting relations from {len(entities)} entities" + ) - min_confidence = options.get("min_confidence", self.min_confidence) - - # Pattern-based extraction - relations = self._extract_with_patterns(text, entities) - - # Filter by confidence - relations = [r for r in relations if r.confidence >= min_confidence] - - return relations + try: + if not text or not entities: + self.progress_tracker.stop_tracking(tracking_id, status="completed", message="No text or entities provided") + return [] + + min_confidence = options.get("min_confidence", self.min_confidence) + + # Pattern-based extraction + self.progress_tracker.update_tracking(tracking_id, message="Extracting relations using patterns...") + relations = self._extract_with_patterns(text, entities) + + # Filter by confidence + self.progress_tracker.update_tracking(tracking_id, message=f"Filtering {len(relations)} relations by confidence...") + relations = [r for r in relations if r.confidence >= min_confidence] + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Extracted {len(relations)} relations") + return relations + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def _extract_with_patterns(self, text: str, entities: List[Entity]) -> List[Relation]: """Extract relations using pattern matching.""" diff --git a/semantica/semantic_extract/semantic_analyzer.py b/semantica/semantic_extract/semantic_analyzer.py index 26be5c1d..e6071602 100644 --- a/semantica/semantic_extract/semantic_analyzer.py +++ b/semantica/semantic_extract/semantic_analyzer.py @@ -35,6 +35,7 @@ from typing import Any, Dict, List, Optional, Tuple from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker @dataclass @@ -66,6 +67,7 @@ class SemanticAnalyzer: self.logger = get_logger("semantic_analyzer") self.config = config or {} self.config.update(kwargs) + self.progress_tracker = get_progress_tracker() self.similarity_analyzer = SimilarityAnalyzer(**self.config.get("similarity", {})) self.role_labeler = RoleLabeler(**self.config.get("role", {})) @@ -82,22 +84,36 @@ class SemanticAnalyzer: Returns: dict: Semantic analysis results """ - results = { - "text": text, - "length": len(text), - "word_count": len(text.split()), - "sentence_count": len(text.split('.')) - } + tracking_id = self.progress_tracker.start_tracking( + module="semantic_extract", + submodule="SemanticAnalyzer", + message="Performing comprehensive semantic analysis" + ) - # Semantic role labeling - if options.get("label_roles", False): - roles = self.label_semantic_roles(text, **options) - results["semantic_roles"] = [r.__dict__ for r in roles] - - # Semantic features - results["semantic_features"] = self._extract_features(text) - - return results + try: + results = { + "text": text, + "length": len(text), + "word_count": len(text.split()), + "sentence_count": len(text.split('.')) + } + + # Semantic role labeling + if options.get("label_roles", False): + self.progress_tracker.update_tracking(tracking_id, message="Labeling semantic roles...") + roles = self.label_semantic_roles(text, **options) + results["semantic_roles"] = [r.__dict__ for r in roles] + + # Semantic features + self.progress_tracker.update_tracking(tracking_id, message="Extracting semantic features...") + results["semantic_features"] = self._extract_features(text) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", message="Semantic analysis complete") + return results + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def calculate_similarity(self, text1: str, text2: str, **options) -> float: """ diff --git a/semantica/semantic_extract/semantic_network_extractor.py b/semantica/semantic_extract/semantic_network_extractor.py index f864ebf3..f568f320 100644 --- a/semantica/semantic_extract/semantic_network_extractor.py +++ b/semantica/semantic_extract/semantic_network_extractor.py @@ -36,6 +36,7 @@ import yaml from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .ner_extractor import Entity from .relation_extractor import Relation @@ -83,6 +84,7 @@ class SemanticNetworkExtractor: """ self.logger = get_logger("semantic_network_extractor") self.config = config + self.progress_tracker = get_progress_tracker() def extract_network( self, @@ -103,23 +105,39 @@ class SemanticNetworkExtractor: Returns: SemanticNetwork: Extracted semantic network """ - from .ner_extractor import NERExtractor - from .relation_extractor import RelationExtractor + tracking_id = self.progress_tracker.start_tracking( + module="semantic_extract", + submodule="SemanticNetworkExtractor", + message="Extracting semantic network from text" + ) - # Extract entities if not provided - if entities is None: - ner = NERExtractor(**self.config.get("ner", {})) - entities = ner.extract_entities(text) - - # Extract relations if not provided - if relations is None: - rel_extractor = RelationExtractor(**self.config.get("relation", {})) - relations = rel_extractor.extract_relations(text, entities) - - # Build network - network = self._build_network(entities, relations) - - return network + try: + from .ner_extractor import NERExtractor + from .relation_extractor import RelationExtractor + + # Extract entities if not provided + if entities is None: + self.progress_tracker.update_tracking(tracking_id, message="Extracting entities...") + ner = NERExtractor(**self.config.get("ner", {})) + entities = ner.extract_entities(text) + + # Extract relations if not provided + if relations is None: + self.progress_tracker.update_tracking(tracking_id, message="Extracting relations...") + rel_extractor = RelationExtractor(**self.config.get("relation", {})) + relations = rel_extractor.extract_relations(text, entities) + + # Build network + self.progress_tracker.update_tracking(tracking_id, message="Building semantic network...") + network = self._build_network(entities, relations) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Extracted network: {len(network.nodes)} nodes, {len(network.edges)} edges") + return network + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def _build_network(self, entities: List[Entity], relations: List[Relation]) -> SemanticNetwork: """Build semantic network from entities and relations.""" diff --git a/semantica/semantic_extract/triple_extractor.py b/semantica/semantic_extract/triple_extractor.py index c9fa82b0..2c3ea47f 100644 --- a/semantica/semantic_extract/triple_extractor.py +++ b/semantica/semantic_extract/triple_extractor.py @@ -37,6 +37,7 @@ from urllib.parse import quote from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .ner_extractor import Entity from .relation_extractor import Relation @@ -60,6 +61,7 @@ class TripleExtractor: self.logger = get_logger("triple_extractor") self.config = config or {} self.config.update(kwargs) + self.progress_tracker = get_progress_tracker() self.triple_validator = TripleValidator(**self.config.get("validator", {})) self.rdf_serializer = RDFSerializer(**self.config.get("serializer", {})) @@ -86,39 +88,56 @@ class TripleExtractor: Returns: list: List of extracted triples """ - from .ner_extractor import NERExtractor - from .relation_extractor import RelationExtractor + tracking_id = self.progress_tracker.start_tracking( + module="semantic_extract", + submodule="TripleExtractor", + message="Extracting RDF triples from text" + ) - # Extract entities if not provided - if entities is None: - ner = NERExtractor(**self.config.get("ner", {})) - entities = ner.extract_entities(text) - - # Extract relations if not provided - if relationships is None: - rel_extractor = RelationExtractor(**self.config.get("relation", {})) - relationships = rel_extractor.extract_relations(text, entities) - - # Convert relations to triples - triples = [] - for relation in relationships: - triple = Triple( - subject=self._format_uri(relation.subject.text), - predicate=self._format_uri(relation.predicate), - object=self._format_uri(relation.object.text), - confidence=relation.confidence, - metadata={ - "context": relation.context, - **relation.metadata - } - ) - triples.append(triple) - - # Validate triples - if options.get("validate", True): - triples = self.triple_validator.validate_triples(triples) - - return triples + try: + from .ner_extractor import NERExtractor + from .relation_extractor import RelationExtractor + + # Extract entities if not provided + if entities is None: + self.progress_tracker.update_tracking(tracking_id, message="Extracting entities...") + ner = NERExtractor(**self.config.get("ner", {})) + entities = ner.extract_entities(text) + + # Extract relations if not provided + if relationships is None: + self.progress_tracker.update_tracking(tracking_id, message="Extracting relations...") + rel_extractor = RelationExtractor(**self.config.get("relation", {})) + relationships = rel_extractor.extract_relations(text, entities) + + # Convert relations to triples + self.progress_tracker.update_tracking(tracking_id, message=f"Converting {len(relationships)} relations to triples...") + triples = [] + for relation in relationships: + triple = Triple( + subject=self._format_uri(relation.subject.text), + predicate=self._format_uri(relation.predicate), + object=self._format_uri(relation.object.text), + confidence=relation.confidence, + metadata={ + "context": relation.context, + **relation.metadata + } + ) + triples.append(triple) + + # Validate triples + if options.get("validate", True): + self.progress_tracker.update_tracking(tracking_id, message="Validating triples...") + triples = self.triple_validator.validate_triples(triples) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Extracted {len(triples)} triples") + return triples + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def _format_uri(self, value: str) -> str: """Format value as URI.""" diff --git a/semantica/split/chunk_validator.py b/semantica/split/chunk_validator.py index a6ad956b..c45bbe70 100644 --- a/semantica/split/chunk_validator.py +++ b/semantica/split/chunk_validator.py @@ -33,6 +33,7 @@ from typing import Any, Dict, List, Optional from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .semantic_chunker import Chunk @@ -62,6 +63,7 @@ class ChunkValidator: """ self.logger = get_logger("chunk_validator") self.config = config + self.progress_tracker = get_progress_tracker() self.min_size = config.get("min_size", 10) self.max_size = config.get("max_size", 10000) @@ -78,48 +80,66 @@ class ChunkValidator: Returns: ValidationResult: Validation result """ - errors = [] - warnings = [] - metrics = {} - - # Size validation - chunk_size = len(chunk.text) - metrics["size"] = chunk_size - - if chunk_size < self.min_size: - errors.append(f"Chunk too small: {chunk_size} < {self.min_size}") - - if chunk_size > self.max_size: - errors.append(f"Chunk too large: {chunk_size} > {self.max_size}") - - # Content validation - if not chunk.text.strip(): - errors.append("Chunk is empty or whitespace only") - - # Coherence validation - coherence_score = self._check_coherence(chunk.text) - metrics["coherence"] = coherence_score - - if coherence_score < 0.3: - warnings.append(f"Low semantic coherence: {coherence_score:.2f}") - - # Structure validation - structure_score = self._check_structure(chunk.text) - metrics["structure"] = structure_score - - # Calculate overall score - score = self._calculate_score(chunk_size, coherence_score, structure_score, errors) - metrics["overall_score"] = score - - valid = len(errors) == 0 and score >= self.min_score - - return ValidationResult( - valid=valid, - score=score, - errors=errors, - warnings=warnings, - metrics=metrics + tracking_id = self.progress_tracker.start_tracking( + module="split", + submodule="ChunkValidator", + message="Validating chunk" ) + + try: + errors = [] + warnings = [] + metrics = {} + + # Size validation + self.progress_tracker.update_tracking(tracking_id, message="Validating chunk size...") + chunk_size = len(chunk.text) + metrics["size"] = chunk_size + + if chunk_size < self.min_size: + errors.append(f"Chunk too small: {chunk_size} < {self.min_size}") + + if chunk_size > self.max_size: + errors.append(f"Chunk too large: {chunk_size} > {self.max_size}") + + # Content validation + if not chunk.text.strip(): + errors.append("Chunk is empty or whitespace only") + + # Coherence validation + self.progress_tracker.update_tracking(tracking_id, message="Checking semantic coherence...") + coherence_score = self._check_coherence(chunk.text) + metrics["coherence"] = coherence_score + + if coherence_score < 0.3: + warnings.append(f"Low semantic coherence: {coherence_score:.2f}") + + # Structure validation + self.progress_tracker.update_tracking(tracking_id, message="Checking structure...") + structure_score = self._check_structure(chunk.text) + metrics["structure"] = structure_score + + # Calculate overall score + score = self._calculate_score(chunk_size, coherence_score, structure_score, errors) + metrics["overall_score"] = score + + valid = len(errors) == 0 and score >= self.min_score + + result = ValidationResult( + valid=valid, + score=score, + errors=errors, + warnings=warnings, + metrics=metrics + ) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Validation complete: {'valid' if valid else 'invalid'}") + return result + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def validate_batch(self, chunks: List[Chunk], **options) -> Dict[str, Any]: """ diff --git a/semantica/split/provenance_tracker.py b/semantica/split/provenance_tracker.py index d9242018..2c8dce56 100644 --- a/semantica/split/provenance_tracker.py +++ b/semantica/split/provenance_tracker.py @@ -33,6 +33,7 @@ from uuid import uuid4 from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .semantic_chunker import Chunk @@ -65,6 +66,7 @@ class ProvenanceTracker: """ self.logger = get_logger("provenance_tracker") self.config = config + self.progress_tracker = get_progress_tracker() self.store_metadata = config.get("store_metadata", True) self.track_versions = config.get("track_versions", False) @@ -136,21 +138,35 @@ class ProvenanceTracker: Returns: list: List of provenance IDs """ - provenance_ids = [] - parent_chunk_id = None + tracking_id = self.progress_tracker.start_tracking( + module="split", + submodule="ProvenanceTracker", + message=f"Tracking provenance for {len(chunks)} chunks" + ) - for chunk in chunks: - provenance_id = self.track_chunk( - chunk, - source_document, - source_path, - parent_chunk_id, - **metadata - ) - provenance_ids.append(provenance_id) - parent_chunk_id = provenance_id - - return provenance_ids + try: + provenance_ids = [] + parent_chunk_id = None + + for i, chunk in enumerate(chunks): + self.progress_tracker.update_tracking(tracking_id, message=f"Tracking chunk {i+1}/{len(chunks)}...") + provenance_id = self.track_chunk( + chunk, + source_document, + source_path, + parent_chunk_id, + **metadata + ) + provenance_ids.append(provenance_id) + parent_chunk_id = provenance_id + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Tracked {len(provenance_ids)} chunks") + return provenance_ids + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def get_provenance(self, chunk_id: str) -> Optional[ProvenanceInfo]: """ diff --git a/semantica/split/semantic_chunker.py b/semantica/split/semantic_chunker.py index 93a49772..d63cba9e 100644 --- a/semantica/split/semantic_chunker.py +++ b/semantica/split/semantic_chunker.py @@ -33,6 +33,7 @@ from typing import Any, Dict, List, Optional, Tuple from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker try: import spacy @@ -71,6 +72,7 @@ class SemanticChunker: self.chunk_size = config.get("chunk_size", 1000) self.chunk_overlap = config.get("chunk_overlap", 200) self.language = config.get("language", "en") + self.progress_tracker = get_progress_tracker() # Initialize spaCy model if available self.nlp = None @@ -94,14 +96,32 @@ class SemanticChunker: Returns: list: List of chunks """ - if not text: - return [] + tracking_id = self.progress_tracker.start_tracking( + module="split", + submodule="SemanticChunker", + message="Splitting text into semantic chunks" + ) - # Use spaCy if available - if self.nlp: - return self._chunk_with_spacy(text, **options) - else: - return self._chunk_fallback(text, **options) + try: + if not text: + self.progress_tracker.stop_tracking(tracking_id, status="completed", message="No text provided") + return [] + + # Use spaCy if available + if self.nlp: + self.progress_tracker.update_tracking(tracking_id, message="Chunking with spaCy...") + chunks = self._chunk_with_spacy(text, **options) + else: + self.progress_tracker.update_tracking(tracking_id, message="Chunking with fallback method...") + chunks = self._chunk_fallback(text, **options) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Created {len(chunks)} chunks") + return chunks + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def _chunk_with_spacy(self, text: str, **options) -> List[Chunk]: """Chunk text using spaCy.""" diff --git a/semantica/split/sliding_window_chunker.py b/semantica/split/sliding_window_chunker.py index 7240d04e..513bc74f 100644 --- a/semantica/split/sliding_window_chunker.py +++ b/semantica/split/sliding_window_chunker.py @@ -30,6 +30,7 @@ from typing import Any, Dict, List, Optional from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .semantic_chunker import Chunk @@ -48,6 +49,7 @@ class SlidingWindowChunker: """ self.logger = get_logger("sliding_window_chunker") self.config = config + self.progress_tracker = get_progress_tracker() self.chunk_size = config.get("chunk_size", 1000) self.overlap = config.get("overlap", 0) @@ -72,15 +74,33 @@ class SlidingWindowChunker: Returns: list: List of chunks """ - if not text: - return [] + tracking_id = self.progress_tracker.start_tracking( + module="split", + submodule="SlidingWindowChunker", + message="Splitting text using sliding window" + ) - preserve_boundaries = options.get("preserve_boundaries", True) - - if preserve_boundaries: - return self._chunk_with_boundaries(text) - else: - return self._chunk_fixed_size(text) + try: + if not text: + self.progress_tracker.stop_tracking(tracking_id, status="completed", message="No text provided") + return [] + + preserve_boundaries = options.get("preserve_boundaries", True) + + if preserve_boundaries: + self.progress_tracker.update_tracking(tracking_id, message="Chunking with boundary preservation...") + chunks = self._chunk_with_boundaries(text) + else: + self.progress_tracker.update_tracking(tracking_id, message="Chunking with fixed-size windows...") + chunks = self._chunk_fixed_size(text) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Created {len(chunks)} chunks") + return chunks + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def _chunk_fixed_size(self, text: str) -> List[Chunk]: """Chunk text with fixed-size windows.""" diff --git a/semantica/split/structural_chunker.py b/semantica/split/structural_chunker.py index 38e896ff..1409dcdd 100644 --- a/semantica/split/structural_chunker.py +++ b/semantica/split/structural_chunker.py @@ -32,6 +32,7 @@ from typing import Any, Dict, List, Optional, Tuple from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .semantic_chunker import Chunk @@ -60,6 +61,7 @@ class StructuralChunker: """ self.logger = get_logger("structural_chunker") self.config = config + self.progress_tracker = get_progress_tracker() self.respect_headers = config.get("respect_headers", True) self.respect_sections = config.get("respect_sections", True) @@ -76,16 +78,32 @@ class StructuralChunker: Returns: list: List of chunks """ - if not text: - return [] + tracking_id = self.progress_tracker.start_tracking( + module="split", + submodule="StructuralChunker", + message="Splitting text based on document structure" + ) - # Extract structural elements - elements = self._extract_structure(text) - - # Group elements into chunks - chunks = self._group_elements(elements, text) - - return chunks + try: + if not text: + self.progress_tracker.stop_tracking(tracking_id, status="completed", message="No text provided") + return [] + + # Extract structural elements + self.progress_tracker.update_tracking(tracking_id, message="Extracting structural elements...") + elements = self._extract_structure(text) + + # Group elements into chunks + self.progress_tracker.update_tracking(tracking_id, message="Grouping elements into chunks...") + chunks = self._group_elements(elements, text) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Created {len(chunks)} chunks") + return chunks + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def _extract_structure(self, text: str) -> List[StructuralElement]: """Extract structural elements from text.""" diff --git a/semantica/split/table_chunker.py b/semantica/split/table_chunker.py index 220a5d43..d3f2603b 100644 --- a/semantica/split/table_chunker.py +++ b/semantica/split/table_chunker.py @@ -32,6 +32,7 @@ from typing import Any, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker from .semantic_chunker import Chunk @@ -61,6 +62,7 @@ class TableChunker: """ self.logger = get_logger("table_chunker") self.config = config + self.progress_tracker = get_progress_tracker() self.max_rows = config.get("max_rows", 100) self.preserve_headers = config.get("preserve_headers", True) @@ -77,25 +79,44 @@ class TableChunker: Returns: list: List of table chunks """ - # Parse table data - if isinstance(table_data, dict): - headers = table_data.get("headers", []) - rows = table_data.get("rows", []) - elif isinstance(table_data, list) and len(table_data) > 0: - # First row as headers if not provided - if options.get("first_row_as_header", True): - headers = table_data[0] - rows = table_data[1:] - else: - headers = [f"Column_{i+1}" for i in range(len(table_data[0]))] - rows = table_data - else: - raise ValidationError("Invalid table data format") + tracking_id = self.progress_tracker.start_tracking( + module="split", + submodule="TableChunker", + message="Chunking table data" + ) - if self.chunk_by_columns: - return self._chunk_by_columns(headers, rows, **options) - else: - return self._chunk_by_rows(headers, rows, **options) + try: + # Parse table data + self.progress_tracker.update_tracking(tracking_id, message="Parsing table data...") + if isinstance(table_data, dict): + headers = table_data.get("headers", []) + rows = table_data.get("rows", []) + elif isinstance(table_data, list) and len(table_data) > 0: + # First row as headers if not provided + if options.get("first_row_as_header", True): + headers = table_data[0] + rows = table_data[1:] + else: + headers = [f"Column_{i+1}" for i in range(len(table_data[0]))] + rows = table_data + else: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message="Invalid table data format") + raise ValidationError("Invalid table data format") + + if self.chunk_by_columns: + self.progress_tracker.update_tracking(tracking_id, message="Chunking by columns...") + chunks = self._chunk_by_columns(headers, rows, **options) + else: + self.progress_tracker.update_tracking(tracking_id, message="Chunking by rows...") + chunks = self._chunk_by_rows(headers, rows, **options) + + self.progress_tracker.stop_tracking(tracking_id, status="completed", + message=f"Created {len(chunks)} table chunks") + return chunks + + except Exception as e: + self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)) + raise def _chunk_by_rows(self, headers: List[str], rows: List[List[str]], **options) -> List[TableChunk]: """Chunk table by rows."""