mengzhx/semantica
1
0
Fork 0
mirror of https://github.com/semantica-agi/semantica.git synced 2026-09-13 04:04:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add progress tracking to seed, semantic_extract, and split modules

Browse Source 
- Added progress tracking to seed module (seed_manager.py)
- Added progress tracking to all semantic_extract files (10 files):
  * triple_extractor.py
  * semantic_network_extractor.py
  * relation_extractor.py
  * semantic_analyzer.py
  * ner_extractor.py
  * named_entity_recognizer.py
  * llm_enhancer.py
  * extraction_validator.py
  * event_detector.py
  * coreference_resolver.py
- Added progress tracking to all split files (6 files):
  * semantic_chunker.py
  * sliding_window_chunker.py
  * structural_chunker.py
  * table_chunker.py
  * chunk_validator.py
  * provenance_tracker.py
- All methods now use start_tracking, update_tracking, and stop_tracking
- Consistent error handling with try-except blocks across all modules
This commit is contained in:
KaifAhmad1
2025-11-12 17:56:45 +05:30
parent 9b7a3275c1
commit 05caff91fa
17 changed files with 685 additions and 350 deletions
Download Patch File Download Diff File Expand all files Collapse all files
semantica/seed/seed_manager.py
+126 -83
View File
@@ -42,6 +42,7 @@ from ..utils.exceptions import ValidationError, ProcessingError
from ..utils.logging import get_logger
from ..utils.helpers import read_json_file, write_json_file
from ..utils.types import EntityDict, RelationshipDict
from ..utils.progress_tracker import get_progress_tracker
@dataclass
@@ -122,6 +123,7 @@ class SeedDataManager:
self.logger = get_logger("seed_manager")
self.config = config or {}
self.config.update(kwargs)
self.progress_tracker = get_progress_tracker()
self.sources: Dict[str, SeedDataSource] = {}
self.seed_data: SeedData = SeedData()
@@ -213,13 +215,21 @@ class SeedDataManager:
>>> records = manager.load_from_csv("data/entities.csv", entity_type="Person")
>>> print(f"Loaded {len(records)} records")
"""
file_path = Path(file_path)
if not file_path.exists():
raise ProcessingError(f"CSV file not found: {file_path}")
records = []
tracking_id = self.progress_tracker.start_tracking(
module="seed",
submodule="SeedDataManager",
message=f"Loading seed data from CSV: {file_path}"
)
try:
file_path = Path(file_path)
if not file_path.exists():
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=f"CSV file not found: {file_path}")
raise ProcessingError(f"CSV file not found: {file_path}")
records = []
self.progress_tracker.update_tracking(tracking_id, message="Reading CSV file...")
with open(file_path, 'r', encoding='utf-8') as f:
reader = csv.DictReader(f)
for row in reader:
@@ -236,9 +246,12 @@ class SeedDataManager:
records.append(record)
self.logger.info(f"Loaded {len(records)} records from CSV: {file_path}")
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Loaded {len(records)} records from CSV")
return records
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise ProcessingError(f"Failed to load CSV: {e}") from e
def load_from_json(
@@ -565,47 +578,62 @@ class SeedDataManager:
>>> foundation = manager.create_foundation_graph()
>>> print(f"Created graph with {len(foundation['entities'])} entities")
"""
foundation = {
"entities": [],
"relationships": [],
"metadata": {
"created_at": datetime.now().isoformat(),
"source_count": len(self.sources),
"verified": True
}
}
# Load data from all sources
for source_name in self.sources:
try:
records = self.load_source(source_name)
for record in records:
# Extract entities
if 'entity_type' in record or 'id' in record:
entity = self._record_to_entity(record)
if entity:
foundation["entities"].append(entity)
# Extract relationships
if 'relationship_type' in record or ('source_id' in record and 'target_id' in record):
relationship = self._record_to_relationship(record)
if relationship:
foundation["relationships"].append(relationship)
except Exception as e:
self.logger.warning(f"Failed to load source '{source_name}': {e}")
# Validate against schema template if provided
if schema_template:
foundation = self._validate_against_template(foundation, schema_template)
self.logger.info(
f"Created foundation graph: {len(foundation['entities'])} entities, "
f"{len(foundation['relationships'])} relationships"
tracking_id = self.progress_tracker.start_tracking(
module="seed",
submodule="SeedDataManager",
message="Creating foundation graph from seed data"
)
return foundation
try:
foundation = {
"entities": [],
"relationships": [],
"metadata": {
"created_at": datetime.now().isoformat(),
"source_count": len(self.sources),
"verified": True
}
}
# Load data from all sources
self.progress_tracker.update_tracking(tracking_id, message=f"Loading data from {len(self.sources)} sources...")
for source_name in self.sources:
try:
self.progress_tracker.update_tracking(tracking_id, message=f"Loading source: {source_name}")
records = self.load_source(source_name)
for record in records:
# Extract entities
if 'entity_type' in record or 'id' in record:
entity = self._record_to_entity(record)
if entity:
foundation["entities"].append(entity)
# Extract relationships
if 'relationship_type' in record or ('source_id' in record and 'target_id' in record):
relationship = self._record_to_relationship(record)
if relationship:
foundation["relationships"].append(relationship)
except Exception as e:
self.logger.warning(f"Failed to load source '{source_name}': {e}")
# Validate against schema template if provided
if schema_template:
self.progress_tracker.update_tracking(tracking_id, message="Validating against schema template...")
foundation = self._validate_against_template(foundation, schema_template)
self.logger.info(
f"Created foundation graph: {len(foundation['entities'])} entities, "
f"{len(foundation['relationships'])} relationships"
)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Created foundation graph: {len(foundation['entities'])} entities, {len(foundation['relationships'])} relationships")
return foundation
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def integrate_with_extracted(
self,
@@ -733,48 +761,63 @@ class SeedDataManager:
>>> if not validation["valid"]:
... print(f"Found {len(validation['errors'])} errors")
"""
results = {
"valid": True,
"errors": [],
"warnings": [],
"metrics": {}
}
tracking_id = self.progress_tracker.start_tracking(
module="seed",
submodule="SeedDataManager",
message="Validating seed data quality"
)
entities = seed_data.get("entities", [])
relationships = seed_data.get("relationships", [])
# Check entities
entity_ids = []
for entity in entities:
if "id" not in entity:
results["errors"].append("Entity missing 'id' field")
results["valid"] = False
else:
if entity["id"] in entity_ids:
results["warnings"].append(f"Duplicate entity ID: {entity['id']}")
entity_ids.append(entity["id"])
try:
results = {
"valid": True,
"errors": [],
"warnings": [],
"metrics": {}
}
if "type" not in entity:
results["warnings"].append(f"Entity {entity.get('id')} missing 'type' field")
# Check relationships
for rel in relationships:
if "source_id" not in rel or "target_id" not in rel:
results["errors"].append("Relationship missing source_id or target_id")
results["valid"] = False
entities = seed_data.get("entities", [])
relationships = seed_data.get("relationships", [])
if "type" not in rel:
results["warnings"].append("Relationship missing 'type' field")
# Calculate metrics
results["metrics"] = {
"entity_count": len(entities),
"relationship_count": len(relationships),
"unique_entity_ids": len(set(entity_ids)),
"duplicate_entities": len(entities) - len(set(entity_ids))
}
return results
# Check entities
self.progress_tracker.update_tracking(tracking_id, message=f"Validating {len(entities)} entities...")
entity_ids = []
for entity in entities:
if "id" not in entity:
results["errors"].append("Entity missing 'id' field")
results["valid"] = False
else:
if entity["id"] in entity_ids:
results["warnings"].append(f"Duplicate entity ID: {entity['id']}")
entity_ids.append(entity["id"])
if "type" not in entity:
results["warnings"].append(f"Entity {entity.get('id')} missing 'type' field")
# Check relationships
self.progress_tracker.update_tracking(tracking_id, message=f"Validating {len(relationships)} relationships...")
for rel in relationships:
if "source_id" not in rel or "target_id" not in rel:
results["errors"].append("Relationship missing source_id or target_id")
results["valid"] = False
if "type" not in rel:
results["warnings"].append("Relationship missing 'type' field")
# Calculate metrics
results["metrics"] = {
"entity_count": len(entities),
"relationship_count": len(relationships),
"unique_entity_ids": len(set(entity_ids)),
"duplicate_entities": len(entities) - len(set(entity_ids))
}
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Validation complete: {len(results['errors'])} errors, {len(results['warnings'])} warnings")
return results
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def _record_to_entity(self, record: Dict[str, Any]) -> Optional[EntityDict]:
"""
semantica/semantic_extract/coreference_resolver.py
+31 -12
View File
@@ -37,6 +37,7 @@ from typing import Any, Dict, List, Optional, Tuple
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .ner_extractor import Entity
@@ -69,6 +70,7 @@ class CoreferenceResolver:
self.logger = get_logger("coreference_resolver")
self.config = config or {}
self.config.update(kwargs)
self.progress_tracker = get_progress_tracker()
self.pronoun_resolver = PronounResolver(**self.config.get("pronoun", {}))
self.entity_detector = EntityCoreferenceDetector(**self.config.get("entity", {}))
@@ -85,19 +87,36 @@ class CoreferenceResolver:
Returns:
list: List of coreference chains
"""
# Extract mentions
mentions = self._extract_mentions(text)
tracking_id = self.progress_tracker.start_tracking(
module="semantic_extract",
submodule="CoreferenceResolver",
message="Resolving coreferences in text"
)
# Resolve pronouns
pronoun_resolutions = self.pronoun_resolver.resolve_pronouns(text, mentions, **options)
# Detect entity coreferences
entity_corefs = self.entity_detector.detect_entity_coreferences(text, mentions, **options)
# Build chains
chains = self.chain_builder.build_coreference_chains(mentions, **options)
return chains
try:
# Extract mentions
self.progress_tracker.update_tracking(tracking_id, message="Extracting mentions...")
mentions = self._extract_mentions(text)
# Resolve pronouns
self.progress_tracker.update_tracking(tracking_id, message="Resolving pronouns...")
pronoun_resolutions = self.pronoun_resolver.resolve_pronouns(text, mentions, **options)
# Detect entity coreferences
self.progress_tracker.update_tracking(tracking_id, message="Detecting entity coreferences...")
entity_corefs = self.entity_detector.detect_entity_coreferences(text, mentions, **options)
# Build chains
self.progress_tracker.update_tracking(tracking_id, message="Building coreference chains...")
chains = self.chain_builder.build_coreference_chains(mentions, **options)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Resolved {len(chains)} coreference chains")
return chains
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def _extract_mentions(self, text: str) -> List[Mention]:
"""Extract all mentions from text."""
semantica/semantic_extract/event_detector.py
+40 -24
View File
@@ -36,6 +36,7 @@ from typing import Any, Dict, List, Optional
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .ner_extractor import Entity
@@ -62,6 +63,7 @@ class EventDetector:
self.logger = get_logger("event_detector")
self.config = config or {}
self.config.update(kwargs)
self.progress_tracker = get_progress_tracker()
self.event_classifier = EventClassifier(**self.config.get("classifier", {}))
self.temporal_processor = TemporalEventProcessor(**self.config.get("temporal", {}))
@@ -87,31 +89,45 @@ class EventDetector:
Returns:
list: List of detected events
"""
events = []
tracking_id = self.progress_tracker.start_tracking(
module="semantic_extract",
submodule="EventDetector",
message="Detecting events in text"
)
# Detect events using patterns
for event_type, pattern in self.event_patterns.items():
for match in re.finditer(pattern, text, re.IGNORECASE):
# Extract surrounding context
start = max(0, match.start() - 50)
end = min(len(text), match.end() + 50)
context = text[start:end]
# Extract participants (simplified)
participants = self._extract_participants(context)
event = Event(
text=match.group(0),
event_type=event_type,
start_char=match.start(),
end_char=match.end(),
participants=participants,
confidence=0.7,
metadata={"context": context}
)
events.append(event)
return events
try:
events = []
# Detect events using patterns
self.progress_tracker.update_tracking(tracking_id, message="Scanning text for event patterns...")
for event_type, pattern in self.event_patterns.items():
for match in re.finditer(pattern, text, re.IGNORECASE):
# Extract surrounding context
start = max(0, match.start() - 50)
end = min(len(text), match.end() + 50)
context = text[start:end]
# Extract participants (simplified)
participants = self._extract_participants(context)
event = Event(
text=match.group(0),
event_type=event_type,
start_char=match.start(),
end_char=match.end(),
participants=participants,
confidence=0.7,
metadata={"context": context}
)
events.append(event)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Detected {len(events)} events")
return events
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def _extract_participants(self, context: str) -> List[str]:
"""Extract event participants from context."""
semantica/semantic_extract/extraction_validator.py
+64 -45
View File
@@ -33,6 +33,7 @@ from typing import Any, Dict, List, Optional
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .ner_extractor import Entity
from .relation_extractor import Relation
@@ -62,6 +63,7 @@ class ExtractionValidator:
"""
self.logger = get_logger("extraction_validator")
self.config = config
self.progress_tracker = get_progress_tracker()
self.min_confidence = config.get("min_confidence", 0.5)
self.validate_consistency = config.get("validate_consistency", True)
@@ -77,52 +79,69 @@ class ExtractionValidator:
Returns:
ValidationResult: Validation result
"""
errors = []
warnings = []
metrics = {}
min_confidence = options.get("min_confidence", self.min_confidence)
# Check confidence scores
low_confidence = [e for e in entities if e.confidence < min_confidence]
if low_confidence:
warnings.append(f"{len(low_confidence)} entities below confidence threshold")
# Check for duplicates
entity_texts = [e.text.lower() for e in entities]
duplicates = len(entity_texts) - len(set(entity_texts))
if duplicates > 0:
warnings.append(f"{duplicates} duplicate entities found")
# Check for empty entities
empty_entities = [e for e in entities if not e.text.strip()]
if empty_entities:
errors.append(f"{len(empty_entities)} empty entities found")
# Calculate metrics
metrics = {
"total_entities": len(entities),
"high_confidence": len([e for e in entities if e.confidence >= 0.8]),
"medium_confidence": len([e for e in entities if min_confidence <= e.confidence < 0.8]),
"low_confidence": len(low_confidence),
"unique_entities": len(set(entity_texts)),
"duplicates": duplicates,
"entity_types": len(set(e.label for e in entities)),
"average_confidence": sum(e.confidence for e in entities) / len(entities) if entities else 0.0
}
# Calculate score
score = self._calculate_entity_score(entities, metrics)
valid = len(errors) == 0
return ValidationResult(
valid=valid,
score=score,
errors=errors,
warnings=warnings,
metrics=metrics
tracking_id = self.progress_tracker.start_tracking(
module="semantic_extract",
submodule="ExtractionValidator",
message=f"Validating {len(entities)} entities"
)
try:
errors = []
warnings = []
metrics = {}
min_confidence = options.get("min_confidence", self.min_confidence)
# Check confidence scores
self.progress_tracker.update_tracking(tracking_id, message="Checking confidence scores...")
low_confidence = [e for e in entities if e.confidence < min_confidence]
if low_confidence:
warnings.append(f"{len(low_confidence)} entities below confidence threshold")
# Check for duplicates
self.progress_tracker.update_tracking(tracking_id, message="Checking for duplicates...")
entity_texts = [e.text.lower() for e in entities]
duplicates = len(entity_texts) - len(set(entity_texts))
if duplicates > 0:
warnings.append(f"{duplicates} duplicate entities found")
# Check for empty entities
empty_entities = [e for e in entities if not e.text.strip()]
if empty_entities:
errors.append(f"{len(empty_entities)} empty entities found")
# Calculate metrics
metrics = {
"total_entities": len(entities),
"high_confidence": len([e for e in entities if e.confidence >= 0.8]),
"medium_confidence": len([e for e in entities if min_confidence <= e.confidence < 0.8]),
"low_confidence": len(low_confidence),
"unique_entities": len(set(entity_texts)),
"duplicates": duplicates,
"entity_types": len(set(e.label for e in entities)),
"average_confidence": sum(e.confidence for e in entities) / len(entities) if entities else 0.0
}
# Calculate score
score = self._calculate_entity_score(entities, metrics)
valid = len(errors) == 0
result = ValidationResult(
valid=valid,
score=score,
errors=errors,
warnings=warnings,
metrics=metrics
)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Validation complete: {len(errors)} errors, {len(warnings)} warnings")
return result
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def validate_relations(self, relations: List[Relation], **options) -> ValidationResult:
"""
semantica/semantic_extract/llm_enhancer.py
+22 -5
View File
@@ -31,6 +31,7 @@ from typing import Any, Dict, List, Optional
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .ner_extractor import Entity
from .relation_extractor import Relation
@@ -61,6 +62,7 @@ class LLMEnhancer:
"""
self.logger = get_logger("llm_enhancer")
self.config = config
self.progress_tracker = get_progress_tracker()
self.provider = config.get("provider", "openai")
self.model = config.get("model", "gpt-3.5-turbo")
@@ -106,18 +108,33 @@ class LLMEnhancer:
Returns:
list: Enhanced entities
"""
if not self.client:
self.logger.warning("LLM client not available. Returning original entities.")
return entities
prompt = self._build_entity_prompt(text, entities)
tracking_id = self.progress_tracker.start_tracking(
module="semantic_extract",
submodule="LLMEnhancer",
message="Enhancing entities using LLM"
)
try:
if not self.client:
self.logger.warning("LLM client not available. Returning original entities.")
self.progress_tracker.stop_tracking(tracking_id, status="completed", message="LLM client not available")
return entities
self.progress_tracker.update_tracking(tracking_id, message="Building prompt...")
prompt = self._build_entity_prompt(text, entities)
self.progress_tracker.update_tracking(tracking_id, message="Calling LLM API...")
response = self._call_llm(prompt, **options)
self.progress_tracker.update_tracking(tracking_id, message="Parsing LLM response...")
enhanced_entities = self._parse_entity_response(response, entities)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Enhanced {len(enhanced_entities)} entities")
return enhanced_entities
except Exception as e:
self.logger.error(f"Failed to enhance entities with LLM: {e}")
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
return entities
def enhance_relations(self, text: str, relations: List[Relation], **options) -> List[Relation]:
semantica/semantic_extract/named_entity_recognizer.py
+16 -1
View File
@@ -35,6 +35,7 @@ from typing import Any, Dict, List, Optional
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .ner_extractor import Entity, NERExtractor
@@ -55,6 +56,7 @@ class NamedEntityRecognizer:
self.logger = get_logger("named_entity_recognizer")
self.config = config or {}
self.config.update(kwargs)
self.progress_tracker = get_progress_tracker()
# Use NERExtractor for actual extraction
self.ner_extractor = NERExtractor(**self.config.get("ner", {}))
@@ -72,7 +74,20 @@ class NamedEntityRecognizer:
Returns:
list: List of extracted entities
"""
return self.ner_extractor.extract_entities(text, **options)
tracking_id = self.progress_tracker.start_tracking(
module="semantic_extract",
submodule="NamedEntityRecognizer",
message="Extracting named entities"
)
try:
entities = self.ner_extractor.extract_entities(text, **options)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Extracted {len(entities)} entities")
return entities
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def classify_entities(self, entities: List[Entity], **context) -> Dict[str, List[Entity]]:
"""
semantica/semantic_extract/ner_extractor.py
+29 -9
View File
@@ -31,6 +31,7 @@ from typing import Any, Dict, List, Optional, Tuple
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
try:
import spacy
@@ -70,6 +71,7 @@ class NERExtractor:
self.model_name = config.get("model", "en_core_web_sm")
self.language = config.get("language", "en")
self.min_confidence = config.get("min_confidence", 0.5)
self.progress_tracker = get_progress_tracker()
# Initialize spaCy model if available
self.nlp = None
@@ -92,16 +94,34 @@ class NERExtractor:
Returns:
list: List of extracted entities
"""
if not text:
return []
tracking_id = self.progress_tracker.start_tracking(
module="semantic_extract",
submodule="NERExtractor",
message="Extracting named entities from text"
)
min_confidence = options.get("min_confidence", self.min_confidence)
entity_types = options.get("entity_types")
if self.nlp:
return self._extract_with_spacy(text, min_confidence, entity_types)
else:
return self._extract_fallback(text)
try:
if not text:
self.progress_tracker.stop_tracking(tracking_id, status="completed", message="No text provided")
return []
min_confidence = options.get("min_confidence", self.min_confidence)
entity_types = options.get("entity_types")
if self.nlp:
self.progress_tracker.update_tracking(tracking_id, message="Extracting entities using spaCy...")
entities = self._extract_with_spacy(text, min_confidence, entity_types)
else:
self.progress_tracker.update_tracking(tracking_id, message="Extracting entities using fallback method...")
entities = self._extract_fallback(text)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Extracted {len(entities)} entities")
return entities
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def _extract_with_spacy(self, text: str, min_confidence: float, entity_types: Optional[List[str]]) -> List[Entity]:
"""Extract entities using spaCy."""
semantica/semantic_extract/relation_extractor.py
+29 -11
View File
@@ -33,6 +33,7 @@ from typing import Any, Dict, List, Optional, Tuple
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .ner_extractor import Entity
@@ -62,6 +63,7 @@ class RelationExtractor:
"""
self.logger = get_logger("relation_extractor")
self.config = config
self.progress_tracker = get_progress_tracker()
self.min_confidence = config.get("min_confidence", 0.5)
self.use_llm = config.get("use_llm", False)
@@ -98,18 +100,34 @@ class RelationExtractor:
Returns:
list: List of extracted relations
"""
if not text or not entities:
return []
tracking_id = self.progress_tracker.start_tracking(
module="semantic_extract",
submodule="RelationExtractor",
message=f"Extracting relations from {len(entities)} entities"
)
min_confidence = options.get("min_confidence", self.min_confidence)
# Pattern-based extraction
relations = self._extract_with_patterns(text, entities)
# Filter by confidence
relations = [r for r in relations if r.confidence >= min_confidence]
return relations
try:
if not text or not entities:
self.progress_tracker.stop_tracking(tracking_id, status="completed", message="No text or entities provided")
return []
min_confidence = options.get("min_confidence", self.min_confidence)
# Pattern-based extraction
self.progress_tracker.update_tracking(tracking_id, message="Extracting relations using patterns...")
relations = self._extract_with_patterns(text, entities)
# Filter by confidence
self.progress_tracker.update_tracking(tracking_id, message=f"Filtering {len(relations)} relations by confidence...")
relations = [r for r in relations if r.confidence >= min_confidence]
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Extracted {len(relations)} relations")
return relations
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def _extract_with_patterns(self, text: str, entities: List[Entity]) -> List[Relation]:
"""Extract relations using pattern matching."""
semantica/semantic_extract/semantic_analyzer.py
+31 -15
View File
@@ -35,6 +35,7 @@ from typing import Any, Dict, List, Optional, Tuple
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
@dataclass
@@ -66,6 +67,7 @@ class SemanticAnalyzer:
self.logger = get_logger("semantic_analyzer")
self.config = config or {}
self.config.update(kwargs)
self.progress_tracker = get_progress_tracker()
self.similarity_analyzer = SimilarityAnalyzer(**self.config.get("similarity", {}))
self.role_labeler = RoleLabeler(**self.config.get("role", {}))
@@ -82,22 +84,36 @@ class SemanticAnalyzer:
Returns:
dict: Semantic analysis results
"""
results = {
"text": text,
"length": len(text),
"word_count": len(text.split()),
"sentence_count": len(text.split('.'))
}
tracking_id = self.progress_tracker.start_tracking(
module="semantic_extract",
submodule="SemanticAnalyzer",
message="Performing comprehensive semantic analysis"
)
# Semantic role labeling
if options.get("label_roles", False):
roles = self.label_semantic_roles(text, **options)
results["semantic_roles"] = [r.__dict__ for r in roles]
# Semantic features
results["semantic_features"] = self._extract_features(text)
return results
try:
results = {
"text": text,
"length": len(text),
"word_count": len(text.split()),
"sentence_count": len(text.split('.'))
}
# Semantic role labeling
if options.get("label_roles", False):
self.progress_tracker.update_tracking(tracking_id, message="Labeling semantic roles...")
roles = self.label_semantic_roles(text, **options)
results["semantic_roles"] = [r.__dict__ for r in roles]
# Semantic features
self.progress_tracker.update_tracking(tracking_id, message="Extracting semantic features...")
results["semantic_features"] = self._extract_features(text)
self.progress_tracker.stop_tracking(tracking_id, status="completed", message="Semantic analysis complete")
return results
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def calculate_similarity(self, text1: str, text2: str, **options) -> float:
"""
semantica/semantic_extract/semantic_network_extractor.py
+34 -16
View File
@@ -36,6 +36,7 @@ import yaml
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .ner_extractor import Entity
from .relation_extractor import Relation
@@ -83,6 +84,7 @@ class SemanticNetworkExtractor:
"""
self.logger = get_logger("semantic_network_extractor")
self.config = config
self.progress_tracker = get_progress_tracker()
def extract_network(
self,
@@ -103,23 +105,39 @@ class SemanticNetworkExtractor:
Returns:
SemanticNetwork: Extracted semantic network
"""
from .ner_extractor import NERExtractor
from .relation_extractor import RelationExtractor
tracking_id = self.progress_tracker.start_tracking(
module="semantic_extract",
submodule="SemanticNetworkExtractor",
message="Extracting semantic network from text"
)
# Extract entities if not provided
if entities is None:
ner = NERExtractor(**self.config.get("ner", {}))
entities = ner.extract_entities(text)
# Extract relations if not provided
if relations is None:
rel_extractor = RelationExtractor(**self.config.get("relation", {}))
relations = rel_extractor.extract_relations(text, entities)
# Build network
network = self._build_network(entities, relations)
return network
try:
from .ner_extractor import NERExtractor
from .relation_extractor import RelationExtractor
# Extract entities if not provided
if entities is None:
self.progress_tracker.update_tracking(tracking_id, message="Extracting entities...")
ner = NERExtractor(**self.config.get("ner", {}))
entities = ner.extract_entities(text)
# Extract relations if not provided
if relations is None:
self.progress_tracker.update_tracking(tracking_id, message="Extracting relations...")
rel_extractor = RelationExtractor(**self.config.get("relation", {}))
relations = rel_extractor.extract_relations(text, entities)
# Build network
self.progress_tracker.update_tracking(tracking_id, message="Building semantic network...")
network = self._build_network(entities, relations)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Extracted network: {len(network.nodes)} nodes, {len(network.edges)} edges")
return network
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def _build_network(self, entities: List[Entity], relations: List[Relation]) -> SemanticNetwork:
"""Build semantic network from entities and relations."""
semantica/semantic_extract/triple_extractor.py
+51 -32
View File
@@ -37,6 +37,7 @@ from urllib.parse import quote
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .ner_extractor import Entity
from .relation_extractor import Relation
@@ -60,6 +61,7 @@ class TripleExtractor:
self.logger = get_logger("triple_extractor")
self.config = config or {}
self.config.update(kwargs)
self.progress_tracker = get_progress_tracker()
self.triple_validator = TripleValidator(**self.config.get("validator", {}))
self.rdf_serializer = RDFSerializer(**self.config.get("serializer", {}))
@@ -86,39 +88,56 @@ class TripleExtractor:
Returns:
list: List of extracted triples
"""
from .ner_extractor import NERExtractor
from .relation_extractor import RelationExtractor
tracking_id = self.progress_tracker.start_tracking(
module="semantic_extract",
submodule="TripleExtractor",
message="Extracting RDF triples from text"
)
# Extract entities if not provided
if entities is None:
ner = NERExtractor(**self.config.get("ner", {}))
entities = ner.extract_entities(text)
# Extract relations if not provided
if relationships is None:
rel_extractor = RelationExtractor(**self.config.get("relation", {}))
relationships = rel_extractor.extract_relations(text, entities)
# Convert relations to triples
triples = []
for relation in relationships:
triple = Triple(
subject=self._format_uri(relation.subject.text),
predicate=self._format_uri(relation.predicate),
object=self._format_uri(relation.object.text),
confidence=relation.confidence,
metadata={
"context": relation.context,
**relation.metadata
}
)
triples.append(triple)
# Validate triples
if options.get("validate", True):
triples = self.triple_validator.validate_triples(triples)
return triples
try:
from .ner_extractor import NERExtractor
from .relation_extractor import RelationExtractor
# Extract entities if not provided
if entities is None:
self.progress_tracker.update_tracking(tracking_id, message="Extracting entities...")
ner = NERExtractor(**self.config.get("ner", {}))
entities = ner.extract_entities(text)
# Extract relations if not provided
if relationships is None:
self.progress_tracker.update_tracking(tracking_id, message="Extracting relations...")
rel_extractor = RelationExtractor(**self.config.get("relation", {}))
relationships = rel_extractor.extract_relations(text, entities)
# Convert relations to triples
self.progress_tracker.update_tracking(tracking_id, message=f"Converting {len(relationships)} relations to triples...")
triples = []
for relation in relationships:
triple = Triple(
subject=self._format_uri(relation.subject.text),
predicate=self._format_uri(relation.predicate),
object=self._format_uri(relation.object.text),
confidence=relation.confidence,
metadata={
"context": relation.context,
**relation.metadata
}
)
triples.append(triple)
# Validate triples
if options.get("validate", True):
self.progress_tracker.update_tracking(tracking_id, message="Validating triples...")
triples = self.triple_validator.validate_triples(triples)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Extracted {len(triples)} triples")
return triples
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def _format_uri(self, value: str) -> str:
"""Format value as URI."""
semantica/split/chunk_validator.py
+61 -41
View File
@@ -33,6 +33,7 @@ from typing import Any, Dict, List, Optional
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .semantic_chunker import Chunk
@@ -62,6 +63,7 @@ class ChunkValidator:
"""
self.logger = get_logger("chunk_validator")
self.config = config
self.progress_tracker = get_progress_tracker()
self.min_size = config.get("min_size", 10)
self.max_size = config.get("max_size", 10000)
@@ -78,48 +80,66 @@ class ChunkValidator:
Returns:
ValidationResult: Validation result
"""
errors = []
warnings = []
metrics = {}
# Size validation
chunk_size = len(chunk.text)
metrics["size"] = chunk_size
if chunk_size < self.min_size:
errors.append(f"Chunk too small: {chunk_size} < {self.min_size}")
if chunk_size > self.max_size:
errors.append(f"Chunk too large: {chunk_size} > {self.max_size}")
# Content validation
if not chunk.text.strip():
errors.append("Chunk is empty or whitespace only")
# Coherence validation
coherence_score = self._check_coherence(chunk.text)
metrics["coherence"] = coherence_score
if coherence_score < 0.3:
warnings.append(f"Low semantic coherence: {coherence_score:.2f}")
# Structure validation
structure_score = self._check_structure(chunk.text)
metrics["structure"] = structure_score
# Calculate overall score
score = self._calculate_score(chunk_size, coherence_score, structure_score, errors)
metrics["overall_score"] = score
valid = len(errors) == 0 and score >= self.min_score
return ValidationResult(
valid=valid,
score=score,
errors=errors,
warnings=warnings,
metrics=metrics
tracking_id = self.progress_tracker.start_tracking(
module="split",
submodule="ChunkValidator",
message="Validating chunk"
)
try:
errors = []
warnings = []
metrics = {}
# Size validation
self.progress_tracker.update_tracking(tracking_id, message="Validating chunk size...")
chunk_size = len(chunk.text)
metrics["size"] = chunk_size
if chunk_size < self.min_size:
errors.append(f"Chunk too small: {chunk_size} < {self.min_size}")
if chunk_size > self.max_size:
errors.append(f"Chunk too large: {chunk_size} > {self.max_size}")
# Content validation
if not chunk.text.strip():
errors.append("Chunk is empty or whitespace only")
# Coherence validation
self.progress_tracker.update_tracking(tracking_id, message="Checking semantic coherence...")
coherence_score = self._check_coherence(chunk.text)
metrics["coherence"] = coherence_score
if coherence_score < 0.3:
warnings.append(f"Low semantic coherence: {coherence_score:.2f}")
# Structure validation
self.progress_tracker.update_tracking(tracking_id, message="Checking structure...")
structure_score = self._check_structure(chunk.text)
metrics["structure"] = structure_score
# Calculate overall score
score = self._calculate_score(chunk_size, coherence_score, structure_score, errors)
metrics["overall_score"] = score
valid = len(errors) == 0 and score >= self.min_score
result = ValidationResult(
valid=valid,
score=score,
errors=errors,
warnings=warnings,
metrics=metrics
)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Validation complete: {'valid' if valid else 'invalid'}")
return result
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def validate_batch(self, chunks: List[Chunk], **options) -> Dict[str, Any]:
"""
semantica/split/provenance_tracker.py
+30 -14
View File
@@ -33,6 +33,7 @@ from uuid import uuid4
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .semantic_chunker import Chunk
@@ -65,6 +66,7 @@ class ProvenanceTracker:
"""
self.logger = get_logger("provenance_tracker")
self.config = config
self.progress_tracker = get_progress_tracker()
self.store_metadata = config.get("store_metadata", True)
self.track_versions = config.get("track_versions", False)
@@ -136,21 +138,35 @@ class ProvenanceTracker:
Returns:
list: List of provenance IDs
"""
provenance_ids = []
parent_chunk_id = None
tracking_id = self.progress_tracker.start_tracking(
module="split",
submodule="ProvenanceTracker",
message=f"Tracking provenance for {len(chunks)} chunks"
)
for chunk in chunks:
provenance_id = self.track_chunk(
chunk,
source_document,
source_path,
parent_chunk_id,
**metadata
)
provenance_ids.append(provenance_id)
parent_chunk_id = provenance_id
return provenance_ids
try:
provenance_ids = []
parent_chunk_id = None
for i, chunk in enumerate(chunks):
self.progress_tracker.update_tracking(tracking_id, message=f"Tracking chunk {i+1}/{len(chunks)}...")
provenance_id = self.track_chunk(
chunk,
source_document,
source_path,
parent_chunk_id,
**metadata
)
provenance_ids.append(provenance_id)
parent_chunk_id = provenance_id
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Tracked {len(provenance_ids)} chunks")
return provenance_ids
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def get_provenance(self, chunk_id: str) -> Optional[ProvenanceInfo]:
"""
semantica/split/semantic_chunker.py
+27 -7
View File
@@ -33,6 +33,7 @@ from typing import Any, Dict, List, Optional, Tuple
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
try:
import spacy
@@ -71,6 +72,7 @@ class SemanticChunker:
self.chunk_size = config.get("chunk_size", 1000)
self.chunk_overlap = config.get("chunk_overlap", 200)
self.language = config.get("language", "en")
self.progress_tracker = get_progress_tracker()
# Initialize spaCy model if available
self.nlp = None
@@ -94,14 +96,32 @@ class SemanticChunker:
Returns:
list: List of chunks
"""
if not text:
return []
tracking_id = self.progress_tracker.start_tracking(
module="split",
submodule="SemanticChunker",
message="Splitting text into semantic chunks"
)
# Use spaCy if available
if self.nlp:
return self._chunk_with_spacy(text, **options)
else:
return self._chunk_fallback(text, **options)
try:
if not text:
self.progress_tracker.stop_tracking(tracking_id, status="completed", message="No text provided")
return []
# Use spaCy if available
if self.nlp:
self.progress_tracker.update_tracking(tracking_id, message="Chunking with spaCy...")
chunks = self._chunk_with_spacy(text, **options)
else:
self.progress_tracker.update_tracking(tracking_id, message="Chunking with fallback method...")
chunks = self._chunk_fallback(text, **options)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Created {len(chunks)} chunks")
return chunks
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def _chunk_with_spacy(self, text: str, **options) -> List[Chunk]:
"""Chunk text using spaCy."""
semantica/split/sliding_window_chunker.py
+28 -8
View File
@@ -30,6 +30,7 @@ from typing import Any, Dict, List, Optional
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .semantic_chunker import Chunk
@@ -48,6 +49,7 @@ class SlidingWindowChunker:
"""
self.logger = get_logger("sliding_window_chunker")
self.config = config
self.progress_tracker = get_progress_tracker()
self.chunk_size = config.get("chunk_size", 1000)
self.overlap = config.get("overlap", 0)
@@ -72,15 +74,33 @@ class SlidingWindowChunker:
Returns:
list: List of chunks
"""
if not text:
return []
tracking_id = self.progress_tracker.start_tracking(
module="split",
submodule="SlidingWindowChunker",
message="Splitting text using sliding window"
)
preserve_boundaries = options.get("preserve_boundaries", True)
if preserve_boundaries:
return self._chunk_with_boundaries(text)
else:
return self._chunk_fixed_size(text)
try:
if not text:
self.progress_tracker.stop_tracking(tracking_id, status="completed", message="No text provided")
return []
preserve_boundaries = options.get("preserve_boundaries", True)
if preserve_boundaries:
self.progress_tracker.update_tracking(tracking_id, message="Chunking with boundary preservation...")
chunks = self._chunk_with_boundaries(text)
else:
self.progress_tracker.update_tracking(tracking_id, message="Chunking with fixed-size windows...")
chunks = self._chunk_fixed_size(text)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Created {len(chunks)} chunks")
return chunks
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def _chunk_fixed_size(self, text: str) -> List[Chunk]:
"""Chunk text with fixed-size windows."""
semantica/split/structural_chunker.py
+27 -9
View File
@@ -32,6 +32,7 @@ from typing import Any, Dict, List, Optional, Tuple
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .semantic_chunker import Chunk
@@ -60,6 +61,7 @@ class StructuralChunker:
"""
self.logger = get_logger("structural_chunker")
self.config = config
self.progress_tracker = get_progress_tracker()
self.respect_headers = config.get("respect_headers", True)
self.respect_sections = config.get("respect_sections", True)
@@ -76,16 +78,32 @@ class StructuralChunker:
Returns:
list: List of chunks
"""
if not text:
return []
tracking_id = self.progress_tracker.start_tracking(
module="split",
submodule="StructuralChunker",
message="Splitting text based on document structure"
)
# Extract structural elements
elements = self._extract_structure(text)
# Group elements into chunks
chunks = self._group_elements(elements, text)
return chunks
try:
if not text:
self.progress_tracker.stop_tracking(tracking_id, status="completed", message="No text provided")
return []
# Extract structural elements
self.progress_tracker.update_tracking(tracking_id, message="Extracting structural elements...")
elements = self._extract_structure(text)
# Group elements into chunks
self.progress_tracker.update_tracking(tracking_id, message="Grouping elements into chunks...")
chunks = self._group_elements(elements, text)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Created {len(chunks)} chunks")
return chunks
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def _extract_structure(self, text: str) -> List[StructuralElement]:
"""Extract structural elements from text."""
semantica/split/table_chunker.py
+39 -18
View File
@@ -32,6 +32,7 @@ from typing import Any, Dict, List, Optional, Union
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
from .semantic_chunker import Chunk
@@ -61,6 +62,7 @@ class TableChunker:
"""
self.logger = get_logger("table_chunker")
self.config = config
self.progress_tracker = get_progress_tracker()
self.max_rows = config.get("max_rows", 100)
self.preserve_headers = config.get("preserve_headers", True)
@@ -77,25 +79,44 @@ class TableChunker:
Returns:
list: List of table chunks
"""
# Parse table data
if isinstance(table_data, dict):
headers = table_data.get("headers", [])
rows = table_data.get("rows", [])
elif isinstance(table_data, list) and len(table_data) > 0:
# First row as headers if not provided
if options.get("first_row_as_header", True):
headers = table_data[0]
rows = table_data[1:]
else:
headers = [f"Column_{i+1}" for i in range(len(table_data[0]))]
rows = table_data
else:
raise ValidationError("Invalid table data format")
tracking_id = self.progress_tracker.start_tracking(
module="split",
submodule="TableChunker",
message="Chunking table data"
)
if self.chunk_by_columns:
return self._chunk_by_columns(headers, rows, **options)
else:
return self._chunk_by_rows(headers, rows, **options)
try:
# Parse table data
self.progress_tracker.update_tracking(tracking_id, message="Parsing table data...")
if isinstance(table_data, dict):
headers = table_data.get("headers", [])
rows = table_data.get("rows", [])
elif isinstance(table_data, list) and len(table_data) > 0:
# First row as headers if not provided
if options.get("first_row_as_header", True):
headers = table_data[0]
rows = table_data[1:]
else:
headers = [f"Column_{i+1}" for i in range(len(table_data[0]))]
rows = table_data
else:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message="Invalid table data format")
raise ValidationError("Invalid table data format")
if self.chunk_by_columns:
self.progress_tracker.update_tracking(tracking_id, message="Chunking by columns...")
chunks = self._chunk_by_columns(headers, rows, **options)
else:
self.progress_tracker.update_tracking(tracking_id, message="Chunking by rows...")
chunks = self._chunk_by_rows(headers, rows, **options)
self.progress_tracker.stop_tracking(tracking_id, status="completed",
message=f"Created {len(chunks)} table chunks")
return chunks
except Exception as e:
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
raise
def _chunk_by_rows(self, headers: List[str], rows: List[List[str]], **options) -> List[TableChunk]:
"""Chunk table by rows."""