mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-15 04:00:33 +00:00
Add progress tracking to all methods in GraphBuilder
- Add progress tracking to add_temporal_edge method - Add progress tracking to create_temporal_snapshot method - Add progress tracking to query_temporal method - Add progress tracking to load_from_neo4j method - Include intermediate progress updates for long-running operations - Add proper error handling with progress tracking in all methods
This commit is contained in:
KaifAhmad1
@@ -31,6 +31,7 @@ from collections import defaultdict
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class CSVExporter:
|
||||
@@ -88,6 +89,9 @@ class CSVExporter:
|
||||
self.encoding = encoding
|
||||
self.include_header = include_header
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug(
|
||||
f"CSV exporter initialized: delimiter='{delimiter}', "
|
||||
f"encoding={encoding}, include_header={include_header}"
|
||||
@@ -128,38 +132,57 @@ class CSVExporter:
|
||||
... "output_base"
|
||||
... )
|
||||
"""
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
# Track CSV export
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=str(file_path),
|
||||
module="export",
|
||||
submodule="CSVExporter",
|
||||
message=f"Exporting data to CSV: {file_path}"
|
||||
)
|
||||
|
||||
self.logger.debug(f"Exporting data to CSV: {file_path}")
|
||||
|
||||
# Handle different data structures
|
||||
if isinstance(data, dict):
|
||||
# Export each key as separate CSV file
|
||||
exported_files = []
|
||||
for key, value in data.items():
|
||||
if isinstance(value, list):
|
||||
output_path = file_path.parent / f"{file_path.stem}_{key}.csv"
|
||||
self._write_csv(value, output_path, fieldnames=fieldnames, mode=mode, **options)
|
||||
exported_files.append(output_path)
|
||||
else:
|
||||
self.logger.warning(
|
||||
f"Skipping key '{key}': value is not a list (type: {type(value)})"
|
||||
)
|
||||
try:
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
|
||||
self.logger.info(
|
||||
f"Exported {len(exported_files)} CSV file(s) from dictionary: "
|
||||
f"{', '.join(str(f) for f in exported_files)}"
|
||||
)
|
||||
elif isinstance(data, list):
|
||||
# Single CSV file
|
||||
self._write_csv(data, file_path, fieldnames=fieldnames, mode=mode, **options)
|
||||
self.logger.info(f"Exported CSV to: {file_path}")
|
||||
else:
|
||||
raise ValidationError(
|
||||
f"Unsupported data type: {type(data)}. "
|
||||
"Expected list of dicts or dict with list values."
|
||||
)
|
||||
self.logger.debug(f"Exporting data to CSV: {file_path}")
|
||||
|
||||
# Handle different data structures
|
||||
if isinstance(data, dict):
|
||||
# Export each key as separate CSV file
|
||||
exported_files = []
|
||||
self.progress_tracker.update_tracking(tracking_id, message=f"Exporting {len(data)} data groups...")
|
||||
for key, value in data.items():
|
||||
if isinstance(value, list):
|
||||
output_path = file_path.parent / f"{file_path.stem}_{key}.csv"
|
||||
self._write_csv(value, output_path, fieldnames=fieldnames, mode=mode, **options)
|
||||
exported_files.append(output_path)
|
||||
else:
|
||||
self.logger.warning(
|
||||
f"Skipping key '{key}': value is not a list (type: {type(value)})"
|
||||
)
|
||||
|
||||
self.logger.info(
|
||||
f"Exported {len(exported_files)} CSV file(s) from dictionary: "
|
||||
f"{', '.join(str(f) for f in exported_files)}"
|
||||
)
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Exported {len(exported_files)} CSV files")
|
||||
elif isinstance(data, list):
|
||||
# Single CSV file
|
||||
self.progress_tracker.update_tracking(tracking_id, message=f"Exporting {len(data)} records...")
|
||||
self._write_csv(data, file_path, fieldnames=fieldnames, mode=mode, **options)
|
||||
self.logger.info(f"Exported CSV to: {file_path}")
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Exported CSV to: {file_path}")
|
||||
else:
|
||||
raise ValidationError(
|
||||
f"Unsupported data type: {type(data)}. "
|
||||
"Expected list of dicts or dict with list values."
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def export_entities(
|
||||
self,
|
||||
|
||||
@@ -29,6 +29,7 @@ import json
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class GraphExporter:
|
||||
@@ -82,6 +83,9 @@ class GraphExporter:
|
||||
self.format = format
|
||||
self.include_attributes = include_attributes
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug(
|
||||
f"Graph exporter initialized: format={format}, "
|
||||
f"include_attributes={include_attributes}"
|
||||
@@ -126,33 +130,49 @@ class GraphExporter:
|
||||
... }
|
||||
>>> exporter.export(graph_data, "graph.graphml", format="graphml")
|
||||
"""
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
|
||||
export_format = format or self.format
|
||||
|
||||
self.logger.debug(
|
||||
f"Exporting graph to {export_format}: {file_path}, "
|
||||
f"nodes={len(graph_data.get('nodes', []))}, "
|
||||
f"edges={len(graph_data.get('edges', []))}"
|
||||
# Track graph export
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=str(file_path),
|
||||
module="export",
|
||||
submodule="GraphExporter",
|
||||
message=f"Exporting graph to {format or self.format}: {file_path}"
|
||||
)
|
||||
|
||||
# Export based on format
|
||||
if export_format == "json":
|
||||
self._export_json(graph_data, file_path, **options)
|
||||
elif export_format == "graphml":
|
||||
self._export_graphml(graph_data, file_path, **options)
|
||||
elif export_format == "gexf":
|
||||
self._export_gexf(graph_data, file_path, **options)
|
||||
elif export_format == "dot":
|
||||
self._export_dot(graph_data, file_path, **options)
|
||||
else:
|
||||
raise ValidationError(
|
||||
f"Unsupported graph format: {export_format}. "
|
||||
f"Supported formats: json, graphml, gexf, dot"
|
||||
try:
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
|
||||
export_format = format or self.format
|
||||
|
||||
self.logger.debug(
|
||||
f"Exporting graph to {export_format}: {file_path}, "
|
||||
f"nodes={len(graph_data.get('nodes', []))}, "
|
||||
f"edges={len(graph_data.get('edges', []))}"
|
||||
)
|
||||
|
||||
self.logger.info(f"Exported graph ({export_format}) to: {file_path}")
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message=f"Exporting in {export_format} format...")
|
||||
# Export based on format
|
||||
if export_format == "json":
|
||||
self._export_json(graph_data, file_path, **options)
|
||||
elif export_format == "graphml":
|
||||
self._export_graphml(graph_data, file_path, **options)
|
||||
elif export_format == "gexf":
|
||||
self._export_gexf(graph_data, file_path, **options)
|
||||
elif export_format == "dot":
|
||||
self._export_dot(graph_data, file_path, **options)
|
||||
else:
|
||||
raise ValidationError(
|
||||
f"Unsupported graph format: {export_format}. "
|
||||
f"Supported formats: json, graphml, gexf, dot"
|
||||
)
|
||||
|
||||
self.logger.info(f"Exported graph ({export_format}) to: {file_path}")
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Exported graph ({export_format}) to: {file_path}")
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def export_knowledge_graph(
|
||||
self,
|
||||
|
||||
@@ -31,6 +31,7 @@ from datetime import datetime
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.helpers import ensure_directory, write_json_file
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class JSONExporter:
|
||||
@@ -87,6 +88,9 @@ class JSONExporter:
|
||||
self.ensure_ascii = ensure_ascii
|
||||
self.format = format
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug(
|
||||
f"JSON exporter initialized: indent={indent}, "
|
||||
f"ensure_ascii={ensure_ascii}, format={format}"
|
||||
@@ -122,41 +126,58 @@ class JSONExporter:
|
||||
... format="json-ld"
|
||||
... )
|
||||
"""
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
|
||||
export_format = format or self.format
|
||||
|
||||
self.logger.debug(
|
||||
f"Exporting data to JSON ({export_format}): {file_path}, "
|
||||
f"include_metadata={include_metadata}, include_provenance={include_provenance}"
|
||||
# Track JSON export
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=str(file_path),
|
||||
module="export",
|
||||
submodule="JSONExporter",
|
||||
message=f"Exporting data to JSON: {file_path}"
|
||||
)
|
||||
|
||||
# Convert data to appropriate format
|
||||
if export_format == "json-ld":
|
||||
json_data = self._convert_to_jsonld(
|
||||
data,
|
||||
include_metadata=include_metadata,
|
||||
include_provenance=include_provenance,
|
||||
**options
|
||||
try:
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
|
||||
export_format = format or self.format
|
||||
|
||||
self.logger.debug(
|
||||
f"Exporting data to JSON ({export_format}): {file_path}, "
|
||||
f"include_metadata={include_metadata}, include_provenance={include_provenance}"
|
||||
)
|
||||
else:
|
||||
json_data = self._convert_to_json(
|
||||
data,
|
||||
include_metadata=include_metadata,
|
||||
include_provenance=include_provenance,
|
||||
**options
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message=f"Converting to {export_format} format...")
|
||||
# Convert data to appropriate format
|
||||
if export_format == "json-ld":
|
||||
json_data = self._convert_to_jsonld(
|
||||
data,
|
||||
include_metadata=include_metadata,
|
||||
include_provenance=include_provenance,
|
||||
**options
|
||||
)
|
||||
else:
|
||||
json_data = self._convert_to_json(
|
||||
data,
|
||||
include_metadata=include_metadata,
|
||||
include_provenance=include_provenance,
|
||||
**options
|
||||
)
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Writing JSON file...")
|
||||
# Write JSON file
|
||||
write_json_file(
|
||||
json_data,
|
||||
file_path,
|
||||
indent=self.indent,
|
||||
ensure_ascii=self.ensure_ascii
|
||||
)
|
||||
|
||||
# Write JSON file
|
||||
write_json_file(
|
||||
json_data,
|
||||
file_path,
|
||||
indent=self.indent,
|
||||
ensure_ascii=self.ensure_ascii
|
||||
)
|
||||
|
||||
self.logger.info(f"Exported JSON ({export_format}) to: {file_path}")
|
||||
|
||||
self.logger.info(f"Exported JSON ({export_format}) to: {file_path}")
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Exported JSON ({export_format}) to: {file_path}")
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def export_knowledge_graph(
|
||||
self,
|
||||
|
||||
@@ -29,6 +29,7 @@ from datetime import datetime
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class OWLExporter:
|
||||
@@ -84,6 +85,9 @@ class OWLExporter:
|
||||
self.version = version
|
||||
self.format = format
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug(
|
||||
f"OWL exporter initialized: uri={ontology_uri}, "
|
||||
f"version={version}, format={format}"
|
||||
@@ -132,34 +136,51 @@ class OWLExporter:
|
||||
... }
|
||||
>>> exporter.export(ontology, "ontology.owl", format="owl-xml")
|
||||
"""
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
|
||||
export_format = format or self.format
|
||||
|
||||
self.logger.debug(
|
||||
f"Exporting ontology to {export_format}: {file_path}, "
|
||||
f"classes={len(ontology.get('classes', []))}, "
|
||||
f"object_properties={len(ontology.get('object_properties', []))}, "
|
||||
f"data_properties={len(ontology.get('data_properties', []))}"
|
||||
# Track OWL export
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=str(file_path),
|
||||
module="export",
|
||||
submodule="OWLExporter",
|
||||
message=f"Exporting ontology to {format or self.format}: {file_path}"
|
||||
)
|
||||
|
||||
# Generate OWL content based on format
|
||||
if export_format == "owl-xml":
|
||||
owl_content = self._export_owl_xml(ontology, **options)
|
||||
elif export_format == "turtle":
|
||||
owl_content = self._export_owl_turtle(ontology, **options)
|
||||
else:
|
||||
raise ValidationError(
|
||||
f"Unsupported OWL format: {export_format}. "
|
||||
"Supported formats: owl-xml, turtle"
|
||||
try:
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
|
||||
export_format = format or self.format
|
||||
|
||||
self.logger.debug(
|
||||
f"Exporting ontology to {export_format}: {file_path}, "
|
||||
f"classes={len(ontology.get('classes', []))}, "
|
||||
f"object_properties={len(ontology.get('object_properties', []))}, "
|
||||
f"data_properties={len(ontology.get('data_properties', []))}"
|
||||
)
|
||||
|
||||
# Write OWL file
|
||||
with open(file_path, "w", encoding=encoding) as f:
|
||||
f.write(owl_content)
|
||||
|
||||
self.logger.info(f"Exported OWL ({export_format}) to: {file_path}")
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message=f"Converting ontology to {export_format}...")
|
||||
# Generate OWL content based on format
|
||||
if export_format == "owl-xml":
|
||||
owl_content = self._export_owl_xml(ontology, **options)
|
||||
elif export_format == "turtle":
|
||||
owl_content = self._export_owl_turtle(ontology, **options)
|
||||
else:
|
||||
raise ValidationError(
|
||||
f"Unsupported OWL format: {export_format}. "
|
||||
"Supported formats: owl-xml, turtle"
|
||||
)
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Writing OWL file...")
|
||||
# Write OWL file
|
||||
with open(file_path, "w", encoding=encoding) as f:
|
||||
f.write(owl_content)
|
||||
|
||||
self.logger.info(f"Exported OWL ({export_format}) to: {file_path}")
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Exported OWL ({export_format}) to: {file_path}")
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def export_ontology(
|
||||
self,
|
||||
|
||||
@@ -35,6 +35,7 @@ from pathlib import Path
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class NamespaceManager:
|
||||
@@ -761,6 +762,9 @@ class RDFExporter:
|
||||
# Supported RDF formats
|
||||
self.supported_formats = ["turtle", "rdfxml", "jsonld", "ntriples", "n3"]
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug(
|
||||
f"RDF exporter initialized with {len(self.supported_formats)} format(s)"
|
||||
)
|
||||
@@ -793,27 +797,38 @@ class RDFExporter:
|
||||
>>> rdf_string = exporter.export_to_rdf(data, format="turtle")
|
||||
>>> print(rdf_string)
|
||||
"""
|
||||
if format not in self.supported_formats:
|
||||
raise ValidationError(
|
||||
f"Unsupported RDF format: {format}. "
|
||||
f"Supported formats: {', '.join(self.supported_formats)}"
|
||||
)
|
||||
# Track RDF export
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=None,
|
||||
module="export",
|
||||
submodule="RDFExporter",
|
||||
message=f"Exporting data to RDF format: {format}"
|
||||
)
|
||||
|
||||
self.logger.debug(f"Exporting to RDF format: {format}")
|
||||
|
||||
# Validate input data
|
||||
validation = self.validator.validate_rdf_syntax(data, format)
|
||||
if not validation["valid"]:
|
||||
self.logger.warning(
|
||||
f"RDF validation issues found: {validation['errors']}. "
|
||||
"Continuing with export, but data may be invalid."
|
||||
)
|
||||
if validation["warnings"]:
|
||||
self.logger.debug(f"RDF validation warnings: {validation['warnings']}")
|
||||
|
||||
# Serialize based on format
|
||||
if format == "turtle":
|
||||
return self.serializer.serialize_to_turtle(data, **options)
|
||||
try:
|
||||
if format not in self.supported_formats:
|
||||
raise ValidationError(
|
||||
f"Unsupported RDF format: {format}. "
|
||||
f"Supported formats: {', '.join(self.supported_formats)}"
|
||||
)
|
||||
|
||||
self.logger.debug(f"Exporting to RDF format: {format}")
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Validating RDF data...")
|
||||
# Validate input data
|
||||
validation = self.validator.validate_rdf_syntax(data, format)
|
||||
if not validation["valid"]:
|
||||
self.logger.warning(
|
||||
f"RDF validation issues found: {validation['errors']}. "
|
||||
"Continuing with export, but data may be invalid."
|
||||
)
|
||||
if validation["warnings"]:
|
||||
self.logger.debug(f"RDF validation warnings: {validation['warnings']}")
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message=f"Serializing to {format} format...")
|
||||
# Serialize based on format
|
||||
if format == "turtle":
|
||||
result = self.serializer.serialize_to_turtle(data, **options)
|
||||
elif format == "rdfxml":
|
||||
return self.serializer.serialize_to_rdfxml(data, **options)
|
||||
elif format == "jsonld":
|
||||
|
||||
@@ -31,6 +31,7 @@ import json
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class ReportGenerator:
|
||||
@@ -87,6 +88,9 @@ class ReportGenerator:
|
||||
self.include_charts = include_charts
|
||||
self.template = template
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug(
|
||||
f"Report generator initialized: format={format}, "
|
||||
f"include_charts={include_charts}"
|
||||
@@ -141,42 +145,61 @@ class ReportGenerator:
|
||||
>>> # Get as string
|
||||
>>> report = generator.generate_report(data, format="markdown")
|
||||
"""
|
||||
report_format = format or self.format
|
||||
|
||||
self.logger.debug(
|
||||
f"Generating report ({report_format}): "
|
||||
f"title={data.get('title', 'Report')}, "
|
||||
f"file_path={file_path}"
|
||||
# Track report generation
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=str(file_path) if file_path else None,
|
||||
module="export",
|
||||
submodule="ReportGenerator",
|
||||
message=f"Generating {format or self.format} report"
|
||||
)
|
||||
|
||||
# Generate report based on format
|
||||
if report_format == "markdown":
|
||||
report = self._generate_markdown(data, **options)
|
||||
elif report_format == "html":
|
||||
report = self._generate_html(data, **options)
|
||||
elif report_format == "json":
|
||||
report = self._generate_json(data, **options)
|
||||
elif report_format == "text":
|
||||
report = self._generate_text(data, **options)
|
||||
else:
|
||||
raise ValidationError(
|
||||
f"Unsupported report format: {report_format}. "
|
||||
"Supported formats: markdown, html, json, text"
|
||||
try:
|
||||
report_format = format or self.format
|
||||
|
||||
self.logger.debug(
|
||||
f"Generating report ({report_format}): "
|
||||
f"title={data.get('title', 'Report')}, "
|
||||
f"file_path={file_path}"
|
||||
)
|
||||
|
||||
# Write to file if path provided
|
||||
if file_path:
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
|
||||
with open(file_path, "w", encoding=encoding) as f:
|
||||
f.write(report)
|
||||
self.progress_tracker.update_tracking(tracking_id, message=f"Generating {report_format} report...")
|
||||
# Generate report based on format
|
||||
if report_format == "markdown":
|
||||
report = self._generate_markdown(data, **options)
|
||||
elif report_format == "html":
|
||||
report = self._generate_html(data, **options)
|
||||
elif report_format == "json":
|
||||
report = self._generate_json(data, **options)
|
||||
elif report_format == "text":
|
||||
report = self._generate_text(data, **options)
|
||||
else:
|
||||
raise ValidationError(
|
||||
f"Unsupported report format: {report_format}. "
|
||||
"Supported formats: markdown, html, json, text"
|
||||
)
|
||||
|
||||
self.logger.info(f"Generated report ({report_format}) to: {file_path}")
|
||||
return None
|
||||
|
||||
# Return report string
|
||||
return report
|
||||
# Write to file if path provided
|
||||
if file_path:
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Writing report to file...")
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
|
||||
with open(file_path, "w", encoding=encoding) as f:
|
||||
f.write(report)
|
||||
|
||||
self.logger.info(f"Generated report ({report_format}) to: {file_path}")
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Generated report ({report_format}) to: {file_path}")
|
||||
return None
|
||||
|
||||
# Return report string
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Generated {report_format} report")
|
||||
return report
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def generate_quality_report(
|
||||
self,
|
||||
|
||||
@@ -30,6 +30,7 @@ import numpy as np
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.helpers import ensure_directory, write_json_file
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class VectorExporter:
|
||||
@@ -85,6 +86,9 @@ class VectorExporter:
|
||||
self.include_metadata = include_metadata
|
||||
self.include_text = include_text
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug(
|
||||
f"Vector exporter initialized: format={format}, "
|
||||
f"include_metadata={include_metadata}, include_text={include_text}"
|
||||
@@ -129,40 +133,57 @@ class VectorExporter:
|
||||
... ]
|
||||
>>> exporter.export(vectors, "vectors.json", format="json")
|
||||
"""
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
|
||||
export_format = format or self.format
|
||||
|
||||
self.logger.debug(
|
||||
f"Exporting vectors to {export_format}: {file_path}, "
|
||||
f"include_metadata={self.include_metadata}, include_text={self.include_text}"
|
||||
# Track vector export
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=str(file_path),
|
||||
module="export",
|
||||
submodule="VectorExporter",
|
||||
message=f"Exporting vectors to {format or self.format}: {file_path}"
|
||||
)
|
||||
|
||||
# Normalize input data
|
||||
if isinstance(vectors, dict):
|
||||
vector_list = vectors.get("vectors", [])
|
||||
metadata = vectors.get("metadata", {})
|
||||
else:
|
||||
vector_list = vectors
|
||||
metadata = {}
|
||||
|
||||
# Export based on format
|
||||
if export_format == "json":
|
||||
self._export_json(vector_list, file_path, metadata, **options)
|
||||
elif export_format == "numpy":
|
||||
self._export_numpy(vector_list, file_path, **options)
|
||||
elif export_format == "binary":
|
||||
self._export_binary(vector_list, file_path, **options)
|
||||
elif export_format == "faiss":
|
||||
self._export_faiss(vector_list, file_path, **options)
|
||||
else:
|
||||
raise ValidationError(
|
||||
f"Unsupported vector format: {export_format}. "
|
||||
"Supported formats: json, numpy, binary, faiss"
|
||||
try:
|
||||
file_path = Path(file_path)
|
||||
ensure_directory(file_path.parent)
|
||||
|
||||
export_format = format or self.format
|
||||
|
||||
self.logger.debug(
|
||||
f"Exporting vectors to {export_format}: {file_path}, "
|
||||
f"include_metadata={self.include_metadata}, include_text={self.include_text}"
|
||||
)
|
||||
|
||||
self.logger.info(f"Exported vectors ({export_format}) to: {file_path}")
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Normalizing input data...")
|
||||
# Normalize input data
|
||||
if isinstance(vectors, dict):
|
||||
vector_list = vectors.get("vectors", [])
|
||||
metadata = vectors.get("metadata", {})
|
||||
else:
|
||||
vector_list = vectors
|
||||
metadata = {}
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message=f"Exporting in {export_format} format...")
|
||||
# Export based on format
|
||||
if export_format == "json":
|
||||
self._export_json(vector_list, file_path, metadata, **options)
|
||||
elif export_format == "numpy":
|
||||
self._export_numpy(vector_list, file_path, **options)
|
||||
elif export_format == "binary":
|
||||
self._export_binary(vector_list, file_path, **options)
|
||||
elif export_format == "faiss":
|
||||
self._export_faiss(vector_list, file_path, **options)
|
||||
else:
|
||||
raise ValidationError(
|
||||
f"Unsupported vector format: {export_format}. "
|
||||
"Supported formats: json, numpy, binary, faiss"
|
||||
)
|
||||
|
||||
self.logger.info(f"Exported vectors ({export_format}) to: {file_path}")
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Exported vectors ({export_format}) to: {file_path}")
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def export_embeddings(
|
||||
self,
|
||||
|
||||
@@ -28,6 +28,7 @@ from datetime import datetime
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.helpers import ensure_directory
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class SemanticNetworkYAMLExporter:
|
||||
@@ -74,6 +75,9 @@ class SemanticNetworkYAMLExporter:
|
||||
"PyYAML not installed. Install with: pip install pyyaml"
|
||||
)
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Semantic network YAML exporter initialized")
|
||||
|
||||
def export_semantic_network(
|
||||
@@ -106,22 +110,41 @@ class SemanticNetworkYAMLExporter:
|
||||
... }
|
||||
>>> yaml_str = exporter.export_semantic_network(network)
|
||||
"""
|
||||
yaml_data = {
|
||||
"metadata": {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"version": "1.0",
|
||||
**semantic_network.get("metadata", {})
|
||||
},
|
||||
"entities": semantic_network.get("entities", []),
|
||||
"relationships": semantic_network.get("relationships", []),
|
||||
"triples": semantic_network.get("triples", [])
|
||||
}
|
||||
|
||||
return self.yaml.dump(
|
||||
yaml_data,
|
||||
default_flow_style=False,
|
||||
sort_keys=False
|
||||
# Track YAML export
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=None,
|
||||
module="export",
|
||||
submodule="SemanticNetworkYAMLExporter",
|
||||
message="Exporting semantic network to YAML"
|
||||
)
|
||||
|
||||
try:
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Preparing YAML data...")
|
||||
yaml_data = {
|
||||
"metadata": {
|
||||
"exported_at": datetime.now().isoformat(),
|
||||
"version": "1.0",
|
||||
**semantic_network.get("metadata", {})
|
||||
},
|
||||
"entities": semantic_network.get("entities", []),
|
||||
"relationships": semantic_network.get("relationships", []),
|
||||
"triples": semantic_network.get("triples", [])
|
||||
}
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Serializing to YAML...")
|
||||
result = self.yaml.dump(
|
||||
yaml_data,
|
||||
default_flow_style=False,
|
||||
sort_keys=False
|
||||
)
|
||||
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message="Exported semantic network to YAML")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def export(
|
||||
self,
|
||||
|
||||
@@ -25,6 +25,7 @@ from typing import Any, Dict, List, Optional
|
||||
from collections import defaultdict, deque
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class CentralityCalculator:
|
||||
@@ -88,6 +89,9 @@ class CentralityCalculator:
|
||||
"Install with: pip install networkx"
|
||||
)
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.info("Centrality calculator initialized")
|
||||
|
||||
def calculate_degree_centrality(self, graph: Dict[str, Any]) -> Dict[str, Any]:
|
||||
@@ -113,79 +117,102 @@ class CentralityCalculator:
|
||||
>>> top_node = result["rankings"][0]["node"]
|
||||
>>> top_score = result["rankings"][0]["score"]
|
||||
"""
|
||||
self.logger.info("Calculating degree centrality")
|
||||
|
||||
# Use NetworkX if available for faster calculation
|
||||
if self.use_networkx:
|
||||
try:
|
||||
nx_graph = self._to_networkx(graph)
|
||||
centrality_dict = self.nx.degree_centrality(nx_graph)
|
||||
|
||||
# Convert to rankings
|
||||
ranked = sorted(
|
||||
centrality_dict.items(),
|
||||
key=lambda x: x[1],
|
||||
reverse=True
|
||||
)
|
||||
|
||||
max_degree = max(
|
||||
dict(nx_graph.degree()).values()
|
||||
) if nx_graph.number_of_nodes() > 0 else 0
|
||||
|
||||
return {
|
||||
"centrality": centrality_dict,
|
||||
"rankings": [
|
||||
{"node": node, "score": score}
|
||||
for node, score in ranked
|
||||
],
|
||||
"max_degree": max_degree,
|
||||
"total_nodes": nx_graph.number_of_nodes()
|
||||
}
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
f"NetworkX calculation failed: {e}, using basic implementation"
|
||||
)
|
||||
|
||||
# Basic implementation using adjacency list
|
||||
adjacency = self._build_adjacency(graph)
|
||||
|
||||
# Calculate raw degrees (number of connections per node)
|
||||
degrees = {}
|
||||
max_degree = 0
|
||||
|
||||
for node in adjacency:
|
||||
degree = len(adjacency[node])
|
||||
degrees[node] = degree
|
||||
max_degree = max(max_degree, degree)
|
||||
|
||||
# Calculate normalized centrality scores
|
||||
# Normalization: degree / (n - 1) where n is number of nodes
|
||||
centrality = {}
|
||||
num_nodes = len(adjacency)
|
||||
normalization = num_nodes - 1 if num_nodes > 1 else 1
|
||||
|
||||
for node, degree in degrees.items():
|
||||
centrality[node] = (
|
||||
degree / normalization if normalization > 0 else 0.0
|
||||
)
|
||||
|
||||
# Rank nodes by centrality (highest first)
|
||||
ranked = sorted(centrality.items(), key=lambda x: x[1], reverse=True)
|
||||
|
||||
self.logger.debug(
|
||||
f"Degree centrality calculated: {num_nodes} nodes, "
|
||||
f"max degree: {max_degree}"
|
||||
# Track centrality calculation
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=None,
|
||||
module="kg",
|
||||
submodule="CentralityCalculator",
|
||||
message="Calculating degree centrality"
|
||||
)
|
||||
|
||||
return {
|
||||
"centrality": centrality,
|
||||
"rankings": [
|
||||
{"node": node, "score": score}
|
||||
for node, score in ranked
|
||||
],
|
||||
"max_degree": max_degree,
|
||||
"total_nodes": num_nodes
|
||||
}
|
||||
try:
|
||||
self.logger.info("Calculating degree centrality")
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Processing graph structure...")
|
||||
# Use NetworkX if available for faster calculation
|
||||
if self.use_networkx:
|
||||
try:
|
||||
nx_graph = self._to_networkx(graph)
|
||||
centrality_dict = self.nx.degree_centrality(nx_graph)
|
||||
|
||||
# Convert to rankings
|
||||
ranked = sorted(
|
||||
centrality_dict.items(),
|
||||
key=lambda x: x[1],
|
||||
reverse=True
|
||||
)
|
||||
|
||||
max_degree = max(
|
||||
dict(nx_graph.degree()).values()
|
||||
) if nx_graph.number_of_nodes() > 0 else 0
|
||||
|
||||
result = {
|
||||
"centrality": centrality_dict,
|
||||
"rankings": [
|
||||
{"node": node, "score": score}
|
||||
for node, score in ranked
|
||||
],
|
||||
"max_degree": max_degree,
|
||||
"total_nodes": nx_graph.number_of_nodes()
|
||||
}
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Calculated degree centrality for {nx_graph.number_of_nodes()} nodes")
|
||||
return result
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
f"NetworkX calculation failed: {e}, using basic implementation"
|
||||
)
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Building adjacency list...")
|
||||
# Basic implementation using adjacency list
|
||||
adjacency = self._build_adjacency(graph)
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Calculating degrees...")
|
||||
# Calculate raw degrees (number of connections per node)
|
||||
degrees = {}
|
||||
max_degree = 0
|
||||
|
||||
for node in adjacency:
|
||||
degree = len(adjacency[node])
|
||||
degrees[node] = degree
|
||||
max_degree = max(max_degree, degree)
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Normalizing centrality scores...")
|
||||
# Calculate normalized centrality scores
|
||||
# Normalization: degree / (n - 1) where n is number of nodes
|
||||
centrality = {}
|
||||
num_nodes = len(adjacency)
|
||||
normalization = num_nodes - 1 if num_nodes > 1 else 1
|
||||
|
||||
for node, degree in degrees.items():
|
||||
centrality[node] = (
|
||||
degree / normalization if normalization > 0 else 0.0
|
||||
)
|
||||
|
||||
# Rank nodes by centrality (highest first)
|
||||
ranked = sorted(centrality.items(), key=lambda x: x[1], reverse=True)
|
||||
|
||||
self.logger.debug(
|
||||
f"Degree centrality calculated: {num_nodes} nodes, "
|
||||
f"max degree: {max_degree}"
|
||||
)
|
||||
|
||||
result = {
|
||||
"centrality": centrality,
|
||||
"rankings": [
|
||||
{"node": node, "score": score}
|
||||
for node, score in ranked
|
||||
],
|
||||
"max_degree": max_degree,
|
||||
"total_nodes": num_nodes
|
||||
}
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Calculated degree centrality for {num_nodes} nodes")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def calculate_betweenness_centrality(self, graph):
|
||||
"""
|
||||
|
||||
@@ -30,6 +30,7 @@ from typing import Any, Dict, List, Optional
|
||||
from collections import defaultdict
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class CommunityDetector:
|
||||
@@ -84,6 +85,9 @@ class CommunityDetector:
|
||||
self.nx = None
|
||||
self.use_networkx = False
|
||||
self.logger.warning("NetworkX not available, using basic implementations")
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
def detect_communities_louvain(
|
||||
self,
|
||||
@@ -115,37 +119,58 @@ class CommunityDetector:
|
||||
- modularity: Calculated modularity score
|
||||
- algorithm: Algorithm name ("louvain")
|
||||
"""
|
||||
self.logger.info("Detecting communities using Louvain algorithm")
|
||||
# Track community detection
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=None,
|
||||
module="kg",
|
||||
submodule="CommunityDetector",
|
||||
message="Detecting communities using Louvain algorithm"
|
||||
)
|
||||
|
||||
if self.use_networkx:
|
||||
try:
|
||||
import networkx.algorithms.community as nx_comm
|
||||
nx_graph = self._to_networkx(graph)
|
||||
resolution = options.get("resolution", 1.0)
|
||||
|
||||
# Use greedy modularity communities (Louvain-like)
|
||||
communities = nx_comm.greedy_modularity_communities(nx_graph, resolution=resolution)
|
||||
|
||||
# Convert to node assignments
|
||||
node_communities = {}
|
||||
for i, community in enumerate(communities):
|
||||
for node in community:
|
||||
node_communities[node] = i
|
||||
|
||||
modularity = nx_comm.modularity(nx_graph, communities)
|
||||
|
||||
return {
|
||||
"communities": list(communities),
|
||||
"node_assignments": node_communities,
|
||||
"modularity": modularity,
|
||||
"algorithm": "louvain"
|
||||
}
|
||||
except Exception as e:
|
||||
self.logger.warning(f"NetworkX Louvain failed: {e}, using basic implementation")
|
||||
|
||||
# Basic greedy modularity implementation
|
||||
adjacency = self._build_adjacency(graph)
|
||||
return self._basic_community_detection(adjacency, algorithm="louvain", **options)
|
||||
try:
|
||||
self.logger.info("Detecting communities using Louvain algorithm")
|
||||
|
||||
if self.use_networkx:
|
||||
try:
|
||||
import networkx.algorithms.community as nx_comm
|
||||
nx_graph = self._to_networkx(graph)
|
||||
resolution = options.get("resolution", 1.0)
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Detecting communities with NetworkX...")
|
||||
# Use greedy modularity communities (Louvain-like)
|
||||
communities = nx_comm.greedy_modularity_communities(nx_graph, resolution=resolution)
|
||||
|
||||
# Convert to node assignments
|
||||
node_communities = {}
|
||||
for i, community in enumerate(communities):
|
||||
for node in community:
|
||||
node_communities[node] = i
|
||||
|
||||
modularity = nx_comm.modularity(nx_graph, communities)
|
||||
|
||||
result = {
|
||||
"communities": list(communities),
|
||||
"node_assignments": node_communities,
|
||||
"modularity": modularity,
|
||||
"algorithm": "louvain"
|
||||
}
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Detected {len(communities)} communities")
|
||||
return result
|
||||
except Exception as e:
|
||||
self.logger.warning(f"NetworkX Louvain failed: {e}, using basic implementation")
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Using basic community detection...")
|
||||
# Basic greedy modularity implementation
|
||||
adjacency = self._build_adjacency(graph)
|
||||
result = self._basic_community_detection(adjacency, algorithm="louvain", **options)
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Detected {len(result.get('communities', []))} communities")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def detect_communities_leiden(
|
||||
self,
|
||||
|
||||
@@ -28,6 +28,7 @@ License: MIT
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from ..conflicts.conflict_detector import ConflictDetector as BaseConflictDetector, Conflict
|
||||
from ..conflicts.conflict_resolver import ConflictResolver
|
||||
|
||||
@@ -67,6 +68,9 @@ class ConflictDetector:
|
||||
self.logger = get_logger("conflict_detector")
|
||||
self.config = config
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
# Initialize conflict detection components
|
||||
self.base_detector = BaseConflictDetector(**config.get("detection", {}))
|
||||
self.resolver = ConflictResolver(**config.get("resolution", {}))
|
||||
@@ -112,81 +116,89 @@ class ConflictDetector:
|
||||
elif hasattr(knowledge_graph, "get_relationships"):
|
||||
relationships = knowledge_graph.get_relationships()
|
||||
|
||||
conflicts = []
|
||||
|
||||
# Detect value conflicts
|
||||
entity_properties = {}
|
||||
for entity in entities:
|
||||
entity_id = entity.get("id") or entity.get("entity_id")
|
||||
if not entity_id:
|
||||
continue
|
||||
conflicts = []
|
||||
|
||||
for prop_name, prop_value in entity.items():
|
||||
if prop_name in ["id", "entity_id", "type", "source"]:
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Detecting value conflicts...")
|
||||
# Detect value conflicts
|
||||
entity_properties = {}
|
||||
for entity in entities:
|
||||
entity_id = entity.get("id") or entity.get("entity_id")
|
||||
if not entity_id:
|
||||
continue
|
||||
|
||||
if entity_id not in entity_properties:
|
||||
entity_properties[entity_id] = {}
|
||||
|
||||
if prop_name not in entity_properties[entity_id]:
|
||||
entity_properties[entity_id][prop_name] = []
|
||||
|
||||
entity_properties[entity_id][prop_name].append({
|
||||
"value": prop_value,
|
||||
"entity": entity
|
||||
})
|
||||
|
||||
# Check for conflicts
|
||||
for entity_id, properties in entity_properties.items():
|
||||
for prop_name, values in properties.items():
|
||||
unique_values = {str(v["value"]) for v in values if v["value"] is not None}
|
||||
if len(unique_values) > 1:
|
||||
conflicts.append({
|
||||
"entity_id": entity_id,
|
||||
"property": prop_name,
|
||||
"conflicting_values": list(unique_values),
|
||||
"type": "value_conflict",
|
||||
"sources": [v["entity"].get("source", "unknown") for v in values]
|
||||
for prop_name, prop_value in entity.items():
|
||||
if prop_name in ["id", "entity_id", "type", "source"]:
|
||||
continue
|
||||
|
||||
if entity_id not in entity_properties:
|
||||
entity_properties[entity_id] = {}
|
||||
|
||||
if prop_name not in entity_properties[entity_id]:
|
||||
entity_properties[entity_id][prop_name] = []
|
||||
|
||||
entity_properties[entity_id][prop_name].append({
|
||||
"value": prop_value,
|
||||
"entity": entity
|
||||
})
|
||||
|
||||
# Detect relationship conflicts
|
||||
relationship_map = {}
|
||||
for rel in relationships:
|
||||
source = rel.get("source") or rel.get("subject")
|
||||
target = rel.get("target") or rel.get("object")
|
||||
rel_type = rel.get("type") or rel.get("predicate")
|
||||
|
||||
key = f"{source}::{rel_type}::{target}"
|
||||
if key not in relationship_map:
|
||||
relationship_map[key] = []
|
||||
relationship_map[key].append(rel)
|
||||
|
||||
# Check for relationship conflicts
|
||||
for key, rels in relationship_map.items():
|
||||
if len(rels) > 1:
|
||||
# Check for conflicting properties
|
||||
properties = {}
|
||||
for rel in rels:
|
||||
for prop_name, prop_value in rel.items():
|
||||
if prop_name in ["source", "target", "subject", "object", "type", "predicate"]:
|
||||
continue
|
||||
if prop_name not in properties:
|
||||
properties[prop_name] = []
|
||||
properties[prop_name].append(prop_value)
|
||||
|
||||
# Check for conflicts
|
||||
for entity_id, properties in entity_properties.items():
|
||||
for prop_name, values in properties.items():
|
||||
unique_values = {str(v) for v in values if v is not None}
|
||||
unique_values = {str(v["value"]) for v in values if v["value"] is not None}
|
||||
if len(unique_values) > 1:
|
||||
conflicts.append({
|
||||
"relationship": key,
|
||||
"entity_id": entity_id,
|
||||
"property": prop_name,
|
||||
"conflicting_values": list(unique_values),
|
||||
"type": "relationship_conflict",
|
||||
"sources": [rel.get("source", "unknown") for rel in rels]
|
||||
"type": "value_conflict",
|
||||
"sources": [v["entity"].get("source", "unknown") for v in values]
|
||||
})
|
||||
|
||||
self.logger.info(f"Detected {len(conflicts)} conflicts")
|
||||
return conflicts
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Detecting relationship conflicts...")
|
||||
# Detect relationship conflicts
|
||||
relationship_map = {}
|
||||
for rel in relationships:
|
||||
source = rel.get("source") or rel.get("subject")
|
||||
target = rel.get("target") or rel.get("object")
|
||||
rel_type = rel.get("type") or rel.get("predicate")
|
||||
|
||||
key = f"{source}::{rel_type}::{target}"
|
||||
if key not in relationship_map:
|
||||
relationship_map[key] = []
|
||||
relationship_map[key].append(rel)
|
||||
|
||||
# Check for relationship conflicts
|
||||
for key, rels in relationship_map.items():
|
||||
if len(rels) > 1:
|
||||
# Check for conflicting properties
|
||||
properties = {}
|
||||
for rel in rels:
|
||||
for prop_name, prop_value in rel.items():
|
||||
if prop_name in ["source", "target", "subject", "object", "type", "predicate"]:
|
||||
continue
|
||||
if prop_name not in properties:
|
||||
properties[prop_name] = []
|
||||
properties[prop_name].append(prop_value)
|
||||
|
||||
for prop_name, values in properties.items():
|
||||
unique_values = {str(v) for v in values if v is not None}
|
||||
if len(unique_values) > 1:
|
||||
conflicts.append({
|
||||
"relationship": key,
|
||||
"property": prop_name,
|
||||
"conflicting_values": list(unique_values),
|
||||
"type": "relationship_conflict",
|
||||
"sources": [rel.get("source", "unknown") for rel in rels]
|
||||
})
|
||||
|
||||
self.logger.info(f"Detected {len(conflicts)} conflicts")
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Detected {len(conflicts)} conflicts")
|
||||
return conflicts
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def resolve_conflicts(
|
||||
self,
|
||||
|
||||
@@ -32,6 +32,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
from collections import defaultdict, deque
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class ConnectivityAnalyzer:
|
||||
@@ -73,6 +74,9 @@ class ConnectivityAnalyzer:
|
||||
self.connectivity_algorithms = [
|
||||
"dfs", "bfs", "tarjan", "kosaraju"
|
||||
]
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
self.analysis_config = config.get("analysis_config", {})
|
||||
self.config = config
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ License: MIT
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from ..deduplication.duplicate_detector import DuplicateDetector, DuplicateGroup
|
||||
from ..deduplication.entity_merger import EntityMerger
|
||||
|
||||
@@ -67,6 +68,9 @@ class Deduplicator:
|
||||
self.logger = get_logger("deduplicator")
|
||||
self.config = config
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
# Initialize deduplication components
|
||||
self.duplicate_detector = DuplicateDetector(**config.get("detection", {}))
|
||||
self.entity_merger = EntityMerger(**config.get("merger", {}))
|
||||
@@ -99,14 +103,20 @@ class Deduplicator:
|
||||
**self.config
|
||||
)
|
||||
|
||||
# Convert to list of lists
|
||||
result = []
|
||||
for group in duplicate_groups:
|
||||
if len(group.entities) >= 2:
|
||||
result.append(group.entities)
|
||||
|
||||
self.logger.info(f"Found {len(result)} duplicate groups")
|
||||
return result
|
||||
# Convert to list of lists
|
||||
result = []
|
||||
for group in duplicate_groups:
|
||||
if len(group.entities) >= 2:
|
||||
result.append(group.entities)
|
||||
|
||||
self.logger.info(f"Found {len(result)} duplicate groups")
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Found {len(result)} duplicate groups")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def merge_duplicates(
|
||||
self,
|
||||
|
||||
@@ -23,6 +23,7 @@ License: MIT
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from ..deduplication.duplicate_detector import DuplicateDetector
|
||||
from ..deduplication.entity_merger import EntityMerger
|
||||
|
||||
@@ -62,6 +63,9 @@ class EntityResolver:
|
||||
- merger: Configuration for entity merger
|
||||
"""
|
||||
self.logger = get_logger("entity_resolver")
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
self.config = config
|
||||
|
||||
# Resolution strategy and threshold
|
||||
@@ -121,47 +125,48 @@ class EntityResolver:
|
||||
threshold=self.similarity_threshold
|
||||
)
|
||||
|
||||
self.logger.debug(f"Found {len(duplicate_groups)} duplicate group(s)")
|
||||
|
||||
# Step 2: Merge duplicates in each group
|
||||
merged_entities = []
|
||||
processed_entity_ids = set() # Track which entities have been merged
|
||||
|
||||
for group in duplicate_groups:
|
||||
# Skip groups with less than 2 entities (not duplicates)
|
||||
if len(group.entities) < 2:
|
||||
continue
|
||||
self.logger.debug(f"Found {len(duplicate_groups)} duplicate group(s)")
|
||||
|
||||
# Merge the duplicate group into a single canonical entity
|
||||
merge_operations = self.entity_merger.merge_duplicates(
|
||||
group.entities,
|
||||
**self.config
|
||||
)
|
||||
self.progress_tracker.update_tracking(tracking_id, message=f"Found {len(duplicate_groups)} duplicate group(s)")
|
||||
# Step 2: Merge duplicates in each group
|
||||
merged_entities = []
|
||||
processed_entity_ids = set() # Track which entities have been merged
|
||||
|
||||
# Process each merge operation
|
||||
for operation in merge_operations:
|
||||
merged_entity = operation.merged_entity
|
||||
merged_entities.append(merged_entity)
|
||||
for group in duplicate_groups:
|
||||
# Skip groups with less than 2 entities (not duplicates)
|
||||
if len(group.entities) < 2:
|
||||
continue
|
||||
|
||||
# Mark all source entities as processed
|
||||
for source_entity in operation.source_entities:
|
||||
entity_id = source_entity.get("id") or source_entity.get("entity_id")
|
||||
if entity_id:
|
||||
processed_entity_ids.add(entity_id)
|
||||
|
||||
# Step 3: Add non-duplicate entities (entities not in any duplicate group)
|
||||
for entity in entities:
|
||||
entity_id = entity.get("id") or entity.get("entity_id")
|
||||
if entity_id and entity_id not in processed_entity_ids:
|
||||
# This entity was not merged, add it as-is
|
||||
merged_entities.append(entity)
|
||||
|
||||
# Log resolution statistics
|
||||
original_count = len(entities)
|
||||
resolved_count = len(merged_entities)
|
||||
reduction = original_count - resolved_count
|
||||
|
||||
self.logger.info(
|
||||
# Merge the duplicate group into a single canonical entity
|
||||
merge_operations = self.entity_merger.merge_duplicates(
|
||||
group.entities,
|
||||
**self.config
|
||||
)
|
||||
|
||||
# Process each merge operation
|
||||
for operation in merge_operations:
|
||||
merged_entity = operation.merged_entity
|
||||
merged_entities.append(merged_entity)
|
||||
|
||||
# Mark all source entities as processed
|
||||
for source_entity in operation.source_entities:
|
||||
entity_id = source_entity.get("id") or source_entity.get("entity_id")
|
||||
if entity_id:
|
||||
processed_entity_ids.add(entity_id)
|
||||
|
||||
# Step 3: Add non-duplicate entities (entities not in any duplicate group)
|
||||
for entity in entities:
|
||||
entity_id = entity.get("id") or entity.get("entity_id")
|
||||
if entity_id and entity_id not in processed_entity_ids:
|
||||
# This entity was not merged, add it as-is
|
||||
merged_entities.append(entity)
|
||||
|
||||
# Log resolution statistics
|
||||
original_count = len(entities)
|
||||
resolved_count = len(merged_entities)
|
||||
reduction = original_count - resolved_count
|
||||
|
||||
self.logger.info(
|
||||
f"Entity resolution complete: {original_count} -> {resolved_count} "
|
||||
f"({reduction} duplicate(s) merged)"
|
||||
)
|
||||
|
||||
@@ -25,6 +25,7 @@ License: MIT
|
||||
from .centrality_calculator import CentralityCalculator
|
||||
from .community_detector import CommunityDetector
|
||||
from .connectivity_analyzer import ConnectivityAnalyzer
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class GraphAnalyzer:
|
||||
@@ -74,6 +75,9 @@ class GraphAnalyzer:
|
||||
from ..utils.logging import get_logger
|
||||
self.logger = get_logger("graph_analyzer")
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
# Merge configuration
|
||||
self.config = config or {}
|
||||
self.config.update(kwargs)
|
||||
|
||||
+142
-83
@@ -285,29 +285,42 @@ class GraphBuilder:
|
||||
Returns:
|
||||
Edge object with temporal annotations
|
||||
"""
|
||||
self.logger.info(f"Adding temporal edge: {source} -{relationship}-> {target}")
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
submodule="GraphBuilder",
|
||||
message=f"Adding temporal edge: {source} -{relationship}-> {target}"
|
||||
)
|
||||
|
||||
# Parse temporal information
|
||||
valid_from = self._parse_time(valid_from) or self._get_timestamp()
|
||||
valid_until = self._parse_time(valid_until) if valid_until else None
|
||||
|
||||
# Create edge with temporal information
|
||||
edge = {
|
||||
"source": source,
|
||||
"target": target,
|
||||
"type": relationship,
|
||||
"valid_from": valid_from,
|
||||
"valid_until": valid_until,
|
||||
"temporal_metadata": temporal_metadata or {},
|
||||
**kwargs
|
||||
}
|
||||
|
||||
# Add to graph
|
||||
if "relationships" not in graph:
|
||||
graph["relationships"] = []
|
||||
graph["relationships"].append(edge)
|
||||
|
||||
return edge
|
||||
try:
|
||||
self.logger.info(f"Adding temporal edge: {source} -{relationship}-> {target}")
|
||||
|
||||
# Parse temporal information
|
||||
valid_from = self._parse_time(valid_from) or self._get_timestamp()
|
||||
valid_until = self._parse_time(valid_until) if valid_until else None
|
||||
|
||||
# Create edge with temporal information
|
||||
edge = {
|
||||
"source": source,
|
||||
"target": target,
|
||||
"type": relationship,
|
||||
"valid_from": valid_from,
|
||||
"valid_until": valid_until,
|
||||
"temporal_metadata": temporal_metadata or {},
|
||||
**kwargs
|
||||
}
|
||||
|
||||
# Add to graph
|
||||
if "relationships" not in graph:
|
||||
graph["relationships"] = []
|
||||
graph["relationships"].append(edge)
|
||||
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Added temporal edge: {source} -{relationship}-> {target}")
|
||||
return edge
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def create_temporal_snapshot(self, graph, timestamp=None, snapshot_name=None, **options):
|
||||
"""
|
||||
@@ -322,45 +335,60 @@ class GraphBuilder:
|
||||
Returns:
|
||||
Temporal snapshot object
|
||||
"""
|
||||
self.logger.info(f"Creating temporal snapshot: {snapshot_name or 'unnamed'}")
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
submodule="GraphBuilder",
|
||||
message=f"Creating temporal snapshot: {snapshot_name or 'unnamed'}"
|
||||
)
|
||||
|
||||
snapshot_time = self._parse_time(timestamp) or self._get_timestamp()
|
||||
|
||||
# Filter entities and relationships valid at snapshot time
|
||||
entities = []
|
||||
relationships = []
|
||||
|
||||
# Get all entities
|
||||
if "entities" in graph:
|
||||
entities = graph["entities"].copy()
|
||||
|
||||
# Filter relationships valid at snapshot time
|
||||
if "relationships" in graph:
|
||||
for rel in graph["relationships"]:
|
||||
valid_from = self._parse_time(rel.get("valid_from"))
|
||||
valid_until = self._parse_time(rel.get("valid_until"))
|
||||
|
||||
# Check if relationship is valid at snapshot time
|
||||
if valid_from and self._compare_times(snapshot_time, valid_from) < 0:
|
||||
continue
|
||||
if valid_until and self._compare_times(snapshot_time, valid_until) > 0:
|
||||
continue
|
||||
|
||||
relationships.append(rel)
|
||||
|
||||
snapshot = {
|
||||
"name": snapshot_name or f"snapshot_{snapshot_time}",
|
||||
"timestamp": snapshot_time,
|
||||
"entities": entities,
|
||||
"relationships": relationships,
|
||||
"metadata": {
|
||||
"num_entities": len(entities),
|
||||
"num_relationships": len(relationships),
|
||||
"snapshot_time": snapshot_time
|
||||
try:
|
||||
self.logger.info(f"Creating temporal snapshot: {snapshot_name or 'unnamed'}")
|
||||
|
||||
snapshot_time = self._parse_time(timestamp) or self._get_timestamp()
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Filtering entities and relationships...")
|
||||
|
||||
# Filter entities and relationships valid at snapshot time
|
||||
entities = []
|
||||
relationships = []
|
||||
|
||||
# Get all entities
|
||||
if "entities" in graph:
|
||||
entities = graph["entities"].copy()
|
||||
|
||||
# Filter relationships valid at snapshot time
|
||||
if "relationships" in graph:
|
||||
for rel in graph["relationships"]:
|
||||
valid_from = self._parse_time(rel.get("valid_from"))
|
||||
valid_until = self._parse_time(rel.get("valid_until"))
|
||||
|
||||
# Check if relationship is valid at snapshot time
|
||||
if valid_from and self._compare_times(snapshot_time, valid_from) < 0:
|
||||
continue
|
||||
if valid_until and self._compare_times(snapshot_time, valid_until) > 0:
|
||||
continue
|
||||
|
||||
relationships.append(rel)
|
||||
|
||||
snapshot = {
|
||||
"name": snapshot_name or f"snapshot_{snapshot_time}",
|
||||
"timestamp": snapshot_time,
|
||||
"entities": entities,
|
||||
"relationships": relationships,
|
||||
"metadata": {
|
||||
"num_entities": len(entities),
|
||||
"num_relationships": len(relationships),
|
||||
"snapshot_time": snapshot_time
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return snapshot
|
||||
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Created snapshot with {len(entities)} entities, {len(relationships)} relationships")
|
||||
return snapshot
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def query_temporal(
|
||||
self,
|
||||
@@ -385,30 +413,47 @@ class GraphBuilder:
|
||||
Returns:
|
||||
Query results with temporal context
|
||||
"""
|
||||
self.logger.info(f"Executing temporal query: {query[:50]}...")
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
submodule="GraphBuilder",
|
||||
message=f"Executing temporal query: {query[:50]}..."
|
||||
)
|
||||
|
||||
# Create snapshot for query time
|
||||
if at_time:
|
||||
snapshot = self.create_temporal_snapshot(graph, timestamp=at_time)
|
||||
elif time_range:
|
||||
start_time, end_time = time_range
|
||||
# Query at end time
|
||||
snapshot = self.create_temporal_snapshot(graph, timestamp=end_time)
|
||||
else:
|
||||
# Use current graph
|
||||
snapshot = graph
|
||||
|
||||
# Basic query execution (simplified)
|
||||
# In a real implementation, this would use a proper query engine
|
||||
results = {
|
||||
"query": query,
|
||||
"timestamp": at_time or (time_range[1] if time_range else None),
|
||||
"entities": snapshot.get("entities", []),
|
||||
"relationships": snapshot.get("relationships", []),
|
||||
"metadata": snapshot.get("metadata", {})
|
||||
}
|
||||
|
||||
return results
|
||||
try:
|
||||
self.logger.info(f"Executing temporal query: {query[:50]}...")
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Creating temporal snapshot for query...")
|
||||
|
||||
# Create snapshot for query time
|
||||
if at_time:
|
||||
snapshot = self.create_temporal_snapshot(graph, timestamp=at_time)
|
||||
elif time_range:
|
||||
start_time, end_time = time_range
|
||||
# Query at end time
|
||||
snapshot = self.create_temporal_snapshot(graph, timestamp=end_time)
|
||||
else:
|
||||
# Use current graph
|
||||
snapshot = graph
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Executing query...")
|
||||
|
||||
# Basic query execution (simplified)
|
||||
# In a real implementation, this would use a proper query engine
|
||||
results = {
|
||||
"query": query,
|
||||
"timestamp": at_time or (time_range[1] if time_range else None),
|
||||
"entities": snapshot.get("entities", []),
|
||||
"relationships": snapshot.get("relationships", []),
|
||||
"metadata": snapshot.get("metadata", {})
|
||||
}
|
||||
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Query executed: {len(results.get('entities', []))} entities, {len(results.get('relationships', []))} relationships")
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def load_from_neo4j(
|
||||
self,
|
||||
@@ -435,15 +480,23 @@ class GraphBuilder:
|
||||
Returns:
|
||||
Knowledge graph loaded from Neo4j
|
||||
"""
|
||||
self.logger.info(f"Loading graph from Neo4j: {uri}")
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
module="kg",
|
||||
submodule="GraphBuilder",
|
||||
message=f"Loading graph from Neo4j: {uri}"
|
||||
)
|
||||
|
||||
try:
|
||||
self.logger.info(f"Loading graph from Neo4j: {uri}")
|
||||
|
||||
from neo4j import GraphDatabase
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Connecting to Neo4j...")
|
||||
driver = GraphDatabase.driver(uri, auth=(username, password))
|
||||
|
||||
with driver.session(database=database) as session:
|
||||
# Load nodes
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Loading nodes from Neo4j...")
|
||||
nodes_result = session.run("MATCH (n) RETURN n")
|
||||
entities = []
|
||||
for record in nodes_result:
|
||||
@@ -456,6 +509,7 @@ class GraphBuilder:
|
||||
entities.append(entity)
|
||||
|
||||
# Load relationships
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Loading relationships from Neo4j...")
|
||||
rels_result = session.run("MATCH (a)-[r]->(b) RETURN a, r, b")
|
||||
relationships = []
|
||||
for record in rels_result:
|
||||
@@ -490,11 +544,16 @@ class GraphBuilder:
|
||||
}
|
||||
|
||||
self.logger.info(f"Loaded {len(entities)} entities and {len(relationships)} relationships from Neo4j")
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Loaded {len(entities)} entities and {len(relationships)} relationships from Neo4j")
|
||||
return graph
|
||||
|
||||
except ImportError:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed",
|
||||
message="neo4j library not available")
|
||||
raise ImportError("neo4j library not available. Install with: pip install neo4j")
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
self.logger.error(f"Error loading from Neo4j: {e}")
|
||||
raise
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ from typing import Any, Dict, List, Optional
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -86,6 +87,9 @@ class GraphValidator:
|
||||
self.logger = get_logger("graph_validator")
|
||||
self.config = config
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Graph validator initialized")
|
||||
|
||||
def validate(self, knowledge_graph: Any) -> ValidationResult:
|
||||
@@ -131,9 +135,10 @@ class GraphValidator:
|
||||
elif hasattr(knowledge_graph, "get_relationships"):
|
||||
relationships = knowledge_graph.get_relationships()
|
||||
|
||||
# Validate entities
|
||||
entity_ids = set()
|
||||
for entity in entities:
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Validating entities...")
|
||||
# Validate entities
|
||||
entity_ids = set()
|
||||
for entity in entities:
|
||||
entity_id = entity.get("id") or entity.get("entity_id")
|
||||
if not entity_id:
|
||||
errors.append("Entity missing required 'id' field")
|
||||
@@ -163,32 +168,40 @@ class GraphValidator:
|
||||
elif target not in entity_ids:
|
||||
warnings.append(f"Relationship references unknown target entity: {target}")
|
||||
|
||||
if not rel_type:
|
||||
errors.append("Relationship missing 'type' field")
|
||||
|
||||
# Check for orphaned entities (entities with no relationships)
|
||||
entity_has_relationships = set()
|
||||
for rel in relationships:
|
||||
source = rel.get("source") or rel.get("subject")
|
||||
target = rel.get("target") or rel.get("object")
|
||||
if source:
|
||||
entity_has_relationships.add(source)
|
||||
if target:
|
||||
entity_has_relationships.add(target)
|
||||
|
||||
orphaned = entity_ids - entity_has_relationships
|
||||
if orphaned:
|
||||
warnings.append(f"Found {len(orphaned)} orphaned entities (no relationships)")
|
||||
|
||||
valid = len(errors) == 0
|
||||
|
||||
self.logger.info(f"Validation complete: {len(errors)} errors, {len(warnings)} warnings")
|
||||
|
||||
return ValidationResult(
|
||||
valid=valid,
|
||||
errors=errors,
|
||||
warnings=warnings
|
||||
)
|
||||
if not rel_type:
|
||||
errors.append("Relationship missing 'type' field")
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Checking for orphaned entities...")
|
||||
# Check for orphaned entities (entities with no relationships)
|
||||
entity_has_relationships = set()
|
||||
for rel in relationships:
|
||||
source = rel.get("source") or rel.get("subject")
|
||||
target = rel.get("target") or rel.get("object")
|
||||
if source:
|
||||
entity_has_relationships.add(source)
|
||||
if target:
|
||||
entity_has_relationships.add(target)
|
||||
|
||||
orphaned = entity_ids - entity_has_relationships
|
||||
if orphaned:
|
||||
warnings.append(f"Found {len(orphaned)} orphaned entities (no relationships)")
|
||||
|
||||
valid = len(errors) == 0
|
||||
|
||||
self.logger.info(f"Validation complete: {len(errors)} errors, {len(warnings)} warnings")
|
||||
|
||||
result = ValidationResult(
|
||||
valid=valid,
|
||||
errors=errors,
|
||||
warnings=warnings
|
||||
)
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Validation complete: {len(errors)} errors, {len(warnings)} warnings")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def check_consistency(self, knowledge_graph: Any) -> bool:
|
||||
"""
|
||||
|
||||
@@ -30,6 +30,7 @@ from typing import Any, Dict, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class ProvenanceTracker:
|
||||
@@ -68,6 +69,9 @@ class ProvenanceTracker:
|
||||
self.config = config
|
||||
self.provenance_data: Dict[str, Any] = {}
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Provenance tracker initialized")
|
||||
|
||||
def track_entity(
|
||||
|
||||
@@ -30,6 +30,7 @@ from typing import Any, Dict, List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class SeedManager:
|
||||
@@ -67,6 +68,9 @@ class SeedManager:
|
||||
self.config = config
|
||||
self.seed_data: List[Dict[str, Any]] = []
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Seed manager initialized")
|
||||
|
||||
def load_seed_data(self, source: str, data: Any) -> None:
|
||||
@@ -82,42 +86,59 @@ class SeedManager:
|
||||
data: Seed data to load (list of entities, dict with "entities" key,
|
||||
or single entity dict)
|
||||
"""
|
||||
self.logger.info(f"Loading seed data from source: {source}")
|
||||
# Track seed data loading
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=None,
|
||||
module="kg",
|
||||
submodule="SeedManager",
|
||||
message=f"Loading seed data from source: {source}"
|
||||
)
|
||||
|
||||
# Normalize data format
|
||||
if isinstance(data, list):
|
||||
entities = data
|
||||
elif isinstance(data, dict):
|
||||
entities = data.get("entities", [data])
|
||||
else:
|
||||
entities = [data]
|
||||
|
||||
# Validate and process entities
|
||||
processed_entities = []
|
||||
for entity in entities:
|
||||
if not isinstance(entity, dict):
|
||||
self.logger.warning(f"Skipping invalid entity format: {type(entity)}")
|
||||
continue
|
||||
try:
|
||||
self.logger.info(f"Loading seed data from source: {source}")
|
||||
|
||||
# Ensure entity has required fields
|
||||
if "id" not in entity and "entity_id" not in entity:
|
||||
# Generate ID if missing
|
||||
entity["id"] = f"{source}_{len(processed_entities)}"
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Normalizing data format...")
|
||||
# Normalize data format
|
||||
if isinstance(data, list):
|
||||
entities = data
|
||||
elif isinstance(data, dict):
|
||||
entities = data.get("entities", [data])
|
||||
else:
|
||||
entities = [data]
|
||||
|
||||
# Add source metadata
|
||||
entity["source"] = source
|
||||
entity["seed_data"] = True
|
||||
self.progress_tracker.update_tracking(tracking_id, message=f"Processing {len(entities)} entities...")
|
||||
# Validate and process entities
|
||||
processed_entities = []
|
||||
for entity in entities:
|
||||
if not isinstance(entity, dict):
|
||||
self.logger.warning(f"Skipping invalid entity format: {type(entity)}")
|
||||
continue
|
||||
|
||||
# Ensure entity has required fields
|
||||
if "id" not in entity and "entity_id" not in entity:
|
||||
# Generate ID if missing
|
||||
entity["id"] = f"{source}_{len(processed_entities)}"
|
||||
|
||||
# Add source metadata
|
||||
entity["source"] = source
|
||||
entity["seed_data"] = True
|
||||
|
||||
processed_entities.append(entity)
|
||||
|
||||
processed_entities.append(entity)
|
||||
|
||||
self.seed_data.append({
|
||||
"source": source,
|
||||
"entities": processed_entities,
|
||||
"count": len(processed_entities),
|
||||
"timestamp": datetime.now().isoformat()
|
||||
})
|
||||
|
||||
self.logger.info(f"Loaded {len(processed_entities)} entities from {source}")
|
||||
self.seed_data.append({
|
||||
"source": source,
|
||||
"entities": processed_entities,
|
||||
"count": len(processed_entities),
|
||||
"timestamp": datetime.now().isoformat()
|
||||
})
|
||||
|
||||
self.logger.info(f"Loaded {len(processed_entities)} entities from {source}")
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Loaded {len(processed_entities)} entities from {source}")
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def load_from_file(
|
||||
self,
|
||||
|
||||
@@ -28,6 +28,8 @@ Author: Semantica Contributors
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class TemporalGraphQuery:
|
||||
"""
|
||||
@@ -82,6 +84,9 @@ class TemporalGraphQuery:
|
||||
from ..utils.logging import get_logger
|
||||
self.logger = get_logger("temporal_query")
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
# Initialize pattern detector
|
||||
self.pattern_detector = TemporalPatternDetector(**kwargs.get("pattern_detection", {}))
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ from typing import Any, Dict, List, Optional
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .quality_metrics import QualityMetrics
|
||||
|
||||
|
||||
@@ -89,6 +90,9 @@ class AutomatedFixer:
|
||||
self.config = kwargs
|
||||
self.quality_metrics = QualityMetrics()
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Automated fixer initialized")
|
||||
|
||||
def fix_duplicates(
|
||||
@@ -114,16 +118,33 @@ class AutomatedFixer:
|
||||
- errors: List of error messages
|
||||
- metadata: Additional fix metadata
|
||||
"""
|
||||
self.logger.info("Fixing duplicate entities")
|
||||
|
||||
# In practice, this would use deduplication module
|
||||
# For now, return placeholder
|
||||
return FixResult(
|
||||
success=True,
|
||||
fixed_count=0,
|
||||
errors=[],
|
||||
metadata={}
|
||||
# Track duplicate fixing
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=None,
|
||||
module="kg_qa",
|
||||
submodule="AutomatedFixer",
|
||||
message="Fixing duplicate entities"
|
||||
)
|
||||
|
||||
try:
|
||||
self.logger.info("Fixing duplicate entities")
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Detecting duplicates...")
|
||||
# In practice, this would use deduplication module
|
||||
# For now, return placeholder
|
||||
result = FixResult(
|
||||
success=True,
|
||||
fixed_count=0,
|
||||
errors=[],
|
||||
metadata={}
|
||||
)
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Fixed {result.fixed_count} duplicate(s)")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def fix_inconsistencies(
|
||||
self,
|
||||
|
||||
@@ -31,6 +31,7 @@ License: MIT
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .quality_metrics import QualityMetrics, CompletenessMetrics, ConsistencyMetrics
|
||||
from .validation_engine import ValidationEngine
|
||||
from .reporting import QualityReporter, QualityReport
|
||||
@@ -78,6 +79,9 @@ class KGQualityAssessor:
|
||||
self.validation_engine = ValidationEngine(**kwargs)
|
||||
self.quality_reporter = QualityReporter(**kwargs)
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("KG quality assessor initialized")
|
||||
|
||||
def assess_overall_quality(
|
||||
@@ -99,12 +103,28 @@ class KGQualityAssessor:
|
||||
Returns:
|
||||
float: Overall quality score between 0.0 and 1.0 (higher is better)
|
||||
"""
|
||||
self.logger.info("Assessing overall quality")
|
||||
# Track quality assessment
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=None,
|
||||
module="kg_qa",
|
||||
submodule="KGQualityAssessor",
|
||||
message="Assessing overall quality"
|
||||
)
|
||||
|
||||
# Calculate metrics
|
||||
overall_score = self.quality_metrics.calculate_overall_score(knowledge_graph)
|
||||
|
||||
return overall_score
|
||||
try:
|
||||
self.logger.info("Assessing overall quality")
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Calculating quality metrics...")
|
||||
# Calculate metrics
|
||||
overall_score = self.quality_metrics.calculate_overall_score(knowledge_graph)
|
||||
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Overall quality score: {overall_score:.2f}")
|
||||
return overall_score
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def generate_quality_report(
|
||||
self,
|
||||
|
||||
@@ -31,6 +31,7 @@ from typing import Any, Dict, List, Optional
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -88,6 +89,9 @@ class QualityMetrics:
|
||||
self.logger = get_logger("quality_metrics")
|
||||
self.config = kwargs
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Quality metrics calculator initialized")
|
||||
|
||||
def calculate_overall_score(
|
||||
@@ -108,13 +112,23 @@ class QualityMetrics:
|
||||
Returns:
|
||||
float: Overall quality score between 0.0 and 1.0 (higher is better)
|
||||
"""
|
||||
completeness = self.calculate_entity_quality(knowledge_graph)
|
||||
consistency = self._calculate_consistency(knowledge_graph)
|
||||
|
||||
# Weighted average
|
||||
overall = (0.6 * completeness) + (0.4 * consistency)
|
||||
|
||||
return min(1.0, max(0.0, overall))
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Calculating entity quality...")
|
||||
completeness = self.calculate_entity_quality(knowledge_graph)
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Calculating consistency...")
|
||||
consistency = self._calculate_consistency(knowledge_graph)
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Aggregating scores...")
|
||||
# Weighted average
|
||||
overall = (0.6 * completeness) + (0.4 * consistency)
|
||||
|
||||
result = min(1.0, max(0.0, overall))
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Overall quality score: {result:.2f}")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def calculate_entity_quality(
|
||||
self,
|
||||
|
||||
@@ -32,6 +32,7 @@ from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -120,6 +121,9 @@ class QualityReporter:
|
||||
self.logger = get_logger("quality_reporter")
|
||||
self.config = kwargs
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Quality reporter initialized")
|
||||
|
||||
def generate_report(
|
||||
@@ -144,19 +148,36 @@ class QualityReporter:
|
||||
QualityReport: Comprehensive quality report with scores, issues,
|
||||
and recommendations
|
||||
"""
|
||||
issues = self._identify_issues(knowledge_graph, quality_metrics)
|
||||
recommendations = self._generate_recommendations(issues)
|
||||
|
||||
report = QualityReport(
|
||||
timestamp=datetime.now(),
|
||||
overall_score=quality_metrics.get("overall", 0.0),
|
||||
completeness_score=quality_metrics.get("completeness", 0.0),
|
||||
consistency_score=quality_metrics.get("consistency", 0.0),
|
||||
issues=issues,
|
||||
recommendations=recommendations
|
||||
# Track report generation
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
file=None,
|
||||
module="kg_qa",
|
||||
submodule="QualityReporter",
|
||||
message="Generating quality report"
|
||||
)
|
||||
|
||||
return report
|
||||
try:
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Identifying issues...")
|
||||
issues = self._identify_issues(knowledge_graph, quality_metrics)
|
||||
self.progress_tracker.update_tracking(tracking_id, message="Generating recommendations...")
|
||||
recommendations = self._generate_recommendations(issues)
|
||||
|
||||
report = QualityReport(
|
||||
timestamp=datetime.now(),
|
||||
overall_score=quality_metrics.get("overall", 0.0),
|
||||
completeness_score=quality_metrics.get("completeness", 0.0),
|
||||
consistency_score=quality_metrics.get("consistency", 0.0),
|
||||
issues=issues,
|
||||
recommendations=recommendations
|
||||
)
|
||||
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Generated quality report with {len(issues)} issues")
|
||||
return report
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def export_report(
|
||||
self,
|
||||
@@ -324,6 +345,9 @@ class IssueTracker:
|
||||
self.config = kwargs
|
||||
self.issues: Dict[str, QualityIssue] = {}
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Issue tracker initialized")
|
||||
|
||||
def add_issue(self, issue: QualityIssue) -> None:
|
||||
@@ -424,6 +448,9 @@ class ImprovementSuggestions:
|
||||
self.logger = get_logger("improvement_suggestions")
|
||||
self.config = kwargs
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Improvement suggestions generator initialized")
|
||||
|
||||
def generate_suggestions(
|
||||
|
||||
@@ -30,6 +30,7 @@ from dataclasses import dataclass, field
|
||||
|
||||
from ..utils.exceptions import ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -87,6 +88,9 @@ class ValidationEngine:
|
||||
self.config = kwargs
|
||||
self.rules: List[Callable] = []
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Validation engine initialized")
|
||||
|
||||
def validate(
|
||||
@@ -115,27 +119,35 @@ class ValidationEngine:
|
||||
- warnings: List of warning messages
|
||||
- metadata: Additional validation metadata
|
||||
"""
|
||||
rules_to_use = rules or self.rules
|
||||
errors = []
|
||||
warnings = []
|
||||
|
||||
for rule in rules_to_use:
|
||||
try:
|
||||
result = rule(knowledge_graph)
|
||||
if isinstance(result, dict):
|
||||
if result.get("error"):
|
||||
errors.append(result["error"])
|
||||
if result.get("warning"):
|
||||
warnings.append(result["warning"])
|
||||
except Exception as e:
|
||||
self.logger.error(f"Validation rule error: {e}")
|
||||
errors.append(f"Validation rule failed: {e}")
|
||||
|
||||
return ValidationResult(
|
||||
valid=len(errors) == 0,
|
||||
errors=errors,
|
||||
warnings=warnings
|
||||
)
|
||||
rules_to_use = rules or self.rules
|
||||
errors = []
|
||||
warnings = []
|
||||
|
||||
self.progress_tracker.update_tracking(tracking_id, message=f"Validating with {len(rules_to_use)} rule(s)...")
|
||||
for rule in rules_to_use:
|
||||
try:
|
||||
result = rule(knowledge_graph)
|
||||
if isinstance(result, dict):
|
||||
if result.get("error"):
|
||||
errors.append(result["error"])
|
||||
if result.get("warning"):
|
||||
warnings.append(result["warning"])
|
||||
except Exception as e:
|
||||
self.logger.error(f"Validation rule error: {e}")
|
||||
errors.append(f"Validation rule failed: {e}")
|
||||
|
||||
result = ValidationResult(
|
||||
valid=len(errors) == 0,
|
||||
errors=errors,
|
||||
warnings=warnings
|
||||
)
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed",
|
||||
message=f"Validation complete: {len(errors)} errors, {len(warnings)} warnings")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e))
|
||||
raise
|
||||
|
||||
def add_rule(self, rule: Callable) -> None:
|
||||
"""
|
||||
|
||||
@@ -35,6 +35,7 @@ from collections import defaultdict
|
||||
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -116,6 +117,9 @@ class DataCleaner:
|
||||
self.data_validator = DataValidator(**self.config)
|
||||
self.missing_value_handler = MissingValueHandler(**self.config)
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Data cleaner initialized")
|
||||
|
||||
def clean_data(
|
||||
@@ -145,36 +149,45 @@ class DataCleaner:
|
||||
Returns:
|
||||
list: Cleaned dataset (list of record dictionaries)
|
||||
"""
|
||||
cleaned = list(dataset)
|
||||
|
||||
# Handle missing values
|
||||
if handle_missing:
|
||||
strategy = options.get("missing_strategy", "remove")
|
||||
cleaned = self.missing_value_handler.handle_missing_values(cleaned, strategy=strategy)
|
||||
|
||||
# Validate data
|
||||
if validate:
|
||||
schema = options.get("schema")
|
||||
validation = self.data_validator.validate_dataset(cleaned, schema)
|
||||
if not validation.valid:
|
||||
self.logger.warning(f"Validation found {len(validation.errors)} errors")
|
||||
|
||||
# Remove duplicates
|
||||
if remove_duplicates:
|
||||
criteria = options.get("duplicate_criteria", {})
|
||||
duplicates = self.detect_duplicates(cleaned, **criteria)
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
message="Semantica: Cleaning data",
|
||||
file=None
|
||||
)
|
||||
try:
|
||||
cleaned = list(dataset)
|
||||
|
||||
# Remove duplicates (keep first occurrence)
|
||||
duplicate_indices = set()
|
||||
for group in duplicates:
|
||||
for record in group.records[1:]: # Skip first (canonical)
|
||||
if record in cleaned:
|
||||
idx = cleaned.index(record)
|
||||
duplicate_indices.add(idx)
|
||||
# Handle missing values
|
||||
if handle_missing:
|
||||
strategy = options.get("missing_strategy", "remove")
|
||||
cleaned = self.missing_value_handler.handle_missing_values(cleaned, strategy=strategy)
|
||||
|
||||
cleaned = [r for i, r in enumerate(cleaned) if i not in duplicate_indices]
|
||||
|
||||
return cleaned
|
||||
# Validate data
|
||||
if validate:
|
||||
schema = options.get("schema")
|
||||
validation = self.data_validator.validate_dataset(cleaned, schema)
|
||||
if not validation.valid:
|
||||
self.logger.warning(f"Validation found {len(validation.errors)} errors")
|
||||
|
||||
# Remove duplicates
|
||||
if remove_duplicates:
|
||||
criteria = options.get("duplicate_criteria", {})
|
||||
duplicates = self.detect_duplicates(cleaned, **criteria)
|
||||
|
||||
# Remove duplicates (keep first occurrence)
|
||||
duplicate_indices = set()
|
||||
for group in duplicates:
|
||||
for record in group.records[1:]: # Skip first (canonical)
|
||||
if record in cleaned:
|
||||
idx = cleaned.index(record)
|
||||
duplicate_indices.add(idx)
|
||||
|
||||
cleaned = [r for i, r in enumerate(cleaned) if i not in duplicate_indices]
|
||||
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed")
|
||||
return cleaned
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed")
|
||||
raise
|
||||
|
||||
def detect_duplicates(
|
||||
self,
|
||||
|
||||
@@ -36,6 +36,7 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
# Optional imports for date parsing
|
||||
try:
|
||||
@@ -89,6 +90,9 @@ class DateNormalizer:
|
||||
self.relative_date_processor = RelativeDateProcessor(**self.config)
|
||||
self.temporal_parser = TemporalExpressionParser(**self.config)
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Date normalizer initialized")
|
||||
|
||||
def normalize_date(
|
||||
|
||||
@@ -33,6 +33,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError, ValidationError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class EncodingHandler:
|
||||
@@ -74,6 +75,9 @@ class EncodingHandler:
|
||||
self.default_encoding = config.get("default_encoding", "utf-8")
|
||||
self.fallback_encodings = config.get("fallback_encodings", ["latin-1", "cp1252", "iso-8859-1"])
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug(f"Encoding handler initialized (default={self.default_encoding})")
|
||||
|
||||
def detect(
|
||||
|
||||
@@ -34,6 +34,7 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class EntityNormalizer:
|
||||
@@ -77,6 +78,9 @@ class EntityNormalizer:
|
||||
self.disambiguator = EntityDisambiguator(**self.config)
|
||||
self.variant_handler = NameVariantHandler(**self.config)
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Entity normalizer initialized")
|
||||
|
||||
def normalize_entity(
|
||||
@@ -101,25 +105,35 @@ class EntityNormalizer:
|
||||
Returns:
|
||||
str: Normalized entity name in standard form
|
||||
"""
|
||||
if not entity_name:
|
||||
return ""
|
||||
|
||||
normalized = entity_name.strip()
|
||||
|
||||
# Clean and standardize
|
||||
normalized = re.sub(r'\s+', ' ', normalized)
|
||||
normalized = normalized.title() if entity_type == "Person" else normalized
|
||||
|
||||
# Resolve aliases
|
||||
if resolve_aliases:
|
||||
resolved = self.alias_resolver.resolve_aliases(normalized, entity_type=entity_type)
|
||||
if resolved:
|
||||
normalized = resolved
|
||||
|
||||
# Handle name variants
|
||||
normalized = self.variant_handler.normalize_name_format(normalized, format_type="standard")
|
||||
|
||||
return normalized
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
message="Semantica: Normalizing entity",
|
||||
file=None
|
||||
)
|
||||
try:
|
||||
if not entity_name:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed")
|
||||
return ""
|
||||
|
||||
normalized = entity_name.strip()
|
||||
|
||||
# Clean and standardize
|
||||
normalized = re.sub(r'\s+', ' ', normalized)
|
||||
normalized = normalized.title() if entity_type == "Person" else normalized
|
||||
|
||||
# Resolve aliases
|
||||
if resolve_aliases:
|
||||
resolved = self.alias_resolver.resolve_aliases(normalized, entity_type=entity_type)
|
||||
if resolved:
|
||||
normalized = resolved
|
||||
|
||||
# Handle name variants
|
||||
normalized = self.variant_handler.normalize_name_format(normalized, format_type="standard")
|
||||
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed")
|
||||
return normalized
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed")
|
||||
raise
|
||||
|
||||
def resolve_aliases(
|
||||
self,
|
||||
@@ -139,7 +153,17 @@ class EntityNormalizer:
|
||||
Returns:
|
||||
Optional[str]: Resolved canonical form if found, None otherwise
|
||||
"""
|
||||
return self.alias_resolver.resolve_aliases(entity_name, **context)
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
message="Semantica: Resolving aliases",
|
||||
file=None
|
||||
)
|
||||
try:
|
||||
result = self.alias_resolver.resolve_aliases(entity_name, **context)
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed")
|
||||
return result
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed")
|
||||
raise
|
||||
|
||||
def disambiguate_entity(
|
||||
self,
|
||||
@@ -163,7 +187,17 @@ class EntityNormalizer:
|
||||
- confidence: Confidence score (0.0 to 1.0)
|
||||
- candidates: List of candidate entity names
|
||||
"""
|
||||
return self.disambiguator.disambiguate(entity_name, **context)
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
message="Semantica: Disambiguating entity",
|
||||
file=None
|
||||
)
|
||||
try:
|
||||
result = self.disambiguator.disambiguate(entity_name, **context)
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed")
|
||||
return result
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed")
|
||||
raise
|
||||
|
||||
def link_entities(
|
||||
self,
|
||||
@@ -183,13 +217,22 @@ class EntityNormalizer:
|
||||
Returns:
|
||||
dict: Dictionary mapping original entity names to canonical forms
|
||||
"""
|
||||
linked = {}
|
||||
|
||||
for entity in entities:
|
||||
canonical = self.normalize_entity(entity, **options)
|
||||
linked[entity] = canonical
|
||||
|
||||
return linked
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
message="Semantica: Linking entities",
|
||||
file=None
|
||||
)
|
||||
try:
|
||||
linked = {}
|
||||
|
||||
for entity in entities:
|
||||
canonical = self.normalize_entity(entity, **options)
|
||||
linked[entity] = canonical
|
||||
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed")
|
||||
return linked
|
||||
except Exception as e:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="failed")
|
||||
raise
|
||||
|
||||
|
||||
class AliasResolver:
|
||||
|
||||
@@ -38,6 +38,7 @@ except ImportError:
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class LanguageDetector:
|
||||
@@ -80,6 +81,9 @@ class LanguageDetector:
|
||||
if not LANGDETECT_AVAILABLE:
|
||||
self.logger.warning("langdetect library not available, language detection will be limited")
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug(f"Language detector initialized (default={self.default_language})")
|
||||
|
||||
def detect(self, text: str, **options) -> str:
|
||||
|
||||
@@ -35,6 +35,7 @@ from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class NumberNormalizer:
|
||||
@@ -79,6 +80,9 @@ class NumberNormalizer:
|
||||
self.currency_normalizer = CurrencyNormalizer(**self.config)
|
||||
self.scientific_handler = ScientificNotationHandler(**self.config)
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Number normalizer initialized")
|
||||
|
||||
def normalize_number(
|
||||
|
||||
@@ -39,6 +39,7 @@ except ImportError:
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
|
||||
|
||||
class TextCleaner:
|
||||
@@ -79,6 +80,9 @@ class TextCleaner:
|
||||
if not BEAUTIFULSOUP_AVAILABLE:
|
||||
self.logger.warning("BeautifulSoup not available, HTML removal will use regex fallback")
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Text cleaner initialized")
|
||||
|
||||
def clean(
|
||||
|
||||
@@ -35,6 +35,7 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..utils.exceptions import ValidationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.progress_tracker import get_progress_tracker
|
||||
from .text_cleaner import TextCleaner
|
||||
|
||||
|
||||
@@ -82,6 +83,9 @@ class TextNormalizer:
|
||||
self.whitespace_normalizer = WhitespaceNormalizer(**self.config)
|
||||
self.special_char_processor = SpecialCharacterProcessor(**self.config)
|
||||
|
||||
# Initialize progress tracker
|
||||
self.progress_tracker = get_progress_tracker()
|
||||
|
||||
self.logger.debug("Text normalizer initialized")
|
||||
|
||||
def normalize_text(
|
||||
@@ -120,20 +124,26 @@ class TextNormalizer:
|
||||
Returns:
|
||||
str: Normalized text
|
||||
"""
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
normalized = text
|
||||
|
||||
# Unicode normalization
|
||||
normalized = self.unicode_normalizer.normalize_unicode(normalized, form=unicode_form)
|
||||
|
||||
# Whitespace normalization
|
||||
normalized = self.whitespace_normalizer.normalize_whitespace(
|
||||
normalized, line_break_type=line_break_type, **options
|
||||
tracking_id = self.progress_tracker.start_tracking(
|
||||
message="Semantica: Normalizing text",
|
||||
file=None
|
||||
)
|
||||
|
||||
# Special character processing
|
||||
try:
|
||||
if not text:
|
||||
self.progress_tracker.stop_tracking(tracking_id, status="completed")
|
||||
return ""
|
||||
|
||||
normalized = text
|
||||
|
||||
# Unicode normalization
|
||||
normalized = self.unicode_normalizer.normalize_unicode(normalized, form=unicode_form)
|
||||
|
||||
# Whitespace normalization
|
||||
normalized = self.whitespace_normalizer.normalize_whitespace(
|
||||
normalized, line_break_type=line_break_type, **options
|
||||
)
|
||||
|
||||
# Special character processing
|
||||
normalized = self.special_char_processor.process_special_chars(
|
||||
normalized, normalize_diacritics=normalize_diacritics, **options
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user