mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Merge remote-tracking branch 'origin/main' into codex/context-graph-markdown-round-trip
This commit is contained in:
@@ -80,6 +80,7 @@ import numpy as np
|
||||
|
||||
from ..utils.exceptions import ConfigurationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
|
||||
from .config import embeddings_config
|
||||
from .embedding_generator import EmbeddingGenerator
|
||||
from .pooling_strategies import PoolingStrategyFactory
|
||||
@@ -119,12 +120,12 @@ def generate_embeddings(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("generation", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(data, data_type=data_type, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, data_type=data_type, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
if method == "default":
|
||||
@@ -167,12 +168,12 @@ def embed_text(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("text", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(text, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -227,12 +228,12 @@ def calculate_similarity(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("similarity", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(embedding1, embedding2, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, embedding1, embedding2, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
generator = EmbeddingGenerator(**kwargs)
|
||||
@@ -274,12 +275,12 @@ def pool_embeddings(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("pooling", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(embeddings, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, embeddings, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
strategy = PoolingStrategyFactory.create(method, **kwargs)
|
||||
|
||||
+79
-78
@@ -164,6 +164,7 @@ from typing import Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
|
||||
from .arango_aql_exporter import ArangoAQLExporter
|
||||
from .arrow_exporter import ArrowExporter
|
||||
from .config import export_config
|
||||
@@ -221,12 +222,12 @@ def export_rdf(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("rdf", method)
|
||||
if custom_method and custom_method is not export_rdf:
|
||||
try:
|
||||
return custom_method(data, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -270,12 +271,12 @@ def export_json(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("json", method)
|
||||
if custom_method and custom_method is not export_json:
|
||||
try:
|
||||
return custom_method(data, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -316,12 +317,12 @@ def export_csv(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("csv", method)
|
||||
if custom_method and custom_method is not export_csv:
|
||||
try:
|
||||
return custom_method(data, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -361,12 +362,12 @@ def export_arrow(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("arrow", method)
|
||||
if custom_method and custom_method is not export_arrow:
|
||||
try:
|
||||
return custom_method(data, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -421,12 +422,12 @@ def export_parquet(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("parquet", method)
|
||||
if custom_method and custom_method is not export_parquet:
|
||||
try:
|
||||
return custom_method(data, file_path, compression=compression, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, compression=compression, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -472,12 +473,12 @@ def export_graph(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("graph", method)
|
||||
if custom_method and custom_method is not export_graph:
|
||||
try:
|
||||
return custom_method(graph_data, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, graph_data, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -538,12 +539,12 @@ def export_yaml(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("yaml", method)
|
||||
if custom_method and custom_method is not export_yaml:
|
||||
try:
|
||||
return custom_method(data, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -595,12 +596,12 @@ def export_owl(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("owl", method)
|
||||
if custom_method and custom_method is not export_owl:
|
||||
try:
|
||||
return custom_method(ontology, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, ontology, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -647,12 +648,12 @@ def export_vector(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("vector", method)
|
||||
if custom_method and custom_method is not export_vector:
|
||||
try:
|
||||
return custom_method(vectors, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, vectors, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -695,12 +696,12 @@ def export_lpg(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("lpg", method)
|
||||
if custom_method and custom_method is not export_lpg:
|
||||
try:
|
||||
return custom_method(knowledge_graph, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, knowledge_graph, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -738,12 +739,12 @@ def export_neo4j_csv(
|
||||
"""
|
||||
custom_method = method_registry.get("neo4j_csv", method)
|
||||
if custom_method and custom_method is not export_neo4j_csv:
|
||||
try:
|
||||
return custom_method(knowledge_graph, output_dir, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, knowledge_graph, output_dir, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = export_config.get_method_config("neo4j_csv")
|
||||
@@ -808,12 +809,12 @@ def export_arango(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("arango", method)
|
||||
if custom_method and custom_method is not export_arango:
|
||||
try:
|
||||
return custom_method(knowledge_graph, file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, knowledge_graph, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -859,12 +860,12 @@ def generate_report(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("report", method)
|
||||
if custom_method and custom_method is not generate_report:
|
||||
try:
|
||||
return custom_method(data, file_path, format=format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, file_path, format=format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
|
||||
+79
-78
@@ -180,6 +180,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ConfigurationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
|
||||
from .config import ingest_config
|
||||
from .file_ingestor import FileIngestor, FileObject
|
||||
from .registry import method_registry
|
||||
@@ -248,12 +249,12 @@ def ingest_file(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("file", method)
|
||||
if custom_method and custom_method != ingest_file:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -314,12 +315,12 @@ def ingest_parquet(
|
||||
"""
|
||||
custom_method = method_registry.get("parquet", method)
|
||||
if custom_method and custom_method != ingest_parquet:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
try:
|
||||
@@ -393,12 +394,12 @@ def ingest_arrow(
|
||||
"""
|
||||
custom_method = method_registry.get("arrow", method)
|
||||
if custom_method and custom_method != ingest_arrow:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
try:
|
||||
@@ -477,12 +478,12 @@ def ingest_xml(
|
||||
"""
|
||||
custom_method = method_registry.get("xml", method)
|
||||
if custom_method and custom_method != ingest_xml:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .xml_ingestor import XMLIngestor
|
||||
@@ -541,12 +542,12 @@ def ingest_web(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("web", method)
|
||||
if custom_method and custom_method != ingest_web:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
try:
|
||||
@@ -631,12 +632,12 @@ def ingest_public_api(
|
||||
"""
|
||||
custom_method = method_registry.get("public_api", method)
|
||||
if custom_method and custom_method != ingest_public_api:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .public_api_ingestor import PublicAPIExamples, PublicAPIIngestor
|
||||
@@ -718,12 +719,12 @@ def ingest_feed(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("feed", method)
|
||||
if custom_method and custom_method != ingest_feed:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
try:
|
||||
@@ -787,12 +788,12 @@ def ingest_stream(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("stream", method)
|
||||
if custom_method and custom_method != ingest_stream:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .stream_ingestor import StreamIngestor
|
||||
@@ -864,12 +865,12 @@ def ingest_repository(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("repo", method)
|
||||
if custom_method and custom_method != ingest_repository:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
try:
|
||||
@@ -936,12 +937,12 @@ def ingest_email(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("email", method)
|
||||
if custom_method and custom_method != ingest_email:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
try:
|
||||
@@ -1015,12 +1016,12 @@ def ingest_ontology(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("ontology", method)
|
||||
if custom_method and custom_method != ingest_ontology:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .ontology_ingestor import OntologyIngestor
|
||||
@@ -1081,12 +1082,12 @@ def ingest_database(
|
||||
if method:
|
||||
custom_method = method_registry.get("db", method)
|
||||
if custom_method and custom_method != ingest_database:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .db_ingestor import DBIngestor
|
||||
@@ -1188,12 +1189,12 @@ def ingest_mcp(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("mcp", method)
|
||||
if custom_method and custom_method != ingest_mcp:
|
||||
try:
|
||||
return custom_method(source, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, source, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
from .mcp_ingestor import MCPIngestor
|
||||
|
||||
+19
-18
@@ -142,6 +142,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from ..utils.exceptions import ConfigurationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
|
||||
from .centrality_calculator import CentralityCalculator
|
||||
from .community_detector import CommunityDetector
|
||||
from .config import kg_config
|
||||
@@ -189,12 +190,12 @@ def build_kg(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("build", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(sources, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, sources, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -237,12 +238,12 @@ def analyze_graph(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("analyze", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(graph, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, graph, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
@@ -441,12 +442,12 @@ def analyze_connectivity(
|
||||
# Check for custom method in registry
|
||||
custom_method = method_registry.get("connectivity", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(graph, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, graph, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Get config
|
||||
|
||||
@@ -125,6 +125,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from ..utils.exceptions import ConfigurationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
|
||||
from .config import normalize_config
|
||||
from .data_cleaner import DataCleaner
|
||||
from .date_normalizer import DateNormalizer
|
||||
@@ -168,12 +169,12 @@ def normalize_text(text: str, method: str = "default", **kwargs) -> str:
|
||||
"""
|
||||
custom_method = method_registry.get("text", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(text, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("text")
|
||||
@@ -212,12 +213,12 @@ def clean_text(text: str, method: str = "default", **kwargs) -> str:
|
||||
"""
|
||||
custom_method = method_registry.get("clean", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(text, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("clean")
|
||||
@@ -262,12 +263,12 @@ def normalize_entity(
|
||||
"""
|
||||
custom_method = method_registry.get("entity", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(entity_name, entity_type, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, entity_name, entity_type, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("entity")
|
||||
@@ -309,12 +310,12 @@ def resolve_aliases(
|
||||
"""
|
||||
custom_method = method_registry.get("entity", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(entity_name, entity_type, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, entity_name, entity_type, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("entity")
|
||||
@@ -356,12 +357,12 @@ def disambiguate_entity(
|
||||
"""
|
||||
custom_method = method_registry.get("entity", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(entity_name, **context)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = context.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, entity_name, fallback_on_custom_error=fallback, **context
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("entity")
|
||||
@@ -411,12 +412,12 @@ def normalize_date(
|
||||
"""
|
||||
custom_method = method_registry.get("date", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(date_input, format, timezone, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, date_input, format, timezone, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("date")
|
||||
@@ -452,12 +453,12 @@ def normalize_time(time_input: Any, method: str = "default", **kwargs) -> str:
|
||||
"""
|
||||
custom_method = method_registry.get("date", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(time_input, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, time_input, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("date")
|
||||
@@ -498,12 +499,12 @@ def normalize_number(
|
||||
"""
|
||||
custom_method = method_registry.get("number", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(number_input, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, number_input, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("number")
|
||||
@@ -542,12 +543,12 @@ def normalize_quantity(
|
||||
"""
|
||||
custom_method = method_registry.get("number", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(quantity_input, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, quantity_input, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("number")
|
||||
@@ -598,14 +599,12 @@ def clean_data(
|
||||
"""
|
||||
custom_method = method_registry.get("clean", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(
|
||||
dataset, remove_duplicates, validate, handle_missing, **kwargs
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, dataset, remove_duplicates, validate, handle_missing, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("clean")
|
||||
@@ -653,12 +652,12 @@ def detect_duplicates(
|
||||
"""
|
||||
custom_method = method_registry.get("clean", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(dataset, threshold, key_fields, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, dataset, threshold, key_fields, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("clean")
|
||||
@@ -700,12 +699,12 @@ def detect_language(
|
||||
"""
|
||||
custom_method = method_registry.get("language", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(text, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, text, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("language")
|
||||
@@ -761,12 +760,12 @@ def handle_encoding(
|
||||
"""
|
||||
custom_method = method_registry.get("encoding", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(data, operation, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, operation, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = normalize_config.get_method_config("encoding")
|
||||
|
||||
+73
-72
@@ -125,6 +125,7 @@ from typing import Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from ..utils.exceptions import ConfigurationError, ProcessingError
|
||||
from ..utils.logging import get_logger
|
||||
from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method
|
||||
from .code_parser import CodeParser
|
||||
from .config import parse_config
|
||||
from .csv_parser import CSVParser
|
||||
@@ -178,12 +179,12 @@ def parse_document(
|
||||
"""
|
||||
custom_method = method_registry.get("document", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(file_path, file_type, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, file_path, file_type, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = parse_config.get_method_config("document")
|
||||
@@ -289,12 +290,12 @@ def parse_web_content(
|
||||
"""
|
||||
custom_method = method_registry.get("web", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(content, content_type, base_url, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, content, content_type, base_url, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = parse_config.get_method_config("web")
|
||||
@@ -345,12 +346,12 @@ def parse_structured_data(
|
||||
"""
|
||||
custom_method = method_registry.get("structured", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(data, data_format, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, data, data_format, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = parse_config.get_method_config("structured")
|
||||
@@ -392,12 +393,12 @@ def parse_email(
|
||||
"""
|
||||
custom_method = method_registry.get("email", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(email_content, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, email_content, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = parse_config.get_method_config("email")
|
||||
@@ -442,12 +443,12 @@ def parse_code(
|
||||
"""
|
||||
custom_method = method_registry.get("code", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(file_path, language, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, file_path, language, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = parse_config.get_method_config("code")
|
||||
@@ -495,12 +496,12 @@ def parse_media(
|
||||
"""
|
||||
custom_method = method_registry.get("media", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(file_path, media_type, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, file_path, media_type, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = parse_config.get_method_config("media")
|
||||
@@ -541,12 +542,12 @@ def parse_pdf(
|
||||
"""
|
||||
custom_method = method_registry.get("document", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = parse_config.get_method_config("document")
|
||||
@@ -585,12 +586,12 @@ def parse_docx(
|
||||
"""
|
||||
custom_method = method_registry.get("document", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = parse_config.get_method_config("document")
|
||||
@@ -628,12 +629,12 @@ def parse_json(file_path: Union[str, Path], method: str = "default", **kwargs) -
|
||||
"""
|
||||
custom_method = method_registry.get("structured", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = parse_config.get_method_config("structured")
|
||||
@@ -675,12 +676,12 @@ def parse_csv(
|
||||
"""
|
||||
custom_method = method_registry.get("structured", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(file_path, delimiter, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, file_path, delimiter, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = parse_config.get_method_config("structured")
|
||||
@@ -714,12 +715,12 @@ def parse_xml(file_path: Union[str, Path], method: str = "default", **kwargs) ->
|
||||
"""
|
||||
custom_method = method_registry.get("structured", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = parse_config.get_method_config("structured")
|
||||
@@ -762,12 +763,12 @@ def parse_image(
|
||||
"""
|
||||
custom_method = method_registry.get("media", method)
|
||||
if custom_method:
|
||||
try:
|
||||
return custom_method(file_path, **kwargs)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {e}, falling back to default"
|
||||
)
|
||||
fallback = kwargs.pop("fallback_on_custom_error", False)
|
||||
result = call_custom_method(
|
||||
logger, method, custom_method, file_path, fallback_on_custom_error=fallback, **kwargs
|
||||
)
|
||||
if result is not CUSTOM_METHOD_FELL_BACK:
|
||||
return result
|
||||
|
||||
try:
|
||||
config = parse_config.get_method_config("media")
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""
|
||||
Invocation policy for methods registered through a MethodRegistry.
|
||||
|
||||
Every module that supports custom methods used to wrap the registered callable
|
||||
in a bare `except Exception`, log a warning, and carry on into the built-in
|
||||
implementation. That makes a registered method advisory: it can add behaviour,
|
||||
but it cannot decline.
|
||||
|
||||
For a gate, a validator or a policy check that is the whole point. Raising is
|
||||
how such a method says "do not produce this output". Catching the exception and
|
||||
running the default produces exactly the output the caller registered the method
|
||||
to prevent, and the only trace is a warning (issue #1108).
|
||||
|
||||
Exceptions from a registered method therefore propagate by default. Callers who
|
||||
relied on the old behaviour can pass `fallback_on_custom_error=True`, which
|
||||
restores the warn-and-continue path for that call.
|
||||
"""
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
class _FellBack:
|
||||
"""Sentinel: the custom method failed and the caller should use the default."""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover - debugging aid
|
||||
return "CUSTOM_METHOD_FELL_BACK"
|
||||
|
||||
|
||||
#: Returned by :func:`call_custom_method` when a custom method raised and
|
||||
#: ``fallback_on_custom_error=True`` was passed. Compare with ``is``.
|
||||
CUSTOM_METHOD_FELL_BACK = _FellBack()
|
||||
|
||||
|
||||
def call_custom_method(
|
||||
logger: Any,
|
||||
method: Any,
|
||||
custom_method: Callable[..., Any],
|
||||
/,
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""
|
||||
Invoke a registered custom method.
|
||||
|
||||
Args:
|
||||
logger: Module logger, used only on the opt-in fallback path.
|
||||
method: The registered name, for the warning message.
|
||||
custom_method: The registered callable.
|
||||
*args: Positional arguments for the custom method.
|
||||
**kwargs: Keyword arguments for the custom method. The reserved key
|
||||
``fallback_on_custom_error`` is consumed here and never forwarded.
|
||||
|
||||
Returns:
|
||||
Whatever the custom method returns, or :data:`CUSTOM_METHOD_FELL_BACK`
|
||||
when it raised and the caller opted into falling back.
|
||||
|
||||
Raises:
|
||||
Exception: Whatever the custom method raised, unless the caller passed
|
||||
``fallback_on_custom_error=True``.
|
||||
"""
|
||||
fallback = bool(kwargs.pop("fallback_on_custom_error", False))
|
||||
|
||||
if not fallback:
|
||||
return custom_method(*args, **kwargs)
|
||||
|
||||
try:
|
||||
return custom_method(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
f"Custom method {method} failed: {exc}, falling back to default "
|
||||
"because fallback_on_custom_error was set"
|
||||
)
|
||||
return CUSTOM_METHOD_FELL_BACK
|
||||
@@ -0,0 +1,225 @@
|
||||
"""
|
||||
Regression tests for #1108.
|
||||
|
||||
Every module supporting custom methods wrapped the registered callable in a
|
||||
bare `except Exception`, logged a warning, and continued into the built-in
|
||||
implementation. That makes a registered method advisory: it can add behaviour,
|
||||
but it cannot decline.
|
||||
|
||||
For a gate, a validator or a policy check, declining is the entire purpose.
|
||||
The demonstration below is the one from the issue: a verifier rejects invalid
|
||||
RDF and deletes the file, and the swallowed exception lets the default write it
|
||||
straight back.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from semantica.export import methods as export_methods
|
||||
from semantica.export.registry import method_registry
|
||||
from semantica.utils.custom_methods import (
|
||||
CUSTOM_METHOD_FELL_BACK,
|
||||
call_custom_method,
|
||||
)
|
||||
|
||||
|
||||
class Refused(Exception):
|
||||
"""Raised by a gate that declines to produce output."""
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_registry():
|
||||
method_registry.clear("rdf")
|
||||
yield
|
||||
method_registry.clear("rdf")
|
||||
|
||||
|
||||
KG = {"entities": [{"id": "e1", "text": "Acme", "type": "ORG"}], "relationships": []}
|
||||
|
||||
|
||||
def test_a_registered_gate_can_refuse(tmp_path):
|
||||
"""The exception must reach the caller instead of being logged and dropped."""
|
||||
def gate(data, file_path, **kwargs):
|
||||
raise Refused("this graph does not pass validation")
|
||||
|
||||
method_registry.register("rdf", "gate", gate)
|
||||
|
||||
with pytest.raises(Refused):
|
||||
export_methods.export_rdf(KG, str(tmp_path / "out.ttl"), method="gate")
|
||||
|
||||
|
||||
def test_a_refusal_leaves_no_output_behind(tmp_path):
|
||||
"""The issue's demonstration: the default used to write the file back."""
|
||||
target = tmp_path / "out.ttl"
|
||||
|
||||
def gate(data, file_path, **kwargs):
|
||||
Path(file_path).unlink(missing_ok=True)
|
||||
raise Refused("rejected by the verifier")
|
||||
|
||||
method_registry.register("rdf", "gate", gate)
|
||||
|
||||
with pytest.raises(Refused):
|
||||
export_methods.export_rdf(KG, str(target), method="gate")
|
||||
|
||||
assert not target.exists(), (
|
||||
"the default implementation wrote the file the gate refused to produce"
|
||||
)
|
||||
|
||||
|
||||
def test_a_custom_method_that_succeeds_is_unaffected(tmp_path):
|
||||
target = tmp_path / "out.ttl"
|
||||
|
||||
def writer(data, file_path, **kwargs):
|
||||
Path(file_path).write_text("# written by the custom method\n")
|
||||
return {"written_by": "custom"}
|
||||
|
||||
method_registry.register("rdf", "writer", writer)
|
||||
result = export_methods.export_rdf(KG, str(target), method="writer")
|
||||
|
||||
assert result == {"written_by": "custom"}
|
||||
assert target.read_text().startswith("# written by the custom method")
|
||||
|
||||
|
||||
def test_the_old_behaviour_is_available_as_an_explicit_opt_in(tmp_path):
|
||||
target = tmp_path / "out.ttl"
|
||||
|
||||
def gate(data, file_path, **kwargs):
|
||||
raise Refused("rejected")
|
||||
|
||||
method_registry.register("rdf", "gate", gate)
|
||||
export_methods.export_rdf(
|
||||
KG, str(target), method="gate", fallback_on_custom_error=True
|
||||
)
|
||||
|
||||
assert target.exists(), "opting in should still fall through to the default"
|
||||
|
||||
|
||||
def test_the_reserved_keyword_is_never_forwarded():
|
||||
"""`fallback_on_custom_error` is consumed by the policy, not by the method."""
|
||||
seen = {}
|
||||
|
||||
def recorder(**kwargs):
|
||||
seen.update(kwargs)
|
||||
return "ok"
|
||||
|
||||
result = call_custom_method(
|
||||
_NullLogger(), "recorder", recorder, alpha=1, fallback_on_custom_error=True
|
||||
)
|
||||
|
||||
assert result == "ok"
|
||||
assert seen == {"alpha": 1}
|
||||
|
||||
|
||||
class _NullLogger:
|
||||
def warning(self, *args, **kwargs):
|
||||
self.last = args
|
||||
|
||||
|
||||
def test_the_sentinel_is_returned_only_on_the_opt_in_path():
|
||||
def boom():
|
||||
raise Refused("no")
|
||||
|
||||
logger = _NullLogger()
|
||||
assert call_custom_method(
|
||||
logger, "boom", boom, fallback_on_custom_error=True
|
||||
) is CUSTOM_METHOD_FELL_BACK
|
||||
|
||||
with pytest.raises(Refused):
|
||||
call_custom_method(logger, "boom", boom)
|
||||
|
||||
|
||||
def test_a_falsy_return_value_is_not_mistaken_for_a_failure():
|
||||
"""`is not CUSTOM_METHOD_FELL_BACK` matters: None and 0 are real results."""
|
||||
for value in (None, 0, "", False, []):
|
||||
assert call_custom_method(_NullLogger(), "m", lambda: value) is value
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"module_name",
|
||||
["export", "ingest", "parse", "normalize", "embeddings", "kg"],
|
||||
)
|
||||
def test_no_module_still_swallows_custom_method_failures(module_name):
|
||||
"""The swallow was repeated across six modules, not just the one filed."""
|
||||
import semantica
|
||||
|
||||
# Read the file rather than import it: some of these modules pull in
|
||||
# optional third-party dependencies that need not be installed to check
|
||||
# that the swallow is gone.
|
||||
source = (
|
||||
Path(semantica.__file__).parent / module_name / "methods.py"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "falling back to default" not in source, (
|
||||
f"semantica/{module_name}/methods.py still swallows custom method failures"
|
||||
)
|
||||
|
||||
|
||||
# ── Review findings on the first revision of this fix ────────────────────────
|
||||
|
||||
def test_the_reserved_flag_never_reaches_the_default_implementation(monkeypatch, tmp_path):
|
||||
"""
|
||||
`**kwargs` unpacking builds a fresh dict inside the helper, so popping there
|
||||
left the caller's own kwargs untouched and the flag was forwarded on to the
|
||||
default path. The helper documents the flag as never forwarded, so that
|
||||
promise was false for exactly the case the flag exists for.
|
||||
"""
|
||||
seen = {}
|
||||
|
||||
class Spy:
|
||||
def __init__(self, **config):
|
||||
seen.update(config)
|
||||
|
||||
def export(self, *args, **kwargs):
|
||||
(tmp_path / "written").write_text("default ran")
|
||||
|
||||
monkeypatch.setattr(export_methods, "RDFExporter", Spy)
|
||||
|
||||
def gate(data, file_path, **kwargs):
|
||||
raise Refused("rejected")
|
||||
|
||||
method_registry.register("rdf", "gate", gate)
|
||||
export_methods.export_rdf(
|
||||
KG, str(tmp_path / "out.ttl"), method="gate", fallback_on_custom_error=True
|
||||
)
|
||||
|
||||
assert (tmp_path / "written").exists(), "the default path did not run"
|
||||
assert "fallback_on_custom_error" not in seen, (
|
||||
f"the reserved flag was forwarded to the default implementation: {seen}"
|
||||
)
|
||||
|
||||
|
||||
def test_the_reserved_flag_never_reaches_a_successful_custom_method(tmp_path):
|
||||
seen = {}
|
||||
|
||||
def writer(data, file_path, **kwargs):
|
||||
seen.update(kwargs)
|
||||
return "ok"
|
||||
|
||||
method_registry.register("rdf", "writer", writer)
|
||||
result = export_methods.export_rdf(
|
||||
KG, str(tmp_path / "out.ttl"), method="writer", fallback_on_custom_error=True
|
||||
)
|
||||
|
||||
assert result == "ok"
|
||||
assert "fallback_on_custom_error" not in seen, seen
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"module_name", ["export", "ingest", "parse", "normalize", "embeddings", "kg"],
|
||||
)
|
||||
def test_every_site_consumes_the_flag_before_forwarding(module_name):
|
||||
"""A site that forgets the pop reintroduces the leak silently."""
|
||||
import semantica
|
||||
|
||||
source = (
|
||||
Path(semantica.__file__).parent / module_name / "methods.py"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
calls = source.count("result = call_custom_method(")
|
||||
pops = source.count('.pop("fallback_on_custom_error", False)')
|
||||
assert calls == pops, (
|
||||
f"semantica/{module_name}/methods.py has {calls} call site(s) but "
|
||||
f"{pops} consume the reserved flag"
|
||||
)
|
||||
Reference in New Issue
Block a user