diff --git a/semantica/embeddings/methods.py b/semantica/embeddings/methods.py index 30c47279..31b27d18 100644 --- a/semantica/embeddings/methods.py +++ b/semantica/embeddings/methods.py @@ -80,6 +80,7 @@ import numpy as np from ..utils.exceptions import ConfigurationError, ProcessingError from ..utils.logging import get_logger +from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method from .config import embeddings_config from .embedding_generator import EmbeddingGenerator from .pooling_strategies import PoolingStrategyFactory @@ -119,12 +120,11 @@ def generate_embeddings( # Check for custom method in registry custom_method = method_registry.get("generation", method) if custom_method: - try: - return custom_method(data, data_type=data_type, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, data, data_type=data_type, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: if method == "default": @@ -167,12 +167,9 @@ def embed_text( # Check for custom method in registry custom_method = method_registry.get("text", method) if custom_method: - try: - return custom_method(text, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, text, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -227,12 +224,9 @@ def calculate_similarity( # Check for custom method in registry custom_method = method_registry.get("similarity", method) if custom_method: - try: - return custom_method(embedding1, embedding2, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, embedding1, embedding2, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: generator = EmbeddingGenerator(**kwargs) @@ -274,12 +268,9 @@ def pool_embeddings( # Check for custom method in registry custom_method = method_registry.get("pooling", method) if custom_method: - try: - return custom_method(embeddings, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, embeddings, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: strategy = PoolingStrategyFactory.create(method, **kwargs) diff --git a/semantica/export/methods.py b/semantica/export/methods.py index d2af2581..4cb54060 100644 --- a/semantica/export/methods.py +++ b/semantica/export/methods.py @@ -164,6 +164,7 @@ from typing import Any, Callable, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError from ..utils.logging import get_logger +from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method from .arango_aql_exporter import ArangoAQLExporter from .arrow_exporter import ArrowExporter from .config import export_config @@ -221,12 +222,11 @@ def export_rdf( # Check for custom method in registry custom_method = method_registry.get("rdf", method) if custom_method and custom_method is not export_rdf: - try: - return custom_method(data, file_path, format=format, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, data, file_path, format=format, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -270,12 +270,11 @@ def export_json( # Check for custom method in registry custom_method = method_registry.get("json", method) if custom_method and custom_method is not export_json: - try: - return custom_method(data, file_path, format=format, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, data, file_path, format=format, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -316,12 +315,9 @@ def export_csv( # Check for custom method in registry custom_method = method_registry.get("csv", method) if custom_method and custom_method is not export_csv: - try: - return custom_method(data, file_path, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, data, file_path, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -361,12 +357,9 @@ def export_arrow( # Check for custom method in registry custom_method = method_registry.get("arrow", method) if custom_method and custom_method is not export_arrow: - try: - return custom_method(data, file_path, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, data, file_path, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -421,12 +414,11 @@ def export_parquet( # Check for custom method in registry custom_method = method_registry.get("parquet", method) if custom_method and custom_method is not export_parquet: - try: - return custom_method(data, file_path, compression=compression, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, data, file_path, compression=compression, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -472,12 +464,11 @@ def export_graph( # Check for custom method in registry custom_method = method_registry.get("graph", method) if custom_method and custom_method is not export_graph: - try: - return custom_method(graph_data, file_path, format=format, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, graph_data, file_path, format=format, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -538,12 +529,9 @@ def export_yaml( # Check for custom method in registry custom_method = method_registry.get("yaml", method) if custom_method and custom_method is not export_yaml: - try: - return custom_method(data, file_path, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, data, file_path, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -595,12 +583,11 @@ def export_owl( # Check for custom method in registry custom_method = method_registry.get("owl", method) if custom_method and custom_method is not export_owl: - try: - return custom_method(ontology, file_path, format=format, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, ontology, file_path, format=format, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -647,12 +634,11 @@ def export_vector( # Check for custom method in registry custom_method = method_registry.get("vector", method) if custom_method and custom_method is not export_vector: - try: - return custom_method(vectors, file_path, format=format, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, vectors, file_path, format=format, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -695,12 +681,11 @@ def export_lpg( # Check for custom method in registry custom_method = method_registry.get("lpg", method) if custom_method and custom_method is not export_lpg: - try: - return custom_method(knowledge_graph, file_path, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, knowledge_graph, file_path, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -738,12 +723,11 @@ def export_neo4j_csv( """ custom_method = method_registry.get("neo4j_csv", method) if custom_method and custom_method is not export_neo4j_csv: - try: - return custom_method(knowledge_graph, output_dir, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, knowledge_graph, output_dir, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = export_config.get_method_config("neo4j_csv") @@ -808,12 +792,11 @@ def export_arango( # Check for custom method in registry custom_method = method_registry.get("arango", method) if custom_method and custom_method is not export_arango: - try: - return custom_method(knowledge_graph, file_path, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, knowledge_graph, file_path, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -859,12 +842,11 @@ def generate_report( # Check for custom method in registry custom_method = method_registry.get("report", method) if custom_method and custom_method is not generate_report: - try: - return custom_method(data, file_path, format=format, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, data, file_path, format=format, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config diff --git a/semantica/ingest/methods.py b/semantica/ingest/methods.py index 586eb827..84624a45 100644 --- a/semantica/ingest/methods.py +++ b/semantica/ingest/methods.py @@ -180,6 +180,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union from ..utils.exceptions import ConfigurationError, ProcessingError from ..utils.logging import get_logger +from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method from .config import ingest_config from .file_ingestor import FileIngestor, FileObject from .registry import method_registry @@ -248,12 +249,9 @@ def ingest_file( # Check for custom method in registry custom_method = method_registry.get("file", method) if custom_method and custom_method != ingest_file: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -314,12 +312,9 @@ def ingest_parquet( """ custom_method = method_registry.get("parquet", method) if custom_method and custom_method != ingest_parquet: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: try: @@ -393,12 +388,9 @@ def ingest_arrow( """ custom_method = method_registry.get("arrow", method) if custom_method and custom_method != ingest_arrow: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: try: @@ -477,12 +469,9 @@ def ingest_xml( """ custom_method = method_registry.get("xml", method) if custom_method and custom_method != ingest_xml: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: from .xml_ingestor import XMLIngestor @@ -541,12 +530,9 @@ def ingest_web( # Check for custom method in registry custom_method = method_registry.get("web", method) if custom_method and custom_method != ingest_web: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: try: @@ -631,12 +617,9 @@ def ingest_public_api( """ custom_method = method_registry.get("public_api", method) if custom_method and custom_method != ingest_public_api: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: from .public_api_ingestor import PublicAPIExamples, PublicAPIIngestor @@ -718,12 +701,9 @@ def ingest_feed( # Check for custom method in registry custom_method = method_registry.get("feed", method) if custom_method and custom_method != ingest_feed: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: try: @@ -787,12 +767,9 @@ def ingest_stream( # Check for custom method in registry custom_method = method_registry.get("stream", method) if custom_method and custom_method != ingest_stream: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: from .stream_ingestor import StreamIngestor @@ -864,12 +841,9 @@ def ingest_repository( # Check for custom method in registry custom_method = method_registry.get("repo", method) if custom_method and custom_method != ingest_repository: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: try: @@ -936,12 +910,9 @@ def ingest_email( # Check for custom method in registry custom_method = method_registry.get("email", method) if custom_method and custom_method != ingest_email: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: try: @@ -1015,12 +986,9 @@ def ingest_ontology( # Check for custom method in registry custom_method = method_registry.get("ontology", method) if custom_method and custom_method != ingest_ontology: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: from .ontology_ingestor import OntologyIngestor @@ -1081,12 +1049,9 @@ def ingest_database( if method: custom_method = method_registry.get("db", method) if custom_method and custom_method != ingest_database: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: from .db_ingestor import DBIngestor @@ -1188,12 +1153,9 @@ def ingest_mcp( # Check for custom method in registry custom_method = method_registry.get("mcp", method) if custom_method and custom_method != ingest_mcp: - try: - return custom_method(source, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, source, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: from .mcp_ingestor import MCPIngestor diff --git a/semantica/kg/methods.py b/semantica/kg/methods.py index 71a2ef1a..c96c96e1 100644 --- a/semantica/kg/methods.py +++ b/semantica/kg/methods.py @@ -142,6 +142,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union from ..utils.exceptions import ConfigurationError, ProcessingError from ..utils.logging import get_logger +from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method from .centrality_calculator import CentralityCalculator from .community_detector import CommunityDetector from .config import kg_config @@ -189,12 +190,9 @@ def build_kg( # Check for custom method in registry custom_method = method_registry.get("build", method) if custom_method: - try: - return custom_method(sources, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, sources, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -237,12 +235,9 @@ def analyze_graph( # Check for custom method in registry custom_method = method_registry.get("analyze", method) if custom_method: - try: - return custom_method(graph, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, graph, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config @@ -441,12 +436,9 @@ def analyze_connectivity( # Check for custom method in registry custom_method = method_registry.get("connectivity", method) if custom_method: - try: - return custom_method(graph, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, graph, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: # Get config diff --git a/semantica/normalize/methods.py b/semantica/normalize/methods.py index 5b65ced5..74f3c5fa 100644 --- a/semantica/normalize/methods.py +++ b/semantica/normalize/methods.py @@ -125,6 +125,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union from ..utils.exceptions import ConfigurationError, ProcessingError from ..utils.logging import get_logger +from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method from .config import normalize_config from .data_cleaner import DataCleaner from .date_normalizer import DateNormalizer @@ -168,12 +169,9 @@ def normalize_text(text: str, method: str = "default", **kwargs) -> str: """ custom_method = method_registry.get("text", method) if custom_method: - try: - return custom_method(text, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, text, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("text") @@ -212,12 +210,9 @@ def clean_text(text: str, method: str = "default", **kwargs) -> str: """ custom_method = method_registry.get("clean", method) if custom_method: - try: - return custom_method(text, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, text, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("clean") @@ -262,12 +257,11 @@ def normalize_entity( """ custom_method = method_registry.get("entity", method) if custom_method: - try: - return custom_method(entity_name, entity_type, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, entity_name, entity_type, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("entity") @@ -309,12 +303,11 @@ def resolve_aliases( """ custom_method = method_registry.get("entity", method) if custom_method: - try: - return custom_method(entity_name, entity_type, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, entity_name, entity_type, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("entity") @@ -356,12 +349,9 @@ def disambiguate_entity( """ custom_method = method_registry.get("entity", method) if custom_method: - try: - return custom_method(entity_name, **context) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, entity_name, **context) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("entity") @@ -411,12 +401,11 @@ def normalize_date( """ custom_method = method_registry.get("date", method) if custom_method: - try: - return custom_method(date_input, format, timezone, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, date_input, format, timezone, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("date") @@ -452,12 +441,9 @@ def normalize_time(time_input: Any, method: str = "default", **kwargs) -> str: """ custom_method = method_registry.get("date", method) if custom_method: - try: - return custom_method(time_input, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, time_input, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("date") @@ -498,12 +484,9 @@ def normalize_number( """ custom_method = method_registry.get("number", method) if custom_method: - try: - return custom_method(number_input, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, number_input, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("number") @@ -542,12 +525,9 @@ def normalize_quantity( """ custom_method = method_registry.get("number", method) if custom_method: - try: - return custom_method(quantity_input, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, quantity_input, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("number") @@ -598,14 +578,11 @@ def clean_data( """ custom_method = method_registry.get("clean", method) if custom_method: - try: - return custom_method( - dataset, remove_duplicates, validate, handle_missing, **kwargs - ) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, dataset, remove_duplicates, validate, handle_missing, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("clean") @@ -653,12 +630,11 @@ def detect_duplicates( """ custom_method = method_registry.get("clean", method) if custom_method: - try: - return custom_method(dataset, threshold, key_fields, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, dataset, threshold, key_fields, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("clean") @@ -700,12 +676,9 @@ def detect_language( """ custom_method = method_registry.get("language", method) if custom_method: - try: - return custom_method(text, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, text, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("language") @@ -761,12 +734,9 @@ def handle_encoding( """ custom_method = method_registry.get("encoding", method) if custom_method: - try: - return custom_method(data, operation, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, data, operation, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = normalize_config.get_method_config("encoding") diff --git a/semantica/parse/methods.py b/semantica/parse/methods.py index e83cf6ff..ba175041 100644 --- a/semantica/parse/methods.py +++ b/semantica/parse/methods.py @@ -125,6 +125,7 @@ from typing import Any, Callable, Dict, List, Optional, Union from ..utils.exceptions import ConfigurationError, ProcessingError from ..utils.logging import get_logger +from ..utils.custom_methods import CUSTOM_METHOD_FELL_BACK, call_custom_method from .code_parser import CodeParser from .config import parse_config from .csv_parser import CSVParser @@ -178,12 +179,9 @@ def parse_document( """ custom_method = method_registry.get("document", method) if custom_method: - try: - return custom_method(file_path, file_type, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, file_path, file_type, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = parse_config.get_method_config("document") @@ -289,12 +287,11 @@ def parse_web_content( """ custom_method = method_registry.get("web", method) if custom_method: - try: - return custom_method(content, content_type, base_url, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method( + logger, method, custom_method, content, content_type, base_url, **kwargs + ) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = parse_config.get_method_config("web") @@ -345,12 +342,9 @@ def parse_structured_data( """ custom_method = method_registry.get("structured", method) if custom_method: - try: - return custom_method(data, data_format, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, data, data_format, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = parse_config.get_method_config("structured") @@ -392,12 +386,9 @@ def parse_email( """ custom_method = method_registry.get("email", method) if custom_method: - try: - return custom_method(email_content, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, email_content, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = parse_config.get_method_config("email") @@ -442,12 +433,9 @@ def parse_code( """ custom_method = method_registry.get("code", method) if custom_method: - try: - return custom_method(file_path, language, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, file_path, language, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = parse_config.get_method_config("code") @@ -495,12 +483,9 @@ def parse_media( """ custom_method = method_registry.get("media", method) if custom_method: - try: - return custom_method(file_path, media_type, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, file_path, media_type, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = parse_config.get_method_config("media") @@ -541,12 +526,9 @@ def parse_pdf( """ custom_method = method_registry.get("document", method) if custom_method: - try: - return custom_method(file_path, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, file_path, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = parse_config.get_method_config("document") @@ -585,12 +567,9 @@ def parse_docx( """ custom_method = method_registry.get("document", method) if custom_method: - try: - return custom_method(file_path, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, file_path, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = parse_config.get_method_config("document") @@ -628,12 +607,9 @@ def parse_json(file_path: Union[str, Path], method: str = "default", **kwargs) - """ custom_method = method_registry.get("structured", method) if custom_method: - try: - return custom_method(file_path, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, file_path, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = parse_config.get_method_config("structured") @@ -675,12 +651,9 @@ def parse_csv( """ custom_method = method_registry.get("structured", method) if custom_method: - try: - return custom_method(file_path, delimiter, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, file_path, delimiter, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = parse_config.get_method_config("structured") @@ -714,12 +687,9 @@ def parse_xml(file_path: Union[str, Path], method: str = "default", **kwargs) -> """ custom_method = method_registry.get("structured", method) if custom_method: - try: - return custom_method(file_path, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, file_path, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = parse_config.get_method_config("structured") @@ -762,12 +732,9 @@ def parse_image( """ custom_method = method_registry.get("media", method) if custom_method: - try: - return custom_method(file_path, **kwargs) - except Exception as e: - logger.warning( - f"Custom method {method} failed: {e}, falling back to default" - ) + result = call_custom_method(logger, method, custom_method, file_path, **kwargs) + if result is not CUSTOM_METHOD_FELL_BACK: + return result try: config = parse_config.get_method_config("media") diff --git a/semantica/utils/custom_methods.py b/semantica/utils/custom_methods.py new file mode 100644 index 00000000..ed47acb4 --- /dev/null +++ b/semantica/utils/custom_methods.py @@ -0,0 +1,75 @@ +""" +Invocation policy for methods registered through a MethodRegistry. + +Every module that supports custom methods used to wrap the registered callable +in a bare `except Exception`, log a warning, and carry on into the built-in +implementation. That makes a registered method advisory: it can add behaviour, +but it cannot decline. + +For a gate, a validator or a policy check that is the whole point. Raising is +how such a method says "do not produce this output". Catching the exception and +running the default produces exactly the output the caller registered the method +to prevent, and the only trace is a warning (issue #1108). + +Exceptions from a registered method therefore propagate by default. Callers who +relied on the old behaviour can pass `fallback_on_custom_error=True`, which +restores the warn-and-continue path for that call. +""" + +from typing import Any, Callable + + +class _FellBack: + """Sentinel: the custom method failed and the caller should use the default.""" + + __slots__ = () + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return "CUSTOM_METHOD_FELL_BACK" + + +#: Returned by :func:`call_custom_method` when a custom method raised and +#: ``fallback_on_custom_error=True`` was passed. Compare with ``is``. +CUSTOM_METHOD_FELL_BACK = _FellBack() + + +def call_custom_method( + logger: Any, + method: Any, + custom_method: Callable[..., Any], + /, + *args: Any, + **kwargs: Any, +) -> Any: + """ + Invoke a registered custom method. + + Args: + logger: Module logger, used only on the opt-in fallback path. + method: The registered name, for the warning message. + custom_method: The registered callable. + *args: Positional arguments for the custom method. + **kwargs: Keyword arguments for the custom method. The reserved key + ``fallback_on_custom_error`` is consumed here and never forwarded. + + Returns: + Whatever the custom method returns, or :data:`CUSTOM_METHOD_FELL_BACK` + when it raised and the caller opted into falling back. + + Raises: + Exception: Whatever the custom method raised, unless the caller passed + ``fallback_on_custom_error=True``. + """ + fallback = bool(kwargs.pop("fallback_on_custom_error", False)) + + if not fallback: + return custom_method(*args, **kwargs) + + try: + return custom_method(*args, **kwargs) + except Exception as exc: + logger.warning( + f"Custom method {method} failed: {exc}, falling back to default " + "because fallback_on_custom_error was set" + ) + return CUSTOM_METHOD_FELL_BACK diff --git a/tests/utils/test_custom_method_can_refuse.py b/tests/utils/test_custom_method_can_refuse.py new file mode 100644 index 00000000..c736e815 --- /dev/null +++ b/tests/utils/test_custom_method_can_refuse.py @@ -0,0 +1,156 @@ +""" +Regression tests for #1108. + +Every module supporting custom methods wrapped the registered callable in a +bare `except Exception`, logged a warning, and continued into the built-in +implementation. That makes a registered method advisory: it can add behaviour, +but it cannot decline. + +For a gate, a validator or a policy check, declining is the entire purpose. +The demonstration below is the one from the issue: a verifier rejects invalid +RDF and deletes the file, and the swallowed exception lets the default write it +straight back. +""" + +import json +from pathlib import Path + +import pytest + +from semantica.export import methods as export_methods +from semantica.export.registry import method_registry +from semantica.utils.custom_methods import ( + CUSTOM_METHOD_FELL_BACK, + call_custom_method, +) + + +class Refused(Exception): + """Raised by a gate that declines to produce output.""" + + +@pytest.fixture(autouse=True) +def _clean_registry(): + method_registry.clear("rdf") + yield + method_registry.clear("rdf") + + +KG = {"entities": [{"id": "e1", "text": "Acme", "type": "ORG"}], "relationships": []} + + +def test_a_registered_gate_can_refuse(tmp_path): + """The exception must reach the caller instead of being logged and dropped.""" + def gate(data, file_path, **kwargs): + raise Refused("this graph does not pass validation") + + method_registry.register("rdf", "gate", gate) + + with pytest.raises(Refused): + export_methods.export_rdf(KG, str(tmp_path / "out.ttl"), method="gate") + + +def test_a_refusal_leaves_no_output_behind(tmp_path): + """The issue's demonstration: the default used to write the file back.""" + target = tmp_path / "out.ttl" + + def gate(data, file_path, **kwargs): + Path(file_path).unlink(missing_ok=True) + raise Refused("rejected by the verifier") + + method_registry.register("rdf", "gate", gate) + + with pytest.raises(Refused): + export_methods.export_rdf(KG, str(target), method="gate") + + assert not target.exists(), ( + "the default implementation wrote the file the gate refused to produce" + ) + + +def test_a_custom_method_that_succeeds_is_unaffected(tmp_path): + target = tmp_path / "out.ttl" + + def writer(data, file_path, **kwargs): + Path(file_path).write_text("# written by the custom method\n") + return {"written_by": "custom"} + + method_registry.register("rdf", "writer", writer) + result = export_methods.export_rdf(KG, str(target), method="writer") + + assert result == {"written_by": "custom"} + assert target.read_text().startswith("# written by the custom method") + + +def test_the_old_behaviour_is_available_as_an_explicit_opt_in(tmp_path): + target = tmp_path / "out.ttl" + + def gate(data, file_path, **kwargs): + raise Refused("rejected") + + method_registry.register("rdf", "gate", gate) + export_methods.export_rdf( + KG, str(target), method="gate", fallback_on_custom_error=True + ) + + assert target.exists(), "opting in should still fall through to the default" + + +def test_the_reserved_keyword_is_never_forwarded(): + """`fallback_on_custom_error` is consumed by the policy, not by the method.""" + seen = {} + + def recorder(**kwargs): + seen.update(kwargs) + return "ok" + + result = call_custom_method( + _NullLogger(), "recorder", recorder, alpha=1, fallback_on_custom_error=True + ) + + assert result == "ok" + assert seen == {"alpha": 1} + + +class _NullLogger: + def warning(self, *args, **kwargs): + self.last = args + + +def test_the_sentinel_is_returned_only_on_the_opt_in_path(): + def boom(): + raise Refused("no") + + logger = _NullLogger() + assert call_custom_method( + logger, "boom", boom, fallback_on_custom_error=True + ) is CUSTOM_METHOD_FELL_BACK + + with pytest.raises(Refused): + call_custom_method(logger, "boom", boom) + + +def test_a_falsy_return_value_is_not_mistaken_for_a_failure(): + """`is not CUSTOM_METHOD_FELL_BACK` matters: None and 0 are real results.""" + for value in (None, 0, "", False, []): + assert call_custom_method(_NullLogger(), "m", lambda: value) is value + + +@pytest.mark.parametrize( + "module_name", + ["export", "ingest", "parse", "normalize", "embeddings", "kg"], +) +def test_no_module_still_swallows_custom_method_failures(module_name): + """The swallow was repeated across six modules, not just the one filed.""" + import semantica + + # Read the file rather than import it: some of these modules pull in + # optional third-party dependencies that need not be installed to check + # that the swallow is gone. + source = ( + Path(semantica.__file__).parent / module_name / "methods.py" + ).read_text(encoding="utf-8") + + assert "falling back to default" not in source, ( + f"semantica/{module_name}/methods.py still swallows custom method failures" + )