From 7b7d3fa8ad440105165e7e3f6469991d906bf5e0 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Sat, 13 Dec 2025 21:54:48 +0530 Subject: [PATCH] Refactor modules for pipeline API compatibility and fix bugs --- cookbook/introduction/config.yaml | 10 ++ cookbook/introduction/welcome_docs/apple.txt | 1 + semantica/core/__init__.py | 1 + semantica/embeddings/embedding_generator.py | 33 ++++- semantica/ingest/file_ingestor.py | 19 +++ semantica/kg/community_detector.py | 1 + semantica/kg/graph_analyzer.py | 13 ++ semantica/kg/graph_builder.py | 126 +++++++++++------- semantica/normalize/text_normalizer.py | 28 +++- semantica/parse/document_parser.py | 34 +++++ semantica/semantic_extract/ner_extractor.py | 25 +++- .../semantic_extract/relation_extractor.py | 44 +++++- semantica/vector_store/hybrid_search.py | 59 ++++++-- semantica/vector_store/vector_store.py | 48 ++++++- 14 files changed, 374 insertions(+), 68 deletions(-) create mode 100644 cookbook/introduction/config.yaml create mode 100644 cookbook/introduction/welcome_docs/apple.txt diff --git a/cookbook/introduction/config.yaml b/cookbook/introduction/config.yaml new file mode 100644 index 00000000..a95a75af --- /dev/null +++ b/cookbook/introduction/config.yaml @@ -0,0 +1,10 @@ +api_keys: + openai: your_key_here + anthropic: your_key_here +embedding: + provider: openai + model: text-embedding-3-large + dimensions: 3072 +knowledge_graph: + backend: networkx + temporal: true diff --git a/cookbook/introduction/welcome_docs/apple.txt b/cookbook/introduction/welcome_docs/apple.txt new file mode 100644 index 00000000..ea3ed56a --- /dev/null +++ b/cookbook/introduction/welcome_docs/apple.txt @@ -0,0 +1 @@ +Apple Inc. was founded by Steve Jobs, Steve Wozniak and Ronald Wayne in Cupertino, California. \ No newline at end of file diff --git a/semantica/core/__init__.py b/semantica/core/__init__.py index 03a7e1c7..ea258d06 100644 --- a/semantica/core/__init__.py +++ b/semantica/core/__init__.py @@ -86,4 +86,5 @@ __all__ = [ "get_status", "get_orchestration_method", "list_available_methods", +] diff --git a/semantica/embeddings/embedding_generator.py b/semantica/embeddings/embedding_generator.py index a95775c7..6b226b82 100644 --- a/semantica/embeddings/embedding_generator.py +++ b/semantica/embeddings/embedding_generator.py @@ -131,7 +131,7 @@ class EmbeddingGenerator: def generate_embeddings( self, - data: Union[str, Path, List[Union[str, Path]]], + data: Union[str, Path, List[Union[str, Path, Any]]], data_type: Optional[str] = None, **options, ) -> np.ndarray: @@ -144,7 +144,7 @@ class EmbeddingGenerator: Args: data: Input data to embed: - str: Text string - - List: Batch of texts + - List: Batch of texts or FileObjects data_type: Explicit data type ("text") If None, defaults to "text" **options: Additional generation options passed to embedder @@ -168,6 +168,35 @@ class EmbeddingGenerator: data_type = "text" self.logger.debug(f"Using data type: {data_type}") + # Pre-process list if it contains FileObjects or dicts + if isinstance(data, list): + processed_data = [] + for item in data: + if isinstance(item, str): + processed_data.append(item) + elif hasattr(item, "content") and item.content: + # Handle FileObject or similar + if isinstance(item.content, bytes): + try: + processed_data.append(item.content.decode("utf-8")) + except Exception: + # Fallback or skip + processed_data.append("") + else: + processed_data.append(str(item.content)) + elif isinstance(item, dict) and "content" in item: + # Handle parsed/normalized doc + processed_data.append(str(item["content"])) + else: + # Try converting to string + try: + processed_data.append(str(item)) + except Exception: + processed_data.append("") + + # Update data to be list of strings + data = processed_data + # Route to appropriate embedder based on data type if data_type == "text": if isinstance(data, str): diff --git a/semantica/ingest/file_ingestor.py b/semantica/ingest/file_ingestor.py index f1fb8582..20e40837 100644 --- a/semantica/ingest/file_ingestor.py +++ b/semantica/ingest/file_ingestor.py @@ -420,6 +420,25 @@ class FileIngestor: self.logger.info("File ingestor initialized") + def ingest(self, source: Union[str, Path], **options) -> List[FileObject]: + """ + Alias for ingest_directory (for backward compatibility or convenience). + + Args: + source: Path to directory or file + **options: Additional options + + Returns: + List[FileObject]: List of ingested file objects + """ + path = Path(source) + if path.is_dir(): + return self.ingest_directory(path, **options) + elif path.is_file(): + return [self.ingest_file(path, **options)] + else: + raise ValidationError(f"Path not found: {path}") + def ingest_directory( self, directory_path: Union[str, Path], recursive: bool = True, **filters ) -> List[FileObject]: diff --git a/semantica/kg/community_detector.py b/semantica/kg/community_detector.py index 764436e4..02364c5e 100644 --- a/semantica/kg/community_detector.py +++ b/semantica/kg/community_detector.py @@ -481,6 +481,7 @@ class CommunityDetector: changed = True iterations = 0 max_iter = options.get("max_iter", 10) + best_modularity = 0.0 while changed and iterations < max_iter: changed = False diff --git a/semantica/kg/graph_analyzer.py b/semantica/kg/graph_analyzer.py index 9148a59a..e8bb8b7a 100644 --- a/semantica/kg/graph_analyzer.py +++ b/semantica/kg/graph_analyzer.py @@ -96,6 +96,19 @@ class GraphAnalyzer: self.logger.info(f"Graph analyzer initialized (temporal: {enable_temporal})") + def analyze(self, graph: Dict[str, Any], **options) -> Dict[str, Any]: + """ + Alias for analyze_graph. + + Args: + graph: Knowledge graph dictionary + **options: Analysis options + + Returns: + Dict[str, Any]: Analysis results + """ + return self.analyze_graph(graph, **options) + def analyze_graph(self, graph: Dict[str, Any], **options) -> Dict[str, Any]: """ Perform comprehensive graph analysis. diff --git a/semantica/kg/graph_builder.py b/semantica/kg/graph_builder.py index 964d4147..d48ab7cf 100644 --- a/semantica/kg/graph_builder.py +++ b/semantica/kg/graph_builder.py @@ -120,41 +120,84 @@ class GraphBuilder: self.conflict_detector = None self.logger.debug("Conflict resolution disabled") + def _process_item(self, item: Any, all_entities: List[Any], all_relationships: List[Any]): + """Helper to process a single item and add to entities or relationships list.""" + if hasattr(item, "text") and hasattr(item, "label"): + # It's likely an Entity object + # Convert to dict format expected by graph builder + entity_dict = { + "id": getattr(item, "id", item.text), # Use text as ID if no ID + "name": item.text, + "type": item.label, + "confidence": getattr(item, "confidence", 1.0), + "metadata": getattr(item, "metadata", {}) + } + all_entities.append(entity_dict) + elif hasattr(item, "subject") and hasattr(item, "predicate") and hasattr(item, "object"): + # It's likely a Relation object + # Convert to dict format + # Subject and Object in Relation might be Entity objects or strings + subj = item.subject + obj = item.object + + subj_id = getattr(subj, "text", subj) if not isinstance(subj, str) else subj + obj_id = getattr(obj, "text", obj) if not isinstance(obj, str) else obj + + rel_dict = { + "source": subj_id, + "target": obj_id, + "type": item.predicate, + "confidence": getattr(item, "confidence", 1.0), + "metadata": getattr(item, "metadata", {}) + } + all_relationships.append(rel_dict) + elif isinstance(item, dict): + if "entities" in item: + all_entities.extend(item["entities"]) + elif "relationships" in item: + all_relationships.extend(item["relationships"]) + elif "source" in item and "target" in item: + all_relationships.append(item) + elif "id" in item or "entity_id" in item or "name" in item: + all_entities.append(item) + else: + # Unknown type, try to treat as entity if it has string representation + pass + def build( self, sources: Union[List[Any], Any], - entity_resolver: Optional[Any] = None, + second_arg: Optional[Any] = None, **options, ) -> Dict[str, Any]: """ Build knowledge graph from sources. - This method processes various source formats and extracts entities and - relationships to construct a knowledge graph. It handles entity resolution - and conflict detection if enabled. - Args: - sources: List of sources in various formats: - - Dict with "entities" and/or "relationships" keys - - Dict with entity-like structure (has "id" or "entity_id") - - Dict with relationship structure (has "source" and "target") - - List of entity/relationship dicts - entity_resolver: Optional custom entity resolver (overrides default) + sources: Entities or sources list + second_arg: Optional relationships list or entity_resolver (for backward compatibility) **options: Additional build options Returns: - Dictionary containing: - - entities: List of resolved entities - - relationships: List of relationships - - metadata: Graph metadata including counts and timestamps - - Example: - >>> sources = [ - ... {"entities": [{"id": "1", "name": "Alice"}], - ... "relationships": [{"source": "1", "target": "2", "type": "knows"}]} - ... ] - >>> graph = builder.build(sources) + Dictionary containing entities and relationships """ + # Handle arguments + entity_resolver = None + explicit_relationships = None + + # Check if second_arg is entity_resolver or relationships + if second_arg is not None: + if hasattr(second_arg, "resolve"): # Duck typing for EntityResolver + entity_resolver = second_arg + elif isinstance(second_arg, list): + explicit_relationships = second_arg + + # Also check options for named arguments + if "entity_resolver" in options: + entity_resolver = options.pop("entity_resolver") + if "relationships" in options: + explicit_relationships = options.pop("relationships") + # Normalize sources to list if not isinstance(sources, list): sources = [sources] @@ -176,32 +219,23 @@ class GraphBuilder: all_entities = [] all_relationships = [] + # Process sources (which might be entities) for source in sources: - if isinstance(source, dict): - # Source is a dictionary - extract entities and relationships - if "entities" in source: - # Explicit entities list - all_entities.extend(source["entities"]) - elif "id" in source or "entity_id" in source: - # Single entity object - all_entities.append(source) - - if "relationships" in source: - # Explicit relationships list - all_relationships.extend(source["relationships"]) - elif "source" in source and "target" in source: - # Single relationship object - all_relationships.append(source) - - elif isinstance(source, list): - # Source is a list - process each item + if isinstance(source, list): + # List of items (could be entities, relations, or mixed) for item in source: - if isinstance(item, dict): - # Determine if item is a relationship or entity - if "source" in item and "target" in item: - all_relationships.append(item) - else: - all_entities.append(item) + self._process_item(item, all_entities, all_relationships) + else: + self._process_item(source, all_entities, all_relationships) + + # Process explicit relationships if provided + if explicit_relationships: + for rel_item in explicit_relationships: + if isinstance(rel_item, list): + for item in rel_item: + self._process_item(item, all_entities, all_relationships) + else: + self._process_item(rel_item, all_entities, all_relationships) self.logger.debug( f"Extracted {len(all_entities)} entities and " diff --git a/semantica/normalize/text_normalizer.py b/semantica/normalize/text_normalizer.py index 66eda911..3ac7cd57 100644 --- a/semantica/normalize/text_normalizer.py +++ b/semantica/normalize/text_normalizer.py @@ -31,7 +31,7 @@ License: MIT import re import unicodedata -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union from ..utils.exceptions import ProcessingError, ValidationError from ..utils.logging import get_logger @@ -88,6 +88,32 @@ class TextNormalizer: self.logger.debug("Text normalizer initialized") + def normalize( + self, + source: Union[str, List[Dict[str, Any]]], + **options, + ) -> Union[str, List[Dict[str, Any]]]: + """ + Normalize text content or a list of parsed documents. + """ + if isinstance(source, list): + # Handle list of parsed documents + normalized_docs = [] + for doc in source: + if isinstance(doc, dict) and "content" in doc: + new_doc = doc.copy() + new_doc["content"] = self.normalize_text(doc["content"], **options) + normalized_docs.append(new_doc) + else: + # If it's just a string or unknown, try to normalize it directly or skip + try: + normalized_docs.append(self.normalize_text(str(doc), **options)) + except Exception: + normalized_docs.append(doc) + return normalized_docs + else: + return self.normalize_text(str(source), **options) + def normalize_text( self, text: str, diff --git a/semantica/parse/document_parser.py b/semantica/parse/document_parser.py index c284d91f..6b5d41e4 100644 --- a/semantica/parse/document_parser.py +++ b/semantica/parse/document_parser.py @@ -97,6 +97,40 @@ class DocumentParser: # Initialize progress tracker self.progress_tracker = get_progress_tracker() + def parse( + self, + source: Union[str, Path, List[Union[str, Path]], List[Any]], + **options, + ) -> Union[Dict[str, Any], List[Dict[str, Any]]]: + """ + Alias for parse_document or parse_batch. + Handles FileObject lists as well. + + Args: + source: Path(s) or FileObject(s) + **options: Parsing options + + Returns: + Parsed document(s) + """ + if isinstance(source, list): + # Extract paths if FileObjects + paths = [] + for item in source: + if hasattr(item, "path"): + paths.append(item.path) + elif isinstance(item, (str, Path)): + paths.append(item) + else: + raise ValueError(f"Unsupported item type in list: {type(item)}") + + # Use parse_batch + batch_result = self.parse_batch(paths, **options) + # Return list of results (successful ones) + return [item["result"] for item in batch_result["successful"]] + else: + return self.parse_document(source, **options) + def parse_document( self, file_path: Union[str, Path], file_type: Optional[str] = None, **options ) -> Dict[str, Any]: diff --git a/semantica/semantic_extract/ner_extractor.py b/semantica/semantic_extract/ner_extractor.py index 2336b400..020fb8ab 100644 --- a/semantica/semantic_extract/ner_extractor.py +++ b/semantica/semantic_extract/ner_extractor.py @@ -142,18 +142,35 @@ class NERExtractor: f"spaCy model {self.model_name} not found. ML method will fallback." ) - def extract(self, text: str, **kwargs) -> List[Entity]: + def extract(self, text: Union[str, List[Dict[str, Any]], List[str]], **kwargs) -> Union[List[Entity], List[List[Entity]]]: """ Alias for extract_entities. + Handles both single string and list of documents. Args: - text: Input text + text: Input text or list of documents **kwargs: Extraction options Returns: - list: List of extracted entities + Union[List[Entity], List[List[Entity]]]: Extracted entities """ - return self.extract_entities(text, **kwargs) + if isinstance(text, list): + # Handle batch extraction + results = [] + for item in text: + if isinstance(item, dict) and "content" in item: + results.append(self.extract_entities(item["content"], **kwargs)) + elif isinstance(item, str): + results.append(self.extract_entities(item, **kwargs)) + else: + # Try converting to string + try: + results.append(self.extract_entities(str(item), **kwargs)) + except Exception: + results.append([]) + return results + else: + return self.extract_entities(text, **kwargs) def extract_entities(self, text: str, **options) -> List[Entity]: """ diff --git a/semantica/semantic_extract/relation_extractor.py b/semantica/semantic_extract/relation_extractor.py index e51c4bc0..e9a6781c 100644 --- a/semantica/semantic_extract/relation_extractor.py +++ b/semantica/semantic_extract/relation_extractor.py @@ -155,19 +155,53 @@ class RelationExtractor: } - def extract(self, text: str, entities: List[Entity], **kwargs) -> List[Relation]: + def extract( + self, + text: Union[str, List[Dict[str, Any]], List[str]], + entities: Union[List[Entity], List[List[Entity]]], + **kwargs + ) -> Union[List[Relation], List[List[Relation]]]: """ Alias for extract_relations. + Handles both single string/entity-list and list of documents/entity-lists. Args: - text: Input text - entities: List of entities in the text + text: Input text or list of documents + entities: List of entities or list of list of entities **kwargs: Extraction options Returns: - list: List of extracted relations + Union[List[Relation], List[List[Relation]]]: Extracted relations """ - return self.extract_relations(text, entities, **kwargs) + if isinstance(text, list) and isinstance(entities, list): + # Handle batch extraction + results = [] + # Ensure lists are same length + min_len = min(len(text), len(entities)) + for i in range(min_len): + doc_item = text[i] + ent_item = entities[i] + + doc_text = "" + if isinstance(doc_item, dict) and "content" in doc_item: + doc_text = doc_item["content"] + elif isinstance(doc_item, str): + doc_text = doc_item + else: + doc_text = str(doc_item) + + # Ensure ent_item is a list of entities + if not isinstance(ent_item, list): + ent_item = [] # Should not happen if entities is List[List[Entity]] + + results.append(self.extract_relations(doc_text, ent_item, **kwargs)) + return results + elif isinstance(text, str) and isinstance(entities, list): + # Single text, single list of entities (standard case) + return self.extract_relations(text, entities, **kwargs) + else: + # Fallback or invalid input combination + return [] def extract_relations( self, text: str, entities: List[Entity], **options diff --git a/semantica/vector_store/hybrid_search.py b/semantica/vector_store/hybrid_search.py index fb0e77e1..363813d6 100644 --- a/semantica/vector_store/hybrid_search.py +++ b/semantica/vector_store/hybrid_search.py @@ -259,21 +259,23 @@ class HybridSearch: • Advanced search strategies """ - def __init__(self, **config): + def __init__(self, vector_store=None, **config): """Initialize hybrid search.""" self.logger = get_logger("hybrid_search") self.config = config + self.vector_store = vector_store self.progress_tracker = get_progress_tracker() self.ranker = SearchRanker( config.get("ranking_strategy", "reciprocal_rank_fusion") ) + self.embedding_generator = None def search( self, - query_vector: np.ndarray, - vectors: List[np.ndarray], - metadata: List[Dict[str, Any]], - vector_ids: List[str], + query: Union[str, np.ndarray], + vectors: Optional[List[np.ndarray]] = None, + metadata: Optional[List[Dict[str, Any]]] = None, + vector_ids: Optional[List[str]] = None, k: int = 10, metadata_filter: Optional[MetadataFilter] = None, **options, @@ -282,10 +284,10 @@ class HybridSearch: Perform hybrid search. Args: - query_vector: Query vector - vectors: List of vectors to search - metadata: List of metadata dictionaries - vector_ids: Vector IDs + query: Query vector or string + vectors: List of vectors to search (optional if vector_store provided) + metadata: List of metadata dictionaries (optional if vector_store provided) + vector_ids: Vector IDs (optional if vector_store provided) k: Number of results metadata_filter: Optional metadata filter **options: Additional options @@ -293,6 +295,10 @@ class HybridSearch: Returns: List of search results """ + # Handle legacy argument top_k + if "top_k" in options: + k = options["top_k"] + tracking_id = self.progress_tracker.start_tracking( module="vector_store", submodule="HybridSearch", @@ -300,6 +306,41 @@ class HybridSearch: ) try: + # Resolve vector store data if not provided + if vectors is None and self.vector_store: + vector_ids = list(self.vector_store.vectors.keys()) + vectors = [self.vector_store.vectors[vid] for vid in vector_ids] + metadata = [self.vector_store.metadata.get(vid, {}) for vid in vector_ids] + + if vectors is None or metadata is None: + # Check if vectors/metadata are falsy (empty list) but not None + # If they are None, we can't proceed. If they are empty lists, we return empty results. + if vectors is None: + vectors = [] + if metadata is None: + metadata = [] + if vector_ids is None: + vector_ids = [] + + # Handle string query + if isinstance(query, str): + self.progress_tracker.update_tracking( + tracking_id, message="Generating query embedding..." + ) + if not self.embedding_generator: + try: + from ..embeddings import EmbeddingGenerator + self.embedding_generator = EmbeddingGenerator() + except ImportError: + raise ImportError("EmbeddingGenerator not available for string queries") + + query_vector = self.embedding_generator.generate_embeddings(query, data_type="text") + # Handle if it returns batch (2D) or single (1D) + if len(query_vector.shape) == 2: + query_vector = query_vector[0] + else: + query_vector = query + if not vectors or not metadata: self.progress_tracker.stop_tracking( tracking_id, diff --git a/semantica/vector_store/vector_store.py b/semantica/vector_store/vector_store.py index da572626..d3789f6b 100644 --- a/semantica/vector_store/vector_store.py +++ b/semantica/vector_store/vector_store.py @@ -37,7 +37,7 @@ Author: Semantica Contributors License: MIT """ -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, Union import numpy as np @@ -89,6 +89,52 @@ class VectorStore: ) self.retriever = VectorRetriever(backend=backend, **self.config) + def store( + self, + vectors: List[np.ndarray], + documents: Optional[List[Any]] = None, + metadata: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None, + **options, + ) -> List[str]: + """ + Convenience method to store vectors with documents/metadata. + + Args: + vectors: List of embeddings + documents: Optional list of source documents + metadata: Optional metadata (dict for all, or list for each) + **options: Additional options + + Returns: + List[str]: Vector IDs + """ + # Prepare metadata list + num_vectors = len(vectors) + final_metadata = [] + + if isinstance(metadata, list): + if len(metadata) != num_vectors: + raise ValueError("Metadata list length must match vectors length") + final_metadata = metadata + elif isinstance(metadata, dict): + # Apply same metadata to all, copy to avoid shared reference issues + final_metadata = [metadata.copy() for _ in range(num_vectors)] + else: + final_metadata = [{} for _ in range(num_vectors)] + + # Merge document metadata if available + if documents and len(documents) == num_vectors: + for i, doc in enumerate(documents): + doc_meta = {} + if hasattr(doc, "metadata"): + doc_meta = doc.metadata + elif isinstance(doc, dict): + doc_meta = doc.get("metadata", {}) + + final_metadata[i].update(doc_meta) + + return self.store_vectors(vectors, metadata=final_metadata, **options) + def store_vectors( self, vectors: List[np.ndarray],