Refactor modules for pipeline API compatibility and fix bugs

cookbook/introduction/config.yaml

@@ -0,0 +1,10 @@
+api_keys:
+  openai: your_key_here
+  anthropic: your_key_here
+embedding:
+  provider: openai
+  model: text-embedding-3-large
+  dimensions: 3072
+knowledge_graph:
+  backend: networkx
+  temporal: true

cookbook/introduction/welcome_docs/apple.txt

@@ -0,0 +1 @@
+Apple Inc. was founded by Steve Jobs, Steve Wozniak and Ronald Wayne in Cupertino, California.

semantica/core/__init__.py

@@ -86,4 +86,5 @@ __all__ = [
     "get_status",
     "get_orchestration_method",
     "list_available_methods",
+]
 

semantica/embeddings/embedding_generator.py

@@ -131,7 +131,7 @@ class EmbeddingGenerator:
 
     def generate_embeddings(
         self,
-        data: Union[str, Path, List[Union[str, Path]]],
+        data: Union[str, Path, List[Union[str, Path, Any]]],
         data_type: Optional[str] = None,
         **options,
     ) -> np.ndarray:
@@ -144,7 +144,7 @@ class EmbeddingGenerator:
         Args:
             data: Input data to embed:
                 - str: Text string
-                - List: Batch of texts
+                - List: Batch of texts or FileObjects
             data_type: Explicit data type ("text")
                       If None, defaults to "text"
             **options: Additional generation options passed to embedder
@@ -168,6 +168,35 @@ class EmbeddingGenerator:
             data_type = "text"
             self.logger.debug(f"Using data type: {data_type}")
 
+        # Pre-process list if it contains FileObjects or dicts
+        if isinstance(data, list):
+            processed_data = []
+            for item in data:
+                if isinstance(item, str):
+                    processed_data.append(item)
+                elif hasattr(item, "content") and item.content:
+                    # Handle FileObject or similar
+                    if isinstance(item.content, bytes):
+                        try:
+                            processed_data.append(item.content.decode("utf-8"))
+                        except Exception:
+                            # Fallback or skip
+                            processed_data.append("")
+                    else:
+                        processed_data.append(str(item.content))
+                elif isinstance(item, dict) and "content" in item:
+                    # Handle parsed/normalized doc
+                    processed_data.append(str(item["content"]))
+                else:
+                    # Try converting to string
+                    try:
+                        processed_data.append(str(item))
+                    except Exception:
+                        processed_data.append("")
+            
+            # Update data to be list of strings
+            data = processed_data
+
         # Route to appropriate embedder based on data type
         if data_type == "text":
             if isinstance(data, str):

semantica/ingest/file_ingestor.py

@@ -420,6 +420,25 @@ class FileIngestor:
 
         self.logger.info("File ingestor initialized")
 
+    def ingest(self, source: Union[str, Path], **options) -> List[FileObject]:
+        """
+        Alias for ingest_directory (for backward compatibility or convenience).
+
+        Args:
+            source: Path to directory or file
+            **options: Additional options
+
+        Returns:
+            List[FileObject]: List of ingested file objects
+        """
+        path = Path(source)
+        if path.is_dir():
+            return self.ingest_directory(path, **options)
+        elif path.is_file():
+            return [self.ingest_file(path, **options)]
+        else:
+            raise ValidationError(f"Path not found: {path}")
+
     def ingest_directory(
         self, directory_path: Union[str, Path], recursive: bool = True, **filters
     ) -> List[FileObject]:

semantica/kg/community_detector.py

@@ -481,6 +481,7 @@ class CommunityDetector:
         changed = True
         iterations = 0
         max_iter = options.get("max_iter", 10)
+        best_modularity = 0.0
 
         while changed and iterations < max_iter:
             changed = False

semantica/kg/graph_analyzer.py

@@ -96,6 +96,19 @@ class GraphAnalyzer:
 
         self.logger.info(f"Graph analyzer initialized (temporal: {enable_temporal})")
 
+    def analyze(self, graph: Dict[str, Any], **options) -> Dict[str, Any]:
+        """
+        Alias for analyze_graph.
+        
+        Args:
+            graph: Knowledge graph dictionary
+            **options: Analysis options
+            
+        Returns:
+            Dict[str, Any]: Analysis results
+        """
+        return self.analyze_graph(graph, **options)
+
     def analyze_graph(self, graph: Dict[str, Any], **options) -> Dict[str, Any]:
         """
         Perform comprehensive graph analysis.

semantica/kg/graph_builder.py

@@ -120,41 +120,84 @@ class GraphBuilder:
             self.conflict_detector = None
             self.logger.debug("Conflict resolution disabled")
 
+    def _process_item(self, item: Any, all_entities: List[Any], all_relationships: List[Any]):
+        """Helper to process a single item and add to entities or relationships list."""
+        if hasattr(item, "text") and hasattr(item, "label"):
+            # It's likely an Entity object
+            # Convert to dict format expected by graph builder
+            entity_dict = {
+                "id": getattr(item, "id", item.text), # Use text as ID if no ID
+                "name": item.text,
+                "type": item.label,
+                "confidence": getattr(item, "confidence", 1.0),
+                "metadata": getattr(item, "metadata", {})
+            }
+            all_entities.append(entity_dict)
+        elif hasattr(item, "subject") and hasattr(item, "predicate") and hasattr(item, "object"):
+            # It's likely a Relation object
+            # Convert to dict format
+            # Subject and Object in Relation might be Entity objects or strings
+            subj = item.subject
+            obj = item.object
+            
+            subj_id = getattr(subj, "text", subj) if not isinstance(subj, str) else subj
+            obj_id = getattr(obj, "text", obj) if not isinstance(obj, str) else obj
+            
+            rel_dict = {
+                "source": subj_id,
+                "target": obj_id,
+                "type": item.predicate,
+                "confidence": getattr(item, "confidence", 1.0),
+                "metadata": getattr(item, "metadata", {})
+            }
+            all_relationships.append(rel_dict)
+        elif isinstance(item, dict):
+            if "entities" in item:
+                all_entities.extend(item["entities"])
+            elif "relationships" in item:
+                all_relationships.extend(item["relationships"])
+            elif "source" in item and "target" in item:
+                all_relationships.append(item)
+            elif "id" in item or "entity_id" in item or "name" in item:
+                all_entities.append(item)
+        else:
+            # Unknown type, try to treat as entity if it has string representation
+            pass
+
     def build(
         self,
         sources: Union[List[Any], Any],
-        entity_resolver: Optional[Any] = None,
+        second_arg: Optional[Any] = None,
         **options,
     ) -> Dict[str, Any]:
         """
         Build knowledge graph from sources.
 
-        This method processes various source formats and extracts entities and
-        relationships to construct a knowledge graph. It handles entity resolution
-        and conflict detection if enabled.
-
         Args:
-            sources: List of sources in various formats:
-                - Dict with "entities" and/or "relationships" keys
-                - Dict with entity-like structure (has "id" or "entity_id")
-                - Dict with relationship structure (has "source" and "target")
-                - List of entity/relationship dicts
-            entity_resolver: Optional custom entity resolver (overrides default)
+            sources: Entities or sources list
+            second_arg: Optional relationships list or entity_resolver (for backward compatibility)
             **options: Additional build options
 
         Returns:
-            Dictionary containing:
-                - entities: List of resolved entities
-                - relationships: List of relationships
-                - metadata: Graph metadata including counts and timestamps
-
-        Example:
-            >>> sources = [
-            ...     {"entities": [{"id": "1", "name": "Alice"}],
-            ...      "relationships": [{"source": "1", "target": "2", "type": "knows"}]}
-            ... ]
-            >>> graph = builder.build(sources)
+            Dictionary containing entities and relationships
         """
+        # Handle arguments
+        entity_resolver = None
+        explicit_relationships = None
+
+        # Check if second_arg is entity_resolver or relationships
+        if second_arg is not None:
+            if hasattr(second_arg, "resolve"): # Duck typing for EntityResolver
+                entity_resolver = second_arg
+            elif isinstance(second_arg, list):
+                explicit_relationships = second_arg
+        
+        # Also check options for named arguments
+        if "entity_resolver" in options:
+            entity_resolver = options.pop("entity_resolver")
+        if "relationships" in options:
+            explicit_relationships = options.pop("relationships")
+
         # Normalize sources to list
         if not isinstance(sources, list):
             sources = [sources]
@@ -176,32 +219,23 @@ class GraphBuilder:
             all_entities = []
             all_relationships = []
 
+            # Process sources (which might be entities)
             for source in sources:
-                if isinstance(source, dict):
-                    # Source is a dictionary - extract entities and relationships
-                    if "entities" in source:
-                        # Explicit entities list
-                        all_entities.extend(source["entities"])
-                    elif "id" in source or "entity_id" in source:
-                        # Single entity object
-                        all_entities.append(source)
-
-                    if "relationships" in source:
-                        # Explicit relationships list
-                        all_relationships.extend(source["relationships"])
-                    elif "source" in source and "target" in source:
-                        # Single relationship object
-                        all_relationships.append(source)
-
-                elif isinstance(source, list):
-                    # Source is a list - process each item
+                if isinstance(source, list):
+                     # List of items (could be entities, relations, or mixed)
                     for item in source:
-                        if isinstance(item, dict):
-                            # Determine if item is a relationship or entity
-                            if "source" in item and "target" in item:
-                                all_relationships.append(item)
-                            else:
-                                all_entities.append(item)
+                        self._process_item(item, all_entities, all_relationships)
+                else:
+                    self._process_item(source, all_entities, all_relationships)
+
+            # Process explicit relationships if provided
+            if explicit_relationships:
+                for rel_item in explicit_relationships:
+                    if isinstance(rel_item, list):
+                        for item in rel_item:
+                             self._process_item(item, all_entities, all_relationships)
+                    else:
+                        self._process_item(rel_item, all_entities, all_relationships)
 
             self.logger.debug(
                 f"Extracted {len(all_entities)} entities and "

semantica/normalize/text_normalizer.py

@@ -31,7 +31,7 @@ License: MIT
 
 import re
 import unicodedata
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from ..utils.exceptions import ProcessingError, ValidationError
 from ..utils.logging import get_logger
@@ -88,6 +88,32 @@ class TextNormalizer:
 
         self.logger.debug("Text normalizer initialized")
 
+    def normalize(
+        self,
+        source: Union[str, List[Dict[str, Any]]],
+        **options,
+    ) -> Union[str, List[Dict[str, Any]]]:
+        """
+        Normalize text content or a list of parsed documents.
+        """
+        if isinstance(source, list):
+            # Handle list of parsed documents
+            normalized_docs = []
+            for doc in source:
+                if isinstance(doc, dict) and "content" in doc:
+                    new_doc = doc.copy()
+                    new_doc["content"] = self.normalize_text(doc["content"], **options)
+                    normalized_docs.append(new_doc)
+                else:
+                    # If it's just a string or unknown, try to normalize it directly or skip
+                    try:
+                        normalized_docs.append(self.normalize_text(str(doc), **options))
+                    except Exception:
+                        normalized_docs.append(doc)
+            return normalized_docs
+        else:
+            return self.normalize_text(str(source), **options)
+
     def normalize_text(
         self,
         text: str,

semantica/parse/document_parser.py

@@ -97,6 +97,40 @@ class DocumentParser:
         # Initialize progress tracker
         self.progress_tracker = get_progress_tracker()
 
+    def parse(
+        self,
+        source: Union[str, Path, List[Union[str, Path]], List[Any]],
+        **options,
+    ) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
+        """
+        Alias for parse_document or parse_batch.
+        Handles FileObject lists as well.
+
+        Args:
+            source: Path(s) or FileObject(s)
+            **options: Parsing options
+
+        Returns:
+            Parsed document(s)
+        """
+        if isinstance(source, list):
+            # Extract paths if FileObjects
+            paths = []
+            for item in source:
+                if hasattr(item, "path"):
+                    paths.append(item.path)
+                elif isinstance(item, (str, Path)):
+                    paths.append(item)
+                else:
+                    raise ValueError(f"Unsupported item type in list: {type(item)}")
+
+            # Use parse_batch
+            batch_result = self.parse_batch(paths, **options)
+            # Return list of results (successful ones)
+            return [item["result"] for item in batch_result["successful"]]
+        else:
+            return self.parse_document(source, **options)
+
     def parse_document(
         self, file_path: Union[str, Path], file_type: Optional[str] = None, **options
     ) -> Dict[str, Any]:

semantica/semantic_extract/ner_extractor.py

@@ -142,18 +142,35 @@ class NERExtractor:
                     f"spaCy model {self.model_name} not found. ML method will fallback."
                 )
 
-    def extract(self, text: str, **kwargs) -> List[Entity]:
+    def extract(self, text: Union[str, List[Dict[str, Any]], List[str]], **kwargs) -> Union[List[Entity], List[List[Entity]]]:
         """
         Alias for extract_entities.
+        Handles both single string and list of documents.
         
         Args:
-            text: Input text
+            text: Input text or list of documents
             **kwargs: Extraction options
             
         Returns:
-            list: List of extracted entities
+            Union[List[Entity], List[List[Entity]]]: Extracted entities
         """
-        return self.extract_entities(text, **kwargs)
+        if isinstance(text, list):
+            # Handle batch extraction
+            results = []
+            for item in text:
+                if isinstance(item, dict) and "content" in item:
+                    results.append(self.extract_entities(item["content"], **kwargs))
+                elif isinstance(item, str):
+                    results.append(self.extract_entities(item, **kwargs))
+                else:
+                    # Try converting to string
+                    try:
+                        results.append(self.extract_entities(str(item), **kwargs))
+                    except Exception:
+                        results.append([])
+            return results
+        else:
+            return self.extract_entities(text, **kwargs)
 
     def extract_entities(self, text: str, **options) -> List[Entity]:
         """

semantica/semantic_extract/relation_extractor.py

@@ -155,19 +155,53 @@ class RelationExtractor:
         }
 
 
-    def extract(self, text: str, entities: List[Entity], **kwargs) -> List[Relation]:
+    def extract(
+        self,
+        text: Union[str, List[Dict[str, Any]], List[str]],
+        entities: Union[List[Entity], List[List[Entity]]],
+        **kwargs
+    ) -> Union[List[Relation], List[List[Relation]]]:
         """
         Alias for extract_relations.
+        Handles both single string/entity-list and list of documents/entity-lists.
         
         Args:
-            text: Input text
-            entities: List of entities in the text
+            text: Input text or list of documents
+            entities: List of entities or list of list of entities
             **kwargs: Extraction options
             
         Returns:
-            list: List of extracted relations
+            Union[List[Relation], List[List[Relation]]]: Extracted relations
         """
-        return self.extract_relations(text, entities, **kwargs)
+        if isinstance(text, list) and isinstance(entities, list):
+            # Handle batch extraction
+            results = []
+            # Ensure lists are same length
+            min_len = min(len(text), len(entities))
+            for i in range(min_len):
+                doc_item = text[i]
+                ent_item = entities[i]
+                
+                doc_text = ""
+                if isinstance(doc_item, dict) and "content" in doc_item:
+                    doc_text = doc_item["content"]
+                elif isinstance(doc_item, str):
+                    doc_text = doc_item
+                else:
+                    doc_text = str(doc_item)
+                
+                # Ensure ent_item is a list of entities
+                if not isinstance(ent_item, list):
+                    ent_item = [] # Should not happen if entities is List[List[Entity]]
+                
+                results.append(self.extract_relations(doc_text, ent_item, **kwargs))
+            return results
+        elif isinstance(text, str) and isinstance(entities, list):
+             # Single text, single list of entities (standard case)
+            return self.extract_relations(text, entities, **kwargs)
+        else:
+            # Fallback or invalid input combination
+            return []
 
     def extract_relations(
         self, text: str, entities: List[Entity], **options

semantica/vector_store/hybrid_search.py

@@ -259,21 +259,23 @@ class HybridSearch:
     • Advanced search strategies
     """
 
-    def __init__(self, **config):
+    def __init__(self, vector_store=None, **config):
         """Initialize hybrid search."""
         self.logger = get_logger("hybrid_search")
         self.config = config
+        self.vector_store = vector_store
         self.progress_tracker = get_progress_tracker()
         self.ranker = SearchRanker(
             config.get("ranking_strategy", "reciprocal_rank_fusion")
         )
+        self.embedding_generator = None
 
     def search(
         self,
-        query_vector: np.ndarray,
-        vectors: List[np.ndarray],
-        metadata: List[Dict[str, Any]],
-        vector_ids: List[str],
+        query: Union[str, np.ndarray],
+        vectors: Optional[List[np.ndarray]] = None,
+        metadata: Optional[List[Dict[str, Any]]] = None,
+        vector_ids: Optional[List[str]] = None,
         k: int = 10,
         metadata_filter: Optional[MetadataFilter] = None,
         **options,
@@ -282,10 +284,10 @@ class HybridSearch:
         Perform hybrid search.
 
         Args:
-            query_vector: Query vector
-            vectors: List of vectors to search
-            metadata: List of metadata dictionaries
-            vector_ids: Vector IDs
+            query: Query vector or string
+            vectors: List of vectors to search (optional if vector_store provided)
+            metadata: List of metadata dictionaries (optional if vector_store provided)
+            vector_ids: Vector IDs (optional if vector_store provided)
             k: Number of results
             metadata_filter: Optional metadata filter
             **options: Additional options
@@ -293,6 +295,10 @@ class HybridSearch:
         Returns:
             List of search results
         """
+        # Handle legacy argument top_k
+        if "top_k" in options:
+            k = options["top_k"]
+
         tracking_id = self.progress_tracker.start_tracking(
             module="vector_store",
             submodule="HybridSearch",
@@ -300,6 +306,41 @@ class HybridSearch:
         )
 
         try:
+            # Resolve vector store data if not provided
+            if vectors is None and self.vector_store:
+                vector_ids = list(self.vector_store.vectors.keys())
+                vectors = [self.vector_store.vectors[vid] for vid in vector_ids]
+                metadata = [self.vector_store.metadata.get(vid, {}) for vid in vector_ids]
+
+            if vectors is None or metadata is None:
+                 # Check if vectors/metadata are falsy (empty list) but not None
+                 # If they are None, we can't proceed. If they are empty lists, we return empty results.
+                 if vectors is None:
+                     vectors = []
+                 if metadata is None:
+                     metadata = []
+                 if vector_ids is None:
+                     vector_ids = []
+
+            # Handle string query
+            if isinstance(query, str):
+                self.progress_tracker.update_tracking(
+                    tracking_id, message="Generating query embedding..."
+                )
+                if not self.embedding_generator:
+                    try:
+                        from ..embeddings import EmbeddingGenerator
+                        self.embedding_generator = EmbeddingGenerator()
+                    except ImportError:
+                        raise ImportError("EmbeddingGenerator not available for string queries")
+                
+                query_vector = self.embedding_generator.generate_embeddings(query, data_type="text")
+                # Handle if it returns batch (2D) or single (1D)
+                if len(query_vector.shape) == 2:
+                     query_vector = query_vector[0]
+            else:
+                query_vector = query
+
             if not vectors or not metadata:
                 self.progress_tracker.stop_tracking(
                     tracking_id,

semantica/vector_store/vector_store.py

@@ -37,7 +37,7 @@ Author: Semantica Contributors
 License: MIT
 """
 
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import numpy as np
 
@@ -89,6 +89,52 @@ class VectorStore:
         )
         self.retriever = VectorRetriever(backend=backend, **self.config)
 
+    def store(
+        self,
+        vectors: List[np.ndarray],
+        documents: Optional[List[Any]] = None,
+        metadata: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
+        **options,
+    ) -> List[str]:
+        """
+        Convenience method to store vectors with documents/metadata.
+
+        Args:
+            vectors: List of embeddings
+            documents: Optional list of source documents
+            metadata: Optional metadata (dict for all, or list for each)
+            **options: Additional options
+
+        Returns:
+            List[str]: Vector IDs
+        """
+        # Prepare metadata list
+        num_vectors = len(vectors)
+        final_metadata = []
+
+        if isinstance(metadata, list):
+            if len(metadata) != num_vectors:
+                raise ValueError("Metadata list length must match vectors length")
+            final_metadata = metadata
+        elif isinstance(metadata, dict):
+            # Apply same metadata to all, copy to avoid shared reference issues
+            final_metadata = [metadata.copy() for _ in range(num_vectors)]
+        else:
+            final_metadata = [{} for _ in range(num_vectors)]
+
+        # Merge document metadata if available
+        if documents and len(documents) == num_vectors:
+            for i, doc in enumerate(documents):
+                doc_meta = {}
+                if hasattr(doc, "metadata"):
+                    doc_meta = doc.metadata
+                elif isinstance(doc, dict):
+                    doc_meta = doc.get("metadata", {})
+                
+                final_metadata[i].update(doc_meta)
+        
+        return self.store_vectors(vectors, metadata=final_metadata, **options)
+
     def store_vectors(
         self,
         vectors: List[np.ndarray],