From 4cd3ef9aa8be442024efb3d0210ee3578a33ca7d Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Sat, 14 Feb 2026 17:13:38 +0530 Subject: [PATCH 01/50] context_fixes --- README.md | 31 ++ semantica/context/agent_context.py | 249 ++++++--- semantica/context/causal_analyzer.py | 9 +- semantica/context/context_graph.py | 105 +++- semantica/context/context_retriever.py | 27 +- semantica/context/policy_engine.py | 531 +++++++++++++------ semantica/kg/centrality_calculator.py | 5 + semantica/kg/community_detector.py | 6 + semantica/kg/link_predictor.py | 5 +- semantica/kg/node_embeddings.py | 26 +- tests/context/test_agent_context_smoke.py | 66 +++ tests/context/test_policy_engine_fallback.py | 62 +++ 12 files changed, 845 insertions(+), 277 deletions(-) create mode 100644 tests/context/test_agent_context_smoke.py create mode 100644 tests/context/test_policy_engine_fallback.py diff --git a/README.md b/README.md index dee65814..66792a8f 100644 --- a/README.md +++ b/README.md @@ -734,6 +734,37 @@ reasoned_result = context.query_with_reasoning( - **ContextRetriever**: Performs hybrid retrieval combining vector search, graph traversal, and memory for optimal context relevance - **AgentContext**: High-level interface integrating Context Graph and Context Retriever for GraphRAG applications +#### Context Graphs: Decision Tracking + +```python +from semantica.context import AgentContext, ContextGraph +from semantica.vector_store import VectorStore + +context = AgentContext( + vector_store=VectorStore(backend="inmemory", dimension=128), + knowledge_graph=ContextGraph(), + enable_decision_tracking=True, + enable_kg_algorithms=False, # semantic-only precedent search +) + +decision_id = context.record_decision( + category="credit_approval", + scenario="High-risk credit limit increase", + reasoning="Recent velocity-check failure and prior fraud flag", + outcome="rejected", + confidence=0.78, + entities=["customer:jessica_norris"], +) + +precedents = context.find_precedents( + scenario="High-risk customer credit increase", + category="credit_approval", + limit=5, +) +``` + +Runnable script: `examples/context_graphs_decision_tracking.py` + **Core Notebooks:** - [**Context Module Introduction**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/19_Context_Module.ipynb) - Basic memory and storage. - [**Advanced Context Engineering**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/advanced/11_Advanced_Context_Engineering.ipynb) - Hybrid retrieval, graph builders, and custom memory policies. diff --git a/semantica/context/agent_context.py b/semantica/context/agent_context.py index 6a2ecaaa..0fdde432 100644 --- a/semantica/context/agent_context.py +++ b/semantica/context/agent_context.py @@ -209,62 +209,52 @@ class AgentContext: if knowledge_graph and hasattr(knowledge_graph, "build_from_conversations"): self._graph_builder = knowledge_graph - # Store config - self.config = { + self.config.update({ "retention_days": retention_days, "max_memories": max_memories, - "use_graph_expansion": use_graph_expansion, - "max_expansion_hops": max_expansion_hops, - "hybrid_alpha": hybrid_alpha, - "enable_decision_tracking": enable_decision_tracking, - } + }) # Initialize decision tracking components if enabled + self._decision_backend = None self._decision_recorder = None self._decision_query = None self._causal_analyzer = None self._policy_engine = None if enable_decision_tracking and knowledge_graph: - # Validate that knowledge_graph supports required GraphStore interface - if not hasattr(knowledge_graph, 'execute_query'): - self.logger.error( - "Decision tracking requires a GraphStore-compatible knowledge graph with execute_query() method. " - "Provided knowledge_graph type does not support Cypher queries. " - "Use GraphStore (Neo4j, FalkorDB) or disable decision tracking." - ) - raise ValueError( - "Decision tracking requires a GraphStore-compatible knowledge graph. " - "The provided knowledge_graph does not have an execute_query() method. " - "For decision tracking, use a GraphStore backend (Neo4j, FalkorDB) " - "or set enable_decision_tracking=False." - ) - - # Initialize enhanced decision tracking components - try: - self._decision_recorder = DecisionRecorder(knowledge_graph) - - # Enhanced DecisionQuery with KG and vector store integration - self._decision_query = DecisionQuery( - graph_store=knowledge_graph, - vector_store=vector_store if enable_vector_store_features else None, - enable_advanced_analytics=enable_advanced_analytics, - enable_centrality_analysis=enable_kg_algorithms, - enable_community_detection=enable_kg_algorithms, - enable_node_embeddings=enable_kg_algorithms - ) - - self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) + if hasattr(knowledge_graph, "execute_query"): + self._decision_backend = "graph_store" + try: + self._decision_recorder = DecisionRecorder(knowledge_graph) + self._decision_query = DecisionQuery( + graph_store=knowledge_graph, + vector_store=vector_store if enable_vector_store_features else None, + enable_advanced_analytics=enable_advanced_analytics, + enable_centrality_analysis=enable_kg_algorithms, + enable_community_detection=enable_kg_algorithms, + enable_node_embeddings=enable_kg_algorithms + ) + self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) + self._policy_engine = PolicyEngine(knowledge_graph) + self.logger.info("Enhanced decision tracking components initialized successfully") + except Exception as e: + self.logger.warning(f"Failed to initialize enhanced decision tracking: {e}") + self._decision_recorder = DecisionRecorder(knowledge_graph) + self._decision_query = DecisionQuery(knowledge_graph) + self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) + self._policy_engine = PolicyEngine(knowledge_graph) + else: + self._decision_backend = "context_graph" self._policy_engine = PolicyEngine(knowledge_graph) - - self.logger.info("Enhanced decision tracking components initialized successfully") - except Exception as e: - self.logger.warning(f"Failed to initialize enhanced decision tracking: {e}") - # Fallback to basic components - self._decision_recorder = DecisionRecorder(knowledge_graph) - self._decision_query = DecisionQuery(knowledge_graph) self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) - self._policy_engine = PolicyEngine(knowledge_graph) + if enable_vector_store_features and hasattr(self.vector_store, "initialize_decision_pipeline"): + try: + self.vector_store.initialize_decision_pipeline( + graph_store=knowledge_graph if enable_kg_algorithms else None, + use_graph_features=enable_kg_algorithms + ) + except Exception as e: + self.logger.warning(f"Failed to initialize decision pipeline: {e}") @property def memory(self) -> AgentMemory: @@ -1560,7 +1550,7 @@ class AgentContext: Raises: RuntimeError: If decision tracking is not enabled """ - if not self._decision_recorder: + if not self._decision_backend: raise RuntimeError("Decision tracking is not enabled") from .decision_models import Decision @@ -1579,18 +1569,54 @@ class AgentContext: entities = entities or [] source_documents = [] # Could be enhanced to capture source docs - - decision_id = self._decision_recorder.record_decision( - decision, entities, source_documents - ) - - # Capture cross-system context if provided - if cross_system_context: - self._decision_recorder.capture_cross_system_context( - decision_id, cross_system_context + + if self._decision_backend == "graph_store": + decision_id = self._decision_recorder.record_decision( + decision, entities, source_documents ) - - return decision_id + + if cross_system_context: + self._decision_recorder.capture_cross_system_context( + decision_id, cross_system_context + ) + + return decision_id + + if not hasattr(self.knowledge_graph, "add_decision"): + raise RuntimeError("Decision tracking backend does not support decisions") + + self.knowledge_graph.add_decision(decision) + if cross_system_context and hasattr(self.knowledge_graph, "add_node_attribute"): + self.knowledge_graph.add_node_attribute( + decision.decision_id, {"cross_system_context": cross_system_context} + ) + for entity_id in entities: + try: + self.knowledge_graph.add_edge(decision.decision_id, entity_id, edge_type="ABOUT") + except Exception: + continue + + vector_id = None + if hasattr(self.vector_store, "store_decision"): + try: + vector_id = self.vector_store.store_decision( + scenario=scenario, + reasoning=reasoning, + outcome=outcome, + confidence=confidence, + entities=entities, + category=category, + decision_id=decision.decision_id, + decision_maker=decision.decision_maker, + timestamp=decision.timestamp.isoformat() + ) + except Exception: + vector_id = None + + if vector_id and hasattr(self.knowledge_graph, "add_node_attribute"): + self.knowledge_graph.add_node_attribute(decision.decision_id, {"vector_id": vector_id}) + + return decision.decision_id def find_precedents( self, @@ -1618,24 +1644,85 @@ class AgentContext: Raises: RuntimeError: If decision tracking is not enabled """ - if not self._decision_query: + if not self._decision_backend: raise RuntimeError("Decision tracking is not enabled") - - if use_hybrid_search: - try: - return self._decision_query.find_precedents_hybrid( - scenario, category, limit - ) - except Exception: - # Fallback to basic search if hybrid fails - return self._decision_query._find_precedents_basic(scenario, category, limit) - else: - # Simple category-based search + + if self._decision_backend == "graph_store": + if use_hybrid_search: + try: + return self._decision_query.find_precedents_hybrid( + scenario, category, limit + ) + except Exception: + return self._decision_query._find_precedents_basic(scenario, category, limit) if category: return self._decision_query.find_by_category(category, limit) - else: - # Use basic search - return self._decision_query._find_precedents_basic(scenario, category, limit) + return self._decision_query._find_precedents_basic(scenario, category, limit) + + results: List[Decision] = [] + + if use_hybrid_search and hasattr(self.vector_store, "search_decisions"): + filters = {"category": category} if category else None + vector_results = self.vector_store.search_decisions( + query=scenario, + filters=filters, + limit=limit, + use_hybrid_search=True + ) + for r in vector_results: + meta = r.get("metadata") or {} + decision_id = meta.get("decision_id") or meta.get("id") + if decision_id and hasattr(self.knowledge_graph, "nodes") and decision_id in self.knowledge_graph.nodes: + node = self.knowledge_graph.nodes[decision_id] + if getattr(node, "node_type", None) == "Decision": + data = getattr(node, "properties", {}) or {} + decision = Decision( + decision_id=decision_id, + category=data.get("category", ""), + scenario=getattr(node, "content", ""), + reasoning=data.get("reasoning", ""), + outcome=data.get("outcome", ""), + confidence=float(data.get("confidence", 0.0) or 0.0), + timestamp=datetime.fromisoformat(data.get("timestamp")) if data.get("timestamp") else datetime.now(), + decision_maker=data.get("decision_maker", "ai_agent"), + reasoning_embedding=data.get("reasoning_embedding"), + node2vec_embedding=data.get("node2vec_embedding"), + metadata={k: v for k, v in data.items() if k not in [ + "category", "reasoning", "outcome", "confidence", + "timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding" + ]} + ) + decision.metadata["score"] = r.get("score") + results.append(decision) + + if results: + return results[:limit] + + if hasattr(self.knowledge_graph, "find_nodes"): + for node in self.knowledge_graph.find_nodes(node_type="Decision"): + if category and node.get("metadata", {}).get("category") != category: + continue + data = node.get("metadata", {}) or {} + results.append( + Decision( + decision_id=node.get("id", ""), + category=data.get("category", ""), + scenario=node.get("content", ""), + reasoning=data.get("reasoning", ""), + outcome=data.get("outcome", ""), + confidence=float(data.get("confidence", 0.0) or 0.0), + timestamp=datetime.fromisoformat(data.get("timestamp")) if data.get("timestamp") else datetime.now(), + decision_maker=data.get("decision_maker", "ai_agent"), + reasoning_embedding=data.get("reasoning_embedding"), + node2vec_embedding=data.get("node2vec_embedding"), + metadata={k: v for k, v in data.items() if k not in [ + "category", "reasoning", "outcome", "confidence", + "timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding" + ]} + ) + ) + + return results[:limit] def get_causal_chain( self, @@ -1657,12 +1744,22 @@ class AgentContext: Raises: RuntimeError: If decision tracking is not enabled """ - if not self._causal_analyzer: + if not self._decision_backend: raise RuntimeError("Decision tracking is not enabled") - - return self._causal_analyzer.get_causal_chain( - decision_id, direction, max_depth - ) + + if self._decision_backend == "graph_store": + return self._causal_analyzer.get_causal_chain( + decision_id, direction, max_depth + ) + + if hasattr(self.knowledge_graph, "get_causal_chain"): + return self.knowledge_graph.get_causal_chain( + decision_id=decision_id, + direction=direction, + max_depth=max_depth + ) + + raise RuntimeError("Decision tracking backend does not support causal chains") def get_policy_engine(self) -> PolicyEngine: """ diff --git a/semantica/context/causal_analyzer.py b/semantica/context/causal_analyzer.py index ab53699c..9ad64215 100644 --- a/semantica/context/causal_analyzer.py +++ b/semantica/context/causal_analyzer.py @@ -77,7 +77,7 @@ class CausalChainAnalyzer: using graph traversal. """ - def __init__(self, graph_store: GraphStore): + def __init__(self, graph_store: Any): """ Initialize CausalChainAnalyzer. @@ -105,6 +105,13 @@ class CausalChainAnalyzer: List of decisions in causal chain """ try: + if hasattr(self.graph_store, "get_causal_chain") and not hasattr(self.graph_store, "execute_query"): + return self.graph_store.get_causal_chain( + decision_id=decision_id, + direction=direction, + max_depth=max_depth + ) + if direction not in ["upstream", "downstream"]: raise ValueError("Direction must be 'upstream' or 'downstream'") diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 4084391c..6e201a80 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -260,7 +260,69 @@ class ContextGraph: count += 1 return count - def get_neighbors(self, node_id: str, hops: int = 1) -> List[Dict[str, Any]]: + def __contains__(self, node_id: object) -> bool: + if not isinstance(node_id, str): + return False + return node_id in self.nodes + + def has_node(self, node_id: str) -> bool: + return node_id in self.nodes + + def neighbors(self, node_id: str) -> List[str]: + return self.get_neighbor_ids(node_id) + + def get_neighbor_ids( + self, + node_id: str, + relationship_types: Optional[List[str]] = None, + ) -> List[str]: + if node_id not in self.nodes: + return [] + + rel_filter = set(relationship_types) if relationship_types else None + neighbor_ids: List[str] = [] + for edge in self._adjacency.get(node_id, []): + if rel_filter is None or edge.edge_type in rel_filter: + neighbor_ids.append(edge.target_id) + return neighbor_ids + + def get_nodes_by_label(self, label: str) -> List[str]: + return list(self.node_type_index.get(label, set())) + + def get_node_property(self, node_id: str, property_name: str) -> Any: + node = self.nodes.get(node_id) + if not node: + return None + return node.properties.get(property_name) + + def get_node_attributes(self, node_id: str) -> Dict[str, Any]: + node = self.nodes.get(node_id) + if not node: + return {} + return node.properties.copy() + + def add_node_attribute(self, node_id: str, attributes: Dict[str, Any]) -> None: + node = self.nodes.get(node_id) + if not node: + return + node.properties.update(attributes) + node.metadata.update(attributes) + + def get_edge_data(self, source_id: str, target_id: str) -> Dict[str, Any]: + for edge in self._adjacency.get(source_id, []): + if edge.target_id == target_id: + data = edge.metadata.copy() + data["type"] = edge.edge_type + data["weight"] = edge.weight + return data + return {} + + def get_neighbors( + self, + node_id: str, + hops: int = 1, + relationship_types: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: """ Get neighbors of a node. @@ -269,36 +331,39 @@ class ContextGraph: if node_id not in self.nodes: return [] - neighbors = [] + neighbors: List[Dict[str, Any]] = [] visited = {node_id} - queue = deque([(node_id, 0)]) # (current_id, current_hop) + queue = deque([(node_id, 0)]) + rel_filter = set(relationship_types) if relationship_types else None while queue: current_id, current_hop = queue.popleft() - if current_hop >= hops: continue - # Get outgoing edges outgoing_edges = self._adjacency.get(current_id, []) for edge in outgoing_edges: + if rel_filter is not None and edge.edge_type not in rel_filter: + continue neighbor_id = edge.target_id - if neighbor_id not in visited: - visited.add(neighbor_id) - queue.append((neighbor_id, current_hop + 1)) + if neighbor_id in visited: + continue + visited.add(neighbor_id) + queue.append((neighbor_id, current_hop + 1)) - if neighbor_id in self.nodes: - node = self.nodes[neighbor_id] - neighbors.append( - { - "id": node.node_id, - "type": node.node_type, - "content": node.content, - "relationship": edge.edge_type, - "weight": edge.weight, - "hop": current_hop + 1, - } - ) + node = self.nodes.get(neighbor_id) + if not node: + continue + neighbors.append( + { + "id": node.node_id, + "type": node.node_type, + "content": node.content, + "relationship": edge.edge_type, + "weight": edge.weight, + "hop": current_hop + 1, + } + ) return neighbors diff --git a/semantica/context/context_retriever.py b/semantica/context/context_retriever.py index c1131f7d..b72833c9 100644 --- a/semantica/context/context_retriever.py +++ b/semantica/context/context_retriever.py @@ -2011,8 +2011,18 @@ Answer:""" try: # Basic neighbor expansion if hasattr(self.knowledge_graph, 'get_neighbors'): - neighbors = self.knowledge_graph.get_neighbors(entity_name) - for neighbor in neighbors[:5]: # Limit to prevent explosion + if hasattr(self.knowledge_graph, "neighbors"): + neighbor_ids = list(self.knowledge_graph.neighbors(entity_name)) + elif hasattr(self.knowledge_graph, "get_neighbor_ids"): + neighbor_ids = self.knowledge_graph.get_neighbor_ids(entity_name) + else: + neighbor_details = self.knowledge_graph.get_neighbors(entity_name, hops=1) + neighbor_ids = [ + n.get("id") for n in neighbor_details + if isinstance(n, dict) and n.get("id") + ] + + for neighbor in neighbor_ids[:5]: # Limit to prevent explosion expanded_entities.append({ "name": neighbor, "type": "related_entity", @@ -2082,8 +2092,17 @@ Answer:""" if hasattr(self.centrality_calculator, 'calculate_degree_centrality'): # Simplified centrality calculation if hasattr(self.knowledge_graph, 'get_neighbors'): - neighbors = self.knowledge_graph.get_neighbors(entity_name) - centrality_scores[entity_name] = len(neighbors) + if hasattr(self.knowledge_graph, "neighbors"): + neighbor_ids = list(self.knowledge_graph.neighbors(entity_name)) + elif hasattr(self.knowledge_graph, "get_neighbor_ids"): + neighbor_ids = self.knowledge_graph.get_neighbor_ids(entity_name) + else: + neighbor_details = self.knowledge_graph.get_neighbors(entity_name, hops=1) + neighbor_ids = [ + n.get("id") for n in neighbor_details + if isinstance(n, dict) and n.get("id") + ] + centrality_scores[entity_name] = len(neighbor_ids) else: centrality_scores[entity_name] = 1 diff --git a/semantica/context/policy_engine.py b/semantica/context/policy_engine.py index cb7eb669..9cefaad6 100644 --- a/semantica/context/policy_engine.py +++ b/semantica/context/policy_engine.py @@ -84,7 +84,7 @@ class PolicyEngine: records exceptions, and analyzes policy impact. """ - def __init__(self, graph_store: GraphStore): + def __init__(self, graph_store: Any): """ Initialize PolicyEngine. @@ -93,6 +93,7 @@ class PolicyEngine: """ self.graph_store = graph_store self.logger = get_logger(__name__) + self._supports_cypher = hasattr(graph_store, "execute_query") def add_policy(self, policy: Policy) -> str: """ @@ -105,35 +106,54 @@ class PolicyEngine: Policy ID """ try: - # Store policy node - query = """ - CREATE (p:Policy { - policy_id: $policy_id, - name: $name, - description: $description, - rules: $rules, - category: $category, - version: $version, - created_at: $created_at, - updated_at: $updated_at, - metadata: $metadata - }) - """ - self.graph_store.execute_query(query, { - "policy_id": policy.policy_id, - "name": policy.name, - "description": policy.description, - "rules": policy.rules, - "category": policy.category, - "version": policy.version, - "created_at": policy.created_at, - "updated_at": policy.updated_at, - "metadata": policy.metadata - }) - + if self._supports_cypher: + query = """ + CREATE (p:Policy { + policy_id: $policy_id, + name: $name, + description: $description, + rules: $rules, + category: $category, + version: $version, + created_at: $created_at, + updated_at: $updated_at, + metadata: $metadata + }) + """ + self.graph_store.execute_query(query, { + "policy_id": policy.policy_id, + "name": policy.name, + "description": policy.description, + "rules": policy.rules, + "category": policy.category, + "version": policy.version, + "created_at": policy.created_at, + "updated_at": policy.updated_at, + "metadata": policy.metadata + }) + self.logger.info(f"Added policy: {policy.policy_id} version {policy.version}") + return policy.policy_id + + if not hasattr(self.graph_store, "add_node"): + raise RuntimeError("Graph backend does not support policy storage") + + node_id = f"{policy.policy_id}:{policy.version}" + self.graph_store.add_node( + node_id=node_id, + node_type="Policy", + content=policy.name or policy.policy_id, + policy_id=policy.policy_id, + name=policy.name, + description=policy.description, + rules=policy.rules, + category=policy.category, + version=policy.version, + created_at=policy.created_at.isoformat() if hasattr(policy.created_at, "isoformat") else str(policy.created_at), + updated_at=policy.updated_at.isoformat() if hasattr(policy.updated_at, "isoformat") else str(policy.updated_at), + metadata=policy.metadata or {} + ) self.logger.info(f"Added policy: {policy.policy_id} version {policy.version}") return policy.policy_id - except Exception as e: self.logger.error(f"Failed to add policy: {e}") raise @@ -187,17 +207,26 @@ class PolicyEngine: # Store new version self.add_policy(updated_policy) - # Link versions - query = """ - MATCH (old:Policy {policy_id: $policy_id, version: $old_version}) - MATCH (new:Policy {policy_id: $policy_id, version: $new_version}) - MERGE (old)-[:VERSION_OF]->(new) - """ - self.graph_store.execute_query(query, { - "policy_id": policy_id, - "old_version": current_policy.version, - "new_version": new_version - }) + if self._supports_cypher: + query = """ + MATCH (old:Policy {policy_id: $policy_id, version: $old_version}) + MATCH (new:Policy {policy_id: $policy_id, version: $new_version}) + MERGE (old)-[:VERSION_OF]->(new) + """ + self.graph_store.execute_query(query, { + "policy_id": policy_id, + "old_version": current_policy.version, + "new_version": new_version + }) + else: + if hasattr(self.graph_store, "add_edge"): + self.graph_store.add_edge( + f"{policy_id}:{current_policy.version}", + f"{policy_id}:{new_version}", + edge_type="VERSION_OF", + changed_at=datetime.now().isoformat(), + change_reason=change_reason + ) self.logger.info(f"Updated policy {policy_id} to version {new_version}") return new_version @@ -222,26 +251,60 @@ class PolicyEngine: List of applicable policies (latest versions) """ try: + if self._supports_cypher: # Get latest policies for category - query = """ - MATCH (p:Policy {category: $category}) - WHERE NOT (p)-[:VERSION_OF]->(:Policy) - RETURN p - ORDER BY p.updated_at DESC - """ - results = self.graph_store.execute_query(query, {"category": category}) - - policies = [] - for record in results: - policy_data = record.get("p", {}) - policies.append(self._dict_to_policy(policy_data)) - - # Filter by entities if specified - if entities: - # This would require entity-specific policy relationships - # For now, return all category policies - pass - + query = """ + MATCH (p:Policy {category: $category}) + WHERE NOT (p)-[:VERSION_OF]->(:Policy) + RETURN p + ORDER BY p.updated_at DESC + """ + results = self.graph_store.execute_query(query, {"category": category}) + + policies = [] + for record in results: + policy_data = record.get("p", {}) + policies.append(self._dict_to_policy(policy_data)) + + if entities: + pass + + self.logger.info(f"Found {len(policies)} applicable policies for category {category}") + return policies + + if not hasattr(self.graph_store, "find_nodes"): + return [] + + latest_by_policy_id: Dict[str, Dict[str, Any]] = {} + for node in self.graph_store.find_nodes(node_type="Policy"): + data = node.get("metadata", {}) or {} + if data.get("category") != category: + continue + pid = data.get("policy_id") + if not pid: + continue + updated_at = data.get("updated_at") or "" + prev = latest_by_policy_id.get(pid) + if not prev: + latest_by_policy_id[pid] = data + else: + if str(updated_at) > str(prev.get("updated_at") or ""): + latest_by_policy_id[pid] = data + + policies: List[Policy] = [] + for data in latest_by_policy_id.values(): + policies.append(self._dict_to_policy({ + "policy_id": data.get("policy_id"), + "name": data.get("name"), + "description": data.get("description"), + "rules": data.get("rules", {}), + "category": data.get("category"), + "version": data.get("version"), + "created_at": data.get("created_at"), + "updated_at": data.get("updated_at"), + "metadata": data.get("metadata", {}) + })) + self.logger.info(f"Found {len(policies)} applicable policies for category {category}") return policies @@ -303,20 +366,33 @@ class PolicyEngine: version: Policy version that was applied """ try: - query = """ - MATCH (d:Decision {decision_id: $decision_id}) - MATCH (p:Policy {policy_id: $policy_id, version: $version}) - MERGE (d)-[:APPLIED_POLICY]->(p) - SET d.policy_applied_at = timestamp() - """ - self.graph_store.execute_query(query, { - "decision_id": decision_id, - "policy_id": policy_id, - "version": version - }) - + if self._supports_cypher: + query = """ + MATCH (d:Decision {decision_id: $decision_id}) + MATCH (p:Policy {policy_id: $policy_id, version: $version}) + MERGE (d)-[:APPLIED_POLICY]->(p) + SET d.policy_applied_at = timestamp() + """ + self.graph_store.execute_query(query, { + "decision_id": decision_id, + "policy_id": policy_id, + "version": version + }) + self.logger.info(f"Recorded policy application: {policy_id} v{version} to decision {decision_id}") + return + + if not hasattr(self.graph_store, "add_edge"): + raise RuntimeError("Graph backend does not support relationships") + policy_node_id = f"{policy_id}:{version}" + self.graph_store.add_edge( + decision_id, + policy_node_id, + edge_type="APPLIED_POLICY", + applied_at=datetime.now().isoformat(), + policy_id=policy_id, + version=version + ) self.logger.info(f"Recorded policy application: {policy_id} v{version} to decision {decision_id}") - except Exception as e: self.logger.error(f"Failed to record policy application: {e}") raise @@ -340,40 +416,61 @@ class PolicyEngine: """ try: exception_id = str(uuid.uuid4()) - - query = """ - CREATE (e:Exception { - exception_id: $exception_id, - decision_id: $decision_id, - policy_id: $policy_id, - reason: $reason, - created_at: datetime() - }) - """ - self.graph_store.execute_query(query, { - "exception_id": exception_id, - "decision_id": decision_id, - "policy_id": policy_id, - "reason": reason - }) - - # Link to decision and policy - query = """ - MATCH (d:Decision {decision_id: $decision_id}) - MATCH (p:Policy {policy_id: $policy_id}) - MATCH (e:Exception {exception_id: $exception_id}) - MERGE (d)-[:GRANTED_EXCEPTION]->(e) - MERGE (e)-[:OVERRIDDEN_POLICY]->(p) - """ - self.graph_store.execute_query(query, { - "decision_id": decision_id, - "policy_id": policy_id, - "exception_id": exception_id - }) - + + if self._supports_cypher: + query = """ + CREATE (e:Exception { + exception_id: $exception_id, + decision_id: $decision_id, + policy_id: $policy_id, + reason: $reason, + created_at: datetime() + }) + """ + self.graph_store.execute_query(query, { + "exception_id": exception_id, + "decision_id": decision_id, + "policy_id": policy_id, + "reason": reason + }) + + query = """ + MATCH (d:Decision {decision_id: $decision_id}) + MATCH (p:Policy {policy_id: $policy_id}) + MATCH (e:Exception {exception_id: $exception_id}) + MERGE (d)-[:GRANTED_EXCEPTION]->(e) + MERGE (e)-[:OVERRIDDEN_POLICY]->(p) + """ + self.graph_store.execute_query(query, { + "decision_id": decision_id, + "policy_id": policy_id, + "exception_id": exception_id + }) + + self.logger.info(f"Recorded policy exception: {exception_id}") + return exception_id + + if not hasattr(self.graph_store, "add_node") or not hasattr(self.graph_store, "add_edge"): + raise RuntimeError("Graph backend does not support exceptions") + + self.graph_store.add_node( + node_id=exception_id, + node_type="Exception", + content=reason, + exception_id=exception_id, + decision_id=decision_id, + policy_id=policy_id, + reason=reason, + created_at=datetime.now().isoformat() + ) + self.graph_store.add_edge(decision_id, exception_id, edge_type="GRANTED_EXCEPTION") + + policy = self.get_policy(policy_id) + if policy: + self.graph_store.add_edge(exception_id, f"{policy_id}:{policy.version}", edge_type="OVERRIDDEN_POLICY") + self.logger.info(f"Recorded policy exception: {exception_id}") return exception_id - except Exception as e: self.logger.error(f"Failed to record exception: {e}") raise @@ -389,23 +486,45 @@ class PolicyEngine: List of policy versions """ try: - query = """ - MATCH (p:Policy {policy_id: $policy_id}) - OPTIONAL MATCH (p)-[:VERSION_OF*]->(future:Policy) - WITH collect(p) + collect(future) as all_versions - UNWIND all_versions as version - RETURN DISTINCT version - ORDER BY version.updated_at - """ - results = self.graph_store.execute_query(query, {"policy_id": policy_id}) + if self._supports_cypher: + query = """ + MATCH (p:Policy {policy_id: $policy_id}) + OPTIONAL MATCH (p)-[:VERSION_OF*]->(future:Policy) + WITH collect(p) + collect(future) as all_versions + UNWIND all_versions as version + RETURN DISTINCT version + ORDER BY version.updated_at + """ + results = self.graph_store.execute_query(query, {"policy_id": policy_id}) - policies = [] - for record in results: - policy_data = record.get("version", {}) - policies.append(self._dict_to_policy(policy_data)) + policies = [] + for record in results: + policy_data = record.get("version", {}) + policies.append(self._dict_to_policy(policy_data)) - self.logger.info(f"Found {len(policies)} versions for policy {policy_id}") - return policies + self.logger.info(f"Found {len(policies)} versions for policy {policy_id}") + return policies + + if not hasattr(self.graph_store, "find_nodes"): + return [] + versions: List[Policy] = [] + for node in self.graph_store.find_nodes(node_type="Policy"): + data = node.get("metadata", {}) or {} + if data.get("policy_id") != policy_id: + continue + versions.append(self._dict_to_policy({ + "policy_id": data.get("policy_id"), + "name": data.get("name"), + "description": data.get("description"), + "rules": data.get("rules", {}), + "category": data.get("category"), + "version": data.get("version"), + "created_at": data.get("created_at"), + "updated_at": data.get("updated_at"), + "metadata": data.get("metadata", {}) + })) + versions.sort(key=lambda p: str(p.updated_at)) + return versions except Exception as e: self.logger.error(f"Failed to get policy history: {e}") @@ -429,23 +548,33 @@ class PolicyEngine: List of affected decision IDs """ try: - query = """ - MATCH (d:Decision)-[:APPLIED_POLICY]->(p:Policy { - policy_id: $policy_id, - version: $from_version - }) - RETURN d.decision_id as decision_id - """ - results = self.graph_store.execute_query(query, { - "policy_id": policy_id, - "from_version": from_version - }) + if self._supports_cypher: + query = """ + MATCH (d:Decision)-[:APPLIED_POLICY]->(p:Policy { + policy_id: $policy_id, + version: $from_version + }) + RETURN d.decision_id as decision_id + """ + results = self.graph_store.execute_query(query, { + "policy_id": policy_id, + "from_version": from_version + }) - decision_ids = [] - for record in results: - decision_ids.append(record.get("decision_id", "")) + decision_ids = [] + for record in results: + decision_ids.append(record.get("decision_id", "")) - self.logger.info(f"Found {len(decision_ids)} decisions affected by policy change") + self.logger.info(f"Found {len(decision_ids)} decisions affected by policy change") + return decision_ids + + if not hasattr(self.graph_store, "find_edges"): + return [] + policy_node_id = f"{policy_id}:{from_version}" + decision_ids: List[str] = [] + for edge in self.graph_store.find_edges(edge_type="APPLIED_POLICY"): + if edge.get("target") == policy_node_id: + decision_ids.append(edge.get("source")) return decision_ids except Exception as e: @@ -471,16 +600,38 @@ class PolicyEngine: current_policy = self.get_policy(policy_id) if not current_policy: raise ValueError(f"Policy {policy_id} not found") - - # Get decisions that used this policy - query = """ - MATCH (d:Decision)-[:APPLIED_POLICY]->(p:Policy {policy_id: $policy_id}) - RETURN d.decision_id as decision_id, d.confidence as confidence, - d.outcome as outcome, d.category as category - """ - results = self.graph_store.execute_query(query, {"policy_id": policy_id}) - - # Analyze impact + + results: List[Dict[str, Any]] = [] + if self._supports_cypher: + query = """ + MATCH (d:Decision)-[:APPLIED_POLICY]->(p:Policy {policy_id: $policy_id}) + RETURN d.decision_id as decision_id, d.confidence as confidence, + d.outcome as outcome, d.category as category + """ + results = self.graph_store.execute_query(query, {"policy_id": policy_id}) + else: + if hasattr(self.graph_store, "find_edges") and hasattr(self.graph_store, "nodes"): + for edge in self.graph_store.find_edges(edge_type="APPLIED_POLICY"): + target = edge.get("target") + if not target or not isinstance(target, str): + continue + policy_node = self.graph_store.nodes.get(target) + if not policy_node: + continue + props = getattr(policy_node, "properties", {}) or {} + if props.get("policy_id") != policy_id: + continue + decision_node = self.graph_store.nodes.get(edge.get("source")) + if not decision_node: + continue + dprops = getattr(decision_node, "properties", {}) or {} + results.append({ + "decision_id": edge.get("source"), + "confidence": dprops.get("confidence", 0.0), + "outcome": dprops.get("outcome", ""), + "category": dprops.get("category", "") + }) + impact_analysis = { "total_decisions": len(results), "affected_decisions": 0, @@ -488,19 +639,16 @@ class PolicyEngine: "risk_assessment": "low", "recommendations": [] } - + for record in results: decision_data = { "confidence": record.get("confidence", 0.0), "outcome": record.get("outcome", ""), "category": record.get("category", "") } - - # Check if decision would still comply with new rules would_comply = self._check_compliance_with_rules( decision_data, proposed_rules ) - if not would_comply: impact_analysis["affected_decisions"] += 1 @@ -546,29 +694,76 @@ class PolicyEngine: Policy object or None """ try: + if self._supports_cypher: + if version: + query = """ + MATCH (p:Policy {policy_id: $policy_id, version: $version}) + RETURN p + """ + params = {"policy_id": policy_id, "version": version} + else: + query = """ + MATCH (p:Policy {policy_id: $policy_id}) + WHERE NOT (p)-[:VERSION_OF]->(:Policy) + RETURN p + """ + params = {"policy_id": policy_id} + + results = self.graph_store.execute_query(query, params) + + if results: + policy_data = results[0].get("p", {}) + return self._dict_to_policy(policy_data) + return None + + if not hasattr(self.graph_store, "find_nodes"): + return None + + candidates: List[Dict[str, Any]] = [] + for node in self.graph_store.find_nodes(node_type="Policy"): + data = node.get("metadata", {}) or {} + if data.get("policy_id") != policy_id: + continue + if version and data.get("version") != version: + continue + candidates.append(data) + + if not candidates: + return None + if version: - query = """ - MATCH (p:Policy {policy_id: $policy_id, version: $version}) - RETURN p - """ - params = {"policy_id": policy_id, "version": version} + data = candidates[0] else: - # Get latest version - query = """ - MATCH (p:Policy {policy_id: $policy_id}) - WHERE NOT (p)-[:VERSION_OF]->(:Policy) - RETURN p - """ - params = {"policy_id": policy_id} - - results = self.graph_store.execute_query(query, params) - - if results: - policy_data = results[0].get("p", {}) - return self._dict_to_policy(policy_data) - - return None - + # Prefer highest semantic version if available, fallback to updated_at + def _version_key(v: str) -> tuple: + try: + parts = [int(p) for p in str(v).split(".")] + # Normalize length for comparison + while len(parts) < 3: + parts.append(-1) + return tuple(parts[:3]) + except Exception: + return (-1, -1, -1) + + try: + data = max( + candidates, + key=lambda d: (_version_key(d.get("version")), str(d.get("updated_at") or "")), + ) + except Exception: + data = max(candidates, key=lambda d: str(d.get("updated_at") or "")) + + return self._dict_to_policy({ + "policy_id": data.get("policy_id"), + "name": data.get("name"), + "description": data.get("description"), + "rules": data.get("rules", {}), + "category": data.get("category"), + "version": data.get("version"), + "created_at": data.get("created_at"), + "updated_at": data.get("updated_at"), + "metadata": data.get("metadata", {}) + }) except Exception as e: self.logger.error(f"Failed to get policy: {e}") return None diff --git a/semantica/kg/centrality_calculator.py b/semantica/kg/centrality_calculator.py index 95ee72f7..1fda57a7 100644 --- a/semantica/kg/centrality_calculator.py +++ b/semantica/kg/centrality_calculator.py @@ -739,6 +739,11 @@ class CentralityCalculator: neighbors = list(graph.neighbors(node)) elif hasattr(graph, 'get_neighbors'): neighbors = graph.get_neighbors(node) + if neighbors and isinstance(neighbors[0], dict): + neighbors = [ + n.get("id") for n in neighbors + if isinstance(n, dict) and n.get("id") + ] else: neighbors = [] diff --git a/semantica/kg/community_detector.py b/semantica/kg/community_detector.py index cd25eb88..2077e4e1 100644 --- a/semantica/kg/community_detector.py +++ b/semantica/kg/community_detector.py @@ -891,6 +891,12 @@ class CommunityDetector: all_neighbors = graph.get_neighbors(node) else: all_neighbors = [] + + if all_neighbors and isinstance(all_neighbors[0], dict): + all_neighbors = [ + n.get("id") for n in all_neighbors + if isinstance(n, dict) and n.get("id") + ] # Filter by relationship types if specified if relationship_types is not None and hasattr(graph, 'get_edge_data'): diff --git a/semantica/kg/link_predictor.py b/semantica/kg/link_predictor.py index 2144fa59..e811957b 100644 --- a/semantica/kg/link_predictor.py +++ b/semantica/kg/link_predictor.py @@ -455,6 +455,9 @@ class LinkPredictor: if hasattr(graph_store, 'neighbors'): return list(graph_store.neighbors(node_id)) elif hasattr(graph_store, 'get_neighbors'): - return graph_store.get_neighbors(node_id) + neighbors = graph_store.get_neighbors(node_id) + if neighbors and isinstance(neighbors[0], dict): + return [n.get("id") for n in neighbors if isinstance(n, dict) and n.get("id")] + return neighbors else: return [] diff --git a/semantica/kg/node_embeddings.py b/semantica/kg/node_embeddings.py index ca746f51..e4018279 100644 --- a/semantica/kg/node_embeddings.py +++ b/semantica/kg/node_embeddings.py @@ -350,13 +350,25 @@ class NodeEmbedder: # Build adjacency for node in nodes: - if hasattr(graph_store, 'get_neighbors'): - neighbors = graph_store.get_neighbors(node, relationship_types) - adjacency[node] = neighbors - else: - # Fallback for NetworkX - if hasattr(graph_store, 'neighbors'): - adjacency[node] = list(graph_store.neighbors(node)) + if hasattr(graph_store, 'neighbors'): + adjacency[node] = list(graph_store.neighbors(node)) + elif hasattr(graph_store, 'get_neighbor_ids'): + adjacency[node] = graph_store.get_neighbor_ids(node, relationship_types) + elif hasattr(graph_store, 'get_neighbors'): + try: + neighbor_details = graph_store.get_neighbors(node, hops=1, relationship_types=relationship_types) + adjacency[node] = [ + n.get("id") for n in neighbor_details + if isinstance(n, dict) and n.get("id") + ] + except TypeError: + neighbor_details = graph_store.get_neighbors(node) + adjacency[node] = [ + n.get("id") for n in neighbor_details + if isinstance(n, dict) and n.get("id") + ] + elif hasattr(graph_store, 'nodes') and hasattr(graph_store, 'edges'): + adjacency[node] = [] return dict(adjacency) diff --git a/tests/context/test_agent_context_smoke.py b/tests/context/test_agent_context_smoke.py new file mode 100644 index 00000000..d019ef20 --- /dev/null +++ b/tests/context/test_agent_context_smoke.py @@ -0,0 +1,66 @@ +import pytest +from semantica.context import AgentContext, ContextGraph +from semantica.context.decision_models import Policy +from semantica.vector_store import VectorStore +from datetime import datetime + + +def test_agent_context_minimal_decisions_and_chain(): + vs = VectorStore(backend="inmemory", dimension=64) + graph = ContextGraph() + ctx = AgentContext( + vector_store=vs, + knowledge_graph=graph, + enable_decision_tracking=True, + enable_kg_algorithms=False, + enable_vector_store_features=False, + ) + d1 = ctx.record_decision( + category="credit_approval", + scenario="s1", + reasoning="r1", + outcome="rejected", + confidence=0.8, + entities=["e1"], + decision_maker="tester", + ) + d2 = ctx.record_decision( + category="credit_approval", + scenario="s2", + reasoning="r2", + outcome="rejected", + confidence=0.85, + entities=["e1"], + decision_maker="tester", + ) + graph.add_causal_relationship(d1, d2, "INFLUENCED") + chain = ctx.get_causal_chain(d2, direction="upstream", max_depth=5) + assert isinstance(chain, list) + assert len(chain) >= 1 + + +def test_agent_context_policy_engine_with_graph_backend(): + vs = VectorStore(backend="inmemory", dimension=64) + graph = ContextGraph() + ctx = AgentContext( + vector_store=vs, + knowledge_graph=graph, + enable_decision_tracking=True, + enable_kg_algorithms=False, + enable_vector_store_features=False, + ) + pe = ctx.get_policy_engine() + pol = Policy( + policy_id="cp", + name="Credit Policy", + description="d", + rules={"min_confidence": 0.8, "allowed_outcomes": ["approved", "rejected"]}, + category="credit_approval", + version="1.0.0", + created_at=datetime.now(), + updated_at=datetime.now(), + metadata={}, + ) + pe.add_policy(pol) + found = pe.get_policy("cp") + assert found is not None diff --git a/tests/context/test_policy_engine_fallback.py b/tests/context/test_policy_engine_fallback.py new file mode 100644 index 00000000..bc26f5bf --- /dev/null +++ b/tests/context/test_policy_engine_fallback.py @@ -0,0 +1,62 @@ +import pytest +from datetime import datetime + +from semantica.context import ContextGraph, PolicyEngine +from semantica.context.decision_models import Policy, Decision + + +def _make_policy(pid="p1", version="1.0.0", min_conf=0.8): + return Policy( + policy_id=pid, + name="Test Policy", + description="desc", + rules={"min_confidence": min_conf, "allowed_outcomes": ["approved", "rejected"]}, + category="test", + version=version, + created_at=datetime.now(), + updated_at=datetime.now(), + metadata={}, + ) + + +def _make_decision(decision_id, conf=0.9, outcome="approved"): + return Decision( + decision_id=decision_id, + category="test", + scenario="s", + reasoning="r", + outcome=outcome, + confidence=conf, + timestamp=datetime.now(), + decision_maker="tester", + metadata={}, + ) + + +def test_policy_engine_add_get_update_with_context_graph(): + graph = ContextGraph() + engine = PolicyEngine(graph) + p = _make_policy() + engine.add_policy(p) + latest = engine.get_policy("p1") + assert latest is not None + assert latest.version == "1.0.0" + new_ver = engine.update_policy("p1", {"min_confidence": 0.85, "allowed_outcomes": ["approved", "rejected"]}, "raise min") + assert isinstance(new_ver, str) + latest2 = engine.get_policy("p1") + assert latest2 is not None + assert latest2.version != "1.0.0" + + +def test_policy_engine_compliance_and_application_edges(): + graph = ContextGraph() + engine = PolicyEngine(graph) + p = _make_policy() + engine.add_policy(p) + d = _make_decision("d1", conf=0.9, outcome="approved") + graph.add_decision(d) + ok = engine.check_compliance(d, "p1") + assert ok is True + engine.record_policy_application("d1", "p1", "1.0.0") + edges = graph.find_edges(edge_type="APPLIED_POLICY") + assert any(e.get("source") == "d1" and isinstance(e.get("target"), str) and e.get("target").startswith("p1:") for e in edges) From b90ffcca9aec13d8331b7092ffe30d837fa712e6 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Sat, 14 Feb 2026 18:02:06 +0530 Subject: [PATCH 02/50] context_compliance_fixes --- PR_CONTEXT.md | 89 ++++++++++++++++++++ README.md | 2 - semantica/context/agent_context.py | 70 +++++++++++---- semantica/context/context_graph.py | 7 +- tests/context/test_policy_engine_fallback.py | 28 +++--- 5 files changed, 167 insertions(+), 29 deletions(-) create mode 100644 PR_CONTEXT.md diff --git a/PR_CONTEXT.md b/PR_CONTEXT.md new file mode 100644 index 00000000..7edb829c --- /dev/null +++ b/PR_CONTEXT.md @@ -0,0 +1,89 @@ +# Context: PolicyEngine fixes, new context tests, cleanup — all tests passing + +## Summary +This PR improves Context Graph decision tracking reliability by fixing PolicyEngine behavior on non‑Cypher backends, resolving a compile issue, and adding focused tests. It also removes an example script per request. All tests pass. + +- Source branch: `context` +- Base branch: do not push directly to `main`; open PR for review +- Scope: Context graphs decision tracking and policy evaluation + +## Changes + +### Policy engine +- Fix: Correct indentation for Cypher branches to prevent compile errors. +- Fix: `get_policy` (non‑Cypher path) now selects the latest policy by semantic version first, falling back to `updated_at` as a secondary key. + +### Tests +- Added `tests/context/test_policy_engine_fallback.py` for ContextGraph (no Cypher) versioning, compliance, and `APPLIED_POLICY` edges. +- Added `tests/context/test_agent_context_smoke.py` for AgentContext + ContextGraph decision tracking (record decisions, causal chain, policy storage). + +### Cleanup +- Removed `examples/context_graphs_decision_tracking.py`. +- Deleted transient test artifacts from earlier local runs (not tracked). + +## Files Touched (high level) +- `semantica/context/policy_engine.py` +- `tests/context/test_policy_engine_fallback.py` (new) +- `tests/context/test_agent_context_smoke.py` (new) +- `examples/context_graphs_decision_tracking.py` (removed) + +## Rationale +- PolicyEngine compile error: Cypher code path indentation was causing a compile failure; fixed so it executes only on Cypher‑capable backends. +- Latest version retrieval: On the in‑memory ContextGraph backend, latest policy selection relied on `updated_at` strings and could return the wrong version. Updated to prefer semantic version order with robust fallback. + +## Test Coverage + +### New tests +- PolicyEngine fallback on ContextGraph: + - Adds/gets/updates policies and verifies the latest version is returned post‑update. + - Checks compliance and presence of `APPLIED_POLICY` edges. +- AgentContext + ContextGraph smoke: + - Records decisions, links causal relationship, validates upstream chain retrieval. + - Stores and retrieves policies via PolicyEngine obtained from AgentContext. + +### Existing tests (already in repo) cover +- ContextGraph decision nodes, precedents, and edge cases. +- Banking/Healthcare end‑to‑end flows. +- Context retriever hybrid/precedents and AgentContext integration. + +## Results +- Context-only tests: + - `python -m pytest -q tests/context` → passed +- Full test suite: + - `python -m pytest -q tests` → passed +- Example subset (smoke): + - `tests/context/test_agent_context_smoke.py` → passed (deprecation warnings are expected) + - `tests/context/test_policy_engine_fallback.py` → passed after fixes + +## How to Test Locally + +```bash +# Optional: fresh venv +python -m venv .venv +.\.venv\Scripts\Activate.ps1 + +# Install dev deps (includes pytest) +pip install -e ".[dev]" + +# Run context suite +python -m pytest -q tests\context + +# Run full suite +python -m pytest -q tests +``` + +## Backwards Compatibility & Risk +- No breaking API changes; fixes are internal behavior corrections for non‑Cypher PolicyEngine operations. +- Risk is low; covered by new tests and full suite. + +## Follow‑ups (optional) +- Add stricter warnings policy if desired (e.g., `filterwarnings` in `pyproject.toml`/`pytest.ini`). +- Expand negative tests (e.g., cycles in causal traversal, invalid rule sets). + +## Checklist +- [x] No direct pushes to `main`. +- [x] New tests added and passing locally. +- [x] No secrets or credentials added. +- [x] Example file removed per request. +- [x] Clear rationale and instructions provided. + diff --git a/README.md b/README.md index 66792a8f..6fce0ca0 100644 --- a/README.md +++ b/README.md @@ -763,8 +763,6 @@ precedents = context.find_precedents( ) ``` -Runnable script: `examples/context_graphs_decision_tracking.py` - **Core Notebooks:** - [**Context Module Introduction**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/19_Context_Module.ipynb) - Basic memory and storage. - [**Advanced Context Engineering**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/advanced/11_Advanced_Context_Engineering.ipynb) - Hybrid retrieval, graph builders, and custom memory policies. diff --git a/semantica/context/agent_context.py b/semantica/context/agent_context.py index 0fdde432..cfead678 100644 --- a/semantica/context/agent_context.py +++ b/semantica/context/agent_context.py @@ -238,7 +238,9 @@ class AgentContext: self._policy_engine = PolicyEngine(knowledge_graph) self.logger.info("Enhanced decision tracking components initialized successfully") except Exception as e: - self.logger.warning(f"Failed to initialize enhanced decision tracking: {e}") + self.logger.warning( + f"Failed to initialize enhanced decision tracking ({type(e).__name__})" + ) self._decision_recorder = DecisionRecorder(knowledge_graph) self._decision_query = DecisionQuery(knowledge_graph) self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) @@ -254,7 +256,9 @@ class AgentContext: use_graph_features=enable_kg_algorithms ) except Exception as e: - self.logger.warning(f"Failed to initialize decision pipeline: {e}") + self.logger.warning( + f"Failed to initialize decision pipeline ({type(e).__name__})" + ) @property def memory(self) -> AgentMemory: @@ -756,7 +760,7 @@ class AgentContext: "edge_count": graph.get("statistics", {}).get("edge_count", 0), } except Exception as e: - self.logger.warning(f"Failed to build graph from documents: {e}") + self.logger.warning(f"Failed to build graph from documents ({type(e).__name__})") return {"node_count": 0, "edge_count": 0} def _context_to_dict( @@ -1457,7 +1461,7 @@ class AgentContext: if memory_id: imported += 1 except Exception as e: - self.logger.warning(f"Failed to import memory: {e}") + self.logger.warning(f"Failed to import memory ({type(e).__name__})") return imported @@ -1590,13 +1594,30 @@ class AgentContext: self.knowledge_graph.add_node_attribute( decision.decision_id, {"cross_system_context": cross_system_context} ) + about_edge_failures: List[Dict[str, str]] = [] for entity_id in entities: try: self.knowledge_graph.add_edge(decision.decision_id, entity_id, edge_type="ABOUT") - except Exception: - continue + except Exception as e: + about_edge_failures.append( + {"entity_id": str(entity_id), "error_type": type(e).__name__} + ) + + if about_edge_failures: + failure_types = sorted( + {f.get("error_type", "") for f in about_edge_failures if f.get("error_type")} + ) + self.logger.warning( + f"record_decision ABOUT edge creation failures: {len(about_edge_failures)} " + f"({', '.join(failure_types) if failure_types else 'unknown'})" + ) + if hasattr(self.knowledge_graph, "add_node_attribute"): + self.knowledge_graph.add_node_attribute( + decision.decision_id, {"about_edge_failures": about_edge_failures} + ) vector_id = None + vector_store_error_type: Optional[str] = None if hasattr(self.vector_store, "store_decision"): try: vector_id = self.vector_store.store_decision( @@ -1610,8 +1631,17 @@ class AgentContext: decision_maker=decision.decision_maker, timestamp=decision.timestamp.isoformat() ) - except Exception: + except Exception as e: vector_id = None + vector_store_error_type = type(e).__name__ + self.logger.warning( + f"record_decision vector store write failed ({vector_store_error_type})" + ) + if hasattr(self.knowledge_graph, "add_node_attribute"): + self.knowledge_graph.add_node_attribute( + decision.decision_id, + {"vector_store_error_type": vector_store_error_type}, + ) if vector_id and hasattr(self.knowledge_graph, "add_node_attribute"): self.knowledge_graph.add_node_attribute(decision.decision_id, {"vector_id": vector_id}) @@ -1661,6 +1691,16 @@ class AgentContext: results: List[Decision] = [] + def _safe_parse_timestamp(value: Any) -> datetime: + if isinstance(value, datetime): + return value + if not value: + return datetime.now() + try: + return datetime.fromisoformat(str(value)) + except Exception: + return datetime.now() + if use_hybrid_search and hasattr(self.vector_store, "search_decisions"): filters = {"category": category} if category else None vector_results = self.vector_store.search_decisions( @@ -1683,7 +1723,7 @@ class AgentContext: reasoning=data.get("reasoning", ""), outcome=data.get("outcome", ""), confidence=float(data.get("confidence", 0.0) or 0.0), - timestamp=datetime.fromisoformat(data.get("timestamp")) if data.get("timestamp") else datetime.now(), + timestamp=_safe_parse_timestamp(data.get("timestamp")), decision_maker=data.get("decision_maker", "ai_agent"), reasoning_embedding=data.get("reasoning_embedding"), node2vec_embedding=data.get("node2vec_embedding"), @@ -1711,7 +1751,7 @@ class AgentContext: reasoning=data.get("reasoning", ""), outcome=data.get("outcome", ""), confidence=float(data.get("confidence", 0.0) or 0.0), - timestamp=datetime.fromisoformat(data.get("timestamp")) if data.get("timestamp") else datetime.now(), + timestamp=_safe_parse_timestamp(data.get("timestamp")), decision_maker=data.get("decision_maker", "ai_agent"), reasoning_embedding=data.get("reasoning_embedding"), node2vec_embedding=data.get("node2vec_embedding"), @@ -1966,7 +2006,7 @@ class AgentContext: "message": "Basic analysis only - KG features not available" } except Exception as e: - self.logger.error(f"Failed to analyze context graph: {e}") + self.logger.error(f"Failed to analyze context graph ({type(e).__name__})") return {"error": str(e)} def find_similar_entities( @@ -1993,7 +2033,7 @@ class AgentContext: # Fallback to basic content similarity return [] except Exception as e: - self.logger.error(f"Failed to find similar entities: {e}") + self.logger.error(f"Failed to find similar entities ({type(e).__name__})") return [] def get_entity_centrality(self, entity_id: str) -> Dict[str, float]: @@ -2015,7 +2055,7 @@ class AgentContext: else: return {"error": "Centrality analysis not available"} except Exception as e: - self.logger.error(f"Failed to get entity centrality: {e}") + self.logger.error(f"Failed to get entity centrality ({type(e).__name__})") return {"error": str(e)} def find_precedents_advanced( @@ -2055,7 +2095,7 @@ class AgentContext: # Fallback to basic method return self.find_precedents(scenario, category, limit) except Exception as e: - self.logger.error(f"Failed to find advanced precedents: {e}") + self.logger.error(f"Failed to find advanced precedents ({type(e).__name__})") return [] def analyze_decision_influence(self, decision_id: str, max_depth: int = 3) -> Dict[str, Any]: @@ -2084,7 +2124,7 @@ class AgentContext: "message": "Basic analysis only - KG features not available" } except Exception as e: - self.logger.error(f"Failed to analyze decision influence: {e}") + self.logger.error(f"Failed to analyze decision influence ({type(e).__name__})") return {"error": str(e)} def predict_decision_relationships(self, decision_id: str, top_k: int = 5) -> List[Dict]: @@ -2107,7 +2147,7 @@ class AgentContext: else: return [] except Exception as e: - self.logger.error(f"Failed to predict decision relationships: {e}") + self.logger.error(f"Failed to predict decision relationships ({type(e).__name__})") return [] def get_context_insights(self) -> Dict[str, Any]: diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 6e201a80..abf35213 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -509,11 +509,14 @@ class ContextGraph: """Find a node by ID.""" node = self.nodes.get(node_id) if node: + merged_metadata = {} + merged_metadata.update(getattr(node, "metadata", {}) or {}) + merged_metadata.update(getattr(node, "properties", {}) or {}) return { "id": node.node_id, "type": node.node_type, "content": node.content, - "metadata": node.metadata, + "metadata": merged_metadata, } return None @@ -530,7 +533,7 @@ class ContextGraph: "id": n.node_id, "type": n.node_type, "content": n.content, - "metadata": n.metadata, + "metadata": {**(getattr(n, "metadata", {}) or {}), **(getattr(n, "properties", {}) or {})}, } for n in nodes ] diff --git a/tests/context/test_policy_engine_fallback.py b/tests/context/test_policy_engine_fallback.py index bc26f5bf..6c163e09 100644 --- a/tests/context/test_policy_engine_fallback.py +++ b/tests/context/test_policy_engine_fallback.py @@ -1,4 +1,3 @@ -import pytest from datetime import datetime from semantica.context import ContextGraph, PolicyEngine @@ -36,12 +35,16 @@ def _make_decision(decision_id, conf=0.9, outcome="approved"): def test_policy_engine_add_get_update_with_context_graph(): graph = ContextGraph() engine = PolicyEngine(graph) - p = _make_policy() - engine.add_policy(p) + policy = _make_policy() + engine.add_policy(policy) latest = engine.get_policy("p1") assert latest is not None assert latest.version == "1.0.0" - new_ver = engine.update_policy("p1", {"min_confidence": 0.85, "allowed_outcomes": ["approved", "rejected"]}, "raise min") + new_ver = engine.update_policy( + "p1", + {"min_confidence": 0.85, "allowed_outcomes": ["approved", "rejected"]}, + "raise min", + ) assert isinstance(new_ver, str) latest2 = engine.get_policy("p1") assert latest2 is not None @@ -51,12 +54,17 @@ def test_policy_engine_add_get_update_with_context_graph(): def test_policy_engine_compliance_and_application_edges(): graph = ContextGraph() engine = PolicyEngine(graph) - p = _make_policy() - engine.add_policy(p) - d = _make_decision("d1", conf=0.9, outcome="approved") - graph.add_decision(d) - ok = engine.check_compliance(d, "p1") + policy = _make_policy() + engine.add_policy(policy) + decision = _make_decision("d1", conf=0.9, outcome="approved") + graph.add_decision(decision) + ok = engine.check_compliance(decision, "p1") assert ok is True engine.record_policy_application("d1", "p1", "1.0.0") edges = graph.find_edges(edge_type="APPLIED_POLICY") - assert any(e.get("source") == "d1" and isinstance(e.get("target"), str) and e.get("target").startswith("p1:") for e in edges) + assert any( + e.get("source") == "d1" + and isinstance(e.get("target"), str) + and e.get("target").startswith("p1:") + for e in edges + ) From 47c0058dce1c880c9541aed685797963dae9f83e Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Sat, 14 Feb 2026 18:04:37 +0530 Subject: [PATCH 03/50] Delete PR_CONTEXT.md --- PR_CONTEXT.md | 89 --------------------------------------------------- 1 file changed, 89 deletions(-) delete mode 100644 PR_CONTEXT.md diff --git a/PR_CONTEXT.md b/PR_CONTEXT.md deleted file mode 100644 index 7edb829c..00000000 --- a/PR_CONTEXT.md +++ /dev/null @@ -1,89 +0,0 @@ -# Context: PolicyEngine fixes, new context tests, cleanup — all tests passing - -## Summary -This PR improves Context Graph decision tracking reliability by fixing PolicyEngine behavior on non‑Cypher backends, resolving a compile issue, and adding focused tests. It also removes an example script per request. All tests pass. - -- Source branch: `context` -- Base branch: do not push directly to `main`; open PR for review -- Scope: Context graphs decision tracking and policy evaluation - -## Changes - -### Policy engine -- Fix: Correct indentation for Cypher branches to prevent compile errors. -- Fix: `get_policy` (non‑Cypher path) now selects the latest policy by semantic version first, falling back to `updated_at` as a secondary key. - -### Tests -- Added `tests/context/test_policy_engine_fallback.py` for ContextGraph (no Cypher) versioning, compliance, and `APPLIED_POLICY` edges. -- Added `tests/context/test_agent_context_smoke.py` for AgentContext + ContextGraph decision tracking (record decisions, causal chain, policy storage). - -### Cleanup -- Removed `examples/context_graphs_decision_tracking.py`. -- Deleted transient test artifacts from earlier local runs (not tracked). - -## Files Touched (high level) -- `semantica/context/policy_engine.py` -- `tests/context/test_policy_engine_fallback.py` (new) -- `tests/context/test_agent_context_smoke.py` (new) -- `examples/context_graphs_decision_tracking.py` (removed) - -## Rationale -- PolicyEngine compile error: Cypher code path indentation was causing a compile failure; fixed so it executes only on Cypher‑capable backends. -- Latest version retrieval: On the in‑memory ContextGraph backend, latest policy selection relied on `updated_at` strings and could return the wrong version. Updated to prefer semantic version order with robust fallback. - -## Test Coverage - -### New tests -- PolicyEngine fallback on ContextGraph: - - Adds/gets/updates policies and verifies the latest version is returned post‑update. - - Checks compliance and presence of `APPLIED_POLICY` edges. -- AgentContext + ContextGraph smoke: - - Records decisions, links causal relationship, validates upstream chain retrieval. - - Stores and retrieves policies via PolicyEngine obtained from AgentContext. - -### Existing tests (already in repo) cover -- ContextGraph decision nodes, precedents, and edge cases. -- Banking/Healthcare end‑to‑end flows. -- Context retriever hybrid/precedents and AgentContext integration. - -## Results -- Context-only tests: - - `python -m pytest -q tests/context` → passed -- Full test suite: - - `python -m pytest -q tests` → passed -- Example subset (smoke): - - `tests/context/test_agent_context_smoke.py` → passed (deprecation warnings are expected) - - `tests/context/test_policy_engine_fallback.py` → passed after fixes - -## How to Test Locally - -```bash -# Optional: fresh venv -python -m venv .venv -.\.venv\Scripts\Activate.ps1 - -# Install dev deps (includes pytest) -pip install -e ".[dev]" - -# Run context suite -python -m pytest -q tests\context - -# Run full suite -python -m pytest -q tests -``` - -## Backwards Compatibility & Risk -- No breaking API changes; fixes are internal behavior corrections for non‑Cypher PolicyEngine operations. -- Risk is low; covered by new tests and full suite. - -## Follow‑ups (optional) -- Add stricter warnings policy if desired (e.g., `filterwarnings` in `pyproject.toml`/`pytest.ini`). -- Expand negative tests (e.g., cycles in causal traversal, invalid rule sets). - -## Checklist -- [x] No direct pushes to `main`. -- [x] New tests added and passing locally. -- [x] No secrets or credentials added. -- [x] Example file removed per request. -- [x] Clear rationale and instructions provided. - From 5d712d5a627773687412723bc98f5a879e735420 Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Sat, 14 Feb 2026 18:25:24 +0530 Subject: [PATCH 04/50] =?UTF-8?q?Context:=20PolicyEngine=20fixes,=20new=20?= =?UTF-8?q?context=20tests,=20cleanup=20=E2=80=94=20all=20tests=20passing?= =?UTF-8?q?=20(#312)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * context_fixes * context_compliance_fixes * Delete PR_CONTEXT.md --- README.md | 29 + semantica/context/agent_context.py | 305 ++++++++--- semantica/context/causal_analyzer.py | 9 +- semantica/context/context_graph.py | 112 +++- semantica/context/context_retriever.py | 27 +- semantica/context/policy_engine.py | 531 +++++++++++++------ semantica/kg/centrality_calculator.py | 5 + semantica/kg/community_detector.py | 6 + semantica/kg/link_predictor.py | 5 +- semantica/kg/node_embeddings.py | 26 +- tests/context/test_agent_context_smoke.py | 66 +++ tests/context/test_policy_engine_fallback.py | 70 +++ 12 files changed, 904 insertions(+), 287 deletions(-) create mode 100644 tests/context/test_agent_context_smoke.py create mode 100644 tests/context/test_policy_engine_fallback.py diff --git a/README.md b/README.md index dee65814..6fce0ca0 100644 --- a/README.md +++ b/README.md @@ -734,6 +734,35 @@ reasoned_result = context.query_with_reasoning( - **ContextRetriever**: Performs hybrid retrieval combining vector search, graph traversal, and memory for optimal context relevance - **AgentContext**: High-level interface integrating Context Graph and Context Retriever for GraphRAG applications +#### Context Graphs: Decision Tracking + +```python +from semantica.context import AgentContext, ContextGraph +from semantica.vector_store import VectorStore + +context = AgentContext( + vector_store=VectorStore(backend="inmemory", dimension=128), + knowledge_graph=ContextGraph(), + enable_decision_tracking=True, + enable_kg_algorithms=False, # semantic-only precedent search +) + +decision_id = context.record_decision( + category="credit_approval", + scenario="High-risk credit limit increase", + reasoning="Recent velocity-check failure and prior fraud flag", + outcome="rejected", + confidence=0.78, + entities=["customer:jessica_norris"], +) + +precedents = context.find_precedents( + scenario="High-risk customer credit increase", + category="credit_approval", + limit=5, +) +``` + **Core Notebooks:** - [**Context Module Introduction**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/introduction/19_Context_Module.ipynb) - Basic memory and storage. - [**Advanced Context Engineering**](https://github.com/Hawksight-AI/semantica/tree/main/cookbook/advanced/11_Advanced_Context_Engineering.ipynb) - Hybrid retrieval, graph builders, and custom memory policies. diff --git a/semantica/context/agent_context.py b/semantica/context/agent_context.py index 6a2ecaaa..cfead678 100644 --- a/semantica/context/agent_context.py +++ b/semantica/context/agent_context.py @@ -209,62 +209,56 @@ class AgentContext: if knowledge_graph and hasattr(knowledge_graph, "build_from_conversations"): self._graph_builder = knowledge_graph - # Store config - self.config = { + self.config.update({ "retention_days": retention_days, "max_memories": max_memories, - "use_graph_expansion": use_graph_expansion, - "max_expansion_hops": max_expansion_hops, - "hybrid_alpha": hybrid_alpha, - "enable_decision_tracking": enable_decision_tracking, - } + }) # Initialize decision tracking components if enabled + self._decision_backend = None self._decision_recorder = None self._decision_query = None self._causal_analyzer = None self._policy_engine = None if enable_decision_tracking and knowledge_graph: - # Validate that knowledge_graph supports required GraphStore interface - if not hasattr(knowledge_graph, 'execute_query'): - self.logger.error( - "Decision tracking requires a GraphStore-compatible knowledge graph with execute_query() method. " - "Provided knowledge_graph type does not support Cypher queries. " - "Use GraphStore (Neo4j, FalkorDB) or disable decision tracking." - ) - raise ValueError( - "Decision tracking requires a GraphStore-compatible knowledge graph. " - "The provided knowledge_graph does not have an execute_query() method. " - "For decision tracking, use a GraphStore backend (Neo4j, FalkorDB) " - "or set enable_decision_tracking=False." - ) - - # Initialize enhanced decision tracking components - try: - self._decision_recorder = DecisionRecorder(knowledge_graph) - - # Enhanced DecisionQuery with KG and vector store integration - self._decision_query = DecisionQuery( - graph_store=knowledge_graph, - vector_store=vector_store if enable_vector_store_features else None, - enable_advanced_analytics=enable_advanced_analytics, - enable_centrality_analysis=enable_kg_algorithms, - enable_community_detection=enable_kg_algorithms, - enable_node_embeddings=enable_kg_algorithms - ) - - self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) + if hasattr(knowledge_graph, "execute_query"): + self._decision_backend = "graph_store" + try: + self._decision_recorder = DecisionRecorder(knowledge_graph) + self._decision_query = DecisionQuery( + graph_store=knowledge_graph, + vector_store=vector_store if enable_vector_store_features else None, + enable_advanced_analytics=enable_advanced_analytics, + enable_centrality_analysis=enable_kg_algorithms, + enable_community_detection=enable_kg_algorithms, + enable_node_embeddings=enable_kg_algorithms + ) + self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) + self._policy_engine = PolicyEngine(knowledge_graph) + self.logger.info("Enhanced decision tracking components initialized successfully") + except Exception as e: + self.logger.warning( + f"Failed to initialize enhanced decision tracking ({type(e).__name__})" + ) + self._decision_recorder = DecisionRecorder(knowledge_graph) + self._decision_query = DecisionQuery(knowledge_graph) + self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) + self._policy_engine = PolicyEngine(knowledge_graph) + else: + self._decision_backend = "context_graph" self._policy_engine = PolicyEngine(knowledge_graph) - - self.logger.info("Enhanced decision tracking components initialized successfully") - except Exception as e: - self.logger.warning(f"Failed to initialize enhanced decision tracking: {e}") - # Fallback to basic components - self._decision_recorder = DecisionRecorder(knowledge_graph) - self._decision_query = DecisionQuery(knowledge_graph) self._causal_analyzer = CausalChainAnalyzer(knowledge_graph) - self._policy_engine = PolicyEngine(knowledge_graph) + if enable_vector_store_features and hasattr(self.vector_store, "initialize_decision_pipeline"): + try: + self.vector_store.initialize_decision_pipeline( + graph_store=knowledge_graph if enable_kg_algorithms else None, + use_graph_features=enable_kg_algorithms + ) + except Exception as e: + self.logger.warning( + f"Failed to initialize decision pipeline ({type(e).__name__})" + ) @property def memory(self) -> AgentMemory: @@ -766,7 +760,7 @@ class AgentContext: "edge_count": graph.get("statistics", {}).get("edge_count", 0), } except Exception as e: - self.logger.warning(f"Failed to build graph from documents: {e}") + self.logger.warning(f"Failed to build graph from documents ({type(e).__name__})") return {"node_count": 0, "edge_count": 0} def _context_to_dict( @@ -1467,7 +1461,7 @@ class AgentContext: if memory_id: imported += 1 except Exception as e: - self.logger.warning(f"Failed to import memory: {e}") + self.logger.warning(f"Failed to import memory ({type(e).__name__})") return imported @@ -1560,7 +1554,7 @@ class AgentContext: Raises: RuntimeError: If decision tracking is not enabled """ - if not self._decision_recorder: + if not self._decision_backend: raise RuntimeError("Decision tracking is not enabled") from .decision_models import Decision @@ -1579,18 +1573,80 @@ class AgentContext: entities = entities or [] source_documents = [] # Could be enhanced to capture source docs - - decision_id = self._decision_recorder.record_decision( - decision, entities, source_documents - ) - - # Capture cross-system context if provided - if cross_system_context: - self._decision_recorder.capture_cross_system_context( - decision_id, cross_system_context + + if self._decision_backend == "graph_store": + decision_id = self._decision_recorder.record_decision( + decision, entities, source_documents ) - - return decision_id + + if cross_system_context: + self._decision_recorder.capture_cross_system_context( + decision_id, cross_system_context + ) + + return decision_id + + if not hasattr(self.knowledge_graph, "add_decision"): + raise RuntimeError("Decision tracking backend does not support decisions") + + self.knowledge_graph.add_decision(decision) + if cross_system_context and hasattr(self.knowledge_graph, "add_node_attribute"): + self.knowledge_graph.add_node_attribute( + decision.decision_id, {"cross_system_context": cross_system_context} + ) + about_edge_failures: List[Dict[str, str]] = [] + for entity_id in entities: + try: + self.knowledge_graph.add_edge(decision.decision_id, entity_id, edge_type="ABOUT") + except Exception as e: + about_edge_failures.append( + {"entity_id": str(entity_id), "error_type": type(e).__name__} + ) + + if about_edge_failures: + failure_types = sorted( + {f.get("error_type", "") for f in about_edge_failures if f.get("error_type")} + ) + self.logger.warning( + f"record_decision ABOUT edge creation failures: {len(about_edge_failures)} " + f"({', '.join(failure_types) if failure_types else 'unknown'})" + ) + if hasattr(self.knowledge_graph, "add_node_attribute"): + self.knowledge_graph.add_node_attribute( + decision.decision_id, {"about_edge_failures": about_edge_failures} + ) + + vector_id = None + vector_store_error_type: Optional[str] = None + if hasattr(self.vector_store, "store_decision"): + try: + vector_id = self.vector_store.store_decision( + scenario=scenario, + reasoning=reasoning, + outcome=outcome, + confidence=confidence, + entities=entities, + category=category, + decision_id=decision.decision_id, + decision_maker=decision.decision_maker, + timestamp=decision.timestamp.isoformat() + ) + except Exception as e: + vector_id = None + vector_store_error_type = type(e).__name__ + self.logger.warning( + f"record_decision vector store write failed ({vector_store_error_type})" + ) + if hasattr(self.knowledge_graph, "add_node_attribute"): + self.knowledge_graph.add_node_attribute( + decision.decision_id, + {"vector_store_error_type": vector_store_error_type}, + ) + + if vector_id and hasattr(self.knowledge_graph, "add_node_attribute"): + self.knowledge_graph.add_node_attribute(decision.decision_id, {"vector_id": vector_id}) + + return decision.decision_id def find_precedents( self, @@ -1618,24 +1674,95 @@ class AgentContext: Raises: RuntimeError: If decision tracking is not enabled """ - if not self._decision_query: + if not self._decision_backend: raise RuntimeError("Decision tracking is not enabled") - - if use_hybrid_search: - try: - return self._decision_query.find_precedents_hybrid( - scenario, category, limit - ) - except Exception: - # Fallback to basic search if hybrid fails - return self._decision_query._find_precedents_basic(scenario, category, limit) - else: - # Simple category-based search + + if self._decision_backend == "graph_store": + if use_hybrid_search: + try: + return self._decision_query.find_precedents_hybrid( + scenario, category, limit + ) + except Exception: + return self._decision_query._find_precedents_basic(scenario, category, limit) if category: return self._decision_query.find_by_category(category, limit) - else: - # Use basic search - return self._decision_query._find_precedents_basic(scenario, category, limit) + return self._decision_query._find_precedents_basic(scenario, category, limit) + + results: List[Decision] = [] + + def _safe_parse_timestamp(value: Any) -> datetime: + if isinstance(value, datetime): + return value + if not value: + return datetime.now() + try: + return datetime.fromisoformat(str(value)) + except Exception: + return datetime.now() + + if use_hybrid_search and hasattr(self.vector_store, "search_decisions"): + filters = {"category": category} if category else None + vector_results = self.vector_store.search_decisions( + query=scenario, + filters=filters, + limit=limit, + use_hybrid_search=True + ) + for r in vector_results: + meta = r.get("metadata") or {} + decision_id = meta.get("decision_id") or meta.get("id") + if decision_id and hasattr(self.knowledge_graph, "nodes") and decision_id in self.knowledge_graph.nodes: + node = self.knowledge_graph.nodes[decision_id] + if getattr(node, "node_type", None) == "Decision": + data = getattr(node, "properties", {}) or {} + decision = Decision( + decision_id=decision_id, + category=data.get("category", ""), + scenario=getattr(node, "content", ""), + reasoning=data.get("reasoning", ""), + outcome=data.get("outcome", ""), + confidence=float(data.get("confidence", 0.0) or 0.0), + timestamp=_safe_parse_timestamp(data.get("timestamp")), + decision_maker=data.get("decision_maker", "ai_agent"), + reasoning_embedding=data.get("reasoning_embedding"), + node2vec_embedding=data.get("node2vec_embedding"), + metadata={k: v for k, v in data.items() if k not in [ + "category", "reasoning", "outcome", "confidence", + "timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding" + ]} + ) + decision.metadata["score"] = r.get("score") + results.append(decision) + + if results: + return results[:limit] + + if hasattr(self.knowledge_graph, "find_nodes"): + for node in self.knowledge_graph.find_nodes(node_type="Decision"): + if category and node.get("metadata", {}).get("category") != category: + continue + data = node.get("metadata", {}) or {} + results.append( + Decision( + decision_id=node.get("id", ""), + category=data.get("category", ""), + scenario=node.get("content", ""), + reasoning=data.get("reasoning", ""), + outcome=data.get("outcome", ""), + confidence=float(data.get("confidence", 0.0) or 0.0), + timestamp=_safe_parse_timestamp(data.get("timestamp")), + decision_maker=data.get("decision_maker", "ai_agent"), + reasoning_embedding=data.get("reasoning_embedding"), + node2vec_embedding=data.get("node2vec_embedding"), + metadata={k: v for k, v in data.items() if k not in [ + "category", "reasoning", "outcome", "confidence", + "timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding" + ]} + ) + ) + + return results[:limit] def get_causal_chain( self, @@ -1657,12 +1784,22 @@ class AgentContext: Raises: RuntimeError: If decision tracking is not enabled """ - if not self._causal_analyzer: + if not self._decision_backend: raise RuntimeError("Decision tracking is not enabled") - - return self._causal_analyzer.get_causal_chain( - decision_id, direction, max_depth - ) + + if self._decision_backend == "graph_store": + return self._causal_analyzer.get_causal_chain( + decision_id, direction, max_depth + ) + + if hasattr(self.knowledge_graph, "get_causal_chain"): + return self.knowledge_graph.get_causal_chain( + decision_id=decision_id, + direction=direction, + max_depth=max_depth + ) + + raise RuntimeError("Decision tracking backend does not support causal chains") def get_policy_engine(self) -> PolicyEngine: """ @@ -1869,7 +2006,7 @@ class AgentContext: "message": "Basic analysis only - KG features not available" } except Exception as e: - self.logger.error(f"Failed to analyze context graph: {e}") + self.logger.error(f"Failed to analyze context graph ({type(e).__name__})") return {"error": str(e)} def find_similar_entities( @@ -1896,7 +2033,7 @@ class AgentContext: # Fallback to basic content similarity return [] except Exception as e: - self.logger.error(f"Failed to find similar entities: {e}") + self.logger.error(f"Failed to find similar entities ({type(e).__name__})") return [] def get_entity_centrality(self, entity_id: str) -> Dict[str, float]: @@ -1918,7 +2055,7 @@ class AgentContext: else: return {"error": "Centrality analysis not available"} except Exception as e: - self.logger.error(f"Failed to get entity centrality: {e}") + self.logger.error(f"Failed to get entity centrality ({type(e).__name__})") return {"error": str(e)} def find_precedents_advanced( @@ -1958,7 +2095,7 @@ class AgentContext: # Fallback to basic method return self.find_precedents(scenario, category, limit) except Exception as e: - self.logger.error(f"Failed to find advanced precedents: {e}") + self.logger.error(f"Failed to find advanced precedents ({type(e).__name__})") return [] def analyze_decision_influence(self, decision_id: str, max_depth: int = 3) -> Dict[str, Any]: @@ -1987,7 +2124,7 @@ class AgentContext: "message": "Basic analysis only - KG features not available" } except Exception as e: - self.logger.error(f"Failed to analyze decision influence: {e}") + self.logger.error(f"Failed to analyze decision influence ({type(e).__name__})") return {"error": str(e)} def predict_decision_relationships(self, decision_id: str, top_k: int = 5) -> List[Dict]: @@ -2010,7 +2147,7 @@ class AgentContext: else: return [] except Exception as e: - self.logger.error(f"Failed to predict decision relationships: {e}") + self.logger.error(f"Failed to predict decision relationships ({type(e).__name__})") return [] def get_context_insights(self) -> Dict[str, Any]: diff --git a/semantica/context/causal_analyzer.py b/semantica/context/causal_analyzer.py index ab53699c..9ad64215 100644 --- a/semantica/context/causal_analyzer.py +++ b/semantica/context/causal_analyzer.py @@ -77,7 +77,7 @@ class CausalChainAnalyzer: using graph traversal. """ - def __init__(self, graph_store: GraphStore): + def __init__(self, graph_store: Any): """ Initialize CausalChainAnalyzer. @@ -105,6 +105,13 @@ class CausalChainAnalyzer: List of decisions in causal chain """ try: + if hasattr(self.graph_store, "get_causal_chain") and not hasattr(self.graph_store, "execute_query"): + return self.graph_store.get_causal_chain( + decision_id=decision_id, + direction=direction, + max_depth=max_depth + ) + if direction not in ["upstream", "downstream"]: raise ValueError("Direction must be 'upstream' or 'downstream'") diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index 4084391c..abf35213 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -260,7 +260,69 @@ class ContextGraph: count += 1 return count - def get_neighbors(self, node_id: str, hops: int = 1) -> List[Dict[str, Any]]: + def __contains__(self, node_id: object) -> bool: + if not isinstance(node_id, str): + return False + return node_id in self.nodes + + def has_node(self, node_id: str) -> bool: + return node_id in self.nodes + + def neighbors(self, node_id: str) -> List[str]: + return self.get_neighbor_ids(node_id) + + def get_neighbor_ids( + self, + node_id: str, + relationship_types: Optional[List[str]] = None, + ) -> List[str]: + if node_id not in self.nodes: + return [] + + rel_filter = set(relationship_types) if relationship_types else None + neighbor_ids: List[str] = [] + for edge in self._adjacency.get(node_id, []): + if rel_filter is None or edge.edge_type in rel_filter: + neighbor_ids.append(edge.target_id) + return neighbor_ids + + def get_nodes_by_label(self, label: str) -> List[str]: + return list(self.node_type_index.get(label, set())) + + def get_node_property(self, node_id: str, property_name: str) -> Any: + node = self.nodes.get(node_id) + if not node: + return None + return node.properties.get(property_name) + + def get_node_attributes(self, node_id: str) -> Dict[str, Any]: + node = self.nodes.get(node_id) + if not node: + return {} + return node.properties.copy() + + def add_node_attribute(self, node_id: str, attributes: Dict[str, Any]) -> None: + node = self.nodes.get(node_id) + if not node: + return + node.properties.update(attributes) + node.metadata.update(attributes) + + def get_edge_data(self, source_id: str, target_id: str) -> Dict[str, Any]: + for edge in self._adjacency.get(source_id, []): + if edge.target_id == target_id: + data = edge.metadata.copy() + data["type"] = edge.edge_type + data["weight"] = edge.weight + return data + return {} + + def get_neighbors( + self, + node_id: str, + hops: int = 1, + relationship_types: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: """ Get neighbors of a node. @@ -269,36 +331,39 @@ class ContextGraph: if node_id not in self.nodes: return [] - neighbors = [] + neighbors: List[Dict[str, Any]] = [] visited = {node_id} - queue = deque([(node_id, 0)]) # (current_id, current_hop) + queue = deque([(node_id, 0)]) + rel_filter = set(relationship_types) if relationship_types else None while queue: current_id, current_hop = queue.popleft() - if current_hop >= hops: continue - # Get outgoing edges outgoing_edges = self._adjacency.get(current_id, []) for edge in outgoing_edges: + if rel_filter is not None and edge.edge_type not in rel_filter: + continue neighbor_id = edge.target_id - if neighbor_id not in visited: - visited.add(neighbor_id) - queue.append((neighbor_id, current_hop + 1)) + if neighbor_id in visited: + continue + visited.add(neighbor_id) + queue.append((neighbor_id, current_hop + 1)) - if neighbor_id in self.nodes: - node = self.nodes[neighbor_id] - neighbors.append( - { - "id": node.node_id, - "type": node.node_type, - "content": node.content, - "relationship": edge.edge_type, - "weight": edge.weight, - "hop": current_hop + 1, - } - ) + node = self.nodes.get(neighbor_id) + if not node: + continue + neighbors.append( + { + "id": node.node_id, + "type": node.node_type, + "content": node.content, + "relationship": edge.edge_type, + "weight": edge.weight, + "hop": current_hop + 1, + } + ) return neighbors @@ -444,11 +509,14 @@ class ContextGraph: """Find a node by ID.""" node = self.nodes.get(node_id) if node: + merged_metadata = {} + merged_metadata.update(getattr(node, "metadata", {}) or {}) + merged_metadata.update(getattr(node, "properties", {}) or {}) return { "id": node.node_id, "type": node.node_type, "content": node.content, - "metadata": node.metadata, + "metadata": merged_metadata, } return None @@ -465,7 +533,7 @@ class ContextGraph: "id": n.node_id, "type": n.node_type, "content": n.content, - "metadata": n.metadata, + "metadata": {**(getattr(n, "metadata", {}) or {}), **(getattr(n, "properties", {}) or {})}, } for n in nodes ] diff --git a/semantica/context/context_retriever.py b/semantica/context/context_retriever.py index c1131f7d..b72833c9 100644 --- a/semantica/context/context_retriever.py +++ b/semantica/context/context_retriever.py @@ -2011,8 +2011,18 @@ Answer:""" try: # Basic neighbor expansion if hasattr(self.knowledge_graph, 'get_neighbors'): - neighbors = self.knowledge_graph.get_neighbors(entity_name) - for neighbor in neighbors[:5]: # Limit to prevent explosion + if hasattr(self.knowledge_graph, "neighbors"): + neighbor_ids = list(self.knowledge_graph.neighbors(entity_name)) + elif hasattr(self.knowledge_graph, "get_neighbor_ids"): + neighbor_ids = self.knowledge_graph.get_neighbor_ids(entity_name) + else: + neighbor_details = self.knowledge_graph.get_neighbors(entity_name, hops=1) + neighbor_ids = [ + n.get("id") for n in neighbor_details + if isinstance(n, dict) and n.get("id") + ] + + for neighbor in neighbor_ids[:5]: # Limit to prevent explosion expanded_entities.append({ "name": neighbor, "type": "related_entity", @@ -2082,8 +2092,17 @@ Answer:""" if hasattr(self.centrality_calculator, 'calculate_degree_centrality'): # Simplified centrality calculation if hasattr(self.knowledge_graph, 'get_neighbors'): - neighbors = self.knowledge_graph.get_neighbors(entity_name) - centrality_scores[entity_name] = len(neighbors) + if hasattr(self.knowledge_graph, "neighbors"): + neighbor_ids = list(self.knowledge_graph.neighbors(entity_name)) + elif hasattr(self.knowledge_graph, "get_neighbor_ids"): + neighbor_ids = self.knowledge_graph.get_neighbor_ids(entity_name) + else: + neighbor_details = self.knowledge_graph.get_neighbors(entity_name, hops=1) + neighbor_ids = [ + n.get("id") for n in neighbor_details + if isinstance(n, dict) and n.get("id") + ] + centrality_scores[entity_name] = len(neighbor_ids) else: centrality_scores[entity_name] = 1 diff --git a/semantica/context/policy_engine.py b/semantica/context/policy_engine.py index cb7eb669..9cefaad6 100644 --- a/semantica/context/policy_engine.py +++ b/semantica/context/policy_engine.py @@ -84,7 +84,7 @@ class PolicyEngine: records exceptions, and analyzes policy impact. """ - def __init__(self, graph_store: GraphStore): + def __init__(self, graph_store: Any): """ Initialize PolicyEngine. @@ -93,6 +93,7 @@ class PolicyEngine: """ self.graph_store = graph_store self.logger = get_logger(__name__) + self._supports_cypher = hasattr(graph_store, "execute_query") def add_policy(self, policy: Policy) -> str: """ @@ -105,35 +106,54 @@ class PolicyEngine: Policy ID """ try: - # Store policy node - query = """ - CREATE (p:Policy { - policy_id: $policy_id, - name: $name, - description: $description, - rules: $rules, - category: $category, - version: $version, - created_at: $created_at, - updated_at: $updated_at, - metadata: $metadata - }) - """ - self.graph_store.execute_query(query, { - "policy_id": policy.policy_id, - "name": policy.name, - "description": policy.description, - "rules": policy.rules, - "category": policy.category, - "version": policy.version, - "created_at": policy.created_at, - "updated_at": policy.updated_at, - "metadata": policy.metadata - }) - + if self._supports_cypher: + query = """ + CREATE (p:Policy { + policy_id: $policy_id, + name: $name, + description: $description, + rules: $rules, + category: $category, + version: $version, + created_at: $created_at, + updated_at: $updated_at, + metadata: $metadata + }) + """ + self.graph_store.execute_query(query, { + "policy_id": policy.policy_id, + "name": policy.name, + "description": policy.description, + "rules": policy.rules, + "category": policy.category, + "version": policy.version, + "created_at": policy.created_at, + "updated_at": policy.updated_at, + "metadata": policy.metadata + }) + self.logger.info(f"Added policy: {policy.policy_id} version {policy.version}") + return policy.policy_id + + if not hasattr(self.graph_store, "add_node"): + raise RuntimeError("Graph backend does not support policy storage") + + node_id = f"{policy.policy_id}:{policy.version}" + self.graph_store.add_node( + node_id=node_id, + node_type="Policy", + content=policy.name or policy.policy_id, + policy_id=policy.policy_id, + name=policy.name, + description=policy.description, + rules=policy.rules, + category=policy.category, + version=policy.version, + created_at=policy.created_at.isoformat() if hasattr(policy.created_at, "isoformat") else str(policy.created_at), + updated_at=policy.updated_at.isoformat() if hasattr(policy.updated_at, "isoformat") else str(policy.updated_at), + metadata=policy.metadata or {} + ) self.logger.info(f"Added policy: {policy.policy_id} version {policy.version}") return policy.policy_id - except Exception as e: self.logger.error(f"Failed to add policy: {e}") raise @@ -187,17 +207,26 @@ class PolicyEngine: # Store new version self.add_policy(updated_policy) - # Link versions - query = """ - MATCH (old:Policy {policy_id: $policy_id, version: $old_version}) - MATCH (new:Policy {policy_id: $policy_id, version: $new_version}) - MERGE (old)-[:VERSION_OF]->(new) - """ - self.graph_store.execute_query(query, { - "policy_id": policy_id, - "old_version": current_policy.version, - "new_version": new_version - }) + if self._supports_cypher: + query = """ + MATCH (old:Policy {policy_id: $policy_id, version: $old_version}) + MATCH (new:Policy {policy_id: $policy_id, version: $new_version}) + MERGE (old)-[:VERSION_OF]->(new) + """ + self.graph_store.execute_query(query, { + "policy_id": policy_id, + "old_version": current_policy.version, + "new_version": new_version + }) + else: + if hasattr(self.graph_store, "add_edge"): + self.graph_store.add_edge( + f"{policy_id}:{current_policy.version}", + f"{policy_id}:{new_version}", + edge_type="VERSION_OF", + changed_at=datetime.now().isoformat(), + change_reason=change_reason + ) self.logger.info(f"Updated policy {policy_id} to version {new_version}") return new_version @@ -222,26 +251,60 @@ class PolicyEngine: List of applicable policies (latest versions) """ try: + if self._supports_cypher: # Get latest policies for category - query = """ - MATCH (p:Policy {category: $category}) - WHERE NOT (p)-[:VERSION_OF]->(:Policy) - RETURN p - ORDER BY p.updated_at DESC - """ - results = self.graph_store.execute_query(query, {"category": category}) - - policies = [] - for record in results: - policy_data = record.get("p", {}) - policies.append(self._dict_to_policy(policy_data)) - - # Filter by entities if specified - if entities: - # This would require entity-specific policy relationships - # For now, return all category policies - pass - + query = """ + MATCH (p:Policy {category: $category}) + WHERE NOT (p)-[:VERSION_OF]->(:Policy) + RETURN p + ORDER BY p.updated_at DESC + """ + results = self.graph_store.execute_query(query, {"category": category}) + + policies = [] + for record in results: + policy_data = record.get("p", {}) + policies.append(self._dict_to_policy(policy_data)) + + if entities: + pass + + self.logger.info(f"Found {len(policies)} applicable policies for category {category}") + return policies + + if not hasattr(self.graph_store, "find_nodes"): + return [] + + latest_by_policy_id: Dict[str, Dict[str, Any]] = {} + for node in self.graph_store.find_nodes(node_type="Policy"): + data = node.get("metadata", {}) or {} + if data.get("category") != category: + continue + pid = data.get("policy_id") + if not pid: + continue + updated_at = data.get("updated_at") or "" + prev = latest_by_policy_id.get(pid) + if not prev: + latest_by_policy_id[pid] = data + else: + if str(updated_at) > str(prev.get("updated_at") or ""): + latest_by_policy_id[pid] = data + + policies: List[Policy] = [] + for data in latest_by_policy_id.values(): + policies.append(self._dict_to_policy({ + "policy_id": data.get("policy_id"), + "name": data.get("name"), + "description": data.get("description"), + "rules": data.get("rules", {}), + "category": data.get("category"), + "version": data.get("version"), + "created_at": data.get("created_at"), + "updated_at": data.get("updated_at"), + "metadata": data.get("metadata", {}) + })) + self.logger.info(f"Found {len(policies)} applicable policies for category {category}") return policies @@ -303,20 +366,33 @@ class PolicyEngine: version: Policy version that was applied """ try: - query = """ - MATCH (d:Decision {decision_id: $decision_id}) - MATCH (p:Policy {policy_id: $policy_id, version: $version}) - MERGE (d)-[:APPLIED_POLICY]->(p) - SET d.policy_applied_at = timestamp() - """ - self.graph_store.execute_query(query, { - "decision_id": decision_id, - "policy_id": policy_id, - "version": version - }) - + if self._supports_cypher: + query = """ + MATCH (d:Decision {decision_id: $decision_id}) + MATCH (p:Policy {policy_id: $policy_id, version: $version}) + MERGE (d)-[:APPLIED_POLICY]->(p) + SET d.policy_applied_at = timestamp() + """ + self.graph_store.execute_query(query, { + "decision_id": decision_id, + "policy_id": policy_id, + "version": version + }) + self.logger.info(f"Recorded policy application: {policy_id} v{version} to decision {decision_id}") + return + + if not hasattr(self.graph_store, "add_edge"): + raise RuntimeError("Graph backend does not support relationships") + policy_node_id = f"{policy_id}:{version}" + self.graph_store.add_edge( + decision_id, + policy_node_id, + edge_type="APPLIED_POLICY", + applied_at=datetime.now().isoformat(), + policy_id=policy_id, + version=version + ) self.logger.info(f"Recorded policy application: {policy_id} v{version} to decision {decision_id}") - except Exception as e: self.logger.error(f"Failed to record policy application: {e}") raise @@ -340,40 +416,61 @@ class PolicyEngine: """ try: exception_id = str(uuid.uuid4()) - - query = """ - CREATE (e:Exception { - exception_id: $exception_id, - decision_id: $decision_id, - policy_id: $policy_id, - reason: $reason, - created_at: datetime() - }) - """ - self.graph_store.execute_query(query, { - "exception_id": exception_id, - "decision_id": decision_id, - "policy_id": policy_id, - "reason": reason - }) - - # Link to decision and policy - query = """ - MATCH (d:Decision {decision_id: $decision_id}) - MATCH (p:Policy {policy_id: $policy_id}) - MATCH (e:Exception {exception_id: $exception_id}) - MERGE (d)-[:GRANTED_EXCEPTION]->(e) - MERGE (e)-[:OVERRIDDEN_POLICY]->(p) - """ - self.graph_store.execute_query(query, { - "decision_id": decision_id, - "policy_id": policy_id, - "exception_id": exception_id - }) - + + if self._supports_cypher: + query = """ + CREATE (e:Exception { + exception_id: $exception_id, + decision_id: $decision_id, + policy_id: $policy_id, + reason: $reason, + created_at: datetime() + }) + """ + self.graph_store.execute_query(query, { + "exception_id": exception_id, + "decision_id": decision_id, + "policy_id": policy_id, + "reason": reason + }) + + query = """ + MATCH (d:Decision {decision_id: $decision_id}) + MATCH (p:Policy {policy_id: $policy_id}) + MATCH (e:Exception {exception_id: $exception_id}) + MERGE (d)-[:GRANTED_EXCEPTION]->(e) + MERGE (e)-[:OVERRIDDEN_POLICY]->(p) + """ + self.graph_store.execute_query(query, { + "decision_id": decision_id, + "policy_id": policy_id, + "exception_id": exception_id + }) + + self.logger.info(f"Recorded policy exception: {exception_id}") + return exception_id + + if not hasattr(self.graph_store, "add_node") or not hasattr(self.graph_store, "add_edge"): + raise RuntimeError("Graph backend does not support exceptions") + + self.graph_store.add_node( + node_id=exception_id, + node_type="Exception", + content=reason, + exception_id=exception_id, + decision_id=decision_id, + policy_id=policy_id, + reason=reason, + created_at=datetime.now().isoformat() + ) + self.graph_store.add_edge(decision_id, exception_id, edge_type="GRANTED_EXCEPTION") + + policy = self.get_policy(policy_id) + if policy: + self.graph_store.add_edge(exception_id, f"{policy_id}:{policy.version}", edge_type="OVERRIDDEN_POLICY") + self.logger.info(f"Recorded policy exception: {exception_id}") return exception_id - except Exception as e: self.logger.error(f"Failed to record exception: {e}") raise @@ -389,23 +486,45 @@ class PolicyEngine: List of policy versions """ try: - query = """ - MATCH (p:Policy {policy_id: $policy_id}) - OPTIONAL MATCH (p)-[:VERSION_OF*]->(future:Policy) - WITH collect(p) + collect(future) as all_versions - UNWIND all_versions as version - RETURN DISTINCT version - ORDER BY version.updated_at - """ - results = self.graph_store.execute_query(query, {"policy_id": policy_id}) + if self._supports_cypher: + query = """ + MATCH (p:Policy {policy_id: $policy_id}) + OPTIONAL MATCH (p)-[:VERSION_OF*]->(future:Policy) + WITH collect(p) + collect(future) as all_versions + UNWIND all_versions as version + RETURN DISTINCT version + ORDER BY version.updated_at + """ + results = self.graph_store.execute_query(query, {"policy_id": policy_id}) - policies = [] - for record in results: - policy_data = record.get("version", {}) - policies.append(self._dict_to_policy(policy_data)) + policies = [] + for record in results: + policy_data = record.get("version", {}) + policies.append(self._dict_to_policy(policy_data)) - self.logger.info(f"Found {len(policies)} versions for policy {policy_id}") - return policies + self.logger.info(f"Found {len(policies)} versions for policy {policy_id}") + return policies + + if not hasattr(self.graph_store, "find_nodes"): + return [] + versions: List[Policy] = [] + for node in self.graph_store.find_nodes(node_type="Policy"): + data = node.get("metadata", {}) or {} + if data.get("policy_id") != policy_id: + continue + versions.append(self._dict_to_policy({ + "policy_id": data.get("policy_id"), + "name": data.get("name"), + "description": data.get("description"), + "rules": data.get("rules", {}), + "category": data.get("category"), + "version": data.get("version"), + "created_at": data.get("created_at"), + "updated_at": data.get("updated_at"), + "metadata": data.get("metadata", {}) + })) + versions.sort(key=lambda p: str(p.updated_at)) + return versions except Exception as e: self.logger.error(f"Failed to get policy history: {e}") @@ -429,23 +548,33 @@ class PolicyEngine: List of affected decision IDs """ try: - query = """ - MATCH (d:Decision)-[:APPLIED_POLICY]->(p:Policy { - policy_id: $policy_id, - version: $from_version - }) - RETURN d.decision_id as decision_id - """ - results = self.graph_store.execute_query(query, { - "policy_id": policy_id, - "from_version": from_version - }) + if self._supports_cypher: + query = """ + MATCH (d:Decision)-[:APPLIED_POLICY]->(p:Policy { + policy_id: $policy_id, + version: $from_version + }) + RETURN d.decision_id as decision_id + """ + results = self.graph_store.execute_query(query, { + "policy_id": policy_id, + "from_version": from_version + }) - decision_ids = [] - for record in results: - decision_ids.append(record.get("decision_id", "")) + decision_ids = [] + for record in results: + decision_ids.append(record.get("decision_id", "")) - self.logger.info(f"Found {len(decision_ids)} decisions affected by policy change") + self.logger.info(f"Found {len(decision_ids)} decisions affected by policy change") + return decision_ids + + if not hasattr(self.graph_store, "find_edges"): + return [] + policy_node_id = f"{policy_id}:{from_version}" + decision_ids: List[str] = [] + for edge in self.graph_store.find_edges(edge_type="APPLIED_POLICY"): + if edge.get("target") == policy_node_id: + decision_ids.append(edge.get("source")) return decision_ids except Exception as e: @@ -471,16 +600,38 @@ class PolicyEngine: current_policy = self.get_policy(policy_id) if not current_policy: raise ValueError(f"Policy {policy_id} not found") - - # Get decisions that used this policy - query = """ - MATCH (d:Decision)-[:APPLIED_POLICY]->(p:Policy {policy_id: $policy_id}) - RETURN d.decision_id as decision_id, d.confidence as confidence, - d.outcome as outcome, d.category as category - """ - results = self.graph_store.execute_query(query, {"policy_id": policy_id}) - - # Analyze impact + + results: List[Dict[str, Any]] = [] + if self._supports_cypher: + query = """ + MATCH (d:Decision)-[:APPLIED_POLICY]->(p:Policy {policy_id: $policy_id}) + RETURN d.decision_id as decision_id, d.confidence as confidence, + d.outcome as outcome, d.category as category + """ + results = self.graph_store.execute_query(query, {"policy_id": policy_id}) + else: + if hasattr(self.graph_store, "find_edges") and hasattr(self.graph_store, "nodes"): + for edge in self.graph_store.find_edges(edge_type="APPLIED_POLICY"): + target = edge.get("target") + if not target or not isinstance(target, str): + continue + policy_node = self.graph_store.nodes.get(target) + if not policy_node: + continue + props = getattr(policy_node, "properties", {}) or {} + if props.get("policy_id") != policy_id: + continue + decision_node = self.graph_store.nodes.get(edge.get("source")) + if not decision_node: + continue + dprops = getattr(decision_node, "properties", {}) or {} + results.append({ + "decision_id": edge.get("source"), + "confidence": dprops.get("confidence", 0.0), + "outcome": dprops.get("outcome", ""), + "category": dprops.get("category", "") + }) + impact_analysis = { "total_decisions": len(results), "affected_decisions": 0, @@ -488,19 +639,16 @@ class PolicyEngine: "risk_assessment": "low", "recommendations": [] } - + for record in results: decision_data = { "confidence": record.get("confidence", 0.0), "outcome": record.get("outcome", ""), "category": record.get("category", "") } - - # Check if decision would still comply with new rules would_comply = self._check_compliance_with_rules( decision_data, proposed_rules ) - if not would_comply: impact_analysis["affected_decisions"] += 1 @@ -546,29 +694,76 @@ class PolicyEngine: Policy object or None """ try: + if self._supports_cypher: + if version: + query = """ + MATCH (p:Policy {policy_id: $policy_id, version: $version}) + RETURN p + """ + params = {"policy_id": policy_id, "version": version} + else: + query = """ + MATCH (p:Policy {policy_id: $policy_id}) + WHERE NOT (p)-[:VERSION_OF]->(:Policy) + RETURN p + """ + params = {"policy_id": policy_id} + + results = self.graph_store.execute_query(query, params) + + if results: + policy_data = results[0].get("p", {}) + return self._dict_to_policy(policy_data) + return None + + if not hasattr(self.graph_store, "find_nodes"): + return None + + candidates: List[Dict[str, Any]] = [] + for node in self.graph_store.find_nodes(node_type="Policy"): + data = node.get("metadata", {}) or {} + if data.get("policy_id") != policy_id: + continue + if version and data.get("version") != version: + continue + candidates.append(data) + + if not candidates: + return None + if version: - query = """ - MATCH (p:Policy {policy_id: $policy_id, version: $version}) - RETURN p - """ - params = {"policy_id": policy_id, "version": version} + data = candidates[0] else: - # Get latest version - query = """ - MATCH (p:Policy {policy_id: $policy_id}) - WHERE NOT (p)-[:VERSION_OF]->(:Policy) - RETURN p - """ - params = {"policy_id": policy_id} - - results = self.graph_store.execute_query(query, params) - - if results: - policy_data = results[0].get("p", {}) - return self._dict_to_policy(policy_data) - - return None - + # Prefer highest semantic version if available, fallback to updated_at + def _version_key(v: str) -> tuple: + try: + parts = [int(p) for p in str(v).split(".")] + # Normalize length for comparison + while len(parts) < 3: + parts.append(-1) + return tuple(parts[:3]) + except Exception: + return (-1, -1, -1) + + try: + data = max( + candidates, + key=lambda d: (_version_key(d.get("version")), str(d.get("updated_at") or "")), + ) + except Exception: + data = max(candidates, key=lambda d: str(d.get("updated_at") or "")) + + return self._dict_to_policy({ + "policy_id": data.get("policy_id"), + "name": data.get("name"), + "description": data.get("description"), + "rules": data.get("rules", {}), + "category": data.get("category"), + "version": data.get("version"), + "created_at": data.get("created_at"), + "updated_at": data.get("updated_at"), + "metadata": data.get("metadata", {}) + }) except Exception as e: self.logger.error(f"Failed to get policy: {e}") return None diff --git a/semantica/kg/centrality_calculator.py b/semantica/kg/centrality_calculator.py index 95ee72f7..1fda57a7 100644 --- a/semantica/kg/centrality_calculator.py +++ b/semantica/kg/centrality_calculator.py @@ -739,6 +739,11 @@ class CentralityCalculator: neighbors = list(graph.neighbors(node)) elif hasattr(graph, 'get_neighbors'): neighbors = graph.get_neighbors(node) + if neighbors and isinstance(neighbors[0], dict): + neighbors = [ + n.get("id") for n in neighbors + if isinstance(n, dict) and n.get("id") + ] else: neighbors = [] diff --git a/semantica/kg/community_detector.py b/semantica/kg/community_detector.py index cd25eb88..2077e4e1 100644 --- a/semantica/kg/community_detector.py +++ b/semantica/kg/community_detector.py @@ -891,6 +891,12 @@ class CommunityDetector: all_neighbors = graph.get_neighbors(node) else: all_neighbors = [] + + if all_neighbors and isinstance(all_neighbors[0], dict): + all_neighbors = [ + n.get("id") for n in all_neighbors + if isinstance(n, dict) and n.get("id") + ] # Filter by relationship types if specified if relationship_types is not None and hasattr(graph, 'get_edge_data'): diff --git a/semantica/kg/link_predictor.py b/semantica/kg/link_predictor.py index 2144fa59..e811957b 100644 --- a/semantica/kg/link_predictor.py +++ b/semantica/kg/link_predictor.py @@ -455,6 +455,9 @@ class LinkPredictor: if hasattr(graph_store, 'neighbors'): return list(graph_store.neighbors(node_id)) elif hasattr(graph_store, 'get_neighbors'): - return graph_store.get_neighbors(node_id) + neighbors = graph_store.get_neighbors(node_id) + if neighbors and isinstance(neighbors[0], dict): + return [n.get("id") for n in neighbors if isinstance(n, dict) and n.get("id")] + return neighbors else: return [] diff --git a/semantica/kg/node_embeddings.py b/semantica/kg/node_embeddings.py index ca746f51..e4018279 100644 --- a/semantica/kg/node_embeddings.py +++ b/semantica/kg/node_embeddings.py @@ -350,13 +350,25 @@ class NodeEmbedder: # Build adjacency for node in nodes: - if hasattr(graph_store, 'get_neighbors'): - neighbors = graph_store.get_neighbors(node, relationship_types) - adjacency[node] = neighbors - else: - # Fallback for NetworkX - if hasattr(graph_store, 'neighbors'): - adjacency[node] = list(graph_store.neighbors(node)) + if hasattr(graph_store, 'neighbors'): + adjacency[node] = list(graph_store.neighbors(node)) + elif hasattr(graph_store, 'get_neighbor_ids'): + adjacency[node] = graph_store.get_neighbor_ids(node, relationship_types) + elif hasattr(graph_store, 'get_neighbors'): + try: + neighbor_details = graph_store.get_neighbors(node, hops=1, relationship_types=relationship_types) + adjacency[node] = [ + n.get("id") for n in neighbor_details + if isinstance(n, dict) and n.get("id") + ] + except TypeError: + neighbor_details = graph_store.get_neighbors(node) + adjacency[node] = [ + n.get("id") for n in neighbor_details + if isinstance(n, dict) and n.get("id") + ] + elif hasattr(graph_store, 'nodes') and hasattr(graph_store, 'edges'): + adjacency[node] = [] return dict(adjacency) diff --git a/tests/context/test_agent_context_smoke.py b/tests/context/test_agent_context_smoke.py new file mode 100644 index 00000000..d019ef20 --- /dev/null +++ b/tests/context/test_agent_context_smoke.py @@ -0,0 +1,66 @@ +import pytest +from semantica.context import AgentContext, ContextGraph +from semantica.context.decision_models import Policy +from semantica.vector_store import VectorStore +from datetime import datetime + + +def test_agent_context_minimal_decisions_and_chain(): + vs = VectorStore(backend="inmemory", dimension=64) + graph = ContextGraph() + ctx = AgentContext( + vector_store=vs, + knowledge_graph=graph, + enable_decision_tracking=True, + enable_kg_algorithms=False, + enable_vector_store_features=False, + ) + d1 = ctx.record_decision( + category="credit_approval", + scenario="s1", + reasoning="r1", + outcome="rejected", + confidence=0.8, + entities=["e1"], + decision_maker="tester", + ) + d2 = ctx.record_decision( + category="credit_approval", + scenario="s2", + reasoning="r2", + outcome="rejected", + confidence=0.85, + entities=["e1"], + decision_maker="tester", + ) + graph.add_causal_relationship(d1, d2, "INFLUENCED") + chain = ctx.get_causal_chain(d2, direction="upstream", max_depth=5) + assert isinstance(chain, list) + assert len(chain) >= 1 + + +def test_agent_context_policy_engine_with_graph_backend(): + vs = VectorStore(backend="inmemory", dimension=64) + graph = ContextGraph() + ctx = AgentContext( + vector_store=vs, + knowledge_graph=graph, + enable_decision_tracking=True, + enable_kg_algorithms=False, + enable_vector_store_features=False, + ) + pe = ctx.get_policy_engine() + pol = Policy( + policy_id="cp", + name="Credit Policy", + description="d", + rules={"min_confidence": 0.8, "allowed_outcomes": ["approved", "rejected"]}, + category="credit_approval", + version="1.0.0", + created_at=datetime.now(), + updated_at=datetime.now(), + metadata={}, + ) + pe.add_policy(pol) + found = pe.get_policy("cp") + assert found is not None diff --git a/tests/context/test_policy_engine_fallback.py b/tests/context/test_policy_engine_fallback.py new file mode 100644 index 00000000..6c163e09 --- /dev/null +++ b/tests/context/test_policy_engine_fallback.py @@ -0,0 +1,70 @@ +from datetime import datetime + +from semantica.context import ContextGraph, PolicyEngine +from semantica.context.decision_models import Policy, Decision + + +def _make_policy(pid="p1", version="1.0.0", min_conf=0.8): + return Policy( + policy_id=pid, + name="Test Policy", + description="desc", + rules={"min_confidence": min_conf, "allowed_outcomes": ["approved", "rejected"]}, + category="test", + version=version, + created_at=datetime.now(), + updated_at=datetime.now(), + metadata={}, + ) + + +def _make_decision(decision_id, conf=0.9, outcome="approved"): + return Decision( + decision_id=decision_id, + category="test", + scenario="s", + reasoning="r", + outcome=outcome, + confidence=conf, + timestamp=datetime.now(), + decision_maker="tester", + metadata={}, + ) + + +def test_policy_engine_add_get_update_with_context_graph(): + graph = ContextGraph() + engine = PolicyEngine(graph) + policy = _make_policy() + engine.add_policy(policy) + latest = engine.get_policy("p1") + assert latest is not None + assert latest.version == "1.0.0" + new_ver = engine.update_policy( + "p1", + {"min_confidence": 0.85, "allowed_outcomes": ["approved", "rejected"]}, + "raise min", + ) + assert isinstance(new_ver, str) + latest2 = engine.get_policy("p1") + assert latest2 is not None + assert latest2.version != "1.0.0" + + +def test_policy_engine_compliance_and_application_edges(): + graph = ContextGraph() + engine = PolicyEngine(graph) + policy = _make_policy() + engine.add_policy(policy) + decision = _make_decision("d1", conf=0.9, outcome="approved") + graph.add_decision(decision) + ok = engine.check_compliance(decision, "p1") + assert ok is True + engine.record_policy_application("d1", "p1", "1.0.0") + edges = graph.find_edges(edge_type="APPLIED_POLICY") + assert any( + e.get("source") == "d1" + and isinstance(e.get("target"), str) + and e.get("target").startswith("p1:") + for e in edges + ) From 8386d79543ca33a9e593712a70ce497a350e9fb4 Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Sat, 14 Feb 2026 19:38:14 +0530 Subject: [PATCH 05/50] Update CHANGELOG.md (#313) --- CHANGELOG.md | 1097 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1097 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70668163..8ac118ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,1103 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Fixed: PolicyEngine latest version selection on ContextGraph; AgentContext fallback robustness and secure logging (PR #TBD by @KaifAhmad1) +- Tests: Added ContextGraph fallback and AgentContext smoke tests; full suite passing + +- **Context Engineering Enhancement** (PR #307 by @KaifAhmad1): + - Comprehensive decision tracking system with full lifecycle management (record → analyze → query → precedent → influence) + - Advanced KG algorithm integration: centrality analysis, community detection, node embeddings with ContextGraph + - Enhanced AgentContext with granular feature flags for decision tracking, KG algorithms, and vector store features + - PolicyException model replacing conflicting Exception name for meaningful business domain modeling + - GraphStore validation preventing runtime failures with explicit capability checking + - Hybrid search combining semantic, structural, and category similarity with configurable weights + - Decision influence analysis with centrality measures and causal chain tracking + - Policy management with versioning, compliance checking, and exception handling + - Production-ready architecture with audit trails, security, and scalability features + - 9 critical bug fixes: logging, security, audit trails, API compatibility, Cypher queries, centrality access, validation, naming + - Comprehensive documentation with usage guides, production examples, and API references + - 100% test coverage with all validation tests passing (9/9 tests) + - Enterprise-grade features for financial services, healthcare, legal, and business domains + - Complete backward compatibility with existing semantica components + - Performance optimizations: caching, indexing, and efficient graph operations + +- **Added PgVector Store Support** (PR #303 by @Sameer6305, @KaifAhmad1): + - Native PostgreSQL vector storage using pgvector extension with full integration + - Multiple distance metrics: cosine, L2/Euclidean, inner product with automatic score normalization + - Advanced indexing: HNSW and IVFFlat for approximate nearest neighbor search with tunable parameters + - JSONB metadata storage with flexible filtering capabilities and batch operations + - Connection pooling support with psycopg3/psycopg2 fallback and efficient resource management + - Comprehensive VectorStore integration with backend delegation and unified API + - Idempotent index creation and table management with safe migration support + - Production-ready security: SQL injection protection with psycopg_sql.SQL() and input validation + - Performance optimizations: UUID4-based IDs, batch executemany operations, connection pooling + - Full backward compatibility with existing vector store implementations + - 36+ comprehensive test cases with Docker integration and dependency skipping + - Complete documentation with setup guides, examples, and performance tuning + - CI/CD integration: resolved benchmark compatibility and fixed documentation links + +- **Improved Vector Store for Decision Tracking** (PR #293 by @KaifAhmad1): + - Comprehensive decision tracking capabilities with hybrid search combining semantic and structural embeddings + - New DecisionEmbeddingPipeline for generating semantic and structural embeddings with KG algorithm integration + - HybridSimilarityCalculator with configurable weights (semantic: 0.7, structural: 0.3) + - DecisionContext high-level interface for decision management with explainable AI features + - ContextRetriever with hybrid precedent search and multi-hop reasoning + - User-friendly convenience API: quick_decision(), find_precedents(), explain(), similar_to(), batch_decisions(), filter_decisions() + - Knowledge Graph algorithm integration: Node2Vec, PathFinder, CommunityDetector, CentralityCalculator, SimilarityCalculator, ConnectivityAnalyzer + - Explainable AI with path tracing, confidence scoring, and comprehensive decision explanations + - Performance optimizations: 0.028s per decision processing, 0.031s search performance, ~0.8KB per decision memory usage + - 100% backward compatibility maintained with existing VectorStore functionality + - 34+ comprehensive tests covering all functionality including end-to-end scenarios and performance benchmarks + - Real-world validation examples for banking and insurance domains + - Documentation with clear imports, examples, and API references + +- **Improved Graph Algorithms in KG Module** (PR #292 by @KaifAhmad1): + - Complete algorithm suite with 30+ graph algorithms across 7 categories + - Node Embeddings: Node2Vec, DeepWalk, Word2Vec for structural similarity analysis + - Similarity Analysis: Cosine, Euclidean, Manhattan, Correlation metrics with batch processing + - Path Finding: Dijkstra, A*, BFS, K-shortest paths for route and network analysis + - Link Prediction: Preferential attachment, Jaccard, Adamic-Adar for network completion + - Centrality Analysis: Degree, Betweenness, Closeness, PageRank for importance ranking + - Community Detection: Louvain, Leiden, Label propagation for clustering analysis + - Connectivity Analysis: Components, bridges, density for network robustness + - Unified provenance tracking system with GraphBuilderWithProvenance and AlgorithmTrackerWithProvenance + - Complete execution tracking with metadata, timestamps, and reproducibility IDs + - Comprehensive test coverage with 5 test suites and 40+ test methods + - Professional documentation overhaul for all modules and reference documentation + - Enterprise-ready functionality with error handling and NetworkX compatibility + - Performance optimizations with sparse matrix operations and batch processing + - Full backward compatibility maintained with gradual migration support + +- **Improved Security Configuration with Dependabot**: + - Configured bi-weekly security updates with manual review by @KaifAhmad1 + - Implemented automated security scans (Monday & Thursday at 7 AM IST) with Bandit, Safety, Semgrep + - Added security-critical package grouping (cryptography, requests, urllib3, certifi, pyopenssl) + - Enterprise-grade security with audit trail, compliance features, and zero auto-merge + - Optimized IST timezone scheduling (Security scans: 7 AM IST, PRs: 9 AM IST) + - Aligned with new Dependabot features: open-source proxy support, smart dependency grouping for Snowflake/Arrow/benchmark features, private registry support, semantic commit prefixes, and latest GitHub security best practices + +- **ResourceScheduler Deadlock Fix and Performance Improvements** (PR #299, #301 by @d4ndr4d3, @KaifAhmad1): + - Fixed critical deadlock in ResourceScheduler by replacing `threading.Lock()` with `threading.RLock()` + - Resolved nested lock acquisition issue in `allocate_resources()` → `allocate_cpu/memory/gpu()` calls + - Added allocation validation with `ValidationError` when no resources can be allocated + - Improved performance by moving progress tracking updates outside lock scope + - Implemented comprehensive resource cleanup on allocation failures to prevent leaks + - Added complete regression test suite (6 tests) for deadlock prevention and edge cases + - Improved error handling and documentation for better operator visibility + - Zero breaking changes, maintains thread safety and backward compatibility + +## [0.2.7] - 2026-02-09 + +### Added / Changed + +- **Snowflake Connector for Data Ingestion** (PR #276 by @Sameer6305): + - Native Snowflake connector with multi-authentication (password, OAuth, key-pair, SSO) + - Table and query ingestion with pagination, schema introspection, batch processing + - SQL injection prevention via identifier escaping, OAuth token validation + - Progress tracking integration, context manager support, document export + - 24 comprehensive unit tests with mocking, complete documentation and examples + - Added as optional dependency `db-snowflake` with snowflake-connector-python>=3.0.0 + +- **Apache Arrow Export Support** (PR #273 by @Sameer6305): + - Added Apache Arrow exporter with explicit schemas, entity/relationship export, compression support + - Integrated with export module and method registry, Pandas/DuckDB compatible + - 20 unit tests + 1 integration test, complete documentation with examples + +- **Comprehensive Benchmark Suite with Regression CLI** (PR #289 by @ZohaibHassan16, @KaifAhmad1): + - 137+ benchmarks across all 10 Semantica modules (Input, Core, Storage, Context, QA, Ontology, etc.) + - Environment-agnostic design with robust mocking system for CI/CD compatibility + - Statistical regression detection using Z-score analysis with configurable thresholds + - Automated performance auditing via GitHub Actions workflow + - Comprehensive documentation suite (benchmarks.md, architecture guides, usage examples) + - Zero breaking changes, production-ready with ultra-fast text processing (>10,000 ops/s) + - Added benchmark runner CLI: `python benchmarks/benchmark_runner.py` + +## [0.2.6] - 2026-02-03 + +### Added / Changed + +- **W3C PROV-O Compliant Provenance Tracking** (#254, #246): + - Comprehensive provenance tracking system with W3C PROV-O compliance across all 17 Semantica modules + - **Core Module**: `ProvenanceManager`, W3C PROV-O schemas, storage backends (InMemory, SQLite), SHA-256 integrity verification + - **Module Integrations**: Semantic Extract, LLMs (Groq, OpenAI, HuggingFace, LiteLLM), Pipeline, Context, Ingest, Embeddings, Graph/Vector/Triplet stores, Reasoning, Conflicts, Deduplication, Export, Parse, Normalize, Ontology, Visualization + - **Features**: Complete lineage tracking (Document → Chunk → Entity → Relationship → Graph), LLM tracking (tokens, costs, latency), source tracking, bridge axioms for domain transformations + - **Compliance Infrastructure**: W3C PROV-O, FDA 21 CFR Part 11, SOX, HIPAA, TNFD + - **Testing**: 237 tests covering core functionality, all 17 module integrations, edge cases, backward compatibility + - **Design**: Opt-in with `provenance=False` by default, zero breaking changes, no new dependencies + - Contributed by @KaifAhmad1 + +- **Enhanced Change Management Module** (#248, #243): + - Enterprise-grade version control for knowledge graphs and ontologies with persistent storage and audit trails + - **Core Classes**: `TemporalVersionManager` (KG versioning), `OntologyVersionManager` (ontology versioning), `ChangeLogEntry` (metadata) + - **Storage**: SQLite (persistent) and in-memory backends with thread-safe operations + - **Features**: SHA-256 checksums, detailed entity/relationship diffs, structural ontology comparison, email validation + - **Compliance Infrastructure**: HIPAA, SOX, FDA 21 CFR Part 11 with immutable audit trails + - **Testing**: 104 tests (100% pass) - unit, integration, compliance, performance, edge cases + - **Performance**: 17.6ms for 10k entities, 510+ ops/sec concurrent, handles 5k+ entity graphs + - **Migration**: Backward compatible, simplified class names, zero external dependencies + - Contributed by @KaifAhmad1 + +- CSV Ingestion Enhancements (PR #244 by @saloni0318) + - Auto-detect CSV encoding (chardet) and delimiter (csv.Sniffer) + - Tolerant decoding and malformed-row handling (`on_bad_lines='warn'`) + - Optional chunked reading for large files; metadata tracks detected values + - Expanded unit tests covering delimiters, quoted/multiline fields, header overrides, chunks, and NaN preservation + +- Tests: Comprehensive units for TextNormalizer (PR #242 by @ZohaibHassan16) + - Added focused test coverage for TextNormalizer behavior across inputs + +- Tests: Register integration mark and tidy ingest test warnings (PR #241 by @KaifAhmad1) + - Introduced integration test marker and reduced noisy warnings in ingest tests + +- **Ingest Unit Tests** (#239, #232): + - Comprehensive unit tests for ingestion modules (file, web, and feed ingestors) + - **Coverage**: File scanning (local/cloud S3/GCS/Azure), web ingestion (URL/sitemap/robots.txt), RSS/Atom feed parsing + - **Testing**: 998 lines of test code with mocked external dependencies for fast, isolated execution + - **Results**: file_ingestor (86%), web_ingestor (86%), feed_ingestor (80%) coverage + - Covers happy paths, edge cases, and error handling + - Contributed by @Mohammed2372 + +### Fixed + +- **Temperature Compatibility Fix** (#256, #252): + - Fixed hardcoded `temperature=0.3` that broke compatibility with models requiring specific temperature values (e.g., gpt-5-mini) + - Added `_add_if_set` helper method to `BaseProvider` that only passes parameters when explicitly set + - When `temperature=None`, parameter is omitted allowing APIs to use model defaults + - Updated all 5 providers: OpenAI, Groq, Gemini, Ollama, DeepSeek + - Reduced code by ~85 lines with cleaner parameter handling + - Comprehensive test coverage added (10 temperature tests, all passing) + - Backward compatible - no breaking changes + - Contributed by @F0rt1s and @IGES-Institut + +- **JenaStore Empty Graph Bug** (#257, #258): + - Fixed `ProcessingError: Graph not initialized` when operating on empty (but initialized) graphs + - Replaced implicit `if not self.graph:` checks with explicit `if self.graph is None:` validation in 5 methods (`add_triplets`, `get_triplets`, `delete_triplet`, `execute_sparql`, `serialize`) + - Properly distinguishes `None` (uninitialized) from empty graphs (initialized with 0 triplets) + - Unblocks benchmarking suite, fresh deployments, and testing workflows + - Contributed by @ZohaibHassan16 + +## [0.2.5] - 2026-01-27 + +### Added +- **Pinecone Vector Store Support**: + - Implemented native Pinecone support (`PineconeStore`) with full CRUD capabilities. + - Added support for serverless and pod-based indexes, namespaces, and metadata filtering. + - Integrated with `VectorStore` unified interface and registry. + - (Closes #219, Resolves #220) +- **Configurable LLM Retry Logic**: + - Exposed `max_retries` parameter in `NERExtractor`, `RelationExtractor`, `TripletExtractor` and low-level extraction methods (`extract_entities_llm`, `extract_relations_llm`, `extract_triplets_llm`). + - Defaults to 3 retries to prevent infinite loops during JSON validation failures or API timeouts. + - Propagated retry configuration through chunked processing helpers to ensure consistent behavior for long documents. + - Updated `03_Earnings_Call_Analysis.ipynb` to use `max_retries=3` by default. + +### Added +- **Bring Your Own Model (BYOM) Support**: + - Enabled full support for custom Hugging Face models in `NERExtractor`, `RelationExtractor`, and `TripletExtractor`. + - Added support for custom tokenizers in `HuggingFaceModelLoader` to handle models with non-standard tokenization requirements. + - Implemented robust fallback logic for model selection: runtime options (`extract(model=...)`) now correctly override configuration defaults. +- **Enhanced NER Implementation**: + - Added configurable aggregation strategies (`simple`, `first`, `average`, `max`) to `extract_entities_huggingface` for better sub-word token handling. + - Implemented robust IOB/BILOU parsing to reconstruct entities from raw model outputs when structured output is unavailable. + - Added confidence scoring for aggregated entities. +- **Relation Extraction Improvements**: + - Implemented standard entity marker technique (wrapping subject/object with ``, `` tags) in `extract_relations_huggingface` for compatibility with sequence classification models. + - Added structured output parsing to convert raw model predictions into validated `Relation` objects. +- **Triplet Extraction Completion**: + - Added specialized parsing for Seq2Seq models (e.g., REBEL) in `extract_triplets_huggingface` to generate structured triplets directly from text. + - Implemented post-processing logic to clean and validate generated triplets. + +### Fixed +- **LLM Extraction Stability**: + - Fixed infinite retry loops in `BaseProvider` by strictly enforcing `max_retries` limit during structured output generation. + - Resolved stuck execution in earnings call analysis notebooks when using smaller models (e.g., Llama 3 8B) that frequently produce invalid JSON. +- **Model Parameter Precedence**: + - Fixed issue where configuration defaults took precedence over runtime arguments in Hugging Face extractors. Runtime options now correctly override config values. +- **Import Handling**: + - Fixed circular import issues in test suites by implementing robust mocking strategies. + +## [0.2.4] - 2026-01-22 + +### Added +- **Ontology Ingestion Module**: + - Implemented `OntologyIngestor` in `semantica.ingest` for parsing RDF/OWL files (Turtle, RDF/XML, JSON-LD, N3) into standardized `OntologyData` objects. + - Added `ingest_ontology` convenience function and integrated it into the unified `ingest(source_type="ontology")` interface. + - Added recursive directory scanning support for batch ontology ingestion. + - Exposed ingestion tools in `semantica.ontology` for better discoverability. + - Added `OntologyData` dataclass for consistent metadata handling (source path, format, timestamps). +- **Documentation**: + - **Ontology Usage Guide**: Updated `ontology_usage.md` with comprehensive examples for single-file and directory ingestion. + - **API Reference**: Updated `ontology.md` with `OntologyIngestor` class documentation and method details. +- **Tests**: + - **Comprehensive Test Suite**: Added `tests/ingest/test_ontology_ingestor.py` covering all supported formats, error handling, and unified interface integration. + - **Demo Script**: Added `examples/demo_ontology_ingest.py` for end-to-end usage demonstration. + +## [0.2.3] - 2026-01-20 + +### Fixed +- **LLM Relation Extraction Parsing**: + - Fixed relation extraction returning zero relations despite successful API calls to Groq and other providers + - Normalized typed responses from instructor/OpenAI/Groq to consistent dict format before parsing + - Added structured JSON fallback when typed generation yields zero relations to avoid silent empty outputs + - Removed acceptance of extra kwargs (`max_tokens`, `max_entities_prompt`) from relation extraction internals + - Filtered kwargs passed to provider LLM calls to only `temperature` and `verbose` +- **API Parameter Handling**: + - Limited kwargs forwarded in chunked extraction helper to prevent parameter leakage + - Ensured minimal, safe parameters are passed to provider calls +- **Pipeline Circular Import (Issues #192, #193)**: + - Fixed circular import between `pipeline_builder` and `pipeline_validator` triggered during `semantica.pipeline` import + - Lazy-loaded `PipelineValidator` inside `PipelineBuilder.__init__` and guarded type hints with `TYPE_CHECKING` + - Ensured `from semantica.deduplication import DuplicateDetector` no longer fails even when pipeline module is imported +- **JupyterLab Progress Output (Issue #181)**: + - Added `SEMANTICA_DISABLE_JUPYTER_PROGRESS` environment variable to disable rich Jupyter/Colab progress tables + - When enabled, progress falls back to console-style output, preventing infinite scrolling and JupyterLab out-of-memory errors + +### Added +- **Comprehensive Test Suite**: +- - Added unit tests (`tests/test_relations_llm.py`) with mocked LLM provider covering both typed and structured response paths +- - Added integration tests (`tests/integration/test_relations_groq.py`) for real Groq API calls with environment variable API key +- - Tests validate relation extraction completion and result parsing across different response formats +- **Amazon Neptune Dev Environment**: +- - Added CloudFormation template (`cookbook/introduction/neptune-setup.yaml`) to provision a dev Neptune cluster with public endpoint and IAM auth enabled +- - Documented deployment, cost estimates, and IAM User vs IAM Role best practices in `cookbook/introduction/21_Amazon_Neptune_Store.ipynb` +- - Added `cfn-lint` to `.pre-commit-config.yaml` for validating CloudFormation templates while excluding `neptune-setup.yaml` from generic YAML linters +- **Vector Store High-Performance Ingestion**: +- - Added `VectorStore.add_documents` for high-throughput ingestion with automatic embedding generation, batching, and parallel processing +- - Added `VectorStore.embed_batch` helper for generating embeddings for lists of texts without immediately storing them +- - Enabled default parallel ingestion in `VectorStore` with `max_workers=6` for common workloads +- - Added dedicated documentation page `docs/vector_store_usage.md` describing high-performance vector store usage and configuration +- - Added `tests/vector_store/test_vector_store_parallel.py` covering parallel vs sequential performance, error handling, and edge cases for `add_documents` and `embed_batch` + +### Changed +- **Relation Extraction API**: +- - Simplified parameter interface by removing unused kwargs that were previously ignored +- - Improved error handling and verbose logging for debugging relation extraction issues +- - Enhanced robustness of post-response parsing across different LLM providers +- **Vector Store Defaults and Examples**: +- - Standardized `VectorStore` default concurrency to `max_workers=6` for parallel ingestion +- - Updated vector store reference documentation and usage guides to rely on implicit defaults instead of requiring manual `max_workers` configuration in examples + + +## [0.2.2] - 2026-01-15 + +### Added +- **Parallel Extraction Engine**: + - Implemented high-throughput parallel batch processing across all core extractors (`NERExtractor`, `RelationExtractor`, `TripletExtractor`, `EventDetector`, `SemanticNetworkExtractor`) using `concurrent.futures.ThreadPoolExecutor`. + - Added `max_workers` configuration parameter (default: 1) to all extractor `extract()` methods, allowing users to tune concurrency based on available CPU cores or API rate limits. + - **Parallel Chunking**: Implemented parallel processing for large document chunking in `_extract_entities_chunked` and `_extract_relations_chunked`, significantly reducing latency for long-form text analysis. + - **Thread-Safe Progress Tracking**: Enhanced `ProgressTracker` to handle concurrent updates from multiple threads without race conditions during batch processing. +- **Semantic Extract Performance & Regression**: + - Added edge-case regression suite covering max worker defaults, LLM prompt entity filtering, and extractor reuse. + - Added a runnable real-use-case benchmark script for batch latency across `NERExtractor`, `RelationExtractor`, `TripletExtractor`, `EventDetector`, `SemanticAnalyzer`, and `SemanticNetworkExtractor`. + - Added Groq LLM smoke tests that exercise LLM-based entities/relations/triplets when `GROQ_API_KEY` is available via environment configuration. + +### Security +- **Credential Sanitization**: + - Removed hardcoded API keys from 8 cookbook notebooks to prevent secret leakage. + - Enforced environment variable usage for `GROQ_API_KEY` across all examples. +- **Secure Caching**: + - Updated `ExtractionCache` to exclude sensitive parameters (e.g., `api_key`, `token`, `password`) from cache key generation, preventing secret leakage and enabling safe cache sharing. + - Upgraded cache key hashing algorithm from MD5 to **SHA-256** for enhanced collision resistance and security. + +### Changed +- **Gemini SDK Migration**: + - Migrated `GeminiProvider` to use the new `google-genai` SDK (v0.1.0+) to address deprecation warnings. + - Implemented graceful fallback to `google.generativeai` for backward compatibility. +- **Dependency Resolution**: + - Pinned `opentelemetry-api` and `opentelemetry-sdk` to `1.37.0` to resolve pip conflicts. + - Updated `protobuf` and `grpcio` constraints for better stability. +- **Entity Filtering Scope**: + - Removed entity filtering from non-LLM extraction flows to avoid accuracy regressions. + - Applied entity downselection only to LLM relation prompt construction, while matching returned entities against the full original entity list. +- **Batch Concurrency Defaults**: + - Standardized `max_workers` defaulting across `semantic_extract` and tuned for low-latency: ML-backed methods default to single-worker, while pattern/regex/rules/LLM/huggingface methods use a higher parallelism default capped by CPU. + - Raised the global `optimization.max_workers` default to 8 for better throughput on batch workloads. + +### Performance +- **Bottleneck Optimization (GitHub Issue #186)**: + - **Resolved Bottleneck #1 (Sequential Processing)**: Replaced sequential `for` loops with parallel execution for both document-level batches and intra-document chunks. + - **Performance Gains**: Achieved **~1.89x speedup** in real-world extraction scenarios (tested with Groq `llama-3.3-70b-versatile` on standard datasets). + - **Initialization Optimization**: Refactored test suite to use class-level `setUpClass` for LLM provider initialization, eliminating redundant API client creation overhead. +- **Low-Latency Entity Matching**: + - Avoided heavyweight embedding stack imports on common matches by improving fast matching heuristics and short-circuiting before embedding similarity. + - Optimized entity matching to prioritize exact/substring/word-boundary matches and only fall back to embedding similarity when needed, reducing CPU overhead in LLM relation/triplet mapping. + + +## [0.2.1] - 2026-01-12 + +### Fixed +- **LLM Output Stability (Bug #176)**: + - Fixed incomplete JSON output issues by correctly propagating `max_tokens` parameter in `extract_relations_llm`. + - Implemented automatic error handling that halves chunk sizes and retries when LLM context or output limits are exceeded. + - Fixed `AttributeError` in provider integration by ensuring consistent parameter passing via `**kwargs`. +- **Constraint Relaxations**: + - Removed hardcoded `max_length` constraints from `Entity`, `Relation`, and `Triplet` classes to support long-form semantic extraction (e.g., long descriptions or names). +- Fixed orchestrator lazy property initialization and configuration normalization logic in `Orchestrator`. +- Resolved `AssertionError` in orchestrator tests by aligning test mocks with production component usage. +- Fixed dependency compatibility issues by pinning `protobuf>=5.29.1,<7.0` and `grpcio>=1.71.2`. +- Added missing dependencies `GitPython` and `chardet` to `pyproject.toml`. +- Verified and aligned `FileObject.text` property usage in GraphRAG notebooks for consistent content decoding. + +### Changed +- **Chunking Defaults**: + - Increased default `max_text_length` for auto-chunking to **64,000 characters** (from 32k/16k) for OpenAI, Anthropic, Gemini, Groq, and DeepSeek providers. + - Unified chunking logic across `extract_entities_llm`, `extract_relations_llm`, and `extract_triplets_llm`. +- **Groq Support**: + - Standardized Groq provider defaults to use `llama-3.3-70b-versatile` with a 64k context window. + - Added native support for `max_tokens` and `max_completion_tokens` to prevent output truncation. + +### Added +- **Testing**: + - Added `tests/reproduce_issue_176.py` to validate `max_tokens` propagation and chunking behavior across all extractors. + + +## [0.2.0] - 2026-01-10 + +### Added +- **Amazon Neptune Support**: + - Added `AmazonNeptuneStore` providing Amazon Neptune graph database integration via Bolt protocol and OpenCypher. + - Implemented `NeptuneAuthTokenManager` extending Neo4j AuthManager for AWS IAM SigV4 signing with automatic token refresh. + - Added robust connection handling: retry logic with backoff for transient errors (signature expired, connection closed) and driver recreation. + - Added `graph-amazon-neptune` optional dependency group (boto3, neo4j). + - Comprehensive test suite covering all GraphStore interface methods. +- **Docling Integration**: + - Added `DoclingParser` in `semantica.parse` for high-fidelity document parsing using the Docling library. + - Supports multi-format parsing (PDF, DOCX, PPTX, XLSX, HTML, images) with superior table extraction and structure understanding. + - Implemented as a standalone parser supporting local execution, OCR, and multiple export formats (Markdown, HTML, JSON). +- **Robust Extraction Fallbacks**: + - Implemented comprehensive fallback chains ("ML/LLM" -> "Pattern" -> "Last Resort") across `NERExtractor`, `RelationExtractor`, and `TripletExtractor` to prevent empty result lists. + - Added "Last Resort" pattern matching in `NERExtractor` to identify capitalized words as generic entities when all other methods fail. + - Added "Last Resort" adjacency-based relation extraction in `RelationExtractor` to create weak connections between adjacent entities if no relations are found. + - Added fallback logic in `TripletExtractor` to convert relations to triplets or use rule-based extraction if standard methods fail. +- **Provenance & Tracking**: + - Added count tracking to batch processing logs in `NERExtractor`, `RelationExtractor`, and `TripletExtractor`. + - Added `batch_index` and `document_id` to the metadata of all extracted entities, relations, triplets, semantic roles, and clusters for better traceability. +- **Semantic Extract Improvements**: + - Introduced `auto-chunking` for long text processing in LLM extraction methods (`extract_entities_llm`, `extract_relations_llm`, `extract_triplets_llm`). + - Added `silent_fail` parameter to LLM extraction methods for configurable error handling. + - Implemented robust JSON parsing and automatic retry logic (3 attempts with exponential backoff) in `BaseProvider` for all LLM providers. + - Enhanced `GroqProvider` with better diagnostics and connectivity testing. + - Added comprehensive entity, relation, and triplet deduplication for chunked extraction. + - Added `semantica/semantic_extract/schemas.py` with canonical Pydantic models for consistent structured output. +- **Testing**: + - Added comprehensive robustness test suite `tests/semantic_extract/test_robustness_fallback.py` for validating extraction fallbacks and metadata propagation. + - Added comprehensive unit test suite `tests/embeddings/test_model_switching.py` for verifying dynamic model transitions and dimension updates. + - Added end-to-end integration test suite for Knowledge Graph pipeline validation (GraphBuilder -> EntityResolver -> GraphAnalyzer). +- **Other**: + - Added missing dependencies `GitPython` and `chardet` to `pyproject.toml`. + - Robustified ID extraction across `CentralityCalculator`, `CommunityDetector`, and `ConnectivityAnalyzer` to handle various entity formats. + - Improved `Entity` class hashability and equality logic in `utils/types.py`. + +### Changed +- **Deduplication & Conflict Logic**: + - Removed internal deduplication logic from `NERExtractor`, `RelationExtractor`, and `TripletExtractor`. + - Removed consistency/conflict checking from `ExtractionValidator` to defer to dedicated `semantica/conflicts` module. + - Removed `_deduplicate_*` methods from `semantica/semantic_extract/methods.py`. +- **Batch Processing & Consistency**: + - Standardized batch processing across all extractors (`NERExtractor`, `RelationExtractor`, `TripletExtractor`, `SemanticNetworkExtractor`, `EventDetector`, `SemanticAnalyzer`, `CoreferenceResolver`) using a unified `extract`/`analyze`/`resolve` method pattern with progress tracking. + - Added provenance metadata (`batch_index`, `document_id`) to `SemanticNetwork` nodes/edges, `Event` objects, `SemanticRole` results, `CoreferenceChain` mentions, and `SemanticCluster` (tracking source `document_ids`). + - Updated `SemanticClusterer.cluster` and `SemanticAnalyzer.cluster_semantically` to accept list of dictionaries (with `content` and `id` keys) for better document tracking during clustering. + - Removed legacy `check_triplet_consistency` from `TripletExtractor`. + - Removed `validate_consistency` and `_check_consistency` from `ExtractionValidator`. +- **Weighted Scoring**: + - Clarified weighted confidence scoring (50% Method Confidence + 50% Type Similarity) in comments. + - Explicitly labeled "Type Similarity" as "user-provided" in code comments to remove ambiguity. +- **Refactoring**: + - Fixed orchestrator lazy property initialization and configuration normalization logic in `Orchestrator`. + - Verified and aligned `FileObject.text` property usage in GraphRAG notebooks for consistent content decoding. + +### Fixed +- **Critical Fixes**: + - Resolved `NameError` in `extraction_validator.py` by adding missing `Union` import. + - Resolved issues where extractors would return empty lists for valid input text when primary extraction methods failed. + - Fixed metadata initialization issue in batch processing where `batch_index` and `document_id` were occasionally missing from extracted items. + - Ensured `LLMExtraction` methods (`enhance_entities`, `enhance_relations`) return original input instead of failing or returning empty results when LLM providers are unavailable. +- **Component Fixes**: + - Fixed model switching bug in `TextEmbedder` where internal state was not cleared, preventing dynamic updates between `fastembed` and `sentence_transformers` (#160). + - Implemented model-intrinsic embedding dimension detection in `TextEmbedder` to ensure consistency between models and vector databases. + - Updated `set_model` to properly refresh configuration and dimensions during model switches. + - Fixed `TypeError: unhashable type: 'Entity'` in `GraphAnalyzer` when processing graphs with raw `Entity` objects or dictionaries in relationships (#159). + - Resolved `AssertionError` in orchestrator tests by aligning test mocks with production component usage. + - Fixed dependency compatibility issues by pinning `protobuf==4.25.3` and `grpcio==1.67.1`. + - Fixed a bug in `TripletExtractor` where the `validate_triplets` method was shadowed by an internal attribute. + - Fixed incorrect `TextSplitter` import path in the `semantic_extract.methods` module. + +## [0.1.1] - 2026-01-05 + +### Added +- Exported `DoclingParser` and `DoclingMetadata` from `semantica.parse` for easier access. +- Added comprehensive `DoclingParser` usage examples to README and documentation. +- Added Windows-specific troubleshooting note for PyTorch DLL issues. + +### Fixed +- Fixed `DoclingParser` import/export issues across platforms (Windows, Linux, Google Colab). +- Improved error messaging when optional `docling` dependency is missing. +- Fixed versioning inconsistencies across the framework. + +## [0.1.0] - 2025-12-31 + +### Added +- New command-line interface (`semantica` CLI) with support for knowledge base building and info commands. +- Integrated FastAPI-based REST API server for remote access to framework functionality. +- Dedicated background worker component for scalable task processing and pipeline execution. +- Framework-level versioning configuration for PyPI distribution. +- Automated release workflow with Trusted Publishing support. + +### Changed +- Updated versioning across the framework to 0.1.0. +- Refined entry point configurations in `pyproject.toml`. +- Improved lazy module loading for core framework components. + +## [0.0.5] - 2025-11-26 + +### Changed +- Configured Trusted Publishing for secure automated PyPI deployments + +## [0.0.4] - 2025-11-26 + +### Changed +- Fixed PyPI deployment issues from v0.0.3 + +## [0.0.3] - 2025-11-25 + +### Changed +- Simplified CI/CD workflows - removed failing tests and strict linting +- Combined release and PyPI publishing into single workflow +- Simplified security scanning to weekly pip-audit only +- Streamlined GitHub Actions configuration + +### Added +- Comprehensive issue templates (Bug, Feature, Documentation, Support, Grant/Partnership) +- Updated pull request template with clear guidelines +- Community support documentation (SUPPORT.md) +- Funding and sponsorship configuration (FUNDING.yml) +- GitHub configuration README for maintainers +- 10+ new domain-specific cookbook examples (Finance, Healthcare, Cybersecurity, etc.) + +### Removed +- Redundant scripts folder (8 shell/PowerShell scripts) +- Unnecessary automation workflows (label-issues, mark-answered) +- Excessive issue templates + +## [0.0.2] - 2025-11-25 + +### Changed +- Updated README with streamlined content and better examples +- Added more notebooks to cookbook +- Improved documentation structure + +## [0.0.1] - 2024-01-XX + +### Added +- Core framework architecture +- Universal data ingestion (multiple file formats) +- Semantic intelligence engine (NER, relation extraction, event detection) +- Knowledge graph construction with entity resolution +- 6-stage ontology generation pipeline +- GraphRAG engine for hybrid retrieval +- Multi-agent system infrastructure +- Production-ready quality assurance modules +- Comprehensive documentation with MkDocs +- Cookbook with interactive tutorials +- Support for multiple vector stores (Weaviate, Qdrant, FAISS) +- Support for multiple graph databases (Neo4j, NetworkX, RDFLib) +- Temporal knowledge graph support +- Conflict detection and resolution +- Deduplication and entity merging +- Schema template enforcement +- Seed data management +- Multi-format export (RDF, JSON-LD, CSV, GraphML) +- Visualization tools +- Pipeline orchestration +- Streaming support (Kafka, RabbitMQ, Kinesis) +- Context engineering for AI agents +- Reasoning and inference engine + +### Documentation +- Getting started guide +- API reference for all modules +- Concepts and architecture documentation +- Use case examples +- Cookbook tutorials +- Community projects showcase + +--- + +## Types of Changes + +- **Added** for new features +- **Changed** for changes in existing functionality +- **Deprecated** for soon-to-be removed features +- **Removed** for now removed features +- **Fixed** for any bug fixes +- **Security** for vulnerability fixes + +## Migration Guides + +When breaking changes are introduced, migration guides will be provided in the release notes and documentation. + +--- + +For detailed release notes, see [GitHub Releases](https://github.com/Hawksight-AI/semantica/releases). + +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +- Fixed: PolicyEngine latest version selection on ContextGraph; AgentContext fallback robustness and secure logging (PR #TBD by @KaifAhmad1) +- Tests: Added ContextGraph fallback and AgentContext smoke tests; full suite passing + +- **Context Engineering Enhancement** (PR #307 by @KaifAhmad1): + - Comprehensive decision tracking system with full lifecycle management (record → analyze → query → precedent → influence) + - Advanced KG algorithm integration: centrality analysis, community detection, node embeddings with ContextGraph + - Enhanced AgentContext with granular feature flags for decision tracking, KG algorithms, and vector store features + - PolicyException model replacing conflicting Exception name for meaningful business domain modeling + - GraphStore validation preventing runtime failures with explicit capability checking + - Hybrid search combining semantic, structural, and category similarity with configurable weights + - Decision influence analysis with centrality measures and causal chain tracking + - Policy management with versioning, compliance checking, and exception handling + - Production-ready architecture with audit trails, security, and scalability features + - 9 critical bug fixes: logging, security, audit trails, API compatibility, Cypher queries, centrality access, validation, naming + - Comprehensive documentation with usage guides, production examples, and API references + - 100% test coverage with all validation tests passing (9/9 tests) + - Enterprise-grade features for financial services, healthcare, legal, and business domains + - Complete backward compatibility with existing semantica components + - Performance optimizations: caching, indexing, and efficient graph operations + +- **Added PgVector Store Support** (PR #303 by @Sameer6305, @KaifAhmad1): + - Native PostgreSQL vector storage using pgvector extension with full integration + - Multiple distance metrics: cosine, L2/Euclidean, inner product with automatic score normalization + - Advanced indexing: HNSW and IVFFlat for approximate nearest neighbor search with tunable parameters + - JSONB metadata storage with flexible filtering capabilities and batch operations + - Connection pooling support with psycopg3/psycopg2 fallback and efficient resource management + - Comprehensive VectorStore integration with backend delegation and unified API + - Idempotent index creation and table management with safe migration support + - Production-ready security: SQL injection protection with psycopg_sql.SQL() and input validation + - Performance optimizations: UUID4-based IDs, batch executemany operations, connection pooling + - Full backward compatibility with existing vector store implementations + - 36+ comprehensive test cases with Docker integration and dependency skipping + - Complete documentation with setup guides, examples, and performance tuning + - CI/CD integration: resolved benchmark compatibility and fixed documentation links + +- **Improved Vector Store for Decision Tracking** (PR #293 by @KaifAhmad1): + - Comprehensive decision tracking capabilities with hybrid search combining semantic and structural embeddings + - New DecisionEmbeddingPipeline for generating semantic and structural embeddings with KG algorithm integration + - HybridSimilarityCalculator with configurable weights (semantic: 0.7, structural: 0.3) + - DecisionContext high-level interface for decision management with explainable AI features + - ContextRetriever with hybrid precedent search and multi-hop reasoning + - User-friendly convenience API: quick_decision(), find_precedents(), explain(), similar_to(), batch_decisions(), filter_decisions() + - Knowledge Graph algorithm integration: Node2Vec, PathFinder, CommunityDetector, CentralityCalculator, SimilarityCalculator, ConnectivityAnalyzer + - Explainable AI with path tracing, confidence scoring, and comprehensive decision explanations + - Performance optimizations: 0.028s per decision processing, 0.031s search performance, ~0.8KB per decision memory usage + - 100% backward compatibility maintained with existing VectorStore functionality + - 34+ comprehensive tests covering all functionality including end-to-end scenarios and performance benchmarks + - Real-world validation examples for banking and insurance domains + - Documentation with clear imports, examples, and API references + +- **Improved Graph Algorithms in KG Module** (PR #292 by @KaifAhmad1): + - Complete algorithm suite with 30+ graph algorithms across 7 categories + - Node Embeddings: Node2Vec, DeepWalk, Word2Vec for structural similarity analysis + - Similarity Analysis: Cosine, Euclidean, Manhattan, Correlation metrics with batch processing + - Path Finding: Dijkstra, A*, BFS, K-shortest paths for route and network analysis + - Link Prediction: Preferential attachment, Jaccard, Adamic-Adar for network completion + - Centrality Analysis: Degree, Betweenness, Closeness, PageRank for importance ranking + - Community Detection: Louvain, Leiden, Label propagation for clustering analysis + - Connectivity Analysis: Components, bridges, density for network robustness + - Unified provenance tracking system with GraphBuilderWithProvenance and AlgorithmTrackerWithProvenance + - Complete execution tracking with metadata, timestamps, and reproducibility IDs + - Comprehensive test coverage with 5 test suites and 40+ test methods + - Professional documentation overhaul for all modules and reference documentation + - Enterprise-ready functionality with error handling and NetworkX compatibility + - Performance optimizations with sparse matrix operations and batch processing + - Full backward compatibility maintained with gradual migration support + +- **Improved Security Configuration with Dependabot**: + - Configured bi-weekly security updates with manual review by @KaifAhmad1 + - Implemented automated security scans (Monday & Thursday at 7 AM IST) with Bandit, Safety, Semgrep + - Added security-critical package grouping (cryptography, requests, urllib3, certifi, pyopenssl) + - Enterprise-grade security with audit trail, compliance features, and zero auto-merge + - Optimized IST timezone scheduling (Security scans: 7 AM IST, PRs: 9 AM IST) + - Aligned with new Dependabot features: open-source proxy support, smart dependency grouping for Snowflake/Arrow/benchmark features, private registry support, semantic commit prefixes, and latest GitHub security best practices + +- **ResourceScheduler Deadlock Fix and Performance Improvements** (PR #299, #301 by @d4ndr4d3, @KaifAhmad1): + - Fixed critical deadlock in ResourceScheduler by replacing `threading.Lock()` with `threading.RLock()` + - Resolved nested lock acquisition issue in `allocate_resources()` → `allocate_cpu/memory/gpu()` calls + - Added allocation validation with `ValidationError` when no resources can be allocated + - Improved performance by moving progress tracking updates outside lock scope + - Implemented comprehensive resource cleanup on allocation failures to prevent leaks + - Added complete regression test suite (6 tests) for deadlock prevention and edge cases + - Improved error handling and documentation for better operator visibility + - Zero breaking changes, maintains thread safety and backward compatibility + +## [0.2.7] - 2026-02-09 + +### Added / Changed + +- **Snowflake Connector for Data Ingestion** (PR #276 by @Sameer6305): + - Native Snowflake connector with multi-authentication (password, OAuth, key-pair, SSO) + - Table and query ingestion with pagination, schema introspection, batch processing + - SQL injection prevention via identifier escaping, OAuth token validation + - Progress tracking integration, context manager support, document export + - 24 comprehensive unit tests with mocking, complete documentation and examples + - Added as optional dependency `db-snowflake` with snowflake-connector-python>=3.0.0 + +- **Apache Arrow Export Support** (PR #273 by @Sameer6305): + - Added Apache Arrow exporter with explicit schemas, entity/relationship export, compression support + - Integrated with export module and method registry, Pandas/DuckDB compatible + - 20 unit tests + 1 integration test, complete documentation with examples + +- **Comprehensive Benchmark Suite with Regression CLI** (PR #289 by @ZohaibHassan16, @KaifAhmad1): + - 137+ benchmarks across all 10 Semantica modules (Input, Core, Storage, Context, QA, Ontology, etc.) + - Environment-agnostic design with robust mocking system for CI/CD compatibility + - Statistical regression detection using Z-score analysis with configurable thresholds + - Automated performance auditing via GitHub Actions workflow + - Comprehensive documentation suite (benchmarks.md, architecture guides, usage examples) + - Zero breaking changes, production-ready with ultra-fast text processing (>10,000 ops/s) + - Added benchmark runner CLI: `python benchmarks/benchmark_runner.py` + +## [0.2.6] - 2026-02-03 + +### Added / Changed + +- **W3C PROV-O Compliant Provenance Tracking** (#254, #246): + - Comprehensive provenance tracking system with W3C PROV-O compliance across all 17 Semantica modules + - **Core Module**: `ProvenanceManager`, W3C PROV-O schemas, storage backends (InMemory, SQLite), SHA-256 integrity verification + - **Module Integrations**: Semantic Extract, LLMs (Groq, OpenAI, HuggingFace, LiteLLM), Pipeline, Context, Ingest, Embeddings, Graph/Vector/Triplet stores, Reasoning, Conflicts, Deduplication, Export, Parse, Normalize, Ontology, Visualization + - **Features**: Complete lineage tracking (Document → Chunk → Entity → Relationship → Graph), LLM tracking (tokens, costs, latency), source tracking, bridge axioms for domain transformations + - **Compliance Infrastructure**: W3C PROV-O, FDA 21 CFR Part 11, SOX, HIPAA, TNFD + - **Testing**: 237 tests covering core functionality, all 17 module integrations, edge cases, backward compatibility + - **Design**: Opt-in with `provenance=False` by default, zero breaking changes, no new dependencies + - Contributed by @KaifAhmad1 + +- **Enhanced Change Management Module** (#248, #243): + - Enterprise-grade version control for knowledge graphs and ontologies with persistent storage and audit trails + - **Core Classes**: `TemporalVersionManager` (KG versioning), `OntologyVersionManager` (ontology versioning), `ChangeLogEntry` (metadata) + - **Storage**: SQLite (persistent) and in-memory backends with thread-safe operations + - **Features**: SHA-256 checksums, detailed entity/relationship diffs, structural ontology comparison, email validation + - **Compliance Infrastructure**: HIPAA, SOX, FDA 21 CFR Part 11 with immutable audit trails + - **Testing**: 104 tests (100% pass) - unit, integration, compliance, performance, edge cases + - **Performance**: 17.6ms for 10k entities, 510+ ops/sec concurrent, handles 5k+ entity graphs + - **Migration**: Backward compatible, simplified class names, zero external dependencies + - Contributed by @KaifAhmad1 + +- CSV Ingestion Enhancements (PR #244 by @saloni0318) + - Auto-detect CSV encoding (chardet) and delimiter (csv.Sniffer) + - Tolerant decoding and malformed-row handling (`on_bad_lines='warn'`) + - Optional chunked reading for large files; metadata tracks detected values + - Expanded unit tests covering delimiters, quoted/multiline fields, header overrides, chunks, and NaN preservation + +- Tests: Comprehensive units for TextNormalizer (PR #242 by @ZohaibHassan16) + - Added focused test coverage for TextNormalizer behavior across inputs + +- Tests: Register integration mark and tidy ingest test warnings (PR #241 by @KaifAhmad1) + - Introduced integration test marker and reduced noisy warnings in ingest tests + +- **Ingest Unit Tests** (#239, #232): + - Comprehensive unit tests for ingestion modules (file, web, and feed ingestors) + - **Coverage**: File scanning (local/cloud S3/GCS/Azure), web ingestion (URL/sitemap/robots.txt), RSS/Atom feed parsing + - **Testing**: 998 lines of test code with mocked external dependencies for fast, isolated execution + - **Results**: file_ingestor (86%), web_ingestor (86%), feed_ingestor (80%) coverage + - Covers happy paths, edge cases, and error handling + - Contributed by @Mohammed2372 + +### Fixed + +- **Temperature Compatibility Fix** (#256, #252): + - Fixed hardcoded `temperature=0.3` that broke compatibility with models requiring specific temperature values (e.g., gpt-5-mini) + - Added `_add_if_set` helper method to `BaseProvider` that only passes parameters when explicitly set + - When `temperature=None`, parameter is omitted allowing APIs to use model defaults + - Updated all 5 providers: OpenAI, Groq, Gemini, Ollama, DeepSeek + - Reduced code by ~85 lines with cleaner parameter handling + - Comprehensive test coverage added (10 temperature tests, all passing) + - Backward compatible - no breaking changes + - Contributed by @F0rt1s and @IGES-Institut + +- **JenaStore Empty Graph Bug** (#257, #258): + - Fixed `ProcessingError: Graph not initialized` when operating on empty (but initialized) graphs + - Replaced implicit `if not self.graph:` checks with explicit `if self.graph is None:` validation in 5 methods (`add_triplets`, `get_triplets`, `delete_triplet`, `execute_sparql`, `serialize`) + - Properly distinguishes `None` (uninitialized) from empty graphs (initialized with 0 triplets) + - Unblocks benchmarking suite, fresh deployments, and testing workflows + - Contributed by @ZohaibHassan16 + +## [0.2.5] - 2026-01-27 + +### Added +- **Pinecone Vector Store Support**: + - Implemented native Pinecone support (`PineconeStore`) with full CRUD capabilities. + - Added support for serverless and pod-based indexes, namespaces, and metadata filtering. + - Integrated with `VectorStore` unified interface and registry. + - (Closes #219, Resolves #220) +- **Configurable LLM Retry Logic**: + - Exposed `max_retries` parameter in `NERExtractor`, `RelationExtractor`, `TripletExtractor` and low-level extraction methods (`extract_entities_llm`, `extract_relations_llm`, `extract_triplets_llm`). + - Defaults to 3 retries to prevent infinite loops during JSON validation failures or API timeouts. + - Propagated retry configuration through chunked processing helpers to ensure consistent behavior for long documents. + - Updated `03_Earnings_Call_Analysis.ipynb` to use `max_retries=3` by default. + +### Added +- **Bring Your Own Model (BYOM) Support**: + - Enabled full support for custom Hugging Face models in `NERExtractor`, `RelationExtractor`, and `TripletExtractor`. + - Added support for custom tokenizers in `HuggingFaceModelLoader` to handle models with non-standard tokenization requirements. + - Implemented robust fallback logic for model selection: runtime options (`extract(model=...)`) now correctly override configuration defaults. +- **Enhanced NER Implementation**: + - Added configurable aggregation strategies (`simple`, `first`, `average`, `max`) to `extract_entities_huggingface` for better sub-word token handling. + - Implemented robust IOB/BILOU parsing to reconstruct entities from raw model outputs when structured output is unavailable. + - Added confidence scoring for aggregated entities. +- **Relation Extraction Improvements**: + - Implemented standard entity marker technique (wrapping subject/object with ``, `` tags) in `extract_relations_huggingface` for compatibility with sequence classification models. + - Added structured output parsing to convert raw model predictions into validated `Relation` objects. +- **Triplet Extraction Completion**: + - Added specialized parsing for Seq2Seq models (e.g., REBEL) in `extract_triplets_huggingface` to generate structured triplets directly from text. + - Implemented post-processing logic to clean and validate generated triplets. + +### Fixed +- **LLM Extraction Stability**: + - Fixed infinite retry loops in `BaseProvider` by strictly enforcing `max_retries` limit during structured output generation. + - Resolved stuck execution in earnings call analysis notebooks when using smaller models (e.g., Llama 3 8B) that frequently produce invalid JSON. +- **Model Parameter Precedence**: + - Fixed issue where configuration defaults took precedence over runtime arguments in Hugging Face extractors. Runtime options now correctly override config values. +- **Import Handling**: + - Fixed circular import issues in test suites by implementing robust mocking strategies. + +## [0.2.4] - 2026-01-22 + +### Added +- **Ontology Ingestion Module**: + - Implemented `OntologyIngestor` in `semantica.ingest` for parsing RDF/OWL files (Turtle, RDF/XML, JSON-LD, N3) into standardized `OntologyData` objects. + - Added `ingest_ontology` convenience function and integrated it into the unified `ingest(source_type="ontology")` interface. + - Added recursive directory scanning support for batch ontology ingestion. + - Exposed ingestion tools in `semantica.ontology` for better discoverability. + - Added `OntologyData` dataclass for consistent metadata handling (source path, format, timestamps). +- **Documentation**: + - **Ontology Usage Guide**: Updated `ontology_usage.md` with comprehensive examples for single-file and directory ingestion. + - **API Reference**: Updated `ontology.md` with `OntologyIngestor` class documentation and method details. +- **Tests**: + - **Comprehensive Test Suite**: Added `tests/ingest/test_ontology_ingestor.py` covering all supported formats, error handling, and unified interface integration. + - **Demo Script**: Added `examples/demo_ontology_ingest.py` for end-to-end usage demonstration. + +## [0.2.3] - 2026-01-20 + +### Fixed +- **LLM Relation Extraction Parsing**: + - Fixed relation extraction returning zero relations despite successful API calls to Groq and other providers + - Normalized typed responses from instructor/OpenAI/Groq to consistent dict format before parsing + - Added structured JSON fallback when typed generation yields zero relations to avoid silent empty outputs + - Removed acceptance of extra kwargs (`max_tokens`, `max_entities_prompt`) from relation extraction internals + - Filtered kwargs passed to provider LLM calls to only `temperature` and `verbose` +- **API Parameter Handling**: + - Limited kwargs forwarded in chunked extraction helper to prevent parameter leakage + - Ensured minimal, safe parameters are passed to provider calls +- **Pipeline Circular Import (Issues #192, #193)**: + - Fixed circular import between `pipeline_builder` and `pipeline_validator` triggered during `semantica.pipeline` import + - Lazy-loaded `PipelineValidator` inside `PipelineBuilder.__init__` and guarded type hints with `TYPE_CHECKING` + - Ensured `from semantica.deduplication import DuplicateDetector` no longer fails even when pipeline module is imported +- **JupyterLab Progress Output (Issue #181)**: + - Added `SEMANTICA_DISABLE_JUPYTER_PROGRESS` environment variable to disable rich Jupyter/Colab progress tables + - When enabled, progress falls back to console-style output, preventing infinite scrolling and JupyterLab out-of-memory errors + +### Added +- **Comprehensive Test Suite**: +- - Added unit tests (`tests/test_relations_llm.py`) with mocked LLM provider covering both typed and structured response paths +- - Added integration tests (`tests/integration/test_relations_groq.py`) for real Groq API calls with environment variable API key +- - Tests validate relation extraction completion and result parsing across different response formats +- **Amazon Neptune Dev Environment**: +- - Added CloudFormation template (`cookbook/introduction/neptune-setup.yaml`) to provision a dev Neptune cluster with public endpoint and IAM auth enabled +- - Documented deployment, cost estimates, and IAM User vs IAM Role best practices in `cookbook/introduction/21_Amazon_Neptune_Store.ipynb` +- - Added `cfn-lint` to `.pre-commit-config.yaml` for validating CloudFormation templates while excluding `neptune-setup.yaml` from generic YAML linters +- **Vector Store High-Performance Ingestion**: +- - Added `VectorStore.add_documents` for high-throughput ingestion with automatic embedding generation, batching, and parallel processing +- - Added `VectorStore.embed_batch` helper for generating embeddings for lists of texts without immediately storing them +- - Enabled default parallel ingestion in `VectorStore` with `max_workers=6` for common workloads +- - Added dedicated documentation page `docs/vector_store_usage.md` describing high-performance vector store usage and configuration +- - Added `tests/vector_store/test_vector_store_parallel.py` covering parallel vs sequential performance, error handling, and edge cases for `add_documents` and `embed_batch` + +### Changed +- **Relation Extraction API**: +- - Simplified parameter interface by removing unused kwargs that were previously ignored +- - Improved error handling and verbose logging for debugging relation extraction issues +- - Enhanced robustness of post-response parsing across different LLM providers +- **Vector Store Defaults and Examples**: +- - Standardized `VectorStore` default concurrency to `max_workers=6` for parallel ingestion +- - Updated vector store reference documentation and usage guides to rely on implicit defaults instead of requiring manual `max_workers` configuration in examples + + +## [0.2.2] - 2026-01-15 + +### Added +- **Parallel Extraction Engine**: + - Implemented high-throughput parallel batch processing across all core extractors (`NERExtractor`, `RelationExtractor`, `TripletExtractor`, `EventDetector`, `SemanticNetworkExtractor`) using `concurrent.futures.ThreadPoolExecutor`. + - Added `max_workers` configuration parameter (default: 1) to all extractor `extract()` methods, allowing users to tune concurrency based on available CPU cores or API rate limits. + - **Parallel Chunking**: Implemented parallel processing for large document chunking in `_extract_entities_chunked` and `_extract_relations_chunked`, significantly reducing latency for long-form text analysis. + - **Thread-Safe Progress Tracking**: Enhanced `ProgressTracker` to handle concurrent updates from multiple threads without race conditions during batch processing. +- **Semantic Extract Performance & Regression**: + - Added edge-case regression suite covering max worker defaults, LLM prompt entity filtering, and extractor reuse. + - Added a runnable real-use-case benchmark script for batch latency across `NERExtractor`, `RelationExtractor`, `TripletExtractor`, `EventDetector`, `SemanticAnalyzer`, and `SemanticNetworkExtractor`. + - Added Groq LLM smoke tests that exercise LLM-based entities/relations/triplets when `GROQ_API_KEY` is available via environment configuration. + +### Security +- **Credential Sanitization**: + - Removed hardcoded API keys from 8 cookbook notebooks to prevent secret leakage. + - Enforced environment variable usage for `GROQ_API_KEY` across all examples. +- **Secure Caching**: + - Updated `ExtractionCache` to exclude sensitive parameters (e.g., `api_key`, `token`, `password`) from cache key generation, preventing secret leakage and enabling safe cache sharing. + - Upgraded cache key hashing algorithm from MD5 to **SHA-256** for enhanced collision resistance and security. + +### Changed +- **Gemini SDK Migration**: + - Migrated `GeminiProvider` to use the new `google-genai` SDK (v0.1.0+) to address deprecation warnings. + - Implemented graceful fallback to `google.generativeai` for backward compatibility. +- **Dependency Resolution**: + - Pinned `opentelemetry-api` and `opentelemetry-sdk` to `1.37.0` to resolve pip conflicts. + - Updated `protobuf` and `grpcio` constraints for better stability. +- **Entity Filtering Scope**: + - Removed entity filtering from non-LLM extraction flows to avoid accuracy regressions. + - Applied entity downselection only to LLM relation prompt construction, while matching returned entities against the full original entity list. +- **Batch Concurrency Defaults**: + - Standardized `max_workers` defaulting across `semantic_extract` and tuned for low-latency: ML-backed methods default to single-worker, while pattern/regex/rules/LLM/huggingface methods use a higher parallelism default capped by CPU. + - Raised the global `optimization.max_workers` default to 8 for better throughput on batch workloads. + +### Performance +- **Bottleneck Optimization (GitHub Issue #186)**: + - **Resolved Bottleneck #1 (Sequential Processing)**: Replaced sequential `for` loops with parallel execution for both document-level batches and intra-document chunks. + - **Performance Gains**: Achieved **~1.89x speedup** in real-world extraction scenarios (tested with Groq `llama-3.3-70b-versatile` on standard datasets). + - **Initialization Optimization**: Refactored test suite to use class-level `setUpClass` for LLM provider initialization, eliminating redundant API client creation overhead. +- **Low-Latency Entity Matching**: + - Avoided heavyweight embedding stack imports on common matches by improving fast matching heuristics and short-circuiting before embedding similarity. + - Optimized entity matching to prioritize exact/substring/word-boundary matches and only fall back to embedding similarity when needed, reducing CPU overhead in LLM relation/triplet mapping. + + +## [0.2.1] - 2026-01-12 + +### Fixed +- **LLM Output Stability (Bug #176)**: + - Fixed incomplete JSON output issues by correctly propagating `max_tokens` parameter in `extract_relations_llm`. + - Implemented automatic error handling that halves chunk sizes and retries when LLM context or output limits are exceeded. + - Fixed `AttributeError` in provider integration by ensuring consistent parameter passing via `**kwargs`. +- **Constraint Relaxations**: + - Removed hardcoded `max_length` constraints from `Entity`, `Relation`, and `Triplet` classes to support long-form semantic extraction (e.g., long descriptions or names). +- Fixed orchestrator lazy property initialization and configuration normalization logic in `Orchestrator`. +- Resolved `AssertionError` in orchestrator tests by aligning test mocks with production component usage. +- Fixed dependency compatibility issues by pinning `protobuf>=5.29.1,<7.0` and `grpcio>=1.71.2`. +- Added missing dependencies `GitPython` and `chardet` to `pyproject.toml`. +- Verified and aligned `FileObject.text` property usage in GraphRAG notebooks for consistent content decoding. + +### Changed +- **Chunking Defaults**: + - Increased default `max_text_length` for auto-chunking to **64,000 characters** (from 32k/16k) for OpenAI, Anthropic, Gemini, Groq, and DeepSeek providers. + - Unified chunking logic across `extract_entities_llm`, `extract_relations_llm`, and `extract_triplets_llm`. +- **Groq Support**: + - Standardized Groq provider defaults to use `llama-3.3-70b-versatile` with a 64k context window. + - Added native support for `max_tokens` and `max_completion_tokens` to prevent output truncation. + +### Added +- **Testing**: + - Added `tests/reproduce_issue_176.py` to validate `max_tokens` propagation and chunking behavior across all extractors. + + +## [0.2.0] - 2026-01-10 + +### Added +- **Amazon Neptune Support**: + - Added `AmazonNeptuneStore` providing Amazon Neptune graph database integration via Bolt protocol and OpenCypher. + - Implemented `NeptuneAuthTokenManager` extending Neo4j AuthManager for AWS IAM SigV4 signing with automatic token refresh. + - Added robust connection handling: retry logic with backoff for transient errors (signature expired, connection closed) and driver recreation. + - Added `graph-amazon-neptune` optional dependency group (boto3, neo4j). + - Comprehensive test suite covering all GraphStore interface methods. +- **Docling Integration**: + - Added `DoclingParser` in `semantica.parse` for high-fidelity document parsing using the Docling library. + - Supports multi-format parsing (PDF, DOCX, PPTX, XLSX, HTML, images) with superior table extraction and structure understanding. + - Implemented as a standalone parser supporting local execution, OCR, and multiple export formats (Markdown, HTML, JSON). +- **Robust Extraction Fallbacks**: + - Implemented comprehensive fallback chains ("ML/LLM" -> "Pattern" -> "Last Resort") across `NERExtractor`, `RelationExtractor`, and `TripletExtractor` to prevent empty result lists. + - Added "Last Resort" pattern matching in `NERExtractor` to identify capitalized words as generic entities when all other methods fail. + - Added "Last Resort" adjacency-based relation extraction in `RelationExtractor` to create weak connections between adjacent entities if no relations are found. + - Added fallback logic in `TripletExtractor` to convert relations to triplets or use rule-based extraction if standard methods fail. +- **Provenance & Tracking**: + - Added count tracking to batch processing logs in `NERExtractor`, `RelationExtractor`, and `TripletExtractor`. + - Added `batch_index` and `document_id` to the metadata of all extracted entities, relations, triplets, semantic roles, and clusters for better traceability. +- **Semantic Extract Improvements**: + - Introduced `auto-chunking` for long text processing in LLM extraction methods (`extract_entities_llm`, `extract_relations_llm`, `extract_triplets_llm`). + - Added `silent_fail` parameter to LLM extraction methods for configurable error handling. + - Implemented robust JSON parsing and automatic retry logic (3 attempts with exponential backoff) in `BaseProvider` for all LLM providers. + - Enhanced `GroqProvider` with better diagnostics and connectivity testing. + - Added comprehensive entity, relation, and triplet deduplication for chunked extraction. + - Added `semantica/semantic_extract/schemas.py` with canonical Pydantic models for consistent structured output. +- **Testing**: + - Added comprehensive robustness test suite `tests/semantic_extract/test_robustness_fallback.py` for validating extraction fallbacks and metadata propagation. + - Added comprehensive unit test suite `tests/embeddings/test_model_switching.py` for verifying dynamic model transitions and dimension updates. + - Added end-to-end integration test suite for Knowledge Graph pipeline validation (GraphBuilder -> EntityResolver -> GraphAnalyzer). +- **Other**: + - Added missing dependencies `GitPython` and `chardet` to `pyproject.toml`. + - Robustified ID extraction across `CentralityCalculator`, `CommunityDetector`, and `ConnectivityAnalyzer` to handle various entity formats. + - Improved `Entity` class hashability and equality logic in `utils/types.py`. + +### Changed +- **Deduplication & Conflict Logic**: + - Removed internal deduplication logic from `NERExtractor`, `RelationExtractor`, and `TripletExtractor`. + - Removed consistency/conflict checking from `ExtractionValidator` to defer to dedicated `semantica/conflicts` module. + - Removed `_deduplicate_*` methods from `semantica/semantic_extract/methods.py`. +- **Batch Processing & Consistency**: + - Standardized batch processing across all extractors (`NERExtractor`, `RelationExtractor`, `TripletExtractor`, `SemanticNetworkExtractor`, `EventDetector`, `SemanticAnalyzer`, `CoreferenceResolver`) using a unified `extract`/`analyze`/`resolve` method pattern with progress tracking. + - Added provenance metadata (`batch_index`, `document_id`) to `SemanticNetwork` nodes/edges, `Event` objects, `SemanticRole` results, `CoreferenceChain` mentions, and `SemanticCluster` (tracking source `document_ids`). + - Updated `SemanticClusterer.cluster` and `SemanticAnalyzer.cluster_semantically` to accept list of dictionaries (with `content` and `id` keys) for better document tracking during clustering. + - Removed legacy `check_triplet_consistency` from `TripletExtractor`. + - Removed `validate_consistency` and `_check_consistency` from `ExtractionValidator`. +- **Weighted Scoring**: + - Clarified weighted confidence scoring (50% Method Confidence + 50% Type Similarity) in comments. + - Explicitly labeled "Type Similarity" as "user-provided" in code comments to remove ambiguity. +- **Refactoring**: + - Fixed orchestrator lazy property initialization and configuration normalization logic in `Orchestrator`. + - Verified and aligned `FileObject.text` property usage in GraphRAG notebooks for consistent content decoding. + +### Fixed +- **Critical Fixes**: + - Resolved `NameError` in `extraction_validator.py` by adding missing `Union` import. + - Resolved issues where extractors would return empty lists for valid input text when primary extraction methods failed. + - Fixed metadata initialization issue in batch processing where `batch_index` and `document_id` were occasionally missing from extracted items. + - Ensured `LLMExtraction` methods (`enhance_entities`, `enhance_relations`) return original input instead of failing or returning empty results when LLM providers are unavailable. +- **Component Fixes**: + - Fixed model switching bug in `TextEmbedder` where internal state was not cleared, preventing dynamic updates between `fastembed` and `sentence_transformers` (#160). + - Implemented model-intrinsic embedding dimension detection in `TextEmbedder` to ensure consistency between models and vector databases. + - Updated `set_model` to properly refresh configuration and dimensions during model switches. + - Fixed `TypeError: unhashable type: 'Entity'` in `GraphAnalyzer` when processing graphs with raw `Entity` objects or dictionaries in relationships (#159). + - Resolved `AssertionError` in orchestrator tests by aligning test mocks with production component usage. + - Fixed dependency compatibility issues by pinning `protobuf==4.25.3` and `grpcio==1.67.1`. + - Fixed a bug in `TripletExtractor` where the `validate_triplets` method was shadowed by an internal attribute. + - Fixed incorrect `TextSplitter` import path in the `semantic_extract.methods` module. + +## [0.1.1] - 2026-01-05 + +### Added +- Exported `DoclingParser` and `DoclingMetadata` from `semantica.parse` for easier access. +- Added comprehensive `DoclingParser` usage examples to README and documentation. +- Added Windows-specific troubleshooting note for PyTorch DLL issues. + +### Fixed +- Fixed `DoclingParser` import/export issues across platforms (Windows, Linux, Google Colab). +- Improved error messaging when optional `docling` dependency is missing. +- Fixed versioning inconsistencies across the framework. + +## [0.1.0] - 2025-12-31 + +### Added +- New command-line interface (`semantica` CLI) with support for knowledge base building and info commands. +- Integrated FastAPI-based REST API server for remote access to framework functionality. +- Dedicated background worker component for scalable task processing and pipeline execution. +- Framework-level versioning configuration for PyPI distribution. +- Automated release workflow with Trusted Publishing support. + +### Changed +- Updated versioning across the framework to 0.1.0. +- Refined entry point configurations in `pyproject.toml`. +- Improved lazy module loading for core framework components. + +## [0.0.5] - 2025-11-26 + +### Changed +- Configured Trusted Publishing for secure automated PyPI deployments + +## [0.0.4] - 2025-11-26 + +### Changed +- Fixed PyPI deployment issues from v0.0.3 + +## [0.0.3] - 2025-11-25 + +### Changed +- Simplified CI/CD workflows - removed failing tests and strict linting +- Combined release and PyPI publishing into single workflow +- Simplified security scanning to weekly pip-audit only +- Streamlined GitHub Actions configuration + +### Added +- Comprehensive issue templates (Bug, Feature, Documentation, Support, Grant/Partnership) +- Updated pull request template with clear guidelines +- Community support documentation (SUPPORT.md) +- Funding and sponsorship configuration (FUNDING.yml) +- GitHub configuration README for maintainers +- 10+ new domain-specific cookbook examples (Finance, Healthcare, Cybersecurity, etc.) + +### Removed +- Redundant scripts folder (8 shell/PowerShell scripts) +- Unnecessary automation workflows (label-issues, mark-answered) +- Excessive issue templates + +## [0.0.2] - 2025-11-25 + +### Changed +- Updated README with streamlined content and better examples +- Added more notebooks to cookbook +- Improved documentation structure + +## [0.0.1] - 2024-01-XX + +### Added +- Core framework architecture +- Universal data ingestion (multiple file formats) +- Semantic intelligence engine (NER, relation extraction, event detection) +- Knowledge graph construction with entity resolution +- 6-stage ontology generation pipeline +- GraphRAG engine for hybrid retrieval +- Multi-agent system infrastructure +- Production-ready quality assurance modules +- Comprehensive documentation with MkDocs +- Cookbook with interactive tutorials +- Support for multiple vector stores (Weaviate, Qdrant, FAISS) +- Support for multiple graph databases (Neo4j, NetworkX, RDFLib) +- Temporal knowledge graph support +- Conflict detection and resolution +- Deduplication and entity merging +- Schema template enforcement +- Seed data management +- Multi-format export (RDF, JSON-LD, CSV, GraphML) +- Visualization tools +- Pipeline orchestration +- Streaming support (Kafka, RabbitMQ, Kinesis) +- Context engineering for AI agents +- Reasoning and inference engine + +### Documentation +- Getting started guide +- API reference for all modules +- Concepts and architecture documentation +- Use case examples +- Cookbook tutorials +- Community projects showcase + +--- + +## Types of Changes + +- **Added** for new features +- **Changed** for changes in existing functionality +- **Deprecated** for soon-to-be removed features +- **Removed** for now removed features +- **Fixed** for any bug fixes +- **Security** for vulnerability fixes + +## Migration Guides + +When breaking changes are introduced, migration guides will be provided in the release notes and documentation. + +--- + +For detailed release notes, see [GitHub Releases](https://github.com/Hawksight-AI/semantica/releases). + +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +- Fixed: PolicyEngine latest version selection on ContextGraph; AgentContext fallback robustness and secure logging (PR #TBD by @KaifAhmad1) +- Tests: Added ContextGraph fallback and AgentContext smoke tests; full suite passing + - **Context Engineering Enhancement** (PR #307 by @KaifAhmad1): - Comprehensive decision tracking system with full lifecycle management (record → analyze → query → precedent → influence) - Advanced KG algorithm integration: centrality analysis, community detection, node embeddings with ContextGraph From 780f8adfbee3e21850b595c99cb9575701bc133d Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Sat, 14 Feb 2026 22:32:43 +0530 Subject: [PATCH 06/50] Delete .all-contributorsrc (#314) --- .all-contributorsrc | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .all-contributorsrc diff --git a/.all-contributorsrc b/.all-contributorsrc deleted file mode 100644 index 77741d45..00000000 --- a/.all-contributorsrc +++ /dev/null @@ -1,17 +0,0 @@ -{ - "projectName": "Semantica", - "projectOwner": "Hawksight-AI", - "repoType": "github", - "repoHost": "https://github.com", - "files": [ - "CONTRIBUTORS.md" - ], - "imageSize": 100, - "commit": true, - "commitConvention": "conventional", - "contributors": [], - "contributorsPerLine": 7, - "badgeTemplate": "[![All Contributors](https://img.shields.io/badge/all_contributors-<%= contributors.length %>-orange.svg?style=flat-square)](#contributors)", - "skipCi": true -} - From 4e31296c1eb6673b6efc5b3d467ec17bbc149f07 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Sun, 15 Feb 2026 12:52:19 +0530 Subject: [PATCH 07/50] Fix Context Graphs decision tracking and add comprehensive tests - Fix empty/None decision ID handling in ContextGraph.add_decision() - Fix None metadata handling to prevent TypeError - Fix causal chain depth logic and node exclusion - Fix nonexistent node handling in add_causal_relationship() - Add missing properties field in to_dict serialization - Add missing from_dict method for graph deserialization - Fix precedent search direction in find_precedents() - Fix UUID generation logic in all decision models - Add comprehensive test suite with 9 tests covering all features - Test coverage: decision tracking, graph analytics, use cases, performance - All 71 context tests now passing (100% success rate) Resolves critical bugs in Context Graphs feature (#290) implementation --- semantica/context/context_graph.py | 111 +++- semantica/context/decision_models.py | 12 +- tests/context/test_context_graphs_examples.py | 521 ++++++++++++++++++ 3 files changed, 609 insertions(+), 35 deletions(-) create mode 100644 tests/context/test_context_graphs_examples.py diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index abf35213..a6c455aa 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -77,6 +77,7 @@ from collections import defaultdict, deque from dataclasses import dataclass, field from datetime import datetime from typing import Any, Dict, List, Optional, Set, Tuple, Union +import uuid from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -841,6 +842,7 @@ class ContextGraph: "id": n.node_id, "type": n.node_type, "content": n.content, + "properties": n.properties, "metadata": n.metadata, } for n in self.nodes.values() @@ -860,6 +862,34 @@ class ContextGraph: }, } + def from_dict(self, graph_dict: Dict[str, Any]) -> None: + """Load graph from dictionary format.""" + # Clear existing graph + self.nodes.clear() + self.edges.clear() + + # Add nodes + for node_data in graph_dict.get("nodes", []): + node = ContextNode( + node_id=node_data["id"], + node_type=node_data["type"], + content=node_data.get("content", ""), + properties=node_data.get("properties", {}), + metadata=node_data.get("metadata", {}) + ) + self._add_internal_node(node) + + # Add edges + for edge_data in graph_dict.get("edges", []): + edge = ContextEdge( + source_id=edge_data["source"], + target_id=edge_data["target"], + edge_type=edge_data["type"], + weight=edge_data.get("weight", 1.0), + metadata=edge_data.get("metadata", {}) + ) + self._add_internal_edge(edge) + # Decision Support Methods def add_decision(self, decision: "Decision") -> None: """ @@ -870,8 +900,14 @@ class ContextGraph: """ from .decision_models import Decision + # Handle empty decision ID by generating UUID only if None + node_id = decision.decision_id if decision.decision_id is not None else str(uuid.uuid4()) + + # Handle None metadata + metadata = decision.metadata or {} + node = ContextNode( - node_id=decision.decision_id, + node_id=node_id, node_type="Decision", content=decision.scenario, properties={ @@ -883,7 +919,7 @@ class ContextGraph: "decision_maker": decision.decision_maker, "reasoning_embedding": decision.reasoning_embedding, "node2vec_embedding": decision.node2vec_embedding, - **decision.metadata + **metadata } ) self._add_internal_node(node) @@ -906,6 +942,15 @@ class ContextGraph: if relationship_type not in valid_types: raise ValueError(f"Relationship type must be one of: {valid_types}") + # Check if decisions exist - if not, skip adding relationship + if source_decision_id not in self.nodes or target_decision_id not in self.nodes: + return + + # Check if nodes are decision nodes - if not, skip adding relationship + if (self.nodes[source_decision_id].node_type != "Decision" or + self.nodes[target_decision_id].node_type != "Decision"): + return + edge = ContextEdge( source_id=source_decision_id, target_id=target_decision_id, @@ -949,41 +994,49 @@ class ContextGraph: visited.add(current_id) - # Get decision node - if current_id in self.nodes: - node = self.nodes[current_id] - if node.node_type == "Decision": - decision_data = node.properties - decision = Decision( - decision_id=current_id, - category=decision_data.get("category", ""), - scenario=node.content, - reasoning=decision_data.get("reasoning", ""), - outcome=decision_data.get("outcome", ""), - confidence=decision_data.get("confidence", 0.0), - timestamp=datetime.fromisoformat(decision_data.get("timestamp", datetime.now().isoformat())), - decision_maker=decision_data.get("decision_maker", ""), - reasoning_embedding=decision_data.get("reasoning_embedding"), - node2vec_embedding=decision_data.get("node2vec_embedding"), - metadata={k: v for k, v in decision_data.items() if k not in [ - "category", "reasoning", "outcome", "confidence", - "timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding" - ]} - ) - decision.metadata["causal_distance"] = depth - decisions.append(decision) + # Skip the starting decision - only add connected decisions + if current_id != decision_id: + # Get decision node + if current_id in self.nodes: + node = self.nodes[current_id] + if node.node_type == "Decision": + decision_data = node.properties + decision = Decision( + decision_id=current_id, + category=decision_data.get("category", ""), + scenario=node.content, + reasoning=decision_data.get("reasoning", ""), + outcome=decision_data.get("outcome", ""), + confidence=decision_data.get("confidence", 0.0), + timestamp=datetime.fromisoformat(decision_data.get("timestamp", datetime.now().isoformat())), + decision_maker=decision_data.get("decision_maker", ""), + reasoning_embedding=decision_data.get("reasoning_embedding"), + node2vec_embedding=decision_data.get("node2vec_embedding"), + metadata={k: v for k, v in decision_data.items() if k not in [ + "category", "reasoning", "outcome", "confidence", + "timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding" + ]} + ) + decision.metadata["causal_distance"] = depth + decisions.append(decision) # Find connected decisions for edge in self.edges: if direction == "upstream": if edge.target_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]: - if edge.source_id not in visited: + if edge.source_id not in visited and depth < max_depth: queue.append((edge.source_id, depth + 1)) else: # downstream if edge.source_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]: - if edge.target_id not in visited: + if edge.target_id not in visited and depth < max_depth: queue.append((edge.target_id, depth + 1)) + # Sort by depth for upstream (most distant first) and downstream (closest first) + if direction == "upstream": + decisions.sort(key=lambda d: d.metadata.get("causal_distance", 0), reverse=True) + else: + decisions.sort(key=lambda d: d.metadata.get("causal_distance", 0)) + return decisions def find_precedents(self, decision_id: str, limit: int = 10) -> List["Decision"]: @@ -1000,8 +1053,8 @@ class ContextGraph: # Find decisions connected via PRECEDENT_FOR relationships precedent_ids = [] for edge in self.edges: - if edge.source_id == decision_id and edge.edge_type == "PRECEDENT_FOR": - precedent_ids.append(edge.target_id) + if edge.target_id == decision_id and edge.edge_type == "PRECEDENT_FOR": + precedent_ids.append(edge.source_id) # Convert to Decision objects decisions = [] diff --git a/semantica/context/decision_models.py b/semantica/context/decision_models.py index 59fe6069..ca2de217 100644 --- a/semantica/context/decision_models.py +++ b/semantica/context/decision_models.py @@ -101,7 +101,7 @@ class Decision: def __post_init__(self): """Validate decision data.""" - if not self.decision_id: + if self.decision_id is None: self.decision_id = str(uuid.uuid4()) if not 0 <= self.confidence <= 1: raise ValueError("Confidence must be between 0 and 1") @@ -143,7 +143,7 @@ class DecisionContext: def __post_init__(self): """Validate context data.""" - if not self.context_id: + if self.context_id is None: self.context_id = str(uuid.uuid4()) def to_dict(self) -> Dict[str, Any]: @@ -179,7 +179,7 @@ class Policy: def __post_init__(self): """Validate policy data.""" - if not self.policy_id: + if self.policy_id is None: self.policy_id = str(uuid.uuid4()) def to_dict(self) -> Dict[str, Any]: @@ -220,7 +220,7 @@ class PolicyException: def __post_init__(self): """Validate exception data.""" - if not self.exception_id: + if self.exception_id is None: self.exception_id = str(uuid.uuid4()) def to_dict(self) -> Dict[str, Any]: @@ -256,7 +256,7 @@ class Precedent: def __post_init__(self): """Validate precedent data.""" - if not self.precedent_id: + if self.precedent_id is None: self.precedent_id = str(uuid.uuid4()) if not 0 <= self.similarity_score <= 1: raise ValueError("Similarity score must be between 0 and 1") @@ -294,7 +294,7 @@ class ApprovalChain: def __post_init__(self): """Validate approval data.""" - if not self.approval_id: + if self.approval_id is None: self.approval_id = str(uuid.uuid4()) valid_methods = ["slack_dm", "zoom_call", "email", "system"] if self.approval_method not in valid_methods: diff --git a/tests/context/test_context_graphs_examples.py b/tests/context/test_context_graphs_examples.py new file mode 100644 index 00000000..37fc4e15 --- /dev/null +++ b/tests/context/test_context_graphs_examples.py @@ -0,0 +1,521 @@ +#!/usr/bin/env python3 +""" +Comprehensive test suite for Context Graphs feature examples from issue #290. +This tests all the example use cases provided in the feature description. +""" + +import pytest +import sys +import os +from datetime import datetime +from unittest.mock import Mock, patch + +# Add the semantica package to Python path for testing +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + +from semantica.context import AgentContext +from semantica.context.context_graph import ContextGraph +from semantica.context.decision_models import Decision, Policy, PolicyException +from semantica.vector_store import VectorStore +from semantica.embeddings import EmbeddingGenerator + + +class TestContextGraphsExamples: + """Test suite for Context Graphs feature examples.""" + + @pytest.fixture + def mock_vector_store(self): + """Create a mock vector store for testing.""" + store = Mock(spec=VectorStore) + store.store = Mock(return_value="test_memory_id") + store.retrieve = Mock(return_value=[]) + store.embed = Mock(return_value=[0.1] * 384) # Mock embedding + return store + + @pytest.fixture + def mock_knowledge_graph(self): + """Create a mock knowledge graph for testing.""" + kg = Mock(spec=ContextGraph) + kg.execute_query = Mock(return_value=[]) + kg.build_from_conversations = Mock(return_value={"statistics": {"node_count": 0, "edge_count": 0}}) + return kg + + def test_context_graph_direct_functionality(self): + """Test ContextGraph directly with decision support features.""" + print("Testing ContextGraph Direct Functionality...") + + # Create context graph with advanced features + graph = ContextGraph( + enable_advanced_analytics=True, + enable_centrality_analysis=True, + enable_community_detection=True, + enable_node_embeddings=True + ) + + # Add a decision + decision = Decision( + decision_id="test_decision_001", + category="test", + scenario="Test scenario for credit approval", + reasoning="Good credit history and stable income", + outcome="approved", + confidence=0.95, + timestamp=datetime.now(), + decision_maker="ai_agent" + ) + + graph.add_decision(decision) + assert len(graph.nodes) == 1 + print("+ Added decision to context graph") + + # Add another decision and causal relationship + decision2 = Decision( + decision_id="test_decision_002", + category="test", + scenario="Related credit decision", + reasoning="Based on previous approval", + outcome="approved", + confidence=0.90, + timestamp=datetime.now(), + decision_maker="ai_agent" + ) + + graph.add_decision(decision2) + graph.add_causal_relationship("test_decision_001", "test_decision_002", "CAUSED") + assert len(graph.nodes) == 2 + assert len(graph.edges) == 1 + print("+ Added causal relationship") + + # Test causal chain + chain = graph.get_causal_chain("test_decision_002", direction="upstream") + assert len(chain) == 1 + assert chain[0].decision_id == "test_decision_001" + print("+ Found causal chain with decisions") + + # Test precedent search + precedents = graph.find_precedents("test_decision_002") + assert len(precedents) == 0 # No precedent relationships added + + # Add precedent relationship and test again + graph.add_causal_relationship("test_decision_001", "test_decision_002", "PRECEDENT_FOR") + precedents = graph.find_precedents("test_decision_002") + assert len(precedents) == 1 + assert precedents[0].decision_id == "test_decision_001" + print("+ Found precedents") + + # Test serialization + graph_dict = graph.to_dict() + assert len(graph_dict['nodes']) == 2 + assert len(graph_dict['edges']) == 2 + assert 'properties' in graph_dict['nodes'][0] + print("+ Serialized graph correctly") + + # Test deserialization + new_graph = ContextGraph() + new_graph.from_dict(graph_dict) + assert len(new_graph.nodes) == 2 + assert len(new_graph.edges) == 2 + print("+ Deserialized graph correctly") + + print("✓ ContextGraph direct functionality test passed") + + def test_financial_services_example(self, mock_vector_store, mock_knowledge_graph): + """Test the financial services example from the feature description.""" + print("Testing Financial Services Example...") + + # Initialize context with decision tracking + context = AgentContext( + vector_store=mock_vector_store, + knowledge_graph=mock_knowledge_graph, + enable_decision_tracking=True, + enable_advanced_analytics=True, + enable_kg_algorithms=True, + enable_vector_store_features=True + ) + + # Credit decision with precedent search + decision_id = context.record_decision( + category="credit_approval", + scenario="High-risk credit limit increase", + reasoning="Past fraud flag with velocity check failure", + outcome="rejected", + confidence=0.788, + entities=["customer:jessica_norris"] + ) + assert decision_id is not None + print("+ Recorded decision") + + # Find similar precedents + precedents = context.find_precedents( + scenario="High-risk customer credit increase", + category="credit_approval", + limit=5 + ) + assert isinstance(precedents, list) + print("+ Found precedents") + + # Analyze causal chain + causal_chain = context.get_causal_chain(decision_id, max_depth=5) + assert isinstance(causal_chain, list) + print("+ Analyzed causal chain") + + print("✓ Financial services example test passed") + + def test_healthcare_example(self, mock_vector_store, mock_knowledge_graph): + """Test the healthcare example from the feature description.""" + print("Testing Healthcare Example...") + + # Initialize context with decision tracking + context = AgentContext( + vector_store=mock_vector_store, + knowledge_graph=mock_knowledge_graph, + enable_decision_tracking=True, + enable_advanced_analytics=True, + enable_kg_algorithms=True, + enable_vector_store_features=True + ) + + # Treatment decision with policy compliance + decision_id = context.record_decision( + category="treatment_plan", + scenario="Diabetic patient with comorbidities", + reasoning="Standard protocol contraindicated due to renal function", + outcome="modified_treatment", + confidence=0.92 + ) + assert decision_id is not None + print("+ Recorded decision") + + # Check policy engine availability + policy_engine = context.get_policy_engine() + if policy_engine: + # Create a test policy + policy = Policy( + policy_id="diabetes_protocol_v2", + name="Diabetes Treatment Protocol v2", + description="Standard treatment protocol for diabetes patients", + rules={"contraindications": ["renal_impairment"], "max_dosage": 100}, + category="treatment", + version="v2", + created_at=datetime.now(), + updated_at=datetime.now() + ) + + # Test policy operations + assert policy.policy_id == "diabetes_protocol_v2" + assert policy.category == "treatment" + print("+ Policy operations working") + + print("✓ Healthcare example test passed") + + def test_legal_example(self, mock_vector_store, mock_knowledge_graph): + """Test the legal example from the feature description.""" + print("Testing Legal Example...") + + # Initialize context with decision tracking + context = AgentContext( + vector_store=mock_vector_store, + knowledge_graph=mock_knowledge_graph, + enable_decision_tracking=True, + enable_advanced_analytics=True, + enable_kg_algorithms=True, + enable_vector_store_features=True + ) + + # Legal decision with precedent analysis + decision_id = context.record_decision( + category="contract_review", + scenario="Non-standard liability clause", + reasoning="Precedent cases show similar clauses upheld", + outcome="approved_with_modifications", + confidence=0.85 + ) + assert decision_id is not None + print("+ Recorded decision") + + # Find legal precedents + precedents = context.find_precedents( + scenario="Liability limitation clauses", + category="contract_review", + limit=10 + ) + assert isinstance(precedents, list) + print("+ Found legal precedents") + + print("✓ Legal example test passed") + + def test_decision_models_functionality(self): + """Test decision models functionality.""" + print("Testing Decision Models...") + + # Test Decision model + decision = Decision( + decision_id="test_decision", + category="test_category", + scenario="Test scenario", + reasoning="Test reasoning", + outcome="approved", + confidence=0.95, + timestamp=datetime.now(), + decision_maker="test_agent" + ) + + assert decision.decision_id == "test_decision" + assert decision.category == "test_category" + assert 0 <= decision.confidence <= 1 + + # Test serialization + decision_dict = decision.to_dict() + assert decision_dict["decision_id"] == "test_decision" + assert "timestamp" in decision_dict + + # Test deserialization + restored_decision = Decision.from_dict(decision_dict) + assert restored_decision.decision_id == decision.decision_id + assert restored_decision.category == decision.category + print("+ Decision model serialization working") + + # Test Policy model + policy = Policy( + policy_id="test_policy", + name="Test Policy", + description="Test policy description", + rules={"max_amount": 1000}, + category="test", + version="1.0", + created_at=datetime.now(), + updated_at=datetime.now() + ) + + assert policy.policy_id == "test_policy" + assert policy.rules["max_amount"] == 1000 + + # Test PolicyException model + exception = PolicyException( + exception_id="test_exception", + decision_id="test_decision", + policy_id="test_policy", + reason="Test exception", + approver="test_approver", + approval_timestamp=datetime.now(), + justification="Test justification" + ) + + assert exception.exception_id == "test_exception" + assert exception.decision_id == "test_decision" + print("+ Policy models working") + + print("✓ Decision models functionality test passed") + + def test_context_graph_edge_cases(self): + """Test ContextGraph edge cases and error handling.""" + print("Testing ContextGraph Edge Cases...") + + graph = ContextGraph() + + # Test empty decision ID handling + decision_empty_id = Decision( + decision_id="", # Empty ID + category="test", + scenario="test scenario", + reasoning="test reasoning", + outcome="test outcome", + confidence=0.8, + timestamp=datetime.now(), + decision_maker="test_agent" + ) + + graph.add_decision(decision_empty_id) + assert "" in graph.nodes # Empty string should be preserved as key + print("+ Empty decision ID handling working") + + # Test None decision ID handling + decision_none_id = Decision( + decision_id=None, # None ID + category="test", + scenario="test scenario 2", + reasoning="test reasoning 2", + outcome="test outcome 2", + confidence=0.8, + timestamp=datetime.now(), + decision_maker="test_agent" + ) + + graph.add_decision(decision_none_id) + assert len(graph.nodes) == 2 # Should have generated UUID + print("+ None decision ID handling working") + + # Test causal relationship with nonexistent nodes (should not raise error) + graph.add_causal_relationship("nonexistent1", "nonexistent2", "CAUSED") + assert len(graph.edges) == 0 # Should not add relationship + print("+ Nonexistent node handling working") + + # Test invalid relationship type + with pytest.raises(ValueError): + graph.add_causal_relationship("test", "test2", "INVALID_TYPE") + print("+ Invalid relationship type validation working") + + # Test causal chain with nonexistent decision + chain = graph.get_causal_chain("nonexistent", direction="upstream") + assert len(chain) == 0 + print("+ Nonexistent decision handling working") + + print("✓ ContextGraph edge cases test passed") + + def test_advanced_features_integration(self, mock_vector_store, mock_knowledge_graph): + """Test advanced features integration.""" + print("Testing Advanced Features Integration...") + + # Test with all features enabled + context = AgentContext( + vector_store=mock_vector_store, + knowledge_graph=mock_knowledge_graph, + enable_decision_tracking=True, + enable_advanced_analytics=True, + enable_kg_algorithms=True, + enable_vector_store_features=True, + use_graph_expansion=True, + max_expansion_hops=3, + hybrid_alpha=0.7 + ) + + # Verify configuration + assert context.config["enable_decision_tracking"] is True + assert context.config["enable_advanced_analytics"] is True + assert context.config["enable_kg_algorithms"] is True + assert context.config["enable_vector_store_features"] is True + assert context.config["use_graph_expansion"] is True + assert context.config["max_expansion_hops"] == 3 + assert context.config["hybrid_alpha"] == 0.7 + print("+ Configuration validation working") + + # Test decision tracking with advanced features + decision_id = context.record_decision( + category="advanced_test", + scenario="Advanced feature test scenario", + reasoning="Testing advanced analytics integration", + outcome="processed", + confidence=0.88, + entities=["entity1", "entity2"] + ) + + assert decision_id is not None + print("+ Advanced decision recording working") + + # Test context insights + insights = context.get_context_insights() + assert isinstance(insights, dict) + print("+ Context insights working") + + print("✓ Advanced features integration test passed") + + +class TestContextGraphsPerformance: + """Performance tests for Context Graphs feature.""" + + def test_large_decision_network(self): + """Test handling of large decision networks.""" + print("Testing Large Decision Network...") + + graph = ContextGraph() + + # Create a network of 100 decisions + decisions = [] + for i in range(100): + decision = Decision( + decision_id=f"decision_{i:03d}", + category="performance_test", + scenario=f"Performance test scenario {i}", + reasoning=f"Performance test reasoning {i}", + outcome="processed", + confidence=0.8 + (i % 20) * 0.01, # Varying confidence + timestamp=datetime.now(), + decision_maker="performance_agent" + ) + decisions.append(decision) + graph.add_decision(decision) + + assert len(graph.nodes) == 100 + print("+ Created 100 decisions") + + # Add causal relationships to create a network + for i in range(99): + # Create a mix of relationship types + relationship_type = ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"][i % 3] + graph.add_causal_relationship(f"decision_{i:03d}", f"decision_{i+1:03d}", relationship_type) + + assert len(graph.edges) == 99 + print("+ Created 99 causal relationships") + + # Test causal chain performance + chain = graph.get_causal_chain("decision_099", direction="upstream", max_depth=50) + assert len(chain) > 0 + print("+ Causal chain analysis working") + + # Test precedent search performance + precedents = graph.find_precedents("decision_050", limit=20) + assert isinstance(precedents, list) + print("+ Precedent search working") + + # Test serialization performance + graph_dict = graph.to_dict() + assert len(graph_dict['nodes']) == 100 + assert len(graph_dict['edges']) == 99 + print("+ Large graph serialization working") + + print("✓ Large decision network test passed") + + def test_concurrent_operations(self): + """Test concurrent decision operations.""" + print("Testing Concurrent Operations...") + + import threading + import time + + graph = ContextGraph() + results = [] + errors = [] + + def add_decisions(start_id, count): + """Add decisions in a separate thread.""" + try: + for i in range(count): + decision = Decision( + decision_id=f"concurrent_decision_{start_id + i:03d}", + category="concurrent_test", + scenario=f"Concurrent test {start_id + i}", + reasoning="Concurrent reasoning", + outcome="processed", + confidence=0.8, + timestamp=datetime.now(), + decision_maker="concurrent_agent" + ) + graph.add_decision(decision) + time.sleep(0.001) # Small delay to simulate real work + results.append(f"Thread {start_id} completed") + except Exception as e: + errors.append(f"Thread {start_id} error: {e}") + + # Create multiple threads + threads = [] + for i in range(5): + thread = threading.Thread(target=add_decisions, args=(i * 20, 20)) + threads.append(thread) + thread.start() + + # Wait for all threads to complete + for thread in threads: + thread.join() + + # Verify results + assert len(errors) == 0, f"Errors occurred: {errors}" + assert len(results) == 5 + assert len(graph.nodes) == 100 # 5 threads * 20 decisions each + print("+ Concurrent operations completed successfully") + + print("✓ Concurrent operations test passed") + + +if __name__ == "__main__": + # Run tests when script is executed directly + pytest.main([__file__, "-v"]) From d2f8992ca9324cad31851b71ff3517aeee03ca5e Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Sun, 15 Feb 2026 13:20:34 +0530 Subject: [PATCH 08/50] Fix Context Graphs Decision Tracking & Add Comprehensive Tests (#315) * context_fixes * context_compliance_fixes * Delete PR_CONTEXT.md * Fix Context Graphs decision tracking and add comprehensive tests - Fix empty/None decision ID handling in ContextGraph.add_decision() - Fix None metadata handling to prevent TypeError - Fix causal chain depth logic and node exclusion - Fix nonexistent node handling in add_causal_relationship() - Add missing properties field in to_dict serialization - Add missing from_dict method for graph deserialization - Fix precedent search direction in find_precedents() - Fix UUID generation logic in all decision models - Add comprehensive test suite with 9 tests covering all features - Test coverage: decision tracking, graph analytics, use cases, performance - All 71 context tests now passing (100% success rate) Resolves critical bugs in Context Graphs feature (#290) implementation --- semantica/context/context_graph.py | 111 +++- semantica/context/decision_models.py | 12 +- tests/context/test_context_graphs_examples.py | 521 ++++++++++++++++++ 3 files changed, 609 insertions(+), 35 deletions(-) create mode 100644 tests/context/test_context_graphs_examples.py diff --git a/semantica/context/context_graph.py b/semantica/context/context_graph.py index abf35213..a6c455aa 100644 --- a/semantica/context/context_graph.py +++ b/semantica/context/context_graph.py @@ -77,6 +77,7 @@ from collections import defaultdict, deque from dataclasses import dataclass, field from datetime import datetime from typing import Any, Dict, List, Optional, Set, Tuple, Union +import uuid from ..utils.logging import get_logger from ..utils.progress_tracker import get_progress_tracker @@ -841,6 +842,7 @@ class ContextGraph: "id": n.node_id, "type": n.node_type, "content": n.content, + "properties": n.properties, "metadata": n.metadata, } for n in self.nodes.values() @@ -860,6 +862,34 @@ class ContextGraph: }, } + def from_dict(self, graph_dict: Dict[str, Any]) -> None: + """Load graph from dictionary format.""" + # Clear existing graph + self.nodes.clear() + self.edges.clear() + + # Add nodes + for node_data in graph_dict.get("nodes", []): + node = ContextNode( + node_id=node_data["id"], + node_type=node_data["type"], + content=node_data.get("content", ""), + properties=node_data.get("properties", {}), + metadata=node_data.get("metadata", {}) + ) + self._add_internal_node(node) + + # Add edges + for edge_data in graph_dict.get("edges", []): + edge = ContextEdge( + source_id=edge_data["source"], + target_id=edge_data["target"], + edge_type=edge_data["type"], + weight=edge_data.get("weight", 1.0), + metadata=edge_data.get("metadata", {}) + ) + self._add_internal_edge(edge) + # Decision Support Methods def add_decision(self, decision: "Decision") -> None: """ @@ -870,8 +900,14 @@ class ContextGraph: """ from .decision_models import Decision + # Handle empty decision ID by generating UUID only if None + node_id = decision.decision_id if decision.decision_id is not None else str(uuid.uuid4()) + + # Handle None metadata + metadata = decision.metadata or {} + node = ContextNode( - node_id=decision.decision_id, + node_id=node_id, node_type="Decision", content=decision.scenario, properties={ @@ -883,7 +919,7 @@ class ContextGraph: "decision_maker": decision.decision_maker, "reasoning_embedding": decision.reasoning_embedding, "node2vec_embedding": decision.node2vec_embedding, - **decision.metadata + **metadata } ) self._add_internal_node(node) @@ -906,6 +942,15 @@ class ContextGraph: if relationship_type not in valid_types: raise ValueError(f"Relationship type must be one of: {valid_types}") + # Check if decisions exist - if not, skip adding relationship + if source_decision_id not in self.nodes or target_decision_id not in self.nodes: + return + + # Check if nodes are decision nodes - if not, skip adding relationship + if (self.nodes[source_decision_id].node_type != "Decision" or + self.nodes[target_decision_id].node_type != "Decision"): + return + edge = ContextEdge( source_id=source_decision_id, target_id=target_decision_id, @@ -949,41 +994,49 @@ class ContextGraph: visited.add(current_id) - # Get decision node - if current_id in self.nodes: - node = self.nodes[current_id] - if node.node_type == "Decision": - decision_data = node.properties - decision = Decision( - decision_id=current_id, - category=decision_data.get("category", ""), - scenario=node.content, - reasoning=decision_data.get("reasoning", ""), - outcome=decision_data.get("outcome", ""), - confidence=decision_data.get("confidence", 0.0), - timestamp=datetime.fromisoformat(decision_data.get("timestamp", datetime.now().isoformat())), - decision_maker=decision_data.get("decision_maker", ""), - reasoning_embedding=decision_data.get("reasoning_embedding"), - node2vec_embedding=decision_data.get("node2vec_embedding"), - metadata={k: v for k, v in decision_data.items() if k not in [ - "category", "reasoning", "outcome", "confidence", - "timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding" - ]} - ) - decision.metadata["causal_distance"] = depth - decisions.append(decision) + # Skip the starting decision - only add connected decisions + if current_id != decision_id: + # Get decision node + if current_id in self.nodes: + node = self.nodes[current_id] + if node.node_type == "Decision": + decision_data = node.properties + decision = Decision( + decision_id=current_id, + category=decision_data.get("category", ""), + scenario=node.content, + reasoning=decision_data.get("reasoning", ""), + outcome=decision_data.get("outcome", ""), + confidence=decision_data.get("confidence", 0.0), + timestamp=datetime.fromisoformat(decision_data.get("timestamp", datetime.now().isoformat())), + decision_maker=decision_data.get("decision_maker", ""), + reasoning_embedding=decision_data.get("reasoning_embedding"), + node2vec_embedding=decision_data.get("node2vec_embedding"), + metadata={k: v for k, v in decision_data.items() if k not in [ + "category", "reasoning", "outcome", "confidence", + "timestamp", "decision_maker", "reasoning_embedding", "node2vec_embedding" + ]} + ) + decision.metadata["causal_distance"] = depth + decisions.append(decision) # Find connected decisions for edge in self.edges: if direction == "upstream": if edge.target_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]: - if edge.source_id not in visited: + if edge.source_id not in visited and depth < max_depth: queue.append((edge.source_id, depth + 1)) else: # downstream if edge.source_id == current_id and edge.edge_type in ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"]: - if edge.target_id not in visited: + if edge.target_id not in visited and depth < max_depth: queue.append((edge.target_id, depth + 1)) + # Sort by depth for upstream (most distant first) and downstream (closest first) + if direction == "upstream": + decisions.sort(key=lambda d: d.metadata.get("causal_distance", 0), reverse=True) + else: + decisions.sort(key=lambda d: d.metadata.get("causal_distance", 0)) + return decisions def find_precedents(self, decision_id: str, limit: int = 10) -> List["Decision"]: @@ -1000,8 +1053,8 @@ class ContextGraph: # Find decisions connected via PRECEDENT_FOR relationships precedent_ids = [] for edge in self.edges: - if edge.source_id == decision_id and edge.edge_type == "PRECEDENT_FOR": - precedent_ids.append(edge.target_id) + if edge.target_id == decision_id and edge.edge_type == "PRECEDENT_FOR": + precedent_ids.append(edge.source_id) # Convert to Decision objects decisions = [] diff --git a/semantica/context/decision_models.py b/semantica/context/decision_models.py index 59fe6069..ca2de217 100644 --- a/semantica/context/decision_models.py +++ b/semantica/context/decision_models.py @@ -101,7 +101,7 @@ class Decision: def __post_init__(self): """Validate decision data.""" - if not self.decision_id: + if self.decision_id is None: self.decision_id = str(uuid.uuid4()) if not 0 <= self.confidence <= 1: raise ValueError("Confidence must be between 0 and 1") @@ -143,7 +143,7 @@ class DecisionContext: def __post_init__(self): """Validate context data.""" - if not self.context_id: + if self.context_id is None: self.context_id = str(uuid.uuid4()) def to_dict(self) -> Dict[str, Any]: @@ -179,7 +179,7 @@ class Policy: def __post_init__(self): """Validate policy data.""" - if not self.policy_id: + if self.policy_id is None: self.policy_id = str(uuid.uuid4()) def to_dict(self) -> Dict[str, Any]: @@ -220,7 +220,7 @@ class PolicyException: def __post_init__(self): """Validate exception data.""" - if not self.exception_id: + if self.exception_id is None: self.exception_id = str(uuid.uuid4()) def to_dict(self) -> Dict[str, Any]: @@ -256,7 +256,7 @@ class Precedent: def __post_init__(self): """Validate precedent data.""" - if not self.precedent_id: + if self.precedent_id is None: self.precedent_id = str(uuid.uuid4()) if not 0 <= self.similarity_score <= 1: raise ValueError("Similarity score must be between 0 and 1") @@ -294,7 +294,7 @@ class ApprovalChain: def __post_init__(self): """Validate approval data.""" - if not self.approval_id: + if self.approval_id is None: self.approval_id = str(uuid.uuid4()) valid_methods = ["slack_dm", "zoom_call", "email", "system"] if self.approval_method not in valid_methods: diff --git a/tests/context/test_context_graphs_examples.py b/tests/context/test_context_graphs_examples.py new file mode 100644 index 00000000..37fc4e15 --- /dev/null +++ b/tests/context/test_context_graphs_examples.py @@ -0,0 +1,521 @@ +#!/usr/bin/env python3 +""" +Comprehensive test suite for Context Graphs feature examples from issue #290. +This tests all the example use cases provided in the feature description. +""" + +import pytest +import sys +import os +from datetime import datetime +from unittest.mock import Mock, patch + +# Add the semantica package to Python path for testing +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + +from semantica.context import AgentContext +from semantica.context.context_graph import ContextGraph +from semantica.context.decision_models import Decision, Policy, PolicyException +from semantica.vector_store import VectorStore +from semantica.embeddings import EmbeddingGenerator + + +class TestContextGraphsExamples: + """Test suite for Context Graphs feature examples.""" + + @pytest.fixture + def mock_vector_store(self): + """Create a mock vector store for testing.""" + store = Mock(spec=VectorStore) + store.store = Mock(return_value="test_memory_id") + store.retrieve = Mock(return_value=[]) + store.embed = Mock(return_value=[0.1] * 384) # Mock embedding + return store + + @pytest.fixture + def mock_knowledge_graph(self): + """Create a mock knowledge graph for testing.""" + kg = Mock(spec=ContextGraph) + kg.execute_query = Mock(return_value=[]) + kg.build_from_conversations = Mock(return_value={"statistics": {"node_count": 0, "edge_count": 0}}) + return kg + + def test_context_graph_direct_functionality(self): + """Test ContextGraph directly with decision support features.""" + print("Testing ContextGraph Direct Functionality...") + + # Create context graph with advanced features + graph = ContextGraph( + enable_advanced_analytics=True, + enable_centrality_analysis=True, + enable_community_detection=True, + enable_node_embeddings=True + ) + + # Add a decision + decision = Decision( + decision_id="test_decision_001", + category="test", + scenario="Test scenario for credit approval", + reasoning="Good credit history and stable income", + outcome="approved", + confidence=0.95, + timestamp=datetime.now(), + decision_maker="ai_agent" + ) + + graph.add_decision(decision) + assert len(graph.nodes) == 1 + print("+ Added decision to context graph") + + # Add another decision and causal relationship + decision2 = Decision( + decision_id="test_decision_002", + category="test", + scenario="Related credit decision", + reasoning="Based on previous approval", + outcome="approved", + confidence=0.90, + timestamp=datetime.now(), + decision_maker="ai_agent" + ) + + graph.add_decision(decision2) + graph.add_causal_relationship("test_decision_001", "test_decision_002", "CAUSED") + assert len(graph.nodes) == 2 + assert len(graph.edges) == 1 + print("+ Added causal relationship") + + # Test causal chain + chain = graph.get_causal_chain("test_decision_002", direction="upstream") + assert len(chain) == 1 + assert chain[0].decision_id == "test_decision_001" + print("+ Found causal chain with decisions") + + # Test precedent search + precedents = graph.find_precedents("test_decision_002") + assert len(precedents) == 0 # No precedent relationships added + + # Add precedent relationship and test again + graph.add_causal_relationship("test_decision_001", "test_decision_002", "PRECEDENT_FOR") + precedents = graph.find_precedents("test_decision_002") + assert len(precedents) == 1 + assert precedents[0].decision_id == "test_decision_001" + print("+ Found precedents") + + # Test serialization + graph_dict = graph.to_dict() + assert len(graph_dict['nodes']) == 2 + assert len(graph_dict['edges']) == 2 + assert 'properties' in graph_dict['nodes'][0] + print("+ Serialized graph correctly") + + # Test deserialization + new_graph = ContextGraph() + new_graph.from_dict(graph_dict) + assert len(new_graph.nodes) == 2 + assert len(new_graph.edges) == 2 + print("+ Deserialized graph correctly") + + print("✓ ContextGraph direct functionality test passed") + + def test_financial_services_example(self, mock_vector_store, mock_knowledge_graph): + """Test the financial services example from the feature description.""" + print("Testing Financial Services Example...") + + # Initialize context with decision tracking + context = AgentContext( + vector_store=mock_vector_store, + knowledge_graph=mock_knowledge_graph, + enable_decision_tracking=True, + enable_advanced_analytics=True, + enable_kg_algorithms=True, + enable_vector_store_features=True + ) + + # Credit decision with precedent search + decision_id = context.record_decision( + category="credit_approval", + scenario="High-risk credit limit increase", + reasoning="Past fraud flag with velocity check failure", + outcome="rejected", + confidence=0.788, + entities=["customer:jessica_norris"] + ) + assert decision_id is not None + print("+ Recorded decision") + + # Find similar precedents + precedents = context.find_precedents( + scenario="High-risk customer credit increase", + category="credit_approval", + limit=5 + ) + assert isinstance(precedents, list) + print("+ Found precedents") + + # Analyze causal chain + causal_chain = context.get_causal_chain(decision_id, max_depth=5) + assert isinstance(causal_chain, list) + print("+ Analyzed causal chain") + + print("✓ Financial services example test passed") + + def test_healthcare_example(self, mock_vector_store, mock_knowledge_graph): + """Test the healthcare example from the feature description.""" + print("Testing Healthcare Example...") + + # Initialize context with decision tracking + context = AgentContext( + vector_store=mock_vector_store, + knowledge_graph=mock_knowledge_graph, + enable_decision_tracking=True, + enable_advanced_analytics=True, + enable_kg_algorithms=True, + enable_vector_store_features=True + ) + + # Treatment decision with policy compliance + decision_id = context.record_decision( + category="treatment_plan", + scenario="Diabetic patient with comorbidities", + reasoning="Standard protocol contraindicated due to renal function", + outcome="modified_treatment", + confidence=0.92 + ) + assert decision_id is not None + print("+ Recorded decision") + + # Check policy engine availability + policy_engine = context.get_policy_engine() + if policy_engine: + # Create a test policy + policy = Policy( + policy_id="diabetes_protocol_v2", + name="Diabetes Treatment Protocol v2", + description="Standard treatment protocol for diabetes patients", + rules={"contraindications": ["renal_impairment"], "max_dosage": 100}, + category="treatment", + version="v2", + created_at=datetime.now(), + updated_at=datetime.now() + ) + + # Test policy operations + assert policy.policy_id == "diabetes_protocol_v2" + assert policy.category == "treatment" + print("+ Policy operations working") + + print("✓ Healthcare example test passed") + + def test_legal_example(self, mock_vector_store, mock_knowledge_graph): + """Test the legal example from the feature description.""" + print("Testing Legal Example...") + + # Initialize context with decision tracking + context = AgentContext( + vector_store=mock_vector_store, + knowledge_graph=mock_knowledge_graph, + enable_decision_tracking=True, + enable_advanced_analytics=True, + enable_kg_algorithms=True, + enable_vector_store_features=True + ) + + # Legal decision with precedent analysis + decision_id = context.record_decision( + category="contract_review", + scenario="Non-standard liability clause", + reasoning="Precedent cases show similar clauses upheld", + outcome="approved_with_modifications", + confidence=0.85 + ) + assert decision_id is not None + print("+ Recorded decision") + + # Find legal precedents + precedents = context.find_precedents( + scenario="Liability limitation clauses", + category="contract_review", + limit=10 + ) + assert isinstance(precedents, list) + print("+ Found legal precedents") + + print("✓ Legal example test passed") + + def test_decision_models_functionality(self): + """Test decision models functionality.""" + print("Testing Decision Models...") + + # Test Decision model + decision = Decision( + decision_id="test_decision", + category="test_category", + scenario="Test scenario", + reasoning="Test reasoning", + outcome="approved", + confidence=0.95, + timestamp=datetime.now(), + decision_maker="test_agent" + ) + + assert decision.decision_id == "test_decision" + assert decision.category == "test_category" + assert 0 <= decision.confidence <= 1 + + # Test serialization + decision_dict = decision.to_dict() + assert decision_dict["decision_id"] == "test_decision" + assert "timestamp" in decision_dict + + # Test deserialization + restored_decision = Decision.from_dict(decision_dict) + assert restored_decision.decision_id == decision.decision_id + assert restored_decision.category == decision.category + print("+ Decision model serialization working") + + # Test Policy model + policy = Policy( + policy_id="test_policy", + name="Test Policy", + description="Test policy description", + rules={"max_amount": 1000}, + category="test", + version="1.0", + created_at=datetime.now(), + updated_at=datetime.now() + ) + + assert policy.policy_id == "test_policy" + assert policy.rules["max_amount"] == 1000 + + # Test PolicyException model + exception = PolicyException( + exception_id="test_exception", + decision_id="test_decision", + policy_id="test_policy", + reason="Test exception", + approver="test_approver", + approval_timestamp=datetime.now(), + justification="Test justification" + ) + + assert exception.exception_id == "test_exception" + assert exception.decision_id == "test_decision" + print("+ Policy models working") + + print("✓ Decision models functionality test passed") + + def test_context_graph_edge_cases(self): + """Test ContextGraph edge cases and error handling.""" + print("Testing ContextGraph Edge Cases...") + + graph = ContextGraph() + + # Test empty decision ID handling + decision_empty_id = Decision( + decision_id="", # Empty ID + category="test", + scenario="test scenario", + reasoning="test reasoning", + outcome="test outcome", + confidence=0.8, + timestamp=datetime.now(), + decision_maker="test_agent" + ) + + graph.add_decision(decision_empty_id) + assert "" in graph.nodes # Empty string should be preserved as key + print("+ Empty decision ID handling working") + + # Test None decision ID handling + decision_none_id = Decision( + decision_id=None, # None ID + category="test", + scenario="test scenario 2", + reasoning="test reasoning 2", + outcome="test outcome 2", + confidence=0.8, + timestamp=datetime.now(), + decision_maker="test_agent" + ) + + graph.add_decision(decision_none_id) + assert len(graph.nodes) == 2 # Should have generated UUID + print("+ None decision ID handling working") + + # Test causal relationship with nonexistent nodes (should not raise error) + graph.add_causal_relationship("nonexistent1", "nonexistent2", "CAUSED") + assert len(graph.edges) == 0 # Should not add relationship + print("+ Nonexistent node handling working") + + # Test invalid relationship type + with pytest.raises(ValueError): + graph.add_causal_relationship("test", "test2", "INVALID_TYPE") + print("+ Invalid relationship type validation working") + + # Test causal chain with nonexistent decision + chain = graph.get_causal_chain("nonexistent", direction="upstream") + assert len(chain) == 0 + print("+ Nonexistent decision handling working") + + print("✓ ContextGraph edge cases test passed") + + def test_advanced_features_integration(self, mock_vector_store, mock_knowledge_graph): + """Test advanced features integration.""" + print("Testing Advanced Features Integration...") + + # Test with all features enabled + context = AgentContext( + vector_store=mock_vector_store, + knowledge_graph=mock_knowledge_graph, + enable_decision_tracking=True, + enable_advanced_analytics=True, + enable_kg_algorithms=True, + enable_vector_store_features=True, + use_graph_expansion=True, + max_expansion_hops=3, + hybrid_alpha=0.7 + ) + + # Verify configuration + assert context.config["enable_decision_tracking"] is True + assert context.config["enable_advanced_analytics"] is True + assert context.config["enable_kg_algorithms"] is True + assert context.config["enable_vector_store_features"] is True + assert context.config["use_graph_expansion"] is True + assert context.config["max_expansion_hops"] == 3 + assert context.config["hybrid_alpha"] == 0.7 + print("+ Configuration validation working") + + # Test decision tracking with advanced features + decision_id = context.record_decision( + category="advanced_test", + scenario="Advanced feature test scenario", + reasoning="Testing advanced analytics integration", + outcome="processed", + confidence=0.88, + entities=["entity1", "entity2"] + ) + + assert decision_id is not None + print("+ Advanced decision recording working") + + # Test context insights + insights = context.get_context_insights() + assert isinstance(insights, dict) + print("+ Context insights working") + + print("✓ Advanced features integration test passed") + + +class TestContextGraphsPerformance: + """Performance tests for Context Graphs feature.""" + + def test_large_decision_network(self): + """Test handling of large decision networks.""" + print("Testing Large Decision Network...") + + graph = ContextGraph() + + # Create a network of 100 decisions + decisions = [] + for i in range(100): + decision = Decision( + decision_id=f"decision_{i:03d}", + category="performance_test", + scenario=f"Performance test scenario {i}", + reasoning=f"Performance test reasoning {i}", + outcome="processed", + confidence=0.8 + (i % 20) * 0.01, # Varying confidence + timestamp=datetime.now(), + decision_maker="performance_agent" + ) + decisions.append(decision) + graph.add_decision(decision) + + assert len(graph.nodes) == 100 + print("+ Created 100 decisions") + + # Add causal relationships to create a network + for i in range(99): + # Create a mix of relationship types + relationship_type = ["CAUSED", "INFLUENCED", "PRECEDENT_FOR"][i % 3] + graph.add_causal_relationship(f"decision_{i:03d}", f"decision_{i+1:03d}", relationship_type) + + assert len(graph.edges) == 99 + print("+ Created 99 causal relationships") + + # Test causal chain performance + chain = graph.get_causal_chain("decision_099", direction="upstream", max_depth=50) + assert len(chain) > 0 + print("+ Causal chain analysis working") + + # Test precedent search performance + precedents = graph.find_precedents("decision_050", limit=20) + assert isinstance(precedents, list) + print("+ Precedent search working") + + # Test serialization performance + graph_dict = graph.to_dict() + assert len(graph_dict['nodes']) == 100 + assert len(graph_dict['edges']) == 99 + print("+ Large graph serialization working") + + print("✓ Large decision network test passed") + + def test_concurrent_operations(self): + """Test concurrent decision operations.""" + print("Testing Concurrent Operations...") + + import threading + import time + + graph = ContextGraph() + results = [] + errors = [] + + def add_decisions(start_id, count): + """Add decisions in a separate thread.""" + try: + for i in range(count): + decision = Decision( + decision_id=f"concurrent_decision_{start_id + i:03d}", + category="concurrent_test", + scenario=f"Concurrent test {start_id + i}", + reasoning="Concurrent reasoning", + outcome="processed", + confidence=0.8, + timestamp=datetime.now(), + decision_maker="concurrent_agent" + ) + graph.add_decision(decision) + time.sleep(0.001) # Small delay to simulate real work + results.append(f"Thread {start_id} completed") + except Exception as e: + errors.append(f"Thread {start_id} error: {e}") + + # Create multiple threads + threads = [] + for i in range(5): + thread = threading.Thread(target=add_decisions, args=(i * 20, 20)) + threads.append(thread) + thread.start() + + # Wait for all threads to complete + for thread in threads: + thread.join() + + # Verify results + assert len(errors) == 0, f"Errors occurred: {errors}" + assert len(results) == 5 + assert len(graph.nodes) == 100 # 5 threads * 20 decisions each + print("+ Concurrent operations completed successfully") + + print("✓ Concurrent operations test passed") + + +if __name__ == "__main__": + # Run tests when script is executed directly + pytest.main([__file__, "-v"]) From 1e4798ca0dc9a9e3219f092cf2360b2e7fb2cd7c Mon Sep 17 00:00:00 2001 From: Mohd Kaif <98801504+KaifAhmad1@users.noreply.github.com> Date: Sun, 15 Feb 2026 13:48:28 +0530 Subject: [PATCH 09/50] Update CHANGELOG with fixes and enhancements Documented fixes and enhancements related to Context Graphs and PolicyEngine, including comprehensive test coverage and improvements in decision handling. --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ac118ba..84030750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Fixed: Context Graphs decision tracking bugs and added comprehensive test coverage (PR #315 by @KaifAhmad1) + - Fixed empty/None decision ID handling in ContextGraph.add_decision() + - Fixed None metadata handling to prevent TypeError + - Fixed causal chain depth logic and node exclusion + - Fixed nonexistent node handling in add_causal_relationship() + - Added missing properties field in to_dict serialization + - Added missing from_dict method for graph deserialization + - Fixed precedent search direction in find_precedents() + - Fixed UUID generation logic in all decision models + - Added comprehensive test suite with 9 tests covering all features + - All 71 context tests now passing (100% success rate) + - Fixed: PolicyEngine latest version selection on ContextGraph; AgentContext fallback robustness and secure logging (PR #TBD by @KaifAhmad1) - Tests: Added ContextGraph fallback and AgentContext smoke tests; full suite passing From 20755e69e29b16dee50e4ccca2240b05a126baa5 Mon Sep 17 00:00:00 2001 From: Sameer Kadam Date: Sun, 15 Feb 2026 15:57:07 +0530 Subject: [PATCH 10/50] feat(graph): add Apache AGE backend integration with configuration, registration, tests and documentation (#311) --- README.md | 17 +- docs/graph_stores/apache_age.md | 243 +++++ semantica/graph_store/__init__.py | 3 + semantica/graph_store/age_store.py | 1312 ++++++++++++++++++++++++++ semantica/graph_store/config.py | 18 + semantica/graph_store/graph_store.py | 7 + tests/graph_store/__init__.py | 0 tests/graph_store/test_age_store.py | 772 +++++++++++++++ 8 files changed, 2369 insertions(+), 3 deletions(-) create mode 100644 docs/graph_stores/apache_age.md create mode 100644 semantica/graph_store/age_store.py create mode 100644 tests/graph_store/__init__.py create mode 100644 tests/graph_store/test_age_store.py diff --git a/README.md b/README.md index 6fce0ca0..ca507b48 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ print(f"Found {len(precedents)} precedents") - **Docling Support** — Document parsing with table extraction (PDF, DOCX, PPTX, XLSX) - **AWS Neptune** — Amazon Neptune graph database support with IAM authentication +- **Apache AGE** — PostgreSQL graph extension backend (openCypher via SQL) - **Custom Ontology Import** — Import existing ontologies (OWL, RDF, Turtle, JSON-LD) > **Built for environments where every answer must be explainable and governed.** @@ -284,7 +285,8 @@ The **semantic gap** is the fundamental disconnect between what AI systems can p - 📐 **OWL Ontologies** — HermiT/Pellet validated, custom ontology import support - 🔢 **Vector Embeddings** — FastEmbed by default - ☁️ **AWS Neptune** — Amazon Neptune graph database support -- 🔍 **Provenance** — Every AI response links back to: +- � **Apache AGE** — PostgreSQL graph extension with openCypher support +- �🔍 **Provenance** — Every AI response links back to: - 📄 Source documents - 🏷️ Extracted entities & relations - 📐 Ontology rules applied @@ -510,13 +512,13 @@ results = vector_store.search(query="supply chain", top_k=5) ### Graph Store & Triplet Store -> **Neo4j, FalkorDB, Amazon Neptune** • **SPARQL queries** • **RDF triplets** +> **Neo4j, FalkorDB, Amazon Neptune, Apache AGE** • **SPARQL queries** • **RDF triplets** ```python from semantica.graph_store import GraphStore from semantica.triplet_store import TripletStore -# Graph Store (Neo4j, FalkorDB) +# Graph Store (Neo4j, FalkorDB, Apache AGE) graph_store = GraphStore(backend="neo4j", uri="bolt://localhost:7687", user="neo4j", password="password") graph_store.add_nodes([{"id": "n1", "labels": ["Person"], "properties": {"name": "Alice"}}]) @@ -538,6 +540,15 @@ neptune_store.add_nodes([ # Query Operations result = neptune_store.execute_query("MATCH (p:Person) RETURN p.name, p.age") +# Apache AGE Graph Store (PostgreSQL + openCypher) +age_store = GraphStore( + backend="age", + connection_string="host=localhost dbname=agedb user=postgres password=secret", + graph_name="semantica", +) +age_store.connect() +age_store.create_node(labels=["Person"], properties={"name": "Alice", "age": 30}) + # Triplet Store (Blazegraph, Jena, RDF4J) triplet_store = TripletStore(backend="blazegraph", endpoint="http://localhost:9999/blazegraph") triplet_store.add_triplet({"subject": "Alice", "predicate": "knows", "object": "Bob"}) diff --git a/docs/graph_stores/apache_age.md b/docs/graph_stores/apache_age.md new file mode 100644 index 00000000..27c16a9d --- /dev/null +++ b/docs/graph_stores/apache_age.md @@ -0,0 +1,243 @@ +# Apache AGE Graph Store + +**Backend**: PostgreSQL + [Apache AGE](https://age.apache.org/) +**Driver**: `psycopg2` + +Apache AGE is a PostgreSQL extension that adds graph database functionality, enabling you to run openCypher queries alongside traditional SQL. This backend lets Semantica use AGE as a property graph store with the same interface as Neo4j and FalkorDB. + +--- + +## Prerequisites + +| Component | Version | +|-----------|---------| +| PostgreSQL | 12+ | +| Apache AGE | 1.4+ (compiled and installed) | +| psycopg2 | 2.9+ | + +```bash +pip install psycopg2-binary +``` + +> **Note**: Apache AGE must be compiled and installed into your PostgreSQL instance. See the [AGE installation guide](https://age.apache.org/age-manual/master/intro/setup.html). + +--- + +## Quick Start + +```python +from semantica.graph_store import GraphStore + +# Using the unified GraphStore facade +store = GraphStore( + backend="age", + connection_string="host=localhost dbname=agedb user=postgres password=secret", + graph_name="semantica", +) +store.connect() + +# Create nodes +alice = store.create_node(labels=["Person"], properties={"name": "Alice", "age": 30}) +bob = store.create_node(labels=["Person"], properties={"name": "Bob", "age": 25}) + +# Create relationship +rel = store.create_relationship(alice["id"], bob["id"], "KNOWS", {"since": 2023}) + +# Query +result = store.execute_query("MATCH (p:Person) RETURN p", cols="p agtype") +print(result["records"]) + +store.close() +``` + +### Direct Usage (without facade) + +```python +from semantica.graph_store.age_store import ApacheAgeStore + +store = ApacheAgeStore( + connection_string="host=localhost dbname=agedb user=postgres password=secret", + graph_name="my_graph", +) +store.connect() + +node = store.create_node(["Entity"], {"semantica_id": "ent-001", "value": "test"}) +print(node) +# {"id": 844424930131969, "labels": ["Entity"], "properties": {"semantica_id": "ent-001", "value": "test"}} + +store.close() +``` + +--- + +## Configuration + +### Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `GRAPH_STORE_AGE_CONNECTION_STRING` | PostgreSQL connection string | `host=localhost dbname=agedb user=postgres password=postgres` | +| `GRAPH_STORE_AGE_GRAPH_NAME` | AGE graph name | `semantica` | + +### Programmatic Configuration + +```python +from semantica.graph_store.config import graph_store_config + +graph_store_config.set("age_connection_string", "host=db.example.com dbname=prod_age user=app") +graph_store_config.set("age_graph_name", "production") +``` + +--- + +## Connection & Initialization + +On `connect()`, the store performs idempotent setup: + +1. `CREATE EXTENSION IF NOT EXISTS age;` +2. `LOAD 'age';` +3. `SET search_path = ag_catalog, "$user", public;` +4. Creates the named graph if it does not already exist. + +This is safe to call repeatedly. + +--- + +## ID Handling + +Apache AGE auto-generates internal vertex/edge IDs (large integers). These are **not** the same as any semantic or application-level ID you may want to assign. + +| Concept | Description | +|---------|-------------| +| **AGE internal ID** | Auto-generated by AGE. Exposed as `"id"` in all returned dicts. Used in `delete_node()`, `get_node()`, etc. | +| **Semantic ID** | Application-level identifier. Store it in the `semantica_id` property. | + +```python +node = store.create_node( + labels=["Document"], + properties={"semantica_id": "doc-abc-123", "title": "My Doc"}, +) +# node["id"] → AGE internal ID (e.g., 844424930131969) +# node["properties"]["semantica_id"] → "doc-abc-123" +``` + +> **Important**: Never mix AGE internal IDs with semantic IDs. Use `node["id"]` for graph operations (delete, update, traverse) and `node["properties"]["semantica_id"]` for application-level lookups. + +--- + +## Label Handling + +AGE supports exactly **one label per vertex**. Semantica handles this transparently: + +- `labels[0]` → used as the primary AGE vertex label. +- `labels[1:]` → stored in a `labels` property array on the vertex. + +When reading nodes, the store reconstructs the full label list automatically. + +```python +node = store.create_node( + labels=["Person", "Employee", "Admin"], + properties={"name": "Alice"}, +) +# In AGE: vertex with label "Person" and property labels=["Employee", "Admin"] +# Returned: {"id": ..., "labels": ["Person", "Employee", "Admin"], "properties": {"name": "Alice"}} +``` + +--- + +## Cypher Query Execution + +All Cypher queries are executed via AGE's SQL wrapper: + +```sql +SELECT * FROM cypher('graph_name', $$ $$) AS (col1 agtype, ...); +``` + +### Parameter Substitution + +AGE does not support `$param` style binding inside `cypher()` calls. The store safely converts parameters to Cypher literals with proper escaping: + +```python +result = store.execute_query( + "MATCH (p:Person) WHERE p.age > $min_age RETURN p", + parameters={"min_age": 25}, + cols="p agtype", +) +``` + +### Column Specification + +For custom queries, pass the `cols` option to specify the `AS` clause: + +```python +result = store.execute_query( + "MATCH (a)-[r]->(b) RETURN a, r, b", + cols="a agtype, r agtype, b agtype", +) +``` + +If omitted, the store attempts to infer columns from the `RETURN` clause. + +--- + +## Transactions + +The store uses explicit PostgreSQL transactions: + +- **Success** → `COMMIT` +- **Exception** → `ROLLBACK`, then re-raise as `ProcessingError` +- No silent failures + +--- + +## API Reference + +All methods match the standard Semantica graph store backend interface: + +| Method | Description | +|--------|-------------| +| `connect(**options)` | Connect and initialize AGE | +| `close()` | Close the connection | +| `create_node(labels, properties)` | Create a vertex | +| `create_nodes(nodes)` | Batch create vertices | +| `get_node(node_id)` | Get vertex by AGE ID | +| `get_nodes(labels, properties, limit)` | Query vertices | +| `update_node(node_id, properties, merge)` | Update vertex properties | +| `delete_node(node_id, detach)` | Delete a vertex | +| `create_relationship(start_id, end_id, type, properties)` | Create an edge | +| `get_relationships(node_id, rel_type, direction, limit)` | Query edges | +| `delete_relationship(rel_id)` | Delete an edge | +| `execute_query(query, parameters)` | Run arbitrary Cypher | +| `get_neighbors(node_id, rel_type, direction, depth)` | Graph traversal | +| `shortest_path(start_id, end_id, rel_type, max_depth)` | Path finding | +| `create_index(label, property_name, index_type)` | Create a PostgreSQL index | +| `get_stats()` | Graph statistics | + +--- + +## Docker Setup + +```yaml +services: + age: + image: apache/age:latest + ports: + - "5432:5432" + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: secret + POSTGRES_DB: agedb +``` + +```bash +docker compose up -d +``` + +Then connect: + +```python +store = GraphStore( + backend="age", + connection_string="host=localhost port=5432 dbname=agedb user=postgres password=secret", +) +``` diff --git a/semantica/graph_store/__init__.py b/semantica/graph_store/__init__.py index ea952b2e..51b02251 100644 --- a/semantica/graph_store/__init__.py +++ b/semantica/graph_store/__init__.py @@ -122,6 +122,7 @@ Author: Semantica Contributors License: MIT """ +from .age_store import ApacheAgeStore from .amazon_neptune import ( AmazonNeptuneStore, NeptuneAuthTokenManager, @@ -172,6 +173,8 @@ __all__ = [ "Neo4jStore", "Neo4jDriver", "Neo4jTransaction", + # Apache AGE + "ApacheAgeStore", # Amazon Neptune "AmazonNeptuneStore", "NeptuneAuthTokenManager", diff --git a/semantica/graph_store/age_store.py b/semantica/graph_store/age_store.py new file mode 100644 index 00000000..1f4ea8b6 --- /dev/null +++ b/semantica/graph_store/age_store.py @@ -0,0 +1,1312 @@ +""" +Apache AGE Store Module + +This module provides Apache AGE (PostgreSQL graph extension) integration for +property graph storage and Cypher querying in the Semantica framework, supporting +full CRUD operations, transactions, and graph analytics. + +Apache AGE extends PostgreSQL with graph database functionality, enabling +hybrid relational + graph workloads using openCypher queries executed via SQL. + +Key Features: + - OpenCypher query language support via SQL wrapper + - Node and relationship CRUD operations + - Transaction support with explicit commit/rollback + - Parameterized queries to prevent SQL injection + - AGE internal ID / semantic ID separation + - Multi-label emulation (one AGE label + property array) + - Batch operations with progress tracking + - Optional dependency handling (psycopg2) + +Main Classes: + - ApacheAgeStore: Main AGE store for graph operations + +Example Usage: + >>> from semantica.graph_store.age_store import ApacheAgeStore + >>> store = ApacheAgeStore( + ... connection_string="host=localhost dbname=agedb user=postgres password=secret", + ... graph_name="semantica" + ... ) + >>> store.connect() + >>> node = store.create_node(labels=["Person"], properties={"name": "Alice"}) + >>> results = store.execute_query("MATCH (p:Person) RETURN p") + >>> store.close() + +Note: + - AGE auto-generates internal vertex/edge IDs. + - The ``node_id`` parameter in CRUD methods refers to the AGE internal ID. + - Semantic IDs can be stored in the ``semantica_id`` property. + - AGE supports exactly one label per vertex; additional labels are stored + in a ``labels`` property array. + +Author: Semantica Contributors +License: MIT +""" + +import json +import re +from typing import Any, Dict, List, Optional, Union + +from ..utils.exceptions import ProcessingError, ValidationError +from ..utils.logging import get_logger +from ..utils.progress_tracker import get_progress_tracker + +# Optional psycopg2 import +try: + import psycopg2 + import psycopg2.extras + + PSYCOPG2_AVAILABLE = True +except (ImportError, OSError): + PSYCOPG2_AVAILABLE = False + psycopg2 = None + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _sanitize_label(label: str) -> str: + """ + Sanitize a Cypher label to prevent injection. + + Only allows alphanumeric characters and underscores. + + Args: + label: Raw label string. + + Returns: + Sanitized label string. + + Raises: + ValidationError: If the label contains invalid characters. + """ + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", label): + raise ValidationError( + f"Invalid label '{label}': must start with a letter or underscore " + "and contain only alphanumeric characters and underscores." + ) + return label + + +def _sanitize_rel_type(rel_type: str) -> str: + """ + Sanitize a relationship type string. + + Args: + rel_type: Raw relationship type. + + Returns: + Sanitized relationship type. + + Raises: + ValidationError: If the type contains invalid characters. + """ + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", rel_type): + raise ValidationError( + f"Invalid relationship type '{rel_type}': must start with a letter or " + "underscore and contain only alphanumeric characters and underscores." + ) + return rel_type + + +def _props_to_cypher_literal(properties: Dict[str, Any]) -> str: + """ + Convert a Python dict to an AGE-compatible Cypher map literal. + + AGE does not support ``$param`` style parameter binding inside + ``cypher()`` calls, so property values must be inlined as literals + with proper escaping. + + Args: + properties: Dictionary of property key-value pairs. + + Returns: + Cypher map literal string, e.g. ``{name: 'Alice', age: 30}``. + """ + if not properties: + return "{}" + parts = [] + for key, value in properties.items(): + # Validate key + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", key): + raise ValidationError(f"Invalid property key: '{key}'") + parts.append(f"{key}: {_value_to_cypher_literal(value)}") + return "{" + ", ".join(parts) + "}" + + +def _value_to_cypher_literal(value: Any) -> str: + """ + Convert a single Python value to a Cypher literal string. + + Args: + value: Python value. + + Returns: + Cypher literal representation. + """ + if value is None: + return "null" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + return repr(value) + if isinstance(value, str): + # Escape single quotes for Cypher strings + escaped = value.replace("\\", "\\\\").replace("'", "\\'") + return f"'{escaped}'" + if isinstance(value, (list, tuple)): + inner = ", ".join(_value_to_cypher_literal(v) for v in value) + return f"[{inner}]" + if isinstance(value, dict): + return _props_to_cypher_literal(value) + # Fallback: convert to string + escaped = str(value).replace("\\", "\\\\").replace("'", "\\'") + return f"'{escaped}'" + + +def _parse_agtype(raw: Any) -> Any: + """ + Parse an agtype value returned by AGE into a Python object. + + AGE returns results as ``agtype`` which may be a JSON-like string + with an optional ``::vertex`` / ``::edge`` / ``::path`` suffix. + + Args: + raw: Raw value from the cursor. + + Returns: + Parsed Python object (dict, list, or scalar). + """ + if raw is None: + return None + if not isinstance(raw, str): + return raw + + text = raw.strip() + + # Strip AGE type suffixes + for suffix in ("::vertex", "::edge", "::path", "::numeric", + "::integer", "::float", "::boolean", "::text"): + if text.endswith(suffix): + text = text[: -len(suffix)].strip() + break + + # Try JSON parse + try: + return json.loads(text) + except (json.JSONDecodeError, ValueError): + pass + + # Boolean literals + if text.lower() == "true": + return True + if text.lower() == "false": + return False + + # Numeric + try: + if "." in text: + return float(text) + return int(text) + except ValueError: + pass + + return text + + +def _vertex_to_node_dict(vertex: Any) -> Dict[str, Any]: + """ + Convert a parsed AGE vertex dict to the standard node return format. + + Expected vertex dict shape from AGE:: + + {"id": , "label": "