mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-09-15 04:00:33 +00:00
Fix 0 entities/relations issue in GraphRAG notebook and improve GraphBuilder logic
This commit is contained in:
KaifAhmad1
@@ -51,22 +51,7 @@
|
||||
"WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
|
||||
"descript-audiotools 0.7.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 4.25.8 which is incompatible.\n",
|
||||
"langchain-openai 0.2.10 requires langchain-core<0.4.0,>=0.3.21, but you have langchain-core 0.1.23 which is incompatible.\n",
|
||||
"mistral-common 1.5.1 requires tiktoken<0.8.0,>=0.7.0, but you have tiktoken 0.12.0 which is incompatible.\n",
|
||||
"nari-tts 0.1.0 requires numpy>=2.2.4, but you have numpy 1.26.4 which is incompatible.\n",
|
||||
"nari-tts 0.1.0 requires torch>=2.6.0, but you have torch 2.2.1 which is incompatible.\n",
|
||||
"parlant 3.0.2 requires fastapi==0.115.12, but you have fastapi 0.120.4 which is incompatible.\n",
|
||||
"parlant 3.0.2 requires fastmcp==2.6.1, but you have fastmcp 2.14.1 which is incompatible.\n",
|
||||
"parlant 3.0.2 requires opentelemetry-exporter-otlp-proto-grpc==1.27.0, but you have opentelemetry-exporter-otlp-proto-grpc 1.38.0 which is incompatible.\n",
|
||||
"parlant 3.0.2 requires rich<14.0.0,>=13.7.1, but you have rich 14.2.0 which is incompatible.\n",
|
||||
"parlant 3.0.2 requires tiktoken<0.9,>=0.8, but you have tiktoken 0.12.0 which is incompatible.\n",
|
||||
"parlant 3.0.2 requires tokenizers<0.21,>=0.20, but you have tokenizers 0.21.4 which is incompatible.\n",
|
||||
"parlant 3.0.2 requires uvicorn<0.33.0,>=0.32.1, but you have uvicorn 0.38.0 which is incompatible.\n",
|
||||
"vllm 0.6.6.post1 requires torch==2.5.1, but you have torch 2.2.1 which is incompatible.\n",
|
||||
"vllm 0.6.6.post1 requires torchvision==0.20.1, but you have torchvision 0.17.1 which is incompatible.\n"
|
||||
"WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -145,7 +130,7 @@
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div style='font-family: monospace;'><h4>🧠 Semantica - 📊 Current Progress</h4><table style='width: 100%; border-collapse: collapse;'><tr><th>Status</th><th>Action</th><th>Module</th><th>Submodule</th><th>File</th><th>Time</th></tr><tr><td>✅</td><td>Semantica is seeding</td><td>🌱 seed</td><td>SeedDataManager</td><td>-</td><td>0.05s</td></tr></table></div>"
|
||||
"<div style='font-family: monospace;'><h4>🧠 Semantica - 📊 Current Progress</h4><table style='width: 100%; border-collapse: collapse;'><tr><th>Status</th><th>Action</th><th>Module</th><th>Submodule</th><th>File</th><th>Time</th></tr><tr><td>✅</td><td>Semantica is ingesting</td><td>📥 ingest</td><td>FeedIngestor</td><td>rss</td><td>1.12s</td></tr><tr><td>✅</td><td>Semantica is ingesting</td><td>📥 ingest</td><td>WebIngestor</td><td>README.md</td><td>1.85s</td></tr><tr><td>✅</td><td>Semantica is ingesting</td><td>📥 ingest</td><td>WebIngestor</td><td>README.md</td><td>1.51s</td></tr><tr><td>✅</td><td>Semantica is normalizing</td><td>🔧 normalize</td><td>TextNormalizer</td><td>-</td><td>0.00s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>EntityAwareChunker</td><td>-</td><td>1.93s</td></tr><tr><td>✅</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>NERExtractor</td><td>-</td><td>0.87s</td></tr><tr><td>🔄</td><td>Semantica is building</td><td>🧠 kg</td><td>GraphBuilder</td><td>-</td><td>677.02s</td></tr><tr><td>🔄</td><td>Semantica is building</td><td>🧠 kg</td><td>EntityResolver</td><td>-</td><td>656.46s</td></tr><tr><td>🔄</td><td>Semantica is deduplicating</td><td>🔄 deduplication</td><td>DuplicateDetector</td><td>-</td><td>656.46s</td></tr><tr><td>🔄</td><td>Semantica is deduplicating</td><td>🔄 deduplication</td><td>SimilarityCalculator</td><td>-</td><td>0.01s</td></tr></table></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
@@ -204,7 +189,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -295,9 +280,26 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\sentence_transformers\\cross_encoder\\CrossEncoder.py:13: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
|
||||
" from tqdm.autonotebook import tqdm, trange\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Performing Graph-Aware Chunking (preserving entity boundaries)...\n",
|
||||
"Generated 23 Graph-Aware chunks (vs 29 baseline chunks for sample).\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.normalize import TextNormalizer\n",
|
||||
"from semantica.split import TextSplitter, EntityAwareChunker\n",
|
||||
@@ -340,21 +342,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "AttributeError",
|
||||
"evalue": "'dict' object has no attribute 'nodes'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[5], line 11\u001b[0m\n\u001b[0;32m 9\u001b[0m \u001b[38;5;66;03m# 2. Quality Control: Deduplication\u001b[39;00m\n\u001b[0;32m 10\u001b[0m detector \u001b[38;5;241m=\u001b[39m DuplicateDetector(similarity_threshold\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.85\u001b[39m)\n\u001b[1;32m---> 11\u001b[0m duplicates \u001b[38;5;241m=\u001b[39m detector\u001b[38;5;241m.\u001b[39mdetect_duplicates(\u001b[38;5;28mlist\u001b[39m(\u001b[43mkg\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnodes\u001b[49m(data\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)))\n\u001b[0;32m 12\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m duplicates:\n\u001b[0;32m 13\u001b[0m merger \u001b[38;5;241m=\u001b[39m EntityMerger()\n",
|
||||
"\u001b[1;31mAttributeError\u001b[0m: 'dict' object has no attribute 'nodes'"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import GraphBuilder\n",
|
||||
"from semantica.deduplication import DuplicateDetector, EntityMerger\n",
|
||||
@@ -400,7 +390,24 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Graph is empty or has no edges, returning 0 communities\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Top Influential Entities: []\n",
|
||||
"Network Connectivity Profile: sparse\n",
|
||||
"Inference Engine initialized with Domain Rules.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.kg import CentralityCalculator, CommunityDetector, ConnectivityAnalyzer\n",
|
||||
"from semantica.reasoning import InferenceEngine, InferenceStrategy\n",
|
||||
@@ -438,15 +445,34 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"fastembed not available. Install with: pip install fastembed. Using fallback embedding method.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "AttributeError",
|
||||
"evalue": "'Semantica' object has no attribute 'generate_embeddings'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[13], line 5\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01msemantica\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcontext\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m AgentContext\n\u001b[0;32m 4\u001b[0m vs \u001b[38;5;241m=\u001b[39m VectorStore(backend\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfaiss\u001b[39m\u001b[38;5;124m\"\u001b[39m, dimension\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1536\u001b[39m)\n\u001b[1;32m----> 5\u001b[0m embeddings \u001b[38;5;241m=\u001b[39m \u001b[43mcore\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate_embeddings\u001b[49m([\u001b[38;5;28mstr\u001b[39m(c\u001b[38;5;241m.\u001b[39mtext) \u001b[38;5;28;01mfor\u001b[39;00m c \u001b[38;5;129;01min\u001b[39;00m all_chunks[:\u001b[38;5;241m12\u001b[39m]])\n\u001b[0;32m 6\u001b[0m vs\u001b[38;5;241m.\u001b[39mstore_vectors(vectors\u001b[38;5;241m=\u001b[39membeddings, metadata\u001b[38;5;241m=\u001b[39m[{\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mstr\u001b[39m(c\u001b[38;5;241m.\u001b[39mtext)} \u001b[38;5;28;01mfor\u001b[39;00m c \u001b[38;5;129;01min\u001b[39;00m all_chunks[:\u001b[38;5;241m12\u001b[39m]])\n\u001b[0;32m 8\u001b[0m \u001b[38;5;66;03m# Global Context Manager for an Agent\u001b[39;00m\n",
|
||||
"\u001b[1;31mAttributeError\u001b[0m: 'Semantica' object has no attribute 'generate_embeddings'"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.vector_store import VectorStore\n",
|
||||
"from semantica.context import AgentContext\n",
|
||||
"\n",
|
||||
"vs = VectorStore(backend=\"faiss\", dimension=1536)\n",
|
||||
"embeddings = core.embedding_generator.generate_embeddings([str(c.text) for c in all_chunks[:12]])\n",
|
||||
"embeddings = core.generate_embeddings([str(c.text) for c in all_chunks[:12]])\n",
|
||||
"vs.store_vectors(vectors=embeddings, metadata=[{\"text\": str(c.text)} for c in all_chunks[:12]])\n",
|
||||
"\n",
|
||||
"# Global Context Manager for an Agent\n",
|
||||
|
||||
@@ -129,6 +129,48 @@ class ConflictDetector:
|
||||
|
||||
self.detected_conflicts: Dict[str, Conflict] = {}
|
||||
|
||||
def detect_conflicts(
|
||||
self,
|
||||
entities: Union[List[Dict[str, Any]], Dict[str, Any]],
|
||||
method: str = "entity",
|
||||
property_name: Optional[str] = None,
|
||||
entity_type: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> List[Conflict]:
|
||||
"""
|
||||
Detect conflicts using the specified method (convenience method).
|
||||
|
||||
Args:
|
||||
entities: Entities to check (List of dicts or a KG dict)
|
||||
method: Detection method ("entity", "value", "type", "relationship", "temporal", "logical")
|
||||
property_name: Property name for "value" method
|
||||
entity_type: Optional entity type filter
|
||||
**kwargs: Additional arguments
|
||||
|
||||
Returns:
|
||||
List of detected conflicts
|
||||
"""
|
||||
# If passed a KG dict, extract entities
|
||||
if isinstance(entities, dict) and "entities" in entities:
|
||||
entities = entities["entities"]
|
||||
|
||||
if method == "value":
|
||||
if not property_name:
|
||||
raise ValueError("property_name is required for value conflict detection")
|
||||
return self.detect_value_conflicts(entities, property_name, entity_type)
|
||||
elif method == "type":
|
||||
return self.detect_type_conflicts(entities)
|
||||
elif method == "relationship":
|
||||
relationships = kwargs.get("relationships", [])
|
||||
return self.detect_relationship_conflicts(relationships)
|
||||
elif method == "temporal":
|
||||
return self.detect_temporal_conflicts(entities)
|
||||
elif method == "logical":
|
||||
return self.detect_logical_conflicts(entities)
|
||||
else:
|
||||
# Default to entity-wide detection
|
||||
return self.detect_entity_conflicts(entities, entity_type)
|
||||
|
||||
def detect_value_conflicts(
|
||||
self,
|
||||
entities: Union[List[Dict[str, Any]], Dict[str, Any]],
|
||||
|
||||
@@ -120,13 +120,12 @@ class GraphBuilder:
|
||||
self.conflict_detector = None
|
||||
self.logger.debug("Conflict resolution disabled")
|
||||
|
||||
def _process_item(self, item: Any, all_entities: List[Any], all_relationships: List[Any]):
|
||||
def _process_item(self, item: Any, all_entities: List[Any], all_relationships: List[Any], **options):
|
||||
"""Helper to process a single item and add to entities or relationships list."""
|
||||
if hasattr(item, "text") and hasattr(item, "label"):
|
||||
# It's likely an Entity object
|
||||
# Convert to dict format expected by graph builder
|
||||
entity_dict = {
|
||||
"id": getattr(item, "id", item.text), # Use text as ID if no ID
|
||||
"id": getattr(item, "id", item.text),
|
||||
"name": item.text,
|
||||
"type": item.label,
|
||||
"confidence": getattr(item, "confidence", 1.0),
|
||||
@@ -135,14 +134,10 @@ class GraphBuilder:
|
||||
all_entities.append(entity_dict)
|
||||
elif hasattr(item, "subject") and hasattr(item, "predicate") and hasattr(item, "object"):
|
||||
# It's likely a Relation object
|
||||
# Convert to dict format
|
||||
# Subject and Object in Relation might be Entity objects or strings
|
||||
subj = item.subject
|
||||
obj = item.object
|
||||
|
||||
subj_id = getattr(subj, "text", subj) if not isinstance(subj, str) else subj
|
||||
obj_id = getattr(obj, "text", obj) if not isinstance(obj, str) else obj
|
||||
|
||||
rel_dict = {
|
||||
"source": subj_id,
|
||||
"target": obj_id,
|
||||
@@ -153,20 +148,63 @@ class GraphBuilder:
|
||||
all_relationships.append(rel_dict)
|
||||
elif isinstance(item, dict):
|
||||
processed = False
|
||||
if "entities" in item:
|
||||
all_entities.extend(item["entities"])
|
||||
processed = True
|
||||
if "relationships" in item:
|
||||
all_relationships.extend(item["relationships"])
|
||||
processed = True
|
||||
found_something = False
|
||||
|
||||
if not processed:
|
||||
if "entities" in item:
|
||||
entities_list = item["entities"]
|
||||
if entities_list:
|
||||
if isinstance(entities_list, list):
|
||||
for ent in entities_list:
|
||||
self._process_item(ent, all_entities, all_relationships, **options)
|
||||
else:
|
||||
self._process_item(entities_list, all_entities, all_relationships, **options)
|
||||
found_something = True
|
||||
processed = True
|
||||
|
||||
if "relationships" in item:
|
||||
rels_list = item["relationships"]
|
||||
if rels_list:
|
||||
if isinstance(rels_list, list):
|
||||
for rel in rels_list:
|
||||
self._process_item(rel, all_entities, all_relationships, **options)
|
||||
else:
|
||||
self._process_item(rels_list, all_entities, all_relationships, **options)
|
||||
found_something = True
|
||||
processed = True
|
||||
|
||||
if not found_something:
|
||||
if "source" in item and "target" in item:
|
||||
all_relationships.append(item)
|
||||
found_something = True
|
||||
elif "id" in item or "entity_id" in item or "name" in item:
|
||||
all_entities.append(item)
|
||||
found_something = True
|
||||
|
||||
# If still nothing found and has 'text', try extraction
|
||||
if not found_something and "text" in item:
|
||||
text = item["text"]
|
||||
# Perform extraction if requested or if it's the only way
|
||||
if options.get("extract", True):
|
||||
from ..semantic_extract.ner_extractor import NERExtractor
|
||||
from ..semantic_extract.triplet_extractor import TripletExtractor
|
||||
|
||||
ner_method = options.get("ner_method", "ml")
|
||||
triplet_method = options.get("triplet_method", "pattern")
|
||||
|
||||
ner = NERExtractor(method=ner_method)
|
||||
entities = ner.extract_entities(text)
|
||||
for ent in entities:
|
||||
self._process_item(ent, all_entities, all_relationships, **options)
|
||||
|
||||
# Only try triplets if specifically requested or if method provided
|
||||
if "triplet_method" in options or options.get("extract_relations", False):
|
||||
triplet = TripletExtractor(method=triplet_method)
|
||||
relations = triplet.extract_triplets(text)
|
||||
for rel in relations:
|
||||
self._process_item(rel, all_entities, all_relationships, **options)
|
||||
found_something = True
|
||||
else:
|
||||
# Unknown type, try to treat as entity if it has string representation
|
||||
# Unknown type
|
||||
pass
|
||||
|
||||
def build(
|
||||
@@ -182,6 +220,10 @@ class GraphBuilder:
|
||||
sources: Entities or sources list
|
||||
second_arg: Optional relationships list or entity_resolver (for backward compatibility)
|
||||
**options: Additional build options
|
||||
- extract: Whether to extract entities from text (default: True)
|
||||
- extract_relations: Whether to extract relations from text (default: False)
|
||||
- ner_method: NER method to use (default: "ml")
|
||||
- triplet_method: Triplet extraction method (default: "pattern")
|
||||
|
||||
Returns:
|
||||
Dictionary containing entities and relationships
|
||||
@@ -229,18 +271,18 @@ class GraphBuilder:
|
||||
if isinstance(source, list):
|
||||
# List of items (could be entities, relations, or mixed)
|
||||
for item in source:
|
||||
self._process_item(item, all_entities, all_relationships)
|
||||
self._process_item(item, all_entities, all_relationships, **options)
|
||||
else:
|
||||
self._process_item(source, all_entities, all_relationships)
|
||||
self._process_item(source, all_entities, all_relationships, **options)
|
||||
|
||||
# Process explicit relationships if provided
|
||||
if explicit_relationships:
|
||||
for rel_item in explicit_relationships:
|
||||
if isinstance(rel_item, list):
|
||||
for item in rel_item:
|
||||
self._process_item(item, all_entities, all_relationships)
|
||||
self._process_item(item, all_entities, all_relationships, **options)
|
||||
else:
|
||||
self._process_item(rel_item, all_entities, all_relationships)
|
||||
self._process_item(rel_item, all_entities, all_relationships, **options)
|
||||
|
||||
self.logger.debug(
|
||||
f"Extracted {len(all_entities)} entities and "
|
||||
|
||||
Reference in New Issue
Block a user