mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Fix LLM-based entity and relation extraction
- Updated extract_entities_llm to use custom entity_types in prompts - Updated extract_relations_llm to use custom relation_types in prompts - Made entity type filtering case-insensitive and flexible - Added verbose mode to RelationExtractor for progress tracking - Improved error handling and progress reporting in notebook - Made prompts more flexible to accept variations of entity/relation types
This commit is contained in:
KaifAhmad1
@@ -59,9 +59,32 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Note: you may need to restart the kernel to use updated packages.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"WARNING: Ignoring invalid distribution ~gno (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~lotly (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~ython-socketio (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~gno (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~lotly (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~ython-socketio (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~gno (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~lotly (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n",
|
||||
"WARNING: Ignoring invalid distribution ~ython-socketio (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu beautifulsoup4 groq sentence-transformers scikit-learn\n"
|
||||
]
|
||||
@@ -75,13 +98,13 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"GROQ_API_KEY\"] = os.getenv(\"GROQ_API_KEY\", \"your-key-here\")\n",
|
||||
"os.environ[\"GROQ_API_KEY\"] = os.getenv(\"GROQ_API_KEY\", \"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\")\n",
|
||||
"\n",
|
||||
"# Configuration constants\n",
|
||||
"EMBEDDING_DIMENSION = 384\n",
|
||||
@@ -99,9 +122,49 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Ingesting from 6 feed sources...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div style='font-family: monospace;'><h4>🧠 Semantica - 📊 Current Progress</h4><table style='width: 100%; border-collapse: collapse;'><tr><th>Status</th><th>Action</th><th>Module</th><th>Submodule</th><th>File</th><th>Time</th></tr><tr><td>❌</td><td>Semantica is parsing</td><td>🔍 parse</td><td>DocumentParser</td><td>p>\n",
|
||||
"</td><td>0.00s</td></tr><tr><td>❌</td><td>Semantica is parsing</td><td>🔍 parse</td><td>DocumentParser</td><td>p>\n",
|
||||
"</td><td>0.01s</td></tr><tr><td>❌</td><td>Semantica is parsing</td><td>🔍 parse</td><td>DocumentParser</td><td>p>\n",
|
||||
"</td><td>0.01s</td></tr><tr><td>❌</td><td>Semantica is parsing</td><td>🔍 parse</td><td>DocumentParser</td><td>p>\n",
|
||||
"</td><td>0.00s</td></tr><tr><td>❌</td><td>Semantica is parsing</td><td>🔍 parse</td><td>DocumentParser</td><td>p>\n",
|
||||
"</td><td>0.00s</td></tr><tr><td>❌</td><td>Semantica is parsing</td><td>🔍 parse</td><td>DocumentParser</td><td>p>\n",
|
||||
"</td><td>0.00s</td></tr><tr><td>❌</td><td>Semantica is parsing</td><td>🔍 parse</td><td>DocumentParser</td><td>p>\n",
|
||||
"</td><td>0.01s</td></tr><tr><td>✅</td><td>Semantica is normalizing</td><td>🔧 normalize</td><td>TextNormalizer</td><td>-</td><td>0.01s</td></tr><tr><td>✅</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>NERExtractor</td><td>-</td><td>3.68s</td></tr><tr><td>🔄</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>RelationExtractor</td><td>-</td><td>0.00s</td></tr></table></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" [1/6] CoinDesk: 25 documents\n",
|
||||
" [2/6] CoinTelegraph: 30 documents\n",
|
||||
" [3/6] Decrypt: 51 documents\n",
|
||||
" [4/6] The Block: 19 documents\n",
|
||||
" [5/6] CryptoSlate: 10 documents\n",
|
||||
" [6/6] CryptoNews: 20 documents\n",
|
||||
"Ingested 155 documents\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.ingest import FeedIngestor, FileIngestor, WebIngestor\n",
|
||||
"import os\n",
|
||||
@@ -167,9 +230,21 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Parsing 155 documents...\n",
|
||||
" Parsed 50/155 documents...\n",
|
||||
" Parsed 100/155 documents...\n",
|
||||
" Parsed 150/155 documents...\n",
|
||||
" Parsed 155/155 documents...\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.parse import DocumentParser\n",
|
||||
"\n",
|
||||
@@ -201,9 +276,27 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Normalizing 155 documents...\n",
|
||||
" Normalized 50/155 documents...\n",
|
||||
" Normalized 100/155 documents...\n",
|
||||
" Normalized 150/155 documents...\n",
|
||||
" Normalized 155/155 documents...\n",
|
||||
"Chunking 155 documents...\n",
|
||||
" Chunked 50/155 documents (50 chunks so far)\n",
|
||||
" Chunked 100/155 documents (101 chunks so far)\n",
|
||||
" Chunked 150/155 documents (151 chunks so far)\n",
|
||||
" Chunked 155/155 documents (156 chunks so far)\n",
|
||||
"Created 156 chunks from 155 documents\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.normalize import TextNormalizer\n",
|
||||
"from semantica.split import TextSplitter\n",
|
||||
@@ -256,9 +349,26 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Extracting entities from 156 chunks...\n",
|
||||
" Processed 20/156 chunks (32 entities found)\n",
|
||||
" Processed 40/156 chunks (100 entities found)\n",
|
||||
" Processed 60/156 chunks (386 entities found)\n",
|
||||
" Processed 80/156 chunks (410 entities found)\n",
|
||||
" Processed 100/156 chunks (556 entities found)\n",
|
||||
" Processed 120/156 chunks (647 entities found)\n",
|
||||
" Processed 140/156 chunks (717 entities found)\n",
|
||||
" Processed 156/156 chunks (1010 entities found)\n",
|
||||
"Extracted 113 protocols, 792 tokens, 32 risks\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.semantic_extract import NERExtractor\n",
|
||||
"\n",
|
||||
@@ -270,6 +380,7 @@
|
||||
")\n",
|
||||
"\n",
|
||||
"all_entities = []\n",
|
||||
"error_count = 0\n",
|
||||
"print(f\"Extracting entities from {len(chunked_documents)} chunks...\")\n",
|
||||
"for i, chunk in enumerate(chunked_documents, 1):\n",
|
||||
" chunk_text = chunk.text if hasattr(chunk, 'text') else str(chunk)\n",
|
||||
@@ -279,12 +390,19 @@
|
||||
" entity_types=[\"Protocol\", \"Token\", \"Pool\", \"Transaction\", \"Risk\"]\n",
|
||||
" )\n",
|
||||
" all_entities.extend(entities)\n",
|
||||
" except Exception:\n",
|
||||
" except Exception as e:\n",
|
||||
" error_count += 1\n",
|
||||
" # Print first few errors for debugging\n",
|
||||
" if error_count <= 3:\n",
|
||||
" print(f\" Warning: Error processing chunk {i}: {str(e)[:100]}\")\n",
|
||||
" continue\n",
|
||||
" \n",
|
||||
" if i % 20 == 0 or i == len(chunked_documents):\n",
|
||||
" print(f\" Processed {i}/{len(chunked_documents)} chunks ({len(all_entities)} entities found)\")\n",
|
||||
"\n",
|
||||
"if error_count > 0:\n",
|
||||
" print(f\" Note: {error_count} chunks had errors during extraction\")\n",
|
||||
"\n",
|
||||
"protocols = [e for e in all_entities if e.label == \"Protocol\" or \"protocol\" in e.label.lower()]\n",
|
||||
"tokens = [e for e in all_entities if e.label == \"Token\" or \"token\" in e.label.lower()]\n",
|
||||
"risks = [e for e in all_entities if e.label == \"Risk\" or \"risk\" in e.label.lower()]\n",
|
||||
@@ -303,7 +421,15 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Extracting relationships from 156 chunks...\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.semantic_extract import RelationExtractor\n",
|
||||
"\n",
|
||||
@@ -311,21 +437,36 @@
|
||||
" method=\"llm\",\n",
|
||||
" provider=\"groq\",\n",
|
||||
" llm_model=\"llama-3.1-8b-instant\",\n",
|
||||
" temperature=0.0\n",
|
||||
" temperature=0.0,\n",
|
||||
" verbose=True\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"all_relationships = []\n",
|
||||
"for chunk in chunked_documents:\n",
|
||||
"error_count = 0\n",
|
||||
"print(f\"Extracting relationships from {len(chunked_documents)} chunks...\")\n",
|
||||
"\n",
|
||||
"for i, chunk in enumerate(chunked_documents, 1):\n",
|
||||
" chunk_text = chunk.text if hasattr(chunk, 'text') else str(chunk)\n",
|
||||
" try:\n",
|
||||
" relationships = relation_extractor.extract_relations(\n",
|
||||
" chunk_text,\n",
|
||||
" entities=all_entities,\n",
|
||||
" relation_types=[\"uses\", \"governs\", \"provides\", \"has_risk\", \"interacts_with\", \"depends_on\"]\n",
|
||||
" relation_types=[\"uses\", \"governs\", \"provides\", \"has_risk\", \"interacts_with\", \"depends_on\"],\n",
|
||||
" verbose=True\n",
|
||||
" )\n",
|
||||
" all_relationships.extend(relationships)\n",
|
||||
" except Exception:\n",
|
||||
" except Exception as e:\n",
|
||||
" error_count += 1\n",
|
||||
" # Print first few errors for debugging\n",
|
||||
" if error_count <= 3:\n",
|
||||
" print(f\" Warning: Error processing chunk {i}: {str(e)[:100]}\")\n",
|
||||
" continue\n",
|
||||
" \n",
|
||||
" if i % 20 == 0 or i == len(chunked_documents):\n",
|
||||
" print(f\" Processed {i}/{len(chunked_documents)} chunks ({len(all_relationships)} relationships found)\")\n",
|
||||
"\n",
|
||||
"if error_count > 0:\n",
|
||||
" print(f\" Note: {error_count} chunks had errors during relation extraction\")\n",
|
||||
"\n",
|
||||
"print(f\"Extracted {len(all_relationships)} relationships\")\n"
|
||||
]
|
||||
@@ -337,35 +478,6 @@
|
||||
"## Resolving Duplicate Entities\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import EntityResolver\n",
|
||||
"from semantica.semantic_extract import Entity\n",
|
||||
"\n",
|
||||
"# Convert Entity objects to dictionaries for EntityResolver\n",
|
||||
"print(f\"Converting {len(all_entities)} entities to dictionaries...\")\n",
|
||||
"entity_dicts = [{\"name\": e.text, \"type\": e.label, \"confidence\": e.confidence} for e in all_entities]\n",
|
||||
"\n",
|
||||
"# Use EntityResolver class to resolve duplicates\n",
|
||||
"entity_resolver = EntityResolver(strategy=\"fuzzy\", similarity_threshold=0.85)\n",
|
||||
"\n",
|
||||
"print(f\"Resolving duplicates in {len(entity_dicts)} entities...\")\n",
|
||||
"resolved_entities = entity_resolver.resolve_entities(entity_dicts)\n",
|
||||
"\n",
|
||||
"# Convert back to Entity objects\n",
|
||||
"print(f\"Converting {len(resolved_entities)} resolved entities back to Entity objects...\")\n",
|
||||
"merged_entities = [\n",
|
||||
" Entity(text=e[\"name\"], label=e[\"type\"], confidence=e.get(\"confidence\", 1.0))\n",
|
||||
" for e in resolved_entities\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"print(f\"Deduplicated {len(entity_dicts)} entities to {len(merged_entities)} unique entities\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
@@ -688,8 +800,22 @@
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
Reference in New Issue
Block a user