From f7fcfa36915c6cd0c1186a0d03205f1914777c5a Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Fri, 26 Dec 2025 23:00:14 +0530 Subject: [PATCH] Fix LLM-based entity and relation extraction - Updated extract_entities_llm to use custom entity_types in prompts - Updated extract_relations_llm to use custom relation_types in prompts - Made entity type filtering case-insensitive and flexible - Added verbose mode to RelationExtractor for progress tracking - Improved error handling and progress reporting in notebook - Made prompts more flexible to accept variations of entity/relation types --- .../01_DeFi_Protocol_Intelligence.ipynb | 222 ++++++++++++++---- semantica/semantic_extract/methods.py | 36 ++- semantica/semantic_extract/ner_extractor.py | 12 +- .../semantic_extract/relation_extractor.py | 16 ++ 4 files changed, 234 insertions(+), 52 deletions(-) diff --git a/cookbook/use_cases/blockchain/01_DeFi_Protocol_Intelligence.ipynb b/cookbook/use_cases/blockchain/01_DeFi_Protocol_Intelligence.ipynb index db4948bf..10aed39e 100644 --- a/cookbook/use_cases/blockchain/01_DeFi_Protocol_Intelligence.ipynb +++ b/cookbook/use_cases/blockchain/01_DeFi_Protocol_Intelligence.ipynb @@ -59,9 +59,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution ~gno (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n" + ] + } + ], "source": [ "%pip install -qU semantica networkx matplotlib plotly pandas faiss-cpu beautifulsoup4 groq sentence-transformers scikit-learn\n" ] @@ -75,13 +98,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", - "os.environ[\"GROQ_API_KEY\"] = os.getenv(\"GROQ_API_KEY\", \"your-key-here\")\n", + "os.environ[\"GROQ_API_KEY\"] = os.getenv(\"GROQ_API_KEY\", \"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\")\n", "\n", "# Configuration constants\n", "EMBEDDING_DIMENSION = 384\n", @@ -99,9 +122,49 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Ingesting from 6 feed sources...\n" + ] + }, + { + "data": { + "text/html": [ + "

🧠 Semantica - 📊 Current Progress

StatusActionModuleSubmoduleFileTime
❌Semantica is parsing🔍 parseDocumentParserp>\n", + "0.00s
❌Semantica is parsing🔍 parseDocumentParserp>\n", + "0.01s
❌Semantica is parsing🔍 parseDocumentParserp>\n", + "0.01s
❌Semantica is parsing🔍 parseDocumentParserp>\n", + "0.00s
❌Semantica is parsing🔍 parseDocumentParserp>\n", + "0.00s
❌Semantica is parsing🔍 parseDocumentParserp>\n", + "0.00s
❌Semantica is parsing🔍 parseDocumentParserp>\n", + "0.01s
✅Semantica is normalizing🔧 normalizeTextNormalizer-0.01s
✅Semantica is extracting🎯 semantic_extractNERExtractor-3.68s
🔄Semantica is extracting🎯 semantic_extractRelationExtractor-0.00s
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [1/6] CoinDesk: 25 documents\n", + " [2/6] CoinTelegraph: 30 documents\n", + " [3/6] Decrypt: 51 documents\n", + " [4/6] The Block: 19 documents\n", + " [5/6] CryptoSlate: 10 documents\n", + " [6/6] CryptoNews: 20 documents\n", + "Ingested 155 documents\n" + ] + } + ], "source": [ "from semantica.ingest import FeedIngestor, FileIngestor, WebIngestor\n", "import os\n", @@ -167,9 +230,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Parsing 155 documents...\n", + " Parsed 50/155 documents...\n", + " Parsed 100/155 documents...\n", + " Parsed 150/155 documents...\n", + " Parsed 155/155 documents...\n" + ] + } + ], "source": [ "from semantica.parse import DocumentParser\n", "\n", @@ -201,9 +276,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Normalizing 155 documents...\n", + " Normalized 50/155 documents...\n", + " Normalized 100/155 documents...\n", + " Normalized 150/155 documents...\n", + " Normalized 155/155 documents...\n", + "Chunking 155 documents...\n", + " Chunked 50/155 documents (50 chunks so far)\n", + " Chunked 100/155 documents (101 chunks so far)\n", + " Chunked 150/155 documents (151 chunks so far)\n", + " Chunked 155/155 documents (156 chunks so far)\n", + "Created 156 chunks from 155 documents\n" + ] + } + ], "source": [ "from semantica.normalize import TextNormalizer\n", "from semantica.split import TextSplitter\n", @@ -256,9 +349,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting entities from 156 chunks...\n", + " Processed 20/156 chunks (32 entities found)\n", + " Processed 40/156 chunks (100 entities found)\n", + " Processed 60/156 chunks (386 entities found)\n", + " Processed 80/156 chunks (410 entities found)\n", + " Processed 100/156 chunks (556 entities found)\n", + " Processed 120/156 chunks (647 entities found)\n", + " Processed 140/156 chunks (717 entities found)\n", + " Processed 156/156 chunks (1010 entities found)\n", + "Extracted 113 protocols, 792 tokens, 32 risks\n" + ] + } + ], "source": [ "from semantica.semantic_extract import NERExtractor\n", "\n", @@ -270,6 +380,7 @@ ")\n", "\n", "all_entities = []\n", + "error_count = 0\n", "print(f\"Extracting entities from {len(chunked_documents)} chunks...\")\n", "for i, chunk in enumerate(chunked_documents, 1):\n", " chunk_text = chunk.text if hasattr(chunk, 'text') else str(chunk)\n", @@ -279,12 +390,19 @@ " entity_types=[\"Protocol\", \"Token\", \"Pool\", \"Transaction\", \"Risk\"]\n", " )\n", " all_entities.extend(entities)\n", - " except Exception:\n", + " except Exception as e:\n", + " error_count += 1\n", + " # Print first few errors for debugging\n", + " if error_count <= 3:\n", + " print(f\" Warning: Error processing chunk {i}: {str(e)[:100]}\")\n", " continue\n", " \n", " if i % 20 == 0 or i == len(chunked_documents):\n", " print(f\" Processed {i}/{len(chunked_documents)} chunks ({len(all_entities)} entities found)\")\n", "\n", + "if error_count > 0:\n", + " print(f\" Note: {error_count} chunks had errors during extraction\")\n", + "\n", "protocols = [e for e in all_entities if e.label == \"Protocol\" or \"protocol\" in e.label.lower()]\n", "tokens = [e for e in all_entities if e.label == \"Token\" or \"token\" in e.label.lower()]\n", "risks = [e for e in all_entities if e.label == \"Risk\" or \"risk\" in e.label.lower()]\n", @@ -303,7 +421,15 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting relationships from 156 chunks...\n" + ] + } + ], "source": [ "from semantica.semantic_extract import RelationExtractor\n", "\n", @@ -311,21 +437,36 @@ " method=\"llm\",\n", " provider=\"groq\",\n", " llm_model=\"llama-3.1-8b-instant\",\n", - " temperature=0.0\n", + " temperature=0.0,\n", + " verbose=True\n", ")\n", "\n", "all_relationships = []\n", - "for chunk in chunked_documents:\n", + "error_count = 0\n", + "print(f\"Extracting relationships from {len(chunked_documents)} chunks...\")\n", + "\n", + "for i, chunk in enumerate(chunked_documents, 1):\n", " chunk_text = chunk.text if hasattr(chunk, 'text') else str(chunk)\n", " try:\n", " relationships = relation_extractor.extract_relations(\n", " chunk_text,\n", " entities=all_entities,\n", - " relation_types=[\"uses\", \"governs\", \"provides\", \"has_risk\", \"interacts_with\", \"depends_on\"]\n", + " relation_types=[\"uses\", \"governs\", \"provides\", \"has_risk\", \"interacts_with\", \"depends_on\"],\n", + " verbose=True\n", " )\n", " all_relationships.extend(relationships)\n", - " except Exception:\n", + " except Exception as e:\n", + " error_count += 1\n", + " # Print first few errors for debugging\n", + " if error_count <= 3:\n", + " print(f\" Warning: Error processing chunk {i}: {str(e)[:100]}\")\n", " continue\n", + " \n", + " if i % 20 == 0 or i == len(chunked_documents):\n", + " print(f\" Processed {i}/{len(chunked_documents)} chunks ({len(all_relationships)} relationships found)\")\n", + "\n", + "if error_count > 0:\n", + " print(f\" Note: {error_count} chunks had errors during relation extraction\")\n", "\n", "print(f\"Extracted {len(all_relationships)} relationships\")\n" ] @@ -337,35 +478,6 @@ "## Resolving Duplicate Entities\n" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from semantica.kg import EntityResolver\n", - "from semantica.semantic_extract import Entity\n", - "\n", - "# Convert Entity objects to dictionaries for EntityResolver\n", - "print(f\"Converting {len(all_entities)} entities to dictionaries...\")\n", - "entity_dicts = [{\"name\": e.text, \"type\": e.label, \"confidence\": e.confidence} for e in all_entities]\n", - "\n", - "# Use EntityResolver class to resolve duplicates\n", - "entity_resolver = EntityResolver(strategy=\"fuzzy\", similarity_threshold=0.85)\n", - "\n", - "print(f\"Resolving duplicates in {len(entity_dicts)} entities...\")\n", - "resolved_entities = entity_resolver.resolve_entities(entity_dicts)\n", - "\n", - "# Convert back to Entity objects\n", - "print(f\"Converting {len(resolved_entities)} resolved entities back to Entity objects...\")\n", - "merged_entities = [\n", - " Entity(text=e[\"name\"], label=e[\"type\"], confidence=e.get(\"confidence\", 1.0))\n", - " for e in resolved_entities\n", - "]\n", - "\n", - "print(f\"Deduplicated {len(entity_dicts)} entities to {len(merged_entities)} unique entities\")\n" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -688,8 +800,22 @@ } ], "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" } }, "nbformat": 4, diff --git a/semantica/semantic_extract/methods.py b/semantica/semantic_extract/methods.py index b117e1da..2f46b6a4 100644 --- a/semantica/semantic_extract/methods.py +++ b/semantica/semantic_extract/methods.py @@ -320,13 +320,24 @@ def extract_entities_llm( if not llm.is_available(): raise ProcessingError(f"{provider} provider not available") + # Use custom entity types if provided, otherwise use defaults + entity_types = kwargs.get("entity_types") + if entity_types: + entity_types_str = ", ".join(entity_types) + entity_types_instruction = f"""Preferred entity types: {entity_types_str}. +You may also use related or similar entity types if they better match the context (e.g., variations, synonyms, or domain-specific types). +If an entity doesn't fit any of the preferred types, use the most appropriate type from the preferred list or a closely related type.""" + else: + entity_types_instruction = """Entity types should be one of: PERSON, ORG, GPE, DATE, EVENT, PRODUCT, CONCEPT, or related types. +Use the most appropriate type for each entity, including variations or synonyms if they better match the context.""" + prompt = f"""Extract named entities from the following text. Return ONLY a valid JSON list of objects with the following structure: [ {{"text": "entity name", "label": "ENTITY_TYPE", "start": 0, "end": 10, "confidence": 0.9}} ] -Entity types should be one of: PERSON, ORG, GPE, DATE, EVENT, PRODUCT, CONCEPT. +{entity_types_instruction} Do not include any conversational filler, explanations, or markdown formatting outside the JSON block. Text: {text}""" @@ -733,18 +744,37 @@ def extract_relations_llm( **kwargs, ) -> List[Relation]: """LLM-based relation extraction.""" + # Support llm_model parameter to disambiguate from ML model + if "llm_model" in kwargs: + model = kwargs.pop("llm_model") + llm = create_provider(provider, model=model, **kwargs) if not llm.is_available(): raise ProcessingError(f"{provider} provider not available") entities_str = ", ".join([f"{e.text} ({e.label})" for e in entities]) + + # Use custom relation types if provided + relation_types = kwargs.get("relation_types") + if relation_types: + relation_types_str = ", ".join(relation_types) + relation_types_instruction = f""" +Preferred relation types: {relation_types_str}. +You may also use related or similar relation types if they better capture the relationship (e.g., variations, synonyms, or domain-specific relations). +If a relation doesn't fit any of the preferred types, use the most appropriate type from the preferred list or a closely related type that accurately describes the relationship.""" + else: + relation_types_instruction = """ +Extract meaningful relationships between entities. Use appropriate relation types that accurately describe how entities are connected. +Common relation types include: related_to, part_of, located_in, created_by, uses, depends_on, interacts_with, and similar variations.""" + prompt = f"""Extract relations between entities from the following text. Text: {text} -Entities: {entities_str} +Entities: {entities_str}{relation_types_instruction} -Return JSON format: [{{"subject": "...", "predicate": "...", "object": "...", "confidence": 0.9}}]""" +Return JSON format: [{{"subject": "...", "predicate": "...", "object": "...", "confidence": 0.9}}] +Extract all meaningful relationships between the entities, using the most appropriate relation type for each relationship.""" try: result = llm.generate_structured(prompt) diff --git a/semantica/semantic_extract/ner_extractor.py b/semantica/semantic_extract/ner_extractor.py index 020fb8ab..3e02a993 100644 --- a/semantica/semantic_extract/ner_extractor.py +++ b/semantica/semantic_extract/ner_extractor.py @@ -235,13 +235,23 @@ class NERExtractor: method_options["model"] = all_options.get( "llm_model", all_options.get("model") ) + # Pass entity_types to LLM method so it can use them in the prompt + if entity_types: + method_options["entity_types"] = entity_types entities = method_func(text, **method_options) # Filter by confidence and entity types filtered = [e for e in entities if e.confidence >= min_confidence] if entity_types: - filtered = [e for e in filtered if e.label in entity_types] + # Case-insensitive and flexible matching for entity types + entity_types_lower = {et.lower() for et in entity_types} + filtered = [ + e for e in filtered + if e.label.lower() in entity_types_lower + or any(et.lower() in e.label.lower() or e.label.lower() in et.lower() + for et in entity_types) + ] if filtered: all_entities.append((method_name, filtered)) diff --git a/semantica/semantic_extract/relation_extractor.py b/semantica/semantic_extract/relation_extractor.py index 1b6ba33e..59bc32f9 100644 --- a/semantica/semantic_extract/relation_extractor.py +++ b/semantica/semantic_extract/relation_extractor.py @@ -128,6 +128,7 @@ class RelationExtractor: self.bidirectional = bidirectional self.confidence_threshold = confidence_threshold self.max_distance = max_distance + self.verbose = config.get("verbose", False) # Method configuration self.method = method if isinstance(method, list) else [method] @@ -249,6 +250,7 @@ class RelationExtractor: min_confidence = options.get("min_confidence", self.min_confidence) validate = options.get("validate", self.validate) + relation_types = options.get("relation_types", self.relation_types) # Merge config with options all_options = {**self.config, **options} @@ -277,12 +279,26 @@ class RelationExtractor: method_options["model"] = all_options.get( "llm_model", all_options.get("model") ) + # Pass relation_types to LLM method so it can use them in the prompt + if relation_types: + method_options["relation_types"] = relation_types elif method_name == "dependency": method_options["model"] = all_options.get( "model", "en_core_web_sm" ) + # Print progress if verbose mode is enabled (only for LLM method to avoid spam) + verbose_mode = self.verbose or options.get("verbose", False) + if verbose_mode and method_name == "llm": + import sys + print(f" [RelationExtractor] Processing with {method_name}...", flush=True, file=sys.stdout) + relations = method_func(text, entities, **method_options) + + # Print result count if verbose (only for LLM method) + if verbose_mode and method_name == "llm" and len(relations) > 0: + import sys + print(f" [RelationExtractor] Extracted {len(relations)} relations", flush=True, file=sys.stdout) # Filter by confidence filtered = [r for r in relations if r.confidence >= min_confidence]