mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Add all changes
This commit is contained in:
KaifAhmad1
@@ -35,14 +35,14 @@
|
||||
"## End-to-End Workflow\n",
|
||||
"\n",
|
||||
"**Workflow:** \n",
|
||||
"Dual PDF Input → Docling Parsing → Normalization & Chunking → Entity, Relation & Triplet Extraction → Conflict Resolution & Deduplication → Knowledge Graph Construction → Amazon Neptune → GraphRAG → Agent Memory & Context → Strategic Q&A\n",
|
||||
"Dual PDF Input → Docling Parsing → Normalization & Chunking → Entity, Relation Extraction → Conflict Resolution & Deduplication → Knowledge Graph Construction → Amazon Neptune → GraphRAG → Agent Memory & Context → Strategic Q&A\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Pipeline Capabilities\n",
|
||||
"\n",
|
||||
"- High-fidelity PDF parsing (text, tables, structure) \n",
|
||||
"- Semantic extraction of entities, relationships, and triplets \n",
|
||||
"- Semantic extraction of entities, and relationships\n",
|
||||
"- Conflict detection and resolution with confidence awareness \n",
|
||||
"- Entity deduplication and canonicalization \n",
|
||||
"- Knowledge graph construction and validation \n",
|
||||
@@ -280,7 +280,6 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"from semantica.semantic_extract import NERExtractor\n",
|
||||
"\n",
|
||||
"ner = NERExtractor(\n",
|
||||
@@ -291,25 +290,19 @@
|
||||
" api_key=GROQ_API_KEY,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"ENTITY_TYPES = [\n",
|
||||
" \"ORGANIZATION\", \"ORG\", \"PERSON\", \"MONEY\", \"CURRENCY\",\n",
|
||||
" \"PERCENT\", \"PERCENTAGE\", \"DATE\", \"TIME\", \"PRODUCT\",\n",
|
||||
" \"LOCATION\", \"GPE\", \"EVENT\", \"QUANTITY\", \"CARDINAL\",\n",
|
||||
"ENTITY_TYPES = [\"ORGANIZATION\", \"PERSON\", \"MONEY\", \"PERCENT\", \"DATE\", \"EVENT\"]\n",
|
||||
"\n",
|
||||
"all_entities = [\n",
|
||||
" e\n",
|
||||
" for c in chunks\n",
|
||||
" for e in ner.extract_entities(\n",
|
||||
" get_chunk_text(c),\n",
|
||||
" entity_types=ENTITY_TYPES,\n",
|
||||
" )\n",
|
||||
" if get_chunk_text(c).strip()\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"all_entities = []\n",
|
||||
"\n",
|
||||
"for chunk in chunks:\n",
|
||||
" text = get_chunk_text(chunk)\n",
|
||||
" if text.strip():\n",
|
||||
" all_entities += ner.extract_entities(text, entity_types=ENTITY_TYPES)\n",
|
||||
"\n",
|
||||
"print(\"Entity extraction completed\")\n",
|
||||
"print(\"Total entities extracted:\", len(all_entities))\n",
|
||||
"\n",
|
||||
"print(\"\\nSample entities\")\n",
|
||||
"for e in all_entities[:10]:\n",
|
||||
" print(f\"{e.label}: {e.text}\")"
|
||||
"print(\"Entities:\", len(all_entities))"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -382,105 +375,39 @@
|
||||
"\n",
|
||||
"relation_extractor = RelationExtractor(\n",
|
||||
" method=\"llm\",\n",
|
||||
" confidence_threshold=0.5,\n",
|
||||
" confidence_threshold=0.6,\n",
|
||||
" relation_types=[\n",
|
||||
" \"HAS_REVENUE\", \"HAS_EPS\", \"HAS_MARGIN\", \"HAS_PROFIT\", \"HAS_GROWTH\",\n",
|
||||
" \"PROVIDES_GUIDANCE\", \"STATES\", \"ANNOUNCES\", \"REPORTS\", \"EXPECTS\",\n",
|
||||
" \"OPERATES_IN\", \"LOCATED_IN\", \"PARTNERS_WITH\", \"SERVES\",\n",
|
||||
" \"COMPARED_TO\", \"INCREASED_BY\", \"DECREASED_BY\", \"CHANGED_BY\",\n",
|
||||
" \"DURING\", \"IN_QUARTER\", \"FOR_PERIOD\",\n",
|
||||
" \"RELATED_TO\", \"PART_OF\", \"AFFECTS\",\n",
|
||||
" \"HAS_REVENUE\",\n",
|
||||
" \"HAS_GROWTH\",\n",
|
||||
" \"REPORTS\",\n",
|
||||
" \"PROVIDES_GUIDANCE\",\n",
|
||||
" \"IN_QUARTER\",\n",
|
||||
" \"FOR_PERIOD\",\n",
|
||||
" \"RELATED_TO\",\n",
|
||||
" ],\n",
|
||||
" api_key=GROQ_API_KEY,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"def get_chunk_text(chunk):\n",
|
||||
" return getattr(chunk, \"content\", getattr(chunk, \"text\", \"\")) or \"\"\n",
|
||||
"\n",
|
||||
"relationships = []\n",
|
||||
"\n",
|
||||
"for chunk in chunks:\n",
|
||||
" text = get_chunk_text(chunk)\n",
|
||||
"\n",
|
||||
" relations = relation_extractor.extract_relations(\n",
|
||||
" text,\n",
|
||||
"relationships = [\n",
|
||||
" r\n",
|
||||
" for c in chunks\n",
|
||||
" for r in relation_extractor.extract_relations(\n",
|
||||
" text=get_chunk_text(c),\n",
|
||||
" entities=all_entities,\n",
|
||||
" provider=\"groq\",\n",
|
||||
" llm_model=\"llama-3.1-8b-instant\",\n",
|
||||
" temperature=0.0,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" relationships += relations\n",
|
||||
"\n",
|
||||
"print(\"Relationship extraction completed\")\n",
|
||||
"print(\"Total chunks:\", len(chunks))\n",
|
||||
"print(\"Total relationships extracted:\", len(relationships))\n",
|
||||
"\n",
|
||||
"if relationships:\n",
|
||||
" r = relationships[0]\n",
|
||||
" print(\"Sample relationship:\")\n",
|
||||
" print(f\"{r.subject.text} → {r.predicate} → {r.object.text}\")"
|
||||
" if get_chunk_text(c).strip()\n",
|
||||
"]\n",
|
||||
"print(\"Relationships:\", len(relationships))\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 6: Extract RDF Triplets\n",
|
||||
"\n",
|
||||
"Extract RDF triplets (subject-predicate-object) using TripletExtractor with Groq LLM.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.semantic_extract import TripletExtractor\n",
|
||||
"\n",
|
||||
"triplet_extractor = TripletExtractor(\n",
|
||||
" method=\"llm\",\n",
|
||||
" include_temporal=True,\n",
|
||||
" include_provenance=True,\n",
|
||||
" provider=\"groq\",\n",
|
||||
" llm_model=\"llama-3.1-8b-instant\",\n",
|
||||
" temperature=0.0,\n",
|
||||
" api_key=GROQ_API_KEY,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"def get_chunk_text(chunk):\n",
|
||||
" return getattr(chunk, \"content\", getattr(chunk, \"text\", \"\")) or \"\"\n",
|
||||
"\n",
|
||||
"triplets = []\n",
|
||||
"\n",
|
||||
"for chunk in chunks:\n",
|
||||
" text = get_chunk_text(chunk)\n",
|
||||
"\n",
|
||||
" triplets += triplet_extractor.extract_triplets(\n",
|
||||
" text,\n",
|
||||
" entities=all_entities,\n",
|
||||
" relations=relationships if relationships else None,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"if hasattr(triplet_extractor, \"validate_triplets\"):\n",
|
||||
" triplets = triplet_extractor.validate_triplets(triplets)\n",
|
||||
"\n",
|
||||
"print(\"Triplet extraction completed\")\n",
|
||||
"print(\"Total chunks:\", len(chunks))\n",
|
||||
"print(\"Total RDF triplets:\", len(triplets))\n",
|
||||
"\n",
|
||||
"if triplets:\n",
|
||||
" t = triplets[0]\n",
|
||||
" print(\"Sample triplet:\")\n",
|
||||
" print(f\"{t.subject} → {t.predicate} → {t.object}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 7: Detect Conflicts\n",
|
||||
"## Step 6: Detect Conflicts\n",
|
||||
"\n",
|
||||
"Detect conflicts in extracted entities and relationships using ConflictDetector.\n"
|
||||
]
|
||||
@@ -533,7 +460,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 8: Resolve Conflicts\n",
|
||||
"## Step 7: Resolve Conflicts\n",
|
||||
"\n",
|
||||
"Resolve detected conflicts using ConflictResolver with voting strategy.\n"
|
||||
]
|
||||
@@ -571,7 +498,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 9: Deduplicate Entities\n",
|
||||
"## Step 8: Deduplicate Entities\n",
|
||||
"\n",
|
||||
"Detect and merge duplicate entities using DuplicateDetector and EntityMerger.\n"
|
||||
]
|
||||
@@ -621,7 +548,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 10: Build Knowledge Graph\n",
|
||||
"## Step 9: Build Knowledge Graph\n",
|
||||
"\n",
|
||||
"Build knowledge graph from cleaned entities, relationships, and triplets using GraphBuilder.\n"
|
||||
]
|
||||
@@ -676,7 +603,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 11: Analyze Knowledge Graph\n",
|
||||
"## Step 10: Analyze Knowledge Graph\n",
|
||||
"\n",
|
||||
"This step evaluates the structure and quality of the knowledge graph.\n",
|
||||
"\n",
|
||||
@@ -726,7 +653,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 12: Persist Knowledge Graph in Amazon Neptune\n",
|
||||
"## Step 11: Persist Knowledge Graph in Amazon Neptune\n",
|
||||
"\n",
|
||||
"After cleaning, conflict resolution, and deduplication, the final step is to\n",
|
||||
"persist the **canonical knowledge graph** into a production graph database.\n",
|
||||
@@ -841,7 +768,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 13: Context Retrieval\n",
|
||||
"## Step 12: Context Retrieval\n",
|
||||
"\n",
|
||||
"Set up hybrid retrieval (vector + graph) using ContextRetriever for GraphRAG queries.\n"
|
||||
]
|
||||
@@ -894,7 +821,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 14: Agent Memory (Long-Term Context)\n",
|
||||
"## Step 13: Agent Memory (Long-Term Context)\n",
|
||||
"\n",
|
||||
"This step enables long-term memory for agents by storing important facts,\n",
|
||||
"metrics, and entities extracted from the knowledge graph.\n",
|
||||
@@ -965,7 +892,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 15: Agent Context\n",
|
||||
"## Step 14: Agent Context\n",
|
||||
"\n",
|
||||
"**AgentContext** provides a unified context layer that combines **vector-based RAG**\n",
|
||||
"with **graph-based GraphRAG** for grounded and explainable retrieval.\n",
|
||||
@@ -1063,7 +990,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 16: Answer Generation\n",
|
||||
"## Step 15: Answer Generation\n",
|
||||
"\n",
|
||||
"Generate answers to financial questions using Groq LLM with retrieved context and knowledge graph.\n"
|
||||
]
|
||||
@@ -1133,7 +1060,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 17: Export Results\n",
|
||||
"## Step 16: Export Results\n",
|
||||
"\n",
|
||||
"Export knowledge graph and analysis results to JSON and RDF formats.\n"
|
||||
]
|
||||
@@ -1155,7 +1082,6 @@
|
||||
"analysis_summary = {\n",
|
||||
" \"entities\": len(knowledge_graph.get(\"entities\", [])),\n",
|
||||
" \"relationships\": len(knowledge_graph.get(\"relationships\", [])),\n",
|
||||
" \"triplets\": len(triplets),\n",
|
||||
" \"conflicts_resolved\": len(resolved_conflicts),\n",
|
||||
" \"merged_entities\": len(merged_entities),\n",
|
||||
" \"communities\": num_communities,\n",
|
||||
|
||||
Reference in New Issue
Block a user