From cd1435ee103b749a3e6a7f05c62dbb668f3b4a20 Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Sun, 11 Jan 2026 23:25:28 +0530 Subject: [PATCH] Save changes to Earnings Call Analysis notebook --- .../finance/03_Earnings_Call_Analysis.ipynb | 1152 +++++++++-------- 1 file changed, 581 insertions(+), 571 deletions(-) diff --git a/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb b/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb index c6ee873b..8e85d46e 100644 --- a/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb +++ b/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb @@ -6,69 +6,115 @@ "source": [ "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Hawksight-AI/semantica/blob/main/cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb)\n", "\n", - "# Earnings Call Analysis with Docling and Semantica: MDA Space Q3 2025\n", + "# Earnings Call Analysis with Docling, Semantica & AWS Neptune \n", + "MDA Space Ltd. — Q3 2025\n", "\n", - "## Data Sources\n", + "## Data Sources\n", "\n", - "This analysis dual-tracks two critical documents from **MDA Space Ltd.** for the third quarter of 2025 to provide a holistic view of the company's performance and strategy:\n", - "1. **Press Release:** A structured summary of financial results, strategic milestones, and executive commentary.\n", - "2. **Earnings Transcript:** A detailed record of the management presentation and the subsequent Q&A session with analysts, providing deeper qualitative context.\n", + "This notebook analyzes two financial documents from **MDA Space Ltd.** for Q3 2025:\n", "\n", - "### Q3 2025 Strategic Highlights\n", - "* **Robust Backlog:** The company maintained a significant backlog at quarter-end, providing strong revenue visibility and supporting long-term growth objectives.\n", - "* **Strong Revenue Growth:** Substantial year-over-year revenue expansion driven by successful execution across major programs and satellite system deliveries.\n", - "* **Profitability & Margins:** Strong operational performance led to a significant increase in Adjusted EBITDA and healthy overall margin expansion.\n", - "* **Earnings Performance:** Noteworthy growth in adjusted net income and earnings per share, reflecting scaled operations and improved bottom-line efficiency.\n", - "* **Cash Flow & Capital Structure:** Positive operating cash flow and a healthy balance sheet with a very low leverage ratio, positioning the company well for future investment.\n", + "1. **Press Release** — Summary of financial results and management commentary \n", + "2. **Earnings Call Transcript** — Management presentation and analyst Q&A \n", + "\n", + "Together, these documents provide both quantitative results and qualitative context.\n", "\n", "---\n", "\n", "## Overview\n", "\n", - "This notebook demonstrates how to extract deep insights from unstructured financial documents. By combining **Docling** for high-fidelity parsing with **Semantica** for knowledge engineering, we build a structured semantic layer that enables complex analysis and advanced GraphRAG capabilities.\n", + "This notebook demonstrates an end-to-end semantic pipeline for transforming\n", + "unstructured financial documents into structured, queryable knowledge.\n", "\n", - "**Workflow:** `Dual PDF Input → Docling Parsing → Entity & Relation Extraction → Knowledge Graph Construction → GraphRAG → Strategic Q&A`" + "**Docling** is used for high-fidelity document parsing. **Semantica** performs\n", + "semantic extraction, cleaning, validation, and knowledge graph construction.\n", + "The final knowledge graph is stored in **AWS Neptune** and used for hybrid\n", + "retrieval, agent memory, and grounded question answering.\n", + "\n", + "---\n", + "\n", + "## End-to-End Workflow\n", + "\n", + "**Workflow:** \n", + "Dual PDF Input → Docling Parsing → Normalization & Chunking → Entity, Relation & Triplet Extraction → Conflict Resolution & Deduplication → Knowledge Graph Construction → Amazon Neptune → GraphRAG → Agent Memory & Context → Strategic Q&A\n", + "\n", + "---\n", + "\n", + "## Pipeline Capabilities\n", + "\n", + "- High-fidelity PDF parsing (text, tables, structure) \n", + "- Semantic extraction of entities, relationships, and triplets \n", + "- Conflict detection and resolution with confidence awareness \n", + "- Entity deduplication and canonicalization \n", + "- Knowledge graph construction and validation \n", + "- Persistent graph storage in **AWS Neptune** (IAM, OpenCypher) \n", + "- Hybrid retrieval using **GraphRAG** (vector + graph) \n", + "- Long-term agent memory and unified context management \n", + "- Grounded LLM-based question answering \n", + "- Structured export to JSON and RDF formats \n", + "\n", + "---\n", + "\n", + "## Outcome\n", + "\n", + "The output is a cleaned, deduplicated knowledge graph stored in **AWS Neptune**,\n", + "along with supporting context for hybrid retrieval and question answering.\n", + "This enables reliable financial analysis and downstream applications built\n", + "on structured, traceable knowledge." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n" + ] + } + ], "source": [ "!pip install -qU semantica docling pdfplumber groq\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✓ Groq LLM initialized: llama-3.1-8b-instant\n" + ] + } + ], "source": [ - "# Initialize Groq LLM provider\n", + "from getpass import getpass\n", "from semantica.llms import Groq\n", - "import os\n", "\n", - "GROQ_API_KEY = os.getenv(\"GROQ_API_KEY\", \"\")\n", + "GROQ_API_KEY = getpass(\"Enter your GROQ API key: \")\n", "\n", "if not GROQ_API_KEY:\n", - " try:\n", - " from google.colab import userdata\n", - " GROQ_API_KEY = userdata.get(\"GROQ_API_KEY\", \"\")\n", - " except ImportError:\n", - " pass\n", + " raise ValueError(\"GROQ API key is required\")\n", "\n", - "if not GROQ_API_KEY:\n", - " raise ValueError(\"GROQ_API_KEY not found. Please set it as an environment variable or update this cell.\")\n", - "\n", - "os.environ[\"GROQ_API_KEY\"] = GROQ_API_KEY\n", - " \n", "groq_llm = Groq(\n", " model=\"llama-3.1-8b-instant\",\n", - " api_key=GROQ_API_KEY\n", + " api_key=GROQ_API_KEY,\n", ")\n", "\n", - "print(f\"✓ Groq LLM initialized: {groq_llm.model}\")\n" + "print(f\"✓ Groq LLM initialized: {groq_llm.model}\")" ] }, { @@ -82,59 +128,73 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "ImportError", + "evalue": "DoclingParser requires the 'docling' package to be installed and working.\n\nError: [WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies.\n\nInstall it with: pip install docling", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mImportError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[3], line 5\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mpathlib\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Path\n\u001b[0;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01msemantica\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mparse\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m DoclingParser\n\u001b[1;32m----> 5\u001b[0m parser \u001b[38;5;241m=\u001b[39m \u001b[43mDoclingParser\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 7\u001b[0m PRESS_RELEASE_URL \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps://filecache.investorroom.com/mr5ircnw_mda/677/MDA_Space_Ltd_Q3_2025_Press_Release_Nov_14_2025_FINAL.pdf\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 8\u001b[0m TRANSCRIPT_URL \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps://filecache.investorroom.com/mr5ircnw_mda/681/MDA\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124m20Space\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124m20Ltd.\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124m20Q3\u001b[39m\u001b[38;5;132;01m%202025%\u001b[39;00m\u001b[38;5;124m20Earnings\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124m20Conference\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124m20Call\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124m20Transcript\u001b[39m\u001b[38;5;132;01m%20%\u001b[39;00m\u001b[38;5;124m28November\u001b[39m\u001b[38;5;132;01m%2014%\u001b[39;00m\u001b[38;5;124m202025\u001b[39m\u001b[38;5;124m%\u001b[39m\u001b[38;5;124m29.pdf\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", + "File \u001b[1;32mc:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\semantica\\parse\\__init__.py:198\u001b[0m, in \u001b[0;36mDoclingParser.__init__\u001b[1;34m(self, **config)\u001b[0m\n\u001b[0;32m 196\u001b[0m error_msg \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mError: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mimport_error_msg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 197\u001b[0m error_msg \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mInstall it with: pip install docling\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 198\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(error_msg)\n", + "\u001b[1;31mImportError\u001b[0m: DoclingParser requires the 'docling' package to be installed and working.\n\nError: [WinError 1114] A dynamic link library (DLL) initialization routine failed. Error loading \"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\torch\\lib\\c10.dll\" or one of its dependencies.\n\nInstall it with: pip install docling" + ] + } + ], "source": [ "import requests\n", "from pathlib import Path\n", "from semantica.parse import DoclingParser\n", "\n", - "# Initialize DoclingParser with default settings\n", - "# Optional: configure with export_format=\"html\" or enable_ocr=True if needed\n", "parser = DoclingParser()\n", "\n", - "# PDF URLs for MDA Space Q3 2025 earnings documents\n", - "press_release_url = \"https://filecache.investorroom.com/mr5ircnw_mda/677/MDA_Space_Ltd_Q3_2025_Press_Release_Nov_14_2025_FINAL.pdf\"\n", - "transcript_url = \"https://filecache.investorroom.com/mr5ircnw_mda/681/MDA%20Space%20Ltd.%20Q3%202025%20Earnings%20Conference%20Call%20Transcript%20%28November%2014%202025%29.pdf\"\n", + "PRESS_RELEASE_URL = \"https://filecache.investorroom.com/mr5ircnw_mda/677/MDA_Space_Ltd_Q3_2025_Press_Release_Nov_14_2025_FINAL.pdf\"\n", + "TRANSCRIPT_URL = \"https://filecache.investorroom.com/mr5ircnw_mda/681/MDA%20Space%20Ltd.%20Q3%202025%20Earnings%20Conference%20Call%20Transcript%20%28November%2014%202025%29.pdf\"\n", "\n", - "# Setup download directory\n", "download_dir = Path(\"downloads\")\n", "download_dir.mkdir(exist_ok=True)\n", "\n", "press_release_pdf = download_dir / \"mda_space_q3_2025_press_release.pdf\"\n", "transcript_pdf = download_dir / \"mda_space_q3_2025_transcript.pdf\"\n", "\n", - "# Download PDFs if they don't exist\n", "if not press_release_pdf.exists():\n", - " press_release_pdf.write_bytes(requests.get(press_release_url).content)\n", + " press_release_pdf.write_bytes(requests.get(PRESS_RELEASE_URL).content)\n", "\n", "if not transcript_pdf.exists():\n", - " transcript_pdf.write_bytes(requests.get(transcript_url).content)\n", + " transcript_pdf.write_bytes(requests.get(TRANSCRIPT_URL).content)\n", "\n", - "# Parse documents using DoclingParser\n", "try:\n", " press_release = parser.parse(press_release_pdf)\n", " transcript = parser.parse(transcript_pdf)\n", "except Exception as e:\n", - " print(f\"⚠️ Parsing failed: {e}\")\n", - " print(\"Using fallback empty documents for demonstration.\")\n", + " print(\"Parsing failed\")\n", + " print(e)\n", + " print(\"Using fallback empty documents.\")\n", " press_release = {\"full_text\": \"\", \"tables\": []}\n", " transcript = {\"full_text\": \"\", \"tables\": []}\n", "\n", - "# Combine parsed documents\n", "parsed_doc = {\n", - " \"full_text\": f\"# Press Release\\n\\n{press_release['full_text']}\\n\\n# Transcript\\n\\n{transcript['full_text']}\",\n", - " \"tables\": press_release['tables'] + transcript['tables'],\n", + " \"full_text\": (\n", + " \"# Press Release\\n\\n\"\n", + " f\"{press_release['full_text']}\\n\\n\"\n", + " \"# Transcript\\n\\n\"\n", + " f\"{transcript['full_text']}\"\n", + " ),\n", + " \"tables\": press_release[\"tables\"] + transcript[\"tables\"],\n", " \"metadata\": {\n", " \"title\": \"MDA Space Ltd. Q3 2025 Earnings Analysis\",\n", " \"company\": \"MDA Space Ltd.\",\n", " \"quarter\": \"Q3 2025\",\n", - " \"date\": \"November 14, 2025\"\n", - " }\n", + " \"date\": \"November 14, 2025\",\n", + " },\n", "}\n", "\n", - "print(f\"✓ Parsed {len(parsed_doc['tables'])} tables from {len(press_release['tables']) + len(transcript['tables'])} documents\")\n" + "print(\"Parsing completed\")\n", + "print(\"Documents processed: 2\")\n", + "print(\"Tables extracted:\", len(parsed_doc[\"tables\"]))" ] }, { @@ -152,20 +212,19 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 2: Normalize full document text (run ONCE)\n", "from semantica.normalize import TextNormalizer\n", - "# Initialize normalizer\n", + "\n", "normalizer = TextNormalizer()\n", "\n", "normalized_text = normalizer.normalize(\n", " parsed_doc[\"full_text\"],\n", " clean_html=False,\n", " remove_extra_whitespace=False,\n", - " lowercase=False # preserve casing for entities & finance terms\n", + " lowercase=False,\n", ")\n", "\n", - "print(f\"Text normalized: {len(normalized_text)} characters\")\n", - "\n" + "print(\"Normalization completed\")\n", + "print(\"Normalized text length:\", len(normalized_text))" ] }, { @@ -185,25 +244,25 @@ "outputs": [], "source": [ "from semantica.split import TextSplitter\n", + "\n", "CHUNK_SIZE = 1000\n", "CHUNK_OVERLAP = 250\n", "\n", "splitter = TextSplitter(\n", - " method=\"recursive\", # safest for long PDFs\n", + " method=\"recursive\",\n", " chunk_size=CHUNK_SIZE,\n", - " chunk_overlap=CHUNK_OVERLAP\n", + " chunk_overlap=CHUNK_OVERLAP,\n", ")\n", "\n", "chunks = splitter.split(normalized_text)\n", "\n", - "print(f\"✓ Created {len(chunks)} chunks\")\n", - "\n", - "# Version-safe access to chunk text\n", "def get_chunk_text(chunk):\n", " return getattr(chunk, \"content\", getattr(chunk, \"text\", \"\"))\n", "\n", - "# Inspect one chunk\n", - "print(\"Sample chunk:\\n\", get_chunk_text(chunks[0])[:])" + "print(\"Chunking completed\")\n", + "print(\"Total chunks:\", len(chunks))\n", + "print(\"Sample chunk:\")\n", + "print(get_chunk_text(chunks[0]))" ] }, { @@ -221,53 +280,36 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 4: Extract entities from ALL chunks using NERExtractor (Groq)\n", - "\n", - "from semantica.semantic_extract import NERExtractor\n", "import os\n", + "from semantica.semantic_extract import NERExtractor\n", "\n", - "# Initialize NER extractor\n", "ner = NERExtractor(\n", " method=\"llm\",\n", " provider=\"groq\",\n", " llm_model=\"llama-3.1-8b-instant\",\n", - " min_confidence=0.5,\n", " temperature=0.0,\n", - " api_key=os.getenv(\"GROQ_API_KEY\")\n", + " api_key=GROQ_API_KEY,\n", ")\n", "\n", - "entity_types = [\n", + "ENTITY_TYPES = [\n", " \"ORGANIZATION\", \"ORG\", \"PERSON\", \"MONEY\", \"CURRENCY\",\n", " \"PERCENT\", \"PERCENTAGE\", \"DATE\", \"TIME\", \"PRODUCT\",\n", - " \"LOCATION\", \"GPE\", \"EVENT\", \"QUANTITY\", \"CARDINAL\"\n", + " \"LOCATION\", \"GPE\", \"EVENT\", \"QUANTITY\", \"CARDINAL\",\n", "]\n", "\n", - "# Version-safe chunk text accessor\n", - "def get_chunk_text(chunk):\n", - " return getattr(chunk, \"content\", getattr(chunk, \"text\", \"\"))\n", - "\n", "all_entities = []\n", "\n", - "for i, chunk in enumerate(chunks, 1):\n", + "for chunk in chunks:\n", " text = get_chunk_text(chunk)\n", + " if text.strip():\n", + " all_entities += ner.extract_entities(text, entity_types=ENTITY_TYPES)\n", "\n", - " if not text.strip():\n", - " continue\n", + "print(\"Entity extraction completed\")\n", + "print(\"Total entities extracted:\", len(all_entities))\n", "\n", - " try:\n", - " entities = ner.extract_entities(\n", - " text,\n", - " entity_types=entity_types\n", - " )\n", - " all_entities.extend(entities)\n", - "\n", - " except Exception as e:\n", - " print(f\"⚠️ Chunk {i} failed: {e}\")\n", - "\n", - " if i % 10 == 0 or i == len(chunks):\n", - " print(f\"Processed {i}/{len(chunks)} chunks\")\n", - "\n", - "print(f\"✓ Total entities extracted: {len(all_entities)}\")\n" + "print(\"\\nSample entities\")\n", + "for e in all_entities[:10]:\n", + " print(f\"{e.label}: {e.text}\")" ] }, { @@ -285,60 +327,40 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 5: Extract financial metrics from ALL chunks (explicit + safe)\n", - "\n", - "financial_entity_types = [\n", + "FINANCIAL_ENTITY_TYPES = [\n", " \"MONEY\", \"CURRENCY\", \"PERCENT\", \"PERCENTAGE\",\n", - " \"QUANTITY\", \"CARDINAL\"\n", + " \"QUANTITY\", \"CARDINAL\",\n", "]\n", "\n", - "def get_chunk_text(chunk):\n", - " return getattr(chunk, \"content\", getattr(chunk, \"text\", \"\"))\n", - "\n", "financial_entities = []\n", "\n", - "total_chunks = len(chunks)\n", - "print(f\"Processing {total_chunks} chunks for financial entities...\")\n", - "\n", - "for idx, chunk in enumerate(chunks, start=1):\n", + "for chunk in chunks:\n", " text = get_chunk_text(chunk)\n", + " if text.strip():\n", + " financial_entities += ner.extract_entities(\n", + " text,\n", + " entity_types=FINANCIAL_ENTITY_TYPES,\n", + " )\n", "\n", - " # Skip empty chunks but still count them\n", - " if not text.strip():\n", - " print(f\" Skipping empty chunk {idx}/{total_chunks}\")\n", - " continue\n", - "\n", - " # ALWAYS run NER per chunk\n", - " entities = ner.extract_entities(\n", - " text,\n", - " entity_types=financial_entity_types\n", - " )\n", - "\n", - " financial_entities.extend(entities)\n", - "\n", - " if idx % 10 == 0 or idx == total_chunks:\n", - " print(f\" Processed {idx}/{total_chunks} chunks\")\n", - "\n", - "# Aggregate results\n", - "financial_metrics = {\"money\": [], \"percentages\": [], \"quantities\": []}\n", + "money, percentages, quantities = [], [], []\n", "\n", "for e in financial_entities:\n", - " label = e.label.lower()\n", - " if \"money\" in label or \"currency\" in label:\n", - " financial_metrics[\"money\"].append(e.text)\n", - " elif \"percent\" in label:\n", - " financial_metrics[\"percentages\"].append(e.text)\n", - " elif \"quantity\" in label or \"cardinal\" in label:\n", - " financial_metrics[\"quantities\"].append(e.text)\n", + " label = e.label.upper()\n", + " if label in (\"MONEY\", \"CURRENCY\"):\n", + " money.append(e.text)\n", + " elif label in (\"PERCENT\", \"PERCENTAGE\"):\n", + " percentages.append(e.text)\n", + " elif label in (\"CARDINAL\", \"QUANTITY\"):\n", + " quantities.append(e.text)\n", "\n", - "print(f\"✓ Financial entity mentions extracted: {len(financial_entities)}\")\n", - "print(f\" Money/Currency: {len(financial_metrics['money'])}\")\n", - "print(f\" Percentages: {len(financial_metrics['percentages'])}\")\n", - "print(f\" Quantities: {len(financial_metrics['quantities'])}\")\n", + "print(\"\\nFinancial entity extraction completed\")\n", + "print(\"Total financial entities:\", len(financial_entities))\n", + "print(\"Money:\", len(money))\n", + "print(\"Percentages:\", len(percentages))\n", + "print(\"Quantities:\", len(quantities))\n", "\n", "if financial_entities:\n", - " sample = financial_entities[0]\n", - " print(f\" Sample: {sample.text} ({sample.label})\")\n" + " print(\"Sample:\", f\"{financial_entities[0].text} ({financial_entities[0].label})\")" ] }, { @@ -356,10 +378,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 6: Extract relationships from all chunks (minimal handling)\n", - "\n", "from semantica.semantic_extract import RelationExtractor\n", - "import os\n", "\n", "relation_extractor = RelationExtractor(\n", " method=\"llm\",\n", @@ -370,36 +389,37 @@ " \"OPERATES_IN\", \"LOCATED_IN\", \"PARTNERS_WITH\", \"SERVES\",\n", " \"COMPARED_TO\", \"INCREASED_BY\", \"DECREASED_BY\", \"CHANGED_BY\",\n", " \"DURING\", \"IN_QUARTER\", \"FOR_PERIOD\",\n", - " \"RELATED_TO\", \"PART_OF\", \"AFFECTS\"\n", + " \"RELATED_TO\", \"PART_OF\", \"AFFECTS\",\n", " ],\n", - " api_key=os.getenv(\"GROQ_API_KEY\")\n", + " api_key=GROQ_API_KEY,\n", ")\n", "\n", "def get_chunk_text(chunk):\n", - " return getattr(chunk, \"content\", getattr(chunk, \"text\", \"\"))\n", + " return getattr(chunk, \"content\", getattr(chunk, \"text\", \"\")) or \"\"\n", "\n", "relationships = []\n", "\n", "for chunk in chunks:\n", " text = get_chunk_text(chunk)\n", - " if not text.strip():\n", - " continue\n", "\n", - " relationships.extend(\n", - " relation_extractor.extract_relations(\n", - " text,\n", - " entities=all_entities,\n", - " provider=\"groq\",\n", - " llm_model=\"llama-3.1-8b-instant\",\n", - " temperature=0.0\n", - " )\n", + " relations = relation_extractor.extract_relations(\n", + " text,\n", + " entities=all_entities,\n", + " provider=\"groq\",\n", + " llm_model=\"llama-3.1-8b-instant\",\n", + " temperature=0.0,\n", " )\n", "\n", - "print(f\"✓ Relationships extracted: {len(relationships)}\")\n", + " relationships += relations\n", + "\n", + "print(\"Relationship extraction completed\")\n", + "print(\"Total chunks:\", len(chunks))\n", + "print(\"Total relationships extracted:\", len(relationships))\n", "\n", "if relationships:\n", - " sample = relationships[0]\n", - " print(f\"Sample: {sample.subject.text} → {sample.predicate} → {sample.object.text}\")\n" + " r = relationships[0]\n", + " print(\"Sample relationship:\")\n", + " print(f\"{r.subject.text} → {r.predicate} → {r.object.text}\")" ] }, { @@ -417,10 +437,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 7: Extract RDF triplets from all chunks (minimal handling)\n", - "\n", "from semantica.semantic_extract import TripletExtractor\n", - "import os\n", "\n", "triplet_extractor = TripletExtractor(\n", " method=\"llm\",\n", @@ -429,39 +446,34 @@ " provider=\"groq\",\n", " llm_model=\"llama-3.1-8b-instant\",\n", " temperature=0.0,\n", - " api_key=os.getenv(\"GROQ_API_KEY\")\n", + " api_key=GROQ_API_KEY,\n", ")\n", "\n", "def get_chunk_text(chunk):\n", - " return getattr(chunk, \"content\", getattr(chunk, \"text\", \"\"))\n", + " return getattr(chunk, \"content\", getattr(chunk, \"text\", \"\")) or \"\"\n", "\n", "triplets = []\n", "\n", "for chunk in chunks:\n", " text = get_chunk_text(chunk)\n", - " if not text.strip():\n", - " continue\n", "\n", - " triplets.extend(\n", - " triplet_extractor.extract_triplets(\n", - " text,\n", - " entities=all_entities,\n", - " relations=relationships if relationships else None\n", - " )\n", + " triplets += triplet_extractor.extract_triplets(\n", + " text,\n", + " entities=all_entities,\n", + " relations=relationships if relationships else None,\n", " )\n", "\n", - "# Optional validation (if available)\n", - "validated_triplets = (\n", - " triplet_extractor.validate_triplets(triplets)\n", - " if hasattr(triplet_extractor, \"validate_triplets\")\n", - " else triplets\n", - ")\n", + "if hasattr(triplet_extractor, \"validate_triplets\"):\n", + " triplets = triplet_extractor.validate_triplets(triplets)\n", "\n", - "print(f\"✓ RDF triplets extracted: {len(validated_triplets)}\")\n", + "print(\"Triplet extraction completed\")\n", + "print(\"Total chunks:\", len(chunks))\n", + "print(\"Total RDF triplets:\", len(triplets))\n", "\n", - "if validated_triplets:\n", - " t = validated_triplets[0]\n", - " print(f\"Sample: {t.subject} → {t.predicate} → {t.object}\")" + "if triplets:\n", + " t = triplets[0]\n", + " print(\"Sample triplet:\")\n", + " print(f\"{t.subject} → {t.predicate} → {t.object}\")\n" ] }, { @@ -479,42 +491,41 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 7: Detect conflicts in extracted entities and relationships\n", "from semantica.conflicts import ConflictDetector, SourceTracker, SourceReference\n", "\n", "source_tracker = SourceTracker()\n", + "\n", "conflict_detector = ConflictDetector(\n", " source_tracker=source_tracker,\n", - " confidence_threshold=0.7\n", + " confidence_threshold=0.7,\n", ")\n", "\n", - "# Track sources for entities\n", "for entity in all_entities:\n", - " entity_id = getattr(entity, 'id', None) or getattr(entity, 'text', '')\n", - " entity_name = getattr(entity, 'text', '')\n", + " entity_id = getattr(entity, \"id\", None) or getattr(entity, \"text\", \"\")\n", + " entity_text = getattr(entity, \"text\", \"\")\n", + " entity_label = getattr(entity, \"label\", \"UNKNOWN\")\n", + "\n", " source_tracker.track_property_source(\n", " entity_id,\n", - " 'name',\n", - " entity_name,\n", + " \"name\",\n", + " entity_text,\n", " source=SourceReference(\n", - " source='earnings_call',\n", - " timestamp='2024-Q1',\n", - " metadata={'entity_type': getattr(entity, 'label', 'UNKNOWN')}\n", - " )\n", + " source=\"earnings_call\",\n", + " timestamp=\"2024-Q1\",\n", + " metadata={\"entity_type\": entity_label},\n", + " ),\n", " )\n", "\n", - "# Detect value conflicts\n", "value_conflicts = conflict_detector.detect_value_conflicts(\n", - " [{'id': getattr(e, 'id', ''), 'name': getattr(e, 'text', '')} for e in all_entities],\n", - " property_name='name'\n", + " [{\"id\": getattr(e, \"id\", \"\"), \"name\": getattr(e, \"text\", \"\")} for e in all_entities],\n", + " property_name=\"name\",\n", ")\n", "\n", - "# Detect relationship conflicts\n", "relationship_conflicts = conflict_detector.detect_relationship_conflicts(relationships)\n", "\n", - "print(f\"✓ Conflicts detected\")\n", - "print(f\" Value conflicts: {len(value_conflicts)}\")\n", - "print(f\" Relationship conflicts: {len(relationship_conflicts)}\")\n" + "print(\"Conflict detection completed\")\n", + "print(\"Value conflicts:\", len(value_conflicts))\n", + "print(\"Relationship conflicts:\", len(relationship_conflicts))\n" ] }, { @@ -532,28 +543,27 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 8: Resolve conflicts using ConflictResolver\n", "from semantica.conflicts import ConflictResolver\n", "\n", "conflict_resolver = ConflictResolver(\n", - " default_strategy='voting',\n", - " source_tracker=source_tracker\n", + " default_strategy=\"voting\",\n", + " source_tracker=source_tracker,\n", ")\n", "\n", - "# Resolve value conflicts\n", - "resolved_entities = list(all_entities)\n", "resolved_conflicts = []\n", + "\n", "for conflict in value_conflicts:\n", - " resolution = conflict_resolver.resolve_conflict(conflict, strategy='voting')\n", - " resolved_conflicts.append(resolution)\n", + " resolved_conflicts.append(\n", + " conflict_resolver.resolve_conflict(conflict, strategy=\"voting\")\n", + " )\n", "\n", - "# Resolve relationship conflicts\n", - "resolved_relationships = list(relationships)\n", "for conflict in relationship_conflicts:\n", - " resolution = conflict_resolver.resolve_conflict(conflict, strategy='voting')\n", - " resolved_conflicts.append(resolution)\n", + " resolved_conflicts.append(\n", + " conflict_resolver.resolve_conflict(conflict, strategy=\"voting\")\n", + " )\n", "\n", - "print(f\"✓ Conflicts resolved: {len(resolved_conflicts)}\")\n" + "print(\"Conflict resolution completed\")\n", + "print(\"Total conflicts resolved:\", len(resolved_conflicts))\n" ] }, { @@ -571,41 +581,39 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 9: Deduplicate entities using DuplicateDetector and EntityMerger\n", "from semantica.deduplication import DuplicateDetector, EntityMerger\n", "\n", "duplicate_detector = DuplicateDetector(\n", " similarity_threshold=0.8,\n", - " confidence_threshold=0.7\n", + " confidence_threshold=0.7,\n", ")\n", "\n", - "# Convert entities to dict format\n", - "entity_dicts = []\n", - "for entity in resolved_entities:\n", - " entity_dicts.append({\n", - " 'id': getattr(entity, 'id', ''),\n", - " 'name': getattr(entity, 'text', ''),\n", - " 'type': getattr(entity, 'label', 'UNKNOWN'),\n", - " 'confidence': getattr(entity, 'confidence', 1.0),\n", - " 'metadata': getattr(entity, 'metadata', {})\n", - " })\n", + "entity_dicts = [\n", + " {\n", + " \"id\": getattr(e, \"id\", \"\"),\n", + " \"name\": getattr(e, \"text\", \"\"),\n", + " \"type\": getattr(e, \"label\", \"UNKNOWN\"),\n", + " \"confidence\": getattr(e, \"confidence\", 1.0),\n", + " \"metadata\": getattr(e, \"metadata\", {}),\n", + " }\n", + " for e in resolved_entities\n", + "]\n", "\n", - "# Detect duplicates\n", "duplicates = duplicate_detector.detect_duplicates(entity_dicts)\n", "\n", - "# Merge duplicates\n", "entity_merger = EntityMerger(preserve_provenance=True)\n", + "\n", "merge_operations = entity_merger.merge_duplicates(\n", " entity_dicts,\n", - " strategy='keep_most_complete'\n", + " strategy=\"keep_most_complete\",\n", ")\n", "\n", "merged_entities = [op.merged_entity for op in merge_operations]\n", "\n", - "print(f\"✓ Deduplication complete\")\n", - "print(f\" Original entities: {len(entity_dicts)}\")\n", - "print(f\" Merged entities: {len(merged_entities)}\")\n", - "print(f\" Duplicates removed: {len(entity_dicts) - len(merged_entities)}\")\n" + "print(\"Entity deduplication completed\")\n", + "print(\"Original entities:\", len(entity_dicts))\n", + "print(\"Merged entities:\", len(merged_entities))\n", + "print(\"Duplicates removed:\", len(entity_dicts) - len(merged_entities))\n" ] }, { @@ -623,46 +631,44 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 10: Build knowledge graph from cleaned entities, resolved relationships, and triplets\n", "from semantica.kg import GraphBuilder\n", "\n", "graph_builder = GraphBuilder(\n", " merge_entities=True,\n", - " entity_resolution_strategy=\"fuzzy\"\n", + " entity_resolution_strategy=\"fuzzy\",\n", ")\n", "\n", - "# Convert triplets to relationships format\n", - "triplet_relationships = []\n", - "for triplet in validated_triplets:\n", - " triplet_relationships.append({\n", - " \"source\": triplet.subject,\n", - " \"predicate\": triplet.predicate,\n", - " \"target\": triplet.object,\n", - " \"confidence\": triplet.confidence,\n", - " \"metadata\": triplet.metadata\n", - " })\n", + "triplet_relationships = [\n", + " {\n", + " \"source\": t.subject,\n", + " \"predicate\": t.predicate,\n", + " \"target\": t.object,\n", + " \"confidence\": t.confidence,\n", + " \"metadata\": t.metadata,\n", + " }\n", + " for t in validated_triplets\n", + "]\n", "\n", - "all_relationships = resolved_relationships + triplet_relationships\n", + "final_relationships = resolved_relationships + triplet_relationships\n", "\n", "kg_data = {\n", " \"entities\": merged_entities,\n", - " \"relationships\": all_relationships,\n", + " \"relationships\": final_relationships,\n", " \"triplets\": validated_triplets,\n", " \"metadata\": {\n", " \"source\": \"earnings_call_transcript\",\n", - " \"financial_metrics\": financial_metrics,\n", - " \"extraction_method\": \"Groq LLM\"\n", - " }\n", + " \"extraction_method\": \"Groq LLM\",\n", + " },\n", "}\n", "\n", "knowledge_graph = graph_builder.build(\n", " sources=[kg_data],\n", - " merge_entities=True\n", + " merge_entities=True,\n", ")\n", "\n", - "print(f\"✓ Knowledge graph built\")\n", - "print(f\" Entities: {len(knowledge_graph.get('entities', []))}\")\n", - "print(f\" Relationships: {len(knowledge_graph.get('relationships', []))}\")\n" + "print(\"Knowledge graph build completed\")\n", + "print(\"Final entities:\", len(knowledge_graph.get(\"entities\", [])))\n", + "print(\"Final relationships:\", len(knowledge_graph.get(\"relationships\", [])))\n" ] }, { @@ -671,7 +677,18 @@ "source": [ "## Step 11: Analyze Knowledge Graph\n", "\n", - "Analyze graph structure using GraphAnalyzer (centrality, communities, connectivity).\n" + "This step evaluates the structure and quality of the knowledge graph.\n", + "\n", + "- **Centrality** \n", + " Identifies the most influential entities based on connectivity.\n", + "\n", + "- **Communities** \n", + " Groups related entities to reveal themes such as business units, markets, or topics.\n", + "\n", + "- **Connectivity** \n", + " Shows how well the graph is linked and whether information is fragmented.\n", + "\n", + "These metrics help validate extraction quality and guide downstream analysis.\n" ] }, { @@ -680,34 +697,72 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 11: Analyze knowledge graph using GraphAnalyzer\n", "from semantica.kg import GraphAnalyzer\n", "\n", "graph_analyzer = GraphAnalyzer()\n", + "\n", "analysis = graph_analyzer.analyze_graph(knowledge_graph)\n", - "centrality = graph_analyzer.calculate_centrality(knowledge_graph, 'degree')\n", - "communities = graph_analyzer.detect_communities(knowledge_graph, algorithm='louvain')\n", + "centrality = graph_analyzer.calculate_centrality(\n", + " knowledge_graph,\n", + " method=\"degree\",\n", + ")\n", + "communities = graph_analyzer.detect_communities(\n", + " knowledge_graph,\n", + " algorithm=\"louvain\",\n", + ")\n", "connectivity = graph_analyzer.analyze_connectivity(knowledge_graph)\n", "metrics = graph_analyzer.compute_metrics(knowledge_graph)\n", "\n", - "top_entities = []\n", - "if centrality and 'rankings' in centrality:\n", - " top_entities = centrality['rankings'][:5]\n", + "top_entities = centrality.get(\"rankings\", [])[:5]\n", + "num_communities = len(communities.get(\"communities\", []))\n", "\n", - "num_communities = len(communities.get('communities', [])) if isinstance(communities, dict) else 0\n", - "\n", - "print(f\"✓ Graph analysis complete\")\n", - "print(f\" Communities: {num_communities}\")\n", - "print(f\" Top entities: {len(top_entities)}\")\n" + "print(\"Graph analysis completed\")\n", + "print(\"Communities:\", num_communities)\n", + "print(\"Top entities:\", len(top_entities))\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 12: Build Context Graph\n", + "## Step 12: Persist Knowledge Graph in Amazon Neptune\n", "\n", - "Build ContextGraph from knowledge graph for enhanced retrieval and GraphRAG.\n" + "After cleaning, conflict resolution, and deduplication, the final step is to\n", + "persist the **canonical knowledge graph** into a production graph database.\n", + "\n", + "Semantica integrates with **Amazon Neptune** to provide a secure, scalable,\n", + "and query-efficient backend for long-lived knowledge graphs.\n", + "\n", + "- **Canonical Storage** \n", + " Only deduplicated entities and resolved relationships are written to Neptune.\n", + "\n", + "- **Secure Access** \n", + " Uses AWS IAM authentication (SigV4) for production-grade security.\n", + "\n", + "- **Flexible Graph Model** \n", + " Supports property graphs (OpenCypher / Gremlin) and RDF (SPARQL).\n", + "\n", + "- **Efficient Querying** \n", + " Leverages the Bolt protocol for low-latency graph queries and traversal.\n", + "\n", + "- **Production Ready** \n", + " Designed for compliance, provenance, and downstream analytics.\n", + "\n", + "This step enables durable storage, rich querying, and integration with\n", + "analytics and applications on top of the extracted knowledge graph.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ[\"NEPTUNE_ENDPOINT\"] = \"your-cluster.us-east-1.neptune.amazonaws.com\"\n", + "os.environ[\"NEPTUNE_PORT\"] = \"8182\"\n", + "os.environ[\"AWS_REGION\"] = \"us-east-1\"" ] }, { @@ -716,42 +771,69 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 12: Build context graph for enhanced retrieval\n", - "from semantica.context import ContextGraph\n", + "from semantica.graph_store import GraphStore\n", + "import os\n", "\n", - "context_graph = ContextGraph(\n", - " extract_entities=True,\n", - " extract_relationships=True\n", + "neptune_store = GraphStore(\n", + " backend=\"neptune\",\n", + " endpoint=os.environ[\"NEPTUNE_ENDPOINT\"],\n", + " port=int(os.environ.get(\"NEPTUNE_PORT\", 8182)),\n", + " region=os.environ[\"AWS_REGION\"],\n", + " iam_auth=True,\n", ")\n", "\n", - "# Convert knowledge graph to context graph format\n", - "nodes = []\n", - "for entity in knowledge_graph.get('entities', []):\n", - " nodes.append({\n", - " \"id\": entity.get('id', entity.get('name', '')),\n", - " \"type\": entity.get('type', 'entity'),\n", - " \"properties\": {\n", - " \"content\": entity.get('name', ''),\n", - " \"confidence\": entity.get('confidence', 1.0),\n", - " **entity.get('metadata', {})\n", - " }\n", - " })\n", + "neptune_store.connect()\n", + "print(\"Connected to AWS Neptune\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for entity in knowledge_graph.get(\"entities\", []):\n", + " neptune_store.create_node(\n", + " labels=[entity.get(\"type\", \"Entity\")],\n", + " properties=entity,\n", + " )\n", "\n", - "edges = []\n", - "for rel in knowledge_graph.get('relationships', []):\n", - " edges.append({\n", - " \"source_id\": rel.get('source', ''),\n", - " \"target_id\": rel.get('target', ''),\n", - " \"type\": rel.get('predicate', 'related_to'),\n", - " \"weight\": rel.get('confidence', 1.0)\n", - " })\n", + "for rel in knowledge_graph.get(\"relationships\", []):\n", + " neptune_store.create_relationship(\n", + " start_node_id=rel[\"source\"],\n", + " end_node_id=rel[\"target\"],\n", + " rel_type=rel[\"predicate\"],\n", + " properties=rel.get(\"metadata\", {}),\n", + " )\n", "\n", - "node_count = context_graph.add_nodes(nodes)\n", - "edge_count = context_graph.add_edges(edges)\n", + "print(\"Knowledge graph populated to AWS Neptune\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Verify data in Neptune\n", + "results = neptune_store.execute_query(\n", + " \"MATCH (n) RETURN count(n) AS node_count\"\n", + ")\n", + "print(\"Total nodes:\", results.get(\"records\", [{}])[0].get(\"node_count\"))\n", "\n", - "print(f\"✓ Context graph built\")\n", - "print(f\" Nodes: {node_count}\")\n", - "print(f\" Edges: {edge_count}\")\n" + "results = neptune_store.execute_query(\n", + " \"MATCH ()-[r]->() RETURN count(r) AS rel_count\"\n", + ")\n", + "print(\"Total relationships:\", results.get(\"records\", [{}])[0].get(\"rel_count\"))\n", + "\n", + "# Sample query: list a few entities\n", + "results = neptune_store.execute_query(\n", + " \"MATCH (n) RETURN labels(n), n.name LIMIT 5\"\n", + ")\n", + "\n", + "print(\"Sample nodes:\")\n", + "for r in results.get(\"records\", []):\n", + " print(r)\n" ] }, { @@ -769,108 +851,68 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 13: Set up hybrid context retrieval and demonstrate GraphRAG\n", "from semantica.vector_store import VectorStore\n", "from semantica.context import ContextRetriever\n", "\n", - "# Initialize VectorStore\n", "vector_store = VectorStore(backend=\"faiss\")\n", + "\n", "vector_store.add(\n", " texts=[parsed_doc[\"full_text\"]],\n", - " metadata=[{\"source\": \"earnings_call\", \"type\": \"transcript\"}]\n", + " metadata=[{\"source\": \"earnings_call\", \"type\": \"transcript\"}],\n", ")\n", "\n", - "# Initialize ContextRetriever\n", "context_retriever = ContextRetriever(\n", - " knowledge_graph=context_graph,\n", + " knowledge_graph=knowledge_graph,\n", " vector_store=vector_store,\n", " hybrid_alpha=0.6,\n", " use_graph_expansion=True,\n", - " max_expansion_hops=2\n", + " max_expansion_hops=2,\n", ")\n", "\n", - "# Retrieve context for financial queries\n", - "financial_queries = [\n", + "queries = [\n", " \"What was the company's revenue guidance?\",\n", - " \"What were the key financial metrics discussed?\"\n", + " \"What were the key financial metrics discussed?\",\n", "]\n", "\n", "retrieved_contexts = []\n", - "for query in financial_queries:\n", + "\n", + "for query in queries:\n", " results = context_retriever.retrieve(\n", " query=query,\n", " max_results=3,\n", - " min_relevance_score=0.2\n", + " min_relevance_score=0.2,\n", " )\n", - " retrieved_contexts.append({\n", - " \"query\": query,\n", - " \"results\": results,\n", - " \"count\": len(results)\n", - " })\n", + " retrieved_contexts.append(results)\n", "\n", - "print(f\"✓ Hybrid retrieval configured\")\n", - "print(f\" Queries processed: {len(retrieved_contexts)}\")\n" + "print(\"Hybrid GraphRAG configured\")\n", + "print(\"Queries processed:\", len(queries))\n", + "print(\"Sample results:\", len(retrieved_contexts[0]) if retrieved_contexts else 0)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 14: Entity Linking\n", + "## Step 14: Agent Memory (Long-Term Context)\n", "\n", - "Link entities across sources and assign URIs using EntityLinker.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 14: Link entities using EntityLinker\n", - "from semantica.context import EntityLinker\n", + "This step enables long-term memory for agents by storing important facts,\n", + "metrics, and entities extracted from the knowledge graph.\n", "\n", - "entity_linker = EntityLinker(knowledge_graph=knowledge_graph)\n", + "- **Semantic Memory Storage** \n", + " Stores structured memories enriched with entities and relationships,\n", + " not just raw text.\n", "\n", - "# Assign URIs to key entities\n", - "linked_entities = []\n", - "for entity in merged_entities[:10]:\n", - " entity_id = entity.get('id', entity.get('name', ''))\n", - " entity_name = entity.get('name', '')\n", - " entity_type = entity.get('type', 'UNKNOWN')\n", - " \n", - " uri = entity_linker.assign_uri(\n", - " entity_id=entity_id,\n", - " text=entity_name,\n", - " entity_type=entity_type\n", - " )\n", - " linked_entities.append({\n", - " \"entity_id\": entity_id,\n", - " \"name\": entity_name,\n", - " \"uri\": uri,\n", - " \"type\": entity_type\n", - " })\n", + "- **Hybrid Recall** \n", + " Combines vector similarity with graph structure for accurate retrieval.\n", "\n", - "# Build entity web\n", - "entity_web = entity_linker.build_entity_web()\n", + "- **Time-Bound Retention** \n", + " Supports memory expiration policies for freshness and governance.\n", "\n", - "print(f\"✓ Entity linking complete\")\n", - "print(f\" Entities linked: {len(linked_entities)}\")\n", - "print(f\" Entity web nodes: {len(entity_web.get('nodes', []))}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 15: Agent Memory\n", + "- **Agent-Ready Context** \n", + " Allows agents to recall prior earnings, metrics, and entities across sessions.\n", "\n", - "Store and retrieve memories using AgentMemory with RAG integration.\n" + "Agent Memory turns one-off analysis into **persistent, reusable intelligence**\n", + "for downstream agents and workflows.\n" ] }, { @@ -879,52 +921,150 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 15: Store and retrieve memories using AgentMemory\n", "from semantica.context import AgentMemory\n", "\n", "agent_memory = AgentMemory(\n", " vector_store=vector_store,\n", " knowledge_graph=knowledge_graph,\n", - " retention_days=30\n", + " retention_days=30,\n", ")\n", "\n", - "# Store earnings call memories\n", - "memory_ids = []\n", "memory_contents = [\n", - " f\"Earnings call transcript: {parsed_doc['metadata'].get('title', 'Q1 2024')}\",\n", - " f\"Financial metrics extracted: {sum(len(v) for v in financial_metrics.values())} metrics\",\n", - " f\"Key entities identified: {len(merged_entities)} entities\"\n", + " f\"Earnings call transcript: {parsed_doc['metadata'].get('title', 'Earnings Call')}\",\n", + " f\"Financial metrics extracted: {sum(len(v) for v in financial_metrics.values())}\",\n", + " f\"Key entities identified: {len(merged_entities)}\",\n", "]\n", "\n", + "memory_ids = []\n", + "\n", "for content in memory_contents:\n", - " memory_id = agent_memory.store(\n", - " content=content,\n", - " metadata={\"source\": \"earnings_call\", \"type\": \"transcript_analysis\"},\n", - " extract_entities=True,\n", - " extract_relationships=True\n", + " memory_ids.append(\n", + " agent_memory.store(\n", + " content=content,\n", + " metadata={\"source\": \"earnings_call\", \"type\": \"analysis\"},\n", + " extract_entities=True,\n", + " extract_relationships=True,\n", + " )\n", " )\n", - " memory_ids.append(memory_id)\n", "\n", - "# Retrieve memories\n", "financial_memories = agent_memory.retrieve(\n", " query=\"financial metrics and earnings\",\n", - " max_results=5\n", + " max_results=5,\n", ")\n", "\n", "memory_stats = agent_memory.get_statistics()\n", "\n", - "print(f\"✓ Agent memory configured\")\n", - "print(f\" Memories stored: {len(memory_ids)}\")\n", - "print(f\" Total memories: {memory_stats.get('total_memories', 0)}\")\n" + "print(\"Agent memory configured\")\n", + "print(\"Memories stored:\", len(memory_ids))\n", + "print(\"Total memories:\", memory_stats.get(\"total_memories\", 0))\n", + "print(\"Retrieved memories:\", len(financial_memories))\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 16: Agent Context\n", + "## Step 15: Agent Context\n", "\n", - "Unified context management with AgentContext (auto-detects RAG vs GraphRAG).\n" + "**AgentContext** provides a unified context layer that combines **vector-based RAG**\n", + "with **graph-based GraphRAG** for grounded and explainable retrieval.\n", + "\n", + "### Key Controls\n", + "- **Graph Expansion** \n", + " Uses connected entities and relationships from the knowledge graph to\n", + " expand context beyond direct text matches.\n", + "\n", + "- **Hybrid Alpha** \n", + " Balances text similarity with graph structure. Lower values favor text;\n", + " higher values favor graph reasoning.\n", + "\n", + "- **Expansion Hops** \n", + " Limits how far context can expand through the graph. Fewer hops keep\n", + " results focused; more hops increase coverage.\n", + "\n", + "AgentContext enables agents to reason over both **documents** and\n", + "**knowledge graphs** through a single interface.\n", + "\n", + "\n", + "### AgentContext Parameters\n", + "\n", + "- **vector_store** – Vector search over unstructured text \n", + "- **knowledge_graph** – Structured entities and relationships \n", + "- **use_graph_expansion** – Enable GraphRAG (graph-based context expansion) \n", + "- **max_expansion_hops** – How far to traverse the graph \n", + "- **hybrid_alpha** – Balance between vector and graph relevance \n", + "- **retention_days** – How long context is kept \n", + "\n", + "**Store options**\n", + "- **link_entities** – Link to existing graph nodes \n", + "\n", + "**Retrieve options**\n", + "- **max_results** – Number of results returned \n", + "- **expand_graph** – Expand context via the graph \n", + "- **include_entities** – Return related entities \n", + "\n", + "AgentContext unifies **memory, GraphRAG, and retrieval** in one interface.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'vector_store' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[4], line 4\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01msemantica\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcontext\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m AgentContext\n\u001b[0;32m 3\u001b[0m agent_context \u001b[38;5;241m=\u001b[39m AgentContext(\n\u001b[1;32m----> 4\u001b[0m vector_store\u001b[38;5;241m=\u001b[39m\u001b[43mvector_store\u001b[49m,\n\u001b[0;32m 5\u001b[0m knowledge_graph\u001b[38;5;241m=\u001b[39mknowledge_graph,\n\u001b[0;32m 6\u001b[0m use_graph_expansion\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 7\u001b[0m max_expansion_hops\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m,\n\u001b[0;32m 8\u001b[0m hybrid_alpha\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.6\u001b[39m,\n\u001b[0;32m 9\u001b[0m retention_days\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m30\u001b[39m,\n\u001b[0;32m 10\u001b[0m )\n\u001b[0;32m 12\u001b[0m memory_id \u001b[38;5;241m=\u001b[39m agent_context\u001b[38;5;241m.\u001b[39mstore(\n\u001b[0;32m 13\u001b[0m content\u001b[38;5;241m=\u001b[39mparsed_doc[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfull_text\u001b[39m\u001b[38;5;124m\"\u001b[39m][:\u001b[38;5;241m1000\u001b[39m],\n\u001b[0;32m 14\u001b[0m metadata\u001b[38;5;241m=\u001b[39m{\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mearnings_call\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdate\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2024-Q1\u001b[39m\u001b[38;5;124m\"\u001b[39m},\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 17\u001b[0m link_entities\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 18\u001b[0m )\n\u001b[0;32m 20\u001b[0m results \u001b[38;5;241m=\u001b[39m agent_context\u001b[38;5;241m.\u001b[39mretrieve(\n\u001b[0;32m 21\u001b[0m query\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mWhat was discussed about revenue growth?\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 22\u001b[0m max_results\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m5\u001b[39m,\n\u001b[0;32m 23\u001b[0m expand_graph\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 24\u001b[0m include_entities\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 25\u001b[0m )\n", + "\u001b[1;31mNameError\u001b[0m: name 'vector_store' is not defined" + ] + } + ], + "source": [ + "from semantica.context import AgentContext\n", + "\n", + "agent_context = AgentContext(\n", + " vector_store=vector_store,\n", + " knowledge_graph=knowledge_graph,\n", + " use_graph_expansion=True,\n", + " max_expansion_hops=2,\n", + " hybrid_alpha=0.6,\n", + " retention_days=30,\n", + ")\n", + "\n", + "memory_id = agent_context.store(\n", + " content=parsed_doc[\"full_text\"][:1000],\n", + " metadata={\"source\": \"earnings_call\", \"date\": \"2024-Q1\"},\n", + " extract_entities=True,\n", + " extract_relationships=True,\n", + " link_entities=True,\n", + ")\n", + "\n", + "results = agent_context.retrieve(\n", + " query=\"What was discussed about revenue growth?\",\n", + " max_results=5,\n", + " expand_graph=True,\n", + " include_entities=True,\n", + ")\n", + "\n", + "stats = agent_context.stats()\n", + "\n", + "print(\"AgentContext configured\")\n", + "print(\"Memory stored:\", memory_id)\n", + "print(\"GraphRAG results:\", len(results))\n", + "print(\"Total memories:\", stats.get(\"total_memories\", 0))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 16: Answer Generation\n", + "\n", + "Generate answers to financial questions using Groq LLM with retrieved context and knowledge graph.\n" ] }, { @@ -933,57 +1073,66 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 16: High-level context management with AgentContext\n", - "from semantica.context import AgentContext\n", + "financial_questions = [\n", + " \"What were the key financial metrics discussed?\",\n", + " \"What guidance was provided for future quarters?\",\n", + "]\n", "\n", - "agent_context = AgentContext(\n", - " vector_store=vector_store,\n", - " knowledge_graph=context_graph,\n", - " use_graph_expansion=True,\n", - " max_expansion_hops=2,\n", - " hybrid_alpha=0.6,\n", - " retention_days=30\n", - ")\n", + "generated_answers = []\n", "\n", - "# Store content with auto-extraction\n", - "memory_id = agent_context.store(\n", - " content=parsed_doc[\"full_text\"][:1000],\n", - " metadata={\"source\": \"earnings_call\", \"date\": \"2024-Q1\"},\n", - " extract_entities=True,\n", - " extract_relationships=True,\n", - " link_entities=True\n", - ")\n", + "for question in financial_questions:\n", + " retrieved_contexts = context_retriever.retrieve(\n", + " query=question,\n", + " max_results=3,\n", + " min_relevance_score=0.2,\n", + " )\n", "\n", - "# Retrieve with auto-detected GraphRAG\n", - "graphrag_results = agent_context.retrieve(\n", - " query=\"What was discussed about revenue growth?\",\n", - " max_results=5,\n", - " expand_graph=True,\n", - " include_entities=True\n", - ")\n", + " context_text = \"\\n\\n\".join(\n", + " ctx.get(\"content\", ctx.get(\"text\", \"\"))\n", + " for ctx in retrieved_contexts\n", + " )[:1000]\n", "\n", - "context_stats = agent_context.stats()\n", + " entity_names = [\n", + " entity.get(\"name\", \"\")\n", + " for entity in knowledge_graph.get(\"entities\", [])[:5]\n", + " ]\n", + " entities_text = \", \".join(entity_names) or \"N/A\"\n", "\n", - "print(f\"✓ AgentContext configured\")\n", - "print(f\" Memory stored: {memory_id}\")\n", - "print(f\" GraphRAG results: {len(graphrag_results)}\")\n", - "print(f\" Total memories: {context_stats.get('total_memories', 0)}\")\n" + " prompt = f\"\"\"\n", + "Answer the question using only the context below.\n", + "If the answer is not present, say so.\n", + "\n", + "Context:\n", + "{context_text}\n", + "\n", + "Key entities: {entities_text}\n", + "\n", + "Question:\n", + "{question}\n", + "\n", + "Answer:\n", + "\"\"\".strip()\n", + "\n", + " try:\n", + " answer = groq_llm.generate(\n", + " prompt,\n", + " temperature=0.7,\n", + " max_tokens=400,\n", + " )\n", + " except Exception as error:\n", + " answer = f\"Answer generation failed: {error}\"\n", + "\n", + " generated_answers.append(answer)\n", + "\n", + "print(\"Answer generation completed\")\n", + "print(\"Questions answered:\", len(generated_answers))\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Step 17: Answer Generation\n", - "\n", - "Generate answers to financial questions using Groq LLM with retrieved context and knowledge graph.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 18: Export Results\n", + "## Step 17: Export Results\n", "\n", "Export knowledge graph and analysis results to JSON and RDF formats.\n" ] @@ -994,169 +1143,30 @@ "metadata": {}, "outputs": [], "source": [ - "# Step 17: Generate answers using Groq LLM from semantica.llms module\n", - "financial_questions = [\n", - " \"What were the key financial metrics discussed in the earnings call?\",\n", - " \"What guidance did management provide for future quarters?\"\n", - "]\n", - "\n", - "generated_answers = []\n", - "for question in financial_questions:\n", - " # Retrieve relevant context\n", - " context_results = context_retriever.retrieve(\n", - " query=question,\n", - " max_results=3,\n", - " min_relevance_score=0.2\n", - " )\n", - " \n", - " # Build context from retrieved results\n", - " context_text = \"\\n\\n\".join([\n", - " f\"Context {i+1}: {result.get('content', result.get('text', ''))}\"\n", - " for i, result in enumerate(context_results[:3])\n", - " ])\n", - " \n", - " # Extract relevant entities\n", - " relevant_entities = [\n", - " entity.get('name', '') for entity in knowledge_graph.get('entities', [])[:10]\n", - " ]\n", - " entities_text = \", \".join(relevant_entities[:5]) if relevant_entities else \"N/A\"\n", - " \n", - " # Build prompt\n", - " prompt = f\"\"\"Based on the following earnings call transcript context and knowledge graph, answer the question.\n", - "\n", - "Context from transcript:\n", - "{context_text[:1000]}\n", - "\n", - "Key entities identified: {entities_text}\n", - "\n", - "Question: {question}\n", - "\n", - "Provide a comprehensive answer based on the context provided. If information is not available in the context, state that clearly.\n", - "\n", - "Answer:\"\"\"\n", - " \n", - " # Generate answer using Groq LLM\n", - " try:\n", - " answer = groq_llm.generate(\n", - " prompt,\n", - " temperature=0.7,\n", - " max_tokens=500\n", - " )\n", - " generated_answers.append({\n", - " \"question\": question,\n", - " \"answer\": answer,\n", - " \"context_sources\": len(context_results),\n", - " \"model\": groq_llm.model\n", - " })\n", - " except Exception as e:\n", - " generated_answers.append({\n", - " \"question\": question,\n", - " \"answer\": f\"Error generating answer: {str(e)}\",\n", - " \"context_sources\": len(context_results),\n", - " \"model\": groq_llm.model\n", - " })\n", - "\n", - "print(f\"✓ Answer generation complete using Groq LLM\")\n", - "print(f\" LLM Provider: Groq ({groq_llm.model})\")\n", - "print(f\" Questions answered: {len(generated_answers)}\")\n", - "if generated_answers:\n", - " print(f\" Sample question: '{generated_answers[0]['question']}'\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Step 18: Export structured outputs including triplets\n", "from semantica.export import JSONExporter, RDFExporter\n", "\n", "json_exporter = JSONExporter()\n", "rdf_exporter = RDFExporter()\n", "\n", - "# Export knowledge graph to JSON\n", "kg_json = json_exporter.export(knowledge_graph, format=\"json\")\n", + "kg_rdf = rdf_exporter.export_to_rdf(knowledge_graph, format=\"turtle\")\n", "\n", - "# Export knowledge graph to RDF (Turtle format)\n", - "rdf_output = rdf_exporter.export_to_rdf(knowledge_graph, format=\"turtle\")\n", - "\n", - "# Create analysis summary\n", "analysis_summary = {\n", - " \"financial_metrics\": financial_metrics,\n", - " \"extraction_stats\": {\n", - " \"entities\": len(all_entities),\n", - " \"relationships\": len(relationships),\n", - " \"triplets\": len(triplets),\n", - " \"provider\": f\"Groq LLM (semantica.llms module) - {groq_llm.model}\"\n", - " },\n", - " \"conflict_resolution\": {\n", - " \"conflicts_detected\": len(value_conflicts) + len(relationship_conflicts),\n", - " \"conflicts_resolved\": len(resolved_conflicts),\n", - " \"strategy\": \"voting\"\n", - " },\n", - " \"deduplication\": {\n", - " \"original_entities\": len(entity_dicts),\n", - " \"duplicates_detected\": len(duplicates),\n", - " \"merged_entities\": len(merged_entities),\n", - " \"strategy\": \"keep_most_complete\"\n", - " },\n", - " \"knowledge_graph\": {\n", - " \"entities\": len(knowledge_graph.get('entities', [])),\n", - " \"relationships\": len(knowledge_graph.get('relationships', []))\n", - " },\n", - " \"graph_analytics\": {\n", - " \"metrics\": metrics,\n", - " \"communities\": num_communities,\n", - " \"top_entities\": top_entities[:5] if top_entities else []\n", - " },\n", - " \"context_graph\": {\n", - " \"nodes\": len(context_graph.nodes),\n", - " \"edges\": len(context_graph.edges)\n", - " },\n", - " \"context_retrieval\": {\n", - " \"queries_processed\": len(retrieved_contexts),\n", - " \"total_results\": sum(c[\"count\"] for c in retrieved_contexts)\n", - " },\n", - " \"entity_linking\": {\n", - " \"entities_linked\": len(linked_entities),\n", - " \"entity_web_nodes\": len(entity_web.get('nodes', [])),\n", - " \"entity_web_edges\": len(entity_web.get('edges', []))\n", - " },\n", - " \"agent_memory\": {\n", - " \"memories_stored\": len(memory_ids),\n", - " \"total_memories\": memory_stats.get('total_memories', 0)\n", - " },\n", - " \"agent_context\": {\n", - " \"graphrag_results\": len(graphrag_results),\n", - " \"total_memories\": context_stats.get('total_memories', 0)\n", - " },\n", - " \"answer_generation\": {\n", - " \"questions_answered\": len(generated_answers),\n", - " \"llm_provider\": \"Groq\",\n", - " \"llm_model\": groq_llm.model,\n", - " \"answers\": [\n", - " {\n", - " \"question\": ans[\"question\"],\n", - " \"answer_length\": len(ans[\"answer\"]),\n", - " \"context_sources\": ans[\"context_sources\"]\n", - " }\n", - " for ans in generated_answers\n", - " ]\n", - " }\n", + " \"entities\": len(knowledge_graph.get(\"entities\", [])),\n", + " \"relationships\": len(knowledge_graph.get(\"relationships\", [])),\n", + " \"triplets\": len(triplets),\n", + " \"conflicts_resolved\": len(resolved_conflicts),\n", + " \"merged_entities\": len(merged_entities),\n", + " \"communities\": num_communities,\n", + " \"questions_answered\": len(generated_answers),\n", + " \"llm_model\": groq_llm.model,\n", "}\n", "\n", - "print(f\"✓ Export complete\")\n", - "print(f\" Analysis summary: {len(analysis_summary)} sections\")\n", - "print(f\" Knowledge graph (JSON): {len(kg_json) if isinstance(kg_json, dict) else 0} items\")\n", - "print(f\" RDF (Turtle): {len(rdf_output)} characters\")\n", - "print(f\" LLM answers generated: {len(generated_answers)}\")\n", - "print(f\" LLM provider: Groq ({groq_llm.model})\")\n" + "print(\"Export completed\")\n", + "print(\"KG JSON entities:\", analysis_summary[\"entities\"])\n", + "print(\"KG RDF size (chars):\", len(kg_rdf))\n", + "print(\"Questions answered:\", analysis_summary[\"questions_answered\"])\n", + "print(\"LLM model:\", analysis_summary[\"llm_model\"])\n" ] } ],