Merge pull request #212 from Hawksight-AI/docs

Optimize Vector DB Storage in Earnings Call Analysis Notebook

cookbook/use_cases/finance/03_Earnings_Call_Analysis.ipynb

@@ -915,36 +915,62 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import time\n",
     "from semantica.vector_store import VectorStore\n",
     "from semantica.context import ContextRetriever\n",
     "\n",
-    "vector_store = VectorStore(backend=\"faiss\", dimension=768)\n",
+    "if 'chunks' not in locals() or not chunks:\n",
+    "    raise ValueError(\"Chunks not found. Please run Step 3 first.\")\n",
     "\n",
-    "documents = [parsed_doc[\"full_text\"]]\n",
-    "metadata = [{\"source\": \"earnings_call\", \"type\": \"transcript\"}]\n",
+    "# Extract text content safely\n",
+    "chunk_texts = [getattr(c, \"content\", getattr(c, \"text\", \"\")) for c in chunks]\n",
+    "chunk_metadatas = [\n",
+    "    {\n",
+    "        \"source\": \"earnings_call\", \n",
+    "        \"type\": \"transcript\", \n",
+    "        \"chunk_index\": i,\n",
+    "        **(getattr(c, \"metadata\", {}) or {})\n",
+    "    }\n",
+    "    for i, c in enumerate(chunks)\n",
+    "]\n",
     "\n",
-    "vector_ids = vector_store.add_documents(\n",
-    "    documents=documents,\n",
-    "    metadata=metadata,\n",
-    "    batch_size=32,\n",
-    "    parallel=True,\n",
+    "# Initialize Vector Store (Optimized for Speed)\n",
+    "# dimension=384 matches the default fast model (BAAI/bge-small-en-v1.5)\n",
+    "vector_store = VectorStore(\n",
+    "    backend=\"faiss\", \n",
+    "    dimension=384,  \n",
+    "    max_workers=16\n",
     ")\n",
     "\n",
+    "print(f\"Storing {len(chunks)} chunks with high-performance settings...\")\n",
+    "start_time = time.time()\n",
+    "\n",
+    "# Store in large batches with parallel processing\n",
+    "vector_ids = vector_store.add_documents(\n",
+    "    documents=chunk_texts,\n",
+    "    metadata=chunk_metadatas,\n",
+    "    batch_size=128,\n",
+    "    parallel=True\n",
+    ")\n",
+    "\n",
+    "print(f\"✅ Stored in {time.time() - start_time:.2f}s\")\n",
+    "\n",
+    "# Initialize Hybrid Retriever\n",
     "context_retriever = ContextRetriever(\n",
-    "    knowledge_graph=knowledge_graph,\n",
+    "    knowledge_graph=knowledge_graph, # Assumes knowledge_graph exists\n",
     "    vector_store=vector_store,\n",
     "    hybrid_alpha=0.6,\n",
     "    use_graph_expansion=True,\n",
     "    max_expansion_hops=2,\n",
     ")\n",
     "\n",
+    "# Test Retrieval\n",
     "queries = [\n",
     "    \"What was the company's revenue guidance?\",\n",
     "    \"What were the key financial metrics discussed?\",\n",
     "]\n",
     "\n",
     "retrieved_contexts = []\n",
-    "\n",
     "for query in queries:\n",
     "    results = context_retriever.retrieve(\n",
     "        query=query,\n",
@@ -954,7 +980,6 @@
     "    retrieved_contexts.append(results)\n",
     "\n",
     "print(\"Hybrid GraphRAG configured\")\n",
-    "print(\"Queries processed:\", len(queries))\n",
     "print(\"Sample results:\", len(retrieved_contexts[0]) if retrieved_contexts else 0)"
    ]
   },