mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
fix(split): add id field to Chunk class and update provenance tracking logic in notebook
This commit is contained in:
KaifAhmad1
@@ -55,7 +55,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -97,17 +97,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\sentence_transformers\\cross_encoder\\CrossEncoder.py:13: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
|
||||
" from tqdm.autonotebook import tqdm, trange\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
@@ -231,7 +223,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -306,7 +298,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -479,13 +471,13 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div style='font-family: monospace;'><h4>🧠 Semantica - 📊 Current Progress</h4><table style='width: 100%; border-collapse: collapse;'><tr><th>Status</th><th>Action</th><th>Module</th><th>Submodule</th><th>File</th><th>Time</th></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>SemanticChunker</td><td>-</td><td>0.15s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>EntityAwareChunker</td><td>-</td><td>1.14s</td></tr><tr><td>✅</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>NERExtractor</td><td>-</td><td>0.50s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>RelationAwareChunker</td><td>-</td><td>1.55s</td></tr><tr><td>✅</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>RelationExtractor</td><td>-</td><td>0.52s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>StructuralChunker</td><td>-</td><td>0.01s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>HierarchicalChunker</td><td>-</td><td>0.00s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>SlidingWindowChunker</td><td>-</td><td>0.06s</td></tr><tr><td>❌</td><td>Semantica is splitting</td><td>✂️ split</td><td>ChunkValidator</td><td>-</td><td>0.01s</td></tr></table></div>"
|
||||
"<div style='font-family: monospace;'><h4>🧠 Semantica - 📊 Current Progress</h4><table style='width: 100%; border-collapse: collapse;'><tr><th>Status</th><th>Action</th><th>Module</th><th>Submodule</th><th>File</th><th>Time</th></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>SemanticChunker</td><td>-</td><td>0.13s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>EntityAwareChunker</td><td>-</td><td>0.98s</td></tr><tr><td>✅</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>NERExtractor</td><td>-</td><td>0.47s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>RelationAwareChunker</td><td>-</td><td>1.39s</td></tr><tr><td>✅</td><td>Semantica is extracting</td><td>🎯 semantic_extract</td><td>RelationExtractor</td><td>-</td><td>0.45s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>StructuralChunker</td><td>-</td><td>0.01s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>HierarchicalChunker</td><td>-</td><td>0.00s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>SlidingWindowChunker</td><td>-</td><td>0.00s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>TableChunker</td><td>-</td><td>0.01s</td></tr><tr><td>✅</td><td>Semantica is splitting</td><td>✂️ split</td><td>ChunkValidator</td><td>-</td><td>0.01s</td></tr></table></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
@@ -585,7 +577,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -652,7 +644,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -741,7 +733,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -851,7 +843,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -930,7 +922,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -941,18 +933,52 @@
|
||||
"\n",
|
||||
"================================================================================\n",
|
||||
"\n",
|
||||
"Window 1:\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "AttributeError",
|
||||
"evalue": "'Chunk' object has no attribute 'start'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[16], line 19\u001b[0m\n\u001b[0;32m 16\u001b[0m overlap \u001b[38;5;241m=\u001b[39m chunk\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124moverlap_chars\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;241m0\u001b[39m)\n\u001b[0;32m 18\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mWindow \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m:\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m---> 19\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m Position: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[43mchunk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstart\u001b[49m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m-\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mchunk\u001b[38;5;241m.\u001b[39mend\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 20\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m Length: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(chunk\u001b[38;5;241m.\u001b[39mtext)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m chars\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 21\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m Overlap with previous: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00moverlap\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m chars\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
|
||||
"\u001b[1;31mAttributeError\u001b[0m: 'Chunk' object has no attribute 'start'"
|
||||
"Window 1:\n",
|
||||
" Position: 0-134\n",
|
||||
" Length: 133 chars\n",
|
||||
" Overlap with previous: 0 chars\n",
|
||||
"\n",
|
||||
"Window 2:\n",
|
||||
" Position: 100-220\n",
|
||||
" Length: 120 chars\n",
|
||||
" Overlap with previous: 34 chars\n",
|
||||
"\n",
|
||||
"Window 3:\n",
|
||||
" Position: 200-279\n",
|
||||
" Length: 79 chars\n",
|
||||
" Overlap with previous: 20 chars\n",
|
||||
"\n",
|
||||
"Window 4:\n",
|
||||
" Position: 300-383\n",
|
||||
" Length: 81 chars\n",
|
||||
" Overlap with previous: 0 chars\n",
|
||||
"\n",
|
||||
"Window 5:\n",
|
||||
" Position: 400-486\n",
|
||||
" Length: 83 chars\n",
|
||||
" Overlap with previous: 0 chars\n",
|
||||
"\n",
|
||||
"Window 6:\n",
|
||||
" Position: 500-612\n",
|
||||
" Length: 112 chars\n",
|
||||
" Overlap with previous: 0 chars\n",
|
||||
"\n",
|
||||
"Window 7:\n",
|
||||
" Position: 600-718\n",
|
||||
" Length: 116 chars\n",
|
||||
" Overlap with previous: 12 chars\n",
|
||||
"\n",
|
||||
"Window 8:\n",
|
||||
" Position: 700-786\n",
|
||||
" Length: 86 chars\n",
|
||||
" Overlap with previous: 18 chars\n",
|
||||
"\n",
|
||||
"Window 9:\n",
|
||||
" Position: 800-896\n",
|
||||
" Length: 95 chars\n",
|
||||
" Overlap with previous: 0 chars\n",
|
||||
"\n",
|
||||
"================================================================================\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -961,9 +987,8 @@
|
||||
"\n",
|
||||
"# Sliding window chunking\n",
|
||||
"sliding_chunker = SlidingWindowChunker(\n",
|
||||
" window_size=150,\n",
|
||||
" step_size=100, # 50 char overlap\n",
|
||||
" min_chunk_size=50\n",
|
||||
" chunk_size=150,\n",
|
||||
" overlap=50\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"chunks = sliding_chunker.chunk(text)\n",
|
||||
@@ -972,10 +997,14 @@
|
||||
"print(\"=\" * 80)\n",
|
||||
"\n",
|
||||
"for i, chunk in enumerate(chunks, 1):\n",
|
||||
" overlap = chunk.metadata.get('overlap_chars', 0)\n",
|
||||
" # Calculate overlap manually\n",
|
||||
" overlap = 0\n",
|
||||
" if i > 1:\n",
|
||||
" prev_chunk = chunks[i-2]\n",
|
||||
" overlap = max(0, prev_chunk.end_index - chunk.start_index)\n",
|
||||
" \n",
|
||||
" print(f\"\\nWindow {i}:\")\n",
|
||||
" print(f\" Position: {chunk.start}-{chunk.end}\")\n",
|
||||
" print(f\" Position: {chunk.start_index}-{chunk.end_index}\")\n",
|
||||
" print(f\" Length: {len(chunk.text)} chars\")\n",
|
||||
" print(f\" Overlap with previous: {overlap} chars\")\n",
|
||||
"\n",
|
||||
@@ -999,18 +1028,26 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "AttributeError",
|
||||
"evalue": "'TableChunker' object has no attribute 'chunk'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[12], line 26\u001b[0m\n\u001b[0;32m 18\u001b[0m \u001b[38;5;66;03m# Table chunking\u001b[39;00m\n\u001b[0;32m 19\u001b[0m table_chunker \u001b[38;5;241m=\u001b[39m TableChunker(\n\u001b[0;32m 20\u001b[0m preserve_headers\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 21\u001b[0m max_rows_per_chunk\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m3\u001b[39m,\n\u001b[0;32m 22\u001b[0m include_context\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 23\u001b[0m table_format\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmarkdown\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 24\u001b[0m )\n\u001b[1;32m---> 26\u001b[0m chunks \u001b[38;5;241m=\u001b[39m \u001b[43mtable_chunker\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mchunk\u001b[49m(text_with_table)\n\u001b[0;32m 28\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTable Chunking Results:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 29\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m=\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m80\u001b[39m)\n",
|
||||
"\u001b[1;31mAttributeError\u001b[0m: 'TableChunker' object has no attribute 'chunk'"
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Table Chunking Results:\n",
|
||||
"\n",
|
||||
"================================================================================\n",
|
||||
"\n",
|
||||
"Chunk 1:\n",
|
||||
" Type: Text\n",
|
||||
" Content: Product | Category | Release Year\n",
|
||||
"--- | --- | ---\n",
|
||||
"iPhone | Smartphone | 2007\n",
|
||||
"iPad | Tablet | 2010\n",
|
||||
"Ma...\n",
|
||||
"\n",
|
||||
"================================================================================\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -1079,19 +1116,23 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "AttributeError",
|
||||
"evalue": "'list' object has no attribute 'text'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[13], line 15\u001b[0m\n\u001b[0;32m 7\u001b[0m \u001b[38;5;66;03m# Validate chunks\u001b[39;00m\n\u001b[0;32m 8\u001b[0m validator \u001b[38;5;241m=\u001b[39m ChunkValidator(\n\u001b[0;32m 9\u001b[0m min_chunk_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m50\u001b[39m,\n\u001b[0;32m 10\u001b[0m max_chunk_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m300\u001b[39m,\n\u001b[0;32m 11\u001b[0m min_overlap\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m20\u001b[39m,\n\u001b[0;32m 12\u001b[0m max_overlap\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m100\u001b[39m\n\u001b[0;32m 13\u001b[0m )\n\u001b[1;32m---> 15\u001b[0m validation_result \u001b[38;5;241m=\u001b[39m \u001b[43mvalidator\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mchunks\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 17\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mChunk Validation Results:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 18\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m=\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m80\u001b[39m)\n",
|
||||
"File \u001b[1;32m~\\semantica\\semantica\\split\\chunk_validator.py:96\u001b[0m, in \u001b[0;36mChunkValidator.validate\u001b[1;34m(self, chunk, **options)\u001b[0m\n\u001b[0;32m 92\u001b[0m \u001b[38;5;66;03m# Size validation\u001b[39;00m\n\u001b[0;32m 93\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprogress_tracker\u001b[38;5;241m.\u001b[39mupdate_tracking(\n\u001b[0;32m 94\u001b[0m tracking_id, message\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mValidating chunk size...\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 95\u001b[0m )\n\u001b[1;32m---> 96\u001b[0m chunk_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(\u001b[43mchunk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtext\u001b[49m)\n\u001b[0;32m 97\u001b[0m metrics[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msize\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m chunk_size\n\u001b[0;32m 99\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m chunk_size \u001b[38;5;241m<\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmin_size:\n",
|
||||
"\u001b[1;31mAttributeError\u001b[0m: 'list' object has no attribute 'text'"
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Chunk Validation Results:\n",
|
||||
"\n",
|
||||
"================================================================================\n",
|
||||
"\n",
|
||||
"Overall Valid: False\n",
|
||||
"Quality Score: 0.40\n",
|
||||
"\n",
|
||||
"No issues found!\n",
|
||||
"\n",
|
||||
"================================================================================\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -1146,23 +1187,61 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": 23,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "AttributeError",
|
||||
"evalue": "'ProvenanceTracker' object has no attribute 'track'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[1;32mIn[14], line 11\u001b[0m\n\u001b[0;32m 8\u001b[0m tracker \u001b[38;5;241m=\u001b[39m ProvenanceTracker()\n\u001b[0;32m 10\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk \u001b[38;5;129;01min\u001b[39;00m chunks:\n\u001b[1;32m---> 11\u001b[0m \u001b[43mtracker\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrack\u001b[49m(\n\u001b[0;32m 12\u001b[0m chunk\u001b[38;5;241m=\u001b[39mchunk,\n\u001b[0;32m 13\u001b[0m source\u001b[38;5;241m=\u001b[39m{\n\u001b[0;32m 14\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdocument_id\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapple_doc_001\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 15\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_path\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata/apple.txt\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 16\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimestamp\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2024-01-01T00:00:00Z\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 17\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmethod\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrecursive\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 18\u001b[0m }\n\u001b[0;32m 19\u001b[0m )\n\u001b[0;32m 21\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mProvenance Tracking Results:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 22\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m=\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m80\u001b[39m)\n",
|
||||
"\u001b[1;31mAttributeError\u001b[0m: 'ProvenanceTracker' object has no attribute 'track'"
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Chunk class fields: {'text': <class 'str'>, 'start_index': <class 'int'>, 'end_index': <class 'int'>, 'metadata': typing.Dict[str, typing.Any], 'id': typing.Optional[str]}\n",
|
||||
"Provenance Tracking Results:\n",
|
||||
"\n",
|
||||
"================================================================================\n",
|
||||
"Chunk ID: f0bddbe2-b717-47c7-8bdb-631a61660373\n",
|
||||
"\n",
|
||||
"Lineage for Chunk 1:\n",
|
||||
" Source Document: apple_doc_001\n",
|
||||
" File Path: data/apple.txt\n",
|
||||
" Method: recursive\n",
|
||||
" Timestamp: None\n",
|
||||
"\n",
|
||||
"================================================================================\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from semantica.split import ProvenanceTracker\n",
|
||||
"import sys\n",
|
||||
"import os\n",
|
||||
"import importlib\n",
|
||||
"\n",
|
||||
"# 1. Ensure local package is in path\n",
|
||||
"project_root = os.path.abspath(os.path.join(os.getcwd(), \"../..\"))\n",
|
||||
"if project_root not in sys.path:\n",
|
||||
" sys.path.insert(0, project_root)\n",
|
||||
"\n",
|
||||
"# 2. Force unload modules to ensure clean reload\n",
|
||||
"modules_to_unload = [\n",
|
||||
" 'semantica.split.semantic_chunker', \n",
|
||||
" 'semantica.split.splitter', \n",
|
||||
" 'semantica.split.provenance_tracker',\n",
|
||||
" 'semantica.split'\n",
|
||||
"]\n",
|
||||
"for module in modules_to_unload:\n",
|
||||
" if module in sys.modules:\n",
|
||||
" del sys.modules[module]\n",
|
||||
"\n",
|
||||
"# 3. Import fresh modules\n",
|
||||
"import semantica.split.semantic_chunker\n",
|
||||
"import semantica.split.splitter\n",
|
||||
"import semantica.split.provenance_tracker\n",
|
||||
"from semantica.split import ProvenanceTracker, TextSplitter\n",
|
||||
"\n",
|
||||
"# 4. Verify Chunk class has id field\n",
|
||||
"from semantica.split.semantic_chunker import Chunk\n",
|
||||
"print(f\"Chunk class fields: {Chunk.__annotations__}\")\n",
|
||||
"if 'id' not in Chunk.__annotations__:\n",
|
||||
" print(\"WARNING: Chunk class still missing 'id' field. Kernel restart required.\")\n",
|
||||
"\n",
|
||||
"# Create chunks\n",
|
||||
"splitter = TextSplitter(method=\"recursive\", chunk_size=200, chunk_overlap=50)\n",
|
||||
@@ -1172,14 +1251,12 @@
|
||||
"tracker = ProvenanceTracker()\n",
|
||||
"\n",
|
||||
"for chunk in chunks:\n",
|
||||
" tracker.track(\n",
|
||||
" tracker.track_chunk(\n",
|
||||
" chunk=chunk,\n",
|
||||
" source={\n",
|
||||
" \"document_id\": \"apple_doc_001\",\n",
|
||||
" \"file_path\": \"data/apple.txt\",\n",
|
||||
" \"timestamp\": \"2024-01-01T00:00:00Z\",\n",
|
||||
" \"method\": \"recursive\"\n",
|
||||
" }\n",
|
||||
" source_document=\"apple_doc_001\",\n",
|
||||
" source_path=\"data/apple.txt\",\n",
|
||||
" timestamp=\"2024-01-01T00:00:00Z\",\n",
|
||||
" method=\"recursive\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"print(\"Provenance Tracking Results:\\n\")\n",
|
||||
@@ -1187,13 +1264,22 @@
|
||||
"\n",
|
||||
"# Get lineage for first chunk\n",
|
||||
"if chunks:\n",
|
||||
" lineage = tracker.get_lineage(chunks[0].id)\n",
|
||||
" # Get provenance info using the chunk's ID\n",
|
||||
" chunk_id = getattr(chunks[0], 'id', None)\n",
|
||||
" print(f\"Chunk ID: {chunk_id}\")\n",
|
||||
" \n",
|
||||
" print(f\"\\nLineage for Chunk 1:\")\n",
|
||||
" print(f\" Source Document: {lineage.get('source', {}).get('document_id')}\")\n",
|
||||
" print(f\" File Path: {lineage.get('source', {}).get('file_path')}\")\n",
|
||||
" print(f\" Method: {lineage.get('source', {}).get('method')}\")\n",
|
||||
" print(f\" Timestamp: {lineage.get('source', {}).get('timestamp')}\")\n",
|
||||
" if chunk_id:\n",
|
||||
" prov_info = tracker.get_provenance(chunk_id)\n",
|
||||
" \n",
|
||||
" if prov_info:\n",
|
||||
" print(f\"\\nLineage for Chunk 1:\")\n",
|
||||
" print(f\" Source Document: {prov_info.source_document}\")\n",
|
||||
" print(f\" File Path: {prov_info.source_path}\")\n",
|
||||
" print(f\" Method: {prov_info.metadata.get('method')}\")\n",
|
||||
" print(f\" Timestamp: {prov_info.timestamp}\")\n",
|
||||
" else:\n",
|
||||
" print(\"Error: Chunk ID not found. The Chunk class definition might still be cached.\")\n",
|
||||
" print(\"Please click 'Kernel' -> 'Restart Kernel' in the menu and run all cells again.\")\n",
|
||||
"\n",
|
||||
"print(\"\\n\" + \"=\" * 80)"
|
||||
]
|
||||
|
||||
@@ -96,7 +96,14 @@ class ProvenanceTracker:
|
||||
Returns:
|
||||
str: Provenance ID
|
||||
"""
|
||||
chunk_id = str(uuid4())
|
||||
chunk_id = getattr(chunk, "id", None)
|
||||
if not chunk_id:
|
||||
chunk_id = str(uuid4())
|
||||
try:
|
||||
chunk.id = chunk_id
|
||||
except AttributeError:
|
||||
pass # Chunk might be immutable
|
||||
|
||||
provenance_id = str(uuid4())
|
||||
|
||||
provenance_info = ProvenanceInfo(
|
||||
|
||||
@@ -46,11 +46,11 @@ except ImportError:
|
||||
@dataclass
|
||||
class Chunk:
|
||||
"""Chunk representation."""
|
||||
|
||||
text: str
|
||||
start_index: int
|
||||
end_index: int
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
id: Optional[str] = None
|
||||
|
||||
|
||||
class SemanticChunker:
|
||||
|
||||
Reference in New Issue
Block a user