diff --git a/cookbook/introduction/11_Chunking_and_Splitting.ipynb b/cookbook/introduction/11_Chunking_and_Splitting.ipynb index cdebfc62..21f2ec96 100644 --- a/cookbook/introduction/11_Chunking_and_Splitting.ipynb +++ b/cookbook/introduction/11_Chunking_and_Splitting.ipynb @@ -55,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -97,17 +97,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "c:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\sentence_transformers\\cross_encoder\\CrossEncoder.py:13: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", - " from tqdm.autonotebook import tqdm, trange\n" - ] - }, { "name": "stdout", "output_type": "stream", @@ -231,7 +223,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -306,7 +298,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -479,13 +471,13 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "

🧠 Semantica - 📊 Current Progress

StatusActionModuleSubmoduleFileTime
✅Semantica is splitting✂️ splitSemanticChunker-0.15s
✅Semantica is splitting✂️ splitEntityAwareChunker-1.14s
✅Semantica is extracting🎯 semantic_extractNERExtractor-0.50s
✅Semantica is splitting✂️ splitRelationAwareChunker-1.55s
✅Semantica is extracting🎯 semantic_extractRelationExtractor-0.52s
✅Semantica is splitting✂️ splitStructuralChunker-0.01s
✅Semantica is splitting✂️ splitHierarchicalChunker-0.00s
✅Semantica is splitting✂️ splitSlidingWindowChunker-0.06s
❌Semantica is splitting✂️ splitChunkValidator-0.01s
" + "

🧠 Semantica - 📊 Current Progress

StatusActionModuleSubmoduleFileTime
✅Semantica is splitting✂️ splitSemanticChunker-0.13s
✅Semantica is splitting✂️ splitEntityAwareChunker-0.98s
✅Semantica is extracting🎯 semantic_extractNERExtractor-0.47s
✅Semantica is splitting✂️ splitRelationAwareChunker-1.39s
✅Semantica is extracting🎯 semantic_extractRelationExtractor-0.45s
✅Semantica is splitting✂️ splitStructuralChunker-0.01s
✅Semantica is splitting✂️ splitHierarchicalChunker-0.00s
✅Semantica is splitting✂️ splitSlidingWindowChunker-0.00s
✅Semantica is splitting✂️ splitTableChunker-0.01s
✅Semantica is splitting✂️ splitChunkValidator-0.01s
" ], "text/plain": [ "" @@ -585,7 +577,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -652,7 +644,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -741,7 +733,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -851,7 +843,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -930,7 +922,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -941,18 +933,52 @@ "\n", "================================================================================\n", "\n", - "Window 1:\n" - ] - }, - { - "ename": "AttributeError", - "evalue": "'Chunk' object has no attribute 'start'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[16], line 19\u001b[0m\n\u001b[0;32m 16\u001b[0m overlap \u001b[38;5;241m=\u001b[39m chunk\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124moverlap_chars\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;241m0\u001b[39m)\n\u001b[0;32m 18\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124mWindow \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m:\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m---> 19\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m Position: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[43mchunk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstart\u001b[49m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m-\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mchunk\u001b[38;5;241m.\u001b[39mend\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 20\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m Length: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(chunk\u001b[38;5;241m.\u001b[39mtext)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m chars\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 21\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m Overlap with previous: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00moverlap\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m chars\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "\u001b[1;31mAttributeError\u001b[0m: 'Chunk' object has no attribute 'start'" + "Window 1:\n", + " Position: 0-134\n", + " Length: 133 chars\n", + " Overlap with previous: 0 chars\n", + "\n", + "Window 2:\n", + " Position: 100-220\n", + " Length: 120 chars\n", + " Overlap with previous: 34 chars\n", + "\n", + "Window 3:\n", + " Position: 200-279\n", + " Length: 79 chars\n", + " Overlap with previous: 20 chars\n", + "\n", + "Window 4:\n", + " Position: 300-383\n", + " Length: 81 chars\n", + " Overlap with previous: 0 chars\n", + "\n", + "Window 5:\n", + " Position: 400-486\n", + " Length: 83 chars\n", + " Overlap with previous: 0 chars\n", + "\n", + "Window 6:\n", + " Position: 500-612\n", + " Length: 112 chars\n", + " Overlap with previous: 0 chars\n", + "\n", + "Window 7:\n", + " Position: 600-718\n", + " Length: 116 chars\n", + " Overlap with previous: 12 chars\n", + "\n", + "Window 8:\n", + " Position: 700-786\n", + " Length: 86 chars\n", + " Overlap with previous: 18 chars\n", + "\n", + "Window 9:\n", + " Position: 800-896\n", + " Length: 95 chars\n", + " Overlap with previous: 0 chars\n", + "\n", + "================================================================================\n" ] } ], @@ -961,9 +987,8 @@ "\n", "# Sliding window chunking\n", "sliding_chunker = SlidingWindowChunker(\n", - " window_size=150,\n", - " step_size=100, # 50 char overlap\n", - " min_chunk_size=50\n", + " chunk_size=150,\n", + " overlap=50\n", ")\n", "\n", "chunks = sliding_chunker.chunk(text)\n", @@ -972,10 +997,14 @@ "print(\"=\" * 80)\n", "\n", "for i, chunk in enumerate(chunks, 1):\n", - " overlap = chunk.metadata.get('overlap_chars', 0)\n", + " # Calculate overlap manually\n", + " overlap = 0\n", + " if i > 1:\n", + " prev_chunk = chunks[i-2]\n", + " overlap = max(0, prev_chunk.end_index - chunk.start_index)\n", " \n", " print(f\"\\nWindow {i}:\")\n", - " print(f\" Position: {chunk.start}-{chunk.end}\")\n", + " print(f\" Position: {chunk.start_index}-{chunk.end_index}\")\n", " print(f\" Length: {len(chunk.text)} chars\")\n", " print(f\" Overlap with previous: {overlap} chars\")\n", "\n", @@ -999,18 +1028,26 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 14, "metadata": {}, "outputs": [ { - "ename": "AttributeError", - "evalue": "'TableChunker' object has no attribute 'chunk'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[12], line 26\u001b[0m\n\u001b[0;32m 18\u001b[0m \u001b[38;5;66;03m# Table chunking\u001b[39;00m\n\u001b[0;32m 19\u001b[0m table_chunker \u001b[38;5;241m=\u001b[39m TableChunker(\n\u001b[0;32m 20\u001b[0m preserve_headers\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 21\u001b[0m max_rows_per_chunk\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m3\u001b[39m,\n\u001b[0;32m 22\u001b[0m include_context\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m 23\u001b[0m table_format\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmarkdown\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 24\u001b[0m )\n\u001b[1;32m---> 26\u001b[0m chunks \u001b[38;5;241m=\u001b[39m \u001b[43mtable_chunker\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mchunk\u001b[49m(text_with_table)\n\u001b[0;32m 28\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTable Chunking Results:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 29\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m=\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m80\u001b[39m)\n", - "\u001b[1;31mAttributeError\u001b[0m: 'TableChunker' object has no attribute 'chunk'" + "name": "stdout", + "output_type": "stream", + "text": [ + "Table Chunking Results:\n", + "\n", + "================================================================================\n", + "\n", + "Chunk 1:\n", + " Type: Text\n", + " Content: Product | Category | Release Year\n", + "--- | --- | ---\n", + "iPhone | Smartphone | 2007\n", + "iPad | Tablet | 2010\n", + "Ma...\n", + "\n", + "================================================================================\n" ] } ], @@ -1079,19 +1116,23 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 15, "metadata": {}, "outputs": [ { - "ename": "AttributeError", - "evalue": "'list' object has no attribute 'text'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[13], line 15\u001b[0m\n\u001b[0;32m 7\u001b[0m \u001b[38;5;66;03m# Validate chunks\u001b[39;00m\n\u001b[0;32m 8\u001b[0m validator \u001b[38;5;241m=\u001b[39m ChunkValidator(\n\u001b[0;32m 9\u001b[0m min_chunk_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m50\u001b[39m,\n\u001b[0;32m 10\u001b[0m max_chunk_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m300\u001b[39m,\n\u001b[0;32m 11\u001b[0m min_overlap\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m20\u001b[39m,\n\u001b[0;32m 12\u001b[0m max_overlap\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m100\u001b[39m\n\u001b[0;32m 13\u001b[0m )\n\u001b[1;32m---> 15\u001b[0m validation_result \u001b[38;5;241m=\u001b[39m \u001b[43mvalidator\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mchunks\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 17\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mChunk Validation Results:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 18\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m=\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m80\u001b[39m)\n", - "File \u001b[1;32m~\\semantica\\semantica\\split\\chunk_validator.py:96\u001b[0m, in \u001b[0;36mChunkValidator.validate\u001b[1;34m(self, chunk, **options)\u001b[0m\n\u001b[0;32m 92\u001b[0m \u001b[38;5;66;03m# Size validation\u001b[39;00m\n\u001b[0;32m 93\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprogress_tracker\u001b[38;5;241m.\u001b[39mupdate_tracking(\n\u001b[0;32m 94\u001b[0m tracking_id, message\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mValidating chunk size...\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 95\u001b[0m )\n\u001b[1;32m---> 96\u001b[0m chunk_size \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(\u001b[43mchunk\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtext\u001b[49m)\n\u001b[0;32m 97\u001b[0m metrics[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msize\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m chunk_size\n\u001b[0;32m 99\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m chunk_size \u001b[38;5;241m<\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmin_size:\n", - "\u001b[1;31mAttributeError\u001b[0m: 'list' object has no attribute 'text'" + "name": "stdout", + "output_type": "stream", + "text": [ + "Chunk Validation Results:\n", + "\n", + "================================================================================\n", + "\n", + "Overall Valid: False\n", + "Quality Score: 0.40\n", + "\n", + "No issues found!\n", + "\n", + "================================================================================\n" ] } ], @@ -1146,23 +1187,61 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 23, "metadata": {}, "outputs": [ { - "ename": "AttributeError", - "evalue": "'ProvenanceTracker' object has no attribute 'track'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[14], line 11\u001b[0m\n\u001b[0;32m 8\u001b[0m tracker \u001b[38;5;241m=\u001b[39m ProvenanceTracker()\n\u001b[0;32m 10\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk \u001b[38;5;129;01min\u001b[39;00m chunks:\n\u001b[1;32m---> 11\u001b[0m \u001b[43mtracker\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrack\u001b[49m(\n\u001b[0;32m 12\u001b[0m chunk\u001b[38;5;241m=\u001b[39mchunk,\n\u001b[0;32m 13\u001b[0m source\u001b[38;5;241m=\u001b[39m{\n\u001b[0;32m 14\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdocument_id\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapple_doc_001\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 15\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_path\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdata/apple.txt\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 16\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimestamp\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2024-01-01T00:00:00Z\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m 17\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmethod\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrecursive\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 18\u001b[0m }\n\u001b[0;32m 19\u001b[0m )\n\u001b[0;32m 21\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mProvenance Tracking Results:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 22\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m=\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m80\u001b[39m)\n", - "\u001b[1;31mAttributeError\u001b[0m: 'ProvenanceTracker' object has no attribute 'track'" + "name": "stdout", + "output_type": "stream", + "text": [ + "Chunk class fields: {'text': , 'start_index': , 'end_index': , 'metadata': typing.Dict[str, typing.Any], 'id': typing.Optional[str]}\n", + "Provenance Tracking Results:\n", + "\n", + "================================================================================\n", + "Chunk ID: f0bddbe2-b717-47c7-8bdb-631a61660373\n", + "\n", + "Lineage for Chunk 1:\n", + " Source Document: apple_doc_001\n", + " File Path: data/apple.txt\n", + " Method: recursive\n", + " Timestamp: None\n", + "\n", + "================================================================================\n" ] } ], "source": [ - "from semantica.split import ProvenanceTracker\n", + "import sys\n", + "import os\n", + "import importlib\n", + "\n", + "# 1. Ensure local package is in path\n", + "project_root = os.path.abspath(os.path.join(os.getcwd(), \"../..\"))\n", + "if project_root not in sys.path:\n", + " sys.path.insert(0, project_root)\n", + "\n", + "# 2. Force unload modules to ensure clean reload\n", + "modules_to_unload = [\n", + " 'semantica.split.semantic_chunker', \n", + " 'semantica.split.splitter', \n", + " 'semantica.split.provenance_tracker',\n", + " 'semantica.split'\n", + "]\n", + "for module in modules_to_unload:\n", + " if module in sys.modules:\n", + " del sys.modules[module]\n", + "\n", + "# 3. Import fresh modules\n", + "import semantica.split.semantic_chunker\n", + "import semantica.split.splitter\n", + "import semantica.split.provenance_tracker\n", + "from semantica.split import ProvenanceTracker, TextSplitter\n", + "\n", + "# 4. Verify Chunk class has id field\n", + "from semantica.split.semantic_chunker import Chunk\n", + "print(f\"Chunk class fields: {Chunk.__annotations__}\")\n", + "if 'id' not in Chunk.__annotations__:\n", + " print(\"WARNING: Chunk class still missing 'id' field. Kernel restart required.\")\n", "\n", "# Create chunks\n", "splitter = TextSplitter(method=\"recursive\", chunk_size=200, chunk_overlap=50)\n", @@ -1172,14 +1251,12 @@ "tracker = ProvenanceTracker()\n", "\n", "for chunk in chunks:\n", - " tracker.track(\n", + " tracker.track_chunk(\n", " chunk=chunk,\n", - " source={\n", - " \"document_id\": \"apple_doc_001\",\n", - " \"file_path\": \"data/apple.txt\",\n", - " \"timestamp\": \"2024-01-01T00:00:00Z\",\n", - " \"method\": \"recursive\"\n", - " }\n", + " source_document=\"apple_doc_001\",\n", + " source_path=\"data/apple.txt\",\n", + " timestamp=\"2024-01-01T00:00:00Z\",\n", + " method=\"recursive\"\n", " )\n", "\n", "print(\"Provenance Tracking Results:\\n\")\n", @@ -1187,13 +1264,22 @@ "\n", "# Get lineage for first chunk\n", "if chunks:\n", - " lineage = tracker.get_lineage(chunks[0].id)\n", + " # Get provenance info using the chunk's ID\n", + " chunk_id = getattr(chunks[0], 'id', None)\n", + " print(f\"Chunk ID: {chunk_id}\")\n", " \n", - " print(f\"\\nLineage for Chunk 1:\")\n", - " print(f\" Source Document: {lineage.get('source', {}).get('document_id')}\")\n", - " print(f\" File Path: {lineage.get('source', {}).get('file_path')}\")\n", - " print(f\" Method: {lineage.get('source', {}).get('method')}\")\n", - " print(f\" Timestamp: {lineage.get('source', {}).get('timestamp')}\")\n", + " if chunk_id:\n", + " prov_info = tracker.get_provenance(chunk_id)\n", + " \n", + " if prov_info:\n", + " print(f\"\\nLineage for Chunk 1:\")\n", + " print(f\" Source Document: {prov_info.source_document}\")\n", + " print(f\" File Path: {prov_info.source_path}\")\n", + " print(f\" Method: {prov_info.metadata.get('method')}\")\n", + " print(f\" Timestamp: {prov_info.timestamp}\")\n", + " else:\n", + " print(\"Error: Chunk ID not found. The Chunk class definition might still be cached.\")\n", + " print(\"Please click 'Kernel' -> 'Restart Kernel' in the menu and run all cells again.\")\n", "\n", "print(\"\\n\" + \"=\" * 80)" ] diff --git a/semantica/split/provenance_tracker.py b/semantica/split/provenance_tracker.py index a45e991a..6ca5961b 100644 --- a/semantica/split/provenance_tracker.py +++ b/semantica/split/provenance_tracker.py @@ -96,7 +96,14 @@ class ProvenanceTracker: Returns: str: Provenance ID """ - chunk_id = str(uuid4()) + chunk_id = getattr(chunk, "id", None) + if not chunk_id: + chunk_id = str(uuid4()) + try: + chunk.id = chunk_id + except AttributeError: + pass # Chunk might be immutable + provenance_id = str(uuid4()) provenance_info = ProvenanceInfo( diff --git a/semantica/split/semantic_chunker.py b/semantica/split/semantic_chunker.py index a2253776..6c9968b4 100644 --- a/semantica/split/semantic_chunker.py +++ b/semantica/split/semantic_chunker.py @@ -46,11 +46,11 @@ except ImportError: @dataclass class Chunk: """Chunk representation.""" - text: str start_index: int end_index: int metadata: Dict[str, Any] = field(default_factory=dict) + id: Optional[str] = None class SemanticChunker: