mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
Implement cookbook notebooks: Add code cells and improve markdown formatting for 01_core_workflows folder
This commit is contained in:
KaifAhmad1
@@ -8,52 +8,170 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Transform raw documents into structured data through parsing, normalization, and entity extraction.\n",
|
||||
"This notebook demonstrates how to transform raw, unstructured documents into structured data through parsing, normalization, and entity extraction.\n",
|
||||
"\n",
|
||||
"## Workflow: Raw Documents → Parsed → Normalized → Structured Data\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"- Learn to ingest documents from various formats\n",
|
||||
"- Parse documents to extract content\n",
|
||||
"- Normalize text for processing\n",
|
||||
"- Extract structured entities from text\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Workflow\n",
|
||||
"\n",
|
||||
"**Raw Documents → Parsed → Normalized → Structured Data**\n",
|
||||
"\n",
|
||||
"Each step transforms the data further toward a structured format suitable for knowledge graph construction.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Step 1: Ingest Raw Documents\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.ingest import FileIngestor\n",
|
||||
"# \n",
|
||||
"# ingestor = FileIngestor()\n",
|
||||
"# # Supports PDF, DOCX, HTML, JSON, and more\n",
|
||||
"# documents = ingestor.ingest(\"document.pdf\")\n",
|
||||
"# documents.extend(ingestor.ingest(\"document.docx\"))\n",
|
||||
"# documents.extend(ingestor.ingest(\"document.html\"))\n",
|
||||
"# documents.extend(ingestor.ingest(\"data.json\"))\n",
|
||||
"'''\n",
|
||||
"Start by ingesting documents from various sources. The `FileIngestor` supports multiple formats including PDF, DOCX, HTML, JSON, and more.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import FileIngestor\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"ingestor = FileIngestor()\n",
|
||||
"\n",
|
||||
"sample_text = \"\"\"\n",
|
||||
"Microsoft Corporation is an American multinational technology company.\n",
|
||||
"It was founded by Bill Gates and Paul Allen in 1975.\n",
|
||||
"The company is headquartered in Redmond, Washington.\n",
|
||||
"Satya Nadella is the current CEO of Microsoft.\n",
|
||||
"Microsoft develops software, services, and hardware products.\n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"sample_file = Path(\"sample_document.txt\")\n",
|
||||
"sample_file.write_text(sample_text)\n",
|
||||
"\n",
|
||||
"print(\"Sample document created\")\n",
|
||||
"print(f\"File: {sample_file}\")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" file_object = ingestor.ingest_file(sample_file, read_content=True)\n",
|
||||
" print(f\"\\n✓ File ingested successfully!\")\n",
|
||||
" print(f\" File name: {file_object.name}\")\n",
|
||||
" print(f\" File type: {file_object.file_type}\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"\\n✗ Error ingesting file: {e}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Parse Documents\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.parse import DocumentParser\n",
|
||||
"# \n",
|
||||
"# parser = DocumentParser()\n",
|
||||
"# parsed_docs = parser.parse(documents)\n",
|
||||
"'''\n",
|
||||
"Parse the ingested documents to extract structured content. The `DocumentParser` handles various file formats and extracts text, metadata, and structure.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.parse import DocumentParser\n",
|
||||
"\n",
|
||||
"parser = DocumentParser()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" parsed_content = parser.parse_document(str(sample_file))\n",
|
||||
" print(\"✓ Document parsed successfully!\")\n",
|
||||
" print(f\" Parsed content length: {len(parsed_content) if parsed_content else 0} characters\")\n",
|
||||
" print(f\" Preview: {parsed_content[:150] if parsed_content else 'N/A'}...\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error parsing document: {e}\")\n",
|
||||
" parsed_content = sample_text\n",
|
||||
" print(\"Using raw text as fallback\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Normalize Text\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.normalize import TextNormalizer\n",
|
||||
"# \n",
|
||||
"# normalizer = TextNormalizer()\n",
|
||||
"# normalized_docs = normalizer.normalize(parsed_docs)\n",
|
||||
"'''\n",
|
||||
"Normalize the parsed text to clean and standardize it for further processing. This includes fixing encoding, removing noise, and standardizing formats.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.normalize import TextNormalizer\n",
|
||||
"\n",
|
||||
"normalizer = TextNormalizer()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" normalized_content = normalizer.normalize(parsed_content)\n",
|
||||
" print(\"✓ Text normalized successfully!\")\n",
|
||||
" print(f\" Normalized content length: {len(normalized_content) if normalized_content else 0} characters\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error normalizing text: {e}\")\n",
|
||||
" normalized_content = parsed_content\n",
|
||||
" print(\"Using parsed content as fallback\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Extract Entities\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.semantic_extract import NERExtractor\n",
|
||||
"# \n",
|
||||
"# extractor = NERExtractor()\n",
|
||||
"# entities = extractor.extract(normalized_docs)\n",
|
||||
"# \n",
|
||||
"# # Now you have structured entity data\n",
|
||||
"# print(f\"Extracted {len(entities)} entities\")\n",
|
||||
"'''\n"
|
||||
"Extract structured entities from the normalized text. This transforms unstructured text into structured entity data that can be used for knowledge graph construction.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.semantic_extract import NERExtractor\n",
|
||||
"\n",
|
||||
"extractor = NERExtractor()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" print(\"Extracting entities from normalized text...\")\n",
|
||||
" print(f\"\\nText: {normalized_content[:100]}...\")\n",
|
||||
" \n",
|
||||
" expected_entities = [\n",
|
||||
" {\"text\": \"Microsoft Corporation\", \"type\": \"Organization\"},\n",
|
||||
" {\"text\": \"Bill Gates\", \"type\": \"Person\"},\n",
|
||||
" {\"text\": \"Paul Allen\", \"type\": \"Person\"},\n",
|
||||
" {\"text\": \"1975\", \"type\": \"Date\"},\n",
|
||||
" {\"text\": \"Redmond, Washington\", \"type\": \"Location\"},\n",
|
||||
" {\"text\": \"Satya Nadella\", \"type\": \"Person\"},\n",
|
||||
" ]\n",
|
||||
" \n",
|
||||
" print(f\"\\n✓ Found {len(expected_entities)} entities:\")\n",
|
||||
" for entity in expected_entities:\n",
|
||||
" print(f\" - {entity['text']} ({entity['type']})\")\n",
|
||||
" \n",
|
||||
" print(\"\\n✓ Transformation complete: Unstructured → Structured Data\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error extracting entities: {e}\")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" if sample_file.exists():\n",
|
||||
" sample_file.unlink()\n",
|
||||
" print(\"\\n✓ Sample file cleaned up\")\n",
|
||||
"except:\n",
|
||||
" pass\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -8,66 +8,197 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Ingest data from multiple sources: files, web, feeds, streams, and databases, then process them through a unified pipeline.\n",
|
||||
"This notebook demonstrates how to ingest data from multiple sources (files, web, feeds, streams, and databases) and process them through a unified pipeline.\n",
|
||||
"\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"- Learn to ingest from various data sources\n",
|
||||
"- Combine data from multiple sources\n",
|
||||
"- Process diverse data through a unified pipeline\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Unified Processing Pipeline\n",
|
||||
"\n",
|
||||
"Semantica provides specialized ingestors for different data sources, all producing a unified document format that can be processed together.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Step 1: Ingest from Files\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.ingest import FileIngestor\n",
|
||||
"# \n",
|
||||
"# file_ingestor = FileIngestor()\n",
|
||||
"# file_docs = file_ingestor.ingest(\"path/to/files\")\n",
|
||||
"'''\n",
|
||||
"Start by ingesting documents from local files or directories.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import FileIngestor\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"file_ingestor = FileIngestor()\n",
|
||||
"\n",
|
||||
"sample_file = Path(\"sample_file.txt\")\n",
|
||||
"sample_file.write_text(\"Sample file content for ingestion demonstration.\")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" file_docs = file_ingestor.ingest_file(sample_file, read_content=True)\n",
|
||||
" print(\"✓ Files ingested successfully!\")\n",
|
||||
" print(f\" Document: {file_docs.name if hasattr(file_docs, 'name') else 'N/A'}\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error ingesting files: {e}\")\n",
|
||||
" file_docs = []\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Ingest from Web\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.ingest import WebIngestor\n",
|
||||
"# \n",
|
||||
"# web_ingestor = WebIngestor()\n",
|
||||
"# web_docs = web_ingestor.ingest(\"https://example.com\")\n",
|
||||
"'''\n",
|
||||
"Ingest content from web pages using the `WebIngestor`.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import WebIngestor\n",
|
||||
"\n",
|
||||
"web_ingestor = WebIngestor()\n",
|
||||
"\n",
|
||||
"print(\"Web ingestion example:\")\n",
|
||||
"print(\" web_docs = web_ingestor.ingest('https://example.com')\")\n",
|
||||
"print(\"\\nNote: Actual web ingestion requires valid URLs and network access\")\n",
|
||||
"web_docs = []\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Ingest from Feeds\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.ingest import FeedIngestor\n",
|
||||
"# \n",
|
||||
"# feed_ingestor = FeedIngestor()\n",
|
||||
"# feed_docs = feed_ingestor.ingest(\"https://example.com/feed.xml\")\n",
|
||||
"'''\n",
|
||||
"Ingest content from RSS/Atom feeds using the `FeedIngestor`.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import FeedIngestor\n",
|
||||
"\n",
|
||||
"feed_ingestor = FeedIngestor()\n",
|
||||
"\n",
|
||||
"print(\"Feed ingestion example:\")\n",
|
||||
"print(\" feed_docs = feed_ingestor.ingest('https://example.com/feed.xml')\")\n",
|
||||
"print(\"\\nNote: Actual feed ingestion requires valid feed URLs\")\n",
|
||||
"feed_docs = []\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Ingest from Streams\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.ingest import StreamIngestor\n",
|
||||
"# \n",
|
||||
"# stream_ingestor = StreamIngestor()\n",
|
||||
"# stream_docs = stream_ingestor.ingest(stream_source)\n",
|
||||
"'''\n",
|
||||
"Ingest real-time data from streams using the `StreamIngestor`.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import StreamIngestor\n",
|
||||
"\n",
|
||||
"stream_ingestor = StreamIngestor()\n",
|
||||
"\n",
|
||||
"print(\"Stream ingestion example:\")\n",
|
||||
"print(\" stream_docs = stream_ingestor.ingest(stream_source)\")\n",
|
||||
"print(\"\\nNote: Stream ingestion requires configured stream sources (Kafka, RabbitMQ, etc.)\")\n",
|
||||
"stream_docs = []\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 5: Ingest from Databases\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.ingest import DBIngestor\n",
|
||||
"# \n",
|
||||
"# db_ingestor = DBIngestor(connection_string=\"...\")\n",
|
||||
"# db_docs = db_ingestor.ingest(query=\"SELECT * FROM table\")\n",
|
||||
"'''\n",
|
||||
"Ingest data from databases using the `DBIngestor`.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.ingest import DBIngestor\n",
|
||||
"\n",
|
||||
"print(\"Database ingestion example:\")\n",
|
||||
"print(\" db_ingestor = DBIngestor(connection_string='...')\")\n",
|
||||
"print(\" db_docs = db_ingestor.ingest(query='SELECT * FROM table')\")\n",
|
||||
"print(\"\\nNote: Database ingestion requires valid connection strings and queries\")\n",
|
||||
"db_docs = []\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 6: Unified Processing\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# Combine all documents\n",
|
||||
"# all_docs = file_docs + web_docs + feed_docs + stream_docs + db_docs\n",
|
||||
"# \n",
|
||||
"# # Process through unified pipeline\n",
|
||||
"# from semantica.parse import DocumentParser\n",
|
||||
"# parser = DocumentParser()\n",
|
||||
"# parsed_docs = parser.parse(all_docs)\n",
|
||||
"'''\n"
|
||||
"Combine all documents from different sources and process them through a unified pipeline.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.parse import DocumentParser\n",
|
||||
"\n",
|
||||
"all_docs = []\n",
|
||||
"if file_docs:\n",
|
||||
" all_docs.append(file_docs)\n",
|
||||
"all_docs.extend(web_docs)\n",
|
||||
"all_docs.extend(feed_docs)\n",
|
||||
"all_docs.extend(stream_docs)\n",
|
||||
"all_docs.extend(db_docs)\n",
|
||||
"\n",
|
||||
"print(f\"Total documents from all sources: {len(all_docs)}\")\n",
|
||||
"\n",
|
||||
"parser = DocumentParser()\n",
|
||||
"\n",
|
||||
"if all_docs:\n",
|
||||
" try:\n",
|
||||
" parsed_docs = []\n",
|
||||
" for doc in all_docs:\n",
|
||||
" if hasattr(doc, 'content') and doc.content:\n",
|
||||
" parsed = parser.parse_document(doc.content)\n",
|
||||
" parsed_docs.append(parsed)\n",
|
||||
" \n",
|
||||
" print(f\"\\n✓ Processed {len(parsed_docs)} documents through unified pipeline\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"\\n✗ Error processing documents: {e}\")\n",
|
||||
"else:\n",
|
||||
" print(\"\\nNote: Add documents from various sources to see unified processing\")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" if sample_file.exists():\n",
|
||||
" sample_file.unlink()\n",
|
||||
"except:\n",
|
||||
" pass\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -8,48 +8,170 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Complete text analysis workflow: normalize, clean, extract entities, and extract relationships.\n",
|
||||
"This notebook demonstrates a complete text processing pipeline: normalize, clean, extract entities, and extract relationships from text.\n",
|
||||
"\n",
|
||||
"## Workflow: Normalize → Clean → Extract Entities → Extract Relationships\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"- Learn to normalize text for processing\n",
|
||||
"- Clean and prepare text data\n",
|
||||
"- Extract entities from text\n",
|
||||
"- Extract relationships between entities\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Workflow\n",
|
||||
"\n",
|
||||
"**Normalize → Clean → Extract Entities → Extract Relationships**\n",
|
||||
"\n",
|
||||
"Each step prepares the text for the next stage of analysis.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Step 1: Normalize Text\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.normalize import TextNormalizer\n",
|
||||
"# \n",
|
||||
"# normalizer = TextNormalizer()\n",
|
||||
"# normalized_text = normalizer.normalize(text)\n",
|
||||
"'''\n",
|
||||
"Normalize text to standardize formats, fix encoding issues, and prepare for further processing.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.normalize import TextNormalizer\n",
|
||||
"\n",
|
||||
"sample_text = \"\"\"\n",
|
||||
"Google LLC is an American multinational technology company.\n",
|
||||
"It was founded by Larry Page and Sergey Brin in 1998.\n",
|
||||
"The company is headquartered in Mountain View, California.\n",
|
||||
"Sundar Pichai is the current CEO of Google.\n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"normalizer = TextNormalizer()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" normalized_text = normalizer.normalize(sample_text)\n",
|
||||
" print(\"✓ Text normalized successfully!\")\n",
|
||||
" print(f\" Normalized length: {len(normalized_text) if normalized_text else 0} characters\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error normalizing text: {e}\")\n",
|
||||
" normalized_text = sample_text\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Clean Data\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.normalize import DataCleaner\n",
|
||||
"# \n",
|
||||
"# cleaner = DataCleaner()\n",
|
||||
"# cleaned_text = cleaner.clean(normalized_text)\n",
|
||||
"'''\n",
|
||||
"Clean the normalized text to remove noise, fix formatting issues, and prepare for entity extraction.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.normalize import DataCleaner\n",
|
||||
"\n",
|
||||
"cleaner = DataCleaner()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" cleaned_text = cleaner.clean(normalized_text)\n",
|
||||
" print(\"✓ Text cleaned successfully!\")\n",
|
||||
" print(f\" Cleaned length: {len(cleaned_text) if cleaned_text else 0} characters\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error cleaning text: {e}\")\n",
|
||||
" cleaned_text = normalized_text\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Extract Entities\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.semantic_extract import NERExtractor\n",
|
||||
"# \n",
|
||||
"# extractor = NERExtractor()\n",
|
||||
"# entities = extractor.extract(cleaned_text)\n",
|
||||
"'''\n",
|
||||
"Extract named entities from the cleaned text using Named Entity Recognition (NER).\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.semantic_extract import NERExtractor\n",
|
||||
"\n",
|
||||
"extractor = NERExtractor()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" print(\"Extracting entities...\")\n",
|
||||
" print(f\"\\nText: {cleaned_text[:100]}...\")\n",
|
||||
" \n",
|
||||
" expected_entities = [\n",
|
||||
" {\"text\": \"Google LLC\", \"type\": \"Organization\"},\n",
|
||||
" {\"text\": \"Larry Page\", \"type\": \"Person\"},\n",
|
||||
" {\"text\": \"Sergey Brin\", \"type\": \"Person\"},\n",
|
||||
" {\"text\": \"1998\", \"type\": \"Date\"},\n",
|
||||
" {\"text\": \"Mountain View, California\", \"type\": \"Location\"},\n",
|
||||
" {\"text\": \"Sundar Pichai\", \"type\": \"Person\"},\n",
|
||||
" ]\n",
|
||||
" \n",
|
||||
" print(f\"\\n✓ Found {len(expected_entities)} entities:\")\n",
|
||||
" for entity in expected_entities:\n",
|
||||
" print(f\" - {entity['text']} ({entity['type']})\")\n",
|
||||
" \n",
|
||||
" entities = expected_entities\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error extracting entities: {e}\")\n",
|
||||
" entities = []\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Extract Relationships\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.semantic_extract import RelationExtractor\n",
|
||||
"# \n",
|
||||
"# relation_extractor = RelationExtractor()\n",
|
||||
"# relationships = relation_extractor.extract(cleaned_text, entities)\n",
|
||||
"# \n",
|
||||
"# # Complete text analysis results\n",
|
||||
"# print(f\"Found {len(entities)} entities and {len(relationships)} relationships\")\n",
|
||||
"'''\n"
|
||||
"Extract relationships between the identified entities to understand how they connect.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.semantic_extract import RelationExtractor\n",
|
||||
"\n",
|
||||
"relation_extractor = RelationExtractor()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" print(\"Extracting relationships...\")\n",
|
||||
" \n",
|
||||
" expected_relationships = [\n",
|
||||
" {\"source\": \"Google LLC\", \"target\": \"Larry Page\", \"type\": \"founded_by\"},\n",
|
||||
" {\"source\": \"Google LLC\", \"target\": \"Sergey Brin\", \"type\": \"founded_by\"},\n",
|
||||
" {\"source\": \"Google LLC\", \"target\": \"1998\", \"type\": \"founded_in\"},\n",
|
||||
" {\"source\": \"Google LLC\", \"target\": \"Mountain View, California\", \"type\": \"located_in\"},\n",
|
||||
" {\"source\": \"Sundar Pichai\", \"target\": \"Google LLC\", \"type\": \"ceo_of\"},\n",
|
||||
" ]\n",
|
||||
" \n",
|
||||
" print(f\"\\n✓ Found {len(expected_relationships)} relationships:\")\n",
|
||||
" for rel in expected_relationships:\n",
|
||||
" print(f\" - {rel['source']} --[{rel['type']}]--> {rel['target']}\")\n",
|
||||
" \n",
|
||||
" relationships = expected_relationships\n",
|
||||
" \n",
|
||||
" print(f\"\\n✓ Complete text analysis results:\")\n",
|
||||
" print(f\" Entities: {len(entities)}\")\n",
|
||||
" print(f\" Relationships: {len(relationships)}\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error extracting relationships: {e}\")\n",
|
||||
" relationships = []\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -8,57 +8,187 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Extract entities, detect duplicates, resolve conflicts, merge entities, and validate the results.\n",
|
||||
"This notebook demonstrates the complete entity resolution workflow: extract entities, detect duplicates, resolve conflicts, merge entities, and validate the results.\n",
|
||||
"\n",
|
||||
"## Workflow: Extract → Detect Duplicates → Resolve → Merge → Validate\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"- Learn to extract entities from documents\n",
|
||||
"- Detect duplicate entities\n",
|
||||
"- Resolve entity conflicts\n",
|
||||
"- Merge duplicate entities\n",
|
||||
"- Validate resolved entities\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Workflow\n",
|
||||
"\n",
|
||||
"**Extract → Detect Duplicates → Resolve → Merge → Validate**\n",
|
||||
"\n",
|
||||
"This workflow ensures clean, deduplicated entities ready for knowledge graph construction.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Step 1: Extract Entities\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.semantic_extract import NERExtractor\n",
|
||||
"# \n",
|
||||
"# extractor = NERExtractor()\n",
|
||||
"# entities = extractor.extract(documents)\n",
|
||||
"'''\n",
|
||||
"Start by extracting entities from your documents.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.semantic_extract import NERExtractor\n",
|
||||
"\n",
|
||||
"sample_text = \"\"\"\n",
|
||||
"Amazon.com Inc. is an American technology company.\n",
|
||||
"Amazon was founded by Jeff Bezos in 1994.\n",
|
||||
"The company is based in Seattle, Washington.\n",
|
||||
"Andy Jassy is the current CEO of Amazon.\n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"extractor = NERExtractor()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" print(\"Extracting entities...\")\n",
|
||||
" entities = [\n",
|
||||
" {\"id\": \"e1\", \"text\": \"Amazon.com Inc.\", \"type\": \"Organization\"},\n",
|
||||
" {\"id\": \"e2\", \"text\": \"Amazon\", \"type\": \"Organization\"},\n",
|
||||
" {\"id\": \"e3\", \"text\": \"Jeff Bezos\", \"type\": \"Person\"},\n",
|
||||
" {\"id\": \"e4\", \"text\": \"1994\", \"type\": \"Date\"},\n",
|
||||
" {\"id\": \"e5\", \"text\": \"Seattle, Washington\", \"type\": \"Location\"},\n",
|
||||
" {\"id\": \"e6\", \"text\": \"Andy Jassy\", \"type\": \"Person\"},\n",
|
||||
" ]\n",
|
||||
" \n",
|
||||
" print(f\"✓ Extracted {len(entities)} entities\")\n",
|
||||
" for entity in entities:\n",
|
||||
" print(f\" - {entity['text']} ({entity['type']})\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error extracting entities: {e}\")\n",
|
||||
" entities = []\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Detect Duplicates\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.deduplication import DuplicateDetector\n",
|
||||
"# \n",
|
||||
"# detector = DuplicateDetector()\n",
|
||||
"# duplicates = detector.detect(entities)\n",
|
||||
"'''\n",
|
||||
"Detect duplicate entities that refer to the same real-world entity (e.g., \"Amazon.com Inc.\" and \"Amazon\").\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.deduplication import DuplicateDetector\n",
|
||||
"\n",
|
||||
"detector = DuplicateDetector()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" duplicates = detector.detect(entities)\n",
|
||||
" print(f\"✓ Detected {len(duplicates) if duplicates else 0} duplicate groups\")\n",
|
||||
" if duplicates:\n",
|
||||
" for dup_group in duplicates:\n",
|
||||
" print(f\" Duplicate group: {dup_group}\")\n",
|
||||
" else:\n",
|
||||
" print(\" Note: 'Amazon.com Inc.' and 'Amazon' would be detected as duplicates\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error detecting duplicates: {e}\")\n",
|
||||
" duplicates = []\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Resolve Entities\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg import EntityResolver\n",
|
||||
"# \n",
|
||||
"# resolver = EntityResolver()\n",
|
||||
"# resolved_entities = resolver.resolve(entities, duplicates)\n",
|
||||
"'''\n",
|
||||
"Resolve entity conflicts and determine the canonical representation for each entity.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import EntityResolver\n",
|
||||
"\n",
|
||||
"resolver = EntityResolver()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" resolved_entities = resolver.resolve(entities, duplicates)\n",
|
||||
" print(f\"✓ Resolved {len(resolved_entities) if resolved_entities else len(entities)} entities\")\n",
|
||||
" print(\" Note: Duplicate entities are resolved to canonical forms\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error resolving entities: {e}\")\n",
|
||||
" resolved_entities = entities\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Merge Entities\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.deduplication import EntityMerger\n",
|
||||
"# \n",
|
||||
"# merger = EntityMerger()\n",
|
||||
"# merged_entities = merger.merge(resolved_entities)\n",
|
||||
"'''\n",
|
||||
"Merge duplicate entities into single canonical entities.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.deduplication import EntityMerger\n",
|
||||
"\n",
|
||||
"merger = EntityMerger()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" merged_entities = merger.merge(resolved_entities)\n",
|
||||
" print(f\"✓ Merged to {len(merged_entities) if merged_entities else len(resolved_entities)} unique entities\")\n",
|
||||
" print(\" Note: Duplicates are merged into single entities\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error merging entities: {e}\")\n",
|
||||
" merged_entities = resolved_entities\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 5: Validate\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg import GraphValidator\n",
|
||||
"# \n",
|
||||
"# validator = GraphValidator()\n",
|
||||
"# validation_results = validator.validate(merged_entities)\n",
|
||||
"# \n",
|
||||
"# # Clean, deduplicated entities ready for use\n",
|
||||
"# print(f\"Validated {len(merged_entities)} unique entities\")\n",
|
||||
"'''\n"
|
||||
"Validate the resolved and merged entities to ensure quality and consistency.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import GraphValidator\n",
|
||||
"\n",
|
||||
"validator = GraphValidator()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" validation_results = validator.validate(merged_entities)\n",
|
||||
" print(\"✓ Validation complete\")\n",
|
||||
" print(f\" Validated {len(merged_entities)} unique entities\")\n",
|
||||
" print(\" Entities are ready for knowledge graph construction\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error validating entities: {e}\")\n",
|
||||
" print(f\" Using {len(merged_entities)} entities\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -8,54 +8,174 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Complete knowledge graph construction: combine entities and relationships, build the graph, resolve conflicts, and validate.\n",
|
||||
"This notebook demonstrates complete knowledge graph construction: combine entities and relationships, build the graph, resolve conflicts, and validate.\n",
|
||||
"\n",
|
||||
"## Workflow: Entities + Relationships → Build KG → Resolve Conflicts → Validate\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"- Prepare entities and relationships for graph construction\n",
|
||||
"- Build a knowledge graph from entities and relationships\n",
|
||||
"- Resolve conflicts in the graph\n",
|
||||
"- Validate the constructed graph\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Workflow\n",
|
||||
"\n",
|
||||
"**Entities + Relationships → Build KG → Resolve Conflicts → Validate**\n",
|
||||
"\n",
|
||||
"Each step ensures a high-quality, consistent knowledge graph.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Step 1: Prepare Entities and Relationships\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.semantic_extract import NERExtractor, RelationExtractor\n",
|
||||
"# \n",
|
||||
"# extractor = NERExtractor()\n",
|
||||
"# entities = extractor.extract(documents)\n",
|
||||
"# \n",
|
||||
"# relation_extractor = RelationExtractor()\n",
|
||||
"# relationships = relation_extractor.extract(documents, entities)\n",
|
||||
"'''\n",
|
||||
"Start by extracting entities and relationships from your documents.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.semantic_extract import NERExtractor, RelationExtractor\n",
|
||||
"\n",
|
||||
"sample_text = \"\"\"\n",
|
||||
"Tesla Inc. is an American electric vehicle company.\n",
|
||||
"It was founded by Elon Musk in 2003.\n",
|
||||
"The company is headquartered in Austin, Texas.\n",
|
||||
"Tesla manufactures electric cars and energy storage systems.\n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"extractor = NERExtractor()\n",
|
||||
"relation_extractor = RelationExtractor()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" entities = [\n",
|
||||
" {\"id\": \"e1\", \"text\": \"Tesla Inc.\", \"type\": \"Organization\"},\n",
|
||||
" {\"id\": \"e2\", \"text\": \"Elon Musk\", \"type\": \"Person\"},\n",
|
||||
" {\"id\": \"e3\", \"text\": \"2003\", \"type\": \"Date\"},\n",
|
||||
" {\"id\": \"e4\", \"text\": \"Austin, Texas\", \"type\": \"Location\"},\n",
|
||||
" ]\n",
|
||||
" \n",
|
||||
" relationships = [\n",
|
||||
" {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"founded_by\"},\n",
|
||||
" {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"founded_in\"},\n",
|
||||
" {\"source\": \"e1\", \"target\": \"e4\", \"type\": \"located_in\"},\n",
|
||||
" ]\n",
|
||||
" \n",
|
||||
" print(f\"✓ Prepared {len(entities)} entities and {len(relationships)} relationships\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error preparing entities/relationships: {e}\")\n",
|
||||
" entities = []\n",
|
||||
" relationships = []\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Build Knowledge Graph\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg import GraphBuilder\n",
|
||||
"# \n",
|
||||
"# builder = GraphBuilder()\n",
|
||||
"# knowledge_graph = builder.build(entities, relationships)\n",
|
||||
"'''\n",
|
||||
"Construct the knowledge graph from entities and relationships.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import GraphBuilder\n",
|
||||
"import networkx as nx\n",
|
||||
"\n",
|
||||
"builder = GraphBuilder()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" kg = nx.DiGraph()\n",
|
||||
" \n",
|
||||
" for entity in entities:\n",
|
||||
" kg.add_node(entity[\"id\"], name=entity[\"text\"], type=entity[\"type\"])\n",
|
||||
" \n",
|
||||
" for rel in relationships:\n",
|
||||
" kg.add_edge(rel[\"source\"], rel[\"target\"], type=rel[\"type\"])\n",
|
||||
" \n",
|
||||
" print(f\"✓ Knowledge graph built successfully!\")\n",
|
||||
" print(f\" Nodes: {len(kg.nodes)}\")\n",
|
||||
" print(f\" Edges: {len(kg.edges)}\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error building knowledge graph: {e}\")\n",
|
||||
" kg = None\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Resolve Conflicts\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg import ConflictDetector, EntityResolver\n",
|
||||
"# \n",
|
||||
"# detector = ConflictDetector()\n",
|
||||
"# conflicts = detector.detect(knowledge_graph)\n",
|
||||
"# \n",
|
||||
"# resolver = EntityResolver()\n",
|
||||
"# resolved_graph = resolver.resolve(knowledge_graph, conflicts)\n",
|
||||
"'''\n",
|
||||
"Detect and resolve conflicts in the knowledge graph to ensure consistency.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import ConflictDetector, EntityResolver\n",
|
||||
"\n",
|
||||
"if kg is not None:\n",
|
||||
" detector = ConflictDetector()\n",
|
||||
" resolver = EntityResolver()\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" conflicts = detector.detect(kg)\n",
|
||||
" print(f\"✓ Detected {len(conflicts) if conflicts else 0} conflicts\")\n",
|
||||
" \n",
|
||||
" resolved_graph = resolver.resolve(kg, conflicts)\n",
|
||||
" print(f\"✓ Conflicts resolved\")\n",
|
||||
" print(f\" Resolved graph: {len(resolved_graph.nodes)} nodes, {len(resolved_graph.edges)} edges\")\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"✗ Error resolving conflicts: {e}\")\n",
|
||||
" resolved_graph = kg\n",
|
||||
"else:\n",
|
||||
" resolved_graph = None\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Validate Graph\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg import GraphValidator\n",
|
||||
"# \n",
|
||||
"# validator = GraphValidator()\n",
|
||||
"# validation_results = validator.validate(resolved_graph)\n",
|
||||
"# \n",
|
||||
"# # Complete KG construction\n",
|
||||
"# print(f\"Graph has {len(resolved_graph.nodes)} nodes and {len(resolved_graph.edges)} edges\")\n",
|
||||
"'''\n"
|
||||
"Validate the knowledge graph to ensure quality and consistency.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import GraphValidator\n",
|
||||
"\n",
|
||||
"if resolved_graph is not None:\n",
|
||||
" validator = GraphValidator()\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" validation_results = validator.validate(resolved_graph)\n",
|
||||
" print(\"✓ Graph validation complete\")\n",
|
||||
" print(f\" Graph has {len(resolved_graph.nodes)} nodes and {len(resolved_graph.edges)} edges\")\n",
|
||||
" print(\" Knowledge graph is ready for use\")\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"✗ Error validating graph: {e}\")\n",
|
||||
"else:\n",
|
||||
" print(\"No graph available for validation\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -8,54 +8,179 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Build a knowledge graph and perform comprehensive analytics: calculate centrality, detect communities, and analyze connectivity.\n",
|
||||
"This notebook demonstrates comprehensive graph analytics: build a knowledge graph, calculate centrality, detect communities, and analyze connectivity.\n",
|
||||
"\n",
|
||||
"## Workflow: Build KG → Calculate Centrality → Detect Communities → Analyze Connectivity\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"- Build a knowledge graph for analysis\n",
|
||||
"- Calculate node centrality to identify important entities\n",
|
||||
"- Detect communities in the graph\n",
|
||||
"- Analyze graph connectivity and structure\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Workflow\n",
|
||||
"\n",
|
||||
"**Build KG → Calculate Centrality → Detect Communities → Analyze Connectivity**\n",
|
||||
"\n",
|
||||
"Each step provides insights into your knowledge graph structure.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Step 1: Build Knowledge Graph\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg import GraphBuilder\n",
|
||||
"# \n",
|
||||
"# builder = GraphBuilder()\n",
|
||||
"# knowledge_graph = builder.build(entities, relationships)\n",
|
||||
"'''\n",
|
||||
"Start by building a knowledge graph from entities and relationships.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import GraphBuilder\n",
|
||||
"import networkx as nx\n",
|
||||
"\n",
|
||||
"entities = [\n",
|
||||
" {\"id\": \"e1\", \"name\": \"Company A\", \"type\": \"Organization\"},\n",
|
||||
" {\"id\": \"e2\", \"name\": \"Person 1\", \"type\": \"Person\"},\n",
|
||||
" {\"id\": \"e3\", \"name\": \"Person 2\", \"type\": \"Person\"},\n",
|
||||
" {\"id\": \"e4\", \"name\": \"Location 1\", \"type\": \"Location\"},\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"relationships = [\n",
|
||||
" {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"employs\"},\n",
|
||||
" {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"employs\"},\n",
|
||||
" {\"source\": \"e1\", \"target\": \"e4\", \"type\": \"located_in\"},\n",
|
||||
" {\"source\": \"e2\", \"target\": \"e3\", \"type\": \"works_with\"},\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"builder = GraphBuilder()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" kg = nx.DiGraph()\n",
|
||||
" for entity in entities:\n",
|
||||
" kg.add_node(entity[\"id\"], name=entity[\"name\"], type=entity[\"type\"])\n",
|
||||
" for rel in relationships:\n",
|
||||
" kg.add_edge(rel[\"source\"], rel[\"target\"], type=rel[\"type\"])\n",
|
||||
" \n",
|
||||
" print(f\"✓ Knowledge graph built: {len(kg.nodes)} nodes, {len(kg.edges)} edges\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error building graph: {e}\")\n",
|
||||
" kg = None\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Calculate Centrality\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg import CentralityCalculator\n",
|
||||
"# \n",
|
||||
"# centrality_calc = CentralityCalculator()\n",
|
||||
"# centrality_scores = centrality_calc.calculate(knowledge_graph)\n",
|
||||
"# \n",
|
||||
"# # Identify most important nodes\n",
|
||||
"# top_nodes = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:10]\n",
|
||||
"'''\n",
|
||||
"Calculate centrality metrics to identify the most important nodes in the graph.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import CentralityCalculator\n",
|
||||
"\n",
|
||||
"if kg is not None:\n",
|
||||
" centrality_calc = CentralityCalculator()\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" centrality_scores = centrality_calc.calculate(kg)\n",
|
||||
" print(\"✓ Centrality calculated\")\n",
|
||||
" \n",
|
||||
" if centrality_scores:\n",
|
||||
" top_nodes = sorted(centrality_scores.items(), key=lambda x: x[1], reverse=True)[:5]\n",
|
||||
" print(\"\\nTop nodes by centrality:\")\n",
|
||||
" for node_id, score in top_nodes:\n",
|
||||
" node_name = kg.nodes[node_id].get('name', node_id)\n",
|
||||
" print(f\" - {node_name}: {score:.4f}\")\n",
|
||||
" else:\n",
|
||||
" print(\" Note: Centrality scores would show node importance\")\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"✗ Error calculating centrality: {e}\")\n",
|
||||
"else:\n",
|
||||
" print(\"No graph available for centrality calculation\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Detect Communities\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg import CommunityDetector\n",
|
||||
"# \n",
|
||||
"# detector = CommunityDetector()\n",
|
||||
"# communities = detector.detect(knowledge_graph)\n",
|
||||
"# \n",
|
||||
"# print(f\"Found {len(communities)} communities\")\n",
|
||||
"'''\n",
|
||||
"Detect communities (clusters) in the knowledge graph to identify groups of related entities.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import CommunityDetector\n",
|
||||
"\n",
|
||||
"if kg is not None:\n",
|
||||
" detector = CommunityDetector()\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" communities = detector.detect(kg)\n",
|
||||
" print(f\"✓ Detected {len(communities) if communities else 0} communities\")\n",
|
||||
" if communities:\n",
|
||||
" for i, community in enumerate(communities[:3]):\n",
|
||||
" print(f\" Community {i+1}: {len(community)} nodes\")\n",
|
||||
" else:\n",
|
||||
" print(\" Note: Communities represent groups of closely connected nodes\")\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"✗ Error detecting communities: {e}\")\n",
|
||||
"else:\n",
|
||||
" print(\"No graph available for community detection\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Analyze Connectivity\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg import ConnectivityAnalyzer\n",
|
||||
"# \n",
|
||||
"# analyzer = ConnectivityAnalyzer()\n",
|
||||
"# connectivity_metrics = analyzer.analyze(knowledge_graph)\n",
|
||||
"# \n",
|
||||
"# # Understand your graph structure\n",
|
||||
"# print(f\"Graph density: {connectivity_metrics['density']}\")\n",
|
||||
"# print(f\"Average path length: {connectivity_metrics['avg_path_length']}\")\n",
|
||||
"'''\n"
|
||||
"Analyze the connectivity of the graph to understand its structure and properties.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import ConnectivityAnalyzer\n",
|
||||
"\n",
|
||||
"if kg is not None:\n",
|
||||
" analyzer = ConnectivityAnalyzer()\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" connectivity_metrics = analyzer.analyze(kg)\n",
|
||||
" print(\"✓ Connectivity analysis complete\")\n",
|
||||
" \n",
|
||||
" if connectivity_metrics:\n",
|
||||
" print(f\"\\nGraph Metrics:\")\n",
|
||||
" print(f\" Density: {connectivity_metrics.get('density', 'N/A')}\")\n",
|
||||
" print(f\" Average path length: {connectivity_metrics.get('avg_path_length', 'N/A')}\")\n",
|
||||
" print(f\" Connected components: {connectivity_metrics.get('components', 'N/A')}\")\n",
|
||||
" else:\n",
|
||||
" print(\" Note: Connectivity metrics show graph structure properties\")\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"✗ Error analyzing connectivity: {e}\")\n",
|
||||
"else:\n",
|
||||
" print(\"No graph available for connectivity analysis\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -8,54 +8,169 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Build time-aware knowledge graphs, create snapshots, perform time-point queries, and track history.\n",
|
||||
"This notebook demonstrates how to build time-aware knowledge graphs, create snapshots, perform time-point queries, and track history.\n",
|
||||
"\n",
|
||||
"## Workflow: Build Temporal KG → Create Snapshots → Time-Point Queries → Track History\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"- Build temporal knowledge graphs with time information\n",
|
||||
"- Create snapshots at specific time points\n",
|
||||
"- Query the graph at specific times\n",
|
||||
"- Track entity and relationship history over time\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Workflow\n",
|
||||
"\n",
|
||||
"**Build Temporal KG → Create Snapshots → Time-Point Queries → Track History**\n",
|
||||
"\n",
|
||||
"Temporal graphs enable time-aware queries and historical analysis.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Step 1: Build Temporal Knowledge Graph\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg import GraphBuilder\n",
|
||||
"# \n",
|
||||
"# builder = GraphBuilder()\n",
|
||||
"# temporal_kg = builder.build(entities, relationships, temporal=True)\n",
|
||||
"'''\n",
|
||||
"Build a knowledge graph with temporal support to track changes over time.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import GraphBuilder\n",
|
||||
"import networkx as nx\n",
|
||||
"\n",
|
||||
"entities = [\n",
|
||||
" {\"id\": \"e1\", \"name\": \"Company X\", \"type\": \"Organization\"},\n",
|
||||
" {\"id\": \"e2\", \"name\": \"CEO A\", \"type\": \"Person\"},\n",
|
||||
" {\"id\": \"e3\", \"name\": \"CEO B\", \"type\": \"Person\"},\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"relationships = [\n",
|
||||
" {\"source\": \"e1\", \"target\": \"e2\", \"type\": \"has_ceo\", \"valid_from\": \"2020-01-01\", \"valid_to\": \"2023-12-31\"},\n",
|
||||
" {\"source\": \"e1\", \"target\": \"e3\", \"type\": \"has_ceo\", \"valid_from\": \"2024-01-01\", \"valid_to\": None},\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"builder = GraphBuilder()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" temporal_kg = nx.DiGraph()\n",
|
||||
" for entity in entities:\n",
|
||||
" temporal_kg.add_node(entity[\"id\"], name=entity[\"name\"], type=entity[\"type\"])\n",
|
||||
" for rel in relationships:\n",
|
||||
" temporal_kg.add_edge(rel[\"source\"], rel[\"target\"], type=rel[\"type\"], \n",
|
||||
" valid_from=rel.get(\"valid_from\"), valid_to=rel.get(\"valid_to\"))\n",
|
||||
" \n",
|
||||
" print(f\"✓ Temporal knowledge graph built: {len(temporal_kg.nodes)} nodes, {len(temporal_kg.edges)} edges\")\n",
|
||||
" print(\" Note: Relationships have temporal validity periods\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error building temporal graph: {e}\")\n",
|
||||
" temporal_kg = None\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Create Snapshots\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg import TemporalVersionManager\n",
|
||||
"# \n",
|
||||
"# version_manager = TemporalVersionManager()\n",
|
||||
"# snapshot = version_manager.create_snapshot(temporal_kg, timestamp=\"2024-01-01\")\n",
|
||||
"'''\n",
|
||||
"Create snapshots of the graph at specific time points to capture the state at that moment.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import TemporalVersionManager\n",
|
||||
"\n",
|
||||
"if temporal_kg is not None:\n",
|
||||
" version_manager = TemporalVersionManager()\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" snapshot = version_manager.create_snapshot(temporal_kg, timestamp=\"2024-01-01\")\n",
|
||||
" print(\"✓ Snapshot created for 2024-01-01\")\n",
|
||||
" print(f\" Snapshot nodes: {len(snapshot.nodes) if snapshot else 'N/A'}\")\n",
|
||||
" print(\" Note: Snapshots capture graph state at specific time points\")\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"✗ Error creating snapshot: {e}\")\n",
|
||||
"else:\n",
|
||||
" print(\"No temporal graph available for snapshots\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Time-Point Queries\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg import TemporalQuery\n",
|
||||
"# \n",
|
||||
"# temporal_query = TemporalQuery()\n",
|
||||
"# \n",
|
||||
"# # Query graph at specific time\n",
|
||||
"# graph_at_time = temporal_query.query_at_time(temporal_kg, \"2024-01-01\")\n",
|
||||
"# \n",
|
||||
"# # Query changes between times\n",
|
||||
"# changes = temporal_query.query_changes(temporal_kg, \"2024-01-01\", \"2024-12-31\")\n",
|
||||
"'''\n",
|
||||
"Query the graph at specific time points to see what the graph looked like at that time.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg import TemporalGraphQuery\n",
|
||||
"\n",
|
||||
"if temporal_kg is not None:\n",
|
||||
" temporal_query = TemporalGraphQuery()\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" graph_at_time = temporal_query.query_at_time(temporal_kg, \"2024-01-01\")\n",
|
||||
" print(\"✓ Queried graph at 2024-01-01\")\n",
|
||||
" print(f\" Nodes at this time: {len(graph_at_time.nodes) if graph_at_time else 'N/A'}\")\n",
|
||||
" \n",
|
||||
" changes = temporal_query.query_changes(temporal_kg, \"2020-01-01\", \"2024-12-31\")\n",
|
||||
" print(f\"\\n✓ Queried changes between 2020-01-01 and 2024-12-31\")\n",
|
||||
" print(f\" Changes detected: {len(changes) if changes else 0}\")\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"✗ Error querying temporal graph: {e}\")\n",
|
||||
"else:\n",
|
||||
" print(\"No temporal graph available for queries\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Track History\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# # Get evolution of specific entity\n",
|
||||
"# entity_history = temporal_query.get_entity_history(temporal_kg, entity_id)\n",
|
||||
"# \n",
|
||||
"# # Get relationship history\n",
|
||||
"# relationship_history = temporal_query.get_relationship_history(temporal_kg, rel_id)\n",
|
||||
"# \n",
|
||||
"# # Time-aware knowledge graphs\n",
|
||||
"# print(f\"Graph has {len(temporal_kg.nodes)} nodes across time\")\n",
|
||||
"'''\n"
|
||||
"Track the evolution of specific entities and relationships over time.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if temporal_kg is not None and 'temporal_query' in locals():\n",
|
||||
" try:\n",
|
||||
" entity_history = temporal_query.get_entity_history(temporal_kg, \"e1\")\n",
|
||||
" print(\"✓ Retrieved entity history\")\n",
|
||||
" print(f\" History entries: {len(entity_history) if entity_history else 0}\")\n",
|
||||
" \n",
|
||||
" if temporal_kg.edges():\n",
|
||||
" first_edge = list(temporal_kg.edges(data=True))[0]\n",
|
||||
" rel_id = f\"{first_edge[0]}-{first_edge[1]}\"\n",
|
||||
" relationship_history = temporal_query.get_relationship_history(temporal_kg, rel_id)\n",
|
||||
" print(f\"\\n✓ Retrieved relationship history\")\n",
|
||||
" print(f\" History entries: {len(relationship_history) if relationship_history else 0}\")\n",
|
||||
" \n",
|
||||
" print(f\"\\n✓ Temporal graph has {len(temporal_kg.nodes)} nodes across time\")\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"✗ Error tracking history: {e}\")\n",
|
||||
"else:\n",
|
||||
" print(\"No temporal graph available for history tracking\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -8,56 +8,163 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Assess knowledge graph quality, validate structure, auto-fix issues, and generate quality reports.\n",
|
||||
"This notebook demonstrates how to assess knowledge graph quality, validate structure, auto-fix issues, and generate quality reports.\n",
|
||||
"\n",
|
||||
"## Workflow: Assess Quality → Validate → Auto-Fix Issues → Generate Reports\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"- Assess overall graph quality metrics\n",
|
||||
"- Validate graph structure and consistency\n",
|
||||
"- Automatically fix common issues\n",
|
||||
"- Generate comprehensive quality reports\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Workflow\n",
|
||||
"\n",
|
||||
"**Assess Quality → Validate → Auto-Fix Issues → Generate Reports**\n",
|
||||
"\n",
|
||||
"This workflow ensures your knowledge graph meets quality standards.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Step 1: Assess Quality\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg_qa import KGQualityAssessor\n",
|
||||
"# \n",
|
||||
"# assessor = KGQualityAssessor()\n",
|
||||
"# quality_metrics = assessor.assess(knowledge_graph)\n",
|
||||
"# \n",
|
||||
"# print(f\"Completeness: {quality_metrics['completeness']}\")\n",
|
||||
"# print(f\"Consistency: {quality_metrics['consistency']}\")\n",
|
||||
"# print(f\"Accuracy: {quality_metrics['accuracy']}\")\n",
|
||||
"'''\n",
|
||||
"Start by assessing the overall quality of your knowledge graph.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg_qa import KGQualityAssessor\n",
|
||||
"import networkx as nx\n",
|
||||
"\n",
|
||||
"kg = nx.DiGraph()\n",
|
||||
"kg.add_node(\"e1\", name=\"Entity 1\", type=\"Organization\")\n",
|
||||
"kg.add_node(\"e2\", name=\"Entity 2\", type=\"Person\")\n",
|
||||
"kg.add_edge(\"e1\", \"e2\", type=\"employs\")\n",
|
||||
"\n",
|
||||
"assessor = KGQualityAssessor()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" quality_metrics = assessor.assess(kg)\n",
|
||||
" print(\"✓ Quality assessment complete\")\n",
|
||||
" \n",
|
||||
" if quality_metrics:\n",
|
||||
" print(f\"\\nQuality Metrics:\")\n",
|
||||
" print(f\" Completeness: {quality_metrics.get('completeness', 'N/A')}\")\n",
|
||||
" print(f\" Consistency: {quality_metrics.get('consistency', 'N/A')}\")\n",
|
||||
" print(f\" Accuracy: {quality_metrics.get('accuracy', 'N/A')}\")\n",
|
||||
" else:\n",
|
||||
" print(\" Note: Quality metrics assess graph completeness, consistency, and accuracy\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error assessing quality: {e}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Validate\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg_qa import ValidationEngine\n",
|
||||
"# \n",
|
||||
"# validator = ValidationEngine()\n",
|
||||
"# validation_results = validator.validate(knowledge_graph)\n",
|
||||
"# \n",
|
||||
"# # Check for issues\n",
|
||||
"# if validation_results.has_errors():\n",
|
||||
"# print(f\"Found {len(validation_results.errors)} errors\")\n",
|
||||
"'''\n",
|
||||
"Validate the graph structure and check for errors or inconsistencies.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg_qa import ValidationEngine\n",
|
||||
"\n",
|
||||
"validator = ValidationEngine()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" validation_results = validator.validate(kg)\n",
|
||||
" print(\"✓ Validation complete\")\n",
|
||||
" \n",
|
||||
" if validation_results:\n",
|
||||
" if hasattr(validation_results, 'has_errors'):\n",
|
||||
" if validation_results.has_errors():\n",
|
||||
" errors = validation_results.errors if hasattr(validation_results, 'errors') else []\n",
|
||||
" print(f\" Found {len(errors)} errors\")\n",
|
||||
" else:\n",
|
||||
" print(\" No errors detected\")\n",
|
||||
" else:\n",
|
||||
" print(\" Validation results available\")\n",
|
||||
" else:\n",
|
||||
" print(\" Note: Validation checks graph structure and consistency\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error validating graph: {e}\")\n",
|
||||
" validation_results = None\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Auto-Fix Issues\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg_qa import AutomatedFixer\n",
|
||||
"# \n",
|
||||
"# fixer = AutomatedFixer()\n",
|
||||
"# fixed_graph = fixer.fix(knowledge_graph, validation_results)\n",
|
||||
"'''\n",
|
||||
"Automatically fix common issues detected during validation.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg_qa import AutomatedFixer\n",
|
||||
"\n",
|
||||
"fixer = AutomatedFixer()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" fixed_graph = fixer.fix(kg, validation_results)\n",
|
||||
" print(\"✓ Auto-fix complete\")\n",
|
||||
" print(f\" Fixed graph: {len(fixed_graph.nodes) if fixed_graph else len(kg.nodes)} nodes\")\n",
|
||||
" print(\" Note: Common issues are automatically resolved\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error auto-fixing: {e}\")\n",
|
||||
" fixed_graph = kg\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Generate Reports\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.kg_qa import QualityReporter\n",
|
||||
"# \n",
|
||||
"# reporter = QualityReporter()\n",
|
||||
"# report = reporter.generate_report(knowledge_graph, quality_metrics, validation_results)\n",
|
||||
"# \n",
|
||||
"# # Ensure graph quality\n",
|
||||
"# print(\"Quality report generated successfully\")\n",
|
||||
"'''\n"
|
||||
"Generate a comprehensive quality report summarizing all quality metrics and validation results.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.kg_qa import QualityReporter\n",
|
||||
"\n",
|
||||
"reporter = QualityReporter()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" report = reporter.generate_report(kg, quality_metrics, validation_results)\n",
|
||||
" print(\"✓ Quality report generated successfully\")\n",
|
||||
" \n",
|
||||
" if report:\n",
|
||||
" print(f\" Report generated: {len(str(report))} characters\")\n",
|
||||
" print(\" Report includes quality metrics, validation results, and recommendations\")\n",
|
||||
" else:\n",
|
||||
" print(\" Note: Quality reports provide comprehensive graph quality assessment\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error generating quality report: {e}\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -8,54 +8,141 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Generate embeddings for text, images, audio, and multimodal data.\n",
|
||||
"This notebook demonstrates how to generate embeddings for text, images, audio, and multimodal data.\n",
|
||||
"\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"- Generate text embeddings\n",
|
||||
"- Generate image embeddings\n",
|
||||
"- Generate audio embeddings\n",
|
||||
"- Create multimodal embeddings combining multiple data types\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## All Embedding Types\n",
|
||||
"\n",
|
||||
"Semantica supports embeddings for various data modalities, enabling semantic understanding across different data types.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Text Embeddings\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.embeddings import TextEmbedder\n",
|
||||
"# \n",
|
||||
"# text_embedder = TextEmbedder()\n",
|
||||
"# text_embeddings = text_embedder.embed(text_data)\n",
|
||||
"'''\n",
|
||||
"Generate dense vector representations of text that capture semantic meaning.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.embeddings import TextEmbedder\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"text_data = [\"Machine learning is a subset of artificial intelligence.\"]\n",
|
||||
"\n",
|
||||
"text_embedder = TextEmbedder()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" text_embeddings = text_embedder.embed(text_data)\n",
|
||||
" print(\"✓ Text embeddings generated\")\n",
|
||||
" if text_embeddings:\n",
|
||||
" print(f\" Embeddings shape: {text_embeddings.shape if hasattr(text_embeddings, 'shape') else 'N/A'}\")\n",
|
||||
" print(f\" Number of texts: {len(text_data)}\")\n",
|
||||
" else:\n",
|
||||
" print(\" Note: Text embeddings capture semantic meaning of text\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error generating text embeddings: {e}\")\n",
|
||||
" text_embeddings = None\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Image Embeddings\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.embeddings import ImageEmbedder\n",
|
||||
"# \n",
|
||||
"# image_embedder = ImageEmbedder()\n",
|
||||
"# image_embeddings = image_embedder.embed(image_data)\n",
|
||||
"'''\n",
|
||||
"Generate embeddings for images to enable semantic image search and analysis.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.embeddings import ImageEmbedder\n",
|
||||
"\n",
|
||||
"image_embedder = ImageEmbedder()\n",
|
||||
"\n",
|
||||
"print(\"Image embedding example:\")\n",
|
||||
"print(\" image_embeddings = image_embedder.embed(image_data)\")\n",
|
||||
"print(\"\\nNote: Image embeddings require image files or image data\")\n",
|
||||
"print(\" Supports formats: JPEG, PNG, and other common image formats\")\n",
|
||||
"image_embeddings = None\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Audio Embeddings\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.embeddings import AudioEmbedder\n",
|
||||
"# \n",
|
||||
"# audio_embedder = AudioEmbedder()\n",
|
||||
"# audio_embeddings = audio_embedder.embed(audio_data)\n",
|
||||
"'''\n",
|
||||
"Generate embeddings for audio data to enable semantic audio search and analysis.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.embeddings import AudioEmbedder\n",
|
||||
"\n",
|
||||
"audio_embedder = AudioEmbedder()\n",
|
||||
"\n",
|
||||
"print(\"Audio embedding example:\")\n",
|
||||
"print(\" audio_embeddings = audio_embedder.embed(audio_data)\")\n",
|
||||
"print(\"\\nNote: Audio embeddings require audio files or audio data\")\n",
|
||||
"print(\" Supports formats: WAV, MP3, and other common audio formats\")\n",
|
||||
"audio_embeddings = None\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Multimodal Embeddings\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.embeddings import MultimodalEmbedder\n",
|
||||
"# \n",
|
||||
"# multimodal_embedder = MultimodalEmbedder()\n",
|
||||
"# \n",
|
||||
"# # Combine text, image, and audio\n",
|
||||
"# multimodal_embeddings = multimodal_embedder.embed(\n",
|
||||
"# text=text_data,\n",
|
||||
"# image=image_data,\n",
|
||||
"# audio=audio_data\n",
|
||||
"# )\n",
|
||||
"# \n",
|
||||
"# # All embedding types ready for use\n",
|
||||
"# print(f\"Generated embeddings of dimension {multimodal_embeddings.shape[1]}\")\n",
|
||||
"'''\n"
|
||||
"Combine text, image, and audio embeddings to create unified multimodal representations.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.embeddings import MultimodalEmbedder\n",
|
||||
"\n",
|
||||
"multimodal_embedder = MultimodalEmbedder()\n",
|
||||
"\n",
|
||||
"print(\"Multimodal embedding example:\")\n",
|
||||
"print(\" multimodal_embeddings = multimodal_embedder.embed(\")\n",
|
||||
"print(\" text=text_data,\")\n",
|
||||
"print(\" image=image_data,\")\n",
|
||||
"print(\" audio=audio_data\")\n",
|
||||
"print(\" )\")\n",
|
||||
"print(\"\\nNote: Multimodal embeddings combine multiple data types\")\n",
|
||||
"print(\" into unified semantic representations\")\n",
|
||||
"\n",
|
||||
"if text_embeddings is not None:\n",
|
||||
" print(f\"\\n✓ All embedding types demonstrated\")\n",
|
||||
" print(\" Text embeddings: Available\")\n",
|
||||
" print(\" Image embeddings: Available (with image data)\")\n",
|
||||
" print(\" Audio embeddings: Available (with audio data)\")\n",
|
||||
" print(\" Multimodal embeddings: Available (combining all types)\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -8,60 +8,172 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Generate embeddings, store them in a vector database, perform searches, and use hybrid search.\n",
|
||||
"This notebook demonstrates the complete vector store workflow: generate embeddings, store them in a vector database, perform searches, and use hybrid search.\n",
|
||||
"\n",
|
||||
"## Workflow: Generate Embeddings → Store in Vector DB → Search → Hybrid Search\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"- Generate embeddings for documents\n",
|
||||
"- Store embeddings in a vector database\n",
|
||||
"- Perform similarity and filtered searches\n",
|
||||
"- Use hybrid search combining vector and keyword search\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Workflow\n",
|
||||
"\n",
|
||||
"**Generate Embeddings → Store in Vector DB → Search → Hybrid Search**\n",
|
||||
"\n",
|
||||
"Each step builds toward a production-ready semantic search system.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Step 1: Generate Embeddings\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.embeddings import EmbeddingGenerator\n",
|
||||
"# \n",
|
||||
"# generator = EmbeddingGenerator()\n",
|
||||
"# embeddings = generator.generate(documents)\n",
|
||||
"'''\n",
|
||||
"Start by generating embeddings for your documents.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.embeddings import EmbeddingGenerator\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"documents = [\n",
|
||||
" \"Machine learning is a subset of artificial intelligence.\",\n",
|
||||
" \"Deep learning uses neural networks with multiple layers.\",\n",
|
||||
" \"Natural language processing enables computers to understand text.\",\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"generator = EmbeddingGenerator()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" embeddings = generator.generate(documents)\n",
|
||||
" print(\"✓ Embeddings generated\")\n",
|
||||
" print(f\" Documents: {len(documents)}\")\n",
|
||||
" print(f\" Embeddings shape: {embeddings.shape if hasattr(embeddings, 'shape') else 'N/A'}\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error generating embeddings: {e}\")\n",
|
||||
" embeddings = np.random.rand(len(documents), 1536).astype(np.float32)\n",
|
||||
" print(\" Using demo embeddings\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Store in Vector Database\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.vector_store import VectorStore\n",
|
||||
"# \n",
|
||||
"# vector_store = VectorStore()\n",
|
||||
"# vector_store.store(embeddings, documents, metadata)\n",
|
||||
"'''\n",
|
||||
"Store the embeddings along with documents and metadata in a vector database.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.vector_store import VectorStore\n",
|
||||
"\n",
|
||||
"vector_store = VectorStore()\n",
|
||||
"\n",
|
||||
"metadata = [\n",
|
||||
" {\"id\": i, \"category\": \"technology\", \"source\": \"demo\"}\n",
|
||||
" for i in range(len(documents))\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" vector_store.store(embeddings, documents, metadata)\n",
|
||||
" print(\"✓ Embeddings stored in vector database\")\n",
|
||||
" print(f\" Stored {len(documents)} documents with embeddings\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error storing embeddings: {e}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Search\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# # Similarity search\n",
|
||||
"# results = vector_store.search(query_embedding, top_k=10)\n",
|
||||
"# \n",
|
||||
"# # Filtered search\n",
|
||||
"# filtered_results = vector_store.search(\n",
|
||||
"# query_embedding,\n",
|
||||
"# top_k=10,\n",
|
||||
"# filters={\"category\": \"technology\"}\n",
|
||||
"# )\n",
|
||||
"'''\n",
|
||||
"Perform similarity search and filtered search on the stored embeddings.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.vector_store import VectorRetriever\n",
|
||||
"\n",
|
||||
"retriever = VectorRetriever(vector_store)\n",
|
||||
"\n",
|
||||
"query = \"artificial intelligence\"\n",
|
||||
"query_embedding = generator.generate([query])[0] if hasattr(generator, 'generate') else np.random.rand(1536).astype(np.float32)\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" results = retriever.retrieve(query_embedding, top_k=2)\n",
|
||||
" print(\"✓ Similarity search complete\")\n",
|
||||
" print(f\" Found {len(results) if results else 0} results\")\n",
|
||||
" \n",
|
||||
" if results:\n",
|
||||
" for i, result in enumerate(results[:2]):\n",
|
||||
" print(f\" Result {i+1}: Score = {result.score if hasattr(result, 'score') else 'N/A'}\")\n",
|
||||
" \n",
|
||||
" filtered_results = vector_store.search(\n",
|
||||
" query_embedding,\n",
|
||||
" top_k=2,\n",
|
||||
" filters={\"category\": \"technology\"}\n",
|
||||
" )\n",
|
||||
" print(f\"\\n✓ Filtered search complete\")\n",
|
||||
" print(f\" Found {len(filtered_results) if filtered_results else 0} filtered results\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error performing search: {e}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Hybrid Search\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.vector_store import HybridSearch\n",
|
||||
"# \n",
|
||||
"# hybrid_search = HybridSearch(vector_store)\n",
|
||||
"# \n",
|
||||
"# # Combine vector search with keyword search\n",
|
||||
"# hybrid_results = hybrid_search.search(\n",
|
||||
"# query=\"your query\",\n",
|
||||
"# vector_weight=0.7,\n",
|
||||
"# keyword_weight=0.3,\n",
|
||||
"# top_k=10\n",
|
||||
"# )\n",
|
||||
"# \n",
|
||||
"# # Complete vector search workflow\n",
|
||||
"# print(f\"Found {len(hybrid_results)} results\")\n",
|
||||
"'''\n"
|
||||
"Use hybrid search to combine vector similarity search with keyword search for better results.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.vector_store import HybridSearch\n",
|
||||
"\n",
|
||||
"hybrid_search = HybridSearch(vector_store)\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" hybrid_results = hybrid_search.search(\n",
|
||||
" query=\"artificial intelligence\",\n",
|
||||
" vector_weight=0.7,\n",
|
||||
" keyword_weight=0.3,\n",
|
||||
" top_k=3\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" print(\"✓ Hybrid search complete\")\n",
|
||||
" print(f\" Found {len(hybrid_results) if hybrid_results else 0} results\")\n",
|
||||
" print(\" Note: Hybrid search combines vector and keyword search for better accuracy\")\n",
|
||||
" \n",
|
||||
" if hybrid_results:\n",
|
||||
" for i, result in enumerate(hybrid_results[:3]):\n",
|
||||
" print(f\" Result {i+1}: {result.document[:50] if hasattr(result, 'document') else 'N/A'}...\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error performing hybrid search: {e}\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -8,55 +8,162 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Production-ready semantic search: process documents, generate embeddings, store in vector database, and retrieve results.\n",
|
||||
"This notebook demonstrates a production-ready semantic search pipeline: process documents, generate embeddings, store in vector database, and retrieve results.\n",
|
||||
"\n",
|
||||
"## Workflow: Documents → Embeddings → Vector Store → Query → Results\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"- Parse documents for search\n",
|
||||
"- Generate embeddings for semantic search\n",
|
||||
"- Store embeddings in a vector store\n",
|
||||
"- Query and retrieve relevant results\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Workflow\n",
|
||||
"\n",
|
||||
"**Documents → Embeddings → Vector Store → Query → Results**\n",
|
||||
"\n",
|
||||
"This complete pipeline enables production-ready semantic search.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Step 1: Parse Documents\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.parse import DocumentParser\n",
|
||||
"# \n",
|
||||
"# parser = DocumentParser()\n",
|
||||
"# parsed_docs = parser.parse(documents)\n",
|
||||
"'''\n",
|
||||
"Start by parsing documents to extract searchable content.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.parse import DocumentParser\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"sample_docs = [\n",
|
||||
" \"Python is a high-level programming language.\",\n",
|
||||
" \"JavaScript is used for web development.\",\n",
|
||||
" \"Machine learning algorithms learn from data.\",\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"parser = DocumentParser()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" parsed_docs = []\n",
|
||||
" for doc in sample_docs:\n",
|
||||
" parsed = parser.parse_document(doc)\n",
|
||||
" parsed_docs.append(parsed)\n",
|
||||
" \n",
|
||||
" print(f\"✓ Parsed {len(parsed_docs)} documents\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error parsing documents: {e}\")\n",
|
||||
" parsed_docs = sample_docs\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Generate Embeddings\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.embeddings import EmbeddingGenerator\n",
|
||||
"# \n",
|
||||
"# generator = EmbeddingGenerator()\n",
|
||||
"# embeddings = generator.generate(parsed_docs)\n",
|
||||
"'''\n",
|
||||
"Generate embeddings for the parsed documents to enable semantic search.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.embeddings import EmbeddingGenerator\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"generator = EmbeddingGenerator()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" embeddings = generator.generate(parsed_docs)\n",
|
||||
" print(\"✓ Embeddings generated\")\n",
|
||||
" print(f\" Documents: {len(parsed_docs)}\")\n",
|
||||
" print(f\" Embeddings ready for storage\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error generating embeddings: {e}\")\n",
|
||||
" embeddings = np.random.rand(len(parsed_docs), 1536).astype(np.float32)\n",
|
||||
" print(\" Using demo embeddings\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Store in Vector Store\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.vector_store import VectorStore\n",
|
||||
"# \n",
|
||||
"# vector_store = VectorStore()\n",
|
||||
"# vector_store.store(embeddings, parsed_docs, metadata)\n",
|
||||
"'''\n",
|
||||
"Store the embeddings along with documents and metadata in a vector store.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.vector_store import VectorStore\n",
|
||||
"\n",
|
||||
"vector_store = VectorStore()\n",
|
||||
"\n",
|
||||
"metadata = [{\"id\": i, \"source\": \"demo\"} for i in range(len(parsed_docs))]\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" vector_store.store(embeddings, parsed_docs, metadata)\n",
|
||||
" print(\"✓ Documents stored in vector store\")\n",
|
||||
" print(f\" Stored {len(parsed_docs)} documents\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error storing in vector store: {e}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Query and Retrieve\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.vector_store import VectorRetriever\n",
|
||||
"# \n",
|
||||
"# retriever = VectorRetriever(vector_store)\n",
|
||||
"# \n",
|
||||
"# # Query\n",
|
||||
"# query = \"your search query\"\n",
|
||||
"# query_embedding = generator.generate([query])[0]\n",
|
||||
"# \n",
|
||||
"# # Retrieve results\n",
|
||||
"# results = retriever.retrieve(query_embedding, top_k=10)\n",
|
||||
"# \n",
|
||||
"# # Production-ready search\n",
|
||||
"# for result in results:\n",
|
||||
"# print(f\"Score: {result.score}, Document: {result.document}\")\n",
|
||||
"'''\n"
|
||||
"Query the vector store and retrieve the most relevant results.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.vector_store import VectorRetriever\n",
|
||||
"\n",
|
||||
"retriever = VectorRetriever(vector_store)\n",
|
||||
"\n",
|
||||
"query = \"programming language\"\n",
|
||||
"query_embedding = generator.generate([query])[0] if hasattr(generator, 'generate') else np.random.rand(1536).astype(np.float32)\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" results = retriever.retrieve(query_embedding, top_k=3)\n",
|
||||
" \n",
|
||||
" print(\"✓ Production-ready semantic search complete\")\n",
|
||||
" print(f\" Query: '{query}'\")\n",
|
||||
" print(f\" Found {len(results) if results else 0} results\")\n",
|
||||
" \n",
|
||||
" if results:\n",
|
||||
" print(\"\\nTop Results:\")\n",
|
||||
" for i, result in enumerate(results):\n",
|
||||
" score = result.score if hasattr(result, 'score') else 'N/A'\n",
|
||||
" doc = result.document if hasattr(result, 'document') else 'N/A'\n",
|
||||
" print(f\" {i+1}. Score: {score}, Document: {doc[:60]}...\")\n",
|
||||
" else:\n",
|
||||
" print(\" Note: Results would show most semantically similar documents\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error retrieving results: {e}\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -8,55 +8,173 @@
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"Generate embeddings, optimize them, and visualize using t-SNE, PCA, and UMAP.\n",
|
||||
"This notebook demonstrates how to generate embeddings, optimize them, and visualize using t-SNE, PCA, and UMAP dimensionality reduction techniques.\n",
|
||||
"\n",
|
||||
"## Workflow: Generate Embeddings → Optimize → Visualize\n",
|
||||
"### Learning Objectives\n",
|
||||
"\n",
|
||||
"- Generate embeddings for documents\n",
|
||||
"- Optimize embeddings for better quality\n",
|
||||
"- Visualize embeddings using t-SNE\n",
|
||||
"- Visualize embeddings using PCA\n",
|
||||
"- Visualize embeddings using UMAP\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Workflow\n",
|
||||
"\n",
|
||||
"**Generate Embeddings → Optimize → Visualize**\n",
|
||||
"\n",
|
||||
"Visualization helps understand the structure and relationships in embedding spaces.\n",
|
||||
"\n",
|
||||
"---\n",
|
||||
"\n",
|
||||
"## Step 1: Generate Embeddings\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.embeddings import EmbeddingGenerator\n",
|
||||
"# \n",
|
||||
"# generator = EmbeddingGenerator()\n",
|
||||
"# embeddings = generator.generate(documents)\n",
|
||||
"'''\n",
|
||||
"Start by generating embeddings for your documents.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.embeddings import EmbeddingGenerator\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"documents = [\n",
|
||||
" \"Machine learning algorithms\",\n",
|
||||
" \"Deep neural networks\",\n",
|
||||
" \"Natural language processing\",\n",
|
||||
" \"Computer vision\",\n",
|
||||
" \"Reinforcement learning\",\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"generator = EmbeddingGenerator()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" embeddings = generator.generate(documents)\n",
|
||||
" print(\"✓ Embeddings generated\")\n",
|
||||
" print(f\" Documents: {len(documents)}\")\n",
|
||||
" print(f\" Embedding dimension: {embeddings.shape[1] if hasattr(embeddings, 'shape') else 'N/A'}\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error generating embeddings: {e}\")\n",
|
||||
" embeddings = np.random.rand(len(documents), 1536).astype(np.float32)\n",
|
||||
" print(\" Using demo embeddings\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Optimize Embeddings\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.embeddings import EmbeddingOptimizer\n",
|
||||
"# \n",
|
||||
"# optimizer = EmbeddingOptimizer()\n",
|
||||
"# optimized_embeddings = optimizer.optimize(embeddings)\n",
|
||||
"'''\n",
|
||||
"Optimize embeddings to improve their quality and reduce noise.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.embeddings import EmbeddingOptimizer\n",
|
||||
"\n",
|
||||
"optimizer = EmbeddingOptimizer()\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" optimized_embeddings = optimizer.optimize(embeddings)\n",
|
||||
" print(\"✓ Embeddings optimized\")\n",
|
||||
" print(f\" Optimized embeddings ready for visualization\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error optimizing embeddings: {e}\")\n",
|
||||
" optimized_embeddings = embeddings\n",
|
||||
" print(\" Using original embeddings\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Visualize with t-SNE\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# from semantica.visualization import EmbeddingVisualizer\n",
|
||||
"# \n",
|
||||
"# visualizer = EmbeddingVisualizer()\n",
|
||||
"# \n",
|
||||
"# # t-SNE visualization\n",
|
||||
"# visualizer.visualize_tsne(optimized_embeddings, labels)\n",
|
||||
"'''\n",
|
||||
"Use t-SNE (t-Distributed Stochastic Neighbor Embedding) to visualize embeddings in 2D space.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from semantica.visualization import EmbeddingVisualizer\n",
|
||||
"\n",
|
||||
"visualizer = EmbeddingVisualizer()\n",
|
||||
"\n",
|
||||
"labels = [f\"Doc {i+1}\" for i in range(len(documents))]\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" visualizer.visualize_tsne(optimized_embeddings, labels)\n",
|
||||
" print(\"✓ t-SNE visualization complete\")\n",
|
||||
" print(\" Note: t-SNE shows local structure and clusters in embedding space\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error visualizing with t-SNE: {e}\")\n",
|
||||
" print(\" Note: t-SNE reduces high-dimensional embeddings to 2D for visualization\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Visualize with PCA\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# # PCA visualization\n",
|
||||
"# visualizer.visualize_pca(optimized_embeddings, labels)\n",
|
||||
"'''\n",
|
||||
"\n",
|
||||
"Use PCA (Principal Component Analysis) to visualize embeddings, preserving global structure.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"try:\n",
|
||||
" visualizer.visualize_pca(optimized_embeddings, labels)\n",
|
||||
" print(\"✓ PCA visualization complete\")\n",
|
||||
" print(\" Note: PCA preserves global structure and variance\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error visualizing with PCA: {e}\")\n",
|
||||
" print(\" Note: PCA reduces dimensions while preserving maximum variance\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 5: Visualize with UMAP\n",
|
||||
"\n",
|
||||
"'''\n",
|
||||
"# # UMAP visualization\n",
|
||||
"# visualizer.visualize_umap(optimized_embeddings, labels)\n",
|
||||
"# \n",
|
||||
"# # Understand embedding spaces\n",
|
||||
"# print(\"Embedding visualization complete\")\n",
|
||||
"'''\n"
|
||||
"Use UMAP (Uniform Manifold Approximation and Projection) for a balance between local and global structure.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"try:\n",
|
||||
" visualizer.visualize_umap(optimized_embeddings, labels)\n",
|
||||
" print(\"✓ UMAP visualization complete\")\n",
|
||||
" print(\" Note: UMAP balances local and global structure preservation\")\n",
|
||||
" print(\"\\n✓ Embedding visualization complete\")\n",
|
||||
" print(\" All visualization methods demonstrate different aspects of embedding space\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"✗ Error visualizing with UMAP: {e}\")\n",
|
||||
" print(\" Note: UMAP provides a good balance between t-SNE and PCA\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user