mirror of
https://github.com/semantica-agi/semantica.git
synced 2026-08-29 04:26:20 +00:00
fix: rename contributing/license pages to avoid Mintlify reserved slug conflict
mint export fails with 'file does not exist' for pages named 'contributing' and 'license' — these are reserved by Mintlify's GitHub integration layer. Renamed to contributing-guide.md and project-license.md and updated all nav entries and cross-links throughout the docs. Also adds .gitattributes LF rules to prevent CRLF issues from Windows devs.
This commit is contained in:
KaifAhmad1
@@ -1,3 +1,11 @@
|
|||||||
|
# Line endings — force LF so Mintlify/Linux CI parses frontmatter correctly
|
||||||
|
* text=auto eol=lf
|
||||||
|
*.md text eol=lf
|
||||||
|
*.json text eol=lf
|
||||||
|
*.yml text eol=lf
|
||||||
|
*.yaml text eol=lf
|
||||||
|
*.py text eol=lf
|
||||||
|
|
||||||
# Linguist documentation and generated files
|
# Linguist documentation and generated files
|
||||||
# This ensures GitHub language statistics reflect the core Python code
|
# This ensures GitHub language statistics reflect the core Python code
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -62,7 +62,7 @@ Published research using Semantica? [Let us know](https://github.com/semantica-a
|
|||||||
## See Also
|
## See Also
|
||||||
|
|
||||||
<CardGroup cols={2}>
|
<CardGroup cols={2}>
|
||||||
<Card title="License" icon="file-contract" href="license">
|
<Card title="License" icon="file-contract" href="project-license">
|
||||||
MIT License details.
|
MIT License details.
|
||||||
</Card>
|
</Card>
|
||||||
<Card title="Community" icon="users" href="community">
|
<Card title="Community" icon="users" href="community">
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ Have a project using Semantica? [Submit it on GitHub](https://github.com/semanti
|
|||||||
See the [Contributing Guide](contributing) for full details.
|
See the [Contributing Guide](contributing) for full details.
|
||||||
|
|
||||||
<CardGroup cols={2}>
|
<CardGroup cols={2}>
|
||||||
<Card title="Contributing" icon="code-pull-request" href="contributing">
|
<Card title="Contributing" icon="code-pull-request" href="contributing-guide">
|
||||||
Submit code, docs, or tests.
|
Submit code, docs, or tests.
|
||||||
</Card>
|
</Card>
|
||||||
<Card title="GitHub Issues" icon="circle-dot" href="https://github.com/semantica-agi/semantica/issues">
|
<Card title="GitHub Issues" icon="circle-dot" href="https://github.com/semantica-agi/semantica/issues">
|
||||||
|
|||||||
+1
-1
@@ -61,7 +61,7 @@ See the [Contributing Guide](contributing) for full details.
|
|||||||
## See Also
|
## See Also
|
||||||
|
|
||||||
<CardGroup cols={2}>
|
<CardGroup cols={2}>
|
||||||
<Card title="Contributing" icon="code-pull-request" href="contributing">
|
<Card title="Contributing" icon="code-pull-request" href="contributing-guide">
|
||||||
Step-by-step guide for submitting PRs.
|
Step-by-step guide for submitting PRs.
|
||||||
</Card>
|
</Card>
|
||||||
<Card title="FAQ" icon="circle-question" href="faq">
|
<Card title="FAQ" icon="circle-question" href="faq">
|
||||||
|
|||||||
+2
-2
@@ -194,10 +194,10 @@
|
|||||||
"pages": [
|
"pages": [
|
||||||
"community",
|
"community",
|
||||||
"community-projects",
|
"community-projects",
|
||||||
"contributing",
|
"contributing-guide",
|
||||||
"governance",
|
"governance",
|
||||||
"citation",
|
"citation",
|
||||||
"license"
|
"project-license"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
+1
-1
@@ -235,7 +235,7 @@ pip install --upgrade semantica
|
|||||||
<Card title="GitHub Issues" icon="github" href="https://github.com/semantica-agi/semantica/issues">
|
<Card title="GitHub Issues" icon="github" href="https://github.com/semantica-agi/semantica/issues">
|
||||||
Bug reports and feature requests.
|
Bug reports and feature requests.
|
||||||
</Card>
|
</Card>
|
||||||
<Card title="Contributing" icon="code-pull-request" href="contributing">
|
<Card title="Contributing" icon="code-pull-request" href="contributing-guide">
|
||||||
Help improve Semantica.
|
Help improve Semantica.
|
||||||
</Card>
|
</Card>
|
||||||
</CardGroup>
|
</CardGroup>
|
||||||
|
|||||||
+1
-1
@@ -98,7 +98,7 @@ MIT License — see [LICENSE](https://github.com/semantica-agi/semantica/blob/ma
|
|||||||
## See Also
|
## See Also
|
||||||
|
|
||||||
<CardGroup cols={2}>
|
<CardGroup cols={2}>
|
||||||
<Card title="Contributing" icon="code-pull-request" href="contributing">
|
<Card title="Contributing" icon="code-pull-request" href="contributing-guide">
|
||||||
How to submit changes.
|
How to submit changes.
|
||||||
</Card>
|
</Card>
|
||||||
<Card title="Community" icon="users" href="community">
|
<Card title="Community" icon="users" href="community">
|
||||||
|
|||||||
@@ -83,7 +83,7 @@ By contributing to Semantica, you agree that your contributions will be licensed
|
|||||||
## See Also
|
## See Also
|
||||||
|
|
||||||
<CardGroup cols={2}>
|
<CardGroup cols={2}>
|
||||||
<Card title="Contributing" icon="code-pull-request" href="contributing">
|
<Card title="Contributing" icon="code-pull-request" href="contributing-guide">
|
||||||
How to contribute to the project.
|
How to contribute to the project.
|
||||||
</Card>
|
</Card>
|
||||||
<Card title="Citation" icon="quote-left" href="citation">
|
<Card title="Citation" icon="quote-left" href="citation">
|
||||||
+213
-213
@@ -1,213 +1,213 @@
|
|||||||
"""
|
"""
|
||||||
Apache Parquet Exporter - Example Usage
|
Apache Parquet Exporter - Example Usage
|
||||||
|
|
||||||
This script demonstrates how to use the ParquetExporter to export
|
This script demonstrates how to use the ParquetExporter to export
|
||||||
knowledge graphs, entities, and relationships to Apache Parquet format.
|
knowledge graphs, entities, and relationships to Apache Parquet format.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from semantica.export import ParquetExporter, export_parquet
|
from semantica.export import ParquetExporter, export_parquet
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
print("=" * 70)
|
print("=" * 70)
|
||||||
print("Apache Parquet Exporter - Example Usage")
|
print("Apache Parquet Exporter - Example Usage")
|
||||||
print("=" * 70)
|
print("=" * 70)
|
||||||
|
|
||||||
# Create a temporary directory for outputs
|
# Create a temporary directory for outputs
|
||||||
temp_dir = Path(tempfile.mkdtemp())
|
temp_dir = Path(tempfile.mkdtemp())
|
||||||
print(f"\n📁 Output directory: {temp_dir}\n")
|
print(f"\n📁 Output directory: {temp_dir}\n")
|
||||||
|
|
||||||
# Sample data
|
# Sample data
|
||||||
entities = [
|
entities = [
|
||||||
{
|
{
|
||||||
"id": "e1",
|
"id": "e1",
|
||||||
"text": "Alice",
|
"text": "Alice",
|
||||||
"type": "Person",
|
"type": "Person",
|
||||||
"confidence": 0.95,
|
"confidence": 0.95,
|
||||||
"start": 0,
|
"start": 0,
|
||||||
"end": 5,
|
"end": 5,
|
||||||
"metadata": {"age": 30, "city": "New York"},
|
"metadata": {"age": 30, "city": "New York"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "e2",
|
"id": "e2",
|
||||||
"text": "Acme Corp",
|
"text": "Acme Corp",
|
||||||
"type": "Organization",
|
"type": "Organization",
|
||||||
"confidence": 0.88,
|
"confidence": 0.88,
|
||||||
"start": 10,
|
"start": 10,
|
||||||
"end": 19,
|
"end": 19,
|
||||||
"metadata": {"location": "NY", "employees": 100},
|
"metadata": {"location": "NY", "employees": 100},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "e3",
|
"id": "e3",
|
||||||
"text": "Bob",
|
"text": "Bob",
|
||||||
"type": "Person",
|
"type": "Person",
|
||||||
"confidence": 0.92,
|
"confidence": 0.92,
|
||||||
"metadata": {"age": 35, "department": "Engineering"},
|
"metadata": {"age": 35, "department": "Engineering"},
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
relationships = [
|
relationships = [
|
||||||
{
|
{
|
||||||
"id": "r1",
|
"id": "r1",
|
||||||
"source_id": "e1",
|
"source_id": "e1",
|
||||||
"target_id": "e2",
|
"target_id": "e2",
|
||||||
"type": "WORKS_FOR",
|
"type": "WORKS_FOR",
|
||||||
"confidence": 0.90,
|
"confidence": 0.90,
|
||||||
"metadata": {"role": "Engineer", "since": 2020},
|
"metadata": {"role": "Engineer", "since": 2020},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "r2",
|
"id": "r2",
|
||||||
"source_id": "e3",
|
"source_id": "e3",
|
||||||
"target_id": "e2",
|
"target_id": "e2",
|
||||||
"type": "WORKS_FOR",
|
"type": "WORKS_FOR",
|
||||||
"confidence": 0.85,
|
"confidence": 0.85,
|
||||||
"metadata": {"role": "Manager", "since": 2018},
|
"metadata": {"role": "Manager", "since": 2018},
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
knowledge_graph = {
|
knowledge_graph = {
|
||||||
"entities": entities,
|
"entities": entities,
|
||||||
"relationships": relationships,
|
"relationships": relationships,
|
||||||
"metadata": {"version": "1.0", "created": "2024-01-01"},
|
"metadata": {"version": "1.0", "created": "2024-01-01"},
|
||||||
}
|
}
|
||||||
|
|
||||||
# Example 1: Export entities using ParquetExporter class
|
# Example 1: Export entities using ParquetExporter class
|
||||||
print("Example 1: Export entities to Parquet")
|
print("Example 1: Export entities to Parquet")
|
||||||
print("-" * 70)
|
print("-" * 70)
|
||||||
exporter = ParquetExporter(compression="snappy")
|
exporter = ParquetExporter(compression="snappy")
|
||||||
entities_path = temp_dir / "entities.parquet"
|
entities_path = temp_dir / "entities.parquet"
|
||||||
exporter.export_entities(entities, entities_path)
|
exporter.export_entities(entities, entities_path)
|
||||||
print(f"✓ Entities exported to: {entities_path}")
|
print(f"✓ Entities exported to: {entities_path}")
|
||||||
print(f" File size: {entities_path.stat().st_size} bytes\n")
|
print(f" File size: {entities_path.stat().st_size} bytes\n")
|
||||||
|
|
||||||
# Example 2: Export relationships
|
# Example 2: Export relationships
|
||||||
print("Example 2: Export relationships to Parquet")
|
print("Example 2: Export relationships to Parquet")
|
||||||
print("-" * 70)
|
print("-" * 70)
|
||||||
rels_path = temp_dir / "relationships.parquet"
|
rels_path = temp_dir / "relationships.parquet"
|
||||||
exporter.export_relationships(relationships, rels_path)
|
exporter.export_relationships(relationships, rels_path)
|
||||||
print(f"✓ Relationships exported to: {rels_path}")
|
print(f"✓ Relationships exported to: {rels_path}")
|
||||||
print(f" File size: {rels_path.stat().st_size} bytes\n")
|
print(f" File size: {rels_path.stat().st_size} bytes\n")
|
||||||
|
|
||||||
# Example 3: Export complete knowledge graph
|
# Example 3: Export complete knowledge graph
|
||||||
print("Example 3: Export knowledge graph to multiple Parquet files")
|
print("Example 3: Export knowledge graph to multiple Parquet files")
|
||||||
print("-" * 70)
|
print("-" * 70)
|
||||||
kg_base_path = temp_dir / "knowledge_graph"
|
kg_base_path = temp_dir / "knowledge_graph"
|
||||||
exporter.export_knowledge_graph(knowledge_graph, kg_base_path)
|
exporter.export_knowledge_graph(knowledge_graph, kg_base_path)
|
||||||
kg_entities = temp_dir / "knowledge_graph_entities.parquet"
|
kg_entities = temp_dir / "knowledge_graph_entities.parquet"
|
||||||
kg_rels = temp_dir / "knowledge_graph_relationships.parquet"
|
kg_rels = temp_dir / "knowledge_graph_relationships.parquet"
|
||||||
print("✓ Knowledge graph exported to:")
|
print("✓ Knowledge graph exported to:")
|
||||||
print(f" - {kg_entities} ({kg_entities.stat().st_size} bytes)")
|
print(f" - {kg_entities} ({kg_entities.stat().st_size} bytes)")
|
||||||
print(f" - {kg_rels} ({kg_rels.stat().st_size} bytes)\n")
|
print(f" - {kg_rels} ({kg_rels.stat().st_size} bytes)\n")
|
||||||
|
|
||||||
# Example 4: Using convenience function
|
# Example 4: Using convenience function
|
||||||
print("Example 4: Using export_parquet convenience function")
|
print("Example 4: Using export_parquet convenience function")
|
||||||
print("-" * 70)
|
print("-" * 70)
|
||||||
conv_path = temp_dir / "convenience_export.parquet"
|
conv_path = temp_dir / "convenience_export.parquet"
|
||||||
export_parquet(entities, conv_path, compression="gzip")
|
export_parquet(entities, conv_path, compression="gzip")
|
||||||
print(f"✓ Exported using convenience function: {conv_path}")
|
print(f"✓ Exported using convenience function: {conv_path}")
|
||||||
print(f" File size: {conv_path.stat().st_size} bytes\n")
|
print(f" File size: {conv_path.stat().st_size} bytes\n")
|
||||||
|
|
||||||
# Example 5: Different compression codecs
|
# Example 5: Different compression codecs
|
||||||
print("Example 5: Compare compression codecs")
|
print("Example 5: Compare compression codecs")
|
||||||
print("-" * 70)
|
print("-" * 70)
|
||||||
|
|
||||||
# Create larger dataset for meaningful comparison
|
# Create larger dataset for meaningful comparison
|
||||||
large_entities = entities * 50
|
large_entities = entities * 50
|
||||||
|
|
||||||
compression_codecs = ["snappy", "gzip", "brotli", "zstd", "lz4", "none"]
|
compression_codecs = ["snappy", "gzip", "brotli", "zstd", "lz4", "none"]
|
||||||
sizes = {}
|
sizes = {}
|
||||||
|
|
||||||
for codec in compression_codecs:
|
for codec in compression_codecs:
|
||||||
codec_exporter = ParquetExporter(compression=codec)
|
codec_exporter = ParquetExporter(compression=codec)
|
||||||
codec_path = temp_dir / f"entities_{codec}.parquet"
|
codec_path = temp_dir / f"entities_{codec}.parquet"
|
||||||
codec_exporter.export_entities(large_entities, codec_path)
|
codec_exporter.export_entities(large_entities, codec_path)
|
||||||
sizes[codec] = codec_path.stat().st_size
|
sizes[codec] = codec_path.stat().st_size
|
||||||
print(f" {codec:8} - {sizes[codec]:,} bytes")
|
print(f" {codec:8} - {sizes[codec]:,} bytes")
|
||||||
|
|
||||||
print()
|
print()
|
||||||
|
|
||||||
# Example 6: Load Parquet with pandas (if available)
|
# Example 6: Load Parquet with pandas (if available)
|
||||||
print("Example 6: Loading Parquet files with pandas")
|
print("Example 6: Loading Parquet files with pandas")
|
||||||
print("-" * 70)
|
print("-" * 70)
|
||||||
try:
|
try:
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
|
|
||||||
df = pd.read_parquet(entities_path)
|
df = pd.read_parquet(entities_path)
|
||||||
print("✓ Loaded entities as pandas DataFrame")
|
print("✓ Loaded entities as pandas DataFrame")
|
||||||
print(f" Shape: {df.shape}")
|
print(f" Shape: {df.shape}")
|
||||||
print(f" Columns: {list(df.columns)}")
|
print(f" Columns: {list(df.columns)}")
|
||||||
print("\nFirst few rows:")
|
print("\nFirst few rows:")
|
||||||
print(df.head())
|
print(df.head())
|
||||||
print()
|
print()
|
||||||
|
|
||||||
except ImportError:
|
except ImportError:
|
||||||
print("⚠ pandas not installed - skipping pandas example\n")
|
print("⚠ pandas not installed - skipping pandas example\n")
|
||||||
|
|
||||||
# Example 7: Load Parquet with pyarrow
|
# Example 7: Load Parquet with pyarrow
|
||||||
print("Example 7: Loading Parquet files with pyarrow")
|
print("Example 7: Loading Parquet files with pyarrow")
|
||||||
print("-" * 70)
|
print("-" * 70)
|
||||||
try:
|
try:
|
||||||
import pyarrow.parquet as pq
|
import pyarrow.parquet as pq
|
||||||
|
|
||||||
table = pq.read_table(entities_path)
|
table = pq.read_table(entities_path)
|
||||||
print("✓ Loaded entities as Arrow Table")
|
print("✓ Loaded entities as Arrow Table")
|
||||||
print(f" Rows: {table.num_rows}")
|
print(f" Rows: {table.num_rows}")
|
||||||
print(f" Columns: {table.num_columns}")
|
print(f" Columns: {table.num_columns}")
|
||||||
print(" Schema:")
|
print(" Schema:")
|
||||||
for i, field in enumerate(table.schema):
|
for i, field in enumerate(table.schema):
|
||||||
print(f" - {field.name}: {field.type}")
|
print(f" - {field.name}: {field.type}")
|
||||||
print()
|
print()
|
||||||
|
|
||||||
except ImportError:
|
except ImportError:
|
||||||
print("⚠ pyarrow not installed - skipping pyarrow example\n")
|
print("⚠ pyarrow not installed - skipping pyarrow example\n")
|
||||||
|
|
||||||
# Example 8: Schema validation
|
# Example 8: Schema validation
|
||||||
print("Example 8: Explicit schema validation")
|
print("Example 8: Explicit schema validation")
|
||||||
print("-" * 70)
|
print("-" * 70)
|
||||||
try:
|
try:
|
||||||
import pyarrow.parquet as pq
|
import pyarrow.parquet as pq
|
||||||
|
|
||||||
# Read parquet file and verify schema
|
# Read parquet file and verify schema
|
||||||
table = pq.read_table(entities_path)
|
table = pq.read_table(entities_path)
|
||||||
|
|
||||||
print("✓ Schema validation:")
|
print("✓ Schema validation:")
|
||||||
print(f" - ID column type: {table.schema.field('id').type}")
|
print(f" - ID column type: {table.schema.field('id').type}")
|
||||||
print(f" - Text column type: {table.schema.field('text').type}")
|
print(f" - Text column type: {table.schema.field('text').type}")
|
||||||
print(f" - Confidence column type: {table.schema.field('confidence').type}")
|
print(f" - Confidence column type: {table.schema.field('confidence').type}")
|
||||||
print(f" - Metadata column type: {table.schema.field('metadata').type}")
|
print(f" - Metadata column type: {table.schema.field('metadata').type}")
|
||||||
print()
|
print()
|
||||||
|
|
||||||
# Verify metadata structure
|
# Verify metadata structure
|
||||||
metadata_field = table.schema.field("metadata")
|
metadata_field = table.schema.field("metadata")
|
||||||
print(" Metadata structure:")
|
print(" Metadata structure:")
|
||||||
if hasattr(metadata_field.type, "num_fields"):
|
if hasattr(metadata_field.type, "num_fields"):
|
||||||
for i in range(metadata_field.type.num_fields):
|
for i in range(metadata_field.type.num_fields):
|
||||||
subfield = metadata_field.type.field(i)
|
subfield = metadata_field.type.field(i)
|
||||||
print(f" - {subfield.name}: {subfield.type}")
|
print(f" - {subfield.name}: {subfield.type}")
|
||||||
print()
|
print()
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"⚠ Schema validation error: {e}\n")
|
print(f"⚠ Schema validation error: {e}\n")
|
||||||
|
|
||||||
# Summary
|
# Summary
|
||||||
print("=" * 70)
|
print("=" * 70)
|
||||||
print("Summary")
|
print("Summary")
|
||||||
print("=" * 70)
|
print("=" * 70)
|
||||||
print("✓ All examples completed successfully")
|
print("✓ All examples completed successfully")
|
||||||
print(f"✓ Output directory: {temp_dir}")
|
print(f"✓ Output directory: {temp_dir}")
|
||||||
print(f"✓ Files created: {len(list(temp_dir.glob('*.parquet')))}")
|
print(f"✓ Files created: {len(list(temp_dir.glob('*.parquet')))}")
|
||||||
print("\nKey Features:")
|
print("\nKey Features:")
|
||||||
print(" - Columnar storage optimized for analytics")
|
print(" - Columnar storage optimized for analytics")
|
||||||
print(" - Multiple compression options (snappy, gzip, brotli, zstd, lz4)")
|
print(" - Multiple compression options (snappy, gzip, brotli, zstd, lz4)")
|
||||||
print(" - Compatible with pandas, Spark, Snowflake, BigQuery, Databricks")
|
print(" - Compatible with pandas, Spark, Snowflake, BigQuery, Databricks")
|
||||||
print(" - Explicit schemas for type safety")
|
print(" - Explicit schemas for type safety")
|
||||||
print(" - Structured metadata handling")
|
print(" - Structured metadata handling")
|
||||||
print("\nFor more information, see the Semantica documentation.")
|
print("\nFor more information, see the Semantica documentation.")
|
||||||
print("=" * 70)
|
print("=" * 70)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+538
-538
File diff suppressed because it is too large
Load Diff
+546
-546
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user