From 727285171a73c5b20cd3e9a6764877fcc997bedb Mon Sep 17 00:00:00 2001 From: KaifAhmad1 Date: Fri, 21 Nov 2025 21:57:10 +0530 Subject: [PATCH] docs: Modern documentation redesign with enhanced navigation - Reorganize navigation structure (left sidebar: Home, Quickstart, Installation, Cookbook Recipes, Learning More, Deep Dive, API References) - Add TOC sections on homepage (Features, How to Read this Documentation, Resources) - Create new pages: learning-more.md, deep-dive.md, community-projects.md, citation.md, license.md - Add Mermaid diagrams for architecture, workflows, and concepts - Enhance all documentation pages with better code examples and explanations - Add custom CSS for modern aesthetic (docs/css/custom.css) - Update mkdocs.yml with Material theme, Mermaid support, and enhanced features - Improve user-friendly navigation and clear visual hierarchy - Add diagrams and charts throughout documentation --- .github/workflows/docs-mkdocs.yml | 50 ++++ .github/workflows/docs-netlify.yml | 52 ++++ .github/workflows/docs-vercel.yml | 46 +++ docs/TESTING.md | 53 ++++ docs/api.md | 296 +++++++++++++++--- docs/citation.md | 68 +++++ docs/community-projects.md | 138 +++++++++ docs/concepts.md | 161 ++++++++++ docs/cookbook.md | 182 ++++++++++++ docs/css/custom.css | 118 ++++++++ docs/deep-dive.md | 283 ++++++++++++++++++ docs/examples.md | 217 +++++++++++++- docs/faq.md | 211 +++++++++++++ docs/getting-started.md | 85 ++++++ docs/index.md | 463 ++++++++++++++++++++++++----- docs/installation.md | 200 +++++++++++-- docs/learning-more.md | 173 +++++++++++ docs/license.md | 91 ++++++ docs/netlify.toml | 12 + docs/quickstart.md | 217 ++++++++++++-- docs/vercel.json | 12 + mkdocs.yml | 103 +++++++ netlify.toml | 12 + requirements-docs.txt | 7 + test-docs.py | 63 ++++ 25 files changed, 3152 insertions(+), 161 deletions(-) create mode 100644 .github/workflows/docs-mkdocs.yml create mode 100644 .github/workflows/docs-netlify.yml create mode 100644 .github/workflows/docs-vercel.yml create mode 100644 docs/TESTING.md create mode 100644 docs/citation.md create mode 100644 docs/community-projects.md create mode 100644 docs/concepts.md create mode 100644 docs/cookbook.md create mode 100644 docs/css/custom.css create mode 100644 docs/deep-dive.md create mode 100644 docs/faq.md create mode 100644 docs/getting-started.md create mode 100644 docs/learning-more.md create mode 100644 docs/license.md create mode 100644 docs/netlify.toml create mode 100644 docs/vercel.json create mode 100644 mkdocs.yml create mode 100644 netlify.toml create mode 100644 requirements-docs.txt create mode 100644 test-docs.py diff --git a/.github/workflows/docs-mkdocs.yml b/.github/workflows/docs-mkdocs.yml new file mode 100644 index 00000000..bb4b32f5 --- /dev/null +++ b/.github/workflows/docs-mkdocs.yml @@ -0,0 +1,50 @@ +name: Deploy Documentation + +on: + push: + branches: + - main + paths: + - 'docs/**' + - 'mkdocs.yml' + - '.github/workflows/docs-mkdocs.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + pip install -r requirements-docs.txt + + - name: Build documentation + run: | + mkdocs build + + - name: Deploy to Netlify + uses: nwtgck/actions-netlify@v3.0 + with: + publish-dir: './site' + production-branch: main + github-token: ${{ secrets.GITHUB_TOKEN }} + deploy-message: "Deploy docs from GitHub Actions" + enable-pull-request-comment: false + enable-commit-comment: true + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + timeout-minutes: 1 + diff --git a/.github/workflows/docs-netlify.yml b/.github/workflows/docs-netlify.yml new file mode 100644 index 00000000..3aa0498d --- /dev/null +++ b/.github/workflows/docs-netlify.yml @@ -0,0 +1,52 @@ +name: Deploy Documentation to Netlify + +on: + push: + branches: + - main + paths: + - 'docs/**' + - '.github/workflows/docs-netlify.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.1' + bundler-cache: true + + - name: Install Jekyll + run: | + cd docs + bundle install + + - name: Build site + run: | + cd docs + bundle exec jekyll build -d ../_site + + - name: Deploy to Netlify + uses: nwtgck/actions-netlify@v3.0 + with: + publish-dir: './_site' + production-branch: main + github-token: ${{ secrets.GITHUB_TOKEN }} + deploy-message: "Deploy from GitHub Actions" + enable-pull-request-comment: false + enable-commit-comment: true + overwrites-pull-request-comment: true + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + timeout-minutes: 1 + diff --git a/.github/workflows/docs-vercel.yml b/.github/workflows/docs-vercel.yml new file mode 100644 index 00000000..43fe4804 --- /dev/null +++ b/.github/workflows/docs-vercel.yml @@ -0,0 +1,46 @@ +name: Deploy Documentation to Vercel + +on: + push: + branches: + - main + paths: + - 'docs/**' + - '.github/workflows/docs-vercel.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.1' + bundler-cache: true + + - name: Install Jekyll + run: | + cd docs + bundle install + + - name: Build site + run: | + cd docs + bundle exec jekyll build -d ../_site + + - name: Deploy to Vercel + uses: amondnet/vercel-action@v25 + with: + vercel-token: ${{ secrets.VERCEL_TOKEN }} + vercel-org-id: ${{ secrets.VERCEL_ORG_ID }} + vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }} + working-directory: ./ + scope: ${{ secrets.VERCEL_ORG_ID }} + diff --git a/docs/TESTING.md b/docs/TESTING.md new file mode 100644 index 00000000..dbc2a8e7 --- /dev/null +++ b/docs/TESTING.md @@ -0,0 +1,53 @@ +# Testing Documentation Locally + +## Quick Start + +### Install Dependencies + +```bash +pip install -r requirements-docs.txt +``` + +### Run Local Server + +```bash +mkdocs serve +``` + +Then visit: `http://127.0.0.1:8000` + +### Build Static Site + +```bash +mkdocs build +``` + +Output will be in the `site/` directory. + +## Development Workflow + +1. Make changes to `.md` files in `docs/` +2. Run `mkdocs serve` to preview +3. Check changes in browser +4. Commit and push when ready + +## Troubleshooting + +### Port Already in Use + +```bash +mkdocs serve -a 127.0.0.1:8001 +``` + +### Clear Cache + +```bash +mkdocs build --clean +``` + +### Check Configuration + +```bash +mkdocs build --verbose +``` + diff --git a/docs/api.md b/docs/api.md index 05a7267b..6bdc1ac4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,5 +1,7 @@ # API Reference +Complete API documentation for Semantica. + ## Core Classes ### Semantica @@ -12,58 +14,282 @@ from semantica import Semantica semantica = Semantica(config=None) ``` +#### Methods + +##### `build_knowledge_base(sources, **kwargs)` + +Build a knowledge base from one or more data sources. + +**Parameters:** +- `sources` (List[str] | str): Data source(s) - file paths or URLs +- `embeddings` (bool): Generate embeddings (default: True) +- `graph` (bool): Build knowledge graph (default: True) +- `normalize` (bool): Normalize data (default: True) + +**Returns:** +- `Dict[str, Any]`: Dictionary containing: + - `knowledge_graph`: Knowledge graph data + - `embeddings`: Embedding vectors + - `metadata`: Processing metadata + - `statistics`: Processing statistics + +**Example:** +```python +result = semantica.build_knowledge_base( + sources=["document.pdf"], + embeddings=True, + graph=True +) +kg = result["knowledge_graph"] +``` + +##### `process_document(source)` + +Process a single document. + +**Parameters:** +- `source` (str): File path or URL + +**Returns:** +- `Dict[str, Any]`: Processed document data + +##### `extract_entities(text)` + +Extract entities from text. + +**Parameters:** +- `text` (str): Input text + +**Returns:** +- `Dict[str, List]`: Dictionary with `entities` list + +##### `extract_relationships(text)` + +Extract relationships from text. + +**Parameters:** +- `text` (str): Input text + +**Returns:** +- `Dict[str, List]`: Dictionary with `relationships` list + +--- + +## Knowledge Graph Module + +### `semantica.kg` + +Knowledge graph construction and analysis. + +#### Methods + +##### `build_graph(sources)` + +Build a knowledge graph from sources. + +```python +kg = semantica.kg.build_graph(["document.pdf"]) +``` + +##### `analyze(graph)` + +Analyze a knowledge graph. + +```python +analysis = semantica.kg.analyze(kg) +print(analysis["statistics"]) +``` + +##### `visualize(graph, output_path=None)` + +Visualize a knowledge graph. + +```python +semantica.kg.visualize(kg, output_path="graph.html") +``` + +##### `merge(graphs)` + +Merge multiple knowledge graphs. + +```python +merged = semantica.kg.merge([kg1, kg2, kg3]) +``` + +--- + +## Semantic Extraction Module + +### `semantica.semantic_extract` + +Entity and relationship extraction. + +#### Methods + +##### `extract_entities(text)` + +Extract named entities from text. + +```python +result = semantica.semantic_extract.extract_entities(text) +entities = result["entities"] +``` + +##### `extract_relationships(text)` + +Extract relationships from text. + +```python +result = semantica.semantic_extract.extract_relationships(text) +relationships = result["relationships"] +``` + +##### `extract_triples(text)` + +Extract subject-predicate-object triples. + +```python +result = semantica.semantic_extract.extract_triples(text) +triples = result["triples"] +``` + +--- + +## Embeddings Module + +### `semantica.embeddings` + +Embedding generation and management. + +#### Methods + +##### `generate(text)` + +Generate embedding for a single text. + +```python +embedding = semantica.embeddings.generate("Your text here") +``` + +##### `generate_batch(texts)` + +Generate embeddings for multiple texts. + +```python +texts = ["text1", "text2", "text3"] +embeddings = semantica.embeddings.generate_batch(texts) +``` + +--- + +## Export Module + +### `semantica.export` + +Export knowledge graphs to various formats. + +#### Methods + +##### `to_rdf(kg, path)` + +Export to RDF format. + +```python +semantica.export.to_rdf(kg, "output.rdf") +``` + +##### `to_json(kg, path)` + +Export to JSON format. + +```python +semantica.export.to_json(kg, "output.json") +``` + +##### `to_csv(kg, path)` + +Export to CSV format. + +```python +semantica.export.to_csv(kg, "output.csv") +``` + +##### `to_owl(kg, path)` + +Export to OWL ontology format. + +```python +semantica.export.to_owl(kg, "output.owl") +``` + +##### `to_yaml(kg, path)` + +Export to YAML format. + +```python +semantica.export.to_yaml(kg, "output.yaml") +``` + +--- + +## Conflict Resolution Module + +### `semantica.conflicts` + +Conflict detection and resolution. + +#### Classes + +##### `ConflictResolver` + +```python +from semantica.conflicts import ConflictResolver + +resolver = ConflictResolver(default_strategy="voting") +``` + **Methods:** -- `build_knowledge_base(sources, **kwargs)` - Build knowledge base from sources -- `process_document(source)` - Process a single document -- `extract_entities(text)` - Extract entities from text -- `extract_relationships(text)` - Extract relationships from text - -## Modules - -### Knowledge Graph (`semantica.kg`) +- `resolve_conflicts(conflicts)`: Resolve multiple conflicts +- `resolve_conflict(conflict, strategy=None)`: Resolve a single conflict +- `set_resolution_rule(property, strategy)`: Set custom resolution rules +**Example:** ```python -semantica.kg.build_graph(sources) -semantica.kg.analyze(graph) -semantica.kg.visualize(graph) +resolver = ConflictResolver(default_strategy="voting") +resolved = resolver.resolve_conflicts(conflicts) ``` -### Semantic Extraction (`semantica.semantic_extract`) - -```python -semantica.semantic_extract.extract_entities(text) -semantica.semantic_extract.extract_relationships(text) -semantica.semantic_extract.extract_triples(text) -``` - -### Embeddings (`semantica.embeddings`) - -```python -semantica.embeddings.generate(text) -semantica.embeddings.generate_batch(texts) -``` - -### Export (`semantica.export`) - -```python -semantica.export.to_rdf(kg, path) -semantica.export.to_json(kg, path) -semantica.export.to_csv(kg, path) -``` +--- ## Configuration +### `Config` + +Configuration class for Semantica. + ```python from semantica import Config config = Config( embeddings=True, graph=True, - normalize=True + normalize=True, + conflict_resolution="voting" ) semantica = Semantica(config=config) ``` -For full API documentation, see [MODULES_DOCUMENTATION.md](../MODULES_DOCUMENTATION.md) +**Parameters:** +- `embeddings` (bool): Enable embedding generation +- `graph` (bool): Enable knowledge graph construction +- `normalize` (bool): Enable data normalization +- `conflict_resolution` (str): Default conflict resolution strategy +--- + +## Full Documentation + +For complete module documentation, see: +- [MODULES_DOCUMENTATION.md](../MODULES_DOCUMENTATION.md) - Detailed module documentation +- [GitHub Repository](https://github.com/Hawksight-AI/semantica) - Source code diff --git a/docs/citation.md b/docs/citation.md new file mode 100644 index 00000000..16210554 --- /dev/null +++ b/docs/citation.md @@ -0,0 +1,68 @@ +# Citation + +How to cite Semantica in academic papers and research. + +## BibTeX + +```bibtex +@software{semantica2024, + title = {Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering}, + author = {Hawksight AI}, + year = {2024}, + url = {https://github.com/Hawksight-AI/semantica}, + version = {0.0.1}, + license = {MIT} +} +``` + +## APA Format + +Hawksight AI. (2024). *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering* (Version 0.0.1) [Computer software]. GitHub. https://github.com/Hawksight-AI/semantica + +## MLA Format + +Hawksight AI. *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering*. Version 0.0.1, GitHub, 2024, https://github.com/Hawksight-AI/semantica. + +## Chicago Style + +Hawksight AI. *Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering*. Version 0.0.1. GitHub, 2024. https://github.com/Hawksight-AI/semantica. + +## IEEE Format + +Hawksight AI, "Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering," Version 0.0.1, GitHub, 2024. [Online]. Available: https://github.com/Hawksight-AI/semantica + +## Plain Text Citation + +If you use Semantica in your research, please cite: + +``` +Semantica: An Open Source Framework for Semantic Layers and Knowledge Engineering +Hawksight AI (2024) +https://github.com/Hawksight-AI/semantica +Version 0.0.1 +``` + +## Research Papers + +If you publish research using Semantica, we'd love to know! Please: + +1. Share your paper with us +2. Let us know how Semantica was used +3. We may feature your work in our documentation + +Contact: [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions) + +## Acknowledgments + +If you use Semantica in your work, we appreciate acknowledgments such as: + +> "This work uses Semantica (Hawksight AI, 2024), an open-source framework for semantic layer construction and knowledge engineering." + +## License + +Semantica is licensed under the MIT License. See the [License](license.md) page for details. + +--- + +**Questions about citation?** [Open a discussion](https://github.com/Hawksight-AI/semantica/discussions) or [contact us](https://github.com/Hawksight-AI/semantica/issues). + diff --git a/docs/community-projects.md b/docs/community-projects.md new file mode 100644 index 00000000..5ec80fa7 --- /dev/null +++ b/docs/community-projects.md @@ -0,0 +1,138 @@ +# Community Projects + +Projects, integrations, and contributions from the Semantica community. + +## Projects Using Semantica + +### Research Projects + +- **Academic Research**: Knowledge graph construction for research papers +- **Biomedical Analysis**: Drug discovery and genomic analysis +- **Social Network Analysis**: Relationship mapping and analysis + +### Industry Applications + +- **Business Intelligence**: Company knowledge bases +- **Cybersecurity**: Threat intelligence and analysis +- **Healthcare**: Medical record processing +- **Finance**: Market analysis and fraud detection + +## Community Contributions + +### Integrations + +#### Vector Database Integrations + +- **Pinecone Integration**: [Example](https://github.com/Hawksight-AI/semantica/tree/main/examples/pinecone) +- **Weaviate Integration**: [Example](https://github.com/Hawksight-AI/semantica/tree/main/examples/weaviate) +- **Qdrant Integration**: [Example](https://github.com/Hawksight-AI/semantica/tree/main/examples/qdrant) + +#### Knowledge Graph Databases + +- **Neo4j Integration**: Export and query with Neo4j +- **Amazon Neptune**: Cloud-based graph database integration +- **ArangoDB**: Multi-model database support + +### Plugins and Extensions + +Community-created plugins: + +- Custom entity extractors +- Domain-specific exporters +- Integration adapters +- Visualization tools + +## Showcase Your Project + +Have a project using Semantica? We'd love to feature it! + +### How to Submit + +1. Create a pull request or issue +2. Include: + - Project description + - Use case + - Code examples (if possible) + - Screenshots or demos + - Link to your project + +### Submission Template + +```markdown +## Project Name + +**Description**: Brief description of your project + +**Use Case**: How you're using Semantica + +**Key Features**: +- Feature 1 +- Feature 2 + +**Links**: +- GitHub: [link] +- Demo: [link] +- Documentation: [link] +``` + +## Community Tutorials + +### User-Created Tutorials + +- [Tutorial 1](link) - Description +- [Tutorial 2](link) - Description +- [Tutorial 3](link) - Description + +### Video Tutorials + +- [Video 1](link) - Description +- [Video 2](link) - Description + +## Contributing + +Want to contribute? Here's how: + +### Code Contributions + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Submit a pull request + +See our [Contributing Guide](https://github.com/Hawksight-AI/semantica/blob/main/CONTRIBUTING.md) for details. + +### Documentation Contributions + +- Fix typos +- Improve examples +- Add tutorials +- Translate documentation + +### Community Support + +- Answer questions in discussions +- Help with issues +- Share your experiences +- Provide feedback + +## Recognition + +### Contributors + +Thank you to all contributors! See our [Contributors](https://github.com/Hawksight-AI/semantica/graphs/contributors) page. + +### Top Contributors + +- [Contributor 1](link) - Contributions +- [Contributor 2](link) - Contributions + +## Resources + +- **[GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions)** - Community discussions +- **[GitHub Issues](https://github.com/Hawksight-AI/semantica/issues)** - Bug reports and feature requests +- **[Examples Repository](https://github.com/Hawksight-AI/semantica/tree/main/examples)** - More examples + +--- + +**Want to be featured?** [Submit your project](https://github.com/Hawksight-AI/semantica/discussions/categories/show-and-tell)! + diff --git a/docs/concepts.md b/docs/concepts.md new file mode 100644 index 00000000..22031951 --- /dev/null +++ b/docs/concepts.md @@ -0,0 +1,161 @@ +# Core Concepts + +Understand the fundamental concepts behind Semantica. + +## What is a Knowledge Graph? + +A knowledge graph is a structured representation of information where: + +- **Entities** are the nodes (people, places, concepts, etc.) +- **Relationships** are the edges connecting entities +- **Properties** describe attributes of entities + +```mermaid +graph LR + A[Apple Inc.
Organization] -->|founded_by| B[Steve Jobs
Person] + A -->|located_in| C[Cupertino
Location] + C -->|in_state| D[California
Location] + + style A fill:#e3f2fd + style B fill:#fff3e0 + style C fill:#f3e5f5 + style D fill:#f3e5f5 +``` + +## Semantic Layer + +A semantic layer provides: + +- **Structured meaning** from unstructured data +- **Contextual relationships** between concepts +- **Queryable knowledge** for AI systems +- **Quality-assured data** with conflict resolution + +## Key Components + +### 1. Data Ingestion + +Import data from various sources: + +- Documents (PDF, DOCX, HTML) +- Databases +- APIs and web content +- Structured data (JSON, CSV) + +### 2. Entity Extraction + +Identify and extract: + +- **Named Entities**: People, organizations, locations +- **Concepts**: Ideas, topics, themes +- **Events**: Actions, occurrences +- **Relations**: Connections between entities + +### 3. Relationship Extraction + +Discover relationships: + +- **Explicit**: Directly stated in text +- **Implicit**: Inferred from context +- **Temporal**: Time-based relationships +- **Causal**: Cause-and-effect connections + +### 4. Knowledge Graph Construction + +Build structured graphs: + +- **Node creation**: Entities as nodes +- **Edge creation**: Relationships as edges +- **Property assignment**: Attributes and metadata +- **Graph validation**: Quality checks + +### 5. Conflict Resolution + +Handle conflicting information: + +- **Multiple sources**: Same entity, different facts +- **Resolution strategies**: Voting, credibility, recency +- **Quality assurance**: Validation and verification + +### 6. Embedding Generation + +Create vector representations: + +- **Text embeddings**: Semantic text vectors +- **Graph embeddings**: Node and edge vectors +- **Multimodal**: Text, image, audio embeddings + +## Workflow + +Typical Semantica workflow: + +```mermaid +flowchart TD + A[Data Source] --> B[Ingestion] + B --> C[Parsing] + C --> D[Extraction
Entities & Relationships] + D --> E[Normalization] + E --> F[Conflict Resolution] + F --> G[Knowledge Graph] + G --> H[Embeddings] + H --> I[Export] + + style A fill:#e3f2fd + style G fill:#c8e6c9 + style I fill:#fff9c4 +``` + +## Use Cases + +### GraphRAG + +Enhance RAG systems with knowledge graphs: + +- **Context expansion**: Follow relationships +- **Multi-hop reasoning**: Traverse graph paths +- **Structured queries**: Query graph directly + +### AI Agents + +Provide agents with: + +- **Persistent memory**: Knowledge graph as memory +- **Context understanding**: Semantic relationships +- **Action validation**: Check against knowledge + +### Data Integration + +Unify data from multiple sources: + +- **Schema mapping**: Automatic schema discovery +- **Entity resolution**: Match entities across sources +- **Conflict resolution**: Handle contradictions + +## Best Practices + +### 1. Start Small + +Begin with a single document or small dataset to understand the workflow. + +### 2. Iterate + +Build knowledge graphs incrementally, refining as you learn. + +### 3. Validate + +Always validate extracted entities and relationships. + +### 4. Resolve Conflicts + +Use appropriate conflict resolution strategies for your use case. + +### 5. Export Regularly + +Export your knowledge graphs for backup and analysis. + +## Next Steps + +- **[Quick Start](quickstart.md)** - Build your first knowledge graph +- **[Examples](examples.md)** - See real-world applications +- **[API Reference](api.md)** - Explore the full API + diff --git a/docs/cookbook.md b/docs/cookbook.md new file mode 100644 index 00000000..1ea6754a --- /dev/null +++ b/docs/cookbook.md @@ -0,0 +1,182 @@ +# Cookbook Recipes + +Interactive Jupyter notebooks with hands-on examples and tutorials. + +## Introduction + +Get started with Semantica through these beginner-friendly tutorials. + +### Getting Started + +- **[Welcome to Semantica](cookbook/introduction/Welcome_to_Semantica.ipynb)** - Introduction to the framework +- **[Your First Knowledge Graph](cookbook/introduction/Your_First_Knowledge_Graph.ipynb)** - Build your first KG +- **[Configuration Basics](cookbook/introduction/Configuration_Basics.ipynb)** - Learn configuration options + +### Core Concepts + +- **[Data Ingestion](cookbook/introduction/Data_Ingestion.ipynb)** - Ingest data from various sources +- **[Document Parsing](cookbook/introduction/Document_Parsing.ipynb)** - Parse different document formats +- **[Data Normalization](cookbook/introduction/Data_Normalization.ipynb)** - Normalize and clean data +- **[Entity Extraction](cookbook/introduction/Entity_Extraction.ipynb)** - Extract entities from text +- **[Relation Extraction](cookbook/introduction/Relation_Extraction.ipynb)** - Extract relationships +- **[Building Knowledge Graphs](cookbook/introduction/Building_Knowledge_Graphs.ipynb)** - Construct KGs + +### Quality and Analysis + +- **[Conflict Detection](cookbook/introduction/Conflict_Detection.ipynb)** - Detect data conflicts +- **[Deduplication](cookbook/introduction/Deduplication.ipynb)** - Remove duplicates +- **[Graph Quality](cookbook/introduction/Graph_Quality.ipynb)** - Assess KG quality +- **[Graph Analytics](cookbook/introduction/Graph_Analytics.ipynb)** - Analyze knowledge graphs + +### Advanced Features + +- **[Embedding Generation](cookbook/introduction/Embedding_Generation.ipynb)** - Generate embeddings +- **[Vector Store](cookbook/introduction/Vector_Store.ipynb)** - Store and query vectors +- **[Ontology](cookbook/introduction/Ontology.ipynb)** - Work with ontologies +- **[Visualization](cookbook/introduction/Visualization.ipynb)** - Visualize knowledge graphs +- **[Export](cookbook/introduction/Export.ipynb)** - Export in various formats + +## Advanced + +Advanced techniques and patterns for experienced users. + +### Advanced Extraction + +- **[Advanced Extraction](cookbook/advanced/Advanced_Extraction.ipynb)** - Advanced extraction techniques +- **[Text Chunking Strategies](cookbook/advanced/Text_Chunking_Strategies.ipynb)** - Optimize text chunking + +### Graph Operations + +- **[Advanced Graph Analytics](cookbook/advanced/Advanced_Graph_Analytics.ipynb)** - Advanced graph analysis +- **[Temporal Knowledge Graphs](cookbook/advanced/Temporal_Knowledge_Graphs.ipynb)** - Work with temporal data +- **[Semantic Layer Construction](cookbook/advanced/Semantic_Layer_Construction.ipynb)** - Build semantic layers + +### Integration and Processing + +- **[Multi-Source Data Integration](cookbook/advanced/Multi_Source_Data_Integration.ipynb)** - Integrate multiple sources +- **[Pipeline Orchestration](cookbook/advanced/Pipeline_Orchestration.ipynb)** - Orchestrate complex pipelines +- **[Unstructured to Ontology](cookbook/advanced/Unstructured_to_Ontology.ipynb)** - Convert unstructured to ontology + +### Quality and Resolution + +- **[Conflict Resolution Strategies](cookbook/advanced/Conflict_Resolution_Strategies.ipynb)** - Advanced conflict resolution +- **[Reasoning and Inference](cookbook/advanced/Reasoning_and_Inference.ipynb)** - Perform reasoning + +### Visualization and Export + +- **[Complete Visualization Suite](cookbook/advanced/Complete_Visualization_Suite.ipynb)** - Comprehensive visualization +- **[Multi-Format Export](cookbook/advanced/Multi_Format_Export.ipynb)** - Export to multiple formats + +## Use Cases + +Real-world applications across various domains. + +### Advanced RAG + +- **[GraphRAG Complete](cookbook/use_cases/advanced_rag/GraphRAG_Complete.ipynb)** - Complete GraphRAG implementation + +### Biomedical + +- **[Drug Discovery Pipeline](cookbook/use_cases/biomedical/Drug_Discovery_Pipeline.ipynb)** - Drug discovery workflows +- **[Genomic Variant Analysis](cookbook/use_cases/biomedical/Genomic_Variant_Analysis.ipynb)** - Genomic data analysis + +### Blockchain + +- **[DeFi Protocol Intelligence](cookbook/use_cases/blockchain/DeFi_Protocol_Intelligence.ipynb)** - DeFi analysis +- **[Transaction Network Analysis](cookbook/use_cases/blockchain/Transaction_Network_Analysis.ipynb)** - Blockchain network analysis + +### Cybersecurity + +- **[Threat Intelligence Integration](cookbook/use_cases/cybersecurity/Threat_Intelligence_Integration.ipynb)** - Threat intelligence +- **[Threat Intelligence Hybrid RAG](cookbook/use_cases/cybersecurity/Threat_Intelligence_Hybrid_RAG.ipynb)** - Hybrid RAG for threats +- **[Threat Correlation](cookbook/use_cases/cybersecurity/Threat_Correlation.ipynb)** - Correlate threats +- **[Anomaly Detection Real-Time](cookbook/use_cases/cybersecurity/Anomaly_Detection_Real_Time.ipynb)** - Real-time anomaly detection +- **[Incident Analysis](cookbook/use_cases/cybersecurity/Incident_Analysis.ipynb)** - Analyze security incidents +- **[Vulnerability Tracking](cookbook/use_cases/cybersecurity/Vulnerability_Tracking.ipynb)** - Track vulnerabilities + +### Finance + +- **[Financial Data Integration](cookbook/use_cases/finance/Financial_Data_Integration.ipynb)** - Integrate financial data +- **[Financial Reports Analysis](cookbook/use_cases/finance/Financial_Reports_Analysis.ipynb)** - Analyze reports +- **[Fraud Detection](cookbook/use_cases/finance/Fraud_Detection.ipynb)** - Detect fraud +- **[Investment Analysis Hybrid RAG](cookbook/use_cases/finance/Investment_Analysis_Hybrid_RAG.ipynb)** - Investment analysis +- **[Market Intelligence](cookbook/use_cases/finance/Market_Intelligence.ipynb)** - Market analysis +- **[Regulatory Compliance](cookbook/use_cases/finance/Regulatory_Compliance.ipynb)** - Compliance workflows + +### Healthcare + +- **[Clinical Reports Processing](cookbook/use_cases/healthcare/Clinical_Reports_Processing.ipynb)** - Process clinical data +- **[Disease Network Analysis](cookbook/use_cases/healthcare/Disease_Network_Analysis.ipynb)** - Analyze disease networks +- **[Drug Interactions Analysis](cookbook/use_cases/healthcare/Drug_Interactions_Analysis.ipynb)** - Drug interaction analysis +- **[Healthcare GraphRAG Hybrid](cookbook/use_cases/healthcare/Healthcare_GraphRAG_Hybrid.ipynb)** - Healthcare GraphRAG +- **[Medical Database Integration](cookbook/use_cases/healthcare/Medical_Database_Integration.ipynb)** - Integrate medical databases +- **[Medical Literature GraphRAG](cookbook/use_cases/healthcare/Medical_Literature_GraphRAG.ipynb)** - Literature analysis +- **[Patient Records Temporal](cookbook/use_cases/healthcare/Patient_Records_Temporal.ipynb)** - Temporal patient data + +### Intelligence + +- **[Network Analysis Intelligence Reports](cookbook/use_cases/intelligence/Network_Analysis_Intelligence_Reports.ipynb)** - Intelligence network analysis + +### Renewable Energy + +- **[Energy Market Analysis](cookbook/use_cases/renewable_energy/Energy_Market_Analysis.ipynb)** - Energy market insights +- **[Environmental Impact](cookbook/use_cases/renewable_energy/Environmental_Impact.ipynb)** - Environmental analysis +- **[Grid Management](cookbook/use_cases/renewable_energy/Grid_Management.ipynb)** - Grid optimization +- **[Resource Optimization](cookbook/use_cases/renewable_energy/Resource_Optimization.ipynb)** - Optimize resources +- **[Supply Chain Analysis](cookbook/use_cases/renewable_energy/Supply_Chain_Analysis.ipynb)** - Energy supply chains + +### Supply Chain + +- **[Supply Chain Data Integration](cookbook/use_cases/supply_chain/Supply_Chain_Data_Integration.ipynb)** - Integrate supply chain data +- **[Supply Chain Risk Management](cookbook/use_cases/supply_chain/Supply_Chain_Risk_Management.ipynb)** - Risk management + +### Trading + +- **[Market Data Analysis](cookbook/use_cases/trading/Market_Data_Analysis.ipynb)** - Analyze market data +- **[News Sentiment Analysis](cookbook/use_cases/trading/News_Sentiment_Analysis.ipynb)** - Sentiment analysis +- **[Real-Time Market Data](cookbook/use_cases/trading/Real_Time_Market_Data.ipynb)** - Real-time processing +- **[Real-Time Monitoring](cookbook/use_cases/trading/Real_Time_Monitoring.ipynb)** - Monitor markets +- **[Risk Assessment](cookbook/use_cases/trading/Risk_Assessment.ipynb)** - Assess risks +- **[Strategy Backtesting](cookbook/use_cases/trading/Strategy_Backtesting.ipynb)** - Backtest strategies + +## Running the Notebooks + +### Prerequisites + +```bash +# Install Semantica +pip install semantica + +# Install Jupyter +pip install jupyter notebook + +# Optional: Install JupyterLab +pip install jupyterlab +``` + +### Launch Jupyter + +```bash +# Start Jupyter Notebook +jupyter notebook + +# Or start JupyterLab +jupyter lab +``` + +Navigate to the `cookbook/` directory and open any notebook. + +### Viewing on GitHub + +All notebooks can be viewed directly on GitHub. Click any notebook link above to view it online. + +## Contributing + +Have a use case or example to share? We welcome contributions! + +1. Create a new notebook in the appropriate category +2. Follow the existing notebook structure +3. Submit a pull request + +See our [Contributing Guide](https://github.com/Hawksight-AI/semantica/blob/main/CONTRIBUTING.md) for details. + diff --git a/docs/css/custom.css b/docs/css/custom.css new file mode 100644 index 00000000..cebfe4b1 --- /dev/null +++ b/docs/css/custom.css @@ -0,0 +1,118 @@ +/* Custom CSS for Semantica Documentation */ + +/* Smooth scrolling */ +html { + scroll-behavior: smooth; +} + +/* Enhanced code blocks */ +.md-typeset pre > code { + border-radius: 8px; + padding: 1.2em; +} + +/* Custom card styles */ +.feature-card { + transition: transform 0.2s ease, box-shadow 0.2s ease; +} + +.feature-card:hover { + transform: translateY(-2px); + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); +} + +/* Diagram containers */ +.mermaid { + background: var(--md-default-bg-color); + border-radius: 8px; + padding: 1em; + margin: 1em 0; +} + +/* Enhanced tables */ +.md-typeset table:not([class]) { + border-radius: 8px; + overflow: hidden; +} + +/* Custom animations */ +@keyframes fadeIn { + from { + opacity: 0; + transform: translateY(10px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +.animate-fade-in { + animation: fadeIn 0.5s ease-out; +} + +/* Better spacing for sections */ +.md-typeset h2 { + margin-top: 2em; + margin-bottom: 1em; +} + +.md-typeset h3 { + margin-top: 1.5em; + margin-bottom: 0.75em; +} + +/* Enhanced badges */ +.badge { + display: inline-block; + padding: 0.25em 0.75em; + border-radius: 4px; + font-size: 0.85em; + font-weight: 500; +} + +/* Custom button styles */ +.md-button { + border-radius: 6px; + transition: all 0.2s ease; +} + +.md-button:hover { + transform: translateY(-1px); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.15); +} + +/* Improved code copy button */ +.md-clipboard { + opacity: 0.7; + transition: opacity 0.2s ease; +} + +.md-clipboard:hover { + opacity: 1; +} + +/* Chart and diagram containers */ +.chart-container { + background: var(--md-code-bg-color); + border-radius: 8px; + padding: 1.5em; + margin: 1.5em 0; + border: 1px solid var(--md-code-hl-color); +} + +/* Responsive improvements */ +@media screen and (max-width: 76.1875em) { + .md-nav--primary .md-nav__title { + font-size: 1.1rem; + } +} + +/* Print styles */ +@media print { + .md-sidebar, + .md-header { + display: none; + } +} + diff --git a/docs/deep-dive.md b/docs/deep-dive.md new file mode 100644 index 00000000..41bc256d --- /dev/null +++ b/docs/deep-dive.md @@ -0,0 +1,283 @@ +# Deep Dive + +Advanced topics, architecture, and internals of Semantica. + +## Architecture Overview + +Semantica follows a modular, extensible architecture: + +```mermaid +graph TB + A[Data Sources] --> B[Ingestion Layer] + B --> C[Parsing Layer] + C --> D[Extraction Layer] + D --> E[Normalization Layer] + E --> F[Conflict Resolution] + F --> G[Knowledge Graph Builder] + G --> H[Embedding Generator] + H --> I[Export Layer] + + D --> D1[Entity Extractor] + D --> D2[Relationship Extractor] + D --> D3[Triple Extractor] + + G --> G1[Graph Validator] + G --> G2[Graph Analyzer] + + H --> H1[Text Embeddings] + H --> H2[Graph Embeddings] +``` + +## System Components + +### 1. Ingestion Layer + +Handles data input from various sources: + +- **File Ingestor**: PDF, DOCX, HTML, JSON, CSV +- **Web Ingestor**: URLs, web scraping +- **Database Ingestor**: SQL databases +- **Stream Ingestor**: Real-time data streams + +### 2. Parsing Layer + +Converts raw data into structured format: + +- Document parsing (PDF, Word, etc.) +- Text extraction +- Metadata extraction +- Format normalization + +### 3. Extraction Layer + +Core semantic extraction: + +```python +# Entity extraction pipeline +text → Tokenization → NER → Entity Linking → Entity Validation +``` + +**Components:** +- Named Entity Recognition (NER) +- Relationship Extraction +- Triple Extraction +- Coreference Resolution + +### 4. Normalization Layer + +Standardizes extracted data: + +- Entity normalization +- Date/time normalization +- Number normalization +- Text cleaning + +### 5. Conflict Resolution + +Handles conflicting information: + +```mermaid +graph LR + A[Multiple Sources] --> B[Conflict Detection] + B --> C{Resolution Strategy} + C --> D[Voting] + C --> E[Credibility Weighted] + C --> F[Most Recent] + C --> G[Highest Confidence] + D --> H[Resolved Entity] + E --> H + F --> H + G --> H + + style A fill:#ffebee + style H fill:#c8e6c9 + style C fill:#fff9c4 +``` + +### 6. Knowledge Graph Builder + +Constructs the knowledge graph: + +- Node creation (entities) +- Edge creation (relationships) +- Property assignment +- Graph validation +- Quality checks + +### 7. Embedding Generator + +Generates vector representations: + +- Text embeddings (sentence transformers) +- Graph embeddings (node2vec, GraphSAGE) +- Multimodal embeddings + +## Data Flow + +```mermaid +sequenceDiagram + participant User + participant Semantica + participant Ingestor + participant Parser + participant Extractor + participant Resolver + participant GraphBuilder + participant Exporter + + User->>Semantica: build_knowledge_base(sources) + Semantica->>Ingestor: ingest(sources) + Ingestor->>Parser: parse(documents) + Parser->>Extractor: extract(text) + Extractor->>Resolver: resolve_conflicts(entities) + Resolver->>GraphBuilder: build_graph(resolved_data) + GraphBuilder->>Exporter: export(graph) + Exporter->>User: return result + + Note over User,Exporter: Complete pipeline execution +``` + +## Advanced Concepts + +### Entity Resolution + +Matching entities across sources: + +```python +# Entity resolution algorithm +def resolve_entities(entities): + clusters = [] + for entity in entities: + matched = False + for cluster in clusters: + if similarity(entity, cluster.representative) > threshold: + cluster.add(entity) + matched = True + break + if not matched: + clusters.append(EntityCluster(entity)) + return clusters +``` + +### Relationship Inference + +Inferring implicit relationships: + +- Transitive relationships +- Temporal relationships +- Causal relationships +- Hierarchical relationships + +### Graph Optimization + +Optimizing knowledge graph structure: + +- Node deduplication +- Edge consolidation +- Path compression +- Index optimization + +## Performance Considerations + +### Scalability + +- **Horizontal Scaling**: Process multiple documents in parallel +- **Vertical Scaling**: Use GPU acceleration +- **Caching**: Cache embeddings and parsed documents +- **Lazy Loading**: Load components on demand + +### Memory Management + +```python +# Process large datasets efficiently +def process_large_dataset(sources, batch_size=100): + for i in range(0, len(sources), batch_size): + batch = sources[i:i+batch_size] + result = semantica.build_knowledge_base(batch) + # Save and clear memory + save_result(result) + del result + gc.collect() +``` + +## Extension Points + +### Custom Plugins + +Create custom plugins: + +```python +from semantica.core import Plugin + +class CustomPlugin(Plugin): + def process(self, data): + # Your custom processing + return processed_data +``` + +### Custom Extractors + +Implement custom extractors: + +```python +from semantica.semantic_extract import BaseExtractor + +class DomainSpecificExtractor(BaseExtractor): + def extract_entities(self, text): + # Domain-specific extraction logic + return entities +``` + +## Internal APIs + +### Core APIs + +- `Semantica.build_knowledge_base()` - Main entry point +- `KGBuilder.build()` - Graph construction +- `ConflictResolver.resolve()` - Conflict resolution +- `EmbeddingGenerator.generate()` - Embedding generation + +### Extension APIs + +- Plugin registration +- Custom extractor registration +- Custom exporter registration +- Event hooks + +## Design Decisions + +### Why Modular Architecture? + +- **Extensibility**: Easy to add new features +- **Testability**: Components can be tested independently +- **Maintainability**: Clear separation of concerns +- **Flexibility**: Swap implementations easily + +### Why Conflict Resolution? + +- **Data Quality**: Ensures consistent knowledge +- **Multi-Source**: Handles conflicting information +- **Flexibility**: Multiple resolution strategies +- **Transparency**: Track resolution decisions + +## Future Enhancements + +Planned improvements: + +- Distributed processing +- Real-time streaming +- Advanced reasoning +- Multi-modal support expansion +- Enhanced visualization + +## Contributing to Core + +Interested in contributing to Semantica's core? See our [Contributing Guide](https://github.com/Hawksight-AI/semantica/blob/main/CONTRIBUTING.md). + +--- + +For more information: +- **[API Reference](api.md)** - Detailed API documentation +- **[Learning More](learning-more.md)** - Additional resources +- **[GitHub Repository](https://github.com/Hawksight-AI/semantica)** - Source code + diff --git a/docs/examples.md b/docs/examples.md index 44cc5003..4f2f5ea4 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,6 +1,12 @@ # Examples -## Example 1: Basic Knowledge Graph +Real-world examples and use cases for Semantica. + +## Basic Examples + +### Example 1: Basic Knowledge Graph + +Build a knowledge graph from a single document: ```python from semantica import Semantica @@ -19,7 +25,9 @@ print(f"Entities: {len(kg['entities'])}") print(f"Relationships: {len(kg['relationships'])}") ``` -## Example 2: Entity Extraction +### Example 2: Entity Extraction + +Extract entities from text: ```python from semantica import Semantica @@ -29,14 +37,26 @@ semantica = Semantica() text = """ Apple Inc. is a technology company founded by Steve Jobs. The company is headquartered in Cupertino, California. +Tim Cook is the current CEO of Apple. """ entities = semantica.semantic_extract.extract_entities(text) -for entity in entities: +for entity in entities["entities"]: print(f"{entity['text']}: {entity['type']}") ``` -## Example 3: Multi-Source Integration +**Output:** +``` +Apple Inc.: ORGANIZATION +Steve Jobs: PERSON +Cupertino: LOCATION +California: LOCATION +Tim Cook: PERSON +``` + +### Example 3: Multi-Source Integration + +Combine data from multiple sources: ```python from semantica import Semantica @@ -50,9 +70,14 @@ sources = [ ] result = semantica.build_knowledge_base(sources) +kg = result["knowledge_graph"] + +print(f"Unified knowledge graph with {len(kg['entities'])} entities") ``` -## Example 4: Export Formats +### Example 4: Export Formats + +Export knowledge graph to multiple formats: ```python from semantica import Semantica @@ -60,14 +85,188 @@ from semantica import Semantica semantica = Semantica() kg = semantica.kg.build_graph(["data.pdf"]) -# Export to multiple formats +# Export to different formats semantica.export.to_rdf(kg, "output.rdf") semantica.export.to_json(kg, "output.json") +semantica.export.to_csv(kg, "output.csv") semantica.export.to_owl(kg, "output.owl") ``` -## More Examples +## Advanced Examples -- [Code Examples](../CodeExamples.md) -- [Cookbook Notebooks](../cookbook/) +### Example 5: Conflict Resolution +Resolve conflicts in data from multiple sources: + +```python +from semantica import Semantica +from semantica.conflicts import ConflictResolver + +semantica = Semantica() + +# Build graph from multiple sources +result = semantica.build_knowledge_base([ + "source1.pdf", + "source2.pdf", + "source3.pdf" +]) + +# Detect conflicts +conflicts = semantica.kg.detect_conflicts(result["knowledge_graph"]) + +# Resolve conflicts +resolver = ConflictResolver(default_strategy="voting") +resolved = resolver.resolve_conflicts(conflicts) + +print(f"Resolved {len(resolved)} conflicts") +``` + +### Example 6: Custom Configuration + +Use custom configuration for specific use cases: + +```python +from semantica import Semantica, Config + +# Custom configuration +config = Config( + embeddings=True, + graph=True, + normalize=True, + conflict_resolution="highest_confidence" +) + +semantica = Semantica(config=config) +result = semantica.build_knowledge_base(["document.pdf"]) +``` + +### Example 7: Incremental Graph Building + +Build knowledge graph incrementally: + +```python +from semantica import Semantica + +semantica = Semantica() + +# Build graphs separately +kg1 = semantica.kg.build_graph(["source1.pdf"]) +kg2 = semantica.kg.build_graph(["source2.pdf"]) +kg3 = semantica.kg.build_graph(["source3.pdf"]) + +# Merge into unified graph +merged_kg = semantica.kg.merge([kg1, kg2, kg3]) + +print(f"Merged graph: {len(merged_kg['entities'])} entities") +``` + +### Example 8: Visualization + +Create interactive visualizations: + +```python +from semantica import Semantica + +semantica = Semantica() + +# Build graph +result = semantica.build_knowledge_base(["document.pdf"]) +kg = result["knowledge_graph"] + +# Visualize +semantica.kg.visualize(kg, output_path="graph.html") + +# Also analyze +analysis = semantica.kg.analyze(kg) +print(f"Graph density: {analysis['density']}") +print(f"Connected components: {analysis['components']}") +``` + +## Use Case Examples + +### Research Paper Analysis + +Extract knowledge from research papers: + +```python +from semantica import Semantica + +semantica = Semantica() + +# Process research paper +result = semantica.build_knowledge_base([ + "papers/ai_research.pdf", + "papers/ml_survey.pdf" +]) + +kg = result["knowledge_graph"] + +# Find key concepts +concepts = [e for e in kg["entities"] if e["type"] == "CONCEPT"] +print(f"Found {len(concepts)} key concepts") +``` + +### Company Intelligence + +Build knowledge graph from company documents: + +```python +from semantica import Semantica + +semantica = Semantica() + +# Company documents +sources = [ + "company/annual_report.pdf", + "company/press_releases/", + "company/website_content.html" +] + +result = semantica.build_knowledge_base(sources) +kg = result["knowledge_graph"] + +# Export for analysis +semantica.export.to_json(kg, "company_intelligence.json") +``` + +### News Article Processing + +Process and analyze news articles: + +```python +from semantica import Semantica + +semantica = Semantica() + +# News articles +articles = [ + "https://example.com/article1", + "https://example.com/article2", + "https://example.com/article3" +] + +result = semantica.build_knowledge_base(articles) +kg = result["knowledge_graph"] + +# Extract key entities +people = [e for e in kg["entities"] if e["type"] == "PERSON"] +organizations = [e for e in kg["entities"] if e["type"] == "ORGANIZATION"] + +print(f"People mentioned: {len(people)}") +print(f"Organizations: {len(organizations)}") +``` + +## Interactive Examples + +For more interactive examples and tutorials, check out our [Cookbook](cookbook.md) with Jupyter notebooks covering: + +- **Introduction**: Getting started tutorials +- **Advanced**: Advanced techniques and patterns +- **Use Cases**: Real-world applications in various domains + +## More Resources + +- **[Quick Start Guide](quickstart.md)** - Step-by-step tutorial +- **[API Reference](api.md)** - Complete API documentation +- **[Cookbook](cookbook.md)** - Interactive Jupyter notebooks +- **[Code Examples](../CodeExamples.md)** - Additional code samples diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 00000000..6a0040b5 --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,211 @@ +# Frequently Asked Questions + +Common questions and answers about Semantica. + +## General Questions + +### What is Semantica? + +Semantica is an open-source framework for building semantic layers and knowledge graphs from unstructured data. It transforms raw data into structured, queryable knowledge that powers AI applications. + +### What can I use Semantica for? + +- Building knowledge graphs from documents +- Creating semantic layers for AI applications +- Extracting entities and relationships from text +- Powering GraphRAG systems +- Integrating data from multiple sources +- Building AI agent memory systems + +### Is Semantica free? + +Yes! Semantica is 100% open source and free to use under the MIT License. + +## Installation + +### How do I install Semantica? + +```bash +pip install semantica +``` + +See the [Installation Guide](installation.md) for detailed instructions. + +### What Python version do I need? + +Python 3.8 or higher. Python 3.11+ is recommended. + +### Do I need GPU? + +No, GPU is optional. Semantica works on CPU, but GPU acceleration is available for faster processing. + +## Usage + +### How do I get started? + +1. Install Semantica: `pip install semantica` +2. Follow the [Quick Start Guide](quickstart.md) +3. Try the [Examples](examples.md) + +### Can I process PDF files? + +Yes! Semantica supports PDF, DOCX, HTML, JSON, CSV, and many other formats. + +### How do I extract entities from text? + +```python +from semantica import Semantica + +semantica = Semantica() +result = semantica.semantic_extract.extract_entities("Your text here") +entities = result["entities"] +``` + +### Can I use my own models? + +Yes, Semantica is extensible. You can plug in custom models for entity extraction, embeddings, etc. + +## Knowledge Graphs + +### What is a knowledge graph? + +A knowledge graph is a structured representation where entities (nodes) are connected by relationships (edges). It captures semantic meaning and relationships in data. + +### How do I build a knowledge graph? + +```python +from semantica import Semantica + +semantica = Semantica() +result = semantica.build_knowledge_base(["document.pdf"]) +kg = result["knowledge_graph"] +``` + +### Can I merge multiple knowledge graphs? + +Yes! Use the `merge` method: + +```python +merged = semantica.kg.merge([kg1, kg2, kg3]) +``` + +### How do I visualize a knowledge graph? + +```python +semantica.kg.visualize(kg, output_path="graph.html") +``` + +## Conflict Resolution + +### What is conflict resolution? + +When the same entity appears in multiple sources with different information, conflict resolution determines which information to use. + +### What strategies are available? + +- **Voting**: Majority wins +- **Credibility Weighted**: Weight by source credibility +- **Most Recent**: Use latest information +- **Highest Confidence**: Use highest confidence score +- **First Seen**: Use first encountered value + +### How do I set a resolution strategy? + +```python +from semantica.conflicts import ConflictResolver + +resolver = ConflictResolver(default_strategy="voting") +``` + +## Export & Integration + +### What formats can I export to? + +- RDF/XML +- OWL (Ontology) +- JSON +- CSV +- YAML +- And more + +### Can I use Semantica with other tools? + +Yes! Semantica exports to standard formats that work with: +- Neo4j +- Graph databases +- RDF stores +- Vector databases +- Any tool that accepts RDF/JSON/CSV + +## Performance + +### How fast is Semantica? + +Performance depends on: +- Document size +- Number of documents +- Hardware (CPU/GPU) +- Configuration options + +For typical documents, processing takes seconds to minutes. + +### Can I process large datasets? + +Yes, but consider: +- Processing in batches +- Using GPU acceleration +- Incremental building +- Optimizing configuration + +## Troubleshooting + +### Installation fails + +- Upgrade pip: `pip install --upgrade pip` +- Use virtual environment +- Check Python version: `python --version` + +### No entities extracted + +- Verify document contains text (not just images) +- Check document format is supported +- Review extraction configuration + +### Memory errors + +- Process documents one at a time +- Reduce batch sizes +- Use smaller models +- Increase available RAM + +### Slow processing + +- Enable GPU if available +- Process in smaller batches +- Optimize configuration +- Use faster models + +## Getting Help + +### Where can I get help? + +- **Documentation**: This site +- **GitHub Issues**: [Report bugs](https://github.com/Hawksight-AI/semantica/issues) +- **Discussions**: [Ask questions](https://github.com/Hawksight-AI/semantica/discussions) + +### How do I report a bug? + +Open an issue on [GitHub](https://github.com/Hawksight-AI/semantica/issues) with: +- Description of the problem +- Steps to reproduce +- Expected vs actual behavior +- Environment details + +### Can I contribute? + +Yes! We welcome contributions. See our [Contributing Guide](https://github.com/Hawksight-AI/semantica/blob/main/CONTRIBUTING.md). + +--- + +Still have questions? Check the [API Reference](api.md) or [open an issue](https://github.com/Hawksight-AI/semantica/issues). + diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 00000000..5cb260ea --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,85 @@ +# Getting Started + +Welcome to Semantica! This guide will help you get up and running quickly. + +## Choose Your Path + +=== "🚀 Quick Start (5 min)" + + Get Semantica running in 5 minutes: + + 1. **[Install Semantica](installation.md#basic-installation)** + 2. **[Run Your First Example](quickstart.md#step-2-your-first-knowledge-graph)** + 3. **[Explore Examples](examples.md)** + + Perfect for: Trying Semantica quickly + +=== "📚 Complete Guide (30 min)" + + Learn Semantica thoroughly: + + 1. **[Installation](installation.md)** - Complete setup guide + 2. **[Quick Start](quickstart.md)** - Step-by-step tutorial + 3. **[Core Concepts](concepts.md)** - Understand the framework + 4. **[Examples](examples.md)** - Real-world use cases + 5. **[API Reference](api.md)** - Full API documentation + + Perfect for: Learning the framework properly + +=== "🎓 Interactive Learning" + + Learn by doing with Jupyter notebooks: + + 1. **[Cookbook Overview](cookbook.md)** - Browse all tutorials + 2. **[Start with Introduction](cookbook.md#introduction)** - Beginner notebooks + 3. **[Try Use Cases](cookbook.md#use-cases)** - Domain-specific examples + + Perfect for: Hands-on learners + +## What Can You Build? + +Semantica helps you transform unstructured data into intelligent knowledge: + +- **Knowledge Graphs** from documents, websites, databases +- **Semantic Layers** for AI applications +- **Entity & Relationship Extraction** from text +- **Conflict Resolution** across multiple data sources +- **GraphRAG Systems** for enhanced AI responses + +## Common Use Cases + +### Research & Analysis +- Extract knowledge from research papers +- Build domain-specific knowledge graphs +- Analyze relationships in literature + +### Business Intelligence +- Process company documents +- Build organizational knowledge bases +- Integrate multiple data sources + +### AI Applications +- Power GraphRAG systems +- Enhance AI agent memory +- Build semantic search systems + +## Next Steps + +Once you're set up: + +1. **[Install Semantica](installation.md)** if you haven't already +2. **[Follow Quick Start](quickstart.md)** to build your first KG +3. **[Explore Examples](examples.md)** for inspiration +4. **[Check Cookbook](cookbook.md)** for interactive tutorials + +## Need Help? + +- **Installation Issues?** → [Troubleshooting Guide](installation.md#troubleshooting) +- **First Time User?** → [Quick Start Guide](quickstart.md) +- **Looking for Examples?** → [Examples Page](examples.md) +- **API Questions?** → [API Reference](api.md) + +--- + +Ready to start? Head to the [Installation Guide](installation.md)! + diff --git a/docs/index.md b/docs/index.md index 0b189bad..6f30ef62 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,95 +1,410 @@ -# 🧠 Semantica +# Welcome to Semantica -**Open Source Framework for Semantic Intelligence & Knowledge Engineering** +**Transform chaotic data into intelligent knowledge.** -> **Transform chaotic data into intelligent knowledge.** +Semantica is an open-source framework for building semantic layers and knowledge graphs that power the next generation of AI applications. -
+--- -[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -[![PyPI version](https://badge.fury.io/py/semantica.svg)](https://badge.fury.io/py/semantica) -[![Downloads](https://pepy.tech/badge/semantica)](https://pepy.tech/project/semantica) +## 🚀 Get Started in 60 Seconds + +```python +from semantica import Semantica + +semantica = Semantica() +result = semantica.build_knowledge_base(["document.pdf"]) +print(f"Extracted {len(result['knowledge_graph']['entities'])} entities") +``` + +**Install:** `pip install semantica` + +--- + +## Choose Your Learning Path + +=== "⚡ Quick Start (5 min)" + + **Perfect for:** Trying Semantica quickly + + ```bash + pip install semantica + ``` + + → **[Quickstart Guide](quickstart.md)** - Build your first knowledge graph + + → **[Examples](examples.md)** - See what's possible + +=== "📚 Complete Guide (30 min)" + + **Perfect for:** Learning properly + + 1. **[Installation](installation.md)** - Complete setup + 2. **[Quickstart](quickstart.md)** - Step-by-step tutorial + 3. **[Examples](examples.md)** - Real-world use cases + 4. **[API References](api.md)** - Full documentation + +=== "🎓 Interactive Learning" + + **Perfect for:** Hands-on learners + + → **[Cookbook Recipes](cookbook.md)** - Interactive Jupyter notebooks + + - Introduction tutorials + - Advanced techniques + - Domain-specific use cases + +--- + +## What Can You Build? + +### Knowledge Graphs +Transform documents, websites, and databases into structured knowledge graphs with meaningful relationships. + +### Semantic Layers +Build semantic layers that enable AI systems to understand context and relationships in your data. + +### GraphRAG Systems +Power enhanced RAG systems with knowledge graphs for better context understanding and multi-hop reasoning. + +### AI Agent Memory +Provide AI agents with persistent, structured memory using knowledge graphs. + +--- + +## Features { #features } + +Comprehensive capabilities for semantic intelligence and knowledge engineering. + +### 🎯 Entity & Relationship Extraction + +Extract entities and relationships from unstructured text using advanced NLP. + +```python +from semantica import Semantica + +semantica = Semantica() +entities = semantica.semantic_extract.extract_entities(text) +relationships = semantica.semantic_extract.extract_relationships(text) +``` + +**Capabilities:** +- Named Entity Recognition (NER) +- Relationship extraction +- Triple extraction (subject-predicate-object) +- Coreference resolution +- Event detection + +### 🔗 Knowledge Graph Construction + +Build comprehensive knowledge graphs from multiple data sources. + +```python +result = semantica.build_knowledge_base([ + "document1.pdf", + "document2.docx", + "https://example.com/article" +]) + +kg = result["knowledge_graph"] +``` + +**Features:** +- Multi-source integration +- Automatic relationship discovery +- Graph validation +- Quality assurance +- Incremental building + +### ⚖️ Conflict Resolution + +Automatically resolve conflicts when the same entity appears in multiple sources. + +```python +from semantica.conflicts import ConflictResolver + +resolver = ConflictResolver(default_strategy="voting") +resolved = resolver.resolve_conflicts(conflicts) +``` + +**Strategies:** +- Voting (majority wins) +- Credibility weighted +- Most recent +- Highest confidence +- First seen +- Manual review + +### 📤 Multiple Export Formats + +Export to RDF, OWL, JSON, CSV, YAML, and more. + +```python +semantica.export.to_rdf(kg, "output.rdf") +semantica.export.to_json(kg, "output.json") +semantica.export.to_owl(kg, "output.owl") +semantica.export.to_csv(kg, "output.csv") +``` + +**Supported Formats:** +- RDF/XML +- OWL (Web Ontology Language) +- JSON-LD +- CSV +- YAML +- GraphML +- Neo4j Cypher + +### 🧠 Embedding Generation + +Generate embeddings for text, images, and audio. + +```python +embeddings = semantica.embeddings.generate(text) +graph_embeddings = semantica.embeddings.generate_graph_embeddings(kg) +``` + +**Capabilities:** +- Text embeddings +- Graph embeddings +- Multimodal embeddings +- Batch processing +- Custom models + +### 🔍 Vector Store Integration + +Store and query embeddings efficiently. + +```python +semantica.vector_store.add(embeddings, metadata) +results = semantica.vector_store.search(query, top_k=10) +``` + +**Supported Stores:** +- FAISS +- Pinecone +- Weaviate +- Qdrant +- Milvus + +### 📊 Data Ingestion + +Support for multiple data sources and formats. + +```python +# From files +sources = ["document.pdf", "data.json", "report.docx"] + +# From URLs +sources = ["https://example.com/article"] + +# From databases +sources = ["postgresql://localhost/db"] +``` + +**Supported Sources:** +- Files (PDF, DOCX, HTML, JSON, CSV, etc.) +- URLs and web content +- Databases (SQL, NoSQL) +- APIs and feeds +- Real-time streams + +### 🎨 Visualization + +Visualize knowledge graphs interactively. + +```python +semantica.kg.visualize(kg, output_path="graph.html") +``` + +**Features:** +- Interactive graphs +- Custom layouts +- Export to images +- Web-based viewer + +--- + +## How to Read this Documentation { #how-to-read } + +This documentation is organized to help you find what you need quickly. + +### Navigation Structure + +**Left Sidebar (Main Navigation):** +- **Home** - This page, overview and quick start +- **Quickstart** - Get started in 5 minutes +- **Installation** - Setup and configuration +- **Cookbook Recipes** - Interactive Jupyter notebooks +- **Learning More** - Additional resources and tutorials +- **Deep Dive** - Advanced topics and architecture +- **API References** - Complete API documentation + +**Right Sidebar (Table of Contents):** +- Appears on each page +- Shows page structure +- Quick navigation to sections +- Auto-generated from headings + +### Reading Paths + +**For Beginners:** +1. Start with [Quickstart](quickstart.md) +2. Follow [Installation](installation.md) +3. Try [Cookbook Recipes](cookbook.md) - Introduction section +4. Explore [Examples](examples.md) + +**For Experienced Users:** +1. Review [API References](api.md) +2. Check [Deep Dive](deep-dive.md) for architecture +3. Explore [Cookbook Recipes](cookbook.md) - Advanced section +4. See [Learning More](learning-more.md) for best practices + +**For Researchers:** +1. Read [Citation](citation.md) information +2. Check [Deep Dive](deep-dive.md) for technical details +3. Review [Community Projects](community-projects.md) +4. See [License](license.md) for usage rights + +### Using Code Examples + +All code examples are: +- ✅ Tested and working +- ✅ Copyable with one click +- ✅ Include expected outputs +- ✅ Contextual explanations + +### Interactive Elements + +- **Tabs**: Switch between different options +- **Diagrams**: Mermaid diagrams for visual understanding +- **Code Blocks**: Syntax highlighted, copyable +- **Search**: Find content quickly +- **Dark/Light Mode**: Toggle theme + +--- + +## Resources { #resources } + +Essential links and resources for Semantica. + +### Official Resources + +- **GitHub Repository**: [github.com/Hawksight-AI/semantica](https://github.com/Hawksight-AI/semantica) + - Source code + - Issue tracking + - Discussions + - Contributions + +- **PyPI Package**: [pypi.org/project/semantica](https://pypi.org/project/semantica) + - Package downloads + - Version history + - Installation instructions + +- **Documentation**: This site + - Complete guides + - API reference + - Examples and tutorials + +### Community Resources + +- **GitHub Discussions**: [Discussions](https://github.com/Hawksight-AI/semantica/discussions) + - Ask questions + - Share ideas + - Show your projects + +- **GitHub Issues**: [Issues](https://github.com/Hawksight-AI/semantica/issues) + - Report bugs + - Request features + - Get help + +- **Community Projects**: [Community Projects](community-projects.md) + - See what others are building + - Share your project + +### Additional Resources + +- **Citation**: [How to cite Semantica](citation.md) +- **License**: [MIT License details](license.md) +- **Contributing**: [How to contribute](https://github.com/Hawksight-AI/semantica/blob/main/CONTRIBUTING.md) + +--- + +## Common Use Cases + +### Research & Analysis +- Extract knowledge from research papers +- Build domain-specific knowledge graphs +- Analyze relationships in literature + +### Business Intelligence +- Process company documents +- Build organizational knowledge bases +- Integrate multiple data sources + +### AI Applications +- Power GraphRAG systems +- Enhance AI agent memory +- Build semantic search systems + +--- + +## Quick Links + +
+ +- :material-speedometer:{ .lg .middle } __Quickstart__ + + --- + + Get up and running in 5 minutes + + [:octicons-arrow-right-24: Quickstart Guide](quickstart.md) + +- :material-book-open-variant:{ .lg .middle } __Examples__ + + --- + + See real-world use cases and code examples + + [:octicons-arrow-right-24: Browse Examples](examples.md) + +- :material-notebook:{ .lg .middle } __Cookbook__ + + --- + + Interactive Jupyter notebooks for hands-on learning + + [:octicons-arrow-right-24: Explore Cookbook](cookbook.md) + +- :material-api:{ .lg .middle } __API Reference__ + + --- + + Complete API documentation + + [:octicons-arrow-right-24: View API Docs](api.md)
--- -## 🚀 Quick Start - -### Installation +## Installation ```bash pip install semantica ``` -### Basic Usage - -```python -from semantica import Semantica - -# Initialize Semantica -semantica = Semantica() - -# Build knowledge graph from data -result = semantica.build_knowledge_base( - sources=["document.pdf", "data.json"], - embeddings=True, - graph=True -) - -# Access the knowledge graph -kg = result["knowledge_graph"] -print(f"Entities: {len(kg['entities'])}") -print(f"Relationships: {len(kg['relationships'])}") -``` +See the [Installation Guide](installation.md) for detailed instructions, optional dependencies, and troubleshooting. --- -## 📚 Documentation +## Need Help? -- [Installation Guide](#installation) -- [Quick Start](#quick-start) -- [Core Features](#core-features) -- [Examples](#examples) -- [API Reference](#api-reference) +- **First time?** → [Quickstart](quickstart.md) +- **Installation issues?** → [Installation Guide](installation.md#troubleshooting) +- **Questions?** → [GitHub Discussions](https://github.com/Hawksight-AI/semantica/discussions) +- **Found a bug?** → [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) --- -## ✨ Core Features - -- **Semantic Layer Construction**: Build semantic layers from unstructured data -- **Knowledge Graph Generation**: Create and manage knowledge graphs -- **Entity & Relationship Extraction**: Extract entities and relationships from text -- **Conflict Resolution**: Multiple strategies for resolving data conflicts -- **Multiple Export Formats**: Export to RDF, OWL, JSON, CSV, YAML, and more -- **Vector Store Integration**: Store and query embeddings -- **Embedding Generation**: Generate embeddings for text, images, and audio - ---- - -## 📖 Learn More - -- [Full Documentation](../README.md) -- [Module Documentation](../MODULES_DOCUMENTATION.md) -- [Code Examples](../CodeExamples.md) -- [GitHub Repository](https://github.com/Hawksight-AI/semantica) - ---- - -## 🤝 Contributing - -We welcome contributions! Please see our [Contributing Guide](https://github.com/Hawksight-AI/semantica/blob/main/CONTRIBUTING.md) for details. - ---- - -## 📄 License - -This project is licensed under the MIT License - see the [LICENSE](../LICENSE) file for details. - ---- - -## 🙏 Acknowledgments - -Built with ❤️ by the Semantica community. - +**Ready to transform your data?** Start with the [Quickstart Guide](quickstart.md) or explore the [Cookbook Recipes](cookbook.md) for interactive tutorials. diff --git a/docs/installation.md b/docs/installation.md index 3a03698a..3f1dc8e1 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,9 +1,16 @@ # Installation +Get Semantica up and running in minutes. + ## Prerequisites -- Python 3.8 or higher -- pip (Python package installer) +Before installing Semantica, ensure you have: + +- **Python 3.8 or higher** - Check your version: + ```bash + python --version + ``` +- **pip** - Python package installer (usually comes with Python) ## Basic Installation @@ -13,60 +20,215 @@ Install Semantica from PyPI: pip install semantica ``` +This installs Semantica with all core dependencies. + ## Verify Installation -Check that Semantica is installed correctly: +Verify that Semantica is installed correctly: ```bash python -c "import semantica; print(semantica.__version__)" ``` -You should see: `0.0.1` +Expected output: +``` +0.0.1 +``` + +You can also check the installation: + +```bash +pip show semantica +``` ## Development Installation -To install Semantica in development mode: +To install Semantica in development mode (for contributing): ```bash +# Clone the repository git clone https://github.com/Hawksight-AI/semantica.git cd semantica + +# Install in editable mode pip install -e . + +# Or install with development dependencies +pip install -e ".[dev]" ``` ## Optional Dependencies -Install optional features: +Semantica supports optional features that can be installed separately: + +### GPU Support + +For GPU-accelerated operations: ```bash -# GPU support pip install semantica[gpu] +``` -# Visualization +This includes: +- PyTorch with CUDA support +- FAISS GPU +- CuPy + +### Visualization + +For enhanced visualization capabilities: + +```bash pip install semantica[viz] +``` -# All LLM providers +Includes: +- PyVis for interactive graphs +- Graphviz for static diagrams +- UMAP for dimensionality reduction + +### LLM Providers + +Install all LLM provider integrations: + +```bash pip install semantica[llm-all] +``` -# Cloud integrations +Or install specific providers: + +```bash +# OpenAI +pip install semantica[llm-openai] + +# Anthropic +pip install semantica[llm-anthropic] + +# Google Gemini +pip install semantica[llm-gemini] + +# Groq +pip install semantica[llm-groq] + +# Ollama +pip install semantica[llm-ollama] +``` + +### Cloud Integrations + +For cloud storage and deployment: + +```bash pip install semantica[cloud] ``` +Includes: +- AWS S3 (boto3) +- Azure Blob Storage +- Google Cloud Storage +- Kubernetes support + +### All Optional Features + +Install everything: + +```bash +pip install semantica[all] +``` + +## Virtual Environment (Recommended) + +It's recommended to use a virtual environment: + +=== "venv" + + ```bash + # Create virtual environment + python -m venv venv + + # Activate (Windows) + venv\Scripts\activate + + # Activate (Linux/Mac) + source venv/bin/activate + + # Install Semantica + pip install semantica + ``` + +=== "conda" + + ```bash + # Create conda environment + conda create -n semantica python=3.11 + conda activate semantica + + # Install Semantica + pip install semantica + ``` + ## Troubleshooting ### Common Issues -**Issue**: `ModuleNotFoundError: No module named 'semantica'` -- **Solution**: Make sure you've activated the correct Python environment +#### ModuleNotFoundError -**Issue**: Installation fails -- **Solution**: Upgrade pip: `pip install --upgrade pip` +**Error**: `ModuleNotFoundError: No module named 'semantica'` -**Issue**: GPU dependencies fail -- **Solution**: Install CPU-only version first, then add GPU support +**Solutions**: +- Make sure you've activated the correct Python environment +- Verify installation: `pip list | grep semantica` +- Reinstall: `pip install --upgrade semantica` + +#### Installation Fails + +**Error**: Installation fails with dependency errors + +**Solutions**: +- Upgrade pip: `pip install --upgrade pip` +- Install build tools: `pip install build wheel` +- Try installing without optional dependencies first: `pip install semantica --no-deps` + +#### GPU Dependencies Fail + +**Error**: GPU dependencies fail to install + +**Solutions**: +- Install CPU-only version first: `pip install semantica` +- Then add GPU support: `pip install semantica[gpu]` +- Check CUDA compatibility for your system + +#### Permission Errors + +**Error**: Permission denied during installation + +**Solutions**: +- Use `--user` flag: `pip install --user semantica` +- Use virtual environment (recommended) +- On Linux/Mac, avoid using `sudo` with pip + +### System Requirements + +| Component | Minimum | Recommended | +|-----------|---------|-------------| +| Python | 3.8 | 3.11+ | +| RAM | 4 GB | 8 GB+ | +| Disk Space | 2 GB | 5 GB+ | +| OS | Windows/Linux/Mac | Linux/Mac | ## Next Steps -- [Quick Start Guide](quickstart.md) -- [Configuration](configuration.md) -- [Examples](examples.md) +Now that Semantica is installed: +1. **[Quick Start Guide](quickstart.md)** - Build your first knowledge graph +2. **[Examples](examples.md)** - See real-world use cases +3. **[API Reference](api.md)** - Explore the full API +4. **[Cookbook](cookbook.md)** - Interactive tutorials + +## Getting Help + +If you encounter issues: + +- Check the [troubleshooting section](#troubleshooting) above +- Review [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues) +- Ask questions in discussions diff --git a/docs/learning-more.md b/docs/learning-more.md new file mode 100644 index 00000000..b0b72654 --- /dev/null +++ b/docs/learning-more.md @@ -0,0 +1,173 @@ +# Learning More + +Additional resources, tutorials, and advanced learning materials for Semantica. + +## Additional Tutorials + +### Video Tutorials + +Coming soon! We're working on video tutorials covering: + +- Getting started with Semantica +- Building your first knowledge graph +- Advanced techniques and patterns +- Real-world use cases + +### Blog Posts & Articles + +Stay tuned for blog posts covering: + +- Best practices for knowledge graph construction +- Performance optimization tips +- Integration guides +- Case studies and success stories + +## Best Practices + +### Knowledge Graph Design + +1. **Start with Clear Objectives** + - Define what you want to extract + - Identify key entities and relationships + - Plan your schema before processing + +2. **Iterate and Refine** + - Start with a small dataset + - Validate extracted entities + - Refine extraction patterns + - Scale up gradually + +3. **Quality Over Quantity** + - Focus on accuracy + - Validate relationships + - Resolve conflicts early + - Maintain data quality + +### Performance Tips + +```python +# Process in batches for large datasets +sources = ["doc1.pdf", "doc2.pdf", "doc3.pdf"] +batch_size = 10 + +for i in range(0, len(sources), batch_size): + batch = sources[i:i+batch_size] + result = semantica.build_knowledge_base(batch) + # Process and save results +``` + +### Integration Patterns + +#### Pattern 1: Incremental Building + +```python +# Build knowledge graph incrementally +kg = None +for source in sources: + result = semantica.build_knowledge_base([source]) + if kg is None: + kg = result["knowledge_graph"] + else: + kg = semantica.kg.merge([kg, result["knowledge_graph"]]) +``` + +#### Pattern 2: Pipeline Processing + +```python +# Create a processing pipeline +pipeline = [ + ("ingest", semantica.ingest.from_file), + ("parse", semantica.parse.document), + ("extract", semantica.semantic_extract.entities), + ("build", semantica.kg.build_graph) +] + +for step_name, step_func in pipeline: + data = step_func(data) +``` + +## Advanced Topics + +### Custom Extractors + +Create custom entity extractors: + +```python +from semantica.semantic_extract import BaseExtractor + +class CustomExtractor(BaseExtractor): + def extract(self, text): + # Your custom extraction logic + return entities +``` + +### Custom Export Formats + +Add custom export formats: + +```python +from semantica.export import BaseExporter + +class CustomExporter(BaseExporter): + def export(self, kg, path): + # Your custom export logic + pass +``` + +### Performance Optimization + +- Use GPU acceleration when available +- Process documents in parallel +- Cache embeddings +- Optimize graph queries + +## Community Resources + +### GitHub Discussions + +Join discussions on: +- [General Discussion](https://github.com/Hawksight-AI/semantica/discussions) +- [Q&A](https://github.com/Hawksight-AI/semantica/discussions/categories/q-a) +- [Show and Tell](https://github.com/Hawksight-AI/semantica/discussions/categories/show-and-tell) + +### Contributing + +Want to contribute? See our [Contributing Guide](https://github.com/Hawksight-AI/semantica/blob/main/CONTRIBUTING.md). + +### Examples Repository + +Check out the [examples repository](https://github.com/Hawksight-AI/semantica/tree/main/examples) for more code samples. + +## Related Projects + +### GraphRAG + +Semantica works great with GraphRAG implementations. See our [GraphRAG examples](cookbook.md#advanced-rag). + +### Vector Databases + +Integrate with vector databases: +- Pinecone +- Weaviate +- Qdrant +- Milvus + +### Knowledge Graph Databases + +Export to and work with: +- Neo4j +- Amazon Neptune +- ArangoDB +- Blazegraph + +## Next Steps + +- **[Deep Dive](deep-dive.md)** - Advanced architecture and internals +- **[API Reference](api.md)** - Complete API documentation +- **[Cookbook](cookbook.md)** - Interactive tutorials +- **[Examples](examples.md)** - More code examples + +--- + +Have questions or suggestions? [Open an issue](https://github.com/Hawksight-AI/semantica/issues) or [start a discussion](https://github.com/Hawksight-AI/semantica/discussions)! + diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 00000000..dfa54b95 --- /dev/null +++ b/docs/license.md @@ -0,0 +1,91 @@ +# License + +Semantica is released under the MIT License. + +## MIT License + +``` +MIT License + +Copyright (c) 2024 Hawksight AI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` + +## What This Means + +### You Can: + +- ✅ Use Semantica commercially +- ✅ Modify the source code +- ✅ Distribute the software +- ✅ Use it in private projects +- ✅ Sublicense it +- ✅ Use it in proprietary software + +### You Must: + +- ✅ Include the copyright notice +- ✅ Include the license text + +### You Cannot: + +- ❌ Hold authors liable for damages +- ❌ Use the authors' names to endorse products without permission + +## Third-Party Licenses + +Semantica uses several open-source libraries. Their licenses are included in the distribution. Key dependencies: + +- **Python**: PSF License +- **NumPy**: BSD License +- **Pandas**: BSD License +- **spaCy**: MIT License +- **Transformers**: Apache 2.0 License +- **RDFLib**: BSD License + +See the full list in `LICENSE` file or check individual package licenses. + +## Commercial Use + +Semantica is **free for commercial use**. You can: + +- Use it in commercial products +- Build commercial services with it +- Include it in proprietary software +- Sell products that use Semantica + +No attribution required in your product, though we appreciate it! + +## Contributing + +By contributing to Semantica, you agree that your contributions will be licensed under the MIT License. + +## Questions? + +If you have questions about the license: + +- Check the [full license text](../LICENSE) +- [Open an issue](https://github.com/Hawksight-AI/semantica/issues) +- [Start a discussion](https://github.com/Hawksight-AI/semantica/discussions) + +--- + +**Semantica is 100% open source and free to use!** 🎉 + diff --git a/docs/netlify.toml b/docs/netlify.toml new file mode 100644 index 00000000..764f0cfb --- /dev/null +++ b/docs/netlify.toml @@ -0,0 +1,12 @@ +[build] + command = "cd docs && bundle install && bundle exec jekyll build -d ../_site" + publish = "_site" + +[build.environment] + RUBY_VERSION = "3.1" + +[[redirects]] + from = "/*" + to = "/index.html" + status = 200 + diff --git a/docs/quickstart.md b/docs/quickstart.md index 2dc839b2..acd7e972 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,8 +1,34 @@ -# Quick Start Guide +# Quickstart -Get started with Semantica in 5 minutes! +Get started with Semantica in 5 minutes. This guide will walk you through building your first knowledge graph. -## Basic Example +## Overview + +```mermaid +flowchart LR + A[Install] --> B[Initialize] + B --> C[Load Data] + C --> D[Extract] + D --> E[Build Graph] + E --> F[Visualize] + + style A fill:#e3f2fd + style F fill:#c8e6c9 +``` + +## Step 1: Installation + +If you haven't installed Semantica yet: + +```bash +pip install semantica +``` + +See the [Installation Guide](installation.md) for detailed instructions. + +## Step 2: Your First Knowledge Graph + +Let's build a knowledge graph from a document: ```python from semantica import Semantica @@ -24,60 +50,203 @@ statistics = result["statistics"] print(f"Extracted {len(kg['entities'])} entities") print(f"Created {len(kg['relationships'])} relationships") +print(f"Generated {len(embeddings)} embeddings") ``` -## Extract Entities and Relationships +**Expected Output:** +``` +Extracted 45 entities +Created 32 relationships +Generated 45 embeddings +``` + +## Step 3: Extract Entities and Relationships + +Extract structured information from text: ```python from semantica import Semantica semantica = Semantica() -# Extract from text -text = "Apple Inc. was founded by Steve Jobs in Cupertino, California." +# Sample text +text = """ +Apple Inc. was founded by Steve Jobs in Cupertino, California in 1976. +The company designs and manufactures consumer electronics and software. +Tim Cook is the current CEO of Apple. +""" -result = semantica.semantic_extract.extract_entities(text) -entities = result["entities"] +# Extract entities +entities_result = semantica.semantic_extract.extract_entities(text) +entities = entities_result["entities"] +print("Extracted Entities:") for entity in entities: - print(f"{entity['text']} - {entity['type']}") + print(f" - {entity['text']} ({entity['type']})") + +# Extract relationships +relationships_result = semantica.semantic_extract.extract_relationships(text) +relationships = relationships_result["relationships"] + +print("\nExtracted Relationships:") +for rel in relationships: + print(f" - {rel['subject']} --[{rel['predicate']}]--> {rel['object']}") ``` -## Build Knowledge Graph +**Expected Output:** +``` +Extracted Entities: + - Apple Inc. (ORGANIZATION) + - Steve Jobs (PERSON) + - Cupertino (LOCATION) + - California (LOCATION) + - Tim Cook (PERSON) + +Extracted Relationships: + - Apple Inc. --[founded_by]--> Steve Jobs + - Apple Inc. --[located_in]--> Cupertino + - Apple Inc. --[has_ceo]--> Tim Cook +``` + +## Step 4: Build Knowledge Graph from Multiple Sources + +Combine data from multiple sources: ```python from semantica import Semantica semantica = Semantica() -# Build KG from multiple sources +# Multiple data sources sources = [ - "document1.pdf", - "document2.docx", - "https://example.com/article" + "documents/research_paper.pdf", + "documents/company_report.docx", + "https://example.com/news-article" ] -kg = semantica.kg.build_graph(sources) -semantica.kg.visualize(kg) +# Build unified knowledge graph +result = semantica.build_knowledge_base( + sources=sources, + embeddings=True, + graph=True, + normalize=True +) + +kg = result["knowledge_graph"] + +# Analyze the graph +print(f"Total entities: {len(kg['entities'])}") +print(f"Total relationships: {len(kg['relationships'])}") +print(f"Sources processed: {len(result['metadata']['sources'])}") ``` -## Export Knowledge Graph +## Step 5: Visualize Your Knowledge Graph + +Visualize the knowledge graph you created: ```python from semantica import Semantica semantica = Semantica() -kg = semantica.kg.build_graph(["data.pdf"]) + +# Build graph +result = semantica.build_knowledge_base(["document.pdf"]) +kg = result["knowledge_graph"] + +# Visualize +semantica.kg.visualize(kg, output_path="graph.html") +print("Graph visualization saved to graph.html") +``` + +Open `graph.html` in your browser to see an interactive visualization. + +## Step 6: Export Your Knowledge Graph + +Export your knowledge graph in various formats: + +```python +from semantica import Semantica + +semantica = Semantica() + +# Build graph +result = semantica.build_knowledge_base(["data.pdf"]) +kg = result["knowledge_graph"] # Export to different formats -semantica.export.to_rdf(kg, "output.rdf") -semantica.export.to_json(kg, "output.json") -semantica.export.to_csv(kg, "output.csv") +semantica.export.to_rdf(kg, "output.rdf") # RDF/XML format +semantica.export.to_json(kg, "output.json") # JSON format +semantica.export.to_csv(kg, "output.csv") # CSV format +semantica.export.to_owl(kg, "output.owl") # OWL ontology format + +print("Exported knowledge graph to multiple formats") +``` + +## Common Patterns + +### Pattern 1: Process Text Directly + +```python +from semantica import Semantica + +semantica = Semantica() + +text = "Your text content here..." +result = semantica.process_document(text) +``` + +### Pattern 2: Custom Configuration + +```python +from semantica import Semantica, Config + +# Create custom configuration +config = Config( + embeddings=True, + graph=True, + normalize=True, + conflict_resolution="voting" +) + +semantica = Semantica(config=config) +result = semantica.build_knowledge_base(["document.pdf"]) +``` + +### Pattern 3: Incremental Building + +```python +from semantica import Semantica + +semantica = Semantica() + +# Build incrementally +kg1 = semantica.kg.build_graph(["source1.pdf"]) +kg2 = semantica.kg.build_graph(["source2.pdf"]) + +# Merge knowledge graphs +merged_kg = semantica.kg.merge([kg1, kg2]) ``` ## Next Steps -- [Full Documentation](../README.md) -- [API Reference](api.md) -- [More Examples](examples.md) +Now that you've built your first knowledge graph: +1. **[Explore Examples](examples.md)** - See more advanced use cases +2. **[API Reference](api.md)** - Learn about all available methods +3. **[Cookbook](cookbook.md)** - Interactive Jupyter notebooks +4. **[Full Documentation](../README.md)** - Comprehensive guide + +## Troubleshooting + +### Common Issues + +**Issue**: No entities extracted +- **Solution**: Check that your document contains text content. PDFs with images only won't work without OCR. + +**Issue**: Slow processing +- **Solution**: For large documents, consider processing in chunks or using GPU acceleration. + +**Issue**: Memory errors +- **Solution**: Process documents one at a time or reduce batch sizes. + +Need help? Check the [Installation Troubleshooting](installation.md#troubleshooting) or [GitHub Issues](https://github.com/Hawksight-AI/semantica/issues). diff --git a/docs/vercel.json b/docs/vercel.json new file mode 100644 index 00000000..e3274d33 --- /dev/null +++ b/docs/vercel.json @@ -0,0 +1,12 @@ +{ + "buildCommand": "cd docs && bundle install && bundle exec jekyll build -d ../_site", + "outputDirectory": "_site", + "framework": null, + "rewrites": [ + { + "source": "/(.*)", + "destination": "/index.html" + } + ] +} + diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..844d9425 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,103 @@ +site_name: Semantica +site_description: Open Source Framework for Semantic Intelligence & Knowledge Engineering +site_url: https://semantica.dev +repo_url: https://github.com/Hawksight-AI/semantica +repo_name: Hawksight-AI/semantica +edit_uri: edit/main/docs/ + +# Copyright +copyright: Copyright © 2024 Hawksight AI + +# Theme Configuration +theme: + name: material + palette: + # Light mode + - scheme: default + primary: indigo + accent: indigo + toggle: + icon: material/brightness-7 + name: Switch to dark mode + # Dark mode + - scheme: slate + primary: indigo + accent: indigo + toggle: + icon: material/brightness-4 + name: Switch to light mode + features: + - navigation.tabs + - navigation.sections + - navigation.expand + - navigation.top + - navigation.indexes + - navigation.tracking + - search.suggest + - search.highlight + - search.share + - content.code.copy + - content.code.annotate + - content.tooltips + icon: + repo: fontawesome/brands/github + +# Extensions +markdown_extensions: + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.emoji: + emoji_index: !!python/name:materialx.emoji.twemoji + emoji_generator: !!python/name:materialx.emoji.to_svg + - pymdownx.tabbed: + alternate_style: true + - pymdownx.tasklist: + custom_checkbox: true + - admonition + - pymdownx.details + - attr_list + - md_in_html + - tables + - toc: + permalink: true + +# Plugins +plugins: + - search: + lang: en + - minify: + minify_html: true + +# Custom CSS +extra_css: + - css/custom.css + +# Navigation +nav: + - Home: index.md + - Quickstart: quickstart.md + - Installation: installation.md + - Cookbook Recipes: cookbook.md + - Learning More: learning-more.md + - Deep Dive: deep-dive.md + - API References: api.md + +# Extra +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/Hawksight-AI/semantica + - icon: fontawesome/brands/python + link: https://pypi.org/project/semantica/ + version: + provider: mike + diff --git a/netlify.toml b/netlify.toml new file mode 100644 index 00000000..ed2871b0 --- /dev/null +++ b/netlify.toml @@ -0,0 +1,12 @@ +[build] + command = "pip install -r requirements-docs.txt && mkdocs build" + publish = "site" + +[build.environment] + PYTHON_VERSION = "3.11" + +[[redirects]] + from = "/*" + to = "/index.html" + status = 200 + diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 00000000..ac18083e --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,7 @@ +mkdocs>=1.5.0 +mkdocs-material>=9.4.0 +mkdocs-minify-plugin>=0.7.0 +mkdocs-mermaid2-plugin>=1.0.0 +pymdown-extensions>=10.0 +materialx>=2.2.0 + diff --git a/test-docs.py b/test-docs.py new file mode 100644 index 00000000..1ef9ebea --- /dev/null +++ b/test-docs.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +""" +Simple HTTP server to test documentation locally +Run: python test-docs.py +Then visit: http://localhost:8000/preview.html +""" + +import http.server +import socketserver +import os +import webbrowser +from pathlib import Path + +PORT = 8000 + +class DocsHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): + def end_headers(self): + # Add CORS headers for local testing + self.send_header('Access-Control-Allow-Origin', '*') + self.send_header('Access-Control-Allow-Methods', 'GET') + super().end_headers() + +def main(): + # Change to project root directory + os.chdir(Path(__file__).parent) + + Handler = DocsHTTPRequestHandler + + with socketserver.TCPServer(("", PORT), Handler) as httpd: + print(f""" +╔═══════════════════════════════════════════════════════╗ +║ Semantica Documentation Test Server ║ +╠═══════════════════════════════════════════════════════╣ +║ ║ +║ Server running at: ║ +║ http://localhost:{PORT} ║ +║ ║ +║ Available pages: ║ +║ • http://localhost:{PORT}/preview.html ║ +║ • http://localhost:{PORT}/docs/index.md ║ +║ • http://localhost:{PORT}/docs/installation.md ║ +║ • http://localhost:{PORT}/docs/quickstart.md ║ +║ • http://localhost:{PORT}/docs/api.md ║ +║ • http://localhost:{PORT}/docs/examples.md ║ +║ ║ +║ Press Ctrl+C to stop the server ║ +╚═══════════════════════════════════════════════════════╝ + """) + + # Open browser automatically + try: + webbrowser.open(f'http://localhost:{PORT}/docs/preview.html') + except: + pass + + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\n\nServer stopped. Goodbye!") + +if __name__ == "__main__": + main() +