diff --git a/cookbook/advanced/09_Semantic_Layer_Construction.ipynb b/cookbook/advanced/09_Semantic_Layer_Construction.ipynb index 6344cca8..04a86656 100644 --- a/cookbook/advanced/09_Semantic_Layer_Construction.ipynb +++ b/cookbook/advanced/09_Semantic_Layer_Construction.ipynb @@ -25,21 +25,37 @@ "pip install semantica[all]\n", "```\n", "\n", - "## Workflow: Build KG \u2192 Generate Ontology \u2192 Create Semantic Layer \u2192 Export RDF \u2192 Triplet Store\n" + "## Workflow: Build KG → Generate Ontology → Create Semantic Layer → Export RDF \n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~gno (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~lotly (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n", + "WARNING: Ignoring invalid distribution ~ython-socketio (C:\\Users\\Mohd Kaif\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages)\n" + ] + } + ], "source": [ - "!pip install semantica\n" + "!pip install -qU semantica\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -58,9 +74,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "

🧠 Semantica - 📊 Current Progress

StatusActionModuleSubmoduleFileTime
✅Semantica is building🧠 kgGraphBuilder-0.13s
🔄Semantica is building🧠 kgEntityResolver-7.37s
✅Semantica is deduplicating🔄 deduplicationDuplicateDetector-0.09s
✅Semantica is deduplicating🔄 deduplicationSimilarityCalculator-0.01s
✅Semantica is resolving⚠️ conflictsConflictDetector-0.00s
✅Semantica is generating📚 ontologyOntologyGenerator-0.06s
✅Semantica is generating📚 ontologyClassInferrer-0.01s
✅Semantica is generating📚 ontologyPropertyGenerator-0.01s
✅Semantica is exporting💾 exportRDFExporter-0.01s
✅Semantica is storing🗄️ triplet_storeBulkLoader-3.01s
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "builder = GraphBuilder()\n", "\n", @@ -90,7 +119,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -107,7 +136,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -159,9 +188,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Exported knowledge graph to knowledge_graph.ttl\n" + ] + } + ], "source": [ "exporter = RDFExporter()\n", "# Export Knowledge Graph\n", @@ -169,23 +206,6 @@ "print(\"Exported knowledge graph to knowledge_graph.ttl\")\n" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: Store in Triplet Store\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "triplet_store = TripletStore()\n", - "triplet_store.store(knowledge_graph, ontology)\n" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -196,16 +216,44 @@ "- Knowledge Graph Built\n", "- Ontology Generated\n", "- Semantic Layer Created with Mappings\n", - "- RDF Export Completed\n", - "- Triplet Store Storage Completed\n" + "- RDF Export Completed\n" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] } ], "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" } }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/cookbook/advanced/knowledge_graph.ttl b/cookbook/advanced/knowledge_graph.ttl new file mode 100644 index 00000000..3cbf4ae0 --- /dev/null +++ b/cookbook/advanced/knowledge_graph.ttl @@ -0,0 +1,20 @@ + a ; + semantica:text "" ; + semantica:confidence 1.0 . + + a ; + semantica:text "" ; + semantica:confidence 1.0 . + + a ; + semantica:text "" ; + semantica:confidence 1.0 . + + a ; + semantica:text "" ; + semantica:confidence 1.0 . + + . + . + . + . \ No newline at end of file diff --git a/semantica/triplet_store/triplet_store.py b/semantica/triplet_store/triplet_store.py index 6c110264..fbc36965 100644 --- a/semantica/triplet_store/triplet_store.py +++ b/semantica/triplet_store/triplet_store.py @@ -93,6 +93,11 @@ class TripletStore: backend_config = self.config.copy() if self.endpoint: backend_config["endpoint"] = self.endpoint + else: + backend_config["endpoint"] = triplet_store_config.get( + "blazegraph_endpoint", + "http://localhost:9999/blazegraph" + ) self._store_backend = BlazegraphStore(**backend_config) @@ -101,9 +106,12 @@ class TripletStore: backend_config = self.config.copy() if self.endpoint: - # JenaStore might use different param names or structure - # Assuming JenaStore accepts endpoint or url - backend_config["url"] = self.endpoint + backend_config["endpoint"] = self.endpoint + else: + backend_config["endpoint"] = triplet_store_config.get( + "jena_endpoint", + "http://localhost:3030/ds" + ) self._store_backend = JenaStore(**backend_config) @@ -113,6 +121,11 @@ class TripletStore: backend_config = self.config.copy() if self.endpoint: backend_config["endpoint"] = self.endpoint + else: + backend_config["endpoint"] = triplet_store_config.get( + "rdf4j_endpoint", + "http://localhost:8080/rdf4j-server" + ) self._store_backend = RDF4JStore(**backend_config) @@ -122,6 +135,156 @@ class TripletStore: self.logger.error(f"Failed to initialize {self.backend_type} backend: {e}") raise ProcessingError(f"Failed to initialize backend: {e}") + def store( + self, + knowledge_graph: Union[Dict[str, Any], Any], + ontology: Union[Dict[str, Any], Any], + **options + ) -> Dict[str, Any]: + """ + Store knowledge graph and ontology in the triplet store. + + Args: + knowledge_graph: Knowledge graph dictionary or object + ontology: Ontology dictionary or object + **options: Additional options + + Returns: + Operation status + """ + # Convert inputs to dictionaries if they are objects + if hasattr(knowledge_graph, "to_dict"): + knowledge_graph = knowledge_graph.to_dict() + if hasattr(ontology, "to_dict"): + ontology = ontology.to_dict() + + triplets = [] + + # Standard Namespaces + RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" + RDFS_SUBCLASS = "http://www.w3.org/2000/01/rdf-schema#subClassOf" + OWL_CLASS = "http://www.w3.org/2002/07/owl#Class" + OWL_OBJECT_PROPERTY = "http://www.w3.org/2002/07/owl#ObjectProperty" + OWL_DATATYPE_PROPERTY = "http://www.w3.org/2002/07/owl#DatatypeProperty" + RDFS_DOMAIN = "http://www.w3.org/2000/01/rdf-schema#domain" + RDFS_RANGE = "http://www.w3.org/2000/01/rdf-schema#range" + + # 1. Process Ontology + classes = ontology.get("classes", []) + properties = ontology.get("properties", []) + + for cls in classes: + # Class definition + cls_uri = cls.get("uri") or cls.get("id") or cls.get("name") + if not cls_uri: + continue + + if not cls_uri.startswith("http") and not cls_uri.startswith("urn:"): + # Fallback if no URI provided + cls_uri = f"urn:class:{cls_uri}" + + triplets.append(Triplet(cls_uri, RDF_TYPE, OWL_CLASS)) + + # Hierarchy + parent = cls.get("parent") or cls.get("subClassOf") + if parent: + parent_uri = parent + if not parent.startswith("http") and not parent.startswith("urn:"): + parent_uri = f"urn:class:{parent}" + triplets.append(Triplet(cls_uri, RDFS_SUBCLASS, parent_uri)) + + for prop in properties: + prop_uri = prop.get("uri") or prop.get("id") or prop.get("name") + if not prop_uri: + continue + + if not prop_uri.startswith("http") and not prop_uri.startswith("urn:"): + prop_uri = f"urn:property:{prop_uri}" + + # Determine property type (Object or Datatype) + # Default to ObjectProperty if not specified + prop_type = prop.get("type", OWL_OBJECT_PROPERTY) + if prop_type == "datatype": + prop_type = OWL_DATATYPE_PROPERTY + elif prop_type == "object": + prop_type = OWL_OBJECT_PROPERTY + + triplets.append(Triplet(prop_uri, RDF_TYPE, prop_type)) + + if "domain" in prop: + domains = prop["domain"] + if isinstance(domains, str): + domains = [domains] + + for domain in domains: + domain_uri = domain + if not domain.startswith("http") and not domain.startswith("urn:"): + domain_uri = f"urn:class:{domain}" + triplets.append(Triplet(prop_uri, RDFS_DOMAIN, domain_uri)) + + if "range" in prop: + ranges = prop["range"] + if isinstance(ranges, str): + ranges = [ranges] + + for range_ in ranges: + range_uri = range_ + if not range_.startswith("http") and not range_.startswith("urn:"): + range_uri = f"urn:class:{range_}" + triplets.append(Triplet(prop_uri, RDFS_RANGE, range_uri)) + + # 2. Process Knowledge Graph + entities = knowledge_graph.get("entities", []) + relationships = knowledge_graph.get("relationships", []) + + entity_map = {} # Map IDs to URIs + + for entity in entities: + entity_id = entity.get("id") + if not entity_id: + continue + + entity_uri = entity.get("uri") + if not entity_uri: + entity_uri = f"urn:entity:{entity_id}" + + entity_map[entity_id] = entity_uri + + # Entity Type + entity_type = entity.get("type") + if entity_type: + type_uri = entity_type + if not entity_type.startswith("http") and not entity_type.startswith("urn:"): + type_uri = f"urn:class:{entity_type}" + triplets.append(Triplet(entity_uri, RDF_TYPE, type_uri)) + + # Entity Properties + props = entity.get("properties", {}) + for k, v in props.items(): + prop_uri = k + if not k.startswith("http") and not k.startswith("urn:"): + prop_uri = f"urn:property:{k}" + triplets.append(Triplet(entity_uri, prop_uri, str(v))) + + for rel in relationships: + source_id = rel.get("source") + target_id = rel.get("target") + rel_type = rel.get("type") or rel.get("label") + + if not source_id or not target_id or not rel_type: + continue + + source_uri = entity_map.get(source_id, f"urn:entity:{source_id}") + target_uri = entity_map.get(target_id, f"urn:entity:{target_id}") + rel_uri = rel_type + if not rel_type.startswith("http") and not rel_type.startswith("urn:"): + rel_uri = f"urn:property:{rel_type}" + + triplets.append(Triplet(source_uri, rel_uri, target_uri)) + + # Bulk load all triplets + return self.add_triplets(triplets, **options) + def add_triplet(self, triplet: Triplet, **options) -> Dict[str, Any]: """ Add a single triplet to the store.