src/2024-09-skos.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Turning OpenAlex topics hierarchy into into SKOS\n",
    "\n",
    "See https://www.w3.org/2004/02/skos/intro\n",
    "\n",
    "Example SKOS fragment in turtle syntax: \n",
    "\n",
    "```\n",
    "@prefix :            <http://ns.nature.com/terms/> .\n",
    "@prefix bibo:        <http://purl.org/ontology/bibo/> .\n",
    "@prefix dc:          <http://purl.org/dc/elements/1.1/> .\n",
    "@prefix dcterms:     <http://purl.org/dc/terms/> .\n",
    "@prefix foaf:        <http://xmlns.com/foaf/0.1/> .\n",
    "@prefix npg:         <http://ns.nature.com/terms/> .\n",
    "@prefix npgd:        <http://ns.nature.com/datasets/> .\n",
    "@prefix npgg:        <http://ns.nature.com/graphs/> .\n",
    "@prefix owl:         <http://www.w3.org/2002/07/owl#> .\n",
    "@prefix prism:       <http://prismstandard.org/namespaces/basic/2.1/> .\n",
    "@prefix rdf:         <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .\n",
    "@prefix rdfs:        <http://www.w3.org/2000/01/rdf-schema#> .\n",
    "@prefix skos:        <http://www.w3.org/2004/02/skos/core#> .\n",
    "@prefix vann:        <http://purl.org/vocab/vann/> .\n",
    "@prefix xsd:         <http://www.w3.org/2001/XMLSchema#> .\n",
    "\n",
    "@prefix article-types: <http://ns.nature.com/article-types/> .\n",
    "\n",
    "article-types:\n",
    "    npg:webpage <http://www.nature.com/ontologies/models/article-types/> ;\n",
    "    dc:date \"2015-08-17T06:12:02.390-04:00\" ;\n",
    "    dc:publisher \"Macmillan Publishers Limited\" ;\n",
    "    dc:rights \"This work is distributed under a Creative Commons Zero 1.0 (CC0 1.0) Public Domain Dedication <http://creativecommons.org/publicdomain/zero/1.0/>.\"@en ;\n",
    "    dcterms:license <http://creativecommons.org/publicdomain/zero/1.0/> ;\n",
    "    vann:preferredNamespacePrefix \"article-types\" ;\n",
    "    vann:preferredNamespaceUri \"http://ns.nature.com/article-types/\" ;\n",
    "    a owl:Ontology, skos:ConceptScheme ;\n",
    "    owl:imports <http://ns.nature.com/terms/> ;\n",
    "    owl:versionInfo \"1.9.45\" ;\n",
    "    skos:definition \"The ArticleTypes Ontology is a categorization of kinds of publication which are used to index and group content published by Macmillan Science and Education.\"@en ;\n",
    "    skos:prefLabel \"ArticleTypes Ontology\"@en .\n",
    "\n",
    "article-types:abstracts-collection\n",
    "    npg:hasRoot article-types:news-and-comment ;\n",
    "    npg:id \"abstracts-collection\" ;\n",
    "    npg:isLeaf true ;\n",
    "    npg:isRoot false ;\n",
    "    npg:treeDepth 2 ;\n",
    "    a npg:ArticleType , skos:Concept ;\n",
    "    rdfs:isDefinedBy <http://ns.nature.com/article-types/> ;\n",
    "    skos:broader article-types:research-highlights ;\n",
    "    skos:definition \"The :abstracts-collection article-type denotes a collection of abstracts from a recent conference.\"@en ;\n",
    "    skos:inScheme <http://ns.nature.com/article-types/> ;\n",
    "    skos:prefLabel \"Abstracts Collection\"@en .\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting started and loading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# !pip install ontospy\n",
    "import ontospy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>display_name</th>\n",
       "      <th>description</th>\n",
       "      <th>ids</th>\n",
       "      <th>display_name_alternatives</th>\n",
       "      <th>fields</th>\n",
       "      <th>siblings</th>\n",
       "      <th>works_count</th>\n",
       "      <th>cited_by_count</th>\n",
       "      <th>works_api_url</th>\n",
       "      <th>updated_date</th>\n",
       "      <th>created_date</th>\n",
       "      <th>entity_type</th>\n",
       "      <th>domain</th>\n",
       "      <th>subfields</th>\n",
       "      <th>field</th>\n",
       "      <th>topics</th>\n",
       "      <th>keywords</th>\n",
       "      <th>subfield</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://openalex.org/domains/3</td>\n",
       "      <td>Physical Sciences</td>\n",
       "      <td>branch of natural science that studies non-liv...</td>\n",
       "      <td>{'wikidata': 'https://www.wikidata.org/wiki/Q1...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'id': 'https://openalex.org/fields/15', 'dis...</td>\n",
       "      <td>[{'id': 'https://openalex.org/domains/4', 'dis...</td>\n",
       "      <td>73237366</td>\n",
       "      <td>770120022</td>\n",
       "      <td>https://api.openalex.org/works?filter=topics.d...</td>\n",
       "      <td>2024-04-01T04:50:55.583031</td>\n",
       "      <td>2024-01-23</td>\n",
       "      <td>domains</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://openalex.org/domains/2</td>\n",
       "      <td>Social Sciences</td>\n",
       "      <td>branch of science focused on societies and the...</td>\n",
       "      <td>{'wikidata': 'https://www.wikidata.org/wiki/Q3...</td>\n",
       "      <td>[political sciences]</td>\n",
       "      <td>[{'id': 'https://openalex.org/fields/12', 'dis...</td>\n",
       "      <td>[{'id': 'https://openalex.org/domains/4', 'dis...</td>\n",
       "      <td>64995390</td>\n",
       "      <td>273064988</td>\n",
       "      <td>https://api.openalex.org/works?filter=topics.d...</td>\n",
       "      <td>2024-04-01T04:50:50.146788</td>\n",
       "      <td>2024-01-23</td>\n",
       "      <td>domains</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://openalex.org/domains/4</td>\n",
       "      <td>Health Sciences</td>\n",
       "      <td>branch of science focused on human health and ...</td>\n",
       "      <td>{'wikidata': 'https://www.wikidata.org/wiki/Q1...</td>\n",
       "      <td>[medical sciences, biomedical sciences, health...</td>\n",
       "      <td>[{'id': 'https://openalex.org/fields/35', 'dis...</td>\n",
       "      <td>[{'id': 'https://openalex.org/domains/1', 'dis...</td>\n",
       "      <td>43957129</td>\n",
       "      <td>480875956</td>\n",
       "      <td>https://api.openalex.org/works?filter=topics.d...</td>\n",
       "      <td>2024-04-01T04:50:50.416213</td>\n",
       "      <td>2024-01-23</td>\n",
       "      <td>domains</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://openalex.org/domains/1</td>\n",
       "      <td>Life Sciences</td>\n",
       "      <td>branch of science that involve the scientific ...</td>\n",
       "      <td>{'wikidata': 'https://www.wikidata.org/wiki/Q8...</td>\n",
       "      <td>[bioscience, biological science disciplines]</td>\n",
       "      <td>[{'id': 'https://openalex.org/fields/11', 'dis...</td>\n",
       "      <td>[{'id': 'https://openalex.org/domains/4', 'dis...</td>\n",
       "      <td>27536918</td>\n",
       "      <td>445784583</td>\n",
       "      <td>https://api.openalex.org/works?filter=topics.d...</td>\n",
       "      <td>2024-04-01T04:50:55.720915</td>\n",
       "      <td>2024-01-23</td>\n",
       "      <td>domains</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://openalex.org/fields/27</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>field of study for diagnosing, treating and pr...</td>\n",
       "      <td>{'wikidata': 'https://www.wikidata.org/wiki/Q1...</td>\n",
       "      <td>[healthcare sciences]</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'id': 'https://openalex.org/fields/35', 'dis...</td>\n",
       "      <td>36272606</td>\n",
       "      <td>433646577</td>\n",
       "      <td>https://api.openalex.org/works?filter=topics.f...</td>\n",
       "      <td>2024-04-01T04:37:08.847205</td>\n",
       "      <td>2024-01-23</td>\n",
       "      <td>fields</td>\n",
       "      <td>{'id': 'https://openalex.org/domains/4', 'disp...</td>\n",
       "      <td>[{'id': 'https://openalex.org/subfields/2702',...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               id       display_name  \\\n",
       "0  https://openalex.org/domains/3  Physical Sciences   \n",
       "1  https://openalex.org/domains/2    Social Sciences   \n",
       "2  https://openalex.org/domains/4    Health Sciences   \n",
       "3  https://openalex.org/domains/1      Life Sciences   \n",
       "4  https://openalex.org/fields/27           Medicine   \n",
       "\n",
       "                                         description  \\\n",
       "0  branch of natural science that studies non-liv...   \n",
       "1  branch of science focused on societies and the...   \n",
       "2  branch of science focused on human health and ...   \n",
       "3  branch of science that involve the scientific ...   \n",
       "4  field of study for diagnosing, treating and pr...   \n",
       "\n",
       "                                                 ids  \\\n",
       "0  {'wikidata': 'https://www.wikidata.org/wiki/Q1...   \n",
       "1  {'wikidata': 'https://www.wikidata.org/wiki/Q3...   \n",
       "2  {'wikidata': 'https://www.wikidata.org/wiki/Q1...   \n",
       "3  {'wikidata': 'https://www.wikidata.org/wiki/Q8...   \n",
       "4  {'wikidata': 'https://www.wikidata.org/wiki/Q1...   \n",
       "\n",
       "                           display_name_alternatives  \\\n",
       "0                                                 []   \n",
       "1                               [political sciences]   \n",
       "2  [medical sciences, biomedical sciences, health...   \n",
       "3       [bioscience, biological science disciplines]   \n",
       "4                              [healthcare sciences]   \n",
       "\n",
       "                                              fields  \\\n",
       "0  [{'id': 'https://openalex.org/fields/15', 'dis...   \n",
       "1  [{'id': 'https://openalex.org/fields/12', 'dis...   \n",
       "2  [{'id': 'https://openalex.org/fields/35', 'dis...   \n",
       "3  [{'id': 'https://openalex.org/fields/11', 'dis...   \n",
       "4                                               None   \n",
       "\n",
       "                                            siblings  works_count  \\\n",
       "0  [{'id': 'https://openalex.org/domains/4', 'dis...     73237366   \n",
       "1  [{'id': 'https://openalex.org/domains/4', 'dis...     64995390   \n",
       "2  [{'id': 'https://openalex.org/domains/1', 'dis...     43957129   \n",
       "3  [{'id': 'https://openalex.org/domains/4', 'dis...     27536918   \n",
       "4  [{'id': 'https://openalex.org/fields/35', 'dis...     36272606   \n",
       "\n",
       "   cited_by_count                                      works_api_url  \\\n",
       "0       770120022  https://api.openalex.org/works?filter=topics.d...   \n",
       "1       273064988  https://api.openalex.org/works?filter=topics.d...   \n",
       "2       480875956  https://api.openalex.org/works?filter=topics.d...   \n",
       "3       445784583  https://api.openalex.org/works?filter=topics.d...   \n",
       "4       433646577  https://api.openalex.org/works?filter=topics.f...   \n",
       "\n",
       "                 updated_date created_date entity_type  \\\n",
       "0  2024-04-01T04:50:55.583031   2024-01-23     domains   \n",
       "1  2024-04-01T04:50:50.146788   2024-01-23     domains   \n",
       "2  2024-04-01T04:50:50.416213   2024-01-23     domains   \n",
       "3  2024-04-01T04:50:55.720915   2024-01-23     domains   \n",
       "4  2024-04-01T04:37:08.847205   2024-01-23      fields   \n",
       "\n",
       "                                              domain  \\\n",
       "0                                               None   \n",
       "1                                               None   \n",
       "2                                               None   \n",
       "3                                               None   \n",
       "4  {'id': 'https://openalex.org/domains/4', 'disp...   \n",
       "\n",
       "                                           subfields field topics keywords  \\\n",
       "0                                               None  None   None     None   \n",
       "1                                               None  None   None     None   \n",
       "2                                               None  None   None     None   \n",
       "3                                               None  None   None     None   \n",
       "4  [{'id': 'https://openalex.org/subfields/2702',...  None   None     None   \n",
       "\n",
       "  subfield  \n",
       "0     None  \n",
       "1     None  \n",
       "2     None  \n",
       "3     None  \n",
       "4     None  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load previously saved files\n",
    "df_merged = pd.read_json(\"data/openalex_merged_cats_2024_09.json\")\n",
    "df_merged.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating a valid TTL file with topics, fields etc.. from OpenAlex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32mReading: <data/openalex-topics-rdf.ttl>\u001b[0m\n",
      ".. trying rdf serialization: <turtle>\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Turtle generated...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m..... success!\u001b[0m\n",
      "\u001b[37m----------\n",
      "Loaded 52785 triples.\n",
      "----------\u001b[0m\n",
      "\u001b[32mRDF sources loaded successfully: 1 of 1.\u001b[0m\n",
      "\u001b[37m..... 'data/openalex-topics-rdf.ttl'\u001b[0m\n",
      "\u001b[37m----------\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Ontologies', 1)\n",
      "('Triples', 52785)\n",
      "('Classes', 0)\n",
      "('Properties', 0)\n",
      "('Annotation Properties', 0)\n",
      "('Object Properties', 0)\n",
      "('Datatype Properties', 0)\n",
      "('Skos Concepts', 4798)\n",
      "('Data Shapes', 0)\n",
      "('Data Sources', 1)\n"
     ]
    }
   ],
   "source": [
    "header = \"\"\"\n",
    "@prefix dc:          <http://purl.org/dc/elements/1.1/> .\n",
    "@prefix dcterms:     <http://purl.org/dc/terms/> .\n",
    "@prefix owl:         <http://www.w3.org/2002/07/owl#> .\n",
    "@prefix rdf:         <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .\n",
    "@prefix rdfs:        <http://www.w3.org/2000/01/rdf-schema#> .\n",
    "@prefix skos:        <http://www.w3.org/2004/02/skos/core#> .\n",
    "@prefix xsd:         <http://www.w3.org/2001/XMLSchema#> .\n",
    "@prefix openalex: <https://lambdamusic.github.io/openalex-hacks/ontology/> .\n",
    "@prefix oatopics: <https://openalex.org/topics/> .\n",
    "@prefix oadomains: <https://openalex.org/domains/> .\n",
    "@prefix oafields: <https://openalex.org/fields/> .\n",
    "@prefix oasubfields: <https://openalex.org/subfields/> .\n",
    "@prefix oakeywords: <https://openalex.org/keywords/> .\n",
    "\n",
    "openalex:\n",
    "    dc:date \"2024-09-24T06:12:02.390-04:00\" ;\n",
    "    dc:publisher \"http://www.michelepasin.org\" ;\n",
    "    dc:rights \"This work is distributed under a Creative Commons Zero 1.0 (CC0 1.0) Public Domain Dedication <http://creativecommons.org/publicdomain/zero/1.0/>.\"@en ;\n",
    "    dcterms:license <http://creativecommons.org/publicdomain/zero/1.0/> ;\n",
    "    a owl:Ontology, skos:ConceptScheme ;\n",
    "    owl:versionInfo \"0.1\" ;\n",
    "    skos:definition \"A non-official SKOS-ification of the OpenAlex scientific topics hierarchy (see https://help.openalex.org/hc/en-us/articles/24736129405719-Topics). Purpose of this exercise is to make it easier to experiment with the model and process it using standard SKOS tools. Export made on 2024-09-24 (for more information, see https://github.com/lambdamusic/openalex-hacks).\"@en ;\n",
    "    skos:prefLabel \"OpenAlex Topics Ontology\"@en ;\n",
    "    rdf:seeAlso <https://help.openalex.org/hc/en-us/articles/24736129405719-Topics> ;\n",
    "    rdf:seeAlso <https://github.com/lambdamusic/openalex-hacks> .\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "\n",
    "# DOMAINS\n",
    "#\n",
    "df = df_merged[df_merged.entity_type == 'domains']\n",
    "\n",
    "template = \"\"\"\n",
    "<{entity_id}>\n",
    "    a skos:Concept ;\n",
    "    rdfs:isDefinedBy openalex: ;\n",
    "    skos:prefLabel \"{label}\"@en ;\n",
    "    rdfs:label \"{label}\"@en ;\n",
    "    skos:definition \"{desc}\"@en ;\n",
    "    skos:inScheme openalex: ;\n",
    "    openalex:works_count {works_count} ;\n",
    "    openalex:cited_by_count {cited_by_count} ;\n",
    "    owl:sameAs <{wikidata_id}> ;\n",
    "    owl:sameAs <{wikipedia_id}> .\n",
    "\"\"\"\n",
    "# out = \"\"\n",
    "for index, row in df.head(5).iterrows():\n",
    "    x = template.format(\n",
    "        entity_id = row['id'],\n",
    "        desc = row['description'],\n",
    "        label = row['display_name'],\n",
    "        works_count = row['works_count'],\n",
    "        cited_by_count = row['cited_by_count'],\n",
    "        wikidata_id = row['ids']['wikidata'],\n",
    "        wikipedia_id = row['ids']['wikipedia'],\n",
    "    )\n",
    "    header += x\n",
    "\n",
    "# FIELDS\n",
    "#  \n",
    "df = df_merged[df_merged.entity_type == 'fields']\n",
    "\n",
    "template = \"\"\"\n",
    "<{topic_id}>\n",
    "    a skos:Concept ;\n",
    "    rdfs:isDefinedBy openalex: ;\n",
    "    skos:prefLabel \"{label}\"@en ;\n",
    "    rdfs:label \"{label}\"@en ;\n",
    "    skos:definition \"{desc}\"@en ;\n",
    "    skos:inScheme openalex: ;\n",
    "    skos:broader <{parent_id}> ;\n",
    "    openalex:works_count {works_count} ;\n",
    "    openalex:cited_by_count {cited_by_count} ;\n",
    "    owl:sameAs <{wikidata_id}> ;\n",
    "    owl:sameAs <{wikipedia_id}> .\n",
    "\"\"\"\n",
    "# out = \"\"\n",
    "for index, row in df.head(500).iterrows():\n",
    "    x = template.format(\n",
    "        topic_id = row['id'],\n",
    "        parent_id = row['domain']['id'],\n",
    "        desc = row['description'],\n",
    "        label = row['display_name'],\n",
    "        works_count = row['works_count'],\n",
    "        cited_by_count = row['cited_by_count'],\n",
    "        wikidata_id = row['ids']['wikidata'],\n",
    "        wikipedia_id = row['ids']['wikipedia'],\n",
    "    )\n",
    "    header += x\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# SUBFIELDS\n",
    "#  \n",
    "df = df_merged[df_merged.entity_type == 'subfields']\n",
    "\n",
    "template = \"\"\"\n",
    "<{topic_id}>\n",
    "    a skos:Concept ;\n",
    "    rdfs:isDefinedBy openalex: ;\n",
    "    skos:prefLabel \"{label}\"@en ;\n",
    "    rdfs:label \"{label}\"@en ;\n",
    "    skos:definition \"{desc}\"@en ;\n",
    "    skos:inScheme openalex: ;\n",
    "    skos:broader <{parent_id}> ;\n",
    "    openalex:works_count {works_count} ;\n",
    "    openalex:cited_by_count {cited_by_count} ;\n",
    "    owl:sameAs <{wikidata_id}> ;\n",
    "    owl:sameAs <{wikipedia_id}> .\n",
    "\"\"\"\n",
    "# out = \"\"\n",
    "for index, row in df.head(5000).iterrows():\n",
    "    x = template.format(\n",
    "        topic_id = row['id'],\n",
    "        parent_id = row['field']['id'],\n",
    "        desc = row['description'],\n",
    "        label = row['display_name'],\n",
    "        works_count = row['works_count'],\n",
    "        cited_by_count = row['cited_by_count'],\n",
    "        wikidata_id = row['ids']['wikidata'],\n",
    "        wikipedia_id = row['ids']['wikipedia'],\n",
    "    )\n",
    "    header += x\n",
    "\n",
    "\n",
    "\n",
    "# TOPICS\n",
    "#  \n",
    "df = df_merged[df_merged.entity_type == 'topics']\n",
    "\n",
    "template = \"\"\"\n",
    "<{topic_id}>\n",
    "    a skos:Concept ;\n",
    "    rdfs:isDefinedBy openalex: ;\n",
    "    skos:prefLabel \"{label}\"@en ;\n",
    "    rdfs:label \"{label}\"@en ;\n",
    "    skos:definition \"{desc}\"@en ;\n",
    "    skos:inScheme openalex: ;\n",
    "    skos:broader <{parent_id}> ;\n",
    "    openalex:works_count {works_count} ;\n",
    "    openalex:cited_by_count {cited_by_count} ;\n",
    "    owl:sameAs <{wikidata_id}> ;\n",
    "    owl:sameAs <{wikipedia_id}> .\n",
    "\"\"\"\n",
    "# out = \"\"\n",
    "for index, row in df.head(100000).iterrows():\n",
    "\n",
    "    # some topics miss the IDs column\n",
    "    wikidata_id = [] if \"wikidata\" not in row['ids'] else row['ids']['wikidata']\n",
    "    wikipedia_id = row['ids']['wikipedia'] if row['ids'] else []\n",
    "\n",
    "    x = template.format(\n",
    "        topic_id = row['id'],\n",
    "        parent_id = row['subfield']['id'],\n",
    "        desc = row['description'],\n",
    "        label = row['display_name'],\n",
    "        works_count = row['works_count'],\n",
    "        cited_by_count = row['cited_by_count'],\n",
    "        # cheating TODO fix me\n",
    "        wikidata_id = row['id'] if \"wikidata\" not in row['ids'] else row['ids']['wikidata'],\n",
    "        wikipedia_id = row['id'] if \"wikipedia\" not in row['ids'] else row['ids']['wikipedia'],\n",
    "    )\n",
    "    header += x\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "## finally\n",
    "\n",
    "with open('data/openalex-topics-rdf.ttl', 'w') as f:\n",
    "    f.write(header)\n",
    "print(\"Turtle generated...\")\n",
    "\n",
    "\n",
    "## test with ontospy\n",
    "if True:\n",
    "    o = ontospy.Ontospy()\n",
    "    o.load_rdf(\"data/openalex-topics-rdf.ttl\")\n",
    "    o.build_all()\n",
    "\n",
    "    for x in o.stats():\n",
    "        print(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extra\n",
    "\n",
    "The implementation below allows to render also 'keywords' as leaf nodes.\n",
    "\n",
    "This generates a much larger graph as each topic has 10 keywords (or so). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# TOPICS+KEYWORDS\n",
    "#  \n",
    "df = df_merged[df_merged.entity_type == 'topics']\n",
    "\n",
    "template = \"\"\"\n",
    "<{topic_id}>\n",
    "    a skos:Concept ;\n",
    "    rdfs:isDefinedBy openalex: ;\n",
    "    skos:prefLabel \"{label}\"@en ;\n",
    "    rdfs:label \"{label}\"@en ;\n",
    "    skos:definition \"{desc}\"@en ;\n",
    "    skos:inScheme openalex: ;\n",
    "    skos:broader <{parent_id}> ;\n",
    "    openalex:works_count {works_count} ;\n",
    "    openalex:cited_by_count {cited_by_count} ;\n",
    "    owl:sameAs <{wikidata_id}> ;\n",
    "    owl:sameAs <{wikipedia_id}> .\n",
    "\"\"\"\n",
    "\n",
    "template_keyword = \"\"\"\n",
    "<{keyword_id}>\n",
    "    a skos:Concept ;\n",
    "    rdfs:isDefinedBy openalex: ;\n",
    "    skos:prefLabel \"{label}\"@en ;\n",
    "    rdfs:label \"{label}\"@en ;\n",
    "    skos:inScheme openalex: ;\n",
    "    skos:broader <{parent_id}> .\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "keyword_index = 0\n",
    "for index, row in df.head(100000).iterrows():\n",
    "\n",
    "    # some topics miss the IDs column\n",
    "    wikidata_id = [] if \"wikidata\" not in row['ids'] else row['ids']['wikidata']\n",
    "    wikipedia_id = row['ids']['wikipedia'] if row['ids'] else []\n",
    "\n",
    "    for k in row['keywords']:\n",
    "        keyword_index += 1\n",
    "        k_id = f\"openalex:keyword-{keyword_index}\"\n",
    "    \n",
    "        x = template_keyword.format(\n",
    "            keyword_id = k_id,\n",
    "            parent_id = row['id'],\n",
    "            label = k,\n",
    "        )\n",
    "        header += x\n",
    "        \n",
    "    x = template.format(\n",
    "        topic_id = row['id'],\n",
    "        parent_id = row['subfield']['id'],\n",
    "        desc = row['description'],\n",
    "        label = row['display_name'],\n",
    "        works_count = row['works_count'],\n",
    "        cited_by_count = row['cited_by_count'],\n",
    "        # cheating TODO fix me\n",
    "        wikidata_id = row['id'] if \"wikidata\" not in row['ids'] else row['ids']['wikidata'],\n",
    "        wikipedia_id = row['id'] if \"wikipedia\" not in row['ids'] else row['ids']['wikipedia'],\n",
    "    )\n",
    "    header += x\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "jupyterlab",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}