notebooks/01_download_in_rolls_age_gap.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataverse API Token\n",
    "# https://dataverse.harvard.edu/dataverseuser.xhtml?selectTab=apiTokenTab\n",
    "#token = 'API TOKEN'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from pyDataverse.api import Api\n",
    "from pyDataverse.models import Dataverse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "api = Api('https://dataverse.harvard.edu/', token)\n",
    "api.status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resp = api.get_dataset('doi:10.7910/DVN/MUEGDT')\n",
    "j = resp.json()\n",
    "j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = []\n",
    "for file in j['data']['latestVersion']['files']:\n",
    "    fid = file['dataFile']['id']\n",
    "    fn = file['dataFile']['filename']\n",
    "    print(fid, fn)\n",
    "    files.append((fid, fn))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "def download_file(url, fn):\n",
    "    local_filename = fn # url.split('/')[-1]\n",
    "    # NOTE the stream=True parameter below\n",
    "    with requests.get(url, stream=True) as r:\n",
    "        r.raise_for_status()\n",
    "        with open(local_filename, 'wb') as f:\n",
    "            for chunk in r.iter_content(chunk_size=8192): \n",
    "                if chunk: # filter out keep-alive new chunks\n",
    "                    f.write(chunk)\n",
    "                    # f.flush()\n",
    "    return local_filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for fid, fn in files:\n",
    "    print(fn)\n",
    "    url = 'https://dataverse.harvard.edu//api/v1/access/datafile/%s?key=%s' % (fid, token)\n",
    "    if not os.path.exists(fn):\n",
    "        download_file(url, fn)\n",
    "    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}