workflow/preprocessing/02_l0_download_from_glims.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GLIMS region files download: Level 0 files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This script creates \"Level 0\" RGI files. These files are fetched directly from the GLIMS database, subsetted with a spatial bounding box around each region.\n",
    "\n",
    "Level 0 files need to be updated to reflect new entries into the GLIMS database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import geopandas as gpd\n",
    "import shutil\n",
    "from utils import mkdir\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RGI Region files "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# go down from rgi7_scripts/workflow/preprocessing\n",
    "data_dir = '../../../rgi7_data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_file = os.path.join(data_dir, '00_rgi70_regions', '00_rgi70_O1Regions')\n",
    "reg_f = gpd.read_file(reg_file)\n",
    "reg_f.plot(alpha=0.5, edgecolor='k');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Region 10 has two boxes with glaciers in it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_f.loc[reg_f.o1region == '10'].plot(alpha=0.5, edgecolor='k');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download loop "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If you want to (re-)download only selected regions\n",
    "# reg_f_sel = reg_f.loc[reg_f.o1region.isin(['04', '19'])] \n",
    "reg_f_sel = reg_f\n",
    "reg_f_sel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With login "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download date: May 23, 2023, 23H CEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glims_database_dump_wlogin import *\n",
    "\n",
    "buffer = 0.5  # in degrees, buffer around the box\n",
    "from_glims = mkdir(os.path.join(data_dir, 'l0_from_glims'))\n",
    "\n",
    "for i, reg in reg_f_sel.iterrows():\n",
    "    \n",
    "    # Prepare bounds \n",
    "    x0, y0, x1, y1 = reg.geometry.bounds\n",
    "    \n",
    "    x0 = np.clip(x0 - buffer, -180., None)\n",
    "    y0 = np.clip(y0 - buffer, -90., None)\n",
    "    x1 = np.clip(x1 + buffer, None, 180.)\n",
    "    y1 = np.clip(y1 + buffer, None, 90.)\n",
    "    \n",
    "    bounds = f' {x0:.2f},{y0:.2f},{x1:.2f},{y1:.2f}'\n",
    "    print('')\n",
    "    print('{}, {}. Bounds: {}'.format(reg.o1region, reg.full_name, bounds))\n",
    "        \n",
    "    # GLIMS request arguments\n",
    "    p = setup_argument_parser()\n",
    "    args = p.parse_args(['--mode', 'glims',\n",
    "                         '--nunataks', 'GLIMS',  # We can't use RGI because it is buggy \n",
    "                         '--archive_type', 'tar',\n",
    "                         '--download_type', 'extent',\n",
    "                         '--clause', bounds,\n",
    "                         '--netrc',\n",
    "                         '--env', 'production'])\n",
    "    \n",
    "    urs_server = \"urs.earthdata.nasa.gov\"\n",
    "    \n",
    "    glims_servers = {\n",
    "        \"production\": \"www.glims.org/services\",\n",
    "        \"blue\": \"blue.glims-services.apps.int.nsidc.org\",\n",
    "        \"integration\": \"integration.glims.org/services\",\n",
    "        \"qa\": \"qa.glims-services.apps.int.nsidc.org\",\n",
    "        \"staging\": \"staging.glims-services.apps.int.nsidc.org\",\n",
    "    }\n",
    "    server = glims_servers[args.env]\n",
    "\n",
    "    if not args.noauth:\n",
    "        # Get credentials from command-line or .netrc file\n",
    "        credentials = get_credentials(args, urs_server)\n",
    "\n",
    "        auth_session = get_authenticated_session(server, credentials[0], credentials[1])\n",
    "        print(\"auth_session: \", auth_session)  # DEBUG\n",
    "\n",
    "        if auth_session is None:\n",
    "            print(\"Authentication failed.\", file=sys.stderr)\n",
    "            sys.exit(1)\n",
    "    else:\n",
    "        auth_session = get_noauth_session(server)\n",
    "\n",
    "    filebasename = issue_order(server, auth_session, args)\n",
    "\n",
    "    poll_readiness(\n",
    "        server, auth_session, filebasename, args.protocol, period=2, tries=args.tries\n",
    "    )  # returns when file is ready\n",
    "    do_download(args, server, auth_session, filebasename)\n",
    "    \n",
    "    shutil.move(filebasename, os.path.join(from_glims, '{:02d}_RGI{:02d}.tgz'.format(i, int(reg['o1region']))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Without login "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glims_database_dump import *\n",
    "\n",
    "servers = {\n",
    "        'production': 'www.glims.org/services',\n",
    "        'blue': 'blue.glims-services.apps.int.nsidc.org',\n",
    "        'integration': 'integration.glims-services.apps.int.nsidc.org',\n",
    "        'qa': 'qa.glims-services.apps.int.nsidc.org',\n",
    "        'staging': 'staging.glims-services.apps.int.nsidc.org',\n",
    "        }\n",
    "\n",
    "buffer = 0.5  # in degrees, buffer around the box\n",
    "from_glims = mkdir(os.path.join(data_dir, 'l0_from_glims'))\n",
    "\n",
    "for i, reg in reg_f_sel.iterrows():\n",
    "    \n",
    "    # Prepare bounds \n",
    "    x0, y0, x1, y1 = reg.geometry.bounds\n",
    "    \n",
    "    x0 = np.clip(x0 - buffer, -180., None)\n",
    "    y0 = np.clip(y0 - buffer, -90., None)\n",
    "    x1 = np.clip(x1 + buffer, None, 180.)\n",
    "    y1 = np.clip(y1 + buffer, None, 90.)\n",
    "    \n",
    "    bounds = f' {x0:.2f},{y0:.2f},{x1:.2f},{y1:.2f}'\n",
    "    print('')\n",
    "    print('{}, {}. Bounds: {}'.format(reg.RGI_CODE, reg.FULL_NAME, bounds))\n",
    "        \n",
    "    # GLIMS request arguments\n",
    "    p = setup_argument_parser()\n",
    "    args = p.parse_args(['--mode', 'glims',\n",
    "                         '--nunataks', 'GLIMS',  # We can't use RGI because it is buggy \n",
    "                         '--archive_type', 'tar',\n",
    "                         '--download_type', 'extent',\n",
    "                         '--clause', bounds])\n",
    "\n",
    "    server = servers[args.env]\n",
    "\n",
    "    filebasename = issue_order(server, args)\n",
    "    poll_readiness(server, filebasename, period=2, tries=args.tries, protocol=args.protocol)    # returns when file is ready\n",
    "    do_download(server, filebasename, args)\n",
    "    \n",
    "    shutil.move(filebasename, os.path.join(from_glims, '{:02d}_RGI{:02d}.tgz'.format(i, int(reg['RGI_CODE']))))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}