01-download-osm-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "aaa1030b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import osmium\n",
    "import subprocess\n",
    "import os\n",
    "os.environ['USE_PYGEOS'] = '0'\n",
    "import geopandas as gpd\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# from geopy.geocoders import Nominatim\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3736af37",
   "metadata": {},
   "outputs": [],
   "source": [
    "LEVEL_HEIGHT = 3.4\n",
    "\n",
    "# https://wiki.openstreetmap.org/wiki/Simple_3D_buildings#Other_roof_tags\n",
    "\n",
    "\n",
    "def _feet_to_meters(s):\n",
    "    r = re.compile(\"([0-9]*\\.?[0-9]+)'([0-9]*\\.?[0-9]+)?\\\"?\")\n",
    "    m = r.findall(s)[0]\n",
    "    if len(m[0]) > 0 and len(m[1]) > 0:\n",
    "        m = float(m[0]) + float(m[1]) / 12.0\n",
    "    elif len(m[0]) > 0:\n",
    "        m = float(m[0])\n",
    "    return m * 0.3048\n",
    "\n",
    "\n",
    "def _get_height(tags):\n",
    "    if 'height' in tags:\n",
    "        # already accounts for roof\n",
    "        if '\\'' in tags['height'] or '\\\"' in tags['height']:\n",
    "            return _feet_to_meters(tags['height'])\n",
    "        r = re.compile(r\"[-+]?\\d*\\.\\d+|\\d+\")\n",
    "        return float(r.findall(tags['height'])[0])\n",
    "    if 'levels' in tags:\n",
    "        roof_height = 0\n",
    "        if 'roof_height' in tags:\n",
    "            if '\\'' in tags['roof_height'] or '\\\"' in tags['roof_height']:\n",
    "                roof_height = _feet_to_meters(tags['roof_height'])\n",
    "            else:\n",
    "                r = re.compile(r\"[-+]?\\d*\\.\\d+|\\d+\")\n",
    "                roof_height = float(r.findall(tags['roof_height'])[0])\n",
    "\n",
    "        # does not account for roof height\n",
    "        height = float(tags['levels']) * LEVEL_HEIGHT\n",
    "        if 'roof_levels' in tags and roof_height == 0:\n",
    "            height += float(tags['roof_levels']) * LEVEL_HEIGHT\n",
    "        return height\n",
    "    return 7.0\n",
    "\n",
    "\n",
    "def _get_min_height(tags):\n",
    "    if 'min_height' in tags:\n",
    "        # already accounts for roof\n",
    "        if '\\'' in tags['min_height'] or '\\\"' in tags['min_height']:\n",
    "            return _feet_to_meters(tags['min_height'])\n",
    "        r = re.compile(r\"[-+]?\\d*\\.\\d+|\\d+\")\n",
    "        return float(r.findall(tags['min_height'])[0])\n",
    "    if 'min_level' in tags:\n",
    "        height = float(tags['min_level']) * LEVEL_HEIGHT\n",
    "        return height\n",
    "    return 0.0\n",
    "\n",
    "\n",
    "class BuildingHandler(osmium.SimpleHandler):\n",
    "\n",
    "    def __init__(self):\n",
    "        osmium.SimpleHandler.__init__(self)\n",
    "        self.geometry = []\n",
    "        self.height = []\n",
    "        self.min_height = []\n",
    "        self.wkbfab = osmium.geom.WKBFactory()\n",
    "\n",
    "    def get_gdf(self):\n",
    "        geometry = gpd.GeoSeries.from_wkb(self.geometry, crs='epsg:4326')\n",
    "        height = pd.Series(self.height, dtype='float')\n",
    "        min_height = pd.Series(self.min_height, dtype='float')\n",
    "\n",
    "        return gpd.GeoDataFrame({\n",
    "            'geometry': geometry,\n",
    "            'min_height': self.min_height,\n",
    "            'height': self.height\n",
    "        }, index=geometry.index)\n",
    "\n",
    "    def area(self, a):\n",
    "        tags = a.tags\n",
    "        # Qualifiers\n",
    "        if not ('building' in tags or 'building:part' in tags or tags.get('type', None) == 'building'):\n",
    "            return\n",
    "        # Disqualifiers\n",
    "        if (tags.get('location', None) == 'underground' or 'bridge' in tags):\n",
    "            return\n",
    "        try:\n",
    "            poly = self.wkbfab.create_multipolygon(a)\n",
    "            height = _get_height(tags)\n",
    "            min_height = _get_min_height(tags)\n",
    "\n",
    "            self.geometry.append(poly)\n",
    "            self.height.append(height)\n",
    "            self.min_height.append(min_height)\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            print(a)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "a6d2fce7",
   "metadata": {},
   "source": [
    "## Option 1: Reading from cities.csv\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8c7c5b47",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('cities.csv')\n",
    "\n",
    "for index, row in df.iterrows():\n",
    "    city = row['abbr']\n",
    "\n",
    "    if city not in ['bei']:\n",
    "        continue\n",
    "\n",
    "    print(city)\n",
    "    bottom_left = eval(row['bottom_left'])\n",
    "    top_right = eval(row['top_right'])\n",
    "    osm_file = 'data/osm/'+row['osm_file']\n",
    "    bbox = [{'lat': float(bottom_left[0]), 'lon': float(bottom_left[1])}, {\n",
    "        'lat': float(top_right[0]), 'lon': float(top_right[1])}]  # bottom left, top right\n",
    "\n",
    "    aux = '%f,%f,%f,%f' % (bbox[0]['lon'], bbox[0]\n",
    "                           ['lat'], bbox[1]['lon'], bbox[1]['lat'])\n",
    "    filename = 'data/osm_new/%s.osm.pbf' % city\n",
    "\n",
    "    if not os.path.exists(os.path.dirname(filename)):\n",
    "        os.makedirs(os.path.dirname(filename))\n",
    "\n",
    "    print(osm_file)\n",
    "    proc = subprocess.call(['wsl', 'osmium', 'extract', '-b',\n",
    "                           aux, '-o', filename, '--overwrite', osm_file], shell=True)\n",
    "    print(city)\n",
    "\n",
    "    h = BuildingHandler()\n",
    "    h.apply_file(filename, locations=True)\n",
    "\n",
    "    gdf = h.get_gdf()\n",
    "    gdf = gdf.to_crs('epsg:3395')\n",
    "    gdf.to_feather('data/osm_new/%s.feather' % city, compression='lz4')\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "e2c18a3f",
   "metadata": {},
   "source": [
    "## Option 2: Using nominatim\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efded0aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "city_full = 'Sao Paulo, Brazil'\n",
    "city = 'sp'\n",
    "filename = 'data/osm/%s.osm.pbf' % (city)\n",
    "input_filename = 'data/osm/south-america-latest.osm.pbf'\n",
    "\n",
    "geolocator = Nominatim(user_agent='uic')\n",
    "location = geolocator.geocode(city_full).raw\n",
    "# print(location)\n",
    "bbox = [float(x) for x in location['boundingbox']]\n",
    "bbox = [{'lat': bbox[0], 'lon': bbox[2]}, {\n",
    "    'lat': bbox[1], 'lon': bbox[3]}]  # bottom left, top right\n",
    "\n",
    "# bbox = [-46.726727,-23.620136,-46.52554,-23.449534]\n",
    "bbox = [{'lat': bbox[0], 'lon': bbox[1]}, {\n",
    "    'lat': bbox[2], 'lon': bbox[3]}]  # bottom left, top right\n",
    "input_filename = 'data/osm/south-america-latest.osm.pbf'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4d4abc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# apt get install osmium-tool\n",
    "aux = '%f,%f,%f,%f' % (bbox[0]['lon'], bbox[0]['lat'],\n",
    "                       bbox[1]['lon'], bbox[1]['lat'])\n",
    "proc = subprocess.call(['wsl', 'osmium', 'extract', '-b', aux,\n",
    "                       '-o', filename, '--overwrite', input_filename], shell=True)\n",
    "\n",
    "h = BuildingHandler()\n",
    "h.apply_file(filename, locations=True)\n",
    "\n",
    "gdf = h.get_gdf()\n",
    "gdf = gdf.to_crs('epsg:3395')\n",
    "gdf.to_feather('data/osm/%s.feather' % city, compression='lz4')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eaba27f9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}