bz2 file support (#72)

support for bz2 input file (compressed OSM files)

README.md

@@ -1,11 +1,11 @@
-# OsmToRoadGraph v.0.5.0
+# OsmToRoadGraph v.0.6.0
 
 [![Build Status](https://travis-ci.org/AndGem/OsmToRoadGraph.svg?branch=master)](https://travis-ci.org/AndGem/OsmToRoadGraph)
 ![Python application](https://github.com/AndGem/OsmToRoadGraph/workflows/Python%20application/badge.svg?branch=master)
 [![codecov](https://codecov.io/gh/AndGem/OsmToRoadGraph/branch/master/graph/badge.svg)](https://codecov.io/gh/AndGem/OsmToRoadGraph)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
-- [OsmToRoadGraph v.0.5.0](#osmtoroadgraph-v050)
+- [OsmToRoadGraph v.0.6.0](#osmtoroadgraph-v060)
   - [Updates](#updates)
   - [Introduction](#introduction)
     - [Motivation](#motivation)
@@ -18,17 +18,16 @@
       - [Output](#output)
         - [Output Format](#output-format)
           - [Example Road Network (*.pycgr)](#example-road-network-pycgr)
-          - [Example Street Names (*.pycgr_names)](#example-street-names-pycgrnames)
+          - [Example Street Names (*.pycgr_names)](#example-street-names-pycgr_names)
       - [Configuring the Accepted OSM Highway Types](#configuring-the-accepted-osm-highway-types)
       - [Indoor Paths](#indoor-paths)
     - [Research](#research)
 
 ## Updates
 
-**Changelog v.0.4.3 -> v.0.5.0:**
+**Changelog v.0.5.0 -> v.0.6.0:**
 
-- [x] Added new output option to write NetworkX compatible files
-- [x] Fixed bugs in the contraction logic
+- [x] Added bz2 support for input files
 
 ## Introduction
 
@@ -80,6 +79,8 @@ optional arguments:
 #### Usage - Explanation
 
 `-f` points to the input filename; the output files will be created in the same folder and using the name of the input file as prefix and adding information as suffix.
+This filename must be either an OSM XML file (usually has the file extension `.osm`), or such a file compressed by bz2.
+If it is a bz2 file, the content will be decompressed in memory.
 
 `-n` sets the network type. This influences the maximum speed saved for the edges. If you care only about connectivity set it to pedestrian.
 

graph/algorithms.py

@@ -31,6 +31,9 @@ def computeLCC(graph):
         found_nodes.append(f_nodes)
         total_nodes = total_nodes - f_nodes
 
+    if len(found_nodes) == 0:
+        return []
+
     # determine largest connected components
     lcc = max(found_nodes, key=lambda component: len(component))
 

osm/read_osm.py

@@ -1,3 +1,4 @@
+import bz2
 import xml.sax
 
 from osm.osm_types import OSMNode, OSMWay
@@ -10,20 +11,46 @@
 
 @timer.timer
 def read_file(osm_filename, configuration, verbose=True) -> Tuple[Dict[int, OSMNode], List[OSMWay]]:
+
     parserHelper = WayParserHelper(configuration)
-    ways, found_node_ids = _read_ways(PercentageFile(osm_filename), parserHelper)
-    nodes = _read_nodes(PercentageFile(osm_filename), found_node_ids)
+    decompressed_content = decompress_content(osm_filename)
+    if decompressed_content:
+        ways, found_node_ids = _read_ways(decompressed_content, parserHelper)
+        nodes = _read_nodes(decompressed_content, found_node_ids)
+    else:
+        ways, found_node_ids = _read_ways(PercentageFile(osm_filename), parserHelper)
+        nodes = _read_nodes(PercentageFile(osm_filename), found_node_ids)
 
     return nodes, ways
 
 
+def decompress_content(osm_filename):
+    magic_bz2 = "\x42\x5a\x68"
+
+    with open(osm_filename, 'r', encoding='utf-8', errors='replace') as f:
+        content_begin = f.read(10)
+
+    if content_begin.startswith(magic_bz2):
+        print("identified bz2 compressed file.. decompressing")
+        f = bz2.open(osm_filename, "rb")
+        content = f.read()
+        print("done!")
+        return content
+
+        print("no compression recognized!")
+        return None
+
+
 @timer.timer
 def _read_ways(osm_file, configuration) -> Tuple[List[OSMWay], Set[int]]:
     parser = xml.sax.make_parser()
     w_handler = WayHandler(configuration)
 
     parser.setContentHandler(w_handler)
-    parser.parse(osm_file)
+    if type(osm_file) == PercentageFile:
+        parser.parse(osm_file)
+    else:
+        xml.sax.parseString(osm_file, w_handler)
 
     return w_handler.found_ways, w_handler.found_nodes
 
@@ -34,6 +61,9 @@ def _read_nodes(osm_file, found_nodes) -> Dict[int, OSMNode]:
     n_handler = NodeHandler(found_nodes)
 
     parser.setContentHandler(n_handler)
-    parser.parse(osm_file)
+    if type(osm_file) == PercentageFile:
+        parser.parse(osm_file)
+    else:
+        xml.sax.parseString(osm_file, n_handler)
 
     return n_handler.nodes

run.py

@@ -1,4 +1,5 @@
 import argparse
+import os
 
 import configuration as config
 import graph.contract_graph as contract_graph
@@ -61,6 +62,10 @@ def convert_osm_to_roadgraph(filename, network_type, options):
         parser.print_help()
         exit()
 
+    if not os.path.isfile(filename):
+        print("ERROR: provided filename {} does not point to a file!".format(filename))
+        exit()
+
     long_network_type = {"p": "pedestrian", "c": "car", "b": "bicycle"}
     if options.network_type in long_network_type.keys():
         network_type = long_network_type[options.network_type]