Switch to osm-reader to handle XML and PBF uniformly

experimental/src/io.rs

@@ -28,7 +28,7 @@ pub fn load_road_network(osm_path: String, timer: &mut Timer) -> Result<RoadNetw
     // TODO Use the same clip
     let clip_pts = None;
     let (mut street_network, _) = streets_reader::osm_to_street_network(
-        &std::fs::read_to_string(osm_path).unwrap(),
+        &std::fs::read(osm_path).unwrap(),
         clip_pts,
         MapConfig::default(),
         timer,

osm2lanes/Cargo.toml

@@ -8,6 +8,7 @@ abstutil = { git = "https://github.com/a-b-street/abstreet" }
 anyhow = { workspace = true }
 enumset = { version = "1.0.12", features=["serde"] }
 geom = { workspace = true }
+osm-reader = { git = "https://github.com/a-b-street/osm-reader", features = ["serde"] }
 serde = { workspace = true }
 
 [dev-dependencies]

osm2lanes/src/osm.rs

@@ -1,8 +1,6 @@
 //! Useful utilities for working with OpenStreetMap.
 
-use std::fmt;
-
-use serde::{Deserialize, Serialize};
+pub use osm_reader::{NodeID, OsmID, RelationID, WayID};
 
 // This is a commonly used key in the codebase, so worthy of a bit of typo-prevention
 pub const HIGHWAY: &str = "highway";
@@ -55,51 +53,3 @@ impl RoadRank {
         0
     }
 }
-
-#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord, Serialize, Deserialize)]
-pub struct NodeID(pub i64);
-#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord, Serialize, Deserialize)]
-pub struct WayID(pub i64);
-#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord, Serialize, Deserialize)]
-pub struct RelationID(pub i64);
-
-impl fmt::Display for NodeID {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "https://www.openstreetmap.org/node/{}", self.0)
-    }
-}
-impl fmt::Display for WayID {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "https://www.openstreetmap.org/way/{}", self.0)
-    }
-}
-impl fmt::Display for RelationID {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "https://www.openstreetmap.org/relation/{}", self.0)
-    }
-}
-
-#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord, Serialize, Deserialize)]
-pub enum OsmID {
-    Node(NodeID),
-    Way(WayID),
-    Relation(RelationID),
-}
-impl fmt::Display for OsmID {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            OsmID::Node(n) => write!(f, "{}", n),
-            OsmID::Way(w) => write!(f, "{}", w),
-            OsmID::Relation(r) => write!(f, "{}", r),
-        }
-    }
-}
-impl OsmID {
-    pub fn inner(self) -> i64 {
-        match self {
-            OsmID::Node(n) => n.0,
-            OsmID::Way(w) => w.0,
-            OsmID::Relation(r) => r.0,
-        }
-    }
-}

osm2streets-java/src/StreetNetwork.java

@@ -36,15 +36,15 @@ public class StreetNetwork {
 		}
 	}
 
-	public static native StreetNetwork create(String osmXmlInput);
+	public static native StreetNetwork create(byte[] osmInput);
 
 	public native List<Surface> getSurfaces();
 
 	public native List<PaintArea> getPaintAreas();
 
 	public static void main(String[] args) throws Exception {
-		String osmXmlInput = new String(Files.readAllBytes(Paths.get("../tests/src/aurora_sausage_link/input.osm")));
-		StreetNetwork network = create(osmXmlInput);
+		byte[] osmInput = Files.readAllBytes(Paths.get("../tests/src/aurora_sausage_link/input.osm"));
+		StreetNetwork network = create(osmInput);
 		System.out.println(network.getSurfaces());
 		System.out.println(network.getPaintAreas());
 	}

osm2streets-java/src/lib.rs

@@ -1,5 +1,5 @@
 use abstutil::Timer;
-use jni::objects::{JClass, JObject, JString, JValue};
+use jni::objects::{JClass, JObject, JValue};
 use jni::sys::{jlong, jobject};
 use jni::JNIEnv;
 
@@ -10,14 +10,13 @@ struct StreetNetwork {
 }
 
 impl StreetNetwork {
-    fn new(osm_xml_input: String) -> Self {
+    fn new(input_bytes: &[u8]) -> Self {
         let cfg = MapConfig::default();
 
         let clip_pts = None;
         let mut timer = Timer::throwaway();
         let (mut network, _) =
-            streets_reader::osm_to_street_network(&osm_xml_input, clip_pts, cfg, &mut timer)
-                .unwrap();
+            streets_reader::osm_to_street_network(input_bytes, clip_pts, cfg, &mut timer).unwrap();
         let transformations = Transformation::standard_for_clipped_areas();
         network.apply_transformations(transformations, &mut timer);
 
@@ -29,10 +28,10 @@ impl StreetNetwork {
 pub extern "system" fn Java_org_osm2streets_StreetNetwork_create(
     env: JNIEnv,
     _: JClass,
-    osm_xml_input: JString,
+    input_bytes: jni::sys::jbyteArray,
 ) -> jobject {
-    let osm_xml_input: String = env.get_string(osm_xml_input).unwrap().into();
-    let network = StreetNetwork::new(osm_xml_input);
+    let input_bytes: Vec<u8> = env.convert_byte_array(input_bytes).unwrap();
+    let network = StreetNetwork::new(&input_bytes);
 
     let pointer = Box::into_raw(Box::new(network)) as jlong;
     let obj_class = env.find_class("org/osm2streets/StreetNetwork").unwrap();

osm2streets-js/src/lib.rs

@@ -32,7 +32,7 @@ impl JsStreetNetwork {
     // TODO clip_pts_geojson should be Option. Empty means None.
     #[wasm_bindgen(constructor)]
     pub fn new(
-        osm_xml_input: &str,
+        osm_input: &[u8],
         clip_pts_geojson: &str,
         input: JsValue,
     ) -> Result<JsStreetNetwork, JsValue> {
@@ -60,7 +60,7 @@ impl JsStreetNetwork {
 
         let mut timer = Timer::throwaway();
         let (mut street_network, doc) =
-            streets_reader::osm_to_street_network(osm_xml_input, clip_pts, cfg, &mut timer)
+            streets_reader::osm_to_street_network(osm_input, clip_pts, cfg, &mut timer)
                 .map_err(|err| JsValue::from_str(&err.to_string()))?;
         let mut transformations = Transformation::standard_for_clipped_areas();
         if input.dual_carriageway_experiment {
@@ -229,9 +229,10 @@ impl JsStreetNetwork {
     pub fn way_to_xml(&self, id: i64) -> String {
         let way = &self.ways[&osm::WayID(id)];
         let mut out = format!(r#"<way id="{id}""#);
-        if let Some(version) = way.version {
+        // TODO Add this to osm-reader
+        /*if let Some(version) = way.version {
             out.push_str(&format!(r#" version="{version}""#));
-        }
+        }*/
         out.push_str(">\n");
         for node in &way.nodes {
             out.push_str(&format!(r#"  <nd ref="{}"/>"#, node.0));

streets_reader/Cargo.toml

@@ -10,6 +10,5 @@ anyhow = { workspace = true }
 country-geocoder = { git = "https://github.com/a-b-street/country-geocoder" }
 geom = { workspace = true }
 log = "0.4.14"
+osm-reader = { git = "https://github.com/a-b-street/osm-reader" }
 osm2streets = { path = "../osm2streets" }
-osmpbf = "0.3.2"
-xmlparser = "0.13.5"

streets_reader/src/lib.rs

@@ -17,14 +17,14 @@ pub mod extract;
 pub mod osm_reader;
 pub mod split_ways;
 
-/// Create a `StreetNetwork` from the contents of an `.osm.xml` file. If `clip_pts` is specified,
-/// use these as a boundary polygon. (Use `LonLat::read_geojson_polygon` or similar to produce
-/// these.)
+/// Create a `StreetNetwork` from the contents of an `.osm.xml` or `.pbf` file. If `clip_pts` is
+/// specified, use these as a boundary polygon. (Use `LonLat::read_geojson_polygon` or similar to
+/// produce these.)
 ///
 /// You probably want to do `StreetNetwork::apply_transformations` on the result to get a useful
 /// result.
 pub fn osm_to_street_network(
-    osm_xml_input: &str,
+    input_bytes: &[u8],
     clip_pts: Option<Vec<LonLat>>,
     cfg: MapConfig,
     timer: &mut Timer,
@@ -34,7 +34,7 @@ pub fn osm_to_street_network(
     // happens in split_ways.
     streets.config = cfg;
 
-    let (extract, doc) = extract_osm(&mut streets, osm_xml_input, clip_pts, timer)?;
+    let (extract, doc) = extract_osm(&mut streets, input_bytes, clip_pts, timer)?;
     split_ways::split_up_roads(&mut streets, extract, timer);
 
     // Cul-de-sacs aren't supported yet.
@@ -43,31 +43,6 @@ pub fn osm_to_street_network(
     Ok((streets, doc))
 }
 
-pub fn pbf_to_street_network(
-    input: &[u8],
-    clip_pts: Option<Vec<LonLat>>,
-    cfg: MapConfig,
-    timer: &mut Timer,
-) -> Result<(StreetNetwork, Document)> {
-    let mut streets = StreetNetwork::blank();
-    // Note that DrivingSide is still incorrect. It'll be set in extract_osm, before Road::new
-    // happens in split_ways.
-    streets.config = cfg;
-
-    let mut doc = Document::read_pbf(
-        input,
-        clip_pts.as_ref().map(|pts| GPSBounds::from(pts.clone())),
-        timer,
-    )?;
-
-    let out = process_data(&mut streets, clip_pts, timer, &mut doc)?;
-    split_ways::split_up_roads(&mut streets, out, timer);
-
-    // Cul-de-sacs aren't supported yet.
-    streets.retain_roads(|r| r.src_i != r.dst_i);
-    Ok((streets, doc))
-}
-
 /// Set up country code and driving side, using an arbitrary point. This must be called after
 /// `gps_bounds` is set.
 pub fn detect_country_code(streets: &mut StreetNetwork) {
@@ -88,27 +63,15 @@ pub fn detect_country_code(streets: &mut StreetNetwork) {
 
 fn extract_osm(
     streets: &mut StreetNetwork,
-    osm_xml_input: &str,
+    input_bytes: &[u8],
     clip_pts: Option<Vec<LonLat>>,
     timer: &mut Timer,
 ) -> Result<(OsmExtract, Document)> {
     let mut doc = Document::read(
-        osm_xml_input,
+        input_bytes,
         clip_pts.as_ref().map(|pts| GPSBounds::from(pts.clone())),
         timer,
     )?;
-    // If GPSBounds aren't provided above, they'll be computed in the Document
-    let out = process_data(streets, clip_pts, timer, &mut doc)?;
-
-    Ok((out, doc))
-}
-
-fn process_data(
-    streets: &mut StreetNetwork,
-    clip_pts: Option<Vec<LonLat>>,
-    timer: &mut Timer,
-    doc: &mut Document,
-) -> Result<OsmExtract> {
     // If GPSBounds aren't provided, they'll be computed in the Document
     streets.gps_bounds = doc.gps_bounds.clone().unwrap();
 
@@ -150,5 +113,6 @@ fn process_data(
         timer.next();
         out.handle_relation(*id, rel);
     }
-    Ok(out)
+
+    Ok((out, doc))
 }

streets_reader/src/osm_reader/mod.rs

@@ -32,7 +32,6 @@ pub struct Way {
     pub nodes: Vec<NodeID>,
     pub pts: Vec<Pt2D>,
     pub tags: Tags,
-    pub version: Option<usize>,
 }
 
 pub struct Relation {