From 4d338bedcc0fe1e8a0111ff3185ee390dd2d4f9e Mon Sep 17 00:00:00 2001 From: Cynthia Coan Date: Sun, 30 Jun 2024 14:50:36 -0700 Subject: [PATCH] introduce mion dump-eeprom & dump-memory (#15) * introduce mion dump-eeprom & dump-memory introduce two new subcommands we found while poking around in telnet memory dumps (telnet has the same username/password as the http interface). these cgi pages allow us to figure out the memory, and eeprom of the MION itself, which should be helpful for certain tasks. * retry dumping memory * fix clap breakage * allow retry on body read too * write as needed + resumption fixes #16 this writes the memory dump file which takes forever (4Gbs at 512 bytes per request, which can sometimes fail and need to be retried) as the requests come in rather than actually waiting for it to all be buffered in memory first. this also lowers the total amount of memory required to do so. also allow resuming a stopped/failed memory dump. * mention how long a mion memory dump takes * concurrently fetch pages --- .../src/commands/mion/dump_eeprom.rs | 76 ++++ .../src/commands/mion/dump_memory.rs | 102 +++++ cmd/bridgectl/src/commands/mion/mod.rs | 7 + cmd/bridgectl/src/commands/mod.rs | 2 + cmd/bridgectl/src/exit_codes.rs | 5 + cmd/bridgectl/src/knobs/cli.rs | 103 ++++- cmd/bridgectl/src/main.rs | 33 +- pkg/cat-dev/src/errors.rs | 6 + pkg/cat-dev/src/mion/cgis/dump_eeprom.rs | 170 +++++++ pkg/cat-dev/src/mion/cgis/dump_memory.rs | 419 ++++++++++++++++++ pkg/cat-dev/src/mion/cgis/mod.rs | 4 + 11 files changed, 924 insertions(+), 3 deletions(-) create mode 100644 cmd/bridgectl/src/commands/mion/dump_eeprom.rs create mode 100644 cmd/bridgectl/src/commands/mion/dump_memory.rs create mode 100644 cmd/bridgectl/src/commands/mion/mod.rs create mode 100644 pkg/cat-dev/src/mion/cgis/dump_eeprom.rs create mode 100644 pkg/cat-dev/src/mion/cgis/dump_memory.rs diff --git a/cmd/bridgectl/src/commands/mion/dump_eeprom.rs b/cmd/bridgectl/src/commands/mion/dump_eeprom.rs new file mode 100644 index 0000000..5258f7b --- /dev/null +++ b/cmd/bridgectl/src/commands/mion/dump_eeprom.rs @@ -0,0 +1,76 @@ +//! Dump the EEPROM for a running CAT-DEV. + +use crate::{ + commands::argv_helpers::get_targeted_bridge_ip, + exit_codes::{DUMP_EEPROM_FAILURE, FAILED_TO_WRITE_TO_DISK}, + SHOULD_LOG_JSON, +}; +use cat_dev::mion::cgis::dump_eeprom; +use miette::miette; +use std::path::PathBuf; +use tokio::fs::write; +use tracing::{error, info}; + +/// Actual command handler for the `mion dump-eeprom` command. +pub async fn handle_dump_eeprom(output_path: Option) { + let bridge_ip = get_targeted_bridge_ip().await; + if SHOULD_LOG_JSON() { + info!( + id = "bridgectl::mion::dump_eeprom::start", + %bridge_ip, + "Dumping MION EEPROM...", + ); + } else { + info!( + %bridge_ip, + "Dumping MION EEPROM...", + ); + } + + match dump_eeprom(bridge_ip).await { + Ok(memory) => { + let err = if let Some(path) = output_path.as_ref() { + write(path, memory).await.err() + } else { + write("eeprom-memory.bin", memory).await.err() + }; + + if let Some(cause) = err { + if SHOULD_LOG_JSON() { + error!( + id = "bridgectl::mion::dump_eeprom::write_failure", + %bridge_ip, + ?cause, + path = output_path.map_or("eeprom-memory.bin".to_owned(), |p| p.to_string_lossy().to_string()), + "Failed to write eeprom memory to disk!", + ); + } else { + error!( + "\n{:?}", + miette!("Could not write successfully dumped MION's EEPROM Memory") + .wrap_err(cause), + ); + } + + std::process::exit(FAILED_TO_WRITE_TO_DISK); + } + } + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + id = "bridgectl::mion::dump_eeprom::failure", + %bridge_ip, + ?cause, + "Failure to dump MION's EEPROM memory", + ); + } else { + error!( + "\n{:?}", + miette!("Could not dump MION's EEPROM Memory.").wrap_err(cause), + ); + } + + std::process::exit(DUMP_EEPROM_FAILURE); + } + } +} diff --git a/cmd/bridgectl/src/commands/mion/dump_memory.rs b/cmd/bridgectl/src/commands/mion/dump_memory.rs new file mode 100644 index 0000000..a9582fe --- /dev/null +++ b/cmd/bridgectl/src/commands/mion/dump_memory.rs @@ -0,0 +1,102 @@ +//! Dump the memory for a running CAT-DEV. + +use crate::{ + commands::argv_helpers::get_targeted_bridge_ip, + exit_codes::{DUMP_MEMORY_FAILURE, FAILED_TO_WRITE_TO_DISK}, + SHOULD_LOG_JSON, +}; +use cat_dev::mion::cgis::dump_memory_with_writer; +use miette::miette; +use std::{ + fs::OpenOptions, + io::{BufWriter, Write}, + path::PathBuf, +}; +use tracing::{error, info}; + +/// Actual command handler for the `mion dump-memory` command. +pub async fn handle_dump_memory(output_path: Option, resume_at: Option) { + let bridge_ip = get_targeted_bridge_ip().await; + if SHOULD_LOG_JSON() { + info!( + id = "bridgectl::mion::dump_memory::start", + %bridge_ip, + "Dumping MION Memory, this will take a LONG time...", + ); + } else { + info!( + %bridge_ip, + "Dumping MION Memory, this will take a LONG time...", + ); + } + + let path = output_path.unwrap_or(PathBuf::from("88F6281-memory.bin")); + let file_writer = match OpenOptions::new() + .write(true) + .append(resume_at.is_some()) + .create(true) + .open(&path) + { + Ok(val) => val, + Err(cause) => { + if SHOULD_LOG_JSON() { + error!( + id = "bridgectl::mion::dump_memory::write_failure", + %bridge_ip, + ?cause, + path = %path.to_string_lossy(), + "Failed to write 88F6281 memory to disk!", + ); + } else { + error!( + "\n{:?}", + miette!("Could not write successfully dumped MION's 88F6281 Memory") + .wrap_err(cause), + ); + } + + std::process::exit(FAILED_TO_WRITE_TO_DISK); + } + }; + let mut buff_writer = BufWriter::new(file_writer); + + if let Err(cause) = dump_memory_with_writer(bridge_ip, resume_at, |bytes: Vec| { + if let Err(cause) = buff_writer.write(&bytes) { + if SHOULD_LOG_JSON() { + error!( + id = "bridgectl::mion::dump_memory::write_failure", + %bridge_ip, + ?cause, + path = %path.to_string_lossy(), + "Failed to write 88F6281 memory to disk!", + ); + } else { + error!( + "\n{:?}", + miette!("Could not write successfully dumped MION's 88F6281 Memory") + .wrap_err(cause), + ); + } + + std::process::exit(FAILED_TO_WRITE_TO_DISK); + } + }) + .await + { + if SHOULD_LOG_JSON() { + error!( + id = "bridgectl::mion::dump_memory::failure", + %bridge_ip, + ?cause, + "Failure to dump MION's memory", + ); + } else { + error!( + "\n{:?}", + miette!("Could not dump MION's Memory.").wrap_err(cause), + ); + } + + std::process::exit(DUMP_MEMORY_FAILURE); + } +} diff --git a/cmd/bridgectl/src/commands/mion/mod.rs b/cmd/bridgectl/src/commands/mion/mod.rs new file mode 100644 index 0000000..fb318e0 --- /dev/null +++ b/cmd/bridgectl/src/commands/mion/mod.rs @@ -0,0 +1,7 @@ +//! Subdirectory containing MION subcommands. + +mod dump_eeprom; +mod dump_memory; + +pub use dump_eeprom::*; +pub use dump_memory::*; diff --git a/cmd/bridgectl/src/commands/mod.rs b/cmd/bridgectl/src/commands/mod.rs index 70fff40..771b0e0 100644 --- a/cmd/bridgectl/src/commands/mod.rs +++ b/cmd/bridgectl/src/commands/mod.rs @@ -28,3 +28,5 @@ pub use remove::*; pub use set_default::*; pub use set_parameters::*; pub use tail::*; + +pub mod mion; diff --git a/cmd/bridgectl/src/exit_codes.rs b/cmd/bridgectl/src/exit_codes.rs index 1dfd6ae..09af196 100644 --- a/cmd/bridgectl/src/exit_codes.rs +++ b/cmd/bridgectl/src/exit_codes.rs @@ -3,6 +3,7 @@ pub const NOT_YET_IMPLEMENTED: i32 = 1; pub const LOGGING_HANDLER_INSTALL_FAILURE: i32 = 2; pub const SHOULD_NEVER_HAPPEN_FAILURE: i32 = 3; +pub const FAILED_TO_WRITE_TO_DISK: i32 = 4; pub const ARGV_PARSE_FAILURE: i32 = 10; pub const ARGV_NO_COMMAND_SPECIFIED: i32 = 11; @@ -49,3 +50,7 @@ pub const SET_PARAMS_FAILED_TO_SET_PARAMS: i32 = 78; pub const TAIL_NEEDS_SERIAL_PORT: i32 = 80; pub const TAIL_COULD_NOT_SPAWN: i32 = 81; + +pub const DUMP_EEPROM_FAILURE: i32 = 90; + +pub const DUMP_MEMORY_FAILURE: i32 = 100; diff --git a/cmd/bridgectl/src/knobs/cli.rs b/cmd/bridgectl/src/knobs/cli.rs index abdd496..5f77ff3 100644 --- a/cmd/bridgectl/src/knobs/cli.rs +++ b/cmd/bridgectl/src/knobs/cli.rs @@ -1,6 +1,6 @@ //! Defines the command line interface a.k.a. all the arguments & flags. -use clap::{Args, Parser}; +use clap::{Args, Parser, Subcommand}; use mac_address::MacAddress; use std::{ fmt::{Display, Formatter, Result as FmtResult}, @@ -58,7 +58,7 @@ pub enum Subcommands { // Add only command arguments. // /////////////////////////////////////////////////// #[arg( - long = "default", + long = "set-default", help = "Makes this bridge the default.", long_help = "Sets the bridge as the default bridge to use when opening new shells, with this you don't need to separately call `set-default`." )] @@ -261,6 +261,8 @@ pub enum Subcommands { visible_aliases = ["ls-serial-ports", "lssp", "list_serial_ports", "ls_serial_ports"], )] ListSerialPorts {}, + #[clap(subcommand)] + Mion(MionSubcommands), /// Remove a bridge from your local configuration file. #[command(name = "remove", visible_alias = "rm")] Remove { @@ -424,6 +426,7 @@ impl Subcommands { || name == "ls_serial_ports" || name == "lssp" } + Self::Mion(_) => name == "mion", Self::Remove { bridge_config_flags, scan_flags, @@ -452,6 +455,102 @@ impl Subcommands { } } +#[derive(Subcommand, Debug)] +pub enum MionSubcommands { + /// Dump the EEPROM on a mion. + #[command(name = "dump-eeprom", alias = "dump_eeprom")] + DumpEeprom { + // /////////////////////////////////////////////////// + // Shared Flags for targeting a single bridge. + // /////////////////////////////////////////////////// + #[command(flatten)] + bridge_config_flags: BridgeConfigurationFlags, + #[command(flatten)] + scan_flags: BridgeScanFlags, + #[command(flatten)] + target_flags: TargetBridgeFlags, + #[arg( + index = 1, + help = "Search for a bridge with a particular name/ip/mac address.", + long_help = "If you don't want to specify what type you're searching for with `--ip`, `--mac-address`, or `--name` you can just pass in a positional argument where we can guess" + )] + bridge_name_positional: Option, + // /////////////////////////////////////////////////// + // Get only command flags. + // /////////////////////////////////////////////////// + #[arg( + short = 'p', + long = "output-path", + alias = "output_path", + help = "The path to output the dumped EEPROM.", + long_help = "The path to the file to write the EEPMROM dump." + )] + output_path: Option, + }, + /// Dump the Memory on a mion. + #[command(name = "dump-memory", alias = "dump_memory")] + DumpMemory { + // /////////////////////////////////////////////////// + // Shared Flags for targeting a single bridge. + // /////////////////////////////////////////////////// + #[command(flatten)] + bridge_config_flags: BridgeConfigurationFlags, + #[command(flatten)] + scan_flags: BridgeScanFlags, + #[command(flatten)] + target_flags: TargetBridgeFlags, + #[arg( + index = 1, + help = "Search for a bridge with a particular name/ip/mac address.", + long_help = "If you don't want to specify what type you're searching for with `--ip`, `--mac-address`, or `--name` you can just pass in a positional argument where we can guess" + )] + bridge_name_positional: Option, + // /////////////////////////////////////////////////// + // Get only command flags. + // /////////////////////////////////////////////////// + #[arg( + short = 'p', + long = "output-path", + alias = "output_path", + help = "The path to output the dumped EEPROM.", + long_help = "The path to the file to write the EEPMROM dump." + )] + output_path: Option, + #[arg( + short = 'r', + long = "resume-at", + alias = "resume_at", + help = "The byte offset to resume reading at.", + long_help = "The byte offset on the page to resume reading at, use debug logs to see where you are if you intend to resume." + )] + resume_at: Option, + }, +} +impl MionSubcommands { + /// If this subcommand matches a particular name. + #[allow(unused)] + #[must_use] + pub fn name_matches(&self, name: &str) -> bool { + match self { + Self::DumpEeprom { + bridge_config_flags, + scan_flags, + target_flags, + bridge_name_positional, + output_path, + } => name == "dump-eeprom" || name == "dump_eeprom", + Self::DumpMemory { + bridge_config_flags, + scan_flags, + target_flags, + bridge_name_positional, + output_path, + resume_at, + } => name == "dump-memory" || name == "dump_memory", + } + } +} + /// Arguments specific to targeting just a single bridge to lookup. #[derive(Args, Debug)] pub struct TargetBridgeFlags { diff --git a/cmd/bridgectl/src/main.rs b/cmd/bridgectl/src/main.rs index a1f1604..6d00893 100644 --- a/cmd/bridgectl/src/main.rs +++ b/cmd/bridgectl/src/main.rs @@ -18,13 +18,15 @@ use crate::{ handle_add_or_update, handle_boot, handle_dump_parameters, handle_get, handle_get_parameters, handle_help, handle_list, handle_list_serial_ports, handle_remove_bridge, handle_set_default_bridge, handle_set_parameters, handle_tail, + mion::handle_dump_eeprom as mion_handle_dump_eeprom, + mion::handle_dump_memory as mion_handle_dump_memory, }, exit_codes::{ ARGV_NO_COMMAND_SPECIFIED, ARGV_PARSE_FAILURE, LOGGING_HANDLER_INSTALL_FAILURE, SHOULD_NEVER_HAPPEN_FAILURE, }, knobs::{ - cli::{CliArguments, Subcommands}, + cli::{CliArguments, MionSubcommands, Subcommands}, env::USE_JSON_OUTPUT, }, }; @@ -185,6 +187,35 @@ async fn main() { Subcommands::ListSerialPorts {} => { handle_list_serial_ports(); } + Subcommands::Mion(mion_subcommands) => match mion_subcommands { + MionSubcommands::DumpEeprom { + bridge_config_flags, + scan_flags, + target_flags, + bridge_name_positional, + output_path, + } => { + initialize_host_bridge(bridge_config_flags).await; + initialize_scan_flags(scan_flags).await; + _ = target_bridge(target_flags, bridge_name_positional.as_deref(), false).await; + + mion_handle_dump_eeprom(output_path).await; + } + MionSubcommands::DumpMemory { + bridge_config_flags, + scan_flags, + target_flags, + bridge_name_positional, + output_path, + resume_at, + } => { + initialize_host_bridge(bridge_config_flags).await; + initialize_scan_flags(scan_flags).await; + _ = target_bridge(target_flags, bridge_name_positional.as_deref(), false).await; + + mion_handle_dump_memory(output_path, resume_at).await; + } + }, Subcommands::Remove { bridge_config_flags, scan_flags, diff --git a/pkg/cat-dev/src/errors.rs b/pkg/cat-dev/src/errors.rs index 0b84a1e..e1ae66b 100644 --- a/pkg/cat-dev/src/errors.rs +++ b/pkg/cat-dev/src/errors.rs @@ -267,4 +267,10 @@ pub enum NetworkParseError { #[error("Could not parse HTML response could not find one of the body tags: ``, or ``: {0}")] #[diagnostic(code(cat_dev::net::parse::html::no_body_tag))] HtmlResponseMissingBody(String), + #[error("Could not find Memory Dump Table Body, failed to find sigils: {0}")] + #[diagnostic(code(cat_dev::net::parse::html::no_mem_dump_sigil))] + HtmlResponseMissingMemoryDumpSigil(String), + #[error("Could not parse byte from memory dump: {0}")] + #[diagnostic(code(cat_dev::net::parse::html::bad_memory_byte))] + HtmlResponseBadByte(String), } diff --git a/pkg/cat-dev/src/mion/cgis/dump_eeprom.rs b/pkg/cat-dev/src/mion/cgis/dump_eeprom.rs new file mode 100644 index 0000000..8c30723 --- /dev/null +++ b/pkg/cat-dev/src/mion/cgis/dump_eeprom.rs @@ -0,0 +1,170 @@ +//! API's for interacting with `/dbg/eeprom_dump.cgi`, a page for live +//! accessing the memory of the EEPROM on the MION devices. + +use crate::{ + errors::{CatBridgeError, NetworkError, NetworkParseError}, + mion::cgis::AUTHZ_HEADER, +}; +use bytes::{BufMut, Bytes, BytesMut}; +use hyper::{ + body::to_bytes as read_http_body_bytes, + client::{connect::Connect, Client}, + Body, Request, Response, Version, +}; +use serde::Serialize; +use std::net::Ipv4Addr; +use tracing::debug; + +const EEPROM_MAX_ADDRESS: usize = 0x1E00; +const TABLE_START_SIGIL: &str = ""; +const TABLE_END_SIGIL: &str = "
"; + +/// Dump the existing EEPROM for a MION. +/// +/// ## Errors +/// +/// - If we cannot encode the parameters as a form url encoded. +/// - If we cannot make the HTTP request. +/// - If the server does not respond with a 200. +/// - If we cannot read the body from HTTP. +/// - If we cannot parse the HTML response. +pub async fn dump_eeprom(mion_ip: Ipv4Addr) -> Result { + dump_eeprom_with_raw_client(&Client::default(), mion_ip).await +} + +/// Perform an EEPROM DUMP request, but with an already existing HTTP client. +/// +/// ## Errors +/// +/// - If we cannot encode the parameters as a form url encoded. +/// - If we cannot make the HTTP request. +/// - If the server does not respond with a 200. +/// - If we cannot read the body from HTTP. +/// - If we cannot parse the HTML response. +pub async fn dump_eeprom_with_raw_client( + client: &Client, + mion_ip: Ipv4Addr, +) -> Result +where + ClientConnectorTy: Clone + Connect + Send + Sync + 'static, +{ + let mut memory_buffer = BytesMut::with_capacity(0x2000); + + while memory_buffer.len() <= EEPROM_MAX_ADDRESS { + debug!( + bridge.ip = %mion_ip, + address = %format!("{:04X}", memory_buffer.len()), + "Dumping eeprom memory area", + ); + + let response = do_raw_eeprom_request( + client, + mion_ip, + &[("start_addr", format!("{:04X}", memory_buffer.len()))], + ) + .await?; + + let status = response.status().as_u16(); + let body_result = read_http_body_bytes(response.into_body()) + .await + .map_err(NetworkError::HyperError); + if status != 200 { + if let Ok(body) = body_result { + return Err(CatBridgeError::NetworkError(NetworkError::ParseError( + NetworkParseError::UnexpectedStatusCode(status, body), + ))); + } + + return Err(CatBridgeError::NetworkError(NetworkError::ParseError( + NetworkParseError::UnexpectedStatusCodeNoBody(status), + ))); + } + let read_body_bytes = body_result?; + let body_as_string = String::from_utf8(read_body_bytes.into()) + .map_err(NetworkParseError::InvalidDataNeedsUTF8) + .map_err(NetworkError::ParseError)?; + + let table = extract_memory_table_body(&body_as_string)?; + for table_row in table.split("").skip(3) { + for table_column in table_row + .trim() + .trim_end_matches("") + .trim_end() + .trim_end_matches("") + .trim_end() + .replace("", "") + .split("") + .skip(3) + { + if table_column.trim().len() != 2 { + return Err(CatBridgeError::NetworkError(NetworkError::ParseError( + NetworkParseError::HtmlResponseBadByte(table_column.to_owned()), + ))); + } + memory_buffer.put_u8(u8::from_str_radix(table_column.trim(), 16).map_err( + |_| { + NetworkError::ParseError(NetworkParseError::HtmlResponseBadByte( + table_column.to_owned(), + )) + }, + )?); + } + } + } + + Ok(memory_buffer.freeze()) +} + +fn extract_memory_table_body(body: &str) -> Result { + let start = body.find(TABLE_START_SIGIL).ok_or_else(|| { + NetworkError::ParseError(NetworkParseError::HtmlResponseMissingMemoryDumpSigil( + body.to_owned(), + )) + })?; + let body_minus_start = &body[start + TABLE_START_SIGIL.len()..]; + let end = body_minus_start.find(TABLE_END_SIGIL).ok_or_else(|| { + NetworkError::ParseError(NetworkParseError::HtmlResponseMissingMemoryDumpSigil( + body.to_owned(), + )) + })?; + + Ok(body_minus_start[..end].to_owned()) +} + +/// Perform a raw request on the MION board's `eeprom_dump.cgi` page. +/// +/// *note: you probably want to call one of the actual methods, as this is +/// basically just a thin wrapper around an HTTP Post Request. Not doing much +/// else more. A lot of it requires that you set things up correctly.* +/// +/// ## Errors +/// +/// - If we cannot make an HTTP request to the MION Request. +/// - If we fail to encode your parameters into a request body. +pub async fn do_raw_eeprom_request<'key, 'value, ClientConnectorTy, UrlEncodableType>( + client: &Client, + mion_ip: Ipv4Addr, + url_parameters: UrlEncodableType, +) -> Result, NetworkError> +where + ClientConnectorTy: Clone + Connect + Send + Sync + 'static, + UrlEncodableType: Serialize, +{ + Ok(client + .request( + Request::post(format!("http://{mion_ip}/dbg/eeprom_dump.cgi")) + .version(Version::HTTP_11) + .header("authorization", format!("Basic {AUTHZ_HEADER}")) + .header("content-type", "application/x-www-form-urlencoded") + .header( + "user-agent", + format!("cat-dev/{}", env!("CARGO_PKG_VERSION")), + ) + .body( + serde_urlencoded::to_string(&url_parameters) + .map_err(NetworkParseError::FormDataEncodeError)? + .into(), + )?, + ) + .await?) +} diff --git a/pkg/cat-dev/src/mion/cgis/dump_memory.rs b/pkg/cat-dev/src/mion/cgis/dump_memory.rs new file mode 100644 index 0000000..b434257 --- /dev/null +++ b/pkg/cat-dev/src/mion/cgis/dump_memory.rs @@ -0,0 +1,419 @@ +//! API's for interacting with `/dbg/mem_dump.cgi`, a page for live +//! accessing the memory of the memory on the main chip of the MION +//! devices. +//! +//! Prefer this over `dbytes` over telnet, as there are some bytes that +//! cannot be read over telnet, that can be read with this CGI interface. + +use crate::{ + errors::{CatBridgeError, NetworkError, NetworkParseError}, + mion::cgis::AUTHZ_HEADER, +}; +use bytes::{Bytes, BytesMut}; +use fnv::FnvHashMap; +use futures::{future::Either, StreamExt}; +use hyper::{ + body::to_bytes as read_http_body_bytes, + client::{connect::Connect, Client}, + Body, Request, Response, Version, +}; +use serde::Serialize; +use std::{ + net::Ipv4Addr, + sync::atomic::{AtomicU8, Ordering as AtomicOrdering}, + time::Duration, +}; +use tokio::{ + sync::mpsc::{ + channel as bounded_channel, Receiver as BoundedReceiver, Sender as BoundedSender, + }, + time::timeout, +}; +use tracing::debug; + +const MEMORY_MAX_ADDRESS: usize = 0xFFFF_FE00; +const TABLE_START_SIGIL: &str = ""; +const TABLE_END_SIGIL: &str = "
"; +const MAX_RETRIES: u8 = 10; +const BACKOFF_SLEEP_SECONDS: u64 = 10; +const MEMORY_TIMEOUT_SECONDS: u64 = 30; +const MAX_MEMORY_CONCURRENCY: usize = 4; + +/// Dump the existing memory for a MION. +/// +/// ## Errors +/// +/// - If we cannot encode the parameters as a form url encoded. +/// - If we cannot make the HTTP request. +/// - If the server does not respond with a 200. +/// - If we cannot read the body from HTTP. +/// - If we cannot parse the HTML response. +pub async fn dump_memory( + mion_ip: Ipv4Addr, + resume_at: Option, +) -> Result { + let mut memory_buffer = BytesMut::with_capacity(0xFFFF_FFFF); + dump_memory_with_raw_client(&Client::default(), mion_ip, resume_at, |bytes: Vec| { + memory_buffer.extend_from_slice(&bytes); + }) + .await?; + Ok(memory_buffer.freeze()) +} + +/// Dump the existing memory for a MION allowing you to write as +/// dumps happen. +/// +/// ## Errors +/// +/// - If we cannot encode the parameters as a form url encoded. +/// - If we cannot make the HTTP request. +/// - If the server does not respond with a 200. +/// - If we cannot read the body from HTTP. +/// - If we cannot parse the HTML response. +pub async fn dump_memory_with_writer( + mion_ip: Ipv4Addr, + resume_at: Option, + callback: FnTy, +) -> Result<(), CatBridgeError> +where + FnTy: FnMut(Vec) + Send + Sync, +{ + dump_memory_with_raw_client(&Client::default(), mion_ip, resume_at, callback).await +} + +/// Perform a memory dump request, but with an already existing HTTP client. +/// +/// ## Errors +/// +/// - If we cannot encode the parameters as a form url encoded. +/// - If we cannot make the HTTP request. +/// - If the server does not respond with a 200. +/// - If we cannot read the body from HTTP. +/// - If we cannot parse the HTML response. +pub async fn dump_memory_with_raw_client( + client: &Client, + mion_ip: Ipv4Addr, + resume_at: Option, + buff_callback: FnTy, +) -> Result<(), CatBridgeError> +where + ClientConnectorTy: Clone + Connect + Send + Sync + 'static, + FnTy: FnMut(Vec) + Send + Sync, +{ + let (stop_requests_sender, mut stop_requests_consumer) = bounded_channel(1); + let retry_counter = AtomicU8::new(0); + + let start_address = resume_at.unwrap_or(0); + let (page_results_sender, page_results_consumer) = bounded_channel(512); + + let retry_counter_ref = &retry_counter; + let sender_ref = &page_results_sender; + // This will make memory page requests concurrently in chunks of + // `MAX_MEMORY_CONCURRENCY`, with each task handling it's own retry + // logic. + // + // If requests start throwing errors, the receiving channel will send + // a message to `stop_requests_sender` which will shut everything down. + let buffered_stream_future = + futures::stream::iter((start_address..=MEMORY_MAX_ADDRESS).step_by(512)) + .map(|page_start| async move { + loop { + if !do_memory_page_fetch( + client, + mion_ip, + page_start, + retry_counter_ref, + sender_ref, + ) + .await + { + break; + } + } + }) + .buffered(MAX_MEMORY_CONCURRENCY) + .collect::>(); + + // As requests finish, they may not necissarily be in order. + // We need to reorder them to ensure they're called back in a + // serial order. + let join_handle = do_memory_page_ordering( + page_results_consumer, + stop_requests_sender, + start_address, + buff_callback, + ); + + { + let recv_future = stop_requests_consumer.recv(); + + futures::pin_mut!(buffered_stream_future); + futures::pin_mut!(recv_future); + + let (select, _joined) = futures::future::join( + // Wait for all requests to finish, or a stop signal caused by an error. + futures::future::select(buffered_stream_future, recv_future), + // Wait for all of our ordering to finish too. + join_handle, + ) + .await; + match select { + Either::Right((error, _)) => { + if let Some(cause) = error { + return Err(cause); + } + } + Either::Left(_) => {} + } + } + + // Double check we didn't get an error ordering all the pages. + if let Ok(cause) = stop_requests_consumer.try_recv() { + return Err(cause); + } + + Ok(()) +} + +async fn do_memory_page_ordering( + mut results: BoundedReceiver), CatBridgeError>>, + stopper: BoundedSender, + start_at: usize, + mut callback: FnTy, +) where + FnTy: FnMut(Vec), +{ + let mut out_of_order_cache: FnvHashMap> = FnvHashMap::default(); + let mut looking_for_page = start_at; + + while looking_for_page <= MEMORY_MAX_ADDRESS { + if let Some(data) = out_of_order_cache.remove(&looking_for_page) { + callback(data); + looking_for_page += 512; + continue; + } + + let Some(page_result) = results.recv().await else { + _ = stopper.send(CatBridgeError::ClosedChannel).await; + break; + }; + match page_result { + Ok((addr, data)) => { + out_of_order_cache.insert(addr, data); + } + Err(cause) => { + _ = stopper.send(cause).await; + break; + } + } + } +} + +async fn do_memory_page_fetch( + client: &Client, + mion_ip: Ipv4Addr, + page_start: usize, + retry_counter: &AtomicU8, + result_stream: &BoundedSender), CatBridgeError>>, +) -> bool +where + ClientConnectorTy: Clone + Connect + Send + Sync + 'static, +{ + let start_addr = format!("{page_start:08X}"); + debug!( + bridge.ip = %mion_ip, + addr = %start_addr, + "Performing memory page fetch", + ); + + let timeout_response = timeout( + Duration::from_secs(MEMORY_TIMEOUT_SECONDS), + do_raw_memory_request(client, mion_ip, &[("start_addr", start_addr)]), + ) + .await; + + let Ok(potential_response) = timeout_response else { + if retry_counter.fetch_add(1, AtomicOrdering::AcqRel) > MAX_RETRIES { + _ = result_stream + .send(Err(NetworkError::TimeoutError.into())) + .await; + return false; + } + debug!(bridge.ip = %mion_ip, "Slamming Memory dump too hard... backing off for a bit"); + tokio::time::sleep(Duration::from_secs(BACKOFF_SLEEP_SECONDS)).await; + return true; + }; + let response = match potential_response { + Ok(value) => value, + Err(cause) => { + if retry_counter.fetch_add(1, AtomicOrdering::AcqRel) > MAX_RETRIES { + _ = result_stream.send(Err(cause.into())).await; + return false; + } + debug!(bridge.ip = %mion_ip, "Slamming Memory dump too hard... backing off for a bit"); + tokio::time::sleep(Duration::from_secs(BACKOFF_SLEEP_SECONDS)).await; + return true; + } + }; + + let status = response.status().as_u16(); + let timeout_body_result = timeout( + Duration::from_secs(MEMORY_TIMEOUT_SECONDS), + read_http_body_bytes(response.into_body()), + ) + .await; + let Ok(body_result) = timeout_body_result else { + if retry_counter.fetch_add(1, AtomicOrdering::AcqRel) > MAX_RETRIES { + _ = result_stream + .send(Err(NetworkError::TimeoutError.into())) + .await; + return false; + } + debug!(bridge.ip = %mion_ip, "Slamming Memory dump too hard... backing off for a bit"); + tokio::time::sleep(Duration::from_secs(BACKOFF_SLEEP_SECONDS)).await; + return true; + }; + + retry_counter.store(0, AtomicOrdering::Release); + if status != 200 { + if let Ok(body) = body_result { + _ = result_stream + .send(Err(CatBridgeError::NetworkError(NetworkError::ParseError( + NetworkParseError::UnexpectedStatusCode(status, body), + )))) + .await; + return false; + } + + _ = result_stream + .send(Err(CatBridgeError::NetworkError(NetworkError::ParseError( + NetworkParseError::UnexpectedStatusCodeNoBody(status), + )))) + .await; + return false; + } + let read_body_bytes = match body_result.map_err(NetworkError::HyperError) { + Ok(value) => value, + Err(cause) => { + _ = result_stream.send(Err(cause.into())).await; + return false; + } + }; + let body_as_string = match String::from_utf8(read_body_bytes.into()) + .map_err(NetworkParseError::InvalidDataNeedsUTF8) + .map_err(NetworkError::ParseError) + { + Ok(value) => value, + Err(cause) => { + _ = result_stream.send(Err(cause.into())).await; + return false; + } + }; + + process_received_page(page_start, result_stream, &body_as_string).await +} + +async fn process_received_page( + page_start: usize, + result_stream: &BoundedSender), CatBridgeError>>, + body_as_string: &str, +) -> bool { + let table = match extract_memory_table_body(body_as_string) { + Ok(value) => value, + Err(cause) => { + _ = result_stream.send(Err(cause)).await; + return false; + } + }; + let mut page_of_bytes = Vec::with_capacity(512); + for table_row in table.split("").skip(3) { + for table_column in table_row + .trim() + .trim_end_matches("") + .trim_end() + .trim_end_matches("") + .trim_end() + .replace("", "") + .split("") + .skip(3) + { + if table_column.trim().len() != 2 { + _ = result_stream + .send(Err(CatBridgeError::NetworkError(NetworkError::ParseError( + NetworkParseError::HtmlResponseBadByte(table_column.to_owned()), + )))) + .await; + return false; + } + let byte = match u8::from_str_radix(table_column.trim(), 16).map_err(|_| { + NetworkError::ParseError(NetworkParseError::HtmlResponseBadByte( + table_column.to_owned(), + )) + }) { + Ok(value) => value, + Err(cause) => { + _ = result_stream.send(Err(cause.into())).await; + return false; + } + }; + page_of_bytes.push(byte); + } + } + + _ = result_stream.send(Ok((page_start, page_of_bytes))).await; + false +} + +fn extract_memory_table_body(body: &str) -> Result { + let start = body.find(TABLE_START_SIGIL).ok_or_else(|| { + NetworkError::ParseError(NetworkParseError::HtmlResponseMissingMemoryDumpSigil( + body.to_owned(), + )) + })?; + let body_minus_start = &body[start + TABLE_START_SIGIL.len()..]; + let end = body_minus_start.find(TABLE_END_SIGIL).ok_or_else(|| { + NetworkError::ParseError(NetworkParseError::HtmlResponseMissingMemoryDumpSigil( + body.to_owned(), + )) + })?; + + Ok(body_minus_start[..end].to_owned()) +} + +/// Perform a raw request on the MION board's `eeprom_dump.cgi` page. +/// +/// *note: you probably want to call one of the actual methods, as this is +/// basically just a thin wrapper around an HTTP Post Request. Not doing much +/// else more. A lot of it requires that you set things up correctly.* +/// +/// ## Errors +/// +/// - If we cannot make an HTTP request to the MION Request. +/// - If we fail to encode your parameters into a request body. +pub async fn do_raw_memory_request<'key, 'value, ClientConnectorTy, UrlEncodableType>( + client: &Client, + mion_ip: Ipv4Addr, + url_parameters: UrlEncodableType, +) -> Result, NetworkError> +where + ClientConnectorTy: Clone + Connect + Send + Sync + 'static, + UrlEncodableType: Serialize, +{ + Ok(client + .request( + Request::post(format!("http://{mion_ip}/dbg/mem_dump.cgi")) + .version(Version::HTTP_11) + .header("authorization", format!("Basic {AUTHZ_HEADER}")) + .header("content-type", "application/x-www-form-urlencoded") + .header( + "user-agent", + format!("cat-dev/{}", env!("CARGO_PKG_VERSION")), + ) + .body( + serde_urlencoded::to_string(&url_parameters) + .map_err(NetworkParseError::FormDataEncodeError)? + .into(), + )?, + ) + .await?) +} diff --git a/pkg/cat-dev/src/mion/cgis/mod.rs b/pkg/cat-dev/src/mion/cgis/mod.rs index 2269d3a..e619cb2 100644 --- a/pkg/cat-dev/src/mion/cgis/mod.rs +++ b/pkg/cat-dev/src/mion/cgis/mod.rs @@ -15,7 +15,11 @@ const AUTHZ_HEADER: &str = "bWlvbjovTXVsdGlfSS9PX05ldHdvcmsv"; mod control; +mod dump_eeprom; +mod dump_memory; mod signal_get; pub use control::*; +pub use dump_eeprom::*; +pub use dump_memory::*; pub use signal_get::*;