From 448bcece61b3115e34072df03421854c87691b70 Mon Sep 17 00:00:00 2001 From: David Chalifoux Date: Thu, 7 Nov 2024 22:47:39 -0500 Subject: [PATCH 1/4] feat: Resumable uploads --- .gitignore | 3 +- Cargo.lock | 190 +++++++++++++++++++++++++++++++-------------- Cargo.toml | 12 +-- src/main.rs | 60 +++++---------- src/put.rs | 1 + src/put/files.rs | 29 +++++++ src/put/tus.rs | 197 +++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 387 insertions(+), 105 deletions(-) create mode 100644 src/put/tus.rs diff --git a/.gitignore b/.gitignore index 2a0038a..96f76cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target -.idea \ No newline at end of file +.idea +.DS_Store diff --git a/Cargo.lock b/Cargo.lock index ee969c6..95c3f15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,15 +125,27 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" -version = "1.3.2" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] -name = "bitflags" -version = "2.6.0" +name = "blake2" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] [[package]] name = "bumpalo" @@ -218,7 +230,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -261,6 +273,16 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "darling" version = "0.20.10" @@ -282,7 +304,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -293,7 +315,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -306,6 +328,17 @@ dependencies = [ "serde", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "directories" version = "5.0.1" @@ -438,6 +471,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -715,6 +758,8 @@ dependencies = [ name = "kaput-cli" version = "2.4.2" dependencies = [ + "base64", + "blake2", "bytefmt", "clap", "confy", @@ -737,7 +782,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.6.0", + "bitflags", "libc", ] @@ -849,7 +894,7 @@ version = "0.10.66" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" dependencies = [ - "bitflags 2.6.0", + "bitflags", "cfg-if", "foreign-types", "libc", @@ -866,7 +911,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -905,9 +950,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "papergrid" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ad43c07024ef767f9160710b3a6773976194758c7919b17e63b863db0bdf7fb" +checksum = "c7419ad52a7de9b60d33e11085a0fe3df1fbd5926aa3f93d3dd53afbc9e86725" dependencies = [ "bytecount", "fnv", @@ -937,7 +982,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1048,9 +1093,9 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" -version = "0.12.5" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64", "bytes", @@ -1088,7 +1133,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg", + "windows-registry", ] [[package]] @@ -1118,7 +1163,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.6.0", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -1186,7 +1231,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.6.0", + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -1205,31 +1250,32 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.204" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -1257,9 +1303,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64", "chrono", @@ -1275,14 +1321,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1341,9 +1387,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.72" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -1355,23 +1401,26 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] [[package]] name = "system-configuration" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 1.3.2", + "bitflags", "core-foundation", "system-configuration-sys", ] [[package]] name = "system-configuration-sys" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ "core-foundation-sys", "libc", @@ -1379,20 +1428,19 @@ dependencies = [ [[package]] name = "tabled" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c998b0c8b921495196a48aabaf1901ff28be0760136e31604f7967b0792050e" +checksum = "77c9303ee60b9bedf722012ea29ae3711ba13a67c9b9ae28993838b63057cb1b" dependencies = [ "papergrid", "tabled_derive", - "unicode-width", ] [[package]] name = "tabled_derive" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c138f99377e5d653a371cdad263615634cfc8467685dfe8e73e2b8e98f44b17" +checksum = "bf0fb8bfdc709786c154e24a66777493fb63ae97e3036d914c8666774c477069" dependencies = [ "heck 0.4.1", "proc-macro-error", @@ -1430,7 +1478,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", ] [[package]] @@ -1614,6 +1662,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicase" version = "2.7.0" @@ -1646,9 +1700,9 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.13" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "untrusted" @@ -1721,7 +1775,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -1755,7 +1809,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1785,6 +1839,36 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -1933,16 +2017,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "zeroize" version = "1.8.1" diff --git a/Cargo.toml b/Cargo.toml index 598e07b..9adccc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,17 +17,19 @@ exclude = [".gitignore", ".github/*"] [dependencies] clap = { version = "4.5.10", features = ["derive"] } confy = "0.6.1" -serde = { version = "1.0.204", features = ["derive"] } -reqwest = { version = "0.12.5", features = [ +serde = { version = "1.0.214", features = ["derive"] } +reqwest = { version = "0.12.9", features = [ "json", "blocking", "multipart", "native-tls-vendored", ] } -tabled = { version = "0.15.0", features = ["derive"] } +tabled = { version = "0.16.0", features = ["derive"] } bytefmt = "0.1.7" -serde_json = { version = "1.0.120", features = ["std"] } -serde_with = { version = "3.9.0", features = [] } +serde_json = { version = "1.0.132", features = ["std"] } +serde_with = { version = "3.11.0", features = [] } +base64 = "0.22.1" +blake2 = "0.10.6" [[bin]] name = "kaput" diff --git a/src/main.rs b/src/main.rs index 7943cf7..e0fbfb6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -158,23 +158,15 @@ fn cli() -> Command { .subcommand( Command::new("upload") .about("Upload file(s) to your account") - .long_about("Uploads file(s) to your account.") + .long_about("Uploads file(s) to your account. It will automatically switch to the resumable upload protocol if the file size is greater than or equal to 50 MB.") .arg_required_else_help(true) .arg( Arg::new("parent_id") .short('p') .long("parent") - .value_parser(value_parser!(i64)) .help("ID of a Put folder to upload to instead of the root folder") .required(false) ) - .arg( - Arg::new("file_name") - .short('n') - .long("name") - .help("Override file name") - .required(false) - ) .arg( Arg::new("is_silent") .short('s') @@ -556,11 +548,13 @@ fn main() { Some(("upload", sub_matches)) => { require_auth(&client, &config); - let parent_id = sub_matches.get_one::("parent_id"); + let path = sub_matches + .get_one::("PATH") + .expect("missing path"); - let file_name = sub_matches.get_one::("file_name"); + let parent_id: Option<&String> = sub_matches.get_one::("parent_id"); - let is_silent = sub_matches.get_one::("is_silent"); + let is_silent: Option<&bool> = sub_matches.get_one::("is_silent"); let mut curl_args: Vec = vec![]; @@ -569,35 +563,19 @@ fn main() { curl_args.push("-s".to_string()); } - let paths = sub_matches - .get_many::("PATH") - .into_iter() - .flatten() - .collect::>(); - - for path in paths { - println!("Uploading: {}\n", path.to_string_lossy()); - - ProcessCommand::new("curl") - .args(curl_args.clone()) - .arg("-H") - .arg(format!("Authorization: Bearer {}", config.api_token)) - .arg("-F") - .arg(format!("file=@{}", path.to_string_lossy())) - .arg("-F") - .arg(format!("filename={}", file_name.unwrap_or(&"".to_string()))) - .arg("-F") - .arg(format!( - "parent_id={}", - parent_id.unwrap_or(&"0".to_string()) - )) - .arg("https://upload.put.io/v2/files/upload") - .stdout(Stdio::piped()) - .spawn() - .expect("failed to run CURL command") - .wait_with_output() - .expect("failed to run CURL command"); - println!("\nUpload finished!") + let metadata: std::fs::Metadata = + std::fs::metadata(&path).expect("reading file metadata"); + + let file_size: u64 = metadata.len(); + + if file_size >= 52_428_800 { + println!("Mode: Resumable"); + + put::tus::upload(&client, &config.api_token, path, parent_id); + } else { + println!("Mode: Non-resumable"); + + put::files::upload(&config.api_token, path, parent_id, &curl_args); } } Some(("move", sub_matches)) => { diff --git a/src/put.rs b/src/put.rs index 96a9bfd..ad7d6b4 100644 --- a/src/put.rs +++ b/src/put.rs @@ -2,4 +2,5 @@ pub mod account; pub mod files; pub mod oob; pub mod transfers; +pub mod tus; pub mod zips; diff --git a/src/put/files.rs b/src/put/files.rs index 58f6b9b..e07da70 100644 --- a/src/put/files.rs +++ b/src/put/files.rs @@ -1,3 +1,4 @@ +use std::path::PathBuf; use std::process::{Command as ProcessCommand, Stdio}; use std::{fmt, fs}; @@ -333,3 +334,31 @@ pub fn download( Ok(()) } + +pub fn upload( + api_token: &String, + path: &PathBuf, + parent_id: Option<&String>, + curl_args: &Vec, +) { + println!("Uploading: {}\n", path.to_string_lossy()); + + ProcessCommand::new("curl") + .args(curl_args.clone()) + .arg("-H") + .arg(format!("Authorization: Bearer {}", api_token)) + .arg("-F") + .arg(format!("file=@{}", path.to_string_lossy())) + .arg("-F") + .arg(format!( + "parent_id={}", + parent_id.unwrap_or(&"0".to_string()) + )) + .arg("https://upload.put.io/v2/files/upload") + .stdout(Stdio::piped()) + .spawn() + .expect("failed to run CURL command") + .wait_with_output() + .expect("failed to run CURL command"); + println!("\nUpload finished!") +} diff --git a/src/put/tus.rs b/src/put/tus.rs new file mode 100644 index 0000000..98ae74a --- /dev/null +++ b/src/put/tus.rs @@ -0,0 +1,197 @@ +use blake2::{Blake2b512, Digest}; +use std::{ + io::{BufReader, Read}, + path::PathBuf, + time::Instant, +}; + +use base64::{engine::general_purpose, Engine as _}; +use reqwest::blocking::Client; + +pub fn upload(client: &Client, api_token: &String, path: &PathBuf, parent_id: Option<&String>) { + if !path.is_file() { + println!("{} is not a file", path.to_string_lossy()); + return; + } + + let file_name: String = path.file_name().unwrap().to_string_lossy().to_string(); + + let absolute_path: String = path + .canonicalize() + .expect("canonicalizing path") + .to_string_lossy() + .to_string(); + + let metadata: std::fs::Metadata = std::fs::metadata(&path).expect("reading file metadata"); + + let file_size: u64 = metadata.len(); + let last_modified: std::time::SystemTime = metadata.modified().expect("reading last modified"); + let last_modified_unix: u64 = last_modified + .duration_since(std::time::UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + + println!("Uploading: {}", file_name); + + let file: std::fs::File = std::fs::File::open(&path).expect("opening file"); + + let location: String; + + // Check if previous location exists + let temp_dir: PathBuf = std::env::temp_dir(); + let mut hasher = Blake2b512::new(); + hasher.update(format!("{}_{}", absolute_path, last_modified_unix)); + let hash: String = format!("{:x}", hasher.finalize()); + let temp_file_path: PathBuf = temp_dir.join(format!("kaput_{hash}")); + + if temp_file_path.exists() { + // Read the location from the file + println!("Resuming upload..."); + + location = std::fs::read_to_string(&temp_file_path).expect("reading temp file"); + } else { + // Get a new upload location and write it to the temp directory + location = create_upload(client, api_token, file_size, file_name.clone(), parent_id) + .expect("creating upload location"); + + std::fs::write(&temp_file_path, location.clone()).expect("writing temp file"); + } + + let mut reader: BufReader = BufReader::new(file); + + let mut total_bytes_read: u64 = 0; + + // When resuming an upload, the server will respond with the offset at which the upload should start + let resume_offset: u64 = get_offset(client, api_token, location.clone()); + + let mut chunk: Vec = vec![0; 52_428_800]; // 50 MB chunk size + + loop { + let bytes_read: usize = reader.read(&mut chunk).expect("Reading chunk from file"); + + if bytes_read == 0 { + // The entire file has been read + + // Delete the temp file + std::fs::remove_file(&temp_file_path).expect("deleting temp file"); + + println!("Upload finished!"); + break; + } + + // The start and end offsets of the current chunk + let chunk_start_offset: u64 = total_bytes_read; + let chunk_end_offset: u64 = total_bytes_read + bytes_read as u64; + + // The total number of bytes read from the file + total_bytes_read += bytes_read as u64; + + // Skip to the initial offset, then start uploading the file from the containing chunk + if total_bytes_read < resume_offset as u64 { + continue; + } + + // The offset at which the current chunk should start uploading + let mut chunk_skip_offset: usize = 0; + + // The file offset to send to the server + let mut upload_offset: u64 = chunk_start_offset; + + if resume_offset >= chunk_start_offset && resume_offset < chunk_end_offset { + // The initial offset is within the current chunk + chunk_skip_offset = resume_offset as usize - chunk_start_offset as usize; + upload_offset = resume_offset; + } + + // Measure the time taken to upload the chunk + let start_time: Instant = Instant::now(); + + // Upload the chunk + let res = client + .patch(location.clone()) + .header("authorization", format!("Bearer {api_token}")) + .header("tus-resumable", "1.0.0") + .header("upload-offset", format!("{upload_offset}")) + .header("content-type", "application/offset+octet-stream") + .header("content-length", format!("{bytes_read}")) + .body(chunk[chunk_skip_offset..bytes_read].to_vec()) + .send(); + + let elapsed_time: f64 = start_time.elapsed().as_secs_f64(); + let upload_speed: f64 = bytes_read as f64 / elapsed_time / 1_048_576.0; // Speed in MB/s + + if let Ok(response) = res { + if response.status() != 204 { + println!("Error: {}", response.status()); + panic!("Upload failed, try again in a few seconds."); + } + } else { + println!("Error: {:?}", res); + panic!("Upload failed, try again in a few seconds."); + } + + let percentage_completed: f64 = (total_bytes_read as f64 / file_size as f64) * 100.0; + println!("{:.0}% ({:.2} MB/s)", percentage_completed, upload_speed); + } + + return; +} + +fn get_offset(client: &Client, api_token: &String, location: String) -> u64 { + let res = client + .head(location) + .header("authorization", format!("Bearer {api_token}")) + .header("tus-resumable", "1.0.0") + .send(); + + if let Ok(response) = res { + if response.status() == 200 { + if let Some(offset) = response.headers().get("upload-offset") { + return offset + .to_str() + .unwrap() + .parse::() + .expect("Invalid offset value"); + } + } + } + + return 0; +} + +/// Creates a new upload using TUS, returns the location +fn create_upload( + client: &Client, + api_token: &String, + file_size: u64, + file_name: String, + parent_id: Option<&String>, +) -> Option { + let base64_name: String = general_purpose::STANDARD.encode(file_name); + let base64_parent_id: String = + general_purpose::STANDARD.encode(parent_id.unwrap_or(&"0".to_string())); + let base64_no_torrent: String = general_purpose::STANDARD.encode("true"); + + let res = client + .post("https://upload.put.io/files/") + .header("authorization", format!("Bearer {api_token}")) + .header("tus-resumable", "1.0.0") + .header("upload-length", format!("{file_size}")) + .header( + "upload-metadata", + format!( + "name {base64_name},no-torrent {base64_no_torrent},parent_id {base64_parent_id}" + ), + ) + .send(); + + if let Ok(response) = res { + if response.status() == 201 { + if let Some(location) = response.headers().get("location") { + return Some(location.to_str().unwrap().to_string()); + } + } + } + + return None; +} From ccb18391a58385a0c327015c603a9b6f5c4f10b9 Mon Sep 17 00:00:00 2001 From: David Chalifoux Date: Thu, 7 Nov 2024 22:55:24 -0500 Subject: [PATCH 2/4] cleanup --- src/main.rs | 2 +- src/put/files.rs | 4 ++-- src/put/tus.rs | 12 +++++------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/main.rs b/src/main.rs index e0fbfb6..c662614 100644 --- a/src/main.rs +++ b/src/main.rs @@ -564,7 +564,7 @@ fn main() { } let metadata: std::fs::Metadata = - std::fs::metadata(&path).expect("reading file metadata"); + std::fs::metadata(path).expect("reading file metadata"); let file_size: u64 = metadata.len(); diff --git a/src/put/files.rs b/src/put/files.rs index e07da70..2186cbf 100644 --- a/src/put/files.rs +++ b/src/put/files.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::Path; use std::process::{Command as ProcessCommand, Stdio}; use std::{fmt, fs}; @@ -337,7 +337,7 @@ pub fn download( pub fn upload( api_token: &String, - path: &PathBuf, + path: &Path, parent_id: Option<&String>, curl_args: &Vec, ) { diff --git a/src/put/tus.rs b/src/put/tus.rs index 98ae74a..3798ecb 100644 --- a/src/put/tus.rs +++ b/src/put/tus.rs @@ -22,7 +22,7 @@ pub fn upload(client: &Client, api_token: &String, path: &PathBuf, parent_id: Op .to_string_lossy() .to_string(); - let metadata: std::fs::Metadata = std::fs::metadata(&path).expect("reading file metadata"); + let metadata: std::fs::Metadata = std::fs::metadata(path).expect("reading file metadata"); let file_size: u64 = metadata.len(); let last_modified: std::time::SystemTime = metadata.modified().expect("reading last modified"); @@ -33,7 +33,7 @@ pub fn upload(client: &Client, api_token: &String, path: &PathBuf, parent_id: Op println!("Uploading: {}", file_name); - let file: std::fs::File = std::fs::File::open(&path).expect("opening file"); + let file: std::fs::File = std::fs::File::open(path).expect("opening file"); let location: String; @@ -87,7 +87,7 @@ pub fn upload(client: &Client, api_token: &String, path: &PathBuf, parent_id: Op total_bytes_read += bytes_read as u64; // Skip to the initial offset, then start uploading the file from the containing chunk - if total_bytes_read < resume_offset as u64 { + if total_bytes_read < resume_offset { continue; } @@ -133,8 +133,6 @@ pub fn upload(client: &Client, api_token: &String, path: &PathBuf, parent_id: Op let percentage_completed: f64 = (total_bytes_read as f64 / file_size as f64) * 100.0; println!("{:.0}% ({:.2} MB/s)", percentage_completed, upload_speed); } - - return; } fn get_offset(client: &Client, api_token: &String, location: String) -> u64 { @@ -156,7 +154,7 @@ fn get_offset(client: &Client, api_token: &String, location: String) -> u64 { } } - return 0; + 0 } /// Creates a new upload using TUS, returns the location @@ -193,5 +191,5 @@ fn create_upload( } } - return None; + None } From e16c57c07cdc6a0d06db59ad180bcac7ad4f30a8 Mon Sep 17 00:00:00 2001 From: David Chalifoux Date: Thu, 7 Nov 2024 22:56:16 -0500 Subject: [PATCH 3/4] cleanup --- src/put/files.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/put/files.rs b/src/put/files.rs index 2186cbf..af80d80 100644 --- a/src/put/files.rs +++ b/src/put/files.rs @@ -335,16 +335,11 @@ pub fn download( Ok(()) } -pub fn upload( - api_token: &String, - path: &Path, - parent_id: Option<&String>, - curl_args: &Vec, -) { +pub fn upload(api_token: &String, path: &Path, parent_id: Option<&String>, curl_args: &[String]) { println!("Uploading: {}\n", path.to_string_lossy()); ProcessCommand::new("curl") - .args(curl_args.clone()) + .args(curl_args) .arg("-H") .arg(format!("Authorization: Bearer {}", api_token)) .arg("-F") From fd3d469deb41625fc49f34d907c1158fe2e28cb1 Mon Sep 17 00:00:00 2001 From: David Chalifoux Date: Thu, 7 Nov 2024 23:04:52 -0500 Subject: [PATCH 4/4] fix: Update release-please --- .github/workflows/create-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 1f32622..895ef83 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -13,7 +13,7 @@ jobs: create-release: runs-on: ubuntu-latest steps: - - uses: google-github-actions/release-please-action@v4 + - uses: googleapis/release-please-action@v4 with: token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} release-type: rust