diff --git a/clean.sh b/clean.sh index bc26e3c..32e51fc 100755 --- a/clean.sh +++ b/clean.sh @@ -59,6 +59,7 @@ rm -rf "$BUILD_DIR" echo -e "${CYAN}Cleaning generated benchmark artifacts...${NC}" BENCH_OUT_DIR="${SCRIPT_DIR}/bench_out" rm -rf $BENCH_OUT_DIR/*.o +rm -rf $BENCH_OUT_DIR/*.c* rm -rf $BENCH_OUT_DIR/bench_bin # Done! diff --git a/crates/bench/Cargo.lock b/crates/bench/Cargo.lock index dccc77f..b70e44c 100644 --- a/crates/bench/Cargo.lock +++ b/crates/bench/Cargo.lock @@ -128,6 +128,7 @@ dependencies = [ "colored", "indicatif", "miette", + "regex", "sanitize-filename", "serde", "serde_json", diff --git a/crates/bench/Cargo.toml b/crates/bench/Cargo.toml index cbaa0c9..648d61c 100644 --- a/crates/bench/Cargo.toml +++ b/crates/bench/Cargo.toml @@ -11,6 +11,7 @@ clap = { version = "4.5.4", features = ["derive"] } colored = "2.1.0" indicatif = "0.17.8" miette = { version = "7.2.0", features = ["fancy"] } +regex = "1.10.4" sanitize-filename = "0.5.0" serde = { version = "1.0.199", features = ["derive"] } serde_json = "1.0.116" diff --git a/crates/bench/src/cli.rs b/crates/bench/src/cli.rs index e2d0edb..975d0b0 100644 --- a/crates/bench/src/cli.rs +++ b/crates/bench/src/cli.rs @@ -38,6 +38,11 @@ pub struct Cli { #[arg(long, short = 'g', action)] pub run_comparative: bool, + /// Whether to run comparative tests against GnuCobol C -> clang. + /// Requires `run_comparative`. + #[arg(long, short = 'r', requires("run_comparative"), action)] + pub run_clang: bool, + /// Whether to run only a build test and not execute benchmarks. #[arg(long, short = 'g', action)] pub build_only: bool, @@ -112,6 +117,25 @@ impl TryInto for Cli { } } + // Check that `clang` is available if required. + if self.run_clang { + match Command::new("clang").output() { + Ok(_) => {} + Err(e) => { + if let ErrorKind::NotFound = e.kind() { + miette::bail!( + "Clang comparisons enabled, but `clang` was not found! Check your PATH." + ); + } else { + miette::bail!( + "An error occurred while verifying if `clang` was available: {}", + e + ); + } + } + } + } + // Verify that the output directory exists. let output_dir = if let Some(path) = self.output_dir { path @@ -223,6 +247,7 @@ impl TryInto for Cli { cobc_force_platform_linker: self.force_platform_linker, disable_hw_security: self.disable_hw_security, run_comparative: self.run_comparative, + run_clang: self.run_clang, build_only: self.build_only, output_dir, output_log, diff --git a/crates/bench/src/log.rs b/crates/bench/src/log.rs index 8a9ce5d..37ea84b 100644 --- a/crates/bench/src/log.rs +++ b/crates/bench/src/log.rs @@ -69,6 +69,10 @@ pub(crate) struct BenchmarkExecution { /// Benchmark results for `cobc`, if present. /// Only generated when executed with `--run-comparative`. pub cobc_results: Option, + + /// Benchmark results for `cobc` -> `clang`, if present. + /// Only generated when executed with `--run-clang`. + pub clang_results: Option, } /// Output for the result of a single benchmark compile/execute pair. diff --git a/crates/bench/src/runner.rs b/crates/bench/src/runner.rs index 66851f9..a57c2ba 100644 --- a/crates/bench/src/runner.rs +++ b/crates/bench/src/runner.rs @@ -7,6 +7,7 @@ use std::{ use colored::Colorize; use miette::Result; +use regex::Regex; use crate::{ bench::Benchmark, @@ -37,6 +38,9 @@ pub(crate) struct Cfg { /// Whether to run comparative tests against GnuCobol's `cobc`. pub run_comparative: bool, + /// Whether to run comparative tests against `cobc` -> `clang`. + pub run_clang: bool, + /// Whether to only build and not execute benchmarks. pub build_only: bool, @@ -72,6 +76,10 @@ pub(crate) fn run_single(cfg: &Cfg, benchmark: &Benchmark) -> Result Result Result { }) } +/// Executes a single benchmark using GnuCobol's `cobc`'s C output followed by +/// binary generation using `clang`. +fn run_clang(cfg: &Cfg, benchmark: &Benchmark) -> Result { + // Calculate output file locations for benchmarking binary, C file. + // We also need a bootstrapping file for `main` since GnuCobol doesn't create + // one for us. + let mut bench_bin_path = cfg.output_dir.clone(); + bench_bin_path.push(BENCH_BIN_NAME); + let mut bench_c_path = cfg.output_dir.clone(); + bench_c_path.push(format!("{}.c", &benchmark.name)); + let mut bootstrap_path = cfg.output_dir.clone(); + bootstrap_path.push("bootstrap.c"); + + // Set up commands for transpiling then compiling from C. + let mut cobc = Command::new("cobc"); + cobc.args(["-C", &format!("-O{}", cfg.cobc_opt_level), "-free"]) + // Required to generate `cob_init()` in the transpiled C. + .arg("-fimplicit-init") + .args(["-o", bench_c_path.to_str().unwrap()]) + .arg(&benchmark.source_file); + let mut clang = Command::new("clang"); + clang + .args(["-lcob", &format!("-O{}", cfg.cobc_opt_level)]) + .args(["-o", bench_bin_path.to_str().unwrap()]) + .arg(bootstrap_path.to_str().unwrap()); + + // We require the program ID of the COBOL file to generate the bootstrapper. + // Attempt to grep for that from the source. + let bench_prog_func = fetch_program_func(benchmark)?; + + // Generate the bootstrapping file. + let bootstrapper = format!( + " + #include \"{}.c\" + int main(void) {{ + {}(); + }} + ", + &benchmark.name, bench_prog_func + ); + std::fs::write(bootstrap_path.clone(), bootstrapper) + .map_err(|e| miette::diagnostic!("Failed to write bootstrap file for `clang`: {e}"))?; + + let before = Instant::now(); + for _ in 0..100 { + // First, transpile to C. + let out = cobc + .output() + .map_err(|e| miette::diagnostic!("Failed to execute `cobc`: {e}"))?; + if !out.status.success() { + miette::bail!( + "Failed benchmark for '{}' with `cobc` compiler error: {}", + benchmark.source_file, + String::from_utf8_lossy(&out.stderr) + ); + } + + // Finally, perform compilation & linkage with `clang`. + let out = clang + .output() + .map_err(|e| miette::diagnostic!("Failed to execute `clang`: {e}"))?; + if !out.status.success() { + miette::bail!( + "Failed benchmark for '{}' with `clang` compiler error: {}", + benchmark.source_file, + String::from_utf8_lossy(&out.stderr) + ); + } + } + let elapsed = before.elapsed(); + println!( + "clang(compile): Total time {:.2?}, average/run of {:.6?}.", + elapsed, + elapsed / 100 + ); + + // Run the target program. + let (execute_time_total, execute_time_avg) = if !cfg.build_only { + let (x, y) = run_bench_bin(cfg, benchmark)?; + (Some(x), Some(y)) + } else { + (None, None) + }; + + Ok(BenchmarkResult { + compile_time_total: elapsed, + compile_time_avg: elapsed / 100, + execute_time_total, + execute_time_avg, + }) +} + +/// Attempts to fetch the output C function name of the given benchmark COBOL file. +/// If not found, throws an error. +fn fetch_program_func(benchmark: &Benchmark) -> Result { + // First, read in source of benchmark. + let source = std::fs::read_to_string(&benchmark.source_file).map_err(|e| { + miette::diagnostic!( + "Failed to read COBOL source for benchmark '{}': {e}", + benchmark.name + ) + })?; + + // Search for a pattern matching "PROGRAM-ID ...". + let prog_id_pat = Regex::new(r"PROGRAM-ID\. [A-Z0-9a-z\-]+").unwrap(); + let prog_id_str = prog_id_pat.find(&source).ok_or(miette::diagnostic!( + "Could not find program ID in sources for benchmark '{}'.", + &benchmark.name + ))?; + + // Extract the program ID, format into final function name. + let mut prog_id = prog_id_str.as_str()["PROGRAM-ID ".len()..].to_string(); + prog_id = prog_id.replace("-", "__"); + Ok(prog_id) +} + /// Executes a single generated benchmarking binary. /// Returns the total execution time and average execution time per iteration. fn run_bench_bin(cfg: &Cfg, benchmark: &Benchmark) -> Result<(Duration, Duration)> {