Skip to content

Commit

Permalink
Create BenchmarkResult, rename to BenchmarkRun
Browse files Browse the repository at this point in the history
  • Loading branch information
gmitch215 committed Sep 14, 2024
1 parent 25e8e1c commit b42623b
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 31 deletions.
48 changes: 18 additions & 30 deletions src/main/kotlin/xyz/gmitch215/benchmarks/measurement/benchmark.kt
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import kotlinx.serialization.Serializable
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.*
import xyz.gmitch215.benchmarks.BenchmarkResult
import xyz.gmitch215.benchmarks.Measurement
import java.io.File

Expand All @@ -27,7 +28,7 @@ suspend fun main(args: Array<String>) = coroutineScope {
val output = File(input, "output")

val file = File(input, "config.yml").readText(Charsets.UTF_8)
val benchmarks = Yaml.default.decodeFromString<List<Benchmark>>(file)
val benchmarkRuns = Yaml.default.decodeFromString<List<BenchmarkRun>>(file)

println("Starting Benchmarks on $os")

Expand All @@ -36,7 +37,7 @@ suspend fun main(args: Array<String>) = coroutineScope {
if (folders.isEmpty())
error("No benchmarks found")

for (benchmark in benchmarks) {
for (benchmark in benchmarkRuns) {
val out = File(output, benchmark.id)
if (!out.exists())
out.mkdirs()
Expand All @@ -48,33 +49,33 @@ suspend fun main(args: Array<String>) = coroutineScope {
}
}

suspend fun runBenchmark(benchmark: Benchmark, folder: File, out: File) = withContext(Dispatchers.IO) {
suspend fun runBenchmark(benchmarkRun: BenchmarkRun, folder: File, out: File) = withContext(Dispatchers.IO) {
val configFile = File(folder, "config.yml").readText(Charsets.UTF_8)
val config = Yaml.default.decodeFromString<BenchmarkConfiguration>(configFile)

val results = mutableListOf<Double>()

launch(Dispatchers.Default) {
var compile = benchmark.compile
var compile = benchmarkRun.compile
val s = File.separator

if (compile != null) {
if (benchmark.location != null) {
val home = System.getenv(benchmark.location)
if (benchmarkRun.location != null) {
val home = System.getenv(benchmarkRun.location)
if (home != null)
compile = "${home}${s}bin${s}$compile"
}

if (os == "windows") {
val executableSuffix = if (benchmark.id.contains("kotlin")) ".bat" else ".exe"
val executableSuffix = if (benchmarkRun.id.contains("kotlin")) ".bat" else ".exe"
compile = compile.replaceFirst(" ", "$executableSuffix ")
}

compile.runCommand(folder)
}

var run = benchmark.run
if (benchmark.id == "kotlin-native")
var run = benchmarkRun.run
if (benchmarkRun.id == "kotlin-native")
run = "$folder${s}$run$kotlinNativeSuffix"

for (i in 0 until RUN_COUNT)
Expand All @@ -86,33 +87,20 @@ suspend fun runBenchmark(benchmark: Benchmark, folder: File, out: File) = withCo
}
}.join()

if (benchmark.cleanup != null)
for (cleanup in benchmark.cleanup) {
if (benchmarkRun.cleanup != null)
for (cleanup in benchmarkRun.cleanup) {
val file = File(folder, cleanup)
if (file.exists())
file.delete()
}

if (results.isEmpty())
error("No results found for '${config.name}' on ${benchmark.language}")

val low = results.min()
val high = results.max()
val avg = results.average()

val data = buildJsonObject {
put("name", config.name)
put("description", config.description)
put("measure", config.measure.name)
put("output", config.output.name)
put("low", config.measure.formatOutput(low, config.output))
put("high", config.measure.formatOutput(high, config.output))
put("avg", config.measure.formatOutput(avg, config.output))

put("results", JsonArray(results.map { JsonPrimitive(it) }))
}
error("No results found for '${config.name}' on ${benchmarkRun.language}")

val id = folder.name
val data = BenchmarkResult(id, config, results)

val file = File(out, "${folder.name}.json")
val file = File(out, "$id.json")
println("Writing to ${file.absolutePath}")

if (!file.exists())
Expand Down Expand Up @@ -153,7 +141,7 @@ data class BenchmarkConfiguration(
)

@Serializable
data class Benchmark(
data class BenchmarkRun(
val language: String,
val id: String,
val location: String? = null,
Expand Down
31 changes: 30 additions & 1 deletion src/main/kotlin/xyz/gmitch215/benchmarks/util.kt
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package xyz.gmitch215.benchmarks

import kotlinx.serialization.Serializable
import xyz.gmitch215.benchmarks.measurement.BenchmarkConfiguration
import java.util.*

@Serializable
enum class Measurement(val multiplier: Double, val unit: String) {
Expand All @@ -18,8 +20,35 @@ enum class Measurement(val multiplier: Double, val unit: String) {
;

fun formatOutput(number: Number, output: Measurement): String {
val value = number.toDouble() * multiplier / output.multiplier
val value = "%,.3f".format(Locale.US, number.toDouble() * multiplier / output.multiplier)
return "$value${output.unit}"
}

}

@Serializable
data class BenchmarkResult(
val id: String,
val name: String,
val description: String,
val measure: Measurement,
val output: Measurement,
val low: String,
val high: String,
val avg: String,
val results: List<Double>
) {

constructor(id: String, config: BenchmarkConfiguration, results: List<Double>) : this(
id,
config.name,
config.description,
config.measure,
config.output,
config.measure.formatOutput(results.min(), config.output),
config.measure.formatOutput(results.max(), config.output),
config.measure.formatOutput(results.average(), config.output),
results
)

}

0 comments on commit b42623b

Please sign in to comment.