From a4e399f6ae3267ab6bebdc14985bd96ca213162c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Fri, 8 Jul 2022 11:23:03 +0200 Subject: [PATCH 1/3] Start refactoring benchmark builder for cross compilation --- benchmarks/tool/compile_benchmarks.dart | 112 +++++++++++++++++------- 1 file changed, 81 insertions(+), 31 deletions(-) diff --git a/benchmarks/tool/compile_benchmarks.dart b/benchmarks/tool/compile_benchmarks.dart index 92fee98b8..3650e866e 100755 --- a/benchmarks/tool/compile_benchmarks.dart +++ b/benchmarks/tool/compile_benchmarks.dart @@ -9,8 +9,9 @@ import 'package:pool/pool.dart' show Pool; Future main(List args) async { final argParser = ArgParser() ..addOption('target', - mandatory: false, defaultsTo: 'aot,exe,jit,js,js-production') - ..addOption('jobs', abbr: 'j', mandatory: false); + mandatory: false, defaultsTo: 'exe,jit,js,js-production') + ..addOption('jobs', abbr: 'j', mandatory: false) + ..addOption('aot-target', mandatory: false, defaultsTo: 'x64'); final parsedArgs = argParser.parse(args); @@ -23,7 +24,22 @@ Future main(List args) async { for (final targetStr in parsedArgs['target'].split(',')) { switch (targetStr) { case 'aot': - targets.add(aotTarget); + if (!bool.hasEnvironment("DART_SDK")) { + print('\$DART_SDK needs to be set when generating aot snapshots'); + exit(1); + } + final dartSdkPath = String.fromEnvironment("DART_SDK"); + + final parsedAotTarget = parsedArgs['aot-target']; + final aotTarget = aotTargets[parsedAotTarget]; + if (aotTarget == null) { + print('Unsupported aot target: $parsedAotTarget'); + print( + 'Supported aot targets: ${aotTargets.keys.toList().join(', ')}'); + exit(1); + } + + targets.add(makeAotTarget(dartSdkPath, aotTarget)); break; case 'exe': @@ -60,7 +76,7 @@ Future main(List args) async { .toList(); } - final commands = >[]; + final commands = []; if (sourceFiles.isNotEmpty && targets.isNotEmpty) { try { @@ -79,11 +95,21 @@ Future main(List args) async { final pool = Pool(jobs); - final stream = pool.forEach, CompileProcess>(commands, - (List command) async { - final commandStr = command.join(' '); + final stream = pool.forEach(commands, + (ProcessInstructions command) async { + var envStr = ''; + if (command.environment != null) { + envStr = command.environment!.entries + .map((entry) => '${entry.key}=${entry.value}') + .join(' ') + + ' '; + } + final commandStr = + '$envStr${command.executable} ${command.arguments.join(' ')}'; print(commandStr); - final result = await Process.run(command[0], command.sublist(1)); + + final result = await Process.run(command.executable, command.arguments, + environment: command.environment); return CompileProcess(commandStr, result); }); @@ -107,6 +133,26 @@ Future main(List args) async { await pool.done; } +class ProcessInstructions { + final String executable; + final List arguments; + final Map? environment; + + ProcessInstructions(this.executable, this.arguments, {this.environment}); +} + +enum AotTarget { + x64, + armv7hf, + armv8, +} + +const aotTargets = { + 'x64': AotTarget.x64, + 'armv7hf': AotTarget.armv7hf, + 'armv8': AotTarget.armv8, +}; + class CompileProcess { final String command; final ProcessResult result; @@ -116,7 +162,7 @@ class CompileProcess { class Target { final String _name; - final List Function(String) _processArgs; + final ProcessInstructions Function(String sourceFile) _processArgs; const Target(this._name, this._processArgs); @@ -125,63 +171,67 @@ class Target { return 'Target($_name)'; } - List compileArgs(String sourceFile) { + ProcessInstructions compileArgs(String sourceFile) { return _processArgs(sourceFile); } } +Target makeAotTarget(String dartSdkPath, AotTarget aotTarget) { + return Target('aot', (sourceFile) { + final baseName = path.basename(sourceFile); + final baseNameNoExt = path.withoutExtension(baseName); + // TODO: Do we need `-Ddart.vm.product=true`? + return ProcessInstructions('$dartSdkPath/pkg/vm/tool/precompiler2', + [sourceFile, 'out/$baseNameNoExt.aot']); + }); +} + const aotTarget = Target('aot', aotProcessArgs); const exeTarget = Target('exe', exeProcessArgs); const jitTarget = Target('jit', jitProcessArgs); const jsTarget = Target('js', jsProcessArgs); const jsProductionTarget = Target('js-production', jsProductionProcessArgs); -List aotProcessArgs(String sourceFile) { +ProcessInstructions aotProcessArgs(String sourceFile) { final baseName = path.basename(sourceFile); final baseNameNoExt = path.withoutExtension(baseName); - return [ - 'dart', - 'compile', - 'aot-snapshot', - sourceFile, - '-o', - 'out/$baseNameNoExt.aot' - ]; + return ProcessInstructions('dart', + ['compile', 'aot-snapshot', sourceFile, '-o', 'out/$baseNameNoExt.aot']); } -List exeProcessArgs(String sourceFile) { +ProcessInstructions exeProcessArgs(String sourceFile) { final baseName = path.basename(sourceFile); final baseNameNoExt = path.withoutExtension(baseName); - return ['dart', 'compile', 'exe', sourceFile, '-o', 'out/$baseNameNoExt.exe']; + return ProcessInstructions( + 'dart', ['compile', 'exe', sourceFile, '-o', 'out/$baseNameNoExt.exe']); } -List jitProcessArgs(String sourceFile) { +ProcessInstructions jitProcessArgs(String sourceFile) { final baseName = path.basename(sourceFile); final baseNameNoExt = path.withoutExtension(baseName); - return [ - 'dart', + return ProcessInstructions('dart', [ '--snapshot-kind=kernel', '--snapshot=out/$baseNameNoExt.dill', sourceFile - ]; + ]); } -List jsProcessArgs(String sourceFile) { +ProcessInstructions jsProcessArgs(String sourceFile) { final baseName = path.basename(sourceFile); final baseNameNoExt = path.withoutExtension(baseName); - return ['dart', 'compile', 'js', sourceFile, '-o', 'out/$baseNameNoExt.js']; + return ProcessInstructions( + 'dart', ['compile', 'js', sourceFile, '-o', 'out/$baseNameNoExt.js']); } -List jsProductionProcessArgs(String sourceFile) { +ProcessInstructions jsProductionProcessArgs(String sourceFile) { final baseName = path.basename(sourceFile); final baseNameNoExt = path.withoutExtension(baseName); - return [ - 'dart', + return ProcessInstructions('dart', [ 'compile', 'js', sourceFile, '-O4', '-o', 'out/$baseNameNoExt.production.js' - ]; + ]); } From 1461c9e11005f64e578af2693b3cfd9bbe9e5615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Fri, 8 Jul 2022 11:27:03 +0200 Subject: [PATCH 2/3] Add some docs --- benchmarks/tool/compile_benchmarks.dart | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/benchmarks/tool/compile_benchmarks.dart b/benchmarks/tool/compile_benchmarks.dart index 3650e866e..0592fc8db 100755 --- a/benchmarks/tool/compile_benchmarks.dart +++ b/benchmarks/tool/compile_benchmarks.dart @@ -133,6 +133,7 @@ Future main(List args) async { await pool.done; } +/// Stores [Process.run] arguments class ProcessInstructions { final String executable; final List arguments; @@ -141,18 +142,22 @@ class ProcessInstructions { ProcessInstructions(this.executable, this.arguments, {this.environment}); } +/// Supported aot snapshot targets enum AotTarget { x64, armv7hf, armv8, } +/// Mapping of `--aot-target` arguments to their [AotTarget]s const aotTargets = { 'x64': AotTarget.x64, 'armv7hf': AotTarget.armv7hf, 'armv8': AotTarget.armv8, }; +/// Packs a debug string for a command being run to the command's result, to be +/// able to show which command failed and why class CompileProcess { final String command; final ProcessResult result; From 4ab1fa78d3d2686eefc18c76cfc13d284a6c3308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Fri, 8 Jul 2022 11:38:43 +0200 Subject: [PATCH 3/3] Pass DART_CONFIGURATION to precompiler2 --- benchmarks/tool/compile_benchmarks.dart | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/benchmarks/tool/compile_benchmarks.dart b/benchmarks/tool/compile_benchmarks.dart index 0592fc8db..2c7c11965 100755 --- a/benchmarks/tool/compile_benchmarks.dart +++ b/benchmarks/tool/compile_benchmarks.dart @@ -14,6 +14,7 @@ Future main(List args) async { ..addOption('aot-target', mandatory: false, defaultsTo: 'x64'); final parsedArgs = argParser.parse(args); + final env = Platform.environment; var jobs = Platform.numberOfProcessors; if (parsedArgs['jobs'] != null) { @@ -24,11 +25,12 @@ Future main(List args) async { for (final targetStr in parsedArgs['target'].split(',')) { switch (targetStr) { case 'aot': - if (!bool.hasEnvironment("DART_SDK")) { + final dartSdkPath = env["DART_SDK"]; + + if (dartSdkPath == null) { print('\$DART_SDK needs to be set when generating aot snapshots'); exit(1); } - final dartSdkPath = String.fromEnvironment("DART_SDK"); final parsedAotTarget = parsedArgs['aot-target']; final aotTarget = aotTargets[parsedAotTarget]; @@ -149,13 +151,21 @@ enum AotTarget { armv8, } -/// Mapping of `--aot-target` arguments to their [AotTarget]s +/// Maps `--aot-target` arguments to their [AotTarget]s const aotTargets = { 'x64': AotTarget.x64, 'armv7hf': AotTarget.armv7hf, 'armv8': AotTarget.armv8, }; +/// Maps [AotTarget]s to `DART_CONFIGURATION` env values when invoking `precompiler2` +// TODO: Product or release? +const aotTargetDartConfiguration = { + AotTarget.x64: "ProductX64", + AotTarget.armv7hf: "ProductXARM/clang_x86", + AotTarget.armv8: "ProductXARM64/clang_x64", +}; + /// Packs a debug string for a command being run to the command's result, to be /// able to show which command failed and why class CompileProcess { @@ -186,8 +196,12 @@ Target makeAotTarget(String dartSdkPath, AotTarget aotTarget) { final baseName = path.basename(sourceFile); final baseNameNoExt = path.withoutExtension(baseName); // TODO: Do we need `-Ddart.vm.product=true`? - return ProcessInstructions('$dartSdkPath/pkg/vm/tool/precompiler2', - [sourceFile, 'out/$baseNameNoExt.aot']); + return ProcessInstructions('$dartSdkPath/pkg/vm/tool/precompiler2', [ + sourceFile, + 'out/$baseNameNoExt.aot' + ], environment: { + 'DART_CONFIGURATION': aotTargetDartConfiguration[aotTarget]! + }); }); }