diff --git a/pkgs/intl4x/example_native/.gitignore b/pkgs/intl4x/example_native/.gitignore new file mode 100644 index 00000000..3be127c5 --- /dev/null +++ b/pkgs/intl4x/example_native/.gitignore @@ -0,0 +1,4 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ +/bin/example_native/ diff --git a/pkgs/intl4x/example_native/README.md b/pkgs/intl4x/example_native/README.md new file mode 100644 index 00000000..3816eca3 --- /dev/null +++ b/pkgs/intl4x/example_native/README.md @@ -0,0 +1,2 @@ +A sample command-line application with an entrypoint in `bin/`, library code +in `lib/`, and example unit test in `test/`. diff --git a/pkgs/intl4x/example_native/analysis_options.yaml b/pkgs/intl4x/example_native/analysis_options.yaml new file mode 100644 index 00000000..dee8927a --- /dev/null +++ b/pkgs/intl4x/example_native/analysis_options.yaml @@ -0,0 +1,30 @@ +# This file configures the static analysis results for your project (errors, +# warnings, and lints). +# +# This enables the 'recommended' set of lints from `package:lints`. +# This set helps identify many issues that may lead to problems when running +# or consuming Dart code, and enforces writing Dart using a single, idiomatic +# style and format. +# +# If you want a smaller set of lints you can change this to specify +# 'package:lints/core.yaml'. These are just the most critical lints +# (the recommended set includes the core lints). +# The core lints are also what is used by pub.dev for scoring packages. + +include: package:lints/recommended.yaml + +# Uncomment the following section to specify additional rules. + +# linter: +# rules: +# - camel_case_types + +# analyzer: +# exclude: +# - path/to/excluded/files/** + +# For more information about the core and recommended set of lints, see +# https://dart.dev/go/core-lints + +# For additional information about configuring this file, see +# https://dart.dev/guides/language/analysis-options diff --git a/pkgs/intl4x/example_native/bin/example_native.dart b/pkgs/intl4x/example_native/bin/example_native.dart new file mode 100644 index 00000000..72e1dd04 --- /dev/null +++ b/pkgs/intl4x/example_native/bin/example_native.dart @@ -0,0 +1,6 @@ +import 'package:intl4x/intl4x.dart'; + +void main(List arguments) { + final intl = Intl(); + print('collation: ${intl.collation().compare('a', 'b')}!'); +} diff --git a/pkgs/intl4x/example_native/pubspec.yaml b/pkgs/intl4x/example_native/pubspec.yaml new file mode 100644 index 00000000..00fc43be --- /dev/null +++ b/pkgs/intl4x/example_native/pubspec.yaml @@ -0,0 +1,17 @@ +name: example_native +description: A sample command-line application. +version: 1.0.0 +# repository: https://github.com/my_org/my_repo + +publish_to: none + +environment: + sdk: ^3.6.0-78.0.dev + +dependencies: + intl4x: + path: ../ + +dev_dependencies: + lints: ^4.0.0 + test: ^1.24.0 diff --git a/pkgs/intl4x/example/.gitignore b/pkgs/intl4x/example_web/.gitignore similarity index 100% rename from pkgs/intl4x/example/.gitignore rename to pkgs/intl4x/example_web/.gitignore diff --git a/pkgs/intl4x/example/README.md b/pkgs/intl4x/example_web/README.md similarity index 100% rename from pkgs/intl4x/example/README.md rename to pkgs/intl4x/example_web/README.md diff --git a/pkgs/intl4x/example/analysis_options.yaml b/pkgs/intl4x/example_web/analysis_options.yaml similarity index 100% rename from pkgs/intl4x/example/analysis_options.yaml rename to pkgs/intl4x/example_web/analysis_options.yaml diff --git a/pkgs/intl4x/example/pubspec.yaml b/pkgs/intl4x/example_web/pubspec.yaml similarity index 100% rename from pkgs/intl4x/example/pubspec.yaml rename to pkgs/intl4x/example_web/pubspec.yaml diff --git a/pkgs/intl4x/example/web/index.html b/pkgs/intl4x/example_web/web/index.html similarity index 100% rename from pkgs/intl4x/example/web/index.html rename to pkgs/intl4x/example_web/web/index.html diff --git a/pkgs/intl4x/example/web/main.dart b/pkgs/intl4x/example_web/web/main.dart similarity index 100% rename from pkgs/intl4x/example/web/main.dart rename to pkgs/intl4x/example_web/web/main.dart diff --git a/pkgs/intl4x/example/web/styles.css b/pkgs/intl4x/example_web/web/styles.css similarity index 100% rename from pkgs/intl4x/example/web/styles.css rename to pkgs/intl4x/example_web/web/styles.css diff --git a/pkgs/intl4x/hook/build.dart b/pkgs/intl4x/hook/build.dart index 9c9832d4..e43d949b 100644 --- a/pkgs/intl4x/hook/build.dart +++ b/pkgs/intl4x/hook/build.dart @@ -6,13 +6,12 @@ import 'dart:io'; import 'package:crypto/crypto.dart' show sha256; import 'package:intl4x/src/hook_helpers/hashes.dart'; +import 'package:intl4x/src/hook_helpers/shared.dart'; import 'package:intl4x/src/hook_helpers/version.dart'; import 'package:native_assets_cli/native_assets_cli.dart'; import 'package:path/path.dart' as path; const crateName = 'icu_capi'; -const package = 'intl4x'; -const assetId = 'src/bindings/lib.g.dart'; final env = 'ICU4X_BUILD_MODE'; @@ -34,26 +33,12 @@ Unknown build mode for icu4x. Set the `ICU4X_BUILD_MODE` environment variable wi '''), }; - final builtLibrary = await buildMode.build(); + final buildResult = await buildMode.build(); // For debugging purposes // ignore: deprecated_member_use output.addMetadatum(env, environmentBuildMode ?? 'fetch'); - - output.addAsset(NativeCodeAsset( - package: package, - name: assetId, - linkMode: DynamicLoadingBundled(), - architecture: config.targetArchitecture, - os: config.targetOS, - file: builtLibrary, - )); - - output.addDependencies( - [ - ...buildMode.dependencies, - config.packageRoot.resolve('hook/build.dart'), - ], - ); + buildResult.addAssets(config, output); + output.addDependencies(buildMode.dependencies); }); } @@ -64,26 +49,74 @@ sealed class BuildMode { List get dependencies; - Future build(); + Future build(); +} + +final class BuildResult { + final Uri library; + final Uri? datagen; + final Uri? postcard; + + BuildResult({ + required this.library, + required this.datagen, + required this.postcard, + }); + + void addAssets(BuildConfig config, BuildOutput output) { + output.addAssets( + [ + NativeCodeAsset( + package: package, + name: assetId, + linkMode: DynamicLoadingBundled(), + architecture: config.targetArchitecture, + os: config.targetOS, + file: library, + ), + if (datagen != null) + DataAsset( + package: package, + name: 'datagen', + file: datagen!, + ), + if (postcard != null) + DataAsset( + package: package, + name: 'postcard', + file: postcard!, + ), + ], + linkInPackage: config.linkingEnabled ? config.packageName : null, + ); + } } final class FetchMode extends BuildMode { FetchMode(super.config); + final httpClient = HttpClient(); @override - Future build() async { + Future build() async { final target = '${config.targetOS}_${config.targetArchitecture}'; - final uri = Uri.parse( + final dylibRemoteUri = Uri.parse( 'https://github.com/dart-lang/i18n/releases/download/$version/$target'); - final request = await HttpClient().getUrl(uri); - final response = await request.close(); - if (response.statusCode != 200) { - throw ArgumentError('The request to $uri failed'); - } - final dynamicLibrary = File.fromUri( - config.outputDirectory.resolve(config.targetOS.dylibFileName('icu4x'))); - await dynamicLibrary.create(); - await response.pipe(dynamicLibrary.openWrite()); + final dynamicLibrary = await fetchToFile( + dylibRemoteUri, + config.outputDirectory.resolve(config.filename('icu4x')), + ); + + final datagen = await fetchToFile( + Uri.parse( + 'https://github.com/dart-lang/i18n/releases/download/$version/$target-datagen'), + config.outputDirectory.resolve('datagen'), + ); + + final postcard = await fetchToFile( + Uri.parse( + 'https://github.com/dart-lang/i18n/releases/download/$version/full.postcard'), + config.outputDirectory.resolve('full.postcard'), + ); final bytes = await dynamicLibrary.readAsBytes(); final fileHash = sha256.convert(bytes).toString(); @@ -92,13 +125,29 @@ final class FetchMode extends BuildMode { config.targetArchitecture, )]; if (fileHash == expectedFileHash) { - return dynamicLibrary.uri; + return BuildResult( + library: dynamicLibrary.uri, + datagen: datagen.uri, + postcard: postcard.uri, + ); } else { throw Exception( - 'The pre-built binary for the target $target at $uri has a hash of ' - '$fileHash, which does not match $expectedFileHash fixed in the ' - 'build hook of package:intl4x.'); + 'The pre-built binary for the target $target at $dylibRemoteUri has a' + ' hash of $fileHash, which does not match $expectedFileHash fixed in' + ' the build hook of package:intl4x.'); + } + } + + Future fetchToFile(Uri uri, Uri fileUri) async { + final request = await httpClient.getUrl(uri); + final response = await request.close(); + if (response.statusCode != 200) { + throw ArgumentError('The request to $uri failed'); } + final file = File.fromUri(fileUri); + await file.create(); + await response.pipe(file.openWrite()); + return file; } @override @@ -106,33 +155,60 @@ final class FetchMode extends BuildMode { } final class LocalMode extends BuildMode { - LocalMode(super.config); - - String get _localBinaryPath { - final localPath = Platform.environment['LOCAL_ICU4X_BINARY']; - if (localPath != null) { + final String localLibraryPath; + final String? localDatagenPath; + final String? localPostcardPath; + + LocalMode(super.config) + : localLibraryPath = _getFromEnvironment( + 'LOCAL_ICU4X_BINARY_${config.linkingEnabled ? 'STATIC' : 'DYNAMIC'}', + true, + )!, + localDatagenPath = _getFromEnvironment('LOCAL_ICU4X_DATAGEN', false), + localPostcardPath = _getFromEnvironment('LOCAL_ICU4X_POSTCARD', false); + + static String? _getFromEnvironment(String key, bool mustExist) { + final localPath = Platform.environment[key]; + if (localPath != null || !mustExist) { return localPath; } - throw ArgumentError('`LOCAL_ICU4X_BINARY` is empty. ' + throw ArgumentError('`$key` is empty. ' 'If the `ICU4X_BUILD_MODE` is set to `local`, the ' - '`LOCAL_ICU4X_BINARY` environment variable must contain the path to ' - 'the binary.'); + '`$key` environment variable must be set.'); } @override - Future build() async { - final dylibFileName = config.targetOS.dylibFileName('icu4x'); - final dylibFileUri = config.outputDirectory.resolve(dylibFileName); - final file = File(_localBinaryPath); - if (!(await file.exists())) { - throw FileSystemException('Could not find binary.', _localBinaryPath); + Future build() async { + final libFileUri = config.outputDirectory.resolve(config.filename('icu4x')); + await copyFile(localLibraryPath, libFileUri); + + final Uri? datagenFileUri; + if (localDatagenPath != null) { + datagenFileUri = config.outputDirectory.resolve('datagen'); + await copyFile(localDatagenPath!, datagenFileUri); + } else { + datagenFileUri = null; + } + + final Uri? postcardFileUri; + if (localPostcardPath != null) { + postcardFileUri = config.outputDirectory.resolve('postcard'); + await copyFile(localPostcardPath!, postcardFileUri); + } else { + postcardFileUri = null; } - await file.copy(dylibFileUri.toFilePath(windows: Platform.isWindows)); - return dylibFileUri; + + return BuildResult( + library: libFileUri, + datagen: datagenFileUri, + postcard: postcardFileUri, + ); } @override - List get dependencies => [Uri.file(_localBinaryPath)]; + List get dependencies => [ + Uri.file(localLibraryPath), + ]; } final class CheckoutMode extends BuildMode { @@ -141,7 +217,7 @@ final class CheckoutMode extends BuildMode { String? get workingDirectory => Platform.environment['LOCAL_ICU4X_CHECKOUT']; @override - Future build() async { + Future build() async { if (workingDirectory == null) { throw ArgumentError('Specify the ICU4X checkout folder' 'with the LOCAL_ICU4X_CHECKOUT variable'); @@ -155,10 +231,14 @@ final class CheckoutMode extends BuildMode { ]; } -Future buildLib(BuildConfig config, String workingDirectory) async { - final dylibFileName = - config.targetOS.dylibFileName(crateName.replaceAll('-', '_')); - final dylibFileUri = config.outputDirectory.resolve(dylibFileName); +Future buildLib( + BuildConfig config, String workingDirectory) async { + final crateNameFixed = crateName.replaceAll('-', '_'); + final libFileName = config.filename(crateNameFixed); + + final libFileUri = config.outputDirectory.resolve(libFileName); + final datagenFileUri = config.outputDirectory.resolve('datagen'); + final postcardFileUri = config.outputDirectory.resolve('postcard'); if (!config.dryRun) { final rustTarget = _asRustTarget( config.targetOS, @@ -204,9 +284,7 @@ Future buildLib(BuildConfig config, String workingDirectory) async { 'panic-handler', 'experimental_components', ]; - final linkModeType = config.linkModePreference == LinkModePreference.static - ? 'staticlib' - : 'cdylib'; + final linkModeType = config.buildStatic ? 'staticlib' : 'cdylib'; final arguments = [ if (isNoStd) '+nightly', 'rustc', @@ -242,15 +320,58 @@ Future buildLib(BuildConfig config, String workingDirectory) async { tempDir.path, rustTarget, 'release', - dylibFileName, + libFileName, ); - final file = File(builtPath); - if (!(await file.exists())) { - throw FileSystemException('Building the dylib failed', builtPath); + await copyFile(builtPath, libFileUri); + + if (config.linkingEnabled) { + final postcardPath = path.join(tempDir.path, 'full.postcard'); + await Process.run( + 'cargo', + [ + 'run', + ...['-p', 'icu_datagen'], + '--', + ...['--locales', 'full'], + ...['--keys', 'all'], + ...['--format', 'blob'], + ...['--out', postcardPath], + ], + workingDirectory: workingDirectory, + ); + await copyFile(postcardPath, postcardFileUri); + + final datagenPath = path.join(tempDir.path, 'datagen'); + final datagenDirectory = path.join(workingDirectory, 'provider/datagen'); + await Process.run( + 'rustup', + ['target', 'add', 'aarch64-unknown-linux-gnu'], + workingDirectory: datagenDirectory, + ); + await Process.run( + 'cargo', + [ + 'build', + '--release', + '--bin', + 'icu4x-datagen', + '--no-default-features', + ...[ + '--features', + 'bin,blob_exporter,blob_input,rayon,experimental_components' + ], + ...['--target', 'aarch64-unknown-linux-gnu'] + ], + workingDirectory: datagenDirectory, + ); + await copyFile(datagenPath, datagenFileUri); } - await file.copy(dylibFileUri.toFilePath(windows: Platform.isWindows)); } - return dylibFileUri; + return BuildResult( + library: libFileUri, + datagen: config.linkingEnabled ? datagenFileUri : null, + postcard: config.linkingEnabled ? postcardFileUri : null, + ); } String _asRustTarget(OS os, Architecture? architecture, bool isSimulator) { @@ -287,3 +408,18 @@ bool _isNoStdTarget((OS os, Architecture? architecture) arg) => [ (OS.android, Architecture.riscv64), (OS.linux, Architecture.riscv64) ].contains(arg); + +extension on BuildConfig { + bool get buildStatic => + linkModePreference == LinkModePreference.static || linkingEnabled; + String Function(String) get filename => + buildStatic ? targetOS.staticlibFileName : targetOS.dylibFileName; +} + +Future copyFile(String path, Uri libFileUri) async { + final file = File(path); + if (!(await file.exists())) { + throw FileSystemException('File does not exist.', path); + } + await file.copy(libFileUri.toFilePath(windows: Platform.isWindows)); +} diff --git a/pkgs/intl4x/hook/link.dart b/pkgs/intl4x/hook/link.dart new file mode 100644 index 00000000..60c65858 --- /dev/null +++ b/pkgs/intl4x/hook/link.dart @@ -0,0 +1,89 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +import 'dart:io'; + +import 'package:collection/collection.dart'; +import 'package:intl4x/src/hook_helpers/locales.dart'; +import 'package:intl4x/src/hook_helpers/shared.dart'; +import 'package:logging/logging.dart'; +import 'package:native_assets_cli/native_assets_cli.dart'; +import 'package:native_toolchain_c/native_toolchain_c.dart'; + +// TODO(mosuem): Use `record_use` to automagically get the used symbols. +const usedSymbols = [ + 'diplomat_buffer_writeable_create', + 'diplomat_buffer_writeable_get_bytes', + 'diplomat_buffer_writeable_len', + 'diplomat_buffer_writeable_destroy', + 'ICU4XCollator_create_v1', + 'ICU4XCollator_compare_utf16_', + 'ICU4XDataProvider_create_compiled', + 'ICU4XDataProvider_create_empty', + 'ICU4XLocale_create_und', + 'ICU4XLocale_set_language', + 'ICU4XLocale_set_region', + 'ICU4XLocale_set_script', + 'ICU4XLocale_to_string', + 'ICU4XLocale_total_cmp_', + //additional + 'ICU4XDataProvider_create_compiled', + 'ICU4XDataProvider_destroy', + 'ICU4XLocale_destroy', + 'ICU4XCollator_destroy', +]; + +void main(List arguments) { + link( + arguments, + (config, output) async { + final staticLib = config.assets + .firstWhereOrNull((asset) => asset.id == 'package:$package/$assetId'); + if (staticLib == null) { + // No static lib built, so assume a dynamic one was already bundled. + return; + } + + final linker = CLinker.library( + name: config.packageName, + assetName: assetId, + sources: [staticLib.file!.path], + linkerOptions: LinkerOptions.treeshake(symbols: usedSymbols), + ); + + await linker.run( + config: config, + output: output, + logger: Logger('') + ..level = Level.ALL + ..onRecord.listen((record) => print(record.message)), + ); + + final postcard = + config.assets.firstWhere((asset) => asset.id.endsWith('postcard')); + final datagenTool = + config.assets.firstWhere((asset) => asset.id.endsWith('datagen')); + final dylib = output.assets.first; + + await Process.run(datagenTool.file!.toFilePath(), [ + '--locales', + (await _customLocales ?? locales).join(','), + '--input', + postcard.file!.toFilePath(), + '--keys-for-bin', + dylib.file!.toFilePath(), + ]); + }, + ); +} + +Future?> get _customLocales async { + final localPath = Platform.environment['ICU4X_LOCALE_LIST']; + if (localPath != null) { + final file = File(localPath); + if (await file.exists()) { + return file.readAsLines(); + } + } + return null; +} diff --git a/pkgs/intl4x/lib/src/hook_helpers/locales.dart b/pkgs/intl4x/lib/src/hook_helpers/locales.dart new file mode 100644 index 00000000..fe46aafd --- /dev/null +++ b/pkgs/intl4x/lib/src/hook_helpers/locales.dart @@ -0,0 +1,5 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +final locales = ['en_US']; diff --git a/pkgs/intl4x/lib/src/hook_helpers/shared.dart b/pkgs/intl4x/lib/src/hook_helpers/shared.dart new file mode 100644 index 00000000..6b00af73 --- /dev/null +++ b/pkgs/intl4x/lib/src/hook_helpers/shared.dart @@ -0,0 +1,6 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +const package = 'intl4x'; +const assetId = 'src/bindings/lib.g.dart'; diff --git a/pkgs/intl4x/lib/src/locale/locale_native.dart b/pkgs/intl4x/lib/src/locale/locale_native.dart index 377b67e7..589dcbc1 100644 --- a/pkgs/intl4x/lib/src/locale/locale_native.dart +++ b/pkgs/intl4x/lib/src/locale/locale_native.dart @@ -7,7 +7,7 @@ import 'locale.dart'; /// This file should be replaced by references to ICU4X when ready. Locale parseLocaleWithSeparatorPlaceholder(String s, [String separator = '-']) { - final parsed = s.split(separator); + final parsed = s.split('.').first.split(separator); // ignore: unused_local_variable final subtags = parsed.skipWhile((value) => value != 'u').toList(); final tags = parsed.takeWhile((value) => value != 'u').toList(); diff --git a/pkgs/intl4x/pubspec.yaml b/pkgs/intl4x/pubspec.yaml index 833e3954..ecc0392c 100644 --- a/pkgs/intl4x/pubspec.yaml +++ b/pkgs/intl4x/pubspec.yaml @@ -20,8 +20,22 @@ dependencies: crypto: ^3.0.3 ffi: ^2.1.0 js: ^0.7.1 + logging: ^1.2.0 meta: ^1.12.0 - native_assets_cli: ^0.7.2 + native_assets_cli: + # path: ../../../native/pkgs/native_assets_cli + git: + url: https://github.com/dart-lang/native.git + path: pkgs/native_assets_cli + ref: fixLinkerSomeMore + + native_toolchain_c: + # path: ../../../native/pkgs/native_toolchain_c + git: + url: https://github.com/dart-lang/native.git + path: pkgs/native_toolchain_c + ref: fixLinkerSomeMore + path: ^1.9.0 dev_dependencies: