diff --git a/pkg/pub_worker/lib/src/bin/dartdoc_wrapper.dart b/pkg/pub_worker/lib/src/bin/dartdoc_wrapper.dart index 5b59dec5b1..4b8ded894a 100644 --- a/pkg/pub_worker/lib/src/bin/dartdoc_wrapper.dart +++ b/pkg/pub_worker/lib/src/bin/dartdoc_wrapper.dart @@ -5,9 +5,9 @@ import 'dart:async'; import 'dart:convert' show json, utf8, Utf8Codec; import 'dart:io'; +import 'dart:isolate'; import 'package:_pub_shared/dartdoc/dartdoc_page.dart'; -import 'package:clock/clock.dart'; import 'package:logging/logging.dart' show Logger; import 'package:path/path.dart' as p; import 'package:stream_transform/stream_transform.dart'; @@ -22,75 +22,76 @@ Future postProcessDartdoc({ required String package, required String version, required String docDir, - required DateTime cutoffTimestamp, }) async { _log.info('Running dartdoc post-processing'); final tmpOutDir = p.join(outputFolder, '_doc'); await Directory(tmpOutDir).create(recursive: true); - final files = Directory(docDir) - .list(recursive: true, followLinks: false) - .whereType(); - await for (final file in files) { - if (cutoffTimestamp.isBefore(clock.now())) { - _log.warning( - 'Cut-off timestamp reached during dartdoc file post-processing.'); - return; - } - final suffix = file.path.substring(docDir.length + 1); - final targetFile = File(p.join(tmpOutDir, suffix)); - await targetFile.parent.create(recursive: true); - final isDartDocSidebar = - file.path.endsWith('.html') && file.path.endsWith('-sidebar.html'); - final isDartDocPage = file.path.endsWith('.html') && !isDartDocSidebar; - if (isDartDocPage) { - final page = DartDocPage.parse(await file.readAsString(encoding: _utf8)); - await targetFile.writeAsBytes(_jsonUtf8.encode(page.toJson())); - } else if (isDartDocSidebar) { - final sidebar = - DartDocSidebar.parse(await file.readAsString(encoding: _utf8)); - await targetFile.writeAsBytes(_jsonUtf8.encode(sidebar.toJson())); - } else { - await file.copy(targetFile.path); - } - } - // Move from temporary output directory to final one, ensuring that - // documentation files won't be present unless all files have been processed. - // This helps if there is a timeout along the way. - await Directory(tmpOutDir).rename(p.join(outputFolder, 'doc')); - _log.info('Finished post-processing'); - - _log.info('Creating .tar.gz archive'); - Stream _list() async* { - final originalDocDir = Directory(docDir); - final originalFiles = originalDocDir + final processingFuture = Isolate.run(() async { + final files = Directory(docDir) .list(recursive: true, followLinks: false) .whereType(); - await for (final file in originalFiles) { - if (cutoffTimestamp.isBefore(clock.now())) { - _log.warning( - 'Cut-off timestamp reached during dartdoc archive building.'); - break; + await for (final file in files) { + final suffix = file.path.substring(docDir.length + 1); + final targetFile = File(p.join(tmpOutDir, suffix)); + await targetFile.parent.create(recursive: true); + final isDartDocSidebar = + file.path.endsWith('.html') && file.path.endsWith('-sidebar.html'); + final isDartDocPage = file.path.endsWith('.html') && !isDartDocSidebar; + if (isDartDocPage) { + final page = + DartDocPage.parse(await file.readAsString(encoding: _utf8)); + await targetFile.writeAsBytes(_jsonUtf8.encode(page.toJson())); + } else if (isDartDocSidebar) { + final sidebar = + DartDocSidebar.parse(await file.readAsString(encoding: _utf8)); + await targetFile.writeAsBytes(_jsonUtf8.encode(sidebar.toJson())); + } else { + await file.copy(targetFile.path); } - // inside the archive prefix the name with /version/ - final relativePath = p.relative(file.path, from: originalDocDir.path); - final tarEntryPath = p.join(package, version, relativePath); - final data = await file.readAsBytes(); - yield TarEntry.data( - TarHeader( - name: tarEntryPath, - size: data.length, - ), - data, - ); } - } + }).whenComplete(() { + _log.info('Finished post-processing'); + }); + _log.info('Creating .tar.gz archive'); final tmpTar = File(p.join(outputFolder, '_package.tar.gz')); - await _list() - .transform(tarWriter) - .transform(gzip.encoder) - .pipe(tmpTar.openWrite()); - await tmpTar.rename(p.join(outputFolder, 'doc', 'package.tar.gz')); + final archiveFuture = Isolate.run(() async { + Stream _list() async* { + final originalDocDir = Directory(docDir); + final originalFiles = originalDocDir + .list(recursive: true, followLinks: false) + .whereType(); + await for (final file in originalFiles) { + // inside the archive prefix the name with /version/ + final relativePath = p.relative(file.path, from: originalDocDir.path); + final tarEntryPath = p.join(package, version, relativePath); + final data = await file.readAsBytes(); + yield TarEntry.data( + TarHeader( + name: tarEntryPath, + size: data.length, + ), + data, + ); + } + } + + await _list() + .transform(tarWriter) + .transform(gzip.encoder) + .pipe(tmpTar.openWrite()); + }).whenComplete(() { + _log.info('Finished .tar.gz archive'); + }); - _log.info('Finished .tar.gz archive'); + await Future.wait([ + processingFuture, + archiveFuture, + ]); + + // Move from temporary output directory to final one, ensuring that + // documentation files won't be present unless all files have been processed. + // This helps if there is a timeout along the way. + await Directory(tmpOutDir).rename(p.join(outputFolder, 'doc')); + await tmpTar.rename(p.join(outputFolder, 'doc', 'package.tar.gz')); } diff --git a/pkg/pub_worker/lib/src/bin/pana_wrapper.dart b/pkg/pub_worker/lib/src/bin/pana_wrapper.dart index 539049658f..8fd1451b68 100644 --- a/pkg/pub_worker/lib/src/bin/pana_wrapper.dart +++ b/pkg/pub_worker/lib/src/bin/pana_wrapper.dart @@ -26,8 +26,8 @@ final _log = Logger('pana'); /// replaced with a placeholder report. final _reportSizeDropThreshold = 32 * 1024; -/// Stop dartdoc if it takes more than 45 minutes. -const _dartdocTimeout = Duration(minutes: 45); +/// Stop dartdoc if it takes more than 30 minutes. +const _dartdocTimeout = Duration(minutes: 30); /// Try to fit analysis into 50 minutes. const _totalTimeout = Duration(minutes: 50); @@ -45,7 +45,7 @@ Future main(List args) async { exit(1); } - final cutoffTimestamp = clock.now().add(_totalTimeout); + final startedTimestamp = clock.now(); final outputFolder = args[0]; final package = args[1]; @@ -122,16 +122,6 @@ Future main(List args) async { logger: _log, ); - if (cutoffTimestamp.isAfter(clock.now())) { - await postProcessDartdoc( - outputFolder: outputFolder, - package: package, - version: version, - docDir: rawDartdocOutputFolder.path, - cutoffTimestamp: cutoffTimestamp, - ); - } - // sanity check on pana report size final reportSize = gzip.encode(utf8.encode(json.encode(summary.toJson()))).length; @@ -161,9 +151,23 @@ Future main(List args) async { p.join(outputFolder, 'summary.json'), ).writeAsString(json.encode(summary)); - if (cutoffTimestamp.isAfter(clock.now())) { - await rawDartdocOutputFolder.delete(recursive: true); + // Post-processing of the dartdoc-generated files seems to take at least as + // much time as running pana+dartdoc in the first place. If we don't seem to + // have enough time to complete, it is better to not start at all. + final cutoffTimestamp = startedTimestamp.add(_totalTimeout * 0.5); + if (cutoffTimestamp.isBefore(clock.now())) { + _log.warning( + 'Cut-off timestamp reached, skipping dartdoc post-processing.'); + return; } + await postProcessDartdoc( + outputFolder: outputFolder, + package: package, + version: version, + docDir: rawDartdocOutputFolder.path, + ); + + await rawDartdocOutputFolder.delete(recursive: true); } final _workerConfigDirectory = Directory('/home/worker/config');