Skip to content

Commit

Permalink
Refactored dartdoc post-processing. (#7553)
Browse files Browse the repository at this point in the history
  • Loading branch information
@isoos
isoos authored Mar 12, 2024
1 parent f905dbb commit e299cf6
Show file tree
Hide file tree
Showing 2 changed files with 81 additions and 76 deletions.
123 changes: 62 additions & 61 deletions pkg/pub_worker/lib/src/bin/dartdoc_wrapper.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
import 'dart:async';
import 'dart:convert' show json, utf8, Utf8Codec;
import 'dart:io';
import 'dart:isolate';

import 'package:_pub_shared/dartdoc/dartdoc_page.dart';
import 'package:clock/clock.dart';
import 'package:logging/logging.dart' show Logger;
import 'package:path/path.dart' as p;
import 'package:stream_transform/stream_transform.dart';
Expand All @@ -22,75 +22,76 @@ Future<void> postProcessDartdoc({
required String package,
required String version,
required String docDir,
required DateTime cutoffTimestamp,
}) async {
_log.info('Running dartdoc post-processing');
final tmpOutDir = p.join(outputFolder, '_doc');
await Directory(tmpOutDir).create(recursive: true);
final files = Directory(docDir)
.list(recursive: true, followLinks: false)
.whereType<File>();
await for (final file in files) {
if (cutoffTimestamp.isBefore(clock.now())) {
_log.warning(
'Cut-off timestamp reached during dartdoc file post-processing.');
return;
}
final suffix = file.path.substring(docDir.length + 1);
final targetFile = File(p.join(tmpOutDir, suffix));
await targetFile.parent.create(recursive: true);
final isDartDocSidebar =
file.path.endsWith('.html') && file.path.endsWith('-sidebar.html');
final isDartDocPage = file.path.endsWith('.html') && !isDartDocSidebar;
if (isDartDocPage) {
final page = DartDocPage.parse(await file.readAsString(encoding: _utf8));
await targetFile.writeAsBytes(_jsonUtf8.encode(page.toJson()));
} else if (isDartDocSidebar) {
final sidebar =
DartDocSidebar.parse(await file.readAsString(encoding: _utf8));
await targetFile.writeAsBytes(_jsonUtf8.encode(sidebar.toJson()));
} else {
await file.copy(targetFile.path);
}
}
// Move from temporary output directory to final one, ensuring that
// documentation files won't be present unless all files have been processed.
// This helps if there is a timeout along the way.
await Directory(tmpOutDir).rename(p.join(outputFolder, 'doc'));
_log.info('Finished post-processing');

_log.info('Creating .tar.gz archive');
Stream<TarEntry> _list() async* {
final originalDocDir = Directory(docDir);
final originalFiles = originalDocDir
final processingFuture = Isolate.run(() async {
final files = Directory(docDir)
.list(recursive: true, followLinks: false)
.whereType<File>();
await for (final file in originalFiles) {
if (cutoffTimestamp.isBefore(clock.now())) {
_log.warning(
'Cut-off timestamp reached during dartdoc archive building.');
break;
await for (final file in files) {
final suffix = file.path.substring(docDir.length + 1);
final targetFile = File(p.join(tmpOutDir, suffix));
await targetFile.parent.create(recursive: true);
final isDartDocSidebar =
file.path.endsWith('.html') && file.path.endsWith('-sidebar.html');
final isDartDocPage = file.path.endsWith('.html') && !isDartDocSidebar;
if (isDartDocPage) {
final page =
DartDocPage.parse(await file.readAsString(encoding: _utf8));
await targetFile.writeAsBytes(_jsonUtf8.encode(page.toJson()));
} else if (isDartDocSidebar) {
final sidebar =
DartDocSidebar.parse(await file.readAsString(encoding: _utf8));
await targetFile.writeAsBytes(_jsonUtf8.encode(sidebar.toJson()));
} else {
await file.copy(targetFile.path);
}
// inside the archive prefix the name with <package>/version/
final relativePath = p.relative(file.path, from: originalDocDir.path);
final tarEntryPath = p.join(package, version, relativePath);
final data = await file.readAsBytes();
yield TarEntry.data(
TarHeader(
name: tarEntryPath,
size: data.length,
),
data,
);
}
}
}).whenComplete(() {
_log.info('Finished post-processing');
});

_log.info('Creating .tar.gz archive');
final tmpTar = File(p.join(outputFolder, '_package.tar.gz'));
await _list()
.transform(tarWriter)
.transform(gzip.encoder)
.pipe(tmpTar.openWrite());
await tmpTar.rename(p.join(outputFolder, 'doc', 'package.tar.gz'));
final archiveFuture = Isolate.run(() async {
Stream<TarEntry> _list() async* {
final originalDocDir = Directory(docDir);
final originalFiles = originalDocDir
.list(recursive: true, followLinks: false)
.whereType<File>();
await for (final file in originalFiles) {
// inside the archive prefix the name with <package>/version/
final relativePath = p.relative(file.path, from: originalDocDir.path);
final tarEntryPath = p.join(package, version, relativePath);
final data = await file.readAsBytes();
yield TarEntry.data(
TarHeader(
name: tarEntryPath,
size: data.length,
),
data,
);
}
}

await _list()
.transform(tarWriter)
.transform(gzip.encoder)
.pipe(tmpTar.openWrite());
}).whenComplete(() {
_log.info('Finished .tar.gz archive');
});

_log.info('Finished .tar.gz archive');
await Future.wait([
processingFuture,
archiveFuture,
]);

// Move from temporary output directory to final one, ensuring that
// documentation files won't be present unless all files have been processed.
// This helps if there is a timeout along the way.
await Directory(tmpOutDir).rename(p.join(outputFolder, 'doc'));
await tmpTar.rename(p.join(outputFolder, 'doc', 'package.tar.gz'));
}
34 changes: 19 additions & 15 deletions pkg/pub_worker/lib/src/bin/pana_wrapper.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ final _log = Logger('pana');
/// replaced with a placeholder report.
final _reportSizeDropThreshold = 32 * 1024;

/// Stop dartdoc if it takes more than 45 minutes.
const _dartdocTimeout = Duration(minutes: 45);
/// Stop dartdoc if it takes more than 30 minutes.
const _dartdocTimeout = Duration(minutes: 30);

/// Try to fit analysis into 50 minutes.
const _totalTimeout = Duration(minutes: 50);
Expand All @@ -45,7 +45,7 @@ Future<void> main(List<String> args) async {
exit(1);
}

final cutoffTimestamp = clock.now().add(_totalTimeout);
final startedTimestamp = clock.now();

final outputFolder = args[0];
final package = args[1];
Expand Down Expand Up @@ -122,16 +122,6 @@ Future<void> main(List<String> args) async {
logger: _log,
);

if (cutoffTimestamp.isAfter(clock.now())) {
await postProcessDartdoc(
outputFolder: outputFolder,
package: package,
version: version,
docDir: rawDartdocOutputFolder.path,
cutoffTimestamp: cutoffTimestamp,
);
}

// sanity check on pana report size
final reportSize =
gzip.encode(utf8.encode(json.encode(summary.toJson()))).length;
Expand Down Expand Up @@ -161,9 +151,23 @@ Future<void> main(List<String> args) async {
p.join(outputFolder, 'summary.json'),
).writeAsString(json.encode(summary));

if (cutoffTimestamp.isAfter(clock.now())) {
await rawDartdocOutputFolder.delete(recursive: true);
// Post-processing of the dartdoc-generated files seems to take at least as
// much time as running pana+dartdoc in the first place. If we don't seem to
// have enough time to complete, it is better to not start at all.
final cutoffTimestamp = startedTimestamp.add(_totalTimeout * 0.5);
if (cutoffTimestamp.isBefore(clock.now())) {
_log.warning(
'Cut-off timestamp reached, skipping dartdoc post-processing.');
return;
}
await postProcessDartdoc(
outputFolder: outputFolder,
package: package,
version: version,
docDir: rawDartdocOutputFolder.path,
);

await rawDartdocOutputFolder.delete(recursive: true);
}

final _workerConfigDirectory = Directory('/home/worker/config');
Expand Down

0 comments on commit e299cf6

Please sign in to comment.