Option to use dockerized worker in fake_server analysis + cleanup. (#…

…7334)

app/lib/fake/tool/init_data_file.dart

@@ -6,6 +6,7 @@ import 'dart:convert';
 import 'dart:io';
 
 import 'package:_pub_shared/data/task_payload.dart';
+import 'package:_pub_shared/worker/docker_utils.dart';
 import 'package:args/command_runner.dart';
 import 'package:gcloud/service_scope.dart';
 import 'package:logging/logging.dart';
@@ -41,8 +42,9 @@ class FakeInitDataFileCommand extends Command {
       )
       ..addOption(
         'analysis',
-        allowed: ['none', 'fake', 'real'],
-        help: 'Analyze the package with fake or real analysis.',
+        allowed: ['none', 'fake', 'local', 'worker'],
+        help:
+            'Analyze the package with fake, local or dockerized worker analysis.',
         defaultsTo: 'none',
       )
       ..addOption('data-file', help: 'The file to store the local state.');
@@ -88,7 +90,9 @@ class FakeInitDataFileCommand extends Command {
                 : ImportSource.autoGenerated(),
           );
 
-          if (analysis == 'real') {
+          if (analysis == 'local') {
+            await _analyzeLocal();
+          } else if (analysis == 'worker') {
             await _analyzeWorker();
           } else if (analysis == 'fake') {
             await processTasksWithFakePanaAndDartdoc();
@@ -98,7 +102,7 @@ class FakeInitDataFileCommand extends Command {
   }
 }
 
-Future<void> _analyzeWorker() async {
+Future<void> _analyzeLocal() async {
   await fork(() async {
     await taskBackend.backfillAndProcessAllPackages((Payload payload) async {
       final arguments = [json.encode(payload.toJson())];
@@ -113,3 +117,17 @@ Future<void> _analyzeWorker() async {
     });
   });
 }
+
+Future<void> _analyzeWorker() async {
+  await buildDockerImage();
+  await fork(() async {
+    await taskBackend.backfillAndProcessAllPackages((Payload payload) async {
+      final p = await startDockerAnalysis(payload);
+      final exitCode = await p.exitCode;
+      if (exitCode != 0) {
+        throw Exception(
+            'Failed to analyze ${payload.package} with exitCode $exitCode');
+      }
+    });
+  });
+}

doc/development.md

@@ -54,14 +54,16 @@ Using the test-profile above, the following process will:
 cd app/
 dart bin/fake_server.dart init-data-file \
   --test-profile=[the file you have created] \
-  --analysis=[ none | fake | real ] \
+  --analysis=[ none | fake | local | worker ] \
   --data-file=dev-data-file.jsonl
 ```
 
-`fake` analysis will use a deterministic, but random-looking
-process to create analysis results quickly, while `real`
-analysis will run `pana` and `dartdoc` the same way as the
-production server would run them.
+- `fake` analysis will use a deterministic, but random-looking
+  process to create analysis results quickly.
+- `local` analysis will use the locally available SDKs from `PATH`.
+- `worker` analysis will run `pana` and `dartdoc` inside a docker container,
+  similarly as the production server would run them, using the configured
+  SDK versions.
 
 ### Using the fake server with the data file
 

pkg/pub_worker/lib/src/testing/docker_utils.dart → pkg/_pub_shared/lib/worker/docker_utils.dart

@@ -5,7 +5,7 @@
 import 'dart:convert';
 import 'dart:io';
 
-import 'package:pub_worker/payload.dart';
+import 'package:_pub_shared/data/task_payload.dart';
 
 final _ciEnv = Platform.environment['CI']?.toLowerCase();
 final _isRunningOnCI = _ciEnv == 'true' || _ciEnv == '1';

pkg/pub_worker/build.yaml

@@ -3,6 +3,5 @@
 targets:
   $default:
     sources:
-      - 'lib/payload.dart'
       - 'lib/pana_report.dart'
       - 'lib/src/testing/server.dart'

pkg/pub_worker/test/dockerized_end2end_test.dart

@@ -5,10 +5,10 @@
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:_pub_shared/data/task_payload.dart';
+import 'package:_pub_shared/worker/docker_utils.dart';
 import 'package:http/http.dart' as http;
 import 'package:pana/pana.dart';
-import 'package:pub_worker/payload.dart';
-import 'package:pub_worker/src/testing/docker_utils.dart';
 import 'package:pub_worker/src/testing/server.dart';
 import 'package:test/test.dart';
 

pkg/pub_worker/tool/trypkg.dart

@@ -1,10 +1,10 @@
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:_pub_shared/data/task_payload.dart';
+import 'package:_pub_shared/worker/docker_utils.dart';
 import 'package:args/args.dart';
 import 'package:path/path.dart' as p;
-import 'package:pub_worker/payload.dart';
-import 'package:pub_worker/src/testing/docker_utils.dart';
 import 'package:pub_worker/src/testing/server.dart';
 
 final _argParser = ArgParser()