From d4f71d512a57bed7d16f150686eaceb01e416a57 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Fri, 1 Dec 2023 10:48:55 +0100 Subject: [PATCH 1/2] Bucket config + initialization for API JSON exporter. --- app/lib/package/export_api_to_bucket.dart | 8 ++++++++ app/lib/service/entrypoint/analyzer.dart | 19 ++++++++++++++++++- app/lib/service/services.dart | 6 ++++++ app/lib/shared/configuration.dart | 7 +++++++ app/lib/shared/configuration.g.dart | 4 ++++ app/lib/tool/neat_task/pub_dev_tasks.dart | 15 +++++++++++++++ 6 files changed, 58 insertions(+), 1 deletion(-) diff --git a/app/lib/package/export_api_to_bucket.dart b/app/lib/package/export_api_to_bucket.dart index 7e7ef1681d..c05c5ab986 100644 --- a/app/lib/package/export_api_to_bucket.dart +++ b/app/lib/package/export_api_to_bucket.dart @@ -7,6 +7,7 @@ import 'dart:io'; import 'package:basics/basics.dart'; import 'package:clock/clock.dart'; import 'package:crypto/crypto.dart'; +import 'package:gcloud/service_scope.dart' as ss; import 'package:gcloud/storage.dart'; import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; @@ -41,6 +42,13 @@ List _apiPkgNameCompletitionDataNames() => [ 'current/api/package-name-completion-data', ]; +/// Sets the API Exporter service. +void registerApiExporter(ApiExporter value) => + ss.register(#_apiExporter, value); + +/// The active API Exporter service or null if it hasn't been initialized. +ApiExporter? get apiExporter => ss.lookup(#_apiExporter) as ApiExporter?; + class ApiExporter { final Bucket _bucket; final int _concurrency; diff --git a/app/lib/service/entrypoint/analyzer.dart b/app/lib/service/entrypoint/analyzer.dart index 4d0264e73e..0476f8b9d1 100644 --- a/app/lib/service/entrypoint/analyzer.dart +++ b/app/lib/service/entrypoint/analyzer.dart @@ -7,7 +7,9 @@ import 'dart:async'; import 'package:args/command_runner.dart'; import 'package:gcloud/service_scope.dart'; import 'package:logging/logging.dart'; +import 'package:pub_dev/package/export_api_to_bucket.dart'; import 'package:pub_dev/search/backend.dart'; +import 'package:pub_dev/shared/configuration.dart'; import '../../analyzer/handlers.dart'; import '../../service/services.dart'; @@ -43,6 +45,15 @@ class AnalyzerCommand extends Command { ); registerScopeExitCallback(indexBuilder.close); + if (activeConfiguration.exportedApiBucketName != null) { + final apiExporterIsolate = await startWorkerIsolate( + logger: logger, + entryPoint: _apiExporterMain, + kind: 'api-exporter', + ); + registerScopeExitCallback(apiExporterIsolate.close); + } + await runHandler(logger, analyzerServiceHandler); }); } @@ -51,11 +62,11 @@ class AnalyzerCommand extends Command { Future _workerMain(EntryMessage message) async { message.protocolSendPort.send(ReadyMessage()); + await popularityStorage.start(); await taskBackend.start(); setupAnalyzerPeriodicTasks(); setupSearchPeriodicTasks(); - await popularityStorage.start(); // wait indefinitely await Completer().future; @@ -66,3 +77,9 @@ Future _indexBuilderMain(EntryMessage message) async { await popularityStorage.start(); await searchBackend.updateSnapshotInForeverLoop(); } + +Future _apiExporterMain(EntryMessage message) async { + message.protocolSendPort.send(ReadyMessage()); + await popularityStorage.start(); + await apiExporter!.uploadInForeverLoop(); +} diff --git a/app/lib/service/services.dart b/app/lib/service/services.dart index dcacf9c6ab..47f75d3a81 100644 --- a/app/lib/service/services.dart +++ b/app/lib/service/services.dart @@ -12,6 +12,7 @@ import 'package:gcloud/service_scope.dart'; import 'package:gcloud/storage.dart'; import 'package:googleapis_auth/auth_io.dart' as auth; import 'package:logging/logging.dart'; +import 'package:pub_dev/package/export_api_to_bucket.dart'; import 'package:pub_dev/service/async_queue/async_queue.dart'; import 'package:pub_dev/service/security_advisories/backend.dart'; import 'package:shelf/shelf.dart' as shelf; @@ -228,6 +229,11 @@ Future _withPubServices(FutureOr Function() fn) async { registerAccountBackend(AccountBackend(dbService)); registerAdminBackend(AdminBackend(dbService)); registerAnnouncementBackend(AnnouncementBackend()); + if (activeConfiguration.exportedApiBucketName != null) { + registerApiExporter(ApiExporter( + bucket: storageService + .bucket(activeConfiguration.exportedApiBucketName!))); + } registerAsyncQueue(AsyncQueue()); registerAuditBackend(AuditBackend(dbService)); registerConsentBackend(ConsentBackend(dbService)); diff --git a/app/lib/shared/configuration.dart b/app/lib/shared/configuration.dart index e359b90672..378c9c18cd 100644 --- a/app/lib/shared/configuration.dart +++ b/app/lib/shared/configuration.dart @@ -85,6 +85,9 @@ class Configuration { /// The name of the Cloud Storage bucket to use for generated reports. final String? reportsBucketName; + /// The name of the Cloud Storage bucket to use for exporting JSON API responses. + final String? exportedApiBucketName; + /// The Cloud project Id. This is only required when using Apiary to access /// Datastore and/or Cloud Storage final String projectId; @@ -275,6 +278,7 @@ class Configuration { required this.dartdocStorageBucketName, required this.popularityDumpBucketName, required this.searchSnapshotBucketName, + required this.exportedApiBucketName, required this.maxTaskInstances, required this.maxTaskRunHours, required this.taskResultBucketName, @@ -341,6 +345,7 @@ class Configuration { dartdocStorageBucketName: 'fake-bucket-dartdoc', popularityDumpBucketName: 'fake-bucket-popularity', searchSnapshotBucketName: 'fake-bucket-search', + exportedApiBucketName: 'fake-exported-apis', maxTaskInstances: 10, maxTaskRunHours: 2, taskResultBucketName: 'fake-bucket-task-result', @@ -392,6 +397,7 @@ class Configuration { dartdocStorageBucketName: 'fake-bucket-dartdoc', popularityDumpBucketName: 'fake-bucket-popularity', searchSnapshotBucketName: 'fake-bucket-search', + exportedApiBucketName: 'fake-exported-apis', taskResultBucketName: 'fake-bucket-task-result', maxTaskInstances: 10, maxTaskRunHours: 2, @@ -442,6 +448,7 @@ class Configuration { publicPackagesBucketName!, searchSnapshotBucketName!, taskResultBucketName!, + if (exportedApiBucketName != null) exportedApiBucketName!, ]); late final isProduction = projectId == 'dartlang-pub'; diff --git a/app/lib/shared/configuration.g.dart b/app/lib/shared/configuration.g.dart index e00df43830..e4df638305 100644 --- a/app/lib/shared/configuration.g.dart +++ b/app/lib/shared/configuration.g.dart @@ -18,6 +18,7 @@ Configuration _$ConfigurationFromJson(Map json) => $checkedCreate( 'incomingPackagesBucketName', 'imageBucketName', 'reportsBucketName', + 'exportedApiBucketName', 'projectId', 'searchServicePrefix', 'fallbackSearchServicePrefix', @@ -68,6 +69,8 @@ Configuration _$ConfigurationFromJson(Map json) => $checkedCreate( $checkedConvert('popularityDumpBucketName', (v) => v as String?), searchSnapshotBucketName: $checkedConvert('searchSnapshotBucketName', (v) => v as String?), + exportedApiBucketName: + $checkedConvert('exportedApiBucketName', (v) => v as String?), maxTaskInstances: $checkedConvert('maxTaskInstances', (v) => v as int), maxTaskRunHours: $checkedConvert('maxTaskRunHours', (v) => v as int), @@ -141,6 +144,7 @@ Map _$ConfigurationToJson(Configuration instance) => 'incomingPackagesBucketName': instance.incomingPackagesBucketName, 'imageBucketName': instance.imageBucketName, 'reportsBucketName': instance.reportsBucketName, + 'exportedApiBucketName': instance.exportedApiBucketName, 'projectId': instance.projectId, 'searchServicePrefix': instance.searchServicePrefix, 'fallbackSearchServicePrefix': instance.fallbackSearchServicePrefix, diff --git a/app/lib/tool/neat_task/pub_dev_tasks.dart b/app/lib/tool/neat_task/pub_dev_tasks.dart index 170db9ac43..6470f205b4 100644 --- a/app/lib/tool/neat_task/pub_dev_tasks.dart +++ b/app/lib/tool/neat_task/pub_dev_tasks.dart @@ -8,6 +8,7 @@ import 'dart:io'; import 'package:gcloud/service_scope.dart' as ss; import 'package:logging/logging.dart'; import 'package:neat_periodic_task/neat_periodic_task.dart'; +import 'package:pub_dev/package/export_api_to_bucket.dart'; import '../../account/backend.dart'; import '../../account/consent_backend.dart'; @@ -114,6 +115,13 @@ void _setupGenericPeriodicTasks() { task: updatePublicArchiveBucket, ); + // Exports the package name completetion data to a bucket. + _daily( + name: 'export-package-name-completition-data-to-bucket', + isRuntimeVersioned: true, + task: () async => await apiExporter?.uploadPkgNameCompletionData(), + ); + // Deletes task status entities where the status hasn't been updated // for more than a month. _weekly( @@ -150,6 +158,13 @@ void _setupGenericPeriodicTasks() { task: taskBackend.garbageCollect, ); + // Deletes exported API data for old runtime versions + _weekly( + name: 'garbage-collect-api-exports', + isRuntimeVersioned: true, + task: () async => apiExporter?.deleteObsoleteRuntimeContent(), + ); + // Delete very old instances that have been abandoned _daily( name: 'garbage-collect-old-instances', From c176636111055842dbab498c2b9626fd713ebaed Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Fri, 1 Dec 2023 11:05:16 +0100 Subject: [PATCH 2/2] Updated config --- app/config/dartlang-pub-dev.yaml | 1 + app/config/dartlang-pub.yaml | 1 + app/test/shared/test_data/foo_config.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/app/config/dartlang-pub-dev.yaml b/app/config/dartlang-pub-dev.yaml index a26ca9584c..05b185aab1 100644 --- a/app/config/dartlang-pub-dev.yaml +++ b/app/config/dartlang-pub-dev.yaml @@ -10,6 +10,7 @@ defaultServiceBaseUrl: https://{{GAE_VERSION}}-dot-{{GOOGLE_CLOUD_PROJECT}}.apps dartdocStorageBucketName: dartlang-pub-dev--dartdoc-storage popularityDumpBucketName: dartlang-pub-dev--popularity searchSnapshotBucketName: dartlang-pub-dev--search-snapshot +exportedApiBucketName: null maxTaskInstances: 50 maxTaskRunHours: 2 taskResultBucketName: dartlang-pub-dev-task-output diff --git a/app/config/dartlang-pub.yaml b/app/config/dartlang-pub.yaml index d576e45682..ef7b9882cd 100644 --- a/app/config/dartlang-pub.yaml +++ b/app/config/dartlang-pub.yaml @@ -13,6 +13,7 @@ defaultServiceBaseUrl: https://{{GAE_VERSION}}-dot-{{GOOGLE_CLOUD_PROJECT}}.apps dartdocStorageBucketName: dartlang-pub--dartdoc-storage popularityDumpBucketName: dartlang-pub--popularity searchSnapshotBucketName: dartlang-pub--search-snapshot +exportedApiBucketName: null maxTaskInstances: 700 maxTaskRunHours: 2 taskResultBucketName: dartlang-pub-task-output diff --git a/app/test/shared/test_data/foo_config.yaml b/app/test/shared/test_data/foo_config.yaml index 00e215c52b..044ed0300f 100644 --- a/app/test/shared/test_data/foo_config.yaml +++ b/app/test/shared/test_data/foo_config.yaml @@ -5,6 +5,7 @@ popularityDumpBucketName: foo searchSnapshotBucketName: foo canonicalPackagesBucketName: foo reportsBucketName: foo +exportedApiBucketName: foo maxTaskInstances: 0 maxTaskRunHours: 2 searchServicePrefix: 'https://search-dot-dartlang-pub-dev.appspot.com'