Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Synchronization logic for exported API #8203

Merged
merged 8 commits into from
Nov 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
378 changes: 201 additions & 177 deletions app/lib/package/api_export/api_exporter.dart

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion app/lib/package/api_export/exported_api.dart
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import '../../shared/storage.dart';
import '../../shared/versions.dart'
show runtimeVersion, runtimeVersionPattern, shouldGCVersion;

final _log = Logger('api_export:exported_bucket');
final _log = Logger('api_export.exported_api');

/// Minimum age before an item can be consider garbage.
///
Expand Down
3 changes: 1 addition & 2 deletions app/lib/package/backend.dart
Original file line number Diff line number Diff line change
Expand Up @@ -1255,8 +1255,7 @@ class PackageBackend {
emailBackend.trySendOutgoingEmail(outgoingEmail),
taskBackend.trackPackage(newVersion.package, updateDependents: true),
if (apiExporter != null)
apiExporter!
.updatePackageVersion(newVersion.package, newVersion.version!),
apiExporter!.synchronizePackage(newVersion.package),
]);
await tarballStorage.updateContentDispositionOnPublicBucket(
newVersion.package, newVersion.version!);
Expand Down
2 changes: 1 addition & 1 deletion app/lib/service/entrypoint/analyzer.dart
Original file line number Diff line number Diff line change
Expand Up @@ -85,5 +85,5 @@ Future _apiExporterMain(EntryMessage message) async {
message.protocolSendPort.send(ReadyMessage());
await popularityStorage.start();
await downloadCountsBackend.start();
await apiExporter!.uploadInForeverLoop();
await apiExporter!.start();
}
24 changes: 18 additions & 6 deletions app/lib/service/security_advisories/backend.dart
Original file line number Diff line number Diff line change
Expand Up @@ -39,22 +39,34 @@ class SecurityAdvisoryBackend {
}

Future<List<SecurityAdvisoryData>> lookupSecurityAdvisories(
String package,
) async {
return (await cache.securityAdvisories(package).get(() async {
String package, {
bool skipCache = false,
}) async {
final loadAdvisories = () async {
final query = _db.query<SecurityAdvisory>()
..filter('affectedPackages =', package);
return query
.run()
.map((SecurityAdvisory adv) => SecurityAdvisoryData.fromModel(adv))
.toList();
}))!;
};
if (skipCache) {
return await loadAdvisories();
}

return (await cache.securityAdvisories(package).get(loadAdvisories))!;
}

/// Create a [ListAdvisoriesResponse] for [package] using advisories from
/// cache.
Future<ListAdvisoriesResponse> listAdvisoriesResponse(String package) async {
final advisories = await lookupSecurityAdvisories(package);
Future<ListAdvisoriesResponse> listAdvisoriesResponse(
String package, {
bool skipCache = false,
}) async {
final advisories = await lookupSecurityAdvisories(
package,
skipCache: skipCache,
);
return ListAdvisoriesResponse(
advisories: advisories.map((a) => a.advisory).toList(),
advisoriesUpdated: advisories.map((a) => a.syncTime).maxOrNull,
Expand Down
11 changes: 2 additions & 9 deletions app/lib/tool/neat_task/pub_dev_tasks.dart
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,9 @@ void _setupGenericPeriodicTasks() {

// Exports the package name completion data to a bucket.
_daily(
name: 'export-package-name-completion-data-to-bucket',
name: 'synchronize-exported-api',
isRuntimeVersioned: true,
task: () async => await apiExporter?.uploadPkgNameCompletionData(),
task: () async => await apiExporter?.synchronizeExportedApi(),
);

// Deletes moderated packages, versions, publishers and users.
Expand Down Expand Up @@ -167,13 +167,6 @@ void _setupGenericPeriodicTasks() {
task: taskBackend.garbageCollect,
);

// Deletes exported API data for old runtime versions
_weekly(
name: 'garbage-collect-api-exports',
isRuntimeVersioned: true,
task: () async => apiExporter?.deleteObsoleteRuntimeContent(),
);

// Delete very old instances that have been abandoned
_daily(
name: 'garbage-collect-old-instances',
Expand Down
Loading
Loading