Skip to content

Commit

Permalink
Remove CachedValue and cleanup of SDK search indexes. (#7161)
Browse files Browse the repository at this point in the history
  • Loading branch information
@isoos
isoos authored Nov 6, 2023
1 parent 4125694 commit e0de45e
Show file tree
Hide file tree
Showing 6 changed files with 77 additions and 141 deletions.
53 changes: 13 additions & 40 deletions app/lib/search/dart_sdk_mem_index.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,8 @@ import 'package:logging/logging.dart';
import 'package:meta/meta.dart';
import 'package:pub_dartdoc_data/dartdoc_index.dart';

import '../shared/cached_value.dart';
import 'backend.dart';
import 'sdk_mem_index.dart';
import 'search_service.dart';

final _logger = Logger('search.dart_sdk_mem_index');

Expand All @@ -23,47 +21,22 @@ const dartSdkLibraryWeights = <String, double>{
};

/// Sets the Dart SDK in-memory index.
void registerDartSdkMemIndex(DartSdkMemIndex updater) =>
ss.register(#_dartSdkMemIndex, updater);

/// The active Dart SDK in-memory index.
DartSdkMemIndex get dartSdkMemIndex =>
ss.lookup(#_dartSdkMemIndex) as DartSdkMemIndex;

/// Dart SDK in-memory index that fetches `index.json` from
/// api.dart.dev and returns search results based on [SdkMemIndex].
class DartSdkMemIndex {
final _index = CachedValue<SdkMemIndex>(
name: 'dart-sdk-index',
interval: Duration(days: 1),
maxAge: Duration(days: 30),
timeout: Duration(hours: 1),
updateFn: _createDartSdkMemIndex,
);

Future<void> start() async {
await _index.start();
}

Future<void> close() async {
await _index.close();
}

List<SdkLibraryHit> search(String query, {int? limit}) {
if (!_index.isAvailable) return <SdkLibraryHit>[];
return _index.value!.search(query, limit: limit);
}

@visibleForTesting
void setDartdocIndex(DartdocIndex index, {String? version}) {
final smi = SdkMemIndex.dart(version: version);
smi.addDartdocIndex(index);
// ignore: invalid_use_of_visible_for_testing_member
_index.setValue(smi);
void registerDartSdkMemIndex(SdkMemIndex? index) {
if (index != null) {
ss.register(#_dartSdkMemIndex, index);
}
}

Future<SdkMemIndex?> _createDartSdkMemIndex() async {
/// The active Dart SDK in-memory index.
SdkMemIndex? get dartSdkMemIndex =>
ss.lookup(#_dartSdkMemIndex) as SdkMemIndex?;

/// Tries to load Dart SDK's dartdoc `index.json` and build
/// a search index from it.
///
/// Returns `null` when the loading of `index.json` failed, or when there
/// was an error parsing the file or building the index.
Future<SdkMemIndex?> createDartSdkMemIndex() async {
try {
final index = SdkMemIndex.dart();
final content = DartdocIndex.parseJsonText(
Expand Down
40 changes: 9 additions & 31 deletions app/lib/search/flutter_sdk_mem_index.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,8 @@ import 'package:gcloud/service_scope.dart' as ss;
import 'package:logging/logging.dart';
import 'package:pub_dartdoc_data/dartdoc_index.dart';

import '../shared/cached_value.dart';
import 'backend.dart';
import 'sdk_mem_index.dart';
import 'search_service.dart';

/// The index.json file contains overlap with the Dart SDK and also repeats
/// regular packages. The selected libraries are unique to the index.json.
Expand Down Expand Up @@ -43,39 +41,19 @@ const _flutterApiPageDirWeights = <String, double>{
final _logger = Logger('search.flutter_sdk_mem_index');

/// Sets the Flutter SDK in-memory index.
void registerFlutterSdkMemIndex(FlutterSdkMemIndex updater) =>
ss.register(#_flutterSdkMemIndex, updater);
void registerFlutterSdkMemIndex(SdkMemIndex? index) {
if (index != null) {
ss.register(#_flutterSdkMemIndex, index);
}
}

/// The active Flutter SDK in-memory index.
FlutterSdkMemIndex get flutterSdkMemIndex =>
ss.lookup(#_flutterSdkMemIndex) as FlutterSdkMemIndex;
SdkMemIndex? get flutterSdkMemIndex =>
ss.lookup(#_flutterSdkMemIndex) as SdkMemIndex?;

/// Flutter SDK in-memory index that fetches `index.json` from
/// Creates Flutter SDK in-memory index that fetches `index.json` from
/// api.flutter.dev and returns search results based on [SdkMemIndex].
class FlutterSdkMemIndex {
final _index = CachedValue<SdkMemIndex>(
name: 'flutter-sdk-index',
interval: Duration(days: 1),
maxAge: Duration(days: 30),
timeout: Duration(hours: 1),
updateFn: _createFlutterSdkMemIndex,
);

Future<void> start() async {
await _index.start();
}

Future<void> close() async {
await _index.close();
}

List<SdkLibraryHit> search(String query, {int? limit}) {
if (!_index.isAvailable) return <SdkLibraryHit>[];
return _index.value!.search(query, limit: limit);
}
}

Future<SdkMemIndex?> _createFlutterSdkMemIndex() async {
Future<SdkMemIndex?> createFlutterSdkMemIndex() async {
try {
final index = SdkMemIndex.flutter();
final content = DartdocIndex.parseJsonText(
Expand Down
14 changes: 7 additions & 7 deletions app/lib/search/result_combiner.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,17 @@
import 'dart:math' as math;

import 'package:_pub_shared/search/tags.dart';
import 'package:pub_dev/search/mem_index.dart';

import 'dart_sdk_mem_index.dart';
import 'flutter_sdk_mem_index.dart';
import 'mem_index.dart';
import 'sdk_mem_index.dart';
import 'search_service.dart';

/// Combines the results from the primary package index and the optional Dart
/// SDK index.
class SearchResultCombiner {
final InMemoryPackageIndex primaryIndex;
final DartSdkMemIndex dartSdkMemIndex;
final FlutterSdkMemIndex flutterSdkMemIndex;
final SdkMemIndex? dartSdkMemIndex;
final SdkMemIndex? flutterSdkMemIndex;

SearchResultCombiner({
required this.primaryIndex,
Expand All @@ -33,8 +32,9 @@ class SearchResultCombiner {
final queryFlutterSdk = query.tagsPredicate.hasNoTagPrefix('sdk:') ||
query.tagsPredicate.hasTag(SdkTag.sdkFlutter);
final sdkLibraryHits = [
...dartSdkMemIndex.search(query.query!, limit: 2),
if (queryFlutterSdk) ...flutterSdkMemIndex.search(query.query!, limit: 2),
...?dartSdkMemIndex?.search(query.query!, limit: 2),
if (queryFlutterSdk)
...?flutterSdkMemIndex?.search(query.query!, limit: 2),
];
if (sdkLibraryHits.isNotEmpty) {
// Do not display low SDK scores if all the first page package hits are more relevant.
Expand Down
4 changes: 2 additions & 2 deletions app/lib/service/entrypoint/search_index.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ Future<void> main(List<String> args, var message) async {
}
await fork(() async {
await servicesWrapperFn(() async {
await dartSdkMemIndex.start();
await flutterSdkMemIndex.start();
registerDartSdkMemIndex(await createDartSdkMemIndex());
registerFlutterSdkMemIndex(await createFlutterSdkMemIndex());
await indexUpdater.init();

final requestReceivePort = ReceivePort();
Expand Down
6 changes: 0 additions & 6 deletions app/lib/service/services.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ import '../publisher/backend.dart';
import '../publisher/domain_verifier.dart';
import '../scorecard/backend.dart';
import '../search/backend.dart';
import '../search/dart_sdk_mem_index.dart';
import '../search/flutter_sdk_mem_index.dart';
import '../search/search_client.dart';
import '../search/top_packages.dart';
import '../search/updater.dart';
Expand Down Expand Up @@ -232,9 +230,7 @@ Future<R> _withPubServices<R>(FutureOr<R> Function() fn) async {
registerAuditBackend(AuditBackend(dbService));
registerConsentBackend(ConsentBackend(dbService));
registerDartdocBackend(DartdocBackend());
registerDartSdkMemIndex(DartSdkMemIndex());
registerEmailBackend(EmailBackend(dbService));
registerFlutterSdkMemIndex(FlutterSdkMemIndex());
registerLikeBackend(LikeBackend(dbService));
registerNameTracker(NameTracker(dbService));
registerPackageIndexHolder(PackageIndexHolder());
Expand Down Expand Up @@ -275,8 +271,6 @@ Future<R> _withPubServices<R>(FutureOr<R> Function() fn) async {
registerScopeExitCallback(announcementBackend.close);
registerScopeExitCallback(searchBackend.close);
registerScopeExitCallback(() async => nameTracker.stopTracking());
registerScopeExitCallback(dartSdkMemIndex.close);
registerScopeExitCallback(flutterSdkMemIndex.close);
registerScopeExitCallback(popularityStorage.close);
registerScopeExitCallback(scoreCardBackend.close);
registerScopeExitCallback(searchClient.close);
Expand Down
101 changes: 46 additions & 55 deletions app/test/search/result_combiner_test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,9 @@ import 'dart:convert';

import 'package:_pub_shared/search/search_form.dart';
import 'package:pub_dartdoc_data/dartdoc_index.dart';
import 'package:pub_dev/search/dart_sdk_mem_index.dart';
import 'package:pub_dev/search/flutter_sdk_mem_index.dart';
import 'package:pub_dev/search/mem_index.dart';
import 'package:pub_dev/search/result_combiner.dart';
import 'package:pub_dev/search/sdk_mem_index.dart';
import 'package:pub_dev/search/search_service.dart';
import 'package:test/test.dart';

Expand All @@ -28,58 +27,52 @@ void main() {
),
],
);
final dartSdkMemIndex = DartSdkMemIndex();
final flutterSdkMemIndex = FlutterSdkMemIndex();
final combiner = SearchResultCombiner(
primaryIndex: primaryIndex,
dartSdkMemIndex: dartSdkMemIndex,
flutterSdkMemIndex: flutterSdkMemIndex,
dartSdkMemIndex: SdkMemIndex.dart()
..addDartdocIndex(
DartdocIndex.fromJsonList([
{
'name': 'dart:core',
'qualifiedName': 'dart:core',
'href': 'dart-core/dart-core-library.html',
'kind': 8,
'overriddenDepth': 0,
'packageName': 'Dart'
},
{
'name': 'String',
'qualifiedName': 'dart:core.String',
'href': 'dart-core/String-class.html',
'kind': 3,
'overriddenDepth': 0,
'packageName': 'Dart',
'enclosedBy': {'name': 'dart:core', 'kind': 8}
},
{
'name': 'substring',
'qualifiedName': 'dart:core.String.substring',
'href': 'dart-core/String/substring.html',
'kind': 9,
'overriddenDepth': 0,
'packageName': 'Dart',
'enclosedBy': {'name': 'String', 'kind': 3}
},
{
// fake method for checking the package name matches
'name': 'stringutils',
'qualifiedName': 'dart:core.String.stringutils',
'href': 'dart-core/String/stringutils.html',
'kind': 9,
'overriddenDepth': 0,
'packageName': 'Dart',
'enclosedBy': {'name': 'String', 'kind': 3}
},
]),
),
flutterSdkMemIndex: null,
);

setUpAll(() async {
dartSdkMemIndex.setDartdocIndex(
DartdocIndex.fromJsonList([
{
'name': 'dart:core',
'qualifiedName': 'dart:core',
'href': 'dart-core/dart-core-library.html',
'kind': 8,
'overriddenDepth': 0,
'packageName': 'Dart'
},
{
'name': 'String',
'qualifiedName': 'dart:core.String',
'href': 'dart-core/String-class.html',
'kind': 3,
'overriddenDepth': 0,
'packageName': 'Dart',
'enclosedBy': {'name': 'dart:core', 'kind': 8}
},
{
'name': 'substring',
'qualifiedName': 'dart:core.String.substring',
'href': 'dart-core/String/substring.html',
'kind': 9,
'overriddenDepth': 0,
'packageName': 'Dart',
'enclosedBy': {'name': 'String', 'kind': 3}
},
{
// fake method for checking the package name matches
'name': 'stringutils',
'qualifiedName': 'dart:core.String.stringutils',
'href': 'dart-core/String/stringutils.html',
'kind': 9,
'overriddenDepth': 0,
'packageName': 'Dart',
'enclosedBy': {'name': 'String', 'kind': 3}
},
]),
version: '2.0.0',
);
});

test('non-text ranking', () async {
final results = combiner
.search(ServiceSearchQuery.parse(order: SearchOrder.popularity));
Expand Down Expand Up @@ -115,16 +108,14 @@ void main() {
'sdkLibraryHits': [
{
'sdk': 'dart',
'version': '2.0.0',
'version': isNotEmpty,
'library': 'dart:core',
'url':
'https://api.dart.dev/stable/2.0.0/dart-core/dart-core-library.html',
'url': contains('dart-core-library.html'),
'score': closeTo(0.98, 0.01),
'apiPages': [
{
'path': 'dart-core/String/substring.html',
'url':
'https://api.dart.dev/stable/2.0.0/dart-core/String/substring.html'
'url': contains('substring.html'),
}
]
},
Expand Down

0 comments on commit e0de45e

Please sign in to comment.