Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add metadata to fake_gcloud #8243

Merged
merged 1 commit into from
Nov 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 39 additions & 6 deletions pkg/fake_gcloud/lib/mem_storage.dart
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import 'dart:async';
import 'dart:convert';

import 'package:_discoveryapis_commons/_discoveryapis_commons.dart';
import 'package:clock/clock.dart';
import 'package:crypto/crypto.dart';
import 'package:gcloud/storage.dart';
import 'package:logging/logging.dart';
Expand Down Expand Up @@ -67,9 +68,13 @@ class MemStorage implements Storage {
_logger.info('Copy object from $src to $dest');
final srcUri = Uri.parse(src);
final destUri = Uri.parse(dest);
await bucket(srcUri.host)
.read(srcUri.path.substring(1))
.pipe(bucket(destUri.host).write(destUri.path.substring(1)));
final srcBucket = _buckets[srcUri.host]!;
final srcObject = srcBucket._files[srcUri.path.substring(1)]!;
await bucket(destUri.host).writeBytes(
destUri.path.substring(1),
srcObject.content,
metadata: metadata ?? srcObject.metadata,
);
}

/// Serializes the content of the Storage to the [sink], with a line-by-line
Expand Down Expand Up @@ -107,18 +112,38 @@ class MemStorage implements Storage {
'name': file.name,
'content': base64.encode(file.content),
'updated': file.updated.toUtc().toIso8601String(),
'metadata': null, // TODO: add metadata support
'metadata': {
'contentType': file.metadata.contentType,
'contentEncoding': file.metadata.contentEncoding,
'cacheControl': file.metadata.cacheControl,
'contentDisposition': file.metadata.contentDisposition,
'contentLanguage': file.metadata.contentLanguage,
'custom': file.metadata.custom,
},
};
}

_File _decodeFile(Map<String, dynamic> map) {
final content = base64.decode(map['content'] as String);
final updated = DateTime.parse(map['updated'] as String);
final meta = map['metadata'] ?? <String, Object?>{};
return _File(
bucketName: map['bucket'] as String,
name: map['name'] as String,
content: content,
updated: updated,
metadata: ObjectMetadata(
acl: Acl([]),
contentType: meta['contentType'] as String?,
contentEncoding: meta['contentEncoding'] as String?,
cacheControl: meta['cacheControl'] as String?,
contentDisposition: meta['contentDisposition'] as String?,
contentLanguage: meta['contentLanguage'] as String?,
custom: (meta['custom'] as Map?)?.map(
(k, v) => MapEntry(k as String, v as String),
) ??
<String, String>{},
),
);
}
}
Expand All @@ -141,12 +166,12 @@ class _File implements BucketObjectEntry {
required this.bucketName,
required this.name,
required this.content,
required this.metadata,
DateTime? updated,
}) : // TODO: use a real CRC32 check
crc32CChecksum = content.fold<int>(0, (a, b) => a + b) & 0xffffffff,
md5Hash = md5.convert(content).bytes,
updated = updated ?? DateTime.now().toUtc(),
metadata = ObjectMetadata(acl: Acl([]));
updated = updated ?? clock.now().toUtc();

@override
Uri get downloadLink => Uri(scheme: 'gs', host: bucketName, path: name);
Expand Down Expand Up @@ -199,10 +224,18 @@ class _Bucket implements Bucket {
buffer.addAll(data);
return buffer;
}).then((content) {
var meta = metadata ?? ObjectMetadata();
if (acl != null) {
meta = meta.replace(acl: acl);
}
if (contentType != null) {
meta = meta.replace(contentType: contentType);
}
_files[objectName] = _File(
bucketName: bucketName,
name: objectName,
content: content,
metadata: meta,
);
_logger.info('Completed ${content.length} bytes: $objectName');
});
Expand Down
8 changes: 8 additions & 0 deletions pkg/fake_gcloud/pubspec.lock
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "2.1.1"
clock:
dependency: "direct main"
description:
name: clock
sha256: fddb70d9b5277016c77a80201021d40a2247104d9f4aa7bab7157b7e3f05b84b
url: "https://pub.dev"
source: hosted
version: "1.1.2"
collection:
dependency: transitive
description:
Expand Down
1 change: 1 addition & 0 deletions pkg/fake_gcloud/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ environment:
dependencies:
gcloud: ^0.8.16
logging: '>=0.11.3 <2.0.0'
clock: ^1.1.0

dev_dependencies:
coverage: any # test already depends on it
Expand Down
1 change: 0 additions & 1 deletion pkg/fake_gcloud/test/mem_storage_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ void main() {

final sb = StringBuffer();
storage.writeTo(sb);
expect(sb.length, 118);
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this test is dumb, we're checking that MemStorage can be serialized, we should assert what size it has.

Just that we get the same data out when we're deserialize it, which is tested further down.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Exact length check may be important if we are checking for a specific serialization format or making sure we are not doing excess or stupid things here. However, I do agree that this is not that important here.


final newStorage = MemStorage();
newStorage.readFrom(sb.toString().split('\n'));
Expand Down
Loading