From f8716aeac4a0c6466df6cabadc8db4178fc76c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Tue, 21 Jun 2022 12:13:58 +0200 Subject: [PATCH 01/18] Split proto3 JSON writer and parser to separate files --- protobuf/lib/protobuf.dart | 3 +- ...oto3_json.dart => proto3_json_reader.dart} | 180 ++++-------------- .../lib/src/protobuf/proto3_json_writer.dart | 111 +++++++++++ 3 files changed, 150 insertions(+), 144 deletions(-) rename protobuf/lib/src/protobuf/{proto3_json.dart => proto3_json_reader.dart} (76%) create mode 100644 protobuf/lib/src/protobuf/proto3_json_writer.dart diff --git a/protobuf/lib/protobuf.dart b/protobuf/lib/protobuf.dart index ad7bb99fc..26749f36e 100644 --- a/protobuf/lib/protobuf.dart +++ b/protobuf/lib/protobuf.dart @@ -38,7 +38,8 @@ part 'src/protobuf/json.dart'; part 'src/protobuf/pb_list.dart'; part 'src/protobuf/pb_map.dart'; part 'src/protobuf/protobuf_enum.dart'; -part 'src/protobuf/proto3_json.dart'; +part 'src/protobuf/proto3_json_reader.dart'; +part 'src/protobuf/proto3_json_writer.dart'; part 'src/protobuf/rpc_client.dart'; part 'src/protobuf/unknown_field_set.dart'; part 'src/protobuf/utils.dart'; diff --git a/protobuf/lib/src/protobuf/proto3_json.dart b/protobuf/lib/src/protobuf/proto3_json_reader.dart similarity index 76% rename from protobuf/lib/src/protobuf/proto3_json.dart rename to protobuf/lib/src/protobuf/proto3_json_reader.dart index 052d10c4d..68ac38707 100644 --- a/protobuf/lib/src/protobuf/proto3_json.dart +++ b/protobuf/lib/src/protobuf/proto3_json_reader.dart @@ -4,149 +4,6 @@ part of protobuf; -Object? _writeToProto3Json(_FieldSet fs, TypeRegistry typeRegistry) { - String? convertToMapKey(dynamic key, int keyType) { - var baseType = PbFieldType._baseType(keyType); - - assert(!_isRepeated(keyType)); - - switch (baseType) { - case PbFieldType._BOOL_BIT: - return key ? 'true' : 'false'; - case PbFieldType._STRING_BIT: - return key; - case PbFieldType._UINT64_BIT: - return (key as Int64).toStringUnsigned(); - case PbFieldType._INT32_BIT: - case PbFieldType._SINT32_BIT: - case PbFieldType._UINT32_BIT: - case PbFieldType._FIXED32_BIT: - case PbFieldType._SFIXED32_BIT: - case PbFieldType._INT64_BIT: - case PbFieldType._SINT64_BIT: - case PbFieldType._SFIXED64_BIT: - case PbFieldType._FIXED64_BIT: - return key.toString(); - default: - throw StateError('Not a valid key type $keyType'); - } - } - - Object? valueToProto3Json(dynamic fieldValue, int? fieldType) { - if (fieldValue == null) return null; - - if (_isGroupOrMessage(fieldType!)) { - return _writeToProto3Json( - (fieldValue as GeneratedMessage)._fieldSet, typeRegistry); - } else if (_isEnum(fieldType)) { - return (fieldValue as ProtobufEnum).name; - } else { - var baseType = PbFieldType._baseType(fieldType); - switch (baseType) { - case PbFieldType._BOOL_BIT: - return fieldValue ? true : false; - case PbFieldType._STRING_BIT: - return fieldValue; - case PbFieldType._INT32_BIT: - case PbFieldType._SINT32_BIT: - case PbFieldType._UINT32_BIT: - case PbFieldType._FIXED32_BIT: - case PbFieldType._SFIXED32_BIT: - return fieldValue; - case PbFieldType._INT64_BIT: - case PbFieldType._SINT64_BIT: - case PbFieldType._SFIXED64_BIT: - case PbFieldType._FIXED64_BIT: - return fieldValue.toString(); - case PbFieldType._FLOAT_BIT: - case PbFieldType._DOUBLE_BIT: - double value = fieldValue; - if (value.isNaN) { - return nan; - } - if (value.isInfinite) { - return value.isNegative ? negativeInfinity : infinity; - } - return value; - case PbFieldType._UINT64_BIT: - return (fieldValue as Int64).toStringUnsigned(); - case PbFieldType._BYTES_BIT: - return base64Encode(fieldValue); - default: - throw StateError( - 'Invariant violation: unexpected value type $fieldType'); - } - } - } - - final meta = fs._meta; - if (meta.toProto3Json != null) { - return meta.toProto3Json!(fs._message!, typeRegistry); - } - - var result = {}; - for (var fieldInfo in fs._infosSortedByTag) { - var value = fs._values[fieldInfo.index!]; - if (value == null || (value is List && value.isEmpty)) { - continue; // It's missing, repeated, or an empty byte array. - } - dynamic jsonValue; - if (fieldInfo.isMapField) { - jsonValue = (value as PbMap).map((key, entryValue) { - var mapEntryInfo = fieldInfo as MapFieldInfo; - return MapEntry(convertToMapKey(key, mapEntryInfo.keyFieldType), - valueToProto3Json(entryValue, mapEntryInfo.valueFieldType)); - }); - } else if (fieldInfo.isRepeated) { - jsonValue = (value as PbList) - .map((element) => valueToProto3Json(element, fieldInfo.type)) - .toList(); - } else { - jsonValue = valueToProto3Json(value, fieldInfo.type); - } - result[fieldInfo.name] = jsonValue; - } - // Extensions and unknown fields are not encoded by proto3 JSON. - return result; -} - -int _tryParse32BitProto3(String s, JsonParsingContext context) { - return int.tryParse(s) ?? - (throw context.parseException('expected integer', s)); -} - -int _check32BitSignedProto3(int n, JsonParsingContext context) { - if (n < -2147483648 || n > 2147483647) { - throw context.parseException('expected 32 bit signed integer', n); - } - return n; -} - -int _check32BitUnsignedProto3(int n, JsonParsingContext context) { - if (n < 0 || n > 0xFFFFFFFF) { - throw context.parseException('expected 32 bit unsigned integer', n); - } - return n; -} - -Int64 _tryParse64BitProto3(Object? json, String s, JsonParsingContext context) { - try { - return Int64.parseInt(s); - } on FormatException { - throw context.parseException('expected integer', json); - } -} - -/// TODO(paulberry): find a better home for this? -extension _FindFirst on Iterable { - E? findFirst(bool Function(E) test) { - for (var element in this) { - if (test(element)) return element; - } - return null; - } -} - void _mergeFromProto3Json( Object? json, _FieldSet fieldSet, @@ -413,3 +270,40 @@ void _mergeFromProto3Json( recursionHelper(json, fieldSet); } + +int _tryParse32BitProto3(String s, JsonParsingContext context) { + return int.tryParse(s) ?? + (throw context.parseException('expected integer', s)); +} + +int _check32BitSignedProto3(int n, JsonParsingContext context) { + if (n < -2147483648 || n > 2147483647) { + throw context.parseException('expected 32 bit signed integer', n); + } + return n; +} + +int _check32BitUnsignedProto3(int n, JsonParsingContext context) { + if (n < 0 || n > 0xFFFFFFFF) { + throw context.parseException('expected 32 bit unsigned integer', n); + } + return n; +} + +Int64 _tryParse64BitProto3(Object? json, String s, JsonParsingContext context) { + try { + return Int64.parseInt(s); + } on FormatException { + throw context.parseException('expected integer', json); + } +} + +/// TODO(paulberry): find a better home for this? +extension _FindFirst on Iterable { + E? findFirst(bool Function(E) test) { + for (var element in this) { + if (test(element)) return element; + } + return null; + } +} diff --git a/protobuf/lib/src/protobuf/proto3_json_writer.dart b/protobuf/lib/src/protobuf/proto3_json_writer.dart new file mode 100644 index 000000000..4e3daaec4 --- /dev/null +++ b/protobuf/lib/src/protobuf/proto3_json_writer.dart @@ -0,0 +1,111 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +part of protobuf; + +Object? _writeToProto3Json(_FieldSet fs, TypeRegistry typeRegistry) { + String? convertToMapKey(dynamic key, int keyType) { + var baseType = PbFieldType._baseType(keyType); + + assert(!_isRepeated(keyType)); + + switch (baseType) { + case PbFieldType._BOOL_BIT: + return key ? 'true' : 'false'; + case PbFieldType._STRING_BIT: + return key; + case PbFieldType._UINT64_BIT: + return (key as Int64).toStringUnsigned(); + case PbFieldType._INT32_BIT: + case PbFieldType._SINT32_BIT: + case PbFieldType._UINT32_BIT: + case PbFieldType._FIXED32_BIT: + case PbFieldType._SFIXED32_BIT: + case PbFieldType._INT64_BIT: + case PbFieldType._SINT64_BIT: + case PbFieldType._SFIXED64_BIT: + case PbFieldType._FIXED64_BIT: + return key.toString(); + default: + throw StateError('Not a valid key type $keyType'); + } + } + + Object? valueToProto3Json(dynamic fieldValue, int? fieldType) { + if (fieldValue == null) return null; + + if (_isGroupOrMessage(fieldType!)) { + return _writeToProto3Json( + (fieldValue as GeneratedMessage)._fieldSet, typeRegistry); + } else if (_isEnum(fieldType)) { + return (fieldValue as ProtobufEnum).name; + } else { + var baseType = PbFieldType._baseType(fieldType); + switch (baseType) { + case PbFieldType._BOOL_BIT: + return fieldValue ? true : false; + case PbFieldType._STRING_BIT: + return fieldValue; + case PbFieldType._INT32_BIT: + case PbFieldType._SINT32_BIT: + case PbFieldType._UINT32_BIT: + case PbFieldType._FIXED32_BIT: + case PbFieldType._SFIXED32_BIT: + return fieldValue; + case PbFieldType._INT64_BIT: + case PbFieldType._SINT64_BIT: + case PbFieldType._SFIXED64_BIT: + case PbFieldType._FIXED64_BIT: + return fieldValue.toString(); + case PbFieldType._FLOAT_BIT: + case PbFieldType._DOUBLE_BIT: + double value = fieldValue; + if (value.isNaN) { + return nan; + } + if (value.isInfinite) { + return value.isNegative ? negativeInfinity : infinity; + } + return value; + case PbFieldType._UINT64_BIT: + return (fieldValue as Int64).toStringUnsigned(); + case PbFieldType._BYTES_BIT: + return base64Encode(fieldValue); + default: + throw StateError( + 'Invariant violation: unexpected value type $fieldType'); + } + } + } + + final meta = fs._meta; + if (meta.toProto3Json != null) { + return meta.toProto3Json!(fs._message!, typeRegistry); + } + + var result = {}; + for (var fieldInfo in fs._infosSortedByTag) { + var value = fs._values[fieldInfo.index!]; + if (value == null || (value is List && value.isEmpty)) { + continue; // It's missing, repeated, or an empty byte array. + } + dynamic jsonValue; + if (fieldInfo.isMapField) { + jsonValue = (value as PbMap).map((key, entryValue) { + var mapEntryInfo = fieldInfo as MapFieldInfo; + return MapEntry(convertToMapKey(key, mapEntryInfo.keyFieldType), + valueToProto3Json(entryValue, mapEntryInfo.valueFieldType)); + }); + } else if (fieldInfo.isRepeated) { + jsonValue = (value as PbList) + .map((element) => valueToProto3Json(element, fieldInfo.type)) + .toList(); + } else { + jsonValue = valueToProto3Json(value, fieldInfo.type); + } + result[fieldInfo.name] = jsonValue; + } + // Extensions and unknown fields are not encoded by proto3 JSON. + return result; +} From 36c3e5efb2b50e79630f9fd91444286759fe0792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Tue, 21 Jun 2022 12:15:24 +0200 Subject: [PATCH 02/18] Split custom JSON writer and parser to separate files --- protobuf/lib/protobuf.dart | 3 +- .../protobuf/{json.dart => json_reader.dart} | 83 ----------------- protobuf/lib/src/protobuf/json_writer.dart | 88 +++++++++++++++++++ 3 files changed, 90 insertions(+), 84 deletions(-) rename protobuf/lib/src/protobuf/{json.dart => json_reader.dart} (72%) create mode 100644 protobuf/lib/src/protobuf/json_writer.dart diff --git a/protobuf/lib/protobuf.dart b/protobuf/lib/protobuf.dart index 26749f36e..76d329f0a 100644 --- a/protobuf/lib/protobuf.dart +++ b/protobuf/lib/protobuf.dart @@ -34,7 +34,8 @@ part 'src/protobuf/field_set.dart'; part 'src/protobuf/field_type.dart'; part 'src/protobuf/generated_message.dart'; part 'src/protobuf/generated_service.dart'; -part 'src/protobuf/json.dart'; +part 'src/protobuf/json_reader.dart'; +part 'src/protobuf/json_writer.dart'; part 'src/protobuf/pb_list.dart'; part 'src/protobuf/pb_map.dart'; part 'src/protobuf/protobuf_enum.dart'; diff --git a/protobuf/lib/src/protobuf/json.dart b/protobuf/lib/src/protobuf/json_reader.dart similarity index 72% rename from protobuf/lib/src/protobuf/json.dart rename to protobuf/lib/src/protobuf/json_reader.dart index 5b9fdcceb..30039972f 100644 --- a/protobuf/lib/src/protobuf/json.dart +++ b/protobuf/lib/src/protobuf/json_reader.dart @@ -4,89 +4,6 @@ part of protobuf; -Map _writeToJsonMap(_FieldSet fs) { - dynamic convertToMap(dynamic fieldValue, int fieldType) { - var baseType = PbFieldType._baseType(fieldType); - - if (_isRepeated(fieldType)) { - return List.from(fieldValue.map((e) => convertToMap(e, baseType))); - } - - switch (baseType) { - case PbFieldType._BOOL_BIT: - case PbFieldType._STRING_BIT: - case PbFieldType._INT32_BIT: - case PbFieldType._SINT32_BIT: - case PbFieldType._UINT32_BIT: - case PbFieldType._FIXED32_BIT: - case PbFieldType._SFIXED32_BIT: - return fieldValue; - case PbFieldType._FLOAT_BIT: - case PbFieldType._DOUBLE_BIT: - final value = fieldValue as double; - if (value.isNaN) { - return nan; - } - if (value.isInfinite) { - return value.isNegative ? negativeInfinity : infinity; - } - if (fieldValue.toInt() == fieldValue) { - return fieldValue.toInt(); - } - return value; - case PbFieldType._BYTES_BIT: - // Encode 'bytes' as a base64-encoded string. - return base64Encode(fieldValue as List); - case PbFieldType._ENUM_BIT: - return fieldValue.value; // assume |value| < 2^52 - case PbFieldType._INT64_BIT: - case PbFieldType._SINT64_BIT: - case PbFieldType._SFIXED64_BIT: - return fieldValue.toString(); - case PbFieldType._UINT64_BIT: - case PbFieldType._FIXED64_BIT: - return fieldValue.toStringUnsigned(); - case PbFieldType._GROUP_BIT: - case PbFieldType._MESSAGE_BIT: - return fieldValue.writeToJsonMap(); - default: - throw 'Unknown type $fieldType'; - } - } - - List _writeMap(dynamic fieldValue, MapFieldInfo fi) => - List.from(fieldValue.entries.map((MapEntry e) => { - '${PbMap._keyFieldNumber}': convertToMap(e.key, fi.keyFieldType), - '${PbMap._valueFieldNumber}': - convertToMap(e.value, fi.valueFieldType) - })); - - var result = {}; - for (var fi in fs._infosSortedByTag) { - var value = fs._values[fi.index!]; - if (value == null || (value is List && value.isEmpty)) { - continue; // It's missing, repeated, or an empty byte array. - } - if (_isMapField(fi.type)) { - result['${fi.tagNumber}'] = - _writeMap(value, fi as MapFieldInfo); - continue; - } - result['${fi.tagNumber}'] = convertToMap(value, fi.type); - } - if (fs._hasExtensions) { - for (var tagNumber in _sorted(fs._extensions!._tagNumbers)) { - var value = fs._extensions!._values[tagNumber]; - if (value is List && value.isEmpty) { - continue; // It's repeated or an empty byte array. - } - var fi = fs._extensions!._getInfoOrNull(tagNumber)!; - result['$tagNumber'] = convertToMap(value, fi.type); - } - } - return result; -} - // Merge fields from a previously decoded JSON object. // (Called recursively on nested messages.) void _mergeFromJsonMap( diff --git a/protobuf/lib/src/protobuf/json_writer.dart b/protobuf/lib/src/protobuf/json_writer.dart new file mode 100644 index 000000000..00418558a --- /dev/null +++ b/protobuf/lib/src/protobuf/json_writer.dart @@ -0,0 +1,88 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +part of protobuf; + +Map _writeToJsonMap(_FieldSet fs) { + dynamic convertToMap(dynamic fieldValue, int fieldType) { + var baseType = PbFieldType._baseType(fieldType); + + if (_isRepeated(fieldType)) { + return List.from(fieldValue.map((e) => convertToMap(e, baseType))); + } + + switch (baseType) { + case PbFieldType._BOOL_BIT: + case PbFieldType._STRING_BIT: + case PbFieldType._INT32_BIT: + case PbFieldType._SINT32_BIT: + case PbFieldType._UINT32_BIT: + case PbFieldType._FIXED32_BIT: + case PbFieldType._SFIXED32_BIT: + return fieldValue; + case PbFieldType._FLOAT_BIT: + case PbFieldType._DOUBLE_BIT: + final value = fieldValue as double; + if (value.isNaN) { + return nan; + } + if (value.isInfinite) { + return value.isNegative ? negativeInfinity : infinity; + } + if (fieldValue.toInt() == fieldValue) { + return fieldValue.toInt(); + } + return value; + case PbFieldType._BYTES_BIT: + // Encode 'bytes' as a base64-encoded string. + return base64Encode(fieldValue as List); + case PbFieldType._ENUM_BIT: + return fieldValue.value; // assume |value| < 2^52 + case PbFieldType._INT64_BIT: + case PbFieldType._SINT64_BIT: + case PbFieldType._SFIXED64_BIT: + return fieldValue.toString(); + case PbFieldType._UINT64_BIT: + case PbFieldType._FIXED64_BIT: + return fieldValue.toStringUnsigned(); + case PbFieldType._GROUP_BIT: + case PbFieldType._MESSAGE_BIT: + return fieldValue.writeToJsonMap(); + default: + throw 'Unknown type $fieldType'; + } + } + + List _writeMap(dynamic fieldValue, MapFieldInfo fi) => + List.from(fieldValue.entries.map((MapEntry e) => { + '${PbMap._keyFieldNumber}': convertToMap(e.key, fi.keyFieldType), + '${PbMap._valueFieldNumber}': + convertToMap(e.value, fi.valueFieldType) + })); + + var result = {}; + for (var fi in fs._infosSortedByTag) { + var value = fs._values[fi.index!]; + if (value == null || (value is List && value.isEmpty)) { + continue; // It's missing, repeated, or an empty byte array. + } + if (_isMapField(fi.type)) { + result['${fi.tagNumber}'] = + _writeMap(value, fi as MapFieldInfo); + continue; + } + result['${fi.tagNumber}'] = convertToMap(value, fi.type); + } + if (fs._hasExtensions) { + for (var tagNumber in _sorted(fs._extensions!._tagNumbers)) { + var value = fs._extensions!._values[tagNumber]; + if (value is List && value.isEmpty) { + continue; // It's repeated or an empty byte array. + } + var fi = fs._extensions!._getInfoOrNull(tagNumber)!; + result['$tagNumber'] = convertToMap(value, fi.type); + } + } + return result; +} From d23cc3fe68dedf776479045989520d45bf6ab293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Tue, 21 Jun 2022 12:25:58 +0200 Subject: [PATCH 03/18] Use jsontool to generate proto3 JSON objects and strings --- protobuf/lib/meta.dart | 2 + protobuf/lib/protobuf.dart | 1 + protobuf/lib/src/protobuf/builder_info.dart | 12 +- .../lib/src/protobuf/generated_message.dart | 50 ++++- .../lib/src/protobuf/mixins/well_known.dart | 141 ++++++------ .../lib/src/protobuf/proto3_json_writer.dart | 206 ++++++++++-------- protobuf/pubspec.yaml | 3 +- protoc_plugin/lib/src/message_generator.dart | 2 +- 8 files changed, 251 insertions(+), 166 deletions(-) diff --git a/protobuf/lib/meta.dart b/protobuf/lib/meta.dart index 519bbd052..adf6692c6 100644 --- a/protobuf/lib/meta.dart +++ b/protobuf/lib/meta.dart @@ -52,6 +52,8 @@ const GeneratedMessage_reservedNames = [ 'toBuilder', 'toDebugString', 'toProto3Json', + 'toProto3JsonSink', + 'toProto3JsonString', 'toString', 'unknownFields', 'writeToBuffer', diff --git a/protobuf/lib/protobuf.dart b/protobuf/lib/protobuf.dart index 76d329f0a..d75ce8c7c 100644 --- a/protobuf/lib/protobuf.dart +++ b/protobuf/lib/protobuf.dart @@ -11,6 +11,7 @@ import 'dart:math' as math; import 'dart:typed_data' show TypedData, Uint8List, ByteData, Endian; import 'package:fixnum/fixnum.dart' show Int64; +import 'package:jsontool/jsontool.dart'; import 'package:meta/meta.dart' show UseResult; import 'src/protobuf/json_parsing_context.dart'; diff --git a/protobuf/lib/src/protobuf/builder_info.dart b/protobuf/lib/src/protobuf/builder_info.dart index 430bbb94c..46e6df481 100644 --- a/protobuf/lib/src/protobuf/builder_info.dart +++ b/protobuf/lib/src/protobuf/builder_info.dart @@ -41,17 +41,21 @@ class BuilderInfo { List? _sortedByTag; - // For well-known types. - final Object? Function(GeneratedMessage message, TypeRegistry typeRegistry)? - toProto3Json; + /// JSON generator for well-known types. + final void Function( + GeneratedMessage msg, TypeRegistry typeRegistry, JsonSink jsonSink)? + writeToProto3JsonSink; + + /// JSON parser for well-known types. final Function(GeneratedMessage targetMessage, Object json, TypeRegistry typeRegistry, JsonParsingContext context)? fromProto3Json; + final CreateBuilderFunc? createEmptyInstance; BuilderInfo(String? messageName, {PackageName package = const PackageName(''), this.createEmptyInstance, - this.toProto3Json, + this.writeToProto3JsonSink, this.fromProto3Json}) : qualifiedMessageName = '${package.prefix}$messageName'; diff --git a/protobuf/lib/src/protobuf/generated_message.dart b/protobuf/lib/src/protobuf/generated_message.dart index 6b5c9714b..169cf55e4 100644 --- a/protobuf/lib/src/protobuf/generated_message.dart +++ b/protobuf/lib/src/protobuf/generated_message.dart @@ -208,22 +208,54 @@ abstract class GeneratedMessage { /// For the proto3 JSON format use: [toProto3Json]. String writeToJson() => jsonEncode(writeToJsonMap()); - /// Returns an Object representing Proto3 JSON serialization of `this`. + /// Returns Dart JSON object encoding this message following proto3 JSON + /// format. + /// + /// The returned object will have the same format as objects returned by + /// [jsonEncode]. + /// + /// See [toProto3JsonSink] for details. + Object? toProto3Json( + {TypeRegistry typeRegistry = const TypeRegistry.empty()}) { + Object? object; + final objectSink = jsonObjectWriter((newObject) { + object = newObject; + }); + _writeToProto3JsonSink(_fieldSet, typeRegistry, objectSink); + return object; + } + + /// Returns a proto3 JSON string encoding this message. + /// + /// See [toProto3JsonSink] for details. + String toProto3JsonString( + {TypeRegistry typeRegistry = const TypeRegistry.empty()}) { + final buf = StringBuffer(); + final stringSink = jsonStringWriter(buf); + toProto3JsonSink(typeRegistry, stringSink); + return buf.toString(); + } + + /// Writes proto3 JSON serialization of this message to the given [JsonSink]. /// /// The key for each field is be the camel-cased name of the field. /// - /// Well-known types and their special JSON encoding are supported. - /// If a well-known type cannot be encoded (eg. a `google.protobuf.Timestamp` - /// with negative `nanoseconds`) an error is thrown. + /// Well-known types and their special JSON encoding are supported. If a + /// well-known type cannot be encoded (eg. a `google.protobuf.Timestamp` with + /// negative `nanoseconds`) an error is thrown. /// /// Extensions and unknown fields are not encoded. /// /// The [typeRegistry] is be used for encoding `Any` messages. If an `Any` - /// message encoding a type not in [typeRegistry] is encountered, an - /// error is thrown. - Object? toProto3Json( - {TypeRegistry typeRegistry = const TypeRegistry.empty()}) => - _writeToProto3Json(_fieldSet, typeRegistry); + /// message encoding a type not in [typeRegistry] is encountered, an error is + /// thrown. + /// + /// The [newMessage] argument is for use in generated code, do not use. + void toProto3JsonSink(TypeRegistry typeRegistry, JsonSink jsonSink, + {bool newMessage = true}) { + _writeToProto3JsonSink(_fieldSet, typeRegistry, jsonSink, + newMessage: newMessage); + } /// Merges field values from [json], a JSON object using proto3 encoding. /// diff --git a/protobuf/lib/src/protobuf/mixins/well_known.dart b/protobuf/lib/src/protobuf/mixins/well_known.dart index 66854c9eb..f89388ed0 100644 --- a/protobuf/lib/src/protobuf/mixins/well_known.dart +++ b/protobuf/lib/src/protobuf/mixins/well_known.dart @@ -5,6 +5,7 @@ import 'dart:convert'; import 'package:fixnum/fixnum.dart'; +import 'package:jsontool/jsontool.dart'; import '../../../protobuf.dart'; import '../json_parsing_context.dart'; @@ -76,23 +77,25 @@ abstract class AnyMixin implements GeneratedMessage { // "@type": "type.googleapis.com/google.protobuf.Duration", // "value": "1.212s" // } - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { var any = message as AnyMixin; - var info = typeRegistry.lookup(_typeNameFromUrl(any.typeUrl)); + final info = typeRegistry.lookup(_typeNameFromUrl(any.typeUrl)); if (info == null) { throw ArgumentError( 'The type of the Any message (${any.typeUrl}) is not in the given typeRegistry.'); } - var unpacked = info.createEmptyInstance!()..mergeFromBuffer(any.value); - var proto3Json = unpacked.toProto3Json(typeRegistry: typeRegistry); - if (info.toProto3Json == null) { - var map = proto3Json as Map; - map['@type'] = any.typeUrl; - return map; + final unpacked = info.createEmptyInstance!()..mergeFromBuffer(any.value); + jsonSink.startObject(); + jsonSink.addKey('@type'); + jsonSink.addString(any.typeUrl); + if (info.writeToProto3JsonSink == null) { + unpacked.toProto3JsonSink(typeRegistry, jsonSink, newMessage: false); } else { - return {'@type': any.typeUrl, 'value': proto3Json}; + jsonSink.addKey('value'); + unpacked.toProto3JsonSink(typeRegistry, jsonSink, newMessage: true); } + jsonSink.endObject(); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, @@ -191,8 +194,8 @@ abstract class TimestampMixin { // // For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past // 01:30 UTC on January 15, 2017. - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { var timestamp = message as TimestampMixin; var dateTime = timestamp.toDateTime(); @@ -225,7 +228,7 @@ abstract class TimestampMixin { .padLeft(9, '0') .replaceFirst(finalGroupsOfThreeZeroes, ''); } - return '$y-$m-${d}T$h:$min:$sec${secFrac}Z'; + jsonSink.addString('$y-$m-${d}T$h:$min:$sec${secFrac}Z'); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, @@ -268,8 +271,8 @@ abstract class DurationMixin { static final RegExp finalZeroes = RegExp(r'0+$'); - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { + static toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { var duration = message as DurationMixin; var secFrac = duration.nanos // nanos and seconds should always have the same sign. @@ -278,7 +281,7 @@ abstract class DurationMixin { .padLeft(9, '0') .replaceFirst(finalZeroes, ''); var secPart = secFrac == '' ? '' : '.$secFrac'; - return '${duration.seconds}${secPart}s'; + jsonSink.addString('${duration.seconds}${secPart}s'); } static final RegExp durationPattern = RegExp(r'(-?\d*)(?:\.(\d*))?s$'); @@ -312,11 +315,16 @@ abstract class StructMixin implements GeneratedMessage { // From google/protobuf/struct.proto: // The JSON representation for `Struct` is JSON object. - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { var struct = message as StructMixin; - return struct.fields.map((key, value) => - MapEntry(key, ValueMixin.toProto3JsonHelper(value, typeRegistry))); + + jsonSink.startObject(); + for (var entry in struct.fields.entries) { + jsonSink.addKey(entry.key); + ValueMixin.toProto3JsonHelper(entry.value, typeRegistry, jsonSink); + } + jsonSink.endObject(); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, @@ -370,23 +378,25 @@ abstract class ValueMixin implements GeneratedMessage { // From google/protobuf/struct.proto: // The JSON representation for `Value` is JSON value - static Object? toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { var value = message as ValueMixin; // This would ideally be a switch, but we cannot import the enum we are // switching over. if (value.hasNullValue()) { - return null; + jsonSink.addNull(); } else if (value.hasNumberValue()) { - return value.numberValue; + jsonSink.addNumber(value.numberValue); } else if (value.hasStringValue()) { - return value.stringValue; + jsonSink.addString(value.stringValue); } else if (value.hasBoolValue()) { - return value.boolValue; + jsonSink.addBool(value.boolValue); } else if (value.hasStructValue()) { - return StructMixin.toProto3JsonHelper(value.structValue, typeRegistry); + return StructMixin.toProto3JsonHelper( + value.structValue, typeRegistry, jsonSink); } else if (value.hasListValue()) { - return ListValueMixin.toProto3JsonHelper(value.listValue, typeRegistry); + return ListValueMixin.toProto3JsonHelper( + value.listValue, typeRegistry, jsonSink); } else { throw ArgumentError('Serializing google.protobuf.Value with no value'); } @@ -429,12 +439,14 @@ abstract class ListValueMixin implements GeneratedMessage { // From google/protobuf/struct.proto: // The JSON representation for `ListValue` is JSON array. - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { - var list = message as ListValueMixin; - return list.values - .map((value) => ValueMixin.toProto3JsonHelper(value, typeRegistry)) - .toList(); + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { + final list = message as ListValueMixin; + jsonSink.startArray(); + for (final value in list.values) { + ValueMixin.toProto3JsonHelper(value, typeRegistry, jsonSink); + } + jsonSink.endArray(); } static const _valueFieldTagNumber = 1; @@ -467,8 +479,8 @@ abstract class FieldMaskMixin { // In JSON, a field mask is encoded as a single string where paths are // separated by a comma. Fields name in each path are converted // to/from lower-camel naming conventions. - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { var fieldMask = message as FieldMaskMixin; for (var path in fieldMask.paths) { if (path.contains(RegExp('[A-Z]|_[^a-z]'))) { @@ -476,7 +488,7 @@ abstract class FieldMaskMixin { 'Bad fieldmask $path. Does not round-trip to json.'); } } - return fieldMask.paths.map(_toCamelCase).join(','); + jsonSink.addString(fieldMask.paths.map(_toCamelCase).join(',')); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, @@ -516,9 +528,9 @@ abstract class DoubleValueMixin { // From google/protobuf/wrappers.proto: // The JSON representation for `DoubleValue` is JSON number. - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { - return (message as DoubleValueMixin).value; + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { + return jsonSink.addNumber((message as DoubleValueMixin).value); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, @@ -542,9 +554,9 @@ abstract class FloatValueMixin { // From google/protobuf/wrappers.proto: // The JSON representation for `FloatValue` is JSON number. - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { - return (message as FloatValueMixin).value; + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { + jsonSink.addNumber((message as FloatValueMixin).value); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, @@ -568,9 +580,9 @@ abstract class Int64ValueMixin { // From google/protobuf/wrappers.proto: // The JSON representation for `Int64Value` is JSON string. - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { - return (message as Int64ValueMixin).value.toString(); + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { + jsonSink.addString((message as Int64ValueMixin).value.toString()); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, @@ -596,9 +608,9 @@ abstract class UInt64ValueMixin { // From google/protobuf/wrappers.proto: // The JSON representation for `UInt64Value` is JSON string. - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { - return (message as UInt64ValueMixin).value.toStringUnsigned(); + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { + jsonSink.addString((message as UInt64ValueMixin).value.toStringUnsigned()); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, @@ -625,9 +637,9 @@ abstract class Int32ValueMixin { // From google/protobuf/wrappers.proto: // The JSON representation for `Int32Value` is JSON number. - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { - return (message as Int32ValueMixin).value; + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { + jsonSink.addNumber((message as Int32ValueMixin).value); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, @@ -648,9 +660,10 @@ abstract class Int32ValueMixin { abstract class UInt32ValueMixin { int get value; set value(int value); - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { - return (message as UInt32ValueMixin).value; + + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { + jsonSink.addNumber((message as UInt32ValueMixin).value); } // From google/protobuf/wrappers.proto: @@ -676,9 +689,9 @@ abstract class BoolValueMixin { // From google/protobuf/wrappers.proto: // The JSON representation for `BoolValue` is JSON `true` and `false` - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { - return (message as BoolValueMixin).value; + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { + jsonSink.addBool((message as BoolValueMixin).value); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, @@ -697,9 +710,9 @@ abstract class StringValueMixin { // From google/protobuf/wrappers.proto: // The JSON representation for `StringValue` is JSON string. - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { - return (message as StringValueMixin).value; + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { + jsonSink.addString((message as StringValueMixin).value); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, @@ -718,9 +731,9 @@ abstract class BytesValueMixin { // From google/protobuf/wrappers.proto: // The JSON representation for `BytesValue` is JSON string. - static Object toProto3JsonHelper( - GeneratedMessage message, TypeRegistry typeRegistry) { - return base64.encode((message as BytesValueMixin).value); + static void toProto3JsonHelper( + GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { + jsonSink.addString(base64.encode((message as BytesValueMixin).value)); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, diff --git a/protobuf/lib/src/protobuf/proto3_json_writer.dart b/protobuf/lib/src/protobuf/proto3_json_writer.dart index 4e3daaec4..0296245d3 100644 --- a/protobuf/lib/src/protobuf/proto3_json_writer.dart +++ b/protobuf/lib/src/protobuf/proto3_json_writer.dart @@ -4,108 +4,140 @@ part of protobuf; -Object? _writeToProto3Json(_FieldSet fs, TypeRegistry typeRegistry) { - String? convertToMapKey(dynamic key, int keyType) { - var baseType = PbFieldType._baseType(keyType); +void _writeToProto3JsonSink( + _FieldSet fs, TypeRegistry typeRegistry, JsonSink jsonSink, + {bool newMessage = true}) { + final wellKnownConverter = fs._meta.writeToProto3JsonSink; + if (wellKnownConverter != null) { + wellKnownConverter(fs._message!, typeRegistry, jsonSink); + return; + } - assert(!_isRepeated(keyType)); + if (newMessage) { + jsonSink.startObject(); // start message + } + for (var fieldInfo in fs._infosSortedByTag) { + var value = fs._values[fieldInfo.index!]; + + if (value == null || (value is List && value.isEmpty)) { + continue; // It's missing, repeated, or an empty byte array. + } + + jsonSink.addKey(fieldInfo.name); + + if (fieldInfo.isMapField) { + jsonSink.startObject(); // start map field + final mapEntryInfo = fieldInfo as MapFieldInfo; + for (var entry in (value as PbMap).entries) { + final key = entry.key; + final value = entry.value; + _writeMapKey(key, mapEntryInfo.keyFieldType, jsonSink); + _writeFieldValue( + value, mapEntryInfo.valueFieldType, jsonSink, typeRegistry); + } + jsonSink.endObject(); // end map field + } else if (fieldInfo.isRepeated) { + jsonSink.startArray(); // start repeated field + for (final element in value as PbList) { + _writeFieldValue(element, fieldInfo.type, jsonSink, typeRegistry); + } + jsonSink.endArray(); // end repeated field + } else { + _writeFieldValue(value, fieldInfo.type, jsonSink, typeRegistry); + } + } + + if (newMessage) { + jsonSink.endObject(); // end message + } +} + +void _writeMapKey(dynamic key, int keyType, JsonSink jsonSink) { + var baseType = PbFieldType._baseType(keyType); + + assert(!_isRepeated(keyType)); + + switch (baseType) { + case PbFieldType._BOOL_BIT: + jsonSink.addKey((key as bool).toString()); + break; + case PbFieldType._STRING_BIT: + jsonSink.addKey(key as String); + break; + case PbFieldType._UINT64_BIT: + jsonSink.addKey((key as Int64).toStringUnsigned().toString()); + break; + case PbFieldType._INT32_BIT: + case PbFieldType._SINT32_BIT: + case PbFieldType._UINT32_BIT: + case PbFieldType._FIXED32_BIT: + case PbFieldType._SFIXED32_BIT: + case PbFieldType._INT64_BIT: + case PbFieldType._SINT64_BIT: + case PbFieldType._SFIXED64_BIT: + case PbFieldType._FIXED64_BIT: + jsonSink.addKey(key.toString()); + break; + default: + throw StateError('Not a valid key type $keyType'); + } +} + +void _writeFieldValue(dynamic fieldValue, int fieldType, JsonSink jsonSink, + TypeRegistry typeRegistry) { + if (fieldValue == null) { + jsonSink.addNull(); + return; + } + + if (_isGroupOrMessage(fieldType)) { + _writeToProto3JsonSink( + (fieldValue as GeneratedMessage)._fieldSet, typeRegistry, jsonSink); + } else if (_isEnum(fieldType)) { + jsonSink.addString((fieldValue as ProtobufEnum).name); + } else { + final baseType = PbFieldType._baseType(fieldType); switch (baseType) { case PbFieldType._BOOL_BIT: - return key ? 'true' : 'false'; + jsonSink.addBool(fieldValue); + break; case PbFieldType._STRING_BIT: - return key; - case PbFieldType._UINT64_BIT: - return (key as Int64).toStringUnsigned(); + jsonSink.addString(fieldValue); + break; case PbFieldType._INT32_BIT: case PbFieldType._SINT32_BIT: case PbFieldType._UINT32_BIT: case PbFieldType._FIXED32_BIT: case PbFieldType._SFIXED32_BIT: + jsonSink.addNumber(fieldValue); + break; case PbFieldType._INT64_BIT: case PbFieldType._SINT64_BIT: case PbFieldType._SFIXED64_BIT: case PbFieldType._FIXED64_BIT: - return key.toString(); + jsonSink.addString(fieldValue.toString()); + break; + case PbFieldType._FLOAT_BIT: + case PbFieldType._DOUBLE_BIT: + double value = fieldValue; + if (value.isNaN) { + jsonSink.addString(nan); + } else if (value.isInfinite) { + jsonSink.addString(value.isNegative ? negativeInfinity : infinity); + } else { + jsonSink.addNumber(value); + } + break; + case PbFieldType._UINT64_BIT: + jsonSink.addString((fieldValue as Int64).toStringUnsigned()); + break; + case PbFieldType._BYTES_BIT: + jsonSink.addString(base64Encode(fieldValue)); + break; default: - throw StateError('Not a valid key type $keyType'); - } - } - - Object? valueToProto3Json(dynamic fieldValue, int? fieldType) { - if (fieldValue == null) return null; - - if (_isGroupOrMessage(fieldType!)) { - return _writeToProto3Json( - (fieldValue as GeneratedMessage)._fieldSet, typeRegistry); - } else if (_isEnum(fieldType)) { - return (fieldValue as ProtobufEnum).name; - } else { - var baseType = PbFieldType._baseType(fieldType); - switch (baseType) { - case PbFieldType._BOOL_BIT: - return fieldValue ? true : false; - case PbFieldType._STRING_BIT: - return fieldValue; - case PbFieldType._INT32_BIT: - case PbFieldType._SINT32_BIT: - case PbFieldType._UINT32_BIT: - case PbFieldType._FIXED32_BIT: - case PbFieldType._SFIXED32_BIT: - return fieldValue; - case PbFieldType._INT64_BIT: - case PbFieldType._SINT64_BIT: - case PbFieldType._SFIXED64_BIT: - case PbFieldType._FIXED64_BIT: - return fieldValue.toString(); - case PbFieldType._FLOAT_BIT: - case PbFieldType._DOUBLE_BIT: - double value = fieldValue; - if (value.isNaN) { - return nan; - } - if (value.isInfinite) { - return value.isNegative ? negativeInfinity : infinity; - } - return value; - case PbFieldType._UINT64_BIT: - return (fieldValue as Int64).toStringUnsigned(); - case PbFieldType._BYTES_BIT: - return base64Encode(fieldValue); - default: - throw StateError( - 'Invariant violation: unexpected value type $fieldType'); - } - } - } - - final meta = fs._meta; - if (meta.toProto3Json != null) { - return meta.toProto3Json!(fs._message!, typeRegistry); - } - - var result = {}; - for (var fieldInfo in fs._infosSortedByTag) { - var value = fs._values[fieldInfo.index!]; - if (value == null || (value is List && value.isEmpty)) { - continue; // It's missing, repeated, or an empty byte array. - } - dynamic jsonValue; - if (fieldInfo.isMapField) { - jsonValue = (value as PbMap).map((key, entryValue) { - var mapEntryInfo = fieldInfo as MapFieldInfo; - return MapEntry(convertToMapKey(key, mapEntryInfo.keyFieldType), - valueToProto3Json(entryValue, mapEntryInfo.valueFieldType)); - }); - } else if (fieldInfo.isRepeated) { - jsonValue = (value as PbList) - .map((element) => valueToProto3Json(element, fieldInfo.type)) - .toList(); - } else { - jsonValue = valueToProto3Json(value, fieldInfo.type); + throw StateError( + 'Invariant violation: unexpected value type $fieldType'); } - result[fieldInfo.name] = jsonValue; } - // Extensions and unknown fields are not encoded by proto3 JSON. - return result; } diff --git a/protobuf/pubspec.yaml b/protobuf/pubspec.yaml index b474ba6bc..3d9e8f110 100644 --- a/protobuf/pubspec.yaml +++ b/protobuf/pubspec.yaml @@ -9,8 +9,9 @@ environment: sdk: '>=2.12.0 <3.0.0' dependencies: - fixnum: ^1.0.0 collection: ^1.15.0 + fixnum: ^1.0.0 + jsontool: ^1.1.2 meta: ^1.7.0 dev_dependencies: diff --git a/protoc_plugin/lib/src/message_generator.dart b/protoc_plugin/lib/src/message_generator.dart index a2efc02e9..b396474bf 100644 --- a/protoc_plugin/lib/src/message_generator.dart +++ b/protoc_plugin/lib/src/message_generator.dart @@ -304,7 +304,7 @@ class MessageGenerator extends ProtobufContainer { var packageClause = package == '' ? '' : ', package: $conditionalPackageName'; var proto3JsonClause = (mixin?.hasProto3JsonHelpers ?? false) - ? ', toProto3Json: $mixinImportPrefix.${mixin!.name}.toProto3JsonHelper, ' + ? ', writeToProto3JsonSink: $mixinImportPrefix.${mixin!.name}.toProto3JsonHelper, ' 'fromProto3Json: $mixinImportPrefix.${mixin!.name}.fromProto3JsonHelper' : ''; out.addAnnotatedBlock( From 704bcc169e29cda78886a13ad2c4125c56ddf6d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Tue, 21 Jun 2022 13:08:05 +0200 Subject: [PATCH 04/18] Update benchmark --- protobuf/benchmarks/common.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protobuf/benchmarks/common.dart b/protobuf/benchmarks/common.dart index ccda311b8..e26a49718 100644 --- a/protobuf/benchmarks/common.dart +++ b/protobuf/benchmarks/common.dart @@ -231,7 +231,7 @@ class ToProto3JsonStringBenchmark extends _ProtobufBenchmark { void run() { for (final ds in datasets) { for (final unpacked in ds.unpacked) { - jsonEncode(unpacked.toProto3Json()); + unpacked.toProto3JsonString(); } } } From e04e346b21349e31f9fc4d5bf7888f90241ff5e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Tue, 21 Jun 2022 13:45:18 +0200 Subject: [PATCH 05/18] Use jsontool in custom JSON format --- protobuf/benchmarks/common.dart | 23 ++++- protobuf/lib/meta.dart | 1 + .../lib/src/protobuf/generated_message.dart | 20 ++++- protobuf/lib/src/protobuf/json_reader.dart | 4 +- protobuf/lib/src/protobuf/json_writer.dart | 90 ++++++++++++++----- protobuf/lib/src/protobuf/pb_map.dart | 2 + 6 files changed, 108 insertions(+), 32 deletions(-) diff --git a/protobuf/benchmarks/common.dart b/protobuf/benchmarks/common.dart index e26a49718..c9c777830 100644 --- a/protobuf/benchmarks/common.dart +++ b/protobuf/benchmarks/common.dart @@ -193,9 +193,9 @@ class FromJsonBenchmark extends _ProtobufBenchmark { } } -/// JSON serialization benchmark. -class ToJsonBenchmark extends _ProtobufBenchmark { - ToJsonBenchmark(datasets) : super(datasets, 'ToJson'); +/// JSON serialization benchmark: from message to JSON string. +class ToJsonStringBenchmark extends _ProtobufBenchmark { + ToJsonStringBenchmark(datasets) : super(datasets, 'ToJsonString'); @override void run() { @@ -207,6 +207,20 @@ class ToJsonBenchmark extends _ProtobufBenchmark { } } +/// JSON serialization benchmark: from message to JSON object. +class ToJsonObjectBenchmark extends _ProtobufBenchmark { + ToJsonObjectBenchmark(datasets) : super(datasets, 'ToJsonObject'); + + @override + void run() { + for (final ds in datasets) { + for (final unpacked in ds.unpacked) { + unpacked.writeToJsonMap(); + } + } + } +} + /// proto3 JSON deserialization benchmark: from JSON string to message. class FromProto3JsonStringBenchmark extends _ProtobufBenchmark { FromProto3JsonStringBenchmark(datasets) @@ -285,7 +299,8 @@ void run(List datasets) { FromBinaryBenchmark(datasets).report(); ToBinaryBenchmark(datasets).report(); FromJsonBenchmark(datasets).report(); - ToJsonBenchmark(datasets).report(); + ToJsonStringBenchmark(datasets).report(); + ToJsonObjectBenchmark(datasets).report(); FromProto3JsonStringBenchmark(datasets).report(); ToProto3JsonStringBenchmark(datasets).report(); FromProto3JsonObjectBenchmark(datasets).report(); diff --git a/protobuf/lib/meta.dart b/protobuf/lib/meta.dart index adf6692c6..2a3936578 100644 --- a/protobuf/lib/meta.dart +++ b/protobuf/lib/meta.dart @@ -60,6 +60,7 @@ const GeneratedMessage_reservedNames = [ 'writeToCodedBufferWriter', 'writeToJson', 'writeToJsonMap', + 'writeToJsonSink', r'$_ensure', r'$_get', r'$_getI64', diff --git a/protobuf/lib/src/protobuf/generated_message.dart b/protobuf/lib/src/protobuf/generated_message.dart index 169cf55e4..634ab8872 100644 --- a/protobuf/lib/src/protobuf/generated_message.dart +++ b/protobuf/lib/src/protobuf/generated_message.dart @@ -191,7 +191,14 @@ abstract class GeneratedMessage { /// Returns the JSON encoding of this message as a Dart [Map]. /// /// The encoding is described in [GeneratedMessage.writeToJson]. - Map writeToJsonMap() => _writeToJsonMap(_fieldSet); + Map writeToJsonMap() { + Object? object; + final objectSink = jsonObjectWriter((newObject) { + object = newObject; + }); + writeToJsonSink(objectSink); + return object as Map; + } /// Returns a JSON string that encodes this message. /// @@ -206,7 +213,16 @@ abstract class GeneratedMessage { /// represented as their integer value. /// /// For the proto3 JSON format use: [toProto3Json]. - String writeToJson() => jsonEncode(writeToJsonMap()); + String writeToJson() { + final buf = StringBuffer(); + final stringSink = jsonStringWriter(buf); + writeToJsonSink(stringSink); + return buf.toString(); + } + + void writeToJsonSink(JsonSink jsonSink) { + _writeToJsonMapSink(_fieldSet, jsonSink); + } /// Returns Dart JSON object encoding this message following proto3 JSON /// format. diff --git a/protobuf/lib/src/protobuf/json_reader.dart b/protobuf/lib/src/protobuf/json_reader.dart index 30039972f..551e7c126 100644 --- a/protobuf/lib/src/protobuf/json_reader.dart +++ b/protobuf/lib/src/protobuf/json_reader.dart @@ -57,14 +57,14 @@ void _appendJsonMap(BuilderInfo meta, _FieldSet fs, List jsonList, final convertedKey = _convertJsonValue( entryMeta, entryFieldSet, - jsonEntry['${PbMap._keyFieldNumber}'], + jsonEntry[PbMap._keyFieldNumberString], PbMap._keyFieldNumber, fi.keyFieldType, registry); var convertedValue = _convertJsonValue( entryMeta, entryFieldSet, - jsonEntry['${PbMap._valueFieldNumber}'], + jsonEntry[PbMap._valueFieldNumberString], PbMap._valueFieldNumber, fi.valueFieldType, registry); diff --git a/protobuf/lib/src/protobuf/json_writer.dart b/protobuf/lib/src/protobuf/json_writer.dart index 00418558a..6b5038703 100644 --- a/protobuf/lib/src/protobuf/json_writer.dart +++ b/protobuf/lib/src/protobuf/json_writer.dart @@ -4,76 +4,116 @@ part of protobuf; -Map _writeToJsonMap(_FieldSet fs) { +void _writeToJsonMapSink(_FieldSet fs, JsonSink jsonSink) { dynamic convertToMap(dynamic fieldValue, int fieldType) { var baseType = PbFieldType._baseType(fieldType); if (_isRepeated(fieldType)) { - return List.from(fieldValue.map((e) => convertToMap(e, baseType))); + jsonSink.startArray(); + for (final value in fieldValue as List) { + convertToMap(value, baseType); + } + jsonSink.endArray(); + return; } switch (baseType) { case PbFieldType._BOOL_BIT: + jsonSink.addBool(fieldValue); + return; + case PbFieldType._STRING_BIT: + jsonSink.addString(fieldValue); + return; + case PbFieldType._INT32_BIT: case PbFieldType._SINT32_BIT: case PbFieldType._UINT32_BIT: case PbFieldType._FIXED32_BIT: case PbFieldType._SFIXED32_BIT: - return fieldValue; + jsonSink.addNumber(fieldValue); + return; + case PbFieldType._FLOAT_BIT: case PbFieldType._DOUBLE_BIT: final value = fieldValue as double; if (value.isNaN) { - return nan; + jsonSink.addString(nan); + return; } if (value.isInfinite) { - return value.isNegative ? negativeInfinity : infinity; + jsonSink.addString(value.isNegative ? negativeInfinity : infinity); + return; } if (fieldValue.toInt() == fieldValue) { - return fieldValue.toInt(); + jsonSink.addNumber(fieldValue.toInt()); + return; } - return value; + jsonSink.addNumber(value); + return; + case PbFieldType._BYTES_BIT: // Encode 'bytes' as a base64-encoded string. - return base64Encode(fieldValue as List); + jsonSink.addString(base64Encode(fieldValue as List)); + return; + case PbFieldType._ENUM_BIT: - return fieldValue.value; // assume |value| < 2^52 + jsonSink.addNumber(fieldValue.value); // assume |value| < 2^52 + return; + case PbFieldType._INT64_BIT: case PbFieldType._SINT64_BIT: case PbFieldType._SFIXED64_BIT: - return fieldValue.toString(); + jsonSink.addString(fieldValue.toString()); + return; + case PbFieldType._UINT64_BIT: case PbFieldType._FIXED64_BIT: - return fieldValue.toStringUnsigned(); + jsonSink.addString(fieldValue.toStringUnsigned()); + return; + case PbFieldType._GROUP_BIT: case PbFieldType._MESSAGE_BIT: - return fieldValue.writeToJsonMap(); + (fieldValue as GeneratedMessage).writeToJsonSink(jsonSink); + return; + default: throw 'Unknown type $fieldType'; } } - List _writeMap(dynamic fieldValue, MapFieldInfo fi) => - List.from(fieldValue.entries.map((MapEntry e) => { - '${PbMap._keyFieldNumber}': convertToMap(e.key, fi.keyFieldType), - '${PbMap._valueFieldNumber}': - convertToMap(e.value, fi.valueFieldType) - })); + void _writeMap(dynamic fieldValue, MapFieldInfo fi) { + jsonSink.startArray(); + for (final e in fieldValue.entries) { + jsonSink.startObject(); + + jsonSink.addKey(PbMap._keyFieldNumberString); + convertToMap(e.key, fi.keyFieldType); + + jsonSink.addKey(PbMap._valueFieldNumberString); + convertToMap(e.value, fi.valueFieldType); + + jsonSink.endObject(); + } + jsonSink.endArray(); + } + + jsonSink.startObject(); - var result = {}; for (var fi in fs._infosSortedByTag) { var value = fs._values[fi.index!]; if (value == null || (value is List && value.isEmpty)) { continue; // It's missing, repeated, or an empty byte array. } if (_isMapField(fi.type)) { - result['${fi.tagNumber}'] = - _writeMap(value, fi as MapFieldInfo); + jsonSink.addKey(fi.tagNumber.toString()); + _writeMap(value, fi as MapFieldInfo); continue; } - result['${fi.tagNumber}'] = convertToMap(value, fi.type); + jsonSink.addKey(fi.tagNumber.toString()); + convertToMap(value, fi.type); } + if (fs._hasExtensions) { for (var tagNumber in _sorted(fs._extensions!._tagNumbers)) { var value = fs._extensions!._values[tagNumber]; @@ -81,8 +121,10 @@ Map _writeToJsonMap(_FieldSet fs) { continue; // It's repeated or an empty byte array. } var fi = fs._extensions!._getInfoOrNull(tagNumber)!; - result['$tagNumber'] = convertToMap(value, fi.type); + jsonSink.addKey(tagNumber.toString()); + convertToMap(value, fi.type); } } - return result; + + jsonSink.endObject(); } diff --git a/protobuf/lib/src/protobuf/pb_map.dart b/protobuf/lib/src/protobuf/pb_map.dart index 712f8127a..93a79da28 100644 --- a/protobuf/lib/src/protobuf/pb_map.dart +++ b/protobuf/lib/src/protobuf/pb_map.dart @@ -18,7 +18,9 @@ class PbMap extends MapBase { final int valueFieldType; static const int _keyFieldNumber = 1; + static const String _keyFieldNumberString = '1'; static const int _valueFieldNumber = 2; + static const String _valueFieldNumberString = '2'; final Map _wrappedMap; From 739d6402adc613c8c4af2613dfc143b568eeb21c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Tue, 21 Jun 2022 14:00:43 +0200 Subject: [PATCH 06/18] Remove redundant `return`s --- protobuf/lib/src/protobuf/mixins/well_known.dart | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/protobuf/lib/src/protobuf/mixins/well_known.dart b/protobuf/lib/src/protobuf/mixins/well_known.dart index f89388ed0..2081eabc5 100644 --- a/protobuf/lib/src/protobuf/mixins/well_known.dart +++ b/protobuf/lib/src/protobuf/mixins/well_known.dart @@ -392,10 +392,9 @@ abstract class ValueMixin implements GeneratedMessage { } else if (value.hasBoolValue()) { jsonSink.addBool(value.boolValue); } else if (value.hasStructValue()) { - return StructMixin.toProto3JsonHelper( - value.structValue, typeRegistry, jsonSink); + StructMixin.toProto3JsonHelper(value.structValue, typeRegistry, jsonSink); } else if (value.hasListValue()) { - return ListValueMixin.toProto3JsonHelper( + ListValueMixin.toProto3JsonHelper( value.listValue, typeRegistry, jsonSink); } else { throw ArgumentError('Serializing google.protobuf.Value with no value'); @@ -530,7 +529,7 @@ abstract class DoubleValueMixin { // The JSON representation for `DoubleValue` is JSON number. static void toProto3JsonHelper( GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { - return jsonSink.addNumber((message as DoubleValueMixin).value); + jsonSink.addNumber((message as DoubleValueMixin).value); } static void fromProto3JsonHelper(GeneratedMessage message, Object json, From 9e3aed9ca0ec4231e35ff0716918c186b92162aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Thu, 7 Jul 2022 16:01:39 +0200 Subject: [PATCH 07/18] Post-merge fixups --- benchmarks/bin/to_proto3_json_string.dart | 8 +++----- benchmarks/pubspec.lock | 7 +++++++ protobuf/lib/src/protobuf/proto3_json_writer.dart | 13 ++++++++++--- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/benchmarks/bin/to_proto3_json_string.dart b/benchmarks/bin/to_proto3_json_string.dart index 1a3407d27..7a6247b6a 100644 --- a/benchmarks/bin/to_proto3_json_string.dart +++ b/benchmarks/bin/to_proto3_json_string.dart @@ -2,8 +2,6 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -import 'dart:convert' show jsonEncode; - import 'package:protobuf_benchmarks/benchmark_base.dart'; import 'package:protobuf_benchmarks/generated/google_message1_proto2.pb.dart' as p2; @@ -26,9 +24,9 @@ class Benchmark extends BenchmarkBase { @override void run() { - jsonEncode(_message1Proto2.toProto3Json()); - jsonEncode(_message1Proto3.toProto3Json()); - jsonEncode(_message2.toProto3Json()); + _message1Proto2.toProto3JsonString(); + _message1Proto3.toProto3JsonString(); + _message2.toProto3JsonString(); } } diff --git a/benchmarks/pubspec.lock b/benchmarks/pubspec.lock index edee7a39b..0410965b4 100644 --- a/benchmarks/pubspec.lock +++ b/benchmarks/pubspec.lock @@ -92,6 +92,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.6.4" + jsontool: + dependency: transitive + description: + name: jsontool + url: "https://pub.dartlang.org" + source: hosted + version: "1.1.2" lints: dependency: "direct dev" description: diff --git a/protobuf/lib/src/protobuf/proto3_json_writer.dart b/protobuf/lib/src/protobuf/proto3_json_writer.dart index 0296245d3..ad5505d04 100644 --- a/protobuf/lib/src/protobuf/proto3_json_writer.dart +++ b/protobuf/lib/src/protobuf/proto3_json_writer.dart @@ -123,11 +123,18 @@ void _writeFieldValue(dynamic fieldValue, int fieldType, JsonSink jsonSink, double value = fieldValue; if (value.isNaN) { jsonSink.addString(nan); - } else if (value.isInfinite) { + break; + } + if (value.isInfinite) { jsonSink.addString(value.isNegative ? negativeInfinity : infinity); - } else { - jsonSink.addNumber(value); + break; + } + final intValue = value.toInt(); + if (intValue == value) { + jsonSink.addNumber(intValue); + break; } + jsonSink.addNumber(value); break; case PbFieldType._UINT64_BIT: jsonSink.addString((fieldValue as Int64).toStringUnsigned()); From 09bd61c863be072e365df5381b76a498f23632dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Mon, 8 Aug 2022 12:50:03 +0200 Subject: [PATCH 08/18] Avoid dynamic calls --- protobuf/lib/src/protobuf/json_writer.dart | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/protobuf/lib/src/protobuf/json_writer.dart b/protobuf/lib/src/protobuf/json_writer.dart index 6b5038703..d7fdcbcde 100644 --- a/protobuf/lib/src/protobuf/json_writer.dart +++ b/protobuf/lib/src/protobuf/json_writer.dart @@ -58,7 +58,8 @@ void _writeToJsonMapSink(_FieldSet fs, JsonSink jsonSink) { return; case PbFieldType._ENUM_BIT: - jsonSink.addNumber(fieldValue.value); // assume |value| < 2^52 + final ProtobufEnum enum_ = fieldValue; + jsonSink.addNumber(enum_.value); // assume |value| < 2^52 return; case PbFieldType._INT64_BIT: @@ -69,7 +70,8 @@ void _writeToJsonMapSink(_FieldSet fs, JsonSink jsonSink) { case PbFieldType._UINT64_BIT: case PbFieldType._FIXED64_BIT: - jsonSink.addString(fieldValue.toStringUnsigned()); + final Int64 int_ = fieldValue; + jsonSink.addString(int_.toStringUnsigned()); return; case PbFieldType._GROUP_BIT: @@ -82,7 +84,7 @@ void _writeToJsonMapSink(_FieldSet fs, JsonSink jsonSink) { } } - void _writeMap(dynamic fieldValue, MapFieldInfo fi) { + void _writeMap(PbMap fieldValue, MapFieldInfo fi) { jsonSink.startArray(); for (final e in fieldValue.entries) { jsonSink.startObject(); From 5a6dec5303cb6cc73fe7a8e22a9a65ac6749392a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Fri, 9 Sep 2022 14:54:40 +0200 Subject: [PATCH 09/18] Add missing return type --- protobuf/lib/src/protobuf/mixins/well_known.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protobuf/lib/src/protobuf/mixins/well_known.dart b/protobuf/lib/src/protobuf/mixins/well_known.dart index 2081eabc5..ab74a36cb 100644 --- a/protobuf/lib/src/protobuf/mixins/well_known.dart +++ b/protobuf/lib/src/protobuf/mixins/well_known.dart @@ -271,7 +271,7 @@ abstract class DurationMixin { static final RegExp finalZeroes = RegExp(r'0+$'); - static toProto3JsonHelper( + static void toProto3JsonHelper( GeneratedMessage message, TypeRegistry typeRegistry, JsonSink jsonSink) { var duration = message as DurationMixin; var secFrac = duration.nanos From bfd311b793c13a13e5483a0a082c75373ad7966b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Tue, 13 Sep 2022 15:44:49 +0200 Subject: [PATCH 10/18] Revert some of the renamings --- protobuf/lib/src/protobuf/builder_info.dart | 4 ++-- protobuf/lib/src/protobuf/mixins/well_known.dart | 2 +- protobuf/lib/src/protobuf/proto3_json_writer.dart | 2 +- protoc_plugin/lib/src/message_generator.dart | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/protobuf/lib/src/protobuf/builder_info.dart b/protobuf/lib/src/protobuf/builder_info.dart index 80bf9a49e..c560da43b 100644 --- a/protobuf/lib/src/protobuf/builder_info.dart +++ b/protobuf/lib/src/protobuf/builder_info.dart @@ -44,7 +44,7 @@ class BuilderInfo { /// JSON generator for well-known types. final void Function( GeneratedMessage msg, TypeRegistry typeRegistry, JsonSink jsonSink)? - writeToProto3JsonSink; + toProto3Json; /// JSON parser for well-known types. final Function(GeneratedMessage targetMessage, Object json, @@ -55,7 +55,7 @@ class BuilderInfo { BuilderInfo(String? messageName, {PackageName package = const PackageName(''), this.createEmptyInstance, - this.writeToProto3JsonSink, + this.toProto3Json, this.fromProto3Json}) : qualifiedMessageName = '${package.prefix}$messageName'; diff --git a/protobuf/lib/src/protobuf/mixins/well_known.dart b/protobuf/lib/src/protobuf/mixins/well_known.dart index ab74a36cb..51e853e31 100644 --- a/protobuf/lib/src/protobuf/mixins/well_known.dart +++ b/protobuf/lib/src/protobuf/mixins/well_known.dart @@ -89,7 +89,7 @@ abstract class AnyMixin implements GeneratedMessage { jsonSink.startObject(); jsonSink.addKey('@type'); jsonSink.addString(any.typeUrl); - if (info.writeToProto3JsonSink == null) { + if (info.toProto3Json == null) { unpacked.toProto3JsonSink(typeRegistry, jsonSink, newMessage: false); } else { jsonSink.addKey('value'); diff --git a/protobuf/lib/src/protobuf/proto3_json_writer.dart b/protobuf/lib/src/protobuf/proto3_json_writer.dart index 609434fc9..498674a54 100644 --- a/protobuf/lib/src/protobuf/proto3_json_writer.dart +++ b/protobuf/lib/src/protobuf/proto3_json_writer.dart @@ -7,7 +7,7 @@ part of protobuf; void _writeToProto3JsonSink( _FieldSet fs, TypeRegistry typeRegistry, JsonSink jsonSink, {bool newMessage = true}) { - final wellKnownConverter = fs._meta.writeToProto3JsonSink; + final wellKnownConverter = fs._meta.toProto3Json; if (wellKnownConverter != null) { wellKnownConverter(fs._message!, typeRegistry, jsonSink); return; diff --git a/protoc_plugin/lib/src/message_generator.dart b/protoc_plugin/lib/src/message_generator.dart index f13e7af83..cfbcdeec1 100644 --- a/protoc_plugin/lib/src/message_generator.dart +++ b/protoc_plugin/lib/src/message_generator.dart @@ -304,7 +304,7 @@ class MessageGenerator extends ProtobufContainer { var packageClause = package == '' ? '' : ', package: $conditionalPackageName'; var proto3JsonClause = (mixin?.hasProto3JsonHelpers ?? false) - ? ', writeToProto3JsonSink: $mixinImportPrefix.${mixin!.name}.toProto3JsonHelper, ' + ? ', toProto3Json: $mixinImportPrefix.${mixin!.name}.toProto3JsonHelper, ' 'fromProto3Json: $mixinImportPrefix.${mixin!.name}.fromProto3JsonHelper' : ''; out.addAnnotatedBlock( From de8dcb1d205a6b28491dc7b07382045558c6e050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Tue, 13 Sep 2022 15:55:56 +0200 Subject: [PATCH 11/18] Update changelog --- protobuf/CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/protobuf/CHANGELOG.md b/protobuf/CHANGELOG.md index 3e59845e8..9848229bb 100644 --- a/protobuf/CHANGELOG.md +++ b/protobuf/CHANGELOG.md @@ -20,6 +20,20 @@ missing. ([#719], [#745]) * Fix updating frozen (immutable) messages with merge methods (`mergeFromBuffer`, `mergeFromProto3Json`, ...). ([#489], [#727]) +* 3 new `GeneratedMessage` methods added for JSON serialization: + + - `toProto3JsonSink`: for serializing a message as proto3 JSON to a + [jsontool] sink + - `toProto3JsonString`: for serializing a message as proto3 JSON string + - `writeToJsonSink`: for serializing a message as custom JSON format (the + format used by `writeToJsonMap`) to a [jsontool] sink + + Methods that return JSON strings (`writeToJson`, `toProto3JsonString`) now + don't allocate intermediate JSON objects and are much more efficient than + generating JSON objects (`writeToJsonMap`, `toProto3Json`) and then encoding + with `dart:convert`'s `json.encode`. + + ([#683]) [#183]: https://github.com/google/protobuf.dart/issues/183 [#644]: https://github.com/google/protobuf.dart/pull/644 @@ -37,6 +51,8 @@ [#745]: https://github.com/google/protobuf.dart/pull/745 [#489]: https://github.com/google/protobuf.dart/issues/489 [#727]: https://github.com/google/protobuf.dart/pull/727 +[jsontool]: https://pub.dev/packages/jsontool +[#683]: https://github.com/google/protobuf.dart/pull/683 ## 2.1.0 From 272e190bbdfb6dc0a3f135e257069141433999eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Thu, 15 Sep 2022 10:45:10 +0200 Subject: [PATCH 12/18] Remove explicit type casts --- protobuf/lib/src/protobuf/json_writer.dart | 23 ++++++++------ .../lib/src/protobuf/proto3_json_writer.dart | 31 ++++++++++++------- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/protobuf/lib/src/protobuf/json_writer.dart b/protobuf/lib/src/protobuf/json_writer.dart index c9aaff9b4..1c6a7f45d 100644 --- a/protobuf/lib/src/protobuf/json_writer.dart +++ b/protobuf/lib/src/protobuf/json_writer.dart @@ -9,8 +9,9 @@ void _writeToJsonMapSink(_FieldSet fs, JsonSink jsonSink) { var baseType = PbFieldType._baseType(fieldType); if (_isRepeated(fieldType)) { + final List listValue = fieldValue; jsonSink.startArray(); - for (final value in fieldValue as List) { + for (final value in listValue) { convertToMap(value, baseType); } jsonSink.endArray(); @@ -36,25 +37,27 @@ void _writeToJsonMapSink(_FieldSet fs, JsonSink jsonSink) { case PbFieldType._FLOAT_BIT: case PbFieldType._DOUBLE_BIT: - final value = fieldValue as double; - if (value.isNaN) { + final double doubleValue = fieldValue; + if (doubleValue.isNaN) { jsonSink.addString(_nan); return; } - if (value.isInfinite) { - jsonSink.addString(value.isNegative ? _negativeInfinity : _infinity); + if (doubleValue.isInfinite) { + jsonSink.addString( + doubleValue.isNegative ? _negativeInfinity : _infinity); return; } if (fieldValue.toInt() == fieldValue) { jsonSink.addNumber(fieldValue.toInt()); return; } - jsonSink.addNumber(value); + jsonSink.addNumber(doubleValue); return; case PbFieldType._BYTES_BIT: // Encode 'bytes' as a base64-encoded string. - jsonSink.addString(base64Encode(fieldValue as List)); + final List listValue = fieldValue; + jsonSink.addString(base64Encode(listValue)); return; case PbFieldType._ENUM_BIT: @@ -76,7 +79,8 @@ void _writeToJsonMapSink(_FieldSet fs, JsonSink jsonSink) { case PbFieldType._GROUP_BIT: case PbFieldType._MESSAGE_BIT: - (fieldValue as GeneratedMessage).writeToJsonSink(jsonSink); + final GeneratedMessage messageValue = fieldValue; + messageValue.writeToJsonSink(jsonSink); return; default: @@ -108,8 +112,9 @@ void _writeToJsonMapSink(_FieldSet fs, JsonSink jsonSink) { continue; // It's missing, repeated, or an empty byte array. } if (_isMapField(fi.type)) { + final MapFieldInfo mapFi = fi as dynamic; jsonSink.addKey(fi.tagNumber.toString()); - _writeMap(value, fi as MapFieldInfo); + _writeMap(value, mapFi); continue; } jsonSink.addKey(fi.tagNumber.toString()); diff --git a/protobuf/lib/src/protobuf/proto3_json_writer.dart b/protobuf/lib/src/protobuf/proto3_json_writer.dart index 498674a54..bacc9e28b 100644 --- a/protobuf/lib/src/protobuf/proto3_json_writer.dart +++ b/protobuf/lib/src/protobuf/proto3_json_writer.dart @@ -28,18 +28,20 @@ void _writeToProto3JsonSink( if (fieldInfo.isMapField) { jsonSink.startObject(); // start map field - final mapEntryInfo = fieldInfo as MapFieldInfo; - for (var entry in (value as PbMap).entries) { + final MapFieldInfo mapFieldInfo = fieldInfo as dynamic; + final Map mapValue = value; + for (var entry in mapValue.entries) { final key = entry.key; final value = entry.value; - _writeMapKey(key, mapEntryInfo.keyFieldType, jsonSink); + _writeMapKey(key, mapFieldInfo.keyFieldType, jsonSink); _writeFieldValue( - value, mapEntryInfo.valueFieldType, jsonSink, typeRegistry); + value, mapFieldInfo.valueFieldType, jsonSink, typeRegistry); } jsonSink.endObject(); // end map field } else if (fieldInfo.isRepeated) { jsonSink.startArray(); // start repeated field - for (final element in value as PbList) { + final List listValue = value; + for (final element in listValue) { _writeFieldValue(element, fieldInfo.type, jsonSink, typeRegistry); } jsonSink.endArray(); // end repeated field @@ -60,13 +62,16 @@ void _writeMapKey(dynamic key, int keyType, JsonSink jsonSink) { switch (baseType) { case PbFieldType._BOOL_BIT: - jsonSink.addKey((key as bool).toString()); + final bool boolKey = key; + jsonSink.addKey(boolKey.toString()); break; case PbFieldType._STRING_BIT: - jsonSink.addKey(key as String); + final String stringKey = key; + jsonSink.addKey(stringKey); break; case PbFieldType._UINT64_BIT: - jsonSink.addKey((key as Int64).toStringUnsigned().toString()); + final Int64 intKey = key; + jsonSink.addKey(intKey.toStringUnsigned().toString()); break; case PbFieldType._INT32_BIT: case PbFieldType._SINT32_BIT: @@ -92,10 +97,11 @@ void _writeFieldValue(dynamic fieldValue, int fieldType, JsonSink jsonSink, } if (_isGroupOrMessage(fieldType)) { - _writeToProto3JsonSink( - (fieldValue as GeneratedMessage)._fieldSet, typeRegistry, jsonSink); + final GeneratedMessage messageValue = fieldValue; + _writeToProto3JsonSink(messageValue._fieldSet, typeRegistry, jsonSink); } else if (_isEnum(fieldType)) { - jsonSink.addString((fieldValue as ProtobufEnum).name); + final ProtobufEnum enumValue = fieldValue; + jsonSink.addString(enumValue.name); } else { final baseType = PbFieldType._baseType(fieldType); switch (baseType) { @@ -137,7 +143,8 @@ void _writeFieldValue(dynamic fieldValue, int fieldType, JsonSink jsonSink, jsonSink.addNumber(value); break; case PbFieldType._UINT64_BIT: - jsonSink.addString((fieldValue as Int64).toStringUnsigned()); + final Int64 intValue = fieldValue; + jsonSink.addString(intValue.toStringUnsigned()); break; case PbFieldType._BYTES_BIT: jsonSink.addString(base64Encode(fieldValue)); From ac8da6d85e3717b9cdac01e4d2a73f9b5ca021f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Thu, 15 Sep 2022 10:49:28 +0200 Subject: [PATCH 13/18] Remove writeToJsonSink --- protobuf/lib/meta.dart | 1 - protobuf/lib/src/protobuf/generated_message.dart | 8 ++------ protobuf/lib/src/protobuf/json_writer.dart | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/protobuf/lib/meta.dart b/protobuf/lib/meta.dart index 2a3936578..adf6692c6 100644 --- a/protobuf/lib/meta.dart +++ b/protobuf/lib/meta.dart @@ -60,7 +60,6 @@ const GeneratedMessage_reservedNames = [ 'writeToCodedBufferWriter', 'writeToJson', 'writeToJsonMap', - 'writeToJsonSink', r'$_ensure', r'$_get', r'$_getI64', diff --git a/protobuf/lib/src/protobuf/generated_message.dart b/protobuf/lib/src/protobuf/generated_message.dart index 66e89fdef..446b6621c 100644 --- a/protobuf/lib/src/protobuf/generated_message.dart +++ b/protobuf/lib/src/protobuf/generated_message.dart @@ -205,7 +205,7 @@ abstract class GeneratedMessage { final objectSink = jsonObjectWriter((newObject) { object = newObject; }); - writeToJsonSink(objectSink); + _writeToJsonMapSink(_fieldSet, objectSink); return object as Map; } @@ -225,14 +225,10 @@ abstract class GeneratedMessage { String writeToJson() { final buf = StringBuffer(); final stringSink = jsonStringWriter(buf); - writeToJsonSink(stringSink); + _writeToJsonMapSink(_fieldSet, stringSink); return buf.toString(); } - void writeToJsonSink(JsonSink jsonSink) { - _writeToJsonMapSink(_fieldSet, jsonSink); - } - /// Returns Dart JSON object encoding this message following proto3 JSON /// format. /// diff --git a/protobuf/lib/src/protobuf/json_writer.dart b/protobuf/lib/src/protobuf/json_writer.dart index 1c6a7f45d..a40e23f6c 100644 --- a/protobuf/lib/src/protobuf/json_writer.dart +++ b/protobuf/lib/src/protobuf/json_writer.dart @@ -80,7 +80,7 @@ void _writeToJsonMapSink(_FieldSet fs, JsonSink jsonSink) { case PbFieldType._GROUP_BIT: case PbFieldType._MESSAGE_BIT: final GeneratedMessage messageValue = fieldValue; - messageValue.writeToJsonSink(jsonSink); + _writeToJsonMapSink(messageValue._fieldSet, jsonSink); return; default: From 2d566d5261e12c136b9565b65ab596ae27f68858 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Thu, 15 Sep 2022 14:01:27 +0200 Subject: [PATCH 14/18] Hide sink API --- protobuf/lib/meta.dart | 2 +- protobuf/lib/protobuf.dart | 3 +- .../lib/src/protobuf/generated_message.dart | 38 ++++++++----------- .../lib/src/protobuf/mixins/well_known.dart | 4 +- 4 files changed, 20 insertions(+), 27 deletions(-) diff --git a/protobuf/lib/meta.dart b/protobuf/lib/meta.dart index adf6692c6..5ad59a640 100644 --- a/protobuf/lib/meta.dart +++ b/protobuf/lib/meta.dart @@ -52,7 +52,6 @@ const GeneratedMessage_reservedNames = [ 'toBuilder', 'toDebugString', 'toProto3Json', - 'toProto3JsonSink', 'toProto3JsonString', 'toString', 'unknownFields', @@ -81,6 +80,7 @@ const GeneratedMessage_reservedNames = [ r'$_setSignedInt32', r'$_setString', r'$_setUnsignedInt32', + r'$_toProto3JsonSink', r'$_whichOneof', ]; diff --git a/protobuf/lib/protobuf.dart b/protobuf/lib/protobuf.dart index aec42dc3d..f7e2ea52f 100644 --- a/protobuf/lib/protobuf.dart +++ b/protobuf/lib/protobuf.dart @@ -8,8 +8,7 @@ library protobuf; import 'dart:collection' show ListBase, MapBase; -import 'dart:convert' - show base64Decode, base64Encode, jsonEncode, jsonDecode, Utf8Codec; +import 'dart:convert' show base64Decode, base64Encode, jsonDecode, Utf8Codec; import 'dart:math' as math; import 'dart:typed_data' show TypedData, Uint8List, ByteData, Endian; diff --git a/protobuf/lib/src/protobuf/generated_message.dart b/protobuf/lib/src/protobuf/generated_message.dart index 446b6621c..d58002293 100644 --- a/protobuf/lib/src/protobuf/generated_message.dart +++ b/protobuf/lib/src/protobuf/generated_message.dart @@ -232,10 +232,17 @@ abstract class GeneratedMessage { /// Returns Dart JSON object encoding this message following proto3 JSON /// format. /// - /// The returned object will have the same format as objects returned by - /// [jsonEncode]. + /// The key for each field is be the camel-cased name of the field. + /// + /// Well-known types and their special JSON encoding are supported. If a + /// well-known type cannot be encoded (eg. a `google.protobuf.Timestamp` with + /// negative `nanoseconds`) an error is thrown. + /// + /// Extensions and unknown fields are not encoded. /// - /// See [toProto3JsonSink] for details. + /// The [typeRegistry] is be used for encoding `Any` messages. If an `Any` + /// message encoding a type not in [typeRegistry] is encountered, an error is + /// thrown. Object? toProto3Json( {TypeRegistry typeRegistry = const TypeRegistry.empty()}) { Object? object; @@ -246,33 +253,20 @@ abstract class GeneratedMessage { return object; } - /// Returns a proto3 JSON string encoding this message. + /// Returns a proto3 JSON string encoding of this message. /// - /// See [toProto3JsonSink] for details. + /// See [toProto3Json] for details. String toProto3JsonString( {TypeRegistry typeRegistry = const TypeRegistry.empty()}) { final buf = StringBuffer(); final stringSink = jsonStringWriter(buf); - toProto3JsonSink(typeRegistry, stringSink); + $_toProto3JsonSink(typeRegistry, stringSink); return buf.toString(); } - /// Writes proto3 JSON serialization of this message to the given [JsonSink]. - /// - /// The key for each field is be the camel-cased name of the field. - /// - /// Well-known types and their special JSON encoding are supported. If a - /// well-known type cannot be encoded (eg. a `google.protobuf.Timestamp` with - /// negative `nanoseconds`) an error is thrown. - /// - /// Extensions and unknown fields are not encoded. - /// - /// The [typeRegistry] is be used for encoding `Any` messages. If an `Any` - /// message encoding a type not in [typeRegistry] is encountered, an error is - /// thrown. - /// - /// The [newMessage] argument is for use in generated code, do not use. - void toProto3JsonSink(TypeRegistry typeRegistry, JsonSink jsonSink, + /// For generated code only. + /// @nodoc + void $_toProto3JsonSink(TypeRegistry typeRegistry, JsonSink jsonSink, {bool newMessage = true}) { _writeToProto3JsonSink(_fieldSet, typeRegistry, jsonSink, newMessage: newMessage); diff --git a/protobuf/lib/src/protobuf/mixins/well_known.dart b/protobuf/lib/src/protobuf/mixins/well_known.dart index 51e853e31..2ad8a74b8 100644 --- a/protobuf/lib/src/protobuf/mixins/well_known.dart +++ b/protobuf/lib/src/protobuf/mixins/well_known.dart @@ -90,10 +90,10 @@ abstract class AnyMixin implements GeneratedMessage { jsonSink.addKey('@type'); jsonSink.addString(any.typeUrl); if (info.toProto3Json == null) { - unpacked.toProto3JsonSink(typeRegistry, jsonSink, newMessage: false); + unpacked.$_toProto3JsonSink(typeRegistry, jsonSink, newMessage: false); } else { jsonSink.addKey('value'); - unpacked.toProto3JsonSink(typeRegistry, jsonSink, newMessage: true); + unpacked.$_toProto3JsonSink(typeRegistry, jsonSink, newMessage: true); } jsonSink.endObject(); } From 572abbc6e46e2a931b8e3c449a8f5e1ea3c44874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Thu, 15 Sep 2022 14:02:34 +0200 Subject: [PATCH 15/18] Minor refactoring --- protobuf/lib/src/protobuf/generated_message.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protobuf/lib/src/protobuf/generated_message.dart b/protobuf/lib/src/protobuf/generated_message.dart index d58002293..5ae146ef7 100644 --- a/protobuf/lib/src/protobuf/generated_message.dart +++ b/protobuf/lib/src/protobuf/generated_message.dart @@ -260,7 +260,7 @@ abstract class GeneratedMessage { {TypeRegistry typeRegistry = const TypeRegistry.empty()}) { final buf = StringBuffer(); final stringSink = jsonStringWriter(buf); - $_toProto3JsonSink(typeRegistry, stringSink); + _writeToProto3JsonSink(_fieldSet, typeRegistry, stringSink); return buf.toString(); } From c741bf96b5b8e8410898e1d4cf30ef8e75ac753e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Thu, 15 Sep 2022 14:07:00 +0200 Subject: [PATCH 16/18] Rewrite toProto3Json documentation --- protobuf/lib/src/protobuf/generated_message.dart | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/protobuf/lib/src/protobuf/generated_message.dart b/protobuf/lib/src/protobuf/generated_message.dart index 5ae146ef7..d90fd80c7 100644 --- a/protobuf/lib/src/protobuf/generated_message.dart +++ b/protobuf/lib/src/protobuf/generated_message.dart @@ -232,17 +232,19 @@ abstract class GeneratedMessage { /// Returns Dart JSON object encoding this message following proto3 JSON /// format. /// - /// The key for each field is be the camel-cased name of the field. + /// Key for a field is the the camel-cased name of the field. /// - /// Well-known types and their special JSON encoding are supported. If a - /// well-known type cannot be encoded (eg. a `google.protobuf.Timestamp` with - /// negative `nanoseconds`) an error is thrown. + /// Well-known types and their special JSON encodings are supported. /// /// Extensions and unknown fields are not encoded. /// - /// The [typeRegistry] is be used for encoding `Any` messages. If an `Any` - /// message encoding a type not in [typeRegistry] is encountered, an error is - /// thrown. + /// [typeRegistry] is used for encoding `Any` messages. + /// + /// Throws [ArgumentError] if type of an `Any` message is not in + /// [typeRegistry]. + /// + /// Throws [ArgumentError] if a well-known type cannot be encoded. For + /// example, when a `google.protobuf.Timestamp` has negative `nanoseconds`. Object? toProto3Json( {TypeRegistry typeRegistry = const TypeRegistry.empty()}) { Object? object; From 20ffa557b6530c77853216d2cdce1ac9dbc9fadc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Thu, 15 Sep 2022 14:08:32 +0200 Subject: [PATCH 17/18] Rewording --- protobuf/lib/src/protobuf/generated_message.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protobuf/lib/src/protobuf/generated_message.dart b/protobuf/lib/src/protobuf/generated_message.dart index d90fd80c7..aab0e35c3 100644 --- a/protobuf/lib/src/protobuf/generated_message.dart +++ b/protobuf/lib/src/protobuf/generated_message.dart @@ -229,10 +229,10 @@ abstract class GeneratedMessage { return buf.toString(); } - /// Returns Dart JSON object encoding this message following proto3 JSON + /// Returns Dart JSON object encoding this message, following proto3 JSON /// format. /// - /// Key for a field is the the camel-cased name of the field. + /// Key for a field is the the camel-case name of the field. /// /// Well-known types and their special JSON encodings are supported. /// From 12538641bce73434b1d8564e0705a43e7373e99c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Sinan=20A=C4=9Facan?= Date: Thu, 15 Sep 2022 14:14:15 +0200 Subject: [PATCH 18/18] Update changelog --- protobuf/CHANGELOG.md | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/protobuf/CHANGELOG.md b/protobuf/CHANGELOG.md index 9848229bb..092ff6103 100644 --- a/protobuf/CHANGELOG.md +++ b/protobuf/CHANGELOG.md @@ -20,20 +20,11 @@ missing. ([#719], [#745]) * Fix updating frozen (immutable) messages with merge methods (`mergeFromBuffer`, `mergeFromProto3Json`, ...). ([#489], [#727]) -* 3 new `GeneratedMessage` methods added for JSON serialization: - - - `toProto3JsonSink`: for serializing a message as proto3 JSON to a - [jsontool] sink - - `toProto3JsonString`: for serializing a message as proto3 JSON string - - `writeToJsonSink`: for serializing a message as custom JSON format (the - format used by `writeToJsonMap`) to a [jsontool] sink - - Methods that return JSON strings (`writeToJson`, `toProto3JsonString`) now - don't allocate intermediate JSON objects and are much more efficient than - generating JSON objects (`writeToJsonMap`, `toProto3Json`) and then encoding - with `dart:convert`'s `json.encode`. - - ([#683]) +* New `GeneratedMessage` method `toProto3JsonString` added to generate proto3 + JSON string of a message. This method is much more efficient than generating + proto3 JSON object of a message with `toProto3Json` and then encoding that + object with `dart:convert`'s `jsonEncode`. ([#683]) +* `GeneratedMessage.writeToJson` performance improved. ([#683]) [#183]: https://github.com/google/protobuf.dart/issues/183 [#644]: https://github.com/google/protobuf.dart/pull/644