Avoid intermediate JSON object allocations when generating JSON strings

benchmarks/bin/to_proto3_json_string.dart

@@ -2,8 +2,6 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'dart:convert' show jsonEncode;
-
 import 'package:protobuf_benchmarks/benchmark_base.dart';
 import 'package:protobuf_benchmarks/generated/google_message1_proto2.pb.dart'
     as p2;
@@ -26,9 +24,9 @@ class Benchmark extends BenchmarkBase {
 
   @override
   void run() {
-    jsonEncode(_message1Proto2.toProto3Json());
-    jsonEncode(_message1Proto3.toProto3Json());
-    jsonEncode(_message2.toProto3Json());
+    _message1Proto2.toProto3JsonString();
+    _message1Proto3.toProto3JsonString();
+    _message2.toProto3JsonString();
   }
 }
 

benchmarks/pubspec.lock

@@ -43,6 +43,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "0.6.4"
+  jsontool:
+    dependency: transitive
+    description:
+      name: jsontool
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.1.2"
   lints:
     dependency: "direct dev"
     description:

protobuf/CHANGELOG.md

@@ -20,6 +20,20 @@
   missing. ([#719], [#745])
 * Fix updating frozen (immutable) messages with merge methods
   (`mergeFromBuffer`, `mergeFromProto3Json`, ...). ([#489], [#727])
+* 3 new `GeneratedMessage` methods added for JSON serialization:
+
+  - `toProto3JsonSink`: for serializing a message as proto3 JSON to a
+    [jsontool] sink
+  - `toProto3JsonString`: for serializing a message as proto3 JSON string
+  - `writeToJsonSink`: for serializing a message as custom JSON format (the
+    format used by `writeToJsonMap`) to a [jsontool] sink
+
+  Methods that return JSON strings (`writeToJson`, `toProto3JsonString`) now
+  don't allocate intermediate JSON objects and are much more efficient than
+  generating JSON objects (`writeToJsonMap`, `toProto3Json`) and then encoding
+  with `dart:convert`'s `json.encode`.
+
+  ([#683])
 
 [#183]: https://github.com/google/protobuf.dart/issues/183
 [#644]: https://github.com/google/protobuf.dart/pull/644
@@ -37,6 +51,8 @@
 [#745]: https://github.com/google/protobuf.dart/pull/745
 [#489]: https://github.com/google/protobuf.dart/issues/489
 [#727]: https://github.com/google/protobuf.dart/pull/727
+[jsontool]: https://pub.dev/packages/jsontool
+[#683]: https://github.com/google/protobuf.dart/pull/683
 
 ## 2.1.0
 

protobuf/lib/meta.dart

@@ -52,12 +52,15 @@ const GeneratedMessage_reservedNames = <String>[
   'toBuilder',
   'toDebugString',
   'toProto3Json',
+  'toProto3JsonSink',
+  'toProto3JsonString',
   'toString',
   'unknownFields',
   'writeToBuffer',
   'writeToCodedBufferWriter',
   'writeToJson',
   'writeToJsonMap',
+  'writeToJsonSink',
   r'$_ensure',
   r'$_get',
   r'$_getI64',

protobuf/lib/protobuf.dart

@@ -14,6 +14,7 @@ import 'dart:math' as math;
 import 'dart:typed_data' show TypedData, Uint8List, ByteData, Endian;
 
 import 'package:fixnum/fixnum.dart' show Int64;
+import 'package:jsontool/jsontool.dart';
 import 'package:meta/meta.dart' show UseResult;
 
 import 'src/protobuf/json_parsing_context.dart';
@@ -38,11 +39,13 @@ part 'src/protobuf/field_set.dart';
 part 'src/protobuf/field_type.dart';
 part 'src/protobuf/generated_message.dart';
 part 'src/protobuf/generated_service.dart';
-part 'src/protobuf/json.dart';
+part 'src/protobuf/json_reader.dart';
+part 'src/protobuf/json_writer.dart';
 part 'src/protobuf/pb_list.dart';
 part 'src/protobuf/pb_map.dart';
 part 'src/protobuf/protobuf_enum.dart';
-part 'src/protobuf/proto3_json.dart';
+part 'src/protobuf/proto3_json_reader.dart';
+part 'src/protobuf/proto3_json_writer.dart';
 part 'src/protobuf/rpc_client.dart';
 part 'src/protobuf/unknown_field_set.dart';
 part 'src/protobuf/utils.dart';

protobuf/lib/src/protobuf/builder_info.dart

@@ -41,11 +41,15 @@ class BuilderInfo {
 
   List<FieldInfo>? _sortedByTag;
 
-  // For well-known types.
-  final Object? Function(GeneratedMessage message, TypeRegistry typeRegistry)?
+  /// JSON generator for well-known types.
+  final void Function(
+          GeneratedMessage msg, TypeRegistry typeRegistry, JsonSink jsonSink)?
       toProto3Json;
+
+  /// JSON parser for well-known types.
   final Function(GeneratedMessage targetMessage, Object json,
       TypeRegistry typeRegistry, JsonParsingContext context)? fromProto3Json;
+
   final CreateBuilderFunc? createEmptyInstance;
 
   BuilderInfo(String? messageName,

protobuf/lib/src/protobuf/generated_message.dart

@@ -200,7 +200,14 @@ abstract class GeneratedMessage {
   /// Returns the JSON encoding of this message as a Dart [Map].
   ///
   /// The encoding is described in [GeneratedMessage.writeToJson].
-  Map<String, dynamic> writeToJsonMap() => _writeToJsonMap(_fieldSet);
+  Map<String, dynamic> writeToJsonMap() {
+    Object? object;
+    final objectSink = jsonObjectWriter((newObject) {
+      object = newObject;
+    });
+    writeToJsonSink(objectSink);
+    return object as Map<String, dynamic>;
+  }
 
   /// Returns a JSON string that encodes this message.
   ///
@@ -215,24 +222,65 @@ abstract class GeneratedMessage {
   /// represented as their integer value.
   ///
   /// For the proto3 JSON format use: [toProto3Json].
-  String writeToJson() => jsonEncode(writeToJsonMap());
+  String writeToJson() {
+    final buf = StringBuffer();
+    final stringSink = jsonStringWriter(buf);
+    writeToJsonSink(stringSink);
+    return buf.toString();
+  }
+
+  void writeToJsonSink(JsonSink jsonSink) {
+    _writeToJsonMapSink(_fieldSet, jsonSink);
+  }
+
+  /// Returns Dart JSON object encoding this message following proto3 JSON
+  /// format.
+  ///
+  /// The returned object will have the same format as objects returned by
+  /// [jsonEncode].
+  ///
+  /// See [toProto3JsonSink] for details.
+  Object? toProto3Json(
+      {TypeRegistry typeRegistry = const TypeRegistry.empty()}) {
+    Object? object;
+    final objectSink = jsonObjectWriter((newObject) {
+      object = newObject;
+    });
+    _writeToProto3JsonSink(_fieldSet, typeRegistry, objectSink);
+    return object;
+  }
 
-  /// Returns an Object representing Proto3 JSON serialization of `this`.
+  /// Returns a proto3 JSON string encoding this message.
+  ///
+  /// See [toProto3JsonSink] for details.
+  String toProto3JsonString(
+      {TypeRegistry typeRegistry = const TypeRegistry.empty()}) {
+    final buf = StringBuffer();
+    final stringSink = jsonStringWriter(buf);
+    toProto3JsonSink(typeRegistry, stringSink);
+    return buf.toString();
+  }
+
+  /// Writes proto3 JSON serialization of this message to the given [JsonSink].
   ///
   /// The key for each field is be the camel-cased name of the field.
   ///
-  /// Well-known types and their special JSON encoding are supported.
-  /// If a well-known type cannot be encoded (eg. a `google.protobuf.Timestamp`
-  /// with negative `nanoseconds`) an error is thrown.
+  /// Well-known types and their special JSON encoding are supported. If a
+  /// well-known type cannot be encoded (eg. a `google.protobuf.Timestamp` with
+  /// negative `nanoseconds`) an error is thrown.
   ///
   /// Extensions and unknown fields are not encoded.
   ///
   /// The [typeRegistry] is be used for encoding `Any` messages. If an `Any`
-  /// message encoding a type not in [typeRegistry] is encountered, an
-  /// error is thrown.
-  Object? toProto3Json(
-          {TypeRegistry typeRegistry = const TypeRegistry.empty()}) =>
-      _writeToProto3Json(_fieldSet, typeRegistry);
+  /// message encoding a type not in [typeRegistry] is encountered, an error is
+  /// thrown.
+  ///
+  /// The [newMessage] argument is for use in generated code, do not use.
+  void toProto3JsonSink(TypeRegistry typeRegistry, JsonSink jsonSink,
+      {bool newMessage = true}) {
+    _writeToProto3JsonSink(_fieldSet, typeRegistry, jsonSink,
+        newMessage: newMessage);
+  }
 
   /// Merges field values from [json], a JSON object using proto3 encoding.
   ///

protobuf/lib/src/protobuf/json.dart → protobuf/lib/src/protobuf/json_reader.dart

@@ -4,94 +4,6 @@
 
 part of protobuf;
 
-Map<String, dynamic> _writeToJsonMap(_FieldSet fs) {
-  dynamic convertToMap(dynamic fieldValue, int fieldType) {
-    var baseType = PbFieldType._baseType(fieldType);
-
-    if (_isRepeated(fieldType)) {
-      final PbList list = fieldValue;
-      return List.from(list.map((e) => convertToMap(e, baseType)));
-    }
-
-    switch (baseType) {
-      case PbFieldType._BOOL_BIT:
-      case PbFieldType._STRING_BIT:
-      case PbFieldType._INT32_BIT:
-      case PbFieldType._SINT32_BIT:
-      case PbFieldType._UINT32_BIT:
-      case PbFieldType._FIXED32_BIT:
-      case PbFieldType._SFIXED32_BIT:
-        return fieldValue;
-      case PbFieldType._FLOAT_BIT:
-      case PbFieldType._DOUBLE_BIT:
-        final value = fieldValue as double;
-        if (value.isNaN) {
-          return _nan;
-        }
-        if (value.isInfinite) {
-          return value.isNegative ? _negativeInfinity : _infinity;
-        }
-        if (fieldValue.toInt() == fieldValue) {
-          return fieldValue.toInt();
-        }
-        return value;
-      case PbFieldType._BYTES_BIT:
-        // Encode 'bytes' as a base64-encoded string.
-        return base64Encode(fieldValue as List<int>);
-      case PbFieldType._ENUM_BIT:
-        final ProtobufEnum enum_ = fieldValue;
-        return enum_.value; // assume |value| < 2^52
-      case PbFieldType._INT64_BIT:
-      case PbFieldType._SINT64_BIT:
-      case PbFieldType._SFIXED64_BIT:
-        return fieldValue.toString();
-      case PbFieldType._UINT64_BIT:
-      case PbFieldType._FIXED64_BIT:
-        final Int64 int_ = fieldValue;
-        return int_.toStringUnsigned();
-      case PbFieldType._GROUP_BIT:
-      case PbFieldType._MESSAGE_BIT:
-        final GeneratedMessage msg = fieldValue;
-        return msg.writeToJsonMap();
-      default:
-        throw 'Unknown type $fieldType';
-    }
-  }
-
-  List _writeMap(PbMap fieldValue, MapFieldInfo fi) =>
-      List.from(fieldValue.entries.map((MapEntry e) => {
-            '${PbMap._keyFieldNumber}': convertToMap(e.key, fi.keyFieldType),
-            '${PbMap._valueFieldNumber}':
-                convertToMap(e.value, fi.valueFieldType)
-          }));
-
-  var result = <String, dynamic>{};
-  for (var fi in fs._infosSortedByTag) {
-    var value = fs._values[fi.index!];
-    if (value == null || (value is List && value.isEmpty)) {
-      continue; // It's missing, repeated, or an empty byte array.
-    }
-    if (_isMapField(fi.type)) {
-      result['${fi.tagNumber}'] =
-          _writeMap(value, fi as MapFieldInfo<dynamic, dynamic>);
-      continue;
-    }
-    result['${fi.tagNumber}'] = convertToMap(value, fi.type);
-  }
-  final extensions = fs._extensions;
-  if (extensions != null) {
-    for (var tagNumber in _sorted(extensions._tagNumbers)) {
-      var value = extensions._values[tagNumber];
-      if (value is List && value.isEmpty) {
-        continue; // It's repeated or an empty byte array.
-      }
-      var fi = extensions._getInfoOrNull(tagNumber)!;
-      result['$tagNumber'] = convertToMap(value, fi.type);
-    }
-  }
-  return result;
-}
-
 // Merge fields from a previously decoded JSON object.
 // (Called recursively on nested messages.)
 void _mergeFromJsonMap(
@@ -146,14 +58,14 @@ void _appendJsonMap(BuilderInfo meta, _FieldSet fs, List jsonList,
     final convertedKey = _convertJsonValue(
         entryMeta,
         entryFieldSet,
-        jsonEntry['${PbMap._keyFieldNumber}'],
+        jsonEntry[PbMap._keyFieldNumberString],
         PbMap._keyFieldNumber,
         fi.keyFieldType,
         registry);
     var convertedValue = _convertJsonValue(
         entryMeta,
         entryFieldSet,
-        jsonEntry['${PbMap._valueFieldNumber}'],
+        jsonEntry[PbMap._valueFieldNumberString],
         PbMap._valueFieldNumber,
         fi.valueFieldType,
         registry);