Support unknown json data and add internal set/clear field methods for generated accessors.

protobuf/CHANGELOG.md

@@ -25,9 +25,20 @@
   To migrate, use `PbMap.unmodifiable(map.keyFieldType, map.valueFieldType)`
   instead of `PbMap.unmodifiable(map)`. ([#902])
 
+* Messages deserialized from JSON now generate the unknown fields when
+  serialized as JSON.
+
+  Note that, as before, unknown fields in JSON messages are not stored in the
+  `unknownFields` of the message. They are only used by the JSON serializers to
+  support roundtripping.
+
+  ([#49], [#918])
+
 [#738]: https://github.com/google/protobuf.dart/issues/738
 [#896]: https://github.com/google/protobuf.dart/issues/896
 [#902]: https://github.com/google/protobuf.dart/issues/902
+[#49]: https://github.com/google/protobuf.dart/issues/49
+[#918]: https://github.com/google/protobuf.dart/pulls/918
 
 ## 3.1.0
 

protobuf/lib/meta.dart

@@ -56,6 +56,7 @@ const GeneratedMessage_reservedNames = <String>[
   'writeToCodedBufferWriter',
   'writeToJson',
   'writeToJsonMap',
+  r'$_clearField',
   r'$_ensure',
   r'$_get',
   r'$_getI64',
@@ -72,6 +73,7 @@ const GeneratedMessage_reservedNames = <String>[
   r'$_setBool',
   r'$_setBytes',
   r'$_setDouble',
+  r'$_setField',
   r'$_setFloat',
   r'$_setInt64',
   r'$_setSignedInt32',

protobuf/lib/src/protobuf/field_set.dart

@@ -33,6 +33,9 @@ class _FieldSet {
   /// Contains all the unknown fields, or null if there aren't any.
   UnknownFieldSet? _unknownFields;
 
+  /// Contains unknown data for messages deserialized from json.
+  Map<String, dynamic>? _unknownJsonData;
+
   /// Encodes whether `this` has been frozen, and if so, also memoizes the
   /// hash code.
   ///
@@ -105,6 +108,7 @@ class _FieldSet {
       if (_isReadOnly) return UnknownFieldSet.emptyUnknownFieldSet;
       _unknownFields = UnknownFieldSet();
     }
+    _unknownJsonData = null;
     return _unknownFields!;
   }
 
@@ -531,6 +535,8 @@ class _FieldSet {
       if (_unknownFields != o._unknownFields) return false;
     }
 
+    // Ignore _unknownJsonData to preserve existing equality behavior.
+
     return true;
   }
 
@@ -597,6 +603,8 @@ class _FieldSet {
     // Hash with unknown fields.
     hash = _HashUtils._combine(hash, _unknownFields?.hashCode ?? 0);
 
+    // Ignore _unknownJsonData to preserve existing hashing behavior.
+
     if (_isReadOnly) {
       _frozenState = hash;
     }
@@ -682,6 +690,11 @@ class _FieldSet {
     } else {
       out.write(UnknownFieldSet().toString());
     }
+
+    final unknownJsonData = _unknownJsonData;
+    if (unknownJsonData != null) {
+      out.write(unknownJsonData.toString());
+    }
   }
 
   /// Merges the contents of the [other] into this message.
@@ -713,6 +726,15 @@ class _FieldSet {
     if (otherUnknownFields != null) {
       _ensureUnknownFields().mergeFromUnknownFieldSet(otherUnknownFields);
     }
+
+    final otherUnknownJsonData = other._unknownJsonData;
+    if (otherUnknownJsonData != null) {
+      final newUnknownJsonData =
+          Map<String, dynamic>.from(_unknownJsonData ?? {});
+      otherUnknownJsonData
+          .forEach((key, value) => newUnknownJsonData[key] = value);
+      _unknownJsonData = newUnknownJsonData.isEmpty ? null : newUnknownJsonData;
+    }
   }
 
   void _mergeField(FieldInfo otherFi, fieldValue, {required bool isExtension}) {
@@ -868,6 +890,11 @@ class _FieldSet {
       _ensureUnknownFields()._fields.addAll(originalUnknownFields._fields);
     }
 
+    final unknownJsonData = original._unknownJsonData;
+    if (unknownJsonData != null) {
+      _unknownJsonData = Map.from(unknownJsonData);
+    }
+
     _oneofCases?.addAll(original._oneofCases!);
   }
 }

protobuf/lib/src/protobuf/generated_message.dart

@@ -173,6 +173,10 @@ abstract class GeneratedMessage {
   }
 
   /// Serialize the message as the protobuf binary format.
+  ///
+  /// Unknown field data, data for which there is no metadata for the associated
+  /// field, will only be included if this message was deserialized from the
+  /// same wire format.
   Uint8List writeToBuffer() {
     final out = CodedBufferWriter();
     writeToCodedBufferWriter(out);
@@ -211,6 +215,10 @@ abstract class GeneratedMessage {
   /// Returns the JSON encoding of this message as a Dart [Map].
   ///
   /// The encoding is described in [GeneratedMessage.writeToJson].
+  ///
+  /// Unknown field data, data for which there is no metadata for the associated
+  /// field, will only be included if this message was deserialized from the
+  /// same wire format.
   Map<String, dynamic> writeToJsonMap() => _writeToJsonMap(_fieldSet);
 
   /// Returns a JSON string that encodes this message.
@@ -226,6 +234,10 @@ abstract class GeneratedMessage {
   /// represented as their integer value.
   ///
   /// For the proto3 JSON format use: [toProto3Json].
+  ///
+  /// Unknown field data, data for which there is no metadata for the associated
+  /// field, will only be included if this message was deserialized from the
+  /// same wire format.
   String writeToJson() => jsonEncode(writeToJsonMap());
 
   /// Returns an Object representing Proto3 JSON serialization of `this`.
@@ -241,6 +253,9 @@ abstract class GeneratedMessage {
   /// The [typeRegistry] is be used for encoding `Any` messages. If an `Any`
   /// message encoding a type not in [typeRegistry] is encountered, an
   /// error is thrown.
+  ///
+  /// Unknown field data, data for which there is no metadata for the associated
+  /// field, will not be included.
   Object? toProto3Json(
           {TypeRegistry typeRegistry = const TypeRegistry.empty()}) =>
       _writeToProto3Json(_fieldSet, typeRegistry);
@@ -509,6 +524,17 @@ abstract class GeneratedMessage {
   /// @nodoc
   void $_setInt64(int index, Int64 value) => _fieldSet._$set(index, value);
 
+  /// For generated code only. Separate from [setField] to distinguish
+  /// reflective accesses.
+  /// @nodoc
+  void $_setField(int tagNumber, Object value) =>
+      _fieldSet._setField(tagNumber, value);
+
+  /// For generated code only. Separate from [clearField] to distinguish
+  /// reflective accesses.
+  /// @nodoc
+  void $_clearField(int tagNumber) => _fieldSet._clearField(tagNumber);
+
   // Support for generating a read-only default singleton instance.
 
   static final Map<Function?, _SingletonMaker<GeneratedMessage>>

protobuf/lib/src/protobuf/json.dart

@@ -89,6 +89,12 @@ Map<String, dynamic> _writeToJsonMap(_FieldSet fs) {
       result['$tagNumber'] = convertToMap(value, fi.type);
     }
   }
+  final unknownJsonData = fs._unknownJsonData;
+  if (unknownJsonData != null) {
+    unknownJsonData.forEach((key, value) {
+      result[key] = value;
+    });
+  }
   return result;
 }
 
@@ -102,9 +108,11 @@ void _mergeFromJsonMap(
   for (final key in keys) {
     var fi = meta.byTagAsString[key];
     if (fi == null) {
-      if (registry == null) continue; // Unknown tag; skip
-      fi = registry.getExtension(fs._messageName, int.parse(key));
-      if (fi == null) continue; // Unknown tag; skip
+      fi = registry?.getExtension(fs._messageName, int.parse(key));
+      if (fi == null) {
+        (fs._unknownJsonData ??= {})[key] = json[key];
+        continue;
+      }
     }
     if (fi.isMapField) {
       _appendJsonMap(

protobuf/test/json_test.dart

@@ -87,7 +87,7 @@ void main() {
     checkJsonMap(m);
   });
 
-  test('testMergeFromJson', () {
+  test('testWriteToJsonMap', () {
     final t = T();
     t.mergeFromJson('''{"1": 123, "2": "hello"}''');
     checkMessage(t);
@@ -118,13 +118,21 @@ void main() {
     final decoded = T()..mergeFromJsonMap(encoded);
     expect(decoded.int64, value);
   });
+
+  test('testJsonMapWithUnknown', () {
+    final m = example.writeToJsonMap();
+    m['9999'] = 'world';
+    final t = T()..mergeFromJsonMap(m);
+    checkJsonMap(t.writeToJsonMap(), unknownFields: {'9999': 'world'});
+  });
 }
 
-void checkJsonMap(Map m) {
-  expect(m.length, 3);
+void checkJsonMap(Map m, {Map<String, dynamic>? unknownFields}) {
+  expect(m.length, 3 + (unknownFields?.length ?? 0));
   expect(m['1'], 123);
   expect(m['2'], 'hello');
   expect(m['4'], [1, 2, 3]);
+  unknownFields?.forEach((k, v) => expect(m[k], v));
 }
 
 void checkMessage(T t) {

protoc_plugin/lib/src/generated/dart_options.pb.dart

@@ -82,7 +82,7 @@ class DartMixin extends $pb.GeneratedMessage {
   @$pb.TagNumber(1)
   $core.bool hasName() => $_has(0);
   @$pb.TagNumber(1)
-  void clearName() => clearField(1);
+  void clearName() => $_clearField(1);
 
   /// A URI pointing to the Dart library that defines the mixin.
   /// The generated Dart code will use this in an import statement.
@@ -96,7 +96,7 @@ class DartMixin extends $pb.GeneratedMessage {
   @$pb.TagNumber(2)
   $core.bool hasImportFrom() => $_has(1);
   @$pb.TagNumber(2)
-  void clearImportFrom() => clearField(2);
+  void clearImportFrom() => $_clearField(2);
 
   /// The name of another mixin to be applied ahead of this one.
   /// The generated class for the message will inherit from all mixins
@@ -111,7 +111,7 @@ class DartMixin extends $pb.GeneratedMessage {
   @$pb.TagNumber(3)
   $core.bool hasParent() => $_has(2);
   @$pb.TagNumber(3)
-  void clearParent() => clearField(3);
+  void clearParent() => $_clearField(3);
 }
 
 /// Defines additional Dart imports to be used with messages in this file.