From f50661854c20c2ba26a02f2b37cb6a99e07d1894 Mon Sep 17 00:00:00 2001 From: slorello89 Date: Tue, 22 Oct 2024 09:34:41 -0400 Subject: [PATCH 1/2] fixing issue with timestamps always being updated --- .../Modeling/DateTimeJsonConvertNewtonsoft.cs | 66 +++++++++++++++++++ .../Modeling/RedisCollectionStateManager.cs | 10 ++- .../ObjectWIthMultipleDateTimes.cs | 24 +++++++ .../RediSearchTests/SearchTests.cs | 61 +++++++++++++++++ 4 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 src/Redis.OM/Modeling/DateTimeJsonConvertNewtonsoft.cs create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/ObjectWIthMultipleDateTimes.cs diff --git a/src/Redis.OM/Modeling/DateTimeJsonConvertNewtonsoft.cs b/src/Redis.OM/Modeling/DateTimeJsonConvertNewtonsoft.cs new file mode 100644 index 00000000..59ef07ba --- /dev/null +++ b/src/Redis.OM/Modeling/DateTimeJsonConvertNewtonsoft.cs @@ -0,0 +1,66 @@ +using System; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Redis.OM.Modeling; + +/// +/// Converter for Newtonsoft. +/// +internal class DateTimeJsonConvertNewtonsoft : JsonConverter +{ + /// + /// Determines is the object is convertable. + /// + /// the object type. + /// whether it can be converted. + public override bool CanConvert(Type objectType) + { + return objectType == typeof(DateTime) || objectType == typeof(DateTime?); + } + + /// + /// writes the object to json. + /// + /// the writer. + /// the value. + /// the serializer. + public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) + { + if (value is DateTime dateTime) + { + // Convert DateTime to Unix timestamp + long unixTimestamp = ((DateTimeOffset)dateTime).ToUnixTimeMilliseconds(); + writer.WriteValue(unixTimestamp); + } + else + { + writer.WriteNull(); + } + } + + /// + /// reads an object back from json. + /// + /// the reader. + /// the object type. + /// the existing value. + /// the serializer. + /// The converted object. + /// thrown if issue comes up deserializing. + public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.Null) + { + return objectType == typeof(DateTime?) ? null : DateTime.MinValue; + } + + if (reader.TokenType == JsonToken.Integer && reader.Value is long unixTimestamp) + { + // Convert Unix timestamp back to DateTime + return DateTimeOffset.FromUnixTimeMilliseconds(unixTimestamp).UtcDateTime; + } + + throw new JsonSerializationException("Invalid token type for Unix timestamp conversion."); + } +} \ No newline at end of file diff --git a/src/Redis.OM/Modeling/RedisCollectionStateManager.cs b/src/Redis.OM/Modeling/RedisCollectionStateManager.cs index 0a39a135..534c6010 100644 --- a/src/Redis.OM/Modeling/RedisCollectionStateManager.cs +++ b/src/Redis.OM/Modeling/RedisCollectionStateManager.cs @@ -2,6 +2,7 @@ using System.Linq; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using Redis.OM.Modeling.Vectors; using JsonSerializer = System.Text.Json.JsonSerializer; namespace Redis.OM.Modeling @@ -11,6 +12,11 @@ namespace Redis.OM.Modeling /// public class RedisCollectionStateManager { + private static JsonSerializerSettings _jsonSerializerSettings = new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, Converters = new List { new DateTimeJsonConvertNewtonsoft() }, + }; + /// /// Initializes a new instance of the class. /// @@ -76,7 +82,7 @@ internal void InsertIntoSnapshot(string key, object value) if (DocumentAttribute.StorageType == StorageType.Json) { - var json = JToken.FromObject(value, Newtonsoft.Json.JsonSerializer.Create(new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore })); + var json = JToken.FromObject(value, Newtonsoft.Json.JsonSerializer.Create(new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore, Converters = new List { new DateTimeJsonConvertNewtonsoft() } })); Snapshot.Add(key, json); } else @@ -115,7 +121,7 @@ internal bool TryDetectDifferencesSingle(string key, object value, out IList)Snapshot[key]; var deletedKeys = snapshotHash.Keys.Except(dataHash.Keys).Select(x => new KeyValuePair(x, string.Empty)); var modifiedKeys = dataHash.Where(x => - !snapshotHash.Keys.Contains(x.Key) || snapshotHash[x.Key] != x.Value).Select(x => + !snapshotHash.Keys.Contains(x.Key) || !snapshotHash[x.Key].Equals(x.Value)).Select(x => new KeyValuePair(x.Key, x.Value.ToString())); differences = new List { diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/ObjectWIthMultipleDateTimes.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/ObjectWIthMultipleDateTimes.cs new file mode 100644 index 00000000..4909f5eb --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/ObjectWIthMultipleDateTimes.cs @@ -0,0 +1,24 @@ +using System; +using Redis.OM.Modeling; + +namespace Redis.OM.Unit.Tests.RediSearchTests; + +[Document(StorageType = StorageType.Json, Prefixes = new []{"obj"})] +public class ObjectWIthMultipleDateTimes +{ + [RedisIdField] + [Indexed] + public string Id { get; set; } + public DateTime DateTime1 { get; set; } + public DateTime DateTime2 { get; set; } +} + +[Document(Prefixes = new []{"obj"})] +public class ObjectWIthMultipleDateTimesHash +{ + [RedisIdField] + [Indexed] + public string Id { get; set; } + public DateTime DateTime1 { get; set; } + public DateTime DateTime2 { get; set; } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs index 9272cb29..c17749aa 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs @@ -48,6 +48,34 @@ public class SearchTests }) }; + private readonly RedisReply _mockedReplyObjectWIthMultipleDateTimes = new[] + { + new RedisReply(1), + new RedisReply( + "obj:01FVN836BNQGYMT80V7RCVY73N"), + new RedisReply(new RedisReply[] + { + "$", + "{\"Id\":\"01FVN836BNQGYMT80V7RCVY73N\",\"DateTime1\":1729592130000,\"DateTime2\":1730475900000}" + }) + }; + + private readonly RedisReply _mockedReplyObjectWIthMultipleDateTimesHash = new[] + { + new RedisReply(1), + new RedisReply( + "obj:01FVN836BNQGYMT80V7RCVY73N"), + new RedisReply(new RedisReply[] + { + "Id", + "01FVN836BNQGYMT80V7RCVY73N", + "DateTime1", + "1729592130000", + "DateTime2", + "1730475900000" + }) + }; + [Fact] public void TestBasicQuery() { @@ -1030,6 +1058,39 @@ public async Task TestUpdateJson() await _substitute.Received().ExecuteAsync("EVALSHA", Arg.Any(), "1", new RedisKey("Redis.OM.Unit.Tests.RediSearchTests.Person:01FVN836BNQGYMT80V7RCVY73N"), "SET", "$.Age", "33"); Scripts.ShaCollection.Clear(); } + + [Fact] + public async Task TestUpdateJsonWithMultipleDateTimes() + { + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockedReplyObjectWIthMultipleDateTimes); + + _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(Task.FromResult(new RedisReply("42"))); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()) + .Returns(Task.FromResult(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb"))); + var collection = new RedisCollection(_substitute); + var obj = (await collection.Where(x => x.Id == "01FVN836BNQGYMT80V7RCVY73N").ToListAsync()).First(); + obj.DateTime1 = obj.DateTime1.AddMilliseconds(1); + await collection.UpdateAsync(obj); + await _substitute.Received().ExecuteAsync("EVALSHA", Arg.Any(), "1", new RedisKey("obj:01FVN836BNQGYMT80V7RCVY73N"), "SET", "$.DateTime1", "1729592130001"); + Scripts.ShaCollection.Clear(); + } + + + [Fact] + public async Task TestUpdateJsonWithMultipleDateTimesHash() + { + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockedReplyObjectWIthMultipleDateTimesHash); + + _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(Task.FromResult(new RedisReply("42"))); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()) + .Returns(Task.FromResult(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb"))); + var collection = new RedisCollection(_substitute); + var obj = (await collection.Where(x => x.Id == "01FVN836BNQGYMT80V7RCVY73N").ToListAsync()).First(); + obj.DateTime1 = obj.DateTime1.AddMilliseconds(1); + await collection.UpdateAsync(obj); + await _substitute.Received().ExecuteAsync("EVALSHA", Arg.Any(), "1", new RedisKey("obj:01FVN836BNQGYMT80V7RCVY73N"), "1", "DateTime1", "1729592130001"); + Scripts.ShaCollection.Clear(); + } [Fact] public async Task TestUpdateJsonUnloadedScriptAsync() From e4ea385d5f7e0ecb20c680cd9f98f1194b36b79b Mon Sep 17 00:00:00 2001 From: slorello89 Date: Wed, 23 Oct 2024 10:11:55 -0400 Subject: [PATCH 2/2] fixing issue with byte array updates --- .../Modeling/RedisCollectionStateManager.cs | 13 +++++++-- .../RediSearchTests/ObjectWithByteArray.cs | 15 +++++++++++ .../RediSearchTests/SearchFunctionalTests.cs | 15 +++++++++++ .../RediSearchTests/SearchTests.cs | 27 +++++++++++++++++++ .../RedisSetupCollection.cs | 2 ++ 5 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 test/Redis.OM.Unit.Tests/RediSearchTests/ObjectWithByteArray.cs diff --git a/src/Redis.OM/Modeling/RedisCollectionStateManager.cs b/src/Redis.OM/Modeling/RedisCollectionStateManager.cs index 534c6010..20ec8ebe 100644 --- a/src/Redis.OM/Modeling/RedisCollectionStateManager.cs +++ b/src/Redis.OM/Modeling/RedisCollectionStateManager.cs @@ -1,8 +1,8 @@ +using System; using System.Collections.Generic; using System.Linq; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using Redis.OM.Modeling.Vectors; using JsonSerializer = System.Text.Json.JsonSerializer; namespace Redis.OM.Modeling @@ -329,7 +329,16 @@ private static JObject FindDiff(JToken currentObject, JToken snapshotObject) break; default: - if (currentObject.ToString() != snapshotObject.ToString()) + if (snapshotObject.Type == JTokenType.Bytes) + { + var snapShotObjectStr = Convert.ToBase64String(snapshotObject.Value()); + + if (snapShotObjectStr != currentObject.ToString()) + { + diff["+"] = currentObject.ToString(); + } + } + else if (currentObject.ToString() != snapshotObject.ToString()) { diff["+"] = currentObject; diff["-"] = snapshotObject; diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/ObjectWithByteArray.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/ObjectWithByteArray.cs new file mode 100644 index 00000000..60756e43 --- /dev/null +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/ObjectWithByteArray.cs @@ -0,0 +1,15 @@ +using Redis.OM.Modeling; + +namespace Redis.OM.Unit.Tests.RediSearchTests; + +[Document(StorageType = StorageType.Json, Prefixes = new []{"obj"})] +public class ObjectWithByteArray +{ + [RedisIdField] + [Indexed] + public string Id { get; set; } + + public byte[] Bytes1 { get; set; } + + public byte[] Bytes2 { get; set; } +} \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/SearchFunctionalTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/SearchFunctionalTests.cs index 6c36fe52..43d9468b 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/SearchFunctionalTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/SearchFunctionalTests.cs @@ -1270,5 +1270,20 @@ public async Task TestSearchByMatchPattern(string pattern, int existingRecordsCo var res = await collection.Where(x => x.Name.MatchPattern(pattern)).ToListAsync(); Assert.Equal(existingRecordsCount, res.Count); } + + [Fact] + public async Task TestUpdateByteArray() + { + var collection = new RedisCollection(_connection); + var obj = new ObjectWithByteArray() { Bytes1 = new byte[] { 1, 2, 3 }, Bytes2 = new byte[] { 4, 5, 6 } }; + var id = await collection.InsertAsync(obj); + var res = (await collection.Where(x => x.Id == obj.Id).ToListAsync()).First(); + res.Bytes1 = new byte[] { 7, 8, 9 }; + res.Bytes2 = new byte[] { 10, 11, 12 }; + await collection.UpdateAsync(res); + var updated = (await collection.Where(x => x.Id == obj.Id).ToListAsync()).First(); + Assert.Equal(new byte[] { 7, 8, 9 }, updated.Bytes1); + Assert.Equal(new byte[] { 10, 11, 12 }, updated.Bytes2); + } } } \ No newline at end of file diff --git a/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs b/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs index c17749aa..6eb09b9f 100644 --- a/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs +++ b/test/Redis.OM.Unit.Tests/RediSearchTests/SearchTests.cs @@ -60,6 +60,18 @@ public class SearchTests }) }; + private readonly RedisReply _mockedReplyObjectWIthMultipleByteArrays = new[] + { + new RedisReply(1), + new RedisReply( + "obj:01FVN836BNQGYMT80V7RCVY73N"), + new RedisReply(new RedisReply[] + { + "$", + "{\"Id\":\"01FVN836BNQGYMT80V7RCVY73N\",\"Bytes1\":\"AQID\",\"Bytes2\":\"BAUG\"}" + }) + }; + private readonly RedisReply _mockedReplyObjectWIthMultipleDateTimesHash = new[] { new RedisReply(1), @@ -1092,6 +1104,21 @@ public async Task TestUpdateJsonWithMultipleDateTimesHash() Scripts.ShaCollection.Clear(); } + [Fact] + public async Task TestUpdateJsonWithByteArrays() + { + _substitute.ExecuteAsync("FT.SEARCH", Arg.Any()).Returns(_mockedReplyObjectWIthMultipleByteArrays); + _substitute.ExecuteAsync("EVALSHA", Arg.Any()).Returns(Task.FromResult(new RedisReply("42"))); + _substitute.ExecuteAsync("SCRIPT", Arg.Any()) + .Returns(Task.FromResult(new RedisReply("cbbf1c4fab5064f419e469cc51c563f8bf51e6fb"))); + + var collection = new RedisCollection(_substitute); + var obj = (await collection.Where(x => x.Id == "01FVN836BNQGYMT80V7RCVY73N").ToListAsync()).First(); + obj.Bytes1 = new byte[] { 4, 5, 6 }; + await collection.UpdateAsync(obj); + await _substitute.Received().ExecuteAsync("EVALSHA", Arg.Any(), "1", new RedisKey("obj:01FVN836BNQGYMT80V7RCVY73N"), "SET", "$.Bytes1","\"BAUG\""); + } + [Fact] public async Task TestUpdateJsonUnloadedScriptAsync() { diff --git a/test/Redis.OM.Unit.Tests/RedisSetupCollection.cs b/test/Redis.OM.Unit.Tests/RedisSetupCollection.cs index c0a3eb1c..7a60efb3 100644 --- a/test/Redis.OM.Unit.Tests/RedisSetupCollection.cs +++ b/test/Redis.OM.Unit.Tests/RedisSetupCollection.cs @@ -31,6 +31,7 @@ public RedisSetup() Connection.CreateIndex(typeof(SelectTestObject)); Connection.CreateIndex(typeof(ObjectWithDateTimeOffsetJson)); Connection.CreateIndex(typeof(ObjectWithMultipleSearchableAttributes)); + Connection.CreateIndex(typeof(ObjectWithByteArray)); } private IRedisConnectionProvider _provider; @@ -64,6 +65,7 @@ public void Dispose() Connection.DropIndexAndAssociatedRecords(typeof(SelectTestObject)); Connection.DropIndexAndAssociatedRecords(typeof(ObjectWithDateTimeOffsetJson)); Connection.DropIndexAndAssociatedRecords(typeof(ObjectWithMultipleSearchableAttributes)); + Connection.DropIndexAndAssociatedRecords(typeof(ObjectWithByteArray)); } } }