Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bugfix/comprehensive index equality #496

Merged
merged 2 commits into from
Oct 28, 2024
Merged

Bugfix/comprehensive index equality #496

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fc156e0
fixing issues with index equality
slorello89 Oct 21, 2024
023050b
fixing hash vector corner case
slorello89 Oct 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 148 additions & 5 deletions src/Redis.OM/Modeling/RedisIndex.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,29 +20,88 @@ public static class RedisIndex
public static bool IndexDefinitionEquals(this RedisIndexInfo redisIndexInfo, Type type)
{
var serialisedDefinition = SerializeIndex(type);
var existingSet = redisIndexInfo.Attributes?.Select(a => (Property: a.Attribute!, a.Type!)).OrderBy(a => a.Property);
var isJson = redisIndexInfo.IndexDefinition?.Identifier == "JSON";

var currentOffset = 0;
if (serialisedDefinition.Length < 5)
{
throw new ArgumentException($"Could not parse the index definition for type: {type.Name}.");
}

if (redisIndexInfo.IndexName != serialisedDefinition[0])
if (redisIndexInfo.IndexDefinition is null)
{
return false;
}

if (redisIndexInfo.IndexDefinition?.Identifier?.Equals(serialisedDefinition[2], StringComparison.OrdinalIgnoreCase) == false)
// these are properties we cannot process because FT.INFO does not respond with them
var unprocessableProperties = new string[] { "EPSILON", "EF_RUNTIME", "PHONETIC", "STOPWORDS" };

foreach (var property in unprocessableProperties)
{
if (serialisedDefinition.Any(x => x.Equals(property)))
{
throw new ArgumentException($"Could not validate index definition that contains {property}");
}
}

if (redisIndexInfo.IndexName != serialisedDefinition[currentOffset])
{
return false;
}

currentOffset += 2; // skip to the index type at index 2

if (redisIndexInfo.IndexDefinition?.Identifier?.Equals(serialisedDefinition[currentOffset], StringComparison.OrdinalIgnoreCase) == false)
{
return false;
}

currentOffset += 2; // skip to prefix count

if (!int.TryParse(serialisedDefinition[currentOffset], out var numPrefixes))
{
throw new ArgumentException("Could not parse index with unknown number of prefixes");
}

currentOffset += 2; // skip to first prefix

if (redisIndexInfo.IndexDefinition?.Prefixes is null || redisIndexInfo.IndexDefinition.Prefixes.Length != numPrefixes || serialisedDefinition.Skip(currentOffset).Take(numPrefixes).SequenceEqual(redisIndexInfo.IndexDefinition.Prefixes))
{
return false;
}

currentOffset += numPrefixes;

if (redisIndexInfo.IndexDefinition?.Filter is not null && !redisIndexInfo.IndexDefinition.Filter.Equals(serialisedDefinition.ElementAt(currentOffset)))
{
return false;
}

if (redisIndexInfo.IndexDefinition?.Filter is not null)
{
currentOffset += 2;
}

if (redisIndexInfo.IndexDefinition?.DefaultLanguage is not null && !redisIndexInfo.IndexDefinition.DefaultLanguage.Equals(serialisedDefinition.ElementAt(currentOffset)))
{
return false;
}

if (redisIndexInfo.IndexDefinition?.Prefixes.FirstOrDefault().Equals(serialisedDefinition[5]) == false)
if (redisIndexInfo.IndexDefinition?.DefaultLanguage is not null)
{
currentOffset += 2;
}

if (redisIndexInfo.IndexDefinition?.LanguageField is not null && !redisIndexInfo.IndexDefinition.LanguageField.Equals(serialisedDefinition.ElementAt(currentOffset)))
{
return false;
}

if (redisIndexInfo.IndexDefinition?.LanguageField is not null)
{
currentOffset += 2;
}

var target = redisIndexInfo.Attributes?.SelectMany(a =>
{
var attr = new List<string>();
Expand All @@ -58,11 +117,81 @@ public static bool IndexDefinitionEquals(this RedisIndexInfo redisIndexInfo, Typ
attr.Add("AS");
}

if (!isJson && a.Type is not null && a.Type == "VECTOR")
{
attr.Add($"{a.Attribute!}.Vector");
attr.Add("AS");
}

attr.Add(a.Attribute!);

if (a.Type != null)
{
attr.Add(a.Type);
if (a.Type == "TAG")
{
attr.Add("SEPARATOR");
attr.Add(a.Separator ?? "|");
}

if (a.Type == "TEXT")
{
if (a.NoStem == true)
{
attr.Add("NOSTEM");
}

if (a.Weight is not null && a.Weight != "1")
{
attr.Add("WEIGHT");
attr.Add(a.Weight);
}
}

if (a.Type == "VECTOR")
{
if (a.Algorithm is null)
{
throw new InvalidOperationException("Encountered Vector field with no algorithm");
}

attr.Add(a.Algorithm);
if (a.VectorType is null)
{
throw new InvalidOperationException("Encountered vector field with no Vector Type");
}

attr.Add(NumVectorArgs(a).ToString());

attr.Add("TYPE");
attr.Add(a.VectorType);

if (a.Dimension is null)
{
throw new InvalidOperationException("Encountered vector field with no dimension");
}

attr.Add("DIM");
attr.Add(a.Dimension);

if (a.DistanceMetric is not null)
{
attr.Add("DISTANCE_METRIC");
attr.Add(a.DistanceMetric);
}

if (a.M is not null)
{
attr.Add("M");
attr.Add(a.M);
}

if (a.EfConstruction is not null)
{
attr.Add("EF_CONSTRUCTION");
attr.Add(a.EfConstruction);
}
}
}

if (a.Sortable == true)
Expand All @@ -73,7 +202,21 @@ public static bool IndexDefinitionEquals(this RedisIndexInfo redisIndexInfo, Typ
return attr.ToArray();
});

return target.SequenceEqual(serialisedDefinition.Skip(7));
return target.SequenceEqual(serialisedDefinition.Skip(currentOffset));
}

/// <summary>
/// calculates the number of arguments that would be required based to reverse engineer the index based off what
/// is in the Info attribute.
/// </summary>
/// <param name="attr">The attribute.</param>
/// <returns>The number of arguments.</returns>
internal static int NumVectorArgs(this RedisIndexInfo.RedisIndexInfoAttribute attr)
{
var numArgs = 6;
numArgs += attr.M is not null ? 2 : 0;
numArgs += attr.EfConstruction is not null ? 2 : 0;
return numArgs;
}

/// <summary>
Expand Down
71 changes: 71 additions & 0 deletions src/Redis.OM/RedisIndexInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using Redis.OM.Modeling;

namespace Redis.OM
{
Expand Down Expand Up @@ -220,6 +221,9 @@ public RedisIndexInfoIndexDefinition(RedisReply redisReply)
case "key_type": Identifier = value.ToString(CultureInfo.InvariantCulture); break;
case "prefixes": Prefixes = value.ToArray().Select(x => x.ToString(CultureInfo.InvariantCulture)).ToArray(); break;
case "default_score": DefaultScore = value.ToString(CultureInfo.InvariantCulture); break;
case "default_language": DefaultLanguage = value.ToString(CultureInfo.InvariantCulture); break;
case "filter": Filter = value.ToString(CultureInfo.InvariantCulture); break;
case "language_field": LanguageField = value.ToString(CultureInfo.InvariantCulture); break;
}
}
}
Expand All @@ -238,6 +242,21 @@ public RedisIndexInfoIndexDefinition(RedisReply redisReply)
/// Gets default_score.
/// </summary>
public string? DefaultScore { get; }

/// <summary>
/// Gets Filter.
/// </summary>
public string? Filter { get; }

/// <summary>
/// Gets language.
/// </summary>
public string? DefaultLanguage { get; }

/// <summary>
/// Gets LanguageField.
/// </summary>
public string? LanguageField { get; }
}

/// <summary>
Expand Down Expand Up @@ -266,9 +285,21 @@ public RedisIndexInfoAttribute(RedisReply redisReply)
case "attribute": Attribute = value; break;
case "type": Type = value; break;
case "SEPARATOR": Separator = value; break;
case "algorithm": Algorithm = value; break;
case "data_type": VectorType = value; break;
case "dim": Dimension = value; break;
case "distance_metric": DistanceMetric = value; break;
case "M": M = value; break;
case "ef_construction": EfConstruction = value; break;
case "WEIGHT": Weight = value; break;
}
}

if (responseArray.Any(x => ((string)x).Equals("NOSTEM", StringComparison.InvariantCultureIgnoreCase)))
{
NoStem = true;
}

if (responseArray.Select(x => x.ToString())
.Any(x => x.Equals("SORTABLE", StringComparison.InvariantCultureIgnoreCase)))
{
Expand Down Expand Up @@ -300,6 +331,46 @@ public RedisIndexInfoAttribute(RedisReply redisReply)
/// Gets SORTABLE.
/// </summary>
public bool? Sortable { get; }

/// <summary>
/// Gets NOSTEM.
/// </summary>
public bool? NoStem { get; }

/// <summary>
/// Gets weight.
/// </summary>
public string? Weight { get; }

/// <summary>
/// Gets Algorithm.
/// </summary>
public string? Algorithm { get; }

/// <summary>
/// Gets the VectorType.
/// </summary>
public string? VectorType { get; }

/// <summary>
/// Gets Dimension.
/// </summary>
public string? Dimension { get; }

/// <summary>
/// Gets DistanceMetric.
/// </summary>
public string? DistanceMetric { get; }

/// <summary>
/// Gets M.
/// </summary>
public string? M { get; }

/// <summary>
/// Gets EF constructor.
/// </summary>
public string? EfConstruction { get; }
}

/// <summary>
Expand Down
Loading
Loading