Merge pull request #32537 from vespa-engine/bratseth/pack_bits

Bratseth/pack bits

config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java

@@ -76,7 +76,7 @@ protected boolean shouldConvert(Expression exp) {
                 String fieldName = ((OutputExpression)exp).getFieldName();
                 if (outputs.contains(fieldName) && !prevNames.contains(fieldName)) {
                     throw new VerificationException(exp, "Attempting to assign conflicting values to field '" +
-                                                         fieldName + "'.");
+                                                         fieldName + "'");
                 }
                 outputs.add(fieldName);
                 prevNames.add(fieldName);
@@ -111,24 +111,24 @@ public DataType getInputType(Expression exp, String fieldName) {
         }
 
         @Override
-        public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
+        public void tryOutputType(Expression expression, String fieldName, DataType valueType) {
             String fieldDesc;
             DataType fieldType;
-            if (exp instanceof AttributeExpression) {
+            if (expression instanceof AttributeExpression) {
                 Attribute attribute = schema.getAttribute(fieldName);
                 if (attribute == null) {
-                    throw new VerificationException(exp, "Attribute '" + fieldName + "' not found.");
+                    throw new VerificationException(expression, "Attribute '" + fieldName + "' not found.");
                 }
                 fieldDesc = "attribute";
                 fieldType = attribute.getDataType();
-            } else if (exp instanceof IndexExpression) {
+            } else if (expression instanceof IndexExpression) {
                 SDField field = schema.getConcreteField(fieldName);
                 if (field == null) {
-                    throw new VerificationException(exp, "Index field '" + fieldName + "' not found.");
+                    throw new VerificationException(expression, "Index field '" + fieldName + "' not found.");
                 }
                 fieldDesc = "index field";
                 fieldType = field.getDataType();
-            } else if (exp instanceof SummaryExpression) {
+            } else if (expression instanceof SummaryExpression) {
                 SummaryField field = schema.getSummaryField(fieldName);
                 if (field == null) {
                     // Use document field if summary field is not found
@@ -137,7 +137,7 @@ public void tryOutputType(Expression exp, String fieldName, DataType valueType)
                         fieldDesc = "document field";
                         fieldType = sdField.getDataType();
                     } else {
-                        throw new VerificationException(exp, "Summary field '" + fieldName + "' not found.");
+                        throw new VerificationException(expression, "Summary field '" + fieldName + "' not found.");
                     }
                 } else {
                     fieldDesc = "summary field";
@@ -148,8 +148,8 @@ public void tryOutputType(Expression exp, String fieldName, DataType valueType)
             }
             if ( ! fieldType.isAssignableFrom(valueType) &&
                  ! fieldType.isAssignableFrom(createCompatType(valueType))) {
-                throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc +
-                                                     " '" + fieldName + "' which is " + fieldType.getName() + ".");
+                throw new VerificationException(expression, "Can not assign " + valueType.getName() + " to " + fieldDesc +
+                                                            " '" + fieldName + "' which is " + fieldType.getName() + ".");
             }
         }
 

config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java

@@ -57,7 +57,7 @@ void requireThatOutputConflictThrows() throws ParseException {
         }
         catch (IllegalArgumentException e) {
             assertEquals("For schema 'indexing_output_confict', field 'bar': For expression 'index bar': Attempting " +
-                         "to assign conflicting values to field 'bar'.",
+                         "to assign conflicting values to field 'bar'",
                          Exceptions.toMessageString(e));
         }
     }

config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java

@@ -36,7 +36,7 @@ void testAttributeChanged() throws ParseException {
         }
         catch (IllegalArgumentException e) {
             assertEquals("For schema 'indexing_attribute_changed', field 'foo': For expression 'attribute foo': " +
-                         "Attempting to assign conflicting values to field 'foo'.",
+                         "Attempting to assign conflicting values to field 'foo'",
                          Exceptions.toMessageString(e));
         }
     }
@@ -80,7 +80,7 @@ void testIndexChanged() throws ParseException {
         }
         catch (IllegalArgumentException e) {
             assertEquals("For schema 'indexing_index_changed', field 'foo': For expression 'index foo': " +
-                         "Attempting to assign conflicting values to field 'foo'.",
+                         "Attempting to assign conflicting values to field 'foo'",
                          Exceptions.toMessageString(e));
         }
     }
@@ -124,7 +124,7 @@ void testSummaryChanged() throws ParseException {
         }
         catch (IllegalArgumentException e) {
             assertEquals("For schema 'indexing_summary_fail', field 'foo': For expression 'summary foo': Attempting " +
-                         "to assign conflicting values to field 'foo'.",
+                         "to assign conflicting values to field 'foo'",
                          Exceptions.toMessageString(e));
         }
     }
@@ -186,7 +186,7 @@ void requireThatMultilineOutputConflictThrows() throws ParseException {
         }
         catch (IllegalArgumentException e) {
             assertEquals("For schema 'indexing_multiline_output_confict', field 'cox': For expression 'index cox': " +
-                         "Attempting to assign conflicting values to field 'cox'.",
+                         "Attempting to assign conflicting values to field 'cox'",
                          Exceptions.toMessageString(e));
         }
     }

document/abi-spec.json

@@ -200,6 +200,7 @@
       "public int getCode()",
       "public com.yahoo.document.FieldPath buildFieldPath(java.lang.String)",
       "public com.yahoo.document.PrimitiveDataType getPrimitiveType()",
+      "public com.yahoo.document.DataType getNestedType()",
       "public void visitMembers(com.yahoo.vespa.objects.ObjectVisitor)",
       "public int compareTo(com.yahoo.document.DataType)",
       "public boolean isMultivalue()",

document/src/main/java/com/yahoo/document/ArrayDataType.java

@@ -24,10 +24,12 @@ public ArrayDataType(DataType nestedType, int code) {
         super("Array<"+nestedType.getName()+">", code, nestedType);
     }
 
+    @Override
     public ArrayDataType clone() {
         return (ArrayDataType) super.clone();
     }
 
+    @Override
     public Array createFieldValue() {
         return new Array(this);
     }

document/src/main/java/com/yahoo/document/CollectionDataType.java

@@ -33,9 +33,8 @@ public CollectionDataType clone() {
         return type;
     }
 
-    public DataType getNestedType() {
-        return nestedType;
-    }
+    @Override
+    public DataType getNestedType() { return nestedType; }
 
     @Override
     protected FieldValue createByReflection(Object arg) { return null; }

document/src/main/java/com/yahoo/document/DataType.java

@@ -291,9 +291,10 @@ public FieldPath buildFieldPath(String fieldPathString) {
      *
      * @return primitive data type, or null
      */
-    public PrimitiveDataType getPrimitiveType() {
-        return null;
-    }
+    public PrimitiveDataType getPrimitiveType() { return null; }
+
+    /** Returns the nested type of this if it is a collection type, null otherwise. */
+    public DataType getNestedType() { return null; }
 
     @Override
     public void visitMembers(ObjectVisitor visitor) {

indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AnyDataType.java

@@ -0,0 +1,30 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.indexinglanguage.expressions;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.datatypes.FieldValue;
+
+/**
+ * A DataType representing "any". This is (so far) only needed during type resolution of indexing pipelines
+ * so it is placed here.
+ *
+ * @author bratseth
+ */
+class AnyDataType extends DataType {
+
+    static final AnyDataType instance = new AnyDataType();
+
+    private AnyDataType() {
+        super("any", DataType.lastPredefinedDataTypeId() + 1);
+    }
+
+    @Override
+    public FieldValue createFieldValue() { throw new UnsupportedOperationException(); }
+
+    @Override
+    public Class<?> getValueClass() { throw new UnsupportedOperationException(); }
+
+    @Override
+    public boolean isValueCompatible(FieldValue value) { return true; }
+
+}

indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java

@@ -10,6 +10,7 @@
 
 import java.math.BigDecimal;
 import java.math.MathContext;
+import java.util.Objects;
 
 /**
  * @author Simon Thoresen Hult
@@ -48,59 +49,45 @@ public String toString() {
 
     public ArithmeticExpression(Expression left, Operator op, Expression right) {
         super(requiredInputType(left, right));
-        left.getClass(); // throws NullPointerException
-        op.getClass();
-        right.getClass();
-        this.left = left;
-        this.op = op;
-        this.right = right;
+        this.left = Objects.requireNonNull(left);
+        this.op = Objects.requireNonNull(op);
+        this.right = Objects.requireNonNull(right);
     }
 
+    public Expression getLeftHandSide() { return left; }
+
+    public Operator getOperator() { return op; }
+
+    public Expression getRightHandSide() { return right; }
+
     @Override
     public ArithmeticExpression convertChildren(ExpressionConverter converter) {
         // TODO: branch()?
         return new ArithmeticExpression(converter.convert(left), op, converter.convert(right));
     }
 
-    public Expression getLeftHandSide() {
-        return left;
-    }
-
-    public Operator getOperator() {
-        return op;
-    }
-
-    public Expression getRightHandSide() {
-        return right;
-    }
-
     @Override
-    protected void doExecute(ExecutionContext context) {
-        FieldValue input = context.getValue();
-        context.setValue(evaluate(context.setValue(input).execute(left).getValue(),
-                                  context.setValue(input).execute(right).getValue()));
+    protected void doVerify(VerificationContext context) {
+        DataType input = context.getCurrentType();
+        context.setCurrentType(evaluate(context.setCurrentType(input).verify(left).getCurrentType(),
+                                        context.setCurrentType(input).verify(right).getCurrentType()));
     }
 
     @Override
-    protected void doVerify(VerificationContext context) {
-        DataType input = context.getValueType();
-        context.setValueType(evaluate(context.setValueType(input).execute(left).getValueType(),
-                                      context.setValueType(input).execute(right).getValueType()));
+    protected void doExecute(ExecutionContext context) {
+        FieldValue input = context.getCurrentValue();
+        context.setCurrentValue(evaluate(context.setCurrentValue(input).execute(left).getCurrentValue(),
+                                         context.setCurrentValue(input).execute(right).getCurrentValue()));
     }
 
     private static DataType requiredInputType(Expression lhs, Expression rhs) {
         DataType lhsType = lhs.requiredInputType();
         DataType rhsType = rhs.requiredInputType();
-        if (lhsType == null) {
-            return rhsType;
-        }
-        if (rhsType == null) {
-            return lhsType;
-        }
-        if (!lhsType.equals(rhsType)) {
+        if (lhsType == null) return rhsType;
+        if (rhsType == null) return lhsType;
+        if (!lhsType.equals(rhsType))
             throw new VerificationException(ArithmeticExpression.class, "Operands require conflicting input types, " +
                                                                         lhsType.getName() + " vs " + rhsType.getName());
-        }
         return lhsType;
     }
 
@@ -115,20 +102,12 @@ public String toString() {
     }
 
     @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof ArithmeticExpression)) {
-            return false;
-        }
-        ArithmeticExpression exp = (ArithmeticExpression)obj;
-        if (!left.equals(exp.left)) {
-            return false;
-        }
-        if (!op.equals(exp.op)) {
-            return false;
-        }
-        if (!right.equals(exp.right)) {
-            return false;
-        }
+    public boolean equals(Object object) {
+        if (!(object instanceof ArithmeticExpression expression)) return false;
+
+        if (!left.equals(expression.left)) return false;
+        if (!op.equals(expression.op)) return false;
+        if (!right.equals(expression.right)) return false;
         return true;
     }
 
@@ -138,39 +117,28 @@ public int hashCode() {
     }
 
     private DataType evaluate(DataType lhs, DataType rhs) {
-        if (lhs == null || rhs == null) {
+        if (lhs == null || rhs == null)
             throw new VerificationException(this, "Attempting to perform arithmetic on a null value");
-        }
-        if (!(lhs instanceof NumericDataType) ||
-            !(rhs instanceof NumericDataType))
-        {
+        if (!(lhs instanceof NumericDataType) || !(rhs instanceof NumericDataType))
             throw new VerificationException(this, "Attempting to perform unsupported arithmetic: [" +
                                                   lhs.getName() + "] " + op + " [" + rhs.getName() + "]");
-        }
-        if (lhs == DataType.FLOAT || lhs == DataType.DOUBLE ||
-            rhs == DataType.FLOAT || rhs == DataType.DOUBLE)
-        {
-            if (lhs == DataType.DOUBLE || rhs == DataType.DOUBLE) {
+
+        if (lhs == DataType.FLOAT || lhs == DataType.DOUBLE || rhs == DataType.FLOAT || rhs == DataType.DOUBLE) {
+            if (lhs == DataType.DOUBLE || rhs == DataType.DOUBLE)
                 return DataType.DOUBLE;
-            }
             return DataType.FLOAT;
         }
-        if (lhs == DataType.LONG || rhs == DataType.LONG) {
+        if (lhs == DataType.LONG || rhs == DataType.LONG)
             return DataType.LONG;
-        }
         return DataType.INT;
     }
 
     private FieldValue evaluate(FieldValue lhs, FieldValue rhs) {
-        if (lhs == null || rhs == null) {
-            return null;
-        }
-        if (!(lhs instanceof NumericFieldValue) ||
-            !(rhs instanceof NumericFieldValue))
-        {
+        if (lhs == null || rhs == null) return null;
+        if (!(lhs instanceof NumericFieldValue) || !(rhs instanceof NumericFieldValue))
             throw new IllegalArgumentException("Unsupported operation: [" + lhs.getDataType().getName() + "] " +
                                                op + " [" + rhs.getDataType().getName() + "]");
-        }
+
         BigDecimal lhsVal = asBigDecimal((NumericFieldValue)lhs);
         BigDecimal rhsVal = asBigDecimal((NumericFieldValue)rhs);
         return switch (op) {
@@ -184,31 +152,27 @@ private FieldValue evaluate(FieldValue lhs, FieldValue rhs) {
 
     private FieldValue createFieldValue(FieldValue lhs, FieldValue rhs, BigDecimal val) {
         if (lhs instanceof FloatFieldValue || lhs instanceof DoubleFieldValue ||
-            rhs instanceof FloatFieldValue || rhs instanceof DoubleFieldValue)
-        {
-            if (lhs instanceof DoubleFieldValue || rhs instanceof DoubleFieldValue) {
+            rhs instanceof FloatFieldValue || rhs instanceof DoubleFieldValue) {
+            if (lhs instanceof DoubleFieldValue || rhs instanceof DoubleFieldValue)
                 return new DoubleFieldValue(val.doubleValue());
-            }
             return new FloatFieldValue(val.floatValue());
         }
-        if (lhs instanceof LongFieldValue || rhs instanceof LongFieldValue) {
+        if (lhs instanceof LongFieldValue || rhs instanceof LongFieldValue)
             return new LongFieldValue(val.longValue());
-        }
         return new IntegerFieldValue(val.intValue());
     }
 
     public static BigDecimal asBigDecimal(NumericFieldValue value) {
-        if (value instanceof ByteFieldValue) {
+        if (value instanceof ByteFieldValue)
             return BigDecimal.valueOf(((ByteFieldValue)value).getByte());
-        } else if (value instanceof DoubleFieldValue) {
+        else if (value instanceof DoubleFieldValue)
             return BigDecimal.valueOf(((DoubleFieldValue)value).getDouble());
-        } else if (value instanceof FloatFieldValue) {
+        else if (value instanceof FloatFieldValue)
             return BigDecimal.valueOf(((FloatFieldValue)value).getFloat());
-        } else if (value instanceof IntegerFieldValue) {
+        else if (value instanceof IntegerFieldValue)
             return BigDecimal.valueOf(((IntegerFieldValue)value).getInteger());
-        } else if (value instanceof LongFieldValue) {
+        else if (value instanceof LongFieldValue)
             return BigDecimal.valueOf(((LongFieldValue)value).getLong());
-        }
         throw new IllegalArgumentException("Unsupported numeric field value type '" +
                                            value.getClass().getName() + "'");
     }
@@ -218,4 +182,5 @@ public void selectMembers(ObjectPredicate predicate, ObjectOperation operation)
         left.select(predicate, operation);
         right.select(predicate, operation);
     }
+
 }

indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpression.java

@@ -14,4 +14,5 @@ public AttributeExpression(String fieldName) {
     public boolean equals(Object obj) {
         return super.equals(obj) && obj instanceof AttributeExpression;
     }
+
 }