diff --git a/CONTRIBUTORS b/CONTRIBUTORS index aed80321e..e3828d8a2 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -17,3 +17,7 @@ Imperial College London David MacIver Dragos Martac Paul Thomson + +Independent contributors via Google Summer of Code + Abel Briggs + Jiradet Ounjai diff --git a/assembly-binaries/pom.xml b/assembly-binaries/pom.xml index 2f78a1432..5d0d93dc1 100644 --- a/assembly-binaries/pom.xml +++ b/assembly-binaries/pom.xml @@ -58,10 +58,10 @@ limitations under the License. - ac0884b2c8dfd214f0d293c12cf3b5931b1eb345 + b97215064186d731eac68adcc5ade4c7b96b265b 9d3202492d78aae5ced08ff14679b81c98d71c15 - 803082da4f73920729938905cb1727e11ff79530 - b938f8045de9556bcd763eca362c2443d25c0b77 + 0565d5aee279a4782689297ada0aa2489e24ad3e + 0f167ce7125795df62ae5893f553e5608c9652f4 d987dc0e5f8aeae0dc2f5aac2013e9a3edd54465 diff --git a/assembly-public/pom.xml b/assembly-public/pom.xml deleted file mode 100644 index 329a7ac4b..000000000 --- a/assembly-public/pom.xml +++ /dev/null @@ -1,196 +0,0 @@ - - - - 4.0.0 - assembly-public - assembly-public - pom - - - com.graphicsfuzz - parent-all - 1.0 - ../parent-all/pom.xml - - - - - - - com.graphicsfuzz - server-public - - - - com.graphicsfuzz - assembly-binaries - zip - provided - - - - com.graphicsfuzz - thrift - python - zip - provided - - - - com.graphicsfuzz - server-static-public - zip - provided - - - - com.graphicsfuzz - thrift - js - zip - provided - - - - com.graphicsfuzz.thirdparty - jquery-js - zip - provided - - - - ant-contrib - ant-contrib - provided - - - - - - - - - pl.project13.maven - git-commit-id-plugin - - - get-the-git-infos - - revision - - initialize - - - - false - - 40 - - - - - - maven-dependency-plugin - - - dep-1 - - copy-dependencies - - generate-sources - - runtime - true - - - - - - - maven-antrun-plugin - - - run-ant - compile - - run - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/assembly-public/src/main/resources/HASH b/assembly-public/src/main/resources/HASH deleted file mode 100644 index f53be555b..000000000 --- a/assembly-public/src/main/resources/HASH +++ /dev/null @@ -1 +0,0 @@ -${git.commit.id.describe} \ No newline at end of file diff --git a/assembly-public/src/main/scripts/README.md b/assembly-public/src/main/scripts/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/ast/pom.xml b/ast/pom.xml index a7d5077c2..ee3f2ba1a 100644 --- a/ast/pom.xml +++ b/ast/pom.xml @@ -68,10 +68,6 @@ limitations under the License. net.sourceforge.argparse4j argparse4j - - org.apache.commons - commons-lang3 - diff --git a/ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 b/ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 index 07efaa322..add4ebe5c 100644 --- a/ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 +++ b/ast/src/main/antlr4/com/graphicsfuzz/parser/GLSL.g4 @@ -59,9 +59,6 @@ variable_identifier: primary_expression: variable_identifier - { - // NEW VARIABLE DECLARATION - } | INTCONSTANT | UINTCONSTANT | FLOATCONSTANT @@ -103,9 +100,8 @@ function_call_header: ; function_identifier: - builtin_type_specifier_nonarray // constructor + builtin_type_specifier_nonarray // type constructor | variable_identifier - | FIELD_SELECTION ; method_call_generic: @@ -123,14 +119,10 @@ method_call_header_with_parameters: | method_call_header_with_parameters COMMA assignment_expression ; - // Grammar Note: Constructors look like methods, but lexical - // analysis recognized most of them as keywords. They are now - // recognized through "type_specifier". method_call_header: variable_identifier LPAREN ; - // Grammar Note: No traditional style type casts. unary_expression: postfix_expression | INC_OP unary_expression @@ -138,7 +130,6 @@ unary_expression: | unary_operator unary_expression ; - // Grammar Note: No '*' or '&' unary ops. Pointers are not supported. unary_operator: PLUS_OP | MINUS_OP @@ -555,8 +546,6 @@ declaration_statement: declaration ; - // Grammar Note: labeled statements for SWITCH only; 'goto' is not - // supported. statement: compound_statement | simple_statement @@ -603,7 +592,7 @@ selection_statement: selection_rest_statement: statement ELSE statement - | statement // REVISIT: binding issue with conditionals + | statement ; condition: @@ -611,10 +600,6 @@ condition: | fully_specified_type IDENTIFIER ASSIGN_OP initializer ; -/// - // switch_statement grammar is based on the syntax described in the body - // of the GLSL spec, not in it's appendix!!! - /// switch_statement: SWITCH LPAREN expression RPAREN switch_body ; @@ -655,23 +640,17 @@ for_init_statement: | declaration_statement ; -conditionopt: - condition - | /// empty /// - ; - for_rest_statement: - conditionopt SEMICOLON - | conditionopt SEMICOLON expression + condition? SEMICOLON + | condition? SEMICOLON expression ; - // Grammar Note: No 'goto'. Gotos are not supported. jump_statement: CONTINUE SEMICOLON | BREAK SEMICOLON | RETURN SEMICOLON | RETURN expression SEMICOLON - | DISCARD SEMICOLON // Fragment shader only. + | DISCARD SEMICOLON ; external_declaration: @@ -685,14 +664,13 @@ function_definition: function_prototype compound_statement_no_new_scope ; -/// layout_qualifieropt is packed into this rule /// interface_block: basic_interface_block | layout_qualifier basic_interface_block ; basic_interface_block: - interface_qualifier IDENTIFIER LBRACE member_list RBRACE instance_name_opt SEMICOLON + interface_qualifier IDENTIFIER LBRACE member_list RBRACE instance_name? SEMICOLON ; interface_qualifier: @@ -702,10 +680,8 @@ interface_qualifier: | BUFFER ; -instance_name_opt: - /// empty /// - | IDENTIFIER - | IDENTIFIER array_specifier +instance_name: + IDENTIFIER array_specifier? ; layout_defaults: @@ -749,7 +725,7 @@ VARYING: 'varying' ; READONLY: 'readonly' ; WRITEONLY: 'writeonly' ; SHARED: 'shared' ; -LAYOUT_TOK: 'layout' ; // REVISIT +LAYOUT_TOK: 'layout' ; UINTCONSTANT: (DECIMAL_DIGITS | OCTAL_DIGITS | HEX_DIGITS) 'u'; ROW_MAJOR: 'row_major' ; PACKED_TOK: 'packed' ; @@ -758,7 +734,6 @@ BOOLCONSTANT: 'true' | 'false' ; INC_OP: '++' ; DEC_OP: '--' ; VOID_TOK: 'void' ; -FIELD_SELECTION: 'C_TODO' ; // REVISIT LEFT_OP: '<<' ; RIGHT_OP: '>>' ; LE_OP: '<=' ; @@ -782,65 +757,26 @@ FLOAT_TOK: 'float' ; INT_TOK: 'int' ; UINT_TOK: 'uint' ; BOOL_TOK: 'bool' ; -// VEC: 'TODO' ; -// BVEC: 'TODO' ; -// IVEC: 'TODO' ; -// UVEC: 'TODO' ; -// MATX: 'TODO' ; -// SAMPLERD: 'TODO' ; -// SAMPLERDRECT: 'TODO' ; SAMPLERCUBE: 'samplerCube' ; -SAMPLEREXTERNALOES: 'samplerExternalOES' ; // REVISIT -// SAMPLERDSHADOW: 'TODO' ; -// SAMPLERDRECTSHADOW: 'TODO' ; SAMPLERCUBESHADOW: 'samplerCubeShadow' ; -// SAMPLERDARRAY: 'TODO' ; -// SAMPLERDARRAYSHADOW: 'TODO' ; SAMPLERBUFFER: 'samplerBuffer' ; SAMPLERCUBEARRAY: 'samplerCubeArray' ; SAMPLERCUBEARRAYSHADOW: 'samplerCubeArrayShadow' ; -// ISAMPLERD: 'TODO' ; -// ISAMPLERDRECT: 'TODO' ; ISAMPLERCUBE: 'isamplerCube' ; -// ISAMPLERDARRAY: 'TODO' ; ISAMPLERBUFFER: 'isamplerBuffer' ; ISAMPLERCUBEARRAY: 'isamplerCubeArray' ; -// USAMPLERD: 'TODO' ; -// USAMPLERDRECT: 'TODO' ; USAMPLERCUBE: 'usamplerCube' ; -// USAMPLERDARRAY: 'TODO' ; USAMPLERBUFFER: 'usamplerBuffer' ; USAMPLERCUBEARRAY: 'usamplerCubeArray' ; -// SAMPLERDMS: 'TODO' ; -// ISAMPLERDMS: 'TODO' ; -// USAMPLERDMS: 'TODO' ; -// SAMPLERDMSARRAY: 'TODO' ; -// ISAMPLERDMSARRAY: 'TODO' ; -// USAMPLERDMSARRAY: 'TODO' ; -// IMAGED: 'TODO' ; -// IMAGEDRECT: 'TODO' ; IMAGECUBE: 'imageCube' ; IMAGEBUFFER: 'imageBuffer' ; -// IMAGEDARRAY: 'TODO' ; IMAGECUBEARRAY: 'imageCubeArray' ; -// IMAGEDMS: 'TODO' ; -// IMAGEDMSARRAY: 'TODO' ; -// IIMAGED: 'TODO' ; -// IIMAGEDRECT: 'TODO' ; IIMAGECUBE: 'iimageCube' ; IIMAGEBUFFER: 'iimageBuffer' ; -// IIMAGEDARRAY: 'TODO' ; IIMAGECUBEARRAY: 'iimageCubeArray' ; -// IIMAGEDMS: 'TODO' ; -// IIMAGEDMSARRAY: 'TODO' ; -// UIMAGED: 'TODO' ; -// UIMAGEDRECT: 'TODO' ; UIMAGECUBE: 'uimageCube' ; UIMAGEBUFFER: 'uimageBuffer' ; -// UIMAGEDARRAY: 'TODO' ; UIMAGECUBEARRAY: 'uimageCubeArray' ; -// UIMAGEDMS: 'TODO' ; -// UIMAGEDMSARRAY: 'TODO' ; ATOMIC_UINT: 'atomic_uint' ; STRUCT: 'struct' ; IF: 'if' ; @@ -889,7 +825,7 @@ SAMPLER1D: 'sampler1D' ; SAMPLER2D: 'sampler2D' ; SAMPLER3D: 'sampler3D' ; SAMPLER2DRECT: 'sampler2DRect' ; - +SAMPLEREXTERNALOES: 'samplerExternalOES' ; SAMPLER1DSHADOW: 'sampler1DShadow' ; SAMPLER2DSHADOW: 'sampler2DShadow' ; SAMPLER2DRECTSHADOW: 'sampler2DRectShadow' ; @@ -968,72 +904,3 @@ COMMENT: ('//' ~('\n'|'\r')* '\r'? '\n' | '/*' (.)*? '*/') -> skip ; WS: [\t\r\u000C ]+ { skip(); } ; EOL: '\n' { if(ignoreNewLine) { skip(); } ignoreNewLine = true; } ; - -/* - -%token ATTRIBUTE CONST_TOK BOOL_TOK FLOAT_TOK INT_TOK UINT_TOK -%token BREAK CONTINUE DO ELSE FOR IF DISCARD RETURN SWITCH CASE DEFAULT -%token BVEC2 BVEC3 BVEC4 IVEC2 IVEC3 IVEC4 UVEC2 UVEC3 UVEC4 VEC2 VEC3 VEC4 -%token CENTROID IN_TOK OUT_TOK INOUT_TOK UNIFORM VARYING SAMPLE -%token NOPERSPECTIVE FLAT SMOOTH -%token MAT2X2 MAT2X3 MAT2X4 -%token MAT3X2 MAT3X3 MAT3X4 -%token MAT4X2 MAT4X3 MAT4X4 -%token SAMPLER1D SAMPLER2D SAMPLER3D SAMPLERCUBE SAMPLER1DSHADOW SAMPLER2DSHADOW -%token SAMPLERCUBESHADOW SAMPLER1DARRAY SAMPLER2DARRAY SAMPLER1DARRAYSHADOW -%token SAMPLER2DARRAYSHADOW SAMPLERCUBEARRAY SAMPLERCUBEARRAYSHADOW -%token ISAMPLER1D ISAMPLER2D ISAMPLER3D ISAMPLERCUBE -%token ISAMPLER1DARRAY ISAMPLER2DARRAY ISAMPLERCUBEARRAY -%token USAMPLER1D USAMPLER2D USAMPLER3D USAMPLERCUBE USAMPLER1DARRAY -%token USAMPLER2DARRAY USAMPLERCUBEARRAY -%token SAMPLER2DRECT ISAMPLER2DRECT USAMPLER2DRECT SAMPLER2DRECTSHADOW -%token SAMPLERBUFFER ISAMPLERBUFFER USAMPLERBUFFER -%token SAMPLER2DMS ISAMPLER2DMS USAMPLER2DMS -%token SAMPLER2DMSARRAY ISAMPLER2DMSARRAY USAMPLER2DMSARRAY -%token SAMPLEREXTERNALOES -%token IMAGE1D IMAGE2D IMAGE3D IMAGE2DRECT IMAGECUBE IMAGEBUFFER -%token IMAGE1DARRAY IMAGE2DARRAY IMAGECUBEARRAY IMAGE2DMS IMAGE2DMSARRAY -%token IIMAGE1D IIMAGE2D IIMAGE3D IIMAGE2DRECT IIMAGECUBE IIMAGEBUFFER -%token IIMAGE1DARRAY IIMAGE2DARRAY IIMAGECUBEARRAY IIMAGE2DMS IIMAGE2DMSARRAY -%token UIMAGE1D UIMAGE2D UIMAGE3D UIMAGE2DRECT UIMAGECUBE UIMAGEBUFFER -%token UIMAGE1DARRAY UIMAGE2DARRAY UIMAGECUBEARRAY UIMAGE2DMS UIMAGE2DMSARRAY -%token IMAGE1DSHADOW IMAGE2DSHADOW IMAGE1DARRAYSHADOW IMAGE2DARRAYSHADOW -%token COHERENT VOLATILE RESTRICT READONLY WRITEONLY -%token ATOMIC_UINT -%token STRUCT VOID_TOK WHILE -%token IDENTIFIER TYPE_IDENTIFIER IDENTIFIER -%token FLOATCONSTANT -%token INTCONSTANT UINTCONSTANT BOOLCONSTANT -%token FIELD_SELECTION -%token LEFT_OP RIGHT_OP -%token INC_OP DEC_OP LE_OP GE_OP EQ_OP NE_OP -%token AND_OP OR_OP XOR_OP MUL_ASSIGN DIV_ASSIGN ADD_ASSIGN -%token MOD_ASSIGN LEFT_ASSIGN RIGHT_ASSIGN AND_ASSIGN XOR_ASSIGN OR_ASSIGN -%token SUB_ASSIGN -%token INVARIANT PRECISE -%token LOWP MEDIUMP HIGHP SUPERP PRECISION - -%token VERSION EXTENSION LINE COLON EOL INTERFACE OUTPUT -%token PRAGMA_DEBUG_ON PRAGMA_DEBUG_OFF -%token PRAGMA_OPTIMIZE_ON PRAGMA_OPTIMIZE_OFF -%token PRAGMA_INVARIANT_ALL -%token LAYOUT_TOK - -// Reserved words that are not actually used in the grammar. -%token ASM CLASS UNION ENUM TYPEDEF TEMPLATE THIS PACKED_TOK GOTO -%token INLINE_TOK NOINLINE PUBLIC_TOK STATIC EXTERN EXTERNAL -%token LONG_TOK SHORT_TOK DOUBLE_TOK HALF FIXED_TOK UNSIGNED INPUT_TOK -%token HVEC2 HVEC3 HVEC4 DVEC2 DVEC3 DVEC4 FVEC2 FVEC3 FVEC4 -%token SAMPLER3DRECT -%token SIZEOF CAST NAMESPACE USING -%token RESOURCE PATCH -%token SUBROUTINE - -%token ERROR_TOK - -%token COMMON PARTITION ACTIVE FILTER ROW_MAJOR - -%right THEN ELSE -%% - -*/ diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInfo.java b/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInfo.java index 2eff72fc8..0bddbf1d4 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInfo.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInfo.java @@ -17,27 +17,102 @@ package com.graphicsfuzz.common.ast.decl; import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.visitors.IAstVisitor; +import com.graphicsfuzz.common.ast.visitors.UnsupportedLanguageFeatureException; import java.util.Optional; public class ArrayInfo implements IAstNode { - private Optional size; + /** + * Size, set after folding. + */ + private Optional constantSize; + /** + * Original size expression before folding, for pretty printing. + */ + private Optional sizeExpr; - public ArrayInfo(Integer size) { - this.size = Optional.of(size); + /** + * "originalSize" is the expression in the original shader representing the array size, + * and is empty if no size was given. If "originalSize" is present, constant folding can + * be used to turn the expression into an integer value, which can be stored in + * "constantSize". It is useful to keep "originalSize" around to allow the shader to + * be pretty-printed in its original form. + * @param originalSize Array size expression + */ + public ArrayInfo(Expr originalSize) { + this.constantSize = Optional.empty(); + this.sizeExpr = Optional.of(originalSize); } + /** + * Private constructor for cloning, needed since the constant size expression may have been + * folded by the time the expression is cloned. + * @param constantSize Possible constant-folded size + * @param originalSize Original size, for pretty printing + */ + private ArrayInfo(Optional constantSize, Optional originalSize) { + this.constantSize = constantSize; + this.sizeExpr = originalSize; + } + + /** + * Default constructor, for arrays with no size definition. + */ public ArrayInfo() { - this.size = Optional.empty(); + this.constantSize = Optional.empty(); + this.sizeExpr = Optional.empty(); + } + + /** + * Query for whether this array has a size definition. + * @return true if size definition is available + */ + public boolean hasConstantSize() { + return constantSize.isPresent(); + } + + /** + * Query whether this array has a size expression + * @return Size expression + */ + public boolean hasSizeExpr() { + return sizeExpr.isPresent(); + } + + /** + * Get the constant size. + * If constant folding was not performed yet (for example, the AST is stil being built) or if + * the array has no size declaration, calling this function will throw an exception. + * @return Integer value of the array size + * @throws UnsupportedLanguageFeatureException if folding was not done + */ + public Integer getConstantSize() throws UnsupportedLanguageFeatureException { + if (hasConstantSize()) { + return constantSize.get(); + } + // TODO(https://github.com/google/graphicsfuzz/issues/784) Until array parameter support + // is overhauled there could be array parameters to which constant folding has not been + // applied. + throw new UnsupportedLanguageFeatureException("Not a constant expression"); } - public boolean hasSize() { - return size.isPresent(); + /** + * Set constant expression after folding. + * @param foldedSize Completely folded size + */ + public void setConstantSizeExpr(int foldedSize) { + constantSize = Optional.of(foldedSize); } - public Integer getSize() { - return size.get(); + /** + * Get the original expression for pretty printing. + * @return Original, non-folded size expression + */ + public Expr getSizeExpr() { + return sizeExpr.get(); } @Override @@ -47,17 +122,17 @@ public void accept(IAstVisitor visitor) { @Override public ArrayInfo clone() { - return hasSize() ? new ArrayInfo(getSize()) : new ArrayInfo(); + return new ArrayInfo(constantSize, sizeExpr); } @Override public boolean equals(Object obj) { - return obj instanceof ArrayInfo && size.equals(((ArrayInfo) obj).size); + return obj instanceof ArrayInfo && sizeExpr.equals(((ArrayInfo) obj).sizeExpr); } @Override public int hashCode() { - return size.hashCode(); + return sizeExpr.hashCode(); } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/Initializer.java b/ast/src/main/java/com/graphicsfuzz/common/ast/decl/Initializer.java index 4c5645672..90e2c14bd 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/Initializer.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/decl/Initializer.java @@ -17,10 +17,46 @@ package com.graphicsfuzz.common.ast.decl; import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.visitors.IAstVisitor; -public abstract class Initializer implements IAstNode { +public class Initializer implements IAstNode { + + private Expr expr; + + public Initializer(Expr expr) { + this.expr = expr; + } + + public Expr getExpr() { + return expr; + } + + public void setExpr(Expr expr) { + this.expr = expr; + } + + @Override + public void accept(IAstVisitor visitor) { + visitor.visitInitializer(this); + } + + @Override + public Initializer clone() { + return new Initializer(expr.clone()); + } + + @Override + public void replaceChild(IAstNode child, IAstNode newChild) { + if (!(child == expr && newChild instanceof Expr)) { + throw new IllegalArgumentException(); + } + setExpr((Expr) newChild); + } @Override - public abstract Initializer clone(); + public boolean hasChild(IAstNode candidateChild) { + return candidateChild == expr; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ScalarInitializer.java b/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ScalarInitializer.java deleted file mode 100644 index 94456a1c2..000000000 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ScalarInitializer.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.common.ast.decl; - -import com.graphicsfuzz.common.ast.IAstNode; -import com.graphicsfuzz.common.ast.expr.Expr; -import com.graphicsfuzz.common.ast.visitors.IAstVisitor; - -public class ScalarInitializer extends Initializer { - - private Expr expr; - - public ScalarInitializer(Expr expr) { - this.expr = expr; - } - - public Expr getExpr() { - return expr; - } - - public void setExpr(Expr expr) { - this.expr = expr; - } - - @Override - public void accept(IAstVisitor visitor) { - visitor.visitScalarInitializer(this); - } - - @Override - public ScalarInitializer clone() { - return new ScalarInitializer(expr.clone()); - } - - @Override - public void replaceChild(IAstNode child, IAstNode newChild) { - if (!(child == expr && newChild instanceof Expr)) { - throw new IllegalArgumentException(); - } - setExpr((Expr) newChild); - } - - @Override - public boolean hasChild(IAstNode candidateChild) { - return candidateChild == expr; - } - -} diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/Inliner.java b/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/Inliner.java index c29a186bf..662f4ac07 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/Inliner.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/Inliner.java @@ -20,8 +20,8 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.ParameterDecl; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -71,7 +71,7 @@ private Inliner(FunctionCallExpr call, TranslationUnit tu, this.call = call; this.tu = tu; this.shadingLanguageVersion = shadingLanguageVersion; - this.typer = new Typer(tu, shadingLanguageVersion); + this.typer = new Typer(tu); this.parentMap = IParentMap.createParentMap(tu); } @@ -186,7 +186,7 @@ private List getInlinedStmts(FunctionDefinition functionDefinition, new VariablesDeclaration( pd.getType().getWithoutQualifiers(), new VariableDeclInfo(pd.getName(), null, - new ScalarInitializer(call.getArg(i).clone()))))); + new Initializer(call.getArg(i).clone()))))); } for (Stmt stmt : functionDefinition.getBody().getStmts()) { if (stmt instanceof ReturnStmt) { diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/ReturnRemover.java b/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/ReturnRemover.java index 1e4ddb8d1..b4b4d3142 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/ReturnRemover.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/inliner/ReturnRemover.java @@ -19,7 +19,7 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.IParentMap; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -170,7 +170,7 @@ private void addSpecialDeclarations() { fd.getBody().insertStmt(0, new DeclarationStmt( new VariablesDeclaration(BasicType.BOOL, new VariableDeclInfo(makeHasReturnedName(), null, - new ScalarInitializer(new BoolConstantExpr(false)))))); + new Initializer(new BoolConstantExpr(false)))))); } private void addReturnInstrumentation() { diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/BlockStmt.java b/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/BlockStmt.java index abfa2b928..16e7de17b 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/BlockStmt.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/BlockStmt.java @@ -58,6 +58,16 @@ public int getNumStmts() { return stmts.size(); } + /** + * Requires the block to be non-empty. + * Yields the final statement in the block. + * @return The final statement of the block. + */ + public Stmt getLastStmt() { + assert !stmts.isEmpty(); + return stmts.get(stmts.size() - 1); + } + public boolean introducesNewScope() { return introducesNewScope; } @@ -154,6 +164,10 @@ public void insertBefore(Stmt originalStmt, Stmt insertedStmt) { throw new IllegalArgumentException("Should be unreachable."); } + /** + * Adds the given statement to the end of the block. + * @param stmt A statement to be added to the block + */ public void addStmt(Stmt stmt) { stmts.add(stmt); } diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/ReturnStmt.java b/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/ReturnStmt.java index 40453de2f..75cfd43b4 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/ReturnStmt.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/stmt/ReturnStmt.java @@ -25,17 +25,22 @@ public class ReturnStmt extends Stmt { private Expr expr; public ReturnStmt(Expr expr) { + assert expr != null; this.expr = expr; } public ReturnStmt() { - this(null); + this.expr = null; } public Expr getExpr() { return expr; } + public void setExpr(Expr expr) { + this.expr = expr; + } + public boolean hasExpr() { return getExpr() != null; } diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java index 78f912fec..af7ef4b33 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/type/BasicType.java @@ -324,6 +324,24 @@ public static List allNumericTypes() { return result; } + /** + * Helper function to return a list of every basic type that is considered an integer. + * + * @return a list of all basic integer types. + */ + public static List allIntegerTypes() { + return new ArrayList(Arrays.asList( + UINT, + UVEC2, + UVEC3, + UVEC4, + INT, + IVEC2, + IVEC3, + IVEC4 + )); + } + public static List allNumericTypesExceptNonSquareMatrices() { final List result = allNumericTypes(); result.removeAll(allNonSquareMatrixTypes()); @@ -344,26 +362,22 @@ private static List allBoolTypes() { BVEC4)); } - public static int numColumns(BasicType basicType) { - assert allMatrixTypes().contains(basicType); - if (Arrays.asList(MAT2X2, MAT2X3, MAT2X4).contains(basicType)) { - return 2; - } - if (Arrays.asList(MAT3X2, MAT3X3, MAT3X4).contains(basicType)) { - return 3; - } - if (Arrays.asList(MAT4X2, MAT4X3, MAT4X4).contains(basicType)) { - return 4; - } - throw new RuntimeException("Camnnot get number of columns for " + basicType); - } - + /** + * Creates a vector type of a specified size from a scalar type. + * + * @return a vector type of numElements dimension, or the scalar type if numElements is 1. + * @throws UnsupportedOperationException if the specified base type is not a scalar, or + * if numElements is outside the bounds of possible dimensions + * (numElements < 0, numElements > 4) + */ public static BasicType makeVectorType(BasicType elementType, int numElements) { - if (!allScalarTypes().contains(elementType)) { - throw new RuntimeException("Cannot make vector type from element type " + elementType); + if (!elementType.isScalar()) { + throw new UnsupportedOperationException( + "Cannot make vector type from element type " + elementType); } if (numElements < 0 || numElements > 4) { - throw new RuntimeException("Cannot make vector type with " + numElements + " elements"); + throw new UnsupportedOperationException( + "Cannot make vector type with " + numElements + " elements"); } if (elementType == FLOAT) { switch (numElements) { @@ -417,10 +431,58 @@ public static BasicType makeVectorType(BasicType elementType, int numElements) { return BVEC4; } } + // Should not be reachable. assert false; return null; } + /** + * Creates a matrix type of a specified size from the given dimensions. + * + * @return a matrix type of numColumns columns and numRows rows. + * @throws UnsupportedOperationException if numColumns or numRows are outside the bounds of + * possible GLSL matrix dimensions (numColumns < 2, numColumns > 4, numRows < 2, numRows > 4) + */ + public static BasicType makeMatrixType(int numColumns, int numRows) { + if (numColumns < 2 || numColumns > 4 || numRows < 2 || numRows > 4) { + throw new UnsupportedOperationException( + "Cannot make matrix type with " + numColumns + " columns and " + numRows + " rows."); + } + switch (numColumns) { + case 2: + switch (numRows) { + case 2: + return MAT2X2; + case 3: + return MAT2X3; + default: + return MAT2X4; + } + case 3: + switch (numRows) { + case 2: + return MAT3X2; + case 3: + return MAT3X3; + default: + return MAT3X4; + } + case 4: + switch (numRows) { + case 2: + return MAT4X2; + case 3: + return MAT4X3; + default: + return MAT4X4; + } + default: + // Should not be reachable. + assert false; + return null; + } + } + public boolean isScalar() { return allScalarTypes().contains(this); } @@ -433,8 +495,34 @@ public boolean isMatrix() { return allMatrixTypes().contains(this); } + /** + * Creates a matrix type that is the transpose of a given matrix type. A matrix type can be + * transposed by swapping its column and row widths. + * + * @return the type that is the transpose of the given matrix type. + * @throws UnsupportedOperationException if the type this method is called on is not a matrix + * type (and thus has no transpose). + */ + public BasicType transposedMatrixType() { + if (!this.isMatrix()) { + throw new UnsupportedOperationException( + "Cannot transpose non-matrix type " + this); + } + return BasicType.makeMatrixType(this.getNumRows(), this.getNumColumns()); + } + + /** + * Determines the vector type of the columns in the matrix. For example, accessing a column of a + * mat2x2 would give you a variable of type vec2. Can only be invoked on a matrix type. + * + * @return the type that represents that the matrix type has. + * @throws UnsupportedOperationException if the type is not a matrix. + */ public BasicType getColumnType() { - assert allMatrixTypes().contains(this); + if (!this.isMatrix()) { + throw new UnsupportedOperationException( + "Type" + this.toString() + " does not have a column type"); + } if (Arrays.asList(BasicType.MAT2X2, BasicType.MAT3X2, BasicType.MAT4X2).contains(this)) { return VEC2; } @@ -445,8 +533,17 @@ public BasicType getColumnType() { return VEC4; } + /** + * Finds the number of columns in a matrix. Can only be invoked on a matrix type. + * + * @return the number of columns that the matrix type has. + * @throws UnsupportedOperationException if the type is not a matrix. + */ public int getNumColumns() { - assert allMatrixTypes().contains(this); + if (!this.isMatrix()) { + throw new UnsupportedOperationException( + "Type" + this.toString() + " does not have columns"); + } if (Arrays.asList(BasicType.MAT2X2, BasicType.MAT2X3, BasicType.MAT2X4).contains(this)) { return 2; } @@ -456,4 +553,25 @@ public int getNumColumns() { assert Arrays.asList(BasicType.MAT4X2, BasicType.MAT4X3, BasicType.MAT4X4).contains(this); return 4; } + + /** + * Finds the number of rows in a matrix. Can only be invoked on a matrix type. + * + * @return the number of rows that the matrix type has. + * @throws UnsupportedOperationException if the type is not a matrix. + */ + public int getNumRows() { + if (!this.isMatrix()) { + throw new UnsupportedOperationException( + "Type" + this.toString() + " does not have rows"); + } + if (Arrays.asList(BasicType.MAT2X2, BasicType.MAT3X2, BasicType.MAT4X2).contains(this)) { + return 2; + } + if (Arrays.asList(BasicType.MAT2X3, BasicType.MAT3X3, BasicType.MAT4X3).contains(this)) { + return 3; + } + assert Arrays.asList(BasicType.MAT2X4, BasicType.MAT3X4, BasicType.MAT4X4).contains(this); + return 4; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java index fa29dc4a9..f0294b93f 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/AstBuilder.java @@ -16,6 +16,8 @@ package com.graphicsfuzz.common.ast.visitors; +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.IParentMap; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.ArrayInfo; import com.graphicsfuzz.common.ast.decl.Declaration; @@ -26,7 +28,6 @@ import com.graphicsfuzz.common.ast.decl.InterfaceBlock; import com.graphicsfuzz.common.ast.decl.ParameterDecl; import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayConstructorExpr; @@ -86,6 +87,7 @@ import com.graphicsfuzz.common.ast.type.UnknownLayoutQualifier; import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.parser.GLSLBaseVisitor; import com.graphicsfuzz.parser.GLSLParser; @@ -196,9 +198,158 @@ void addTopLevelDeclaration(Declaration decl) { topLevelDeclarations.add(decl); } + /** + * Performs constant folding for array size fields + * @param tu Parsed syntax tree to scan and modify + * @return Modified syntax tree + */ + private static TranslationUnit fixUpArraySizes(TranslationUnit tu) { + // Use a private class that extends ScopeTrackingVisitor to do the patching up. + new ScopeTrackingVisitor() { + /** + * Attempt to reduce expression, or throw exception if failed. + * @param Expression to fold + * @return Folded expression + */ + public Expr reduce(Expr expr) { + + if (expr instanceof IntConstantExpr) { + return expr; + } + if (expr instanceof VariableIdentifierExpr) { + Expr newexpr = + getCurrentScope() + .lookupScopeEntry(((VariableIdentifierExpr) expr).getName()) + .getVariableDeclInfo() + .getInitializer() + .getExpr(); + return reduce(newexpr); + } + if (expr instanceof ParenExpr) { + return reduce(((ParenExpr)expr).getExpr()); + } + if (expr instanceof BinaryExpr) { + BinaryExpr bexpr = (BinaryExpr) expr; + + Expr lexpr = reduce(bexpr.getLhs()); + Expr rexpr = reduce(bexpr.getRhs()); + + if (!(lexpr instanceof IntConstantExpr && rexpr instanceof IntConstantExpr)) { + throw new RuntimeException("Unable to fold constant (leaf of binary expression did " + + "not fold)" + bexpr.getText()); + } + + int lval = ((IntConstantExpr) lexpr).getNumericValue(); + int rval = ((IntConstantExpr) rexpr).getNumericValue(); + int fval = 0; + switch (bexpr.getOp()) { + case MOD: + fval = lval % rval; + break; + case MUL: + fval = lval * rval; + break; + case DIV: + if (rval == 0) { + throw new RuntimeException("Division by zero while folding constant " + + bexpr.getText()); + } + fval = lval / rval; + break; + case ADD: + fval = lval + rval; + break; + case SUB: + fval = lval - rval; + break; + case BAND: + fval = lval & rval; + break; + case BOR: + fval = lval | rval; + break; + case BXOR: + fval = lval ^ rval; + break; + case LAND: + fval = ((lval != 0) && (rval != 0)) ? 1 : 0; + break; + case LOR: + fval = ((lval != 0) || (rval != 0)) ? 1 : 0; + break; + case LXOR: + fval = ((lval == 0) != (rval == 0)) ? 1 : 0; + break; + case SHL: + fval = lval << rval; + break; + case SHR: + fval = lval >> rval; + break; + case LT: + fval = (lval < rval) ? 1 : 0; + break; + case GT: + fval = (lval > rval) ? 1 : 0; + break; + case LE: + fval = (lval <= rval) ? 1 : 0; + break; + case GE: + fval = (lval >= rval) ? 1 : 0; + break; + case EQ: + fval = (lval == rval) ? 1 : 0; + break; + case NE: + fval = (lval != rval) ? 1 : 0; + break; + default: + throw new RuntimeException("Unable to fold constant (unimplemented binary " + + "expression) " + bexpr.getText()); + } + return new IntConstantExpr(Integer.toString(fval)); + } + throw new RuntimeException("Unable to fold constant (unimplemented expression) " + + expr.getText()); + } + + private void handleArrayInfo(ArrayInfo arrayInfo) { + if (arrayInfo.hasSizeExpr()) { + arrayInfo.setConstantSizeExpr(((IntConstantExpr)reduce(arrayInfo.getSizeExpr())) + .getNumericValue()); + } + } + + @Override + public void visitVariableDeclInfo(VariableDeclInfo variableDeclInfo) { + if (variableDeclInfo.hasArrayInfo()) { + handleArrayInfo(variableDeclInfo.getArrayInfo()); + } + super.visitVariableDeclInfo(variableDeclInfo); + } + + @Override + public void visitStructDefinitionType(StructDefinitionType structDefinitionType) { + super.visitStructDefinitionType(structDefinitionType); + for (Type fieldType : structDefinitionType.getFieldTypes()) { + if (fieldType.getWithoutQualifiers() instanceof ArrayType) { + handleArrayInfo(((ArrayType) fieldType.getWithoutQualifiers()).getArrayInfo()); + } + } + } + + }.visit(tu); + return tu; + } + public static TranslationUnit getTranslationUnit(Translation_unitContext ctx, ShaderKind shaderKind, boolean hasWebGlHint) { - return new AstBuilder(shaderKind, hasWebGlHint).visitTranslation_unit(ctx); + /* Scan parsed AST and perform constant folding for array sizes. + * Doing the constant folding during AST build is not as convenient as + * not all of the required information may be available when needed. + */ + return fixUpArraySizes(new AstBuilder(shaderKind, hasWebGlHint).visitTranslation_unit(ctx)); } @Override @@ -278,6 +429,8 @@ public Declaration visitDeclaration(DeclarationContext ctx) { if (ctx.interface_block() != null) { return visitInterface_block(ctx.interface_block()); } + // The above captures all the declaration kinds, so this indicates a bad input + // rather than lack of support. throw new RuntimeException("Unknown declaration at line " + ctx.start.getLine() + ": " + getOriginalSourceText(ctx)); } @@ -290,9 +443,9 @@ public Declaration visitInterface_block(Interface_blockContext ctx) { final Basic_interface_blockContext basicCtx = ctx.basic_interface_block(); final TypeQualifier interfaceQualifier = visitInterface_qualifier(basicCtx.interface_qualifier()); - final Optional maybeInstanceName = basicCtx.instance_name_opt() == null + final Optional maybeInstanceName = basicCtx.instance_name() == null ? Optional.empty() - : Optional.of(basicCtx.instance_name_opt().getText()); + : Optional.of(basicCtx.instance_name().getText()); final Pair, List> members = getMembers(basicCtx.member_list()); return new InterfaceBlock( maybeLayoutQualifier, @@ -315,8 +468,9 @@ public TypeQualifier visitInterface_qualifier(Interface_qualifierContext ctx) { case "buffer": return TypeQualifier.BUFFER; default: - throw new RuntimeException("Interface qualifier: " + ctx.getText() - + " unknown or not yet supported."); + // The above is supposed to capture all the interface qualifiers, so this + // indicates that the input is bad (rather than lack of support). + throw new RuntimeException("Unknown interface qualifier: " + ctx.getText()); } } @@ -358,7 +512,8 @@ private Type getType(Type_qualifierContext qualifiersCtx, private Type getType(Type_specifierContext typeSpecifier, List qualifiers) { if (typeSpecifier.array_specifier() != null) { - throw new RuntimeException(); + throw new UnsupportedLanguageFeatureException("Array information specified at the base type" + + ", e.g. 'int[3] v', is not currently supported; use e.g. 'int A[3]' instead"); } if (typeSpecifier.type_specifier_nonarray().builtin_type_specifier_nonarray() != null) { return new QualifiedType(getBuiltinType(typeSpecifier.type_specifier_nonarray() @@ -416,18 +571,14 @@ private StructDefinitionType makeStructDefinition(Optional struc private ArrayInfo getArrayInfo(Array_specifierContext arraySpecifierContext) { if (arraySpecifierContext.array_specifier() != null) { - throw new RuntimeException("Not yet supporting multi-dimmensional arrays"); + throw new UnsupportedLanguageFeatureException("Not yet supporting multi-dimensional arrays"); } if (arraySpecifierContext.constant_expression() == null) { // An array with unspecified length. return new ArrayInfo(); } final Expr expr = (Expr) visit(arraySpecifierContext.constant_expression()); - if (expr instanceof IntConstantExpr) { - return new ArrayInfo(Integer.parseInt(((IntConstantExpr) expr).getValue())); - } - throw new RuntimeException("Unable to construct array info for array with size " - + expr.getText()); + return new ArrayInfo(expr); } private BuiltinType getBuiltinType(Builtin_type_specifier_nonarrayContext ctx) { @@ -792,10 +943,6 @@ public List visitStatement_list(Statement_listContext ctx) { @Override public DeclarationStmt visitDeclaration_statement(Declaration_statementContext ctx) { - if (ctx.declaration().init_declarator_list() == null) { - throw new RuntimeException("Error at line " + ctx.start.getLine() - + ": Only variable declarations are supported in declaration statements"); - } return new DeclarationStmt( visitInit_declarator_list(ctx.declaration().init_declarator_list())); } @@ -835,9 +982,9 @@ public Initializer visitInitializer(InitializerContext ctx) { return null; } if (ctx.assignment_expression() != null) { - return new ScalarInitializer(visitAssignment_expression(ctx.assignment_expression())); + return new Initializer(visitAssignment_expression(ctx.assignment_expression())); } - throw new RuntimeException(); + throw new UnsupportedLanguageFeatureException("Initializer lists are not currently supported."); } @Override @@ -957,8 +1104,8 @@ public Stmt visitIteration_statement(Iteration_statementContext ctx) { } assert ctx.FOR() != null; return new ForStmt(visitFor_init_statement(ctx.for_init_statement()), - ctx.for_rest_statement().conditionopt().condition() == null ? null : - visitCondition(ctx.for_rest_statement().conditionopt().condition()), + ctx.for_rest_statement().condition() == null ? null : + visitCondition(ctx.for_rest_statement().condition()), ctx.for_rest_statement().expression() == null ? null : visitExpression(ctx.for_rest_statement().expression()), visitStatement_no_new_scope(ctx.statement_no_new_scope())); @@ -970,14 +1117,14 @@ public Expr visitCondition(ConditionContext ctx) { return visitExpression(ctx.expression()); } assert ctx.ASSIGN_OP() != null; - throw new RuntimeException( + throw new UnsupportedLanguageFeatureException( "We do not yet support the case where the condition of a 'for' or 'while' introduces a " + "new variable: " + getOriginalSourceText(ctx)); } @Override public Stmt visitFor_rest_statement(For_rest_statementContext ctx) { - throw new RuntimeException(); + throw new RuntimeException("By construction, this visitor method should never get executed."); } @Override @@ -1037,6 +1184,8 @@ public PragmaStatement visitPragma_statement(Pragma_statementContext ctx) { if (ctx.PRAGMA_INVARIANT_ALL() != null) { return PragmaStatement.INVARIANT_ALL; } + // The above captures all the possibilities for a pragma statement, so reaching the following + // line indicates that the shader is invalid, rather than that support is missing. throw new RuntimeException("Unknown pragma statement " + ctx.getText()); } @@ -1098,8 +1247,8 @@ public Expr visitPostfix_expression(Postfix_expressionContext ctx) { visitExpression(ctx.integer_expression().expression())); } if (ctx.method_call_generic() != null) { - // TODO: check grammar - throw new RuntimeException("Not yet supported: " + getOriginalSourceText(ctx)); + throw new UnsupportedLanguageFeatureException("Method calls are not currently supported: " + + getOriginalSourceText(ctx)); } if (ctx.IDENTIFIER() != null) { return new MemberLookupExpr(visitPostfix_expression(ctx.postfix_expression()), @@ -1131,6 +1280,7 @@ public Expr visitFunction_call_header_no_parameters( Function_call_header_no_parametersContext ctx) { if (isBuiltinTypeConstructor(ctx.function_call_header().function_identifier()) || isStructTypeConstructor(ctx.function_call_header().function_identifier())) { + // This is illegal, so indicates an invalid shader rather than lack of support. throw new RuntimeException( "Found type constructor with no arguments at line " + ctx.start.getLine() + ": " + getOriginalSourceText(ctx)); @@ -1139,6 +1289,8 @@ public Expr visitFunction_call_header_no_parameters( return new FunctionCallExpr(getCallee(ctx.function_call_header().function_identifier()), new ArrayList<>()); } + // The above logic is intended to capture all cases, so the following indicates an invalid + // shader, rather than lack of support. throw new RuntimeException("Unsupported function call at line " + ctx.start.getLine() + ": " + getOriginalSourceText(ctx)); } @@ -1176,6 +1328,8 @@ public Expr visitFunction_call_header_with_parameters( if (isRegularFunction(header.function_identifier())) { return new FunctionCallExpr(getCallee(header.function_identifier()), params); } + // The above logic is intended to capture all cases, so the following indicates an invalid + // shader, rather than lack of support. throw new RuntimeException("Unsupported function call: " + getOriginalSourceText(ctx)); } @@ -1412,6 +1566,8 @@ private BinOp getBinOp(Token token) { return op; } } + // The BinOp class includes all binary operators, so the following indicates an invalid shader, + // rather than lack of support. throw new RuntimeException("Unknown binary operator: " + token.getText()); } diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/IAstVisitor.java b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/IAstVisitor.java index 98784af46..1df2911a2 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/IAstVisitor.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/IAstVisitor.java @@ -19,14 +19,13 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.ArrayInfo; -import com.graphicsfuzz.common.ast.decl.ArrayInitializer; import com.graphicsfuzz.common.ast.decl.DefaultLayout; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.InterfaceBlock; import com.graphicsfuzz.common.ast.decl.ParameterDecl; import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayConstructorExpr; @@ -90,9 +89,7 @@ public interface IAstVisitor { void visitPrecisionDeclaration(PrecisionDeclaration precisionDeclaration); - void visitArrayInitializer(ArrayInitializer arrayInitializer); - - void visitScalarInitializer(ScalarInitializer scalarInitializer); + void visitInitializer(Initializer initializer); void visitBinaryExpr(BinaryExpr binaryExpr); diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/StandardVisitor.java b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/StandardVisitor.java index 52b6574d4..f06dadde6 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/StandardVisitor.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/StandardVisitor.java @@ -19,15 +19,14 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.ArrayInfo; -import com.graphicsfuzz.common.ast.decl.ArrayInitializer; import com.graphicsfuzz.common.ast.decl.Declaration; import com.graphicsfuzz.common.ast.decl.DefaultLayout; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.InterfaceBlock; import com.graphicsfuzz.common.ast.decl.ParameterDecl; import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayConstructorExpr; @@ -163,13 +162,8 @@ public void visitPrecisionDeclaration(PrecisionDeclaration precisionDeclaration) } @Override - public void visitArrayInitializer(ArrayInitializer arrayInitializer) { - throw new RuntimeException(); - } - - @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - visitChildFromParent(scalarInitializer.getExpr(), scalarInitializer); + public void visitInitializer(Initializer initializer) { + visitChildFromParent(initializer.getExpr(), initializer); } @Override diff --git a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInitializer.java b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/UnsupportedLanguageFeatureException.java similarity index 59% rename from ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInitializer.java rename to ast/src/main/java/com/graphicsfuzz/common/ast/visitors/UnsupportedLanguageFeatureException.java index df369467b..b63b345ab 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/ast/decl/ArrayInitializer.java +++ b/ast/src/main/java/com/graphicsfuzz/common/ast/visitors/UnsupportedLanguageFeatureException.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 The GraphicsFuzz Project Authors + * Copyright 2019 The GraphicsFuzz Project Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,20 +14,16 @@ * limitations under the License. */ -package com.graphicsfuzz.common.ast.decl; +package com.graphicsfuzz.common.ast.visitors; -import com.graphicsfuzz.common.ast.visitors.IAstVisitor; - -public class ArrayInitializer extends Initializer { - - @Override - public void accept(IAstVisitor visitor) { - visitor.visitArrayInitializer(this); - } +/** + * A RuntimeException to identify cases where AST building fails due to known unsupported + * language features. + */ +public class UnsupportedLanguageFeatureException extends RuntimeException { - @Override - public ArrayInitializer clone() { - return new ArrayInitializer(); + public UnsupportedLanguageFeatureException(String message) { + super(message); } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java index a9135e4da..08e4527bc 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/CompositeShadingLanguageVersion.java @@ -73,6 +73,16 @@ public boolean supportedClampUint() { return prototype.supportedClampUint(); } + @Override + public boolean supportedComputeShaders() { + return prototype.supportedComputeShaders(); + } + + @Override + public boolean supportedDerivativeFunctions() { + return prototype.supportedDerivativeFunctions(); + } + @Override public boolean supportedDeterminant() { return prototype.supportedDeterminant(); @@ -83,6 +93,11 @@ public boolean supportedDoStmt() { return prototype.supportedDoStmt(); } + @Override + public boolean supportedExplicitDerivativeFunctions() { + return prototype.supportedExplicitDerivativeFunctions(); + } + @Override public boolean supportedFloatBitsToInt() { return prototype.supportedFloatBitsToInt(); @@ -108,6 +123,16 @@ public boolean supportedIntBitsToFloat() { return prototype.supportedIntBitsToFloat(); } + @Override + public boolean supportedIntegerFunctions() { + return prototype.supportedIntegerFunctions(); + } + + @Override + public boolean supportedInterpolationFunctions() { + return prototype.supportedInterpolationFunctions(); + } + @Override public boolean supportedInverse() { return prototype.supportedInverse(); @@ -258,4 +283,28 @@ public boolean supportedUnsigned() { return prototype.supportedUnsigned(); } + @Override + public boolean supportedAngleAndTrigonometricFunctions() { + return prototype.supportedAngleAndTrigonometricFunctions(); + } + + @Override + public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { + return prototype.supportedHyperbolicAngleAndTrigonometricFunctions(); + } + + @Override + public boolean supportedModf() { + return prototype.supportedModf(); + } + + @Override + public boolean supportedFrexp() { + return prototype.supportedFrexp(); + } + + @Override + public boolean supportedLdexp() { + return prototype.supportedLdexp(); + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java index 799ccdd6b..fdbce3f9e 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl100.java @@ -74,6 +74,16 @@ public boolean supportedClampUint() { return false; } + @Override + public boolean supportedComputeShaders() { + return false; + } + + @Override + public boolean supportedDerivativeFunctions() { + return false; + } + @Override public boolean supportedDeterminant() { return false; @@ -84,6 +94,11 @@ public boolean supportedDoStmt() { return true; } + @Override + public boolean supportedExplicitDerivativeFunctions() { + return false; + } + @Override public boolean supportedFloatBitsToInt() { return false; @@ -109,6 +124,16 @@ public boolean supportedIntBitsToFloat() { return false; } + @Override + public boolean supportedIntegerFunctions() { + return false; + } + + @Override + public boolean supportedInterpolationFunctions() { + return false; + } + @Override public boolean supportedInverse() { return false; @@ -258,4 +283,29 @@ public boolean supportedUnpackUnorm4x8() { public boolean supportedUnsigned() { return false; } + + @Override + public boolean supportedAngleAndTrigonometricFunctions() { + return true; + } + + @Override + public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { + return false; + } + + @Override + public boolean supportedModf() { + return false; + } + + @Override + public boolean supportedFrexp() { + return false; + } + + @Override + public boolean supportedLdexp() { + return false; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java index 6f27f8dc2..0b9277180 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl300.java @@ -55,6 +55,11 @@ public boolean supportedClampUint() { return true; } + @Override + public boolean supportedDerivativeFunctions() { + return true; + } + @Override public boolean supportedDeterminant() { return true; @@ -210,4 +215,13 @@ public boolean supportedUnsigned() { return true; } + @Override + public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { + return true; + } + + @Override + public boolean supportedModf() { + return true; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java index ab457c7f9..bdbfc039f 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl310.java @@ -30,6 +30,16 @@ public String getVersionString() { return "310 es"; } + @Override + public boolean supportedComputeShaders() { + return true; + } + + @Override + public boolean supportedIntegerFunctions() { + return true; + } + @Override public boolean supportedMixNonfloatBool() { return true; @@ -55,4 +65,14 @@ public boolean supportedUnpackUnorm4x8() { return true; } + @Override + public boolean supportedFrexp() { + return true; + } + + @Override + public boolean supportedLdexp() { + return true; + } + } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl320.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl320.java index 8c0426d3b..776f78023 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl320.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Essl320.java @@ -30,4 +30,9 @@ public String getVersionString() { return "320 es"; } + @Override + public boolean supportedInterpolationFunctions() { + return true; + } + } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java index a30a7540c..d4beaec68 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl110.java @@ -74,6 +74,16 @@ public boolean supportedClampUint() { return false; } + @Override + public boolean supportedComputeShaders() { + return false; + } + + @Override + public boolean supportedDerivativeFunctions() { + return true; + } + @Override public boolean supportedDeterminant() { return false; @@ -84,6 +94,11 @@ public boolean supportedDoStmt() { return true; } + @Override + public boolean supportedExplicitDerivativeFunctions() { + return false; + } + @Override public boolean supportedFloatBitsToInt() { return false; @@ -109,6 +124,16 @@ public boolean supportedIntBitsToFloat() { return false; } + @Override + public boolean supportedIntegerFunctions() { + return false; + } + + @Override + public boolean supportedInterpolationFunctions() { + return false; + } + @Override public boolean supportedInverse() { return false; @@ -261,4 +286,29 @@ public boolean supportedUnpackUnorm4x8() { public boolean supportedUnsigned() { return false; } + + @Override + public boolean supportedAngleAndTrigonometricFunctions() { + return true; + } + + @Override + public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { + return false; + } + + @Override + public boolean supportedModf() { + return false; + } + + @Override + public boolean supportedFrexp() { + return false; + } + + @Override + public boolean supportedLdexp() { + return false; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl130.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl130.java index 5a349711a..36cb869fc 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl130.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl130.java @@ -115,4 +115,13 @@ public boolean supportedUnsigned() { return true; } + @Override + public boolean supportedHyperbolicAngleAndTrigonometricFunctions() { + return true; + } + + @Override + public boolean supportedModf() { + return true; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl150.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl150.java index ef28609c5..c89b6939e 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl150.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl150.java @@ -27,7 +27,16 @@ private Glsl150(ShadingLanguageVersion prototype) { @Override public String getVersionString() { - return "140"; + return "150"; } + @Override + public boolean supportedDeterminant() { + return true; + } + + @Override + public boolean supportedInverse() { + return true; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl330.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl330.java index f15e7ca33..4f188e67b 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl330.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl330.java @@ -30,14 +30,6 @@ public String getVersionString() { return "330"; } - @Override - public boolean supportedDeterminant() { - // According to this page: - // https://www.khronos.org/registry/OpenGL-Refpages/gl4/html/determinant.xhtml - // determinant should be supported from GLSL 1.40, but glslangValidator disagrees. - return true; - } - @Override public boolean supportedFloatBitsToInt() { return true; @@ -53,14 +45,6 @@ public boolean supportedIntBitsToFloat() { return true; } - @Override - public boolean supportedInverse() { - // According to this page: - // https://www.khronos.org/registry/OpenGL-Refpages/gl4/html/inverse.xhtml - // inverse should be supported from GLSL 1.50, but glslangValidator disagrees. - return true; - } - @Override public boolean supportedUintBitsToFloat() { return true; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java index c703e29d7..58ee30cbb 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl400.java @@ -35,6 +35,16 @@ public boolean supportedFma() { return true; } + @Override + public boolean supportedIntegerFunctions() { + return true; + } + + @Override + public boolean supportedInterpolationFunctions() { + return true; + } + @Override public boolean supportedPackSnorm4x8() { return true; @@ -65,4 +75,13 @@ public boolean supportedUnpackUnorm4x8() { return true; } + @Override + public boolean supportedFrexp() { + return true; + } + + @Override + public boolean supportedLdexp() { + return true; + } } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl430.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl430.java index e4859c7b8..e1502c523 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl430.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl430.java @@ -29,4 +29,10 @@ private Glsl430(ShadingLanguageVersion prototype) { public String getVersionString() { return "430"; } + + @Override + public boolean supportedComputeShaders() { + return true; + } + } diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl450.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl450.java index 8aacf7415..6ae02e3f7 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl450.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/Glsl450.java @@ -30,6 +30,11 @@ public String getVersionString() { return "450"; } + @Override + public boolean supportedExplicitDerivativeFunctions() { + return true; + } + @Override public boolean supportedMixNonfloatBool() { return true; diff --git a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java index 924b49b30..6013da153 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java +++ b/ast/src/main/java/com/graphicsfuzz/common/glslversion/ShadingLanguageVersion.java @@ -163,10 +163,41 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { boolean supportedClampUint(); + /** + * GLSL versions 4.3+ and ESSL versions 3.1+ support compute shaders. + * @return true if the shading language version allows compute shaders - false otherwise. + */ + boolean supportedComputeShaders(); + + /** + * Derivative Functions are a subset of fragment processing functions that compute + * the rate of change between pixels in a given fragment. + * GLSL versions 1.1+ and ESSL versions 3.0+ support these functions. + * + * @return true if explicit derivative functions are supported - false otherwise. + */ + boolean supportedDerivativeFunctions(); + + /** + * Determinant Function calculates the determinant of a given square matrix. + * GLSL versions 1.5+ and ESSL versions 3.0+ support this function. + * + * @return true if Determinant Function is supported - false otherwise. + */ boolean supportedDeterminant(); boolean supportedDoStmt(); + /** + * In recent GLSL specifications, new derivative functions were added that allow a user to + * specify how much precision the user wants in the calculation, instead of leaving the choice + * to the compiler. + * GLSL versions 4.5+ support these explicit derivative functions. + * + * @return true if explicit derivative functions are supported - false otherwise. + */ + boolean supportedExplicitDerivativeFunctions(); + boolean supportedFloatBitsToInt(); boolean supportedFloatBitsToUint(); @@ -177,6 +208,31 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { boolean supportedIntBitsToFloat(); + /** + * Integer Functions are a set of built-in functions that allow manipulation of integers and + * their corresponding vectors in ways difficult or impossible with normal GLSL syntax - for + * example, summing two unsigned integers where the result causes an overflow. + * GLSL versions 4.0+ and ESSL versions 3.1+ support these functions. + * + * @return true if Integer Functions are supported - false otherwise. + */ + boolean supportedIntegerFunctions(); + + /** + * Interpolation Functions are a subset of fragment processing functions that + * compute an interpolated value of a fragment shader input variable at a specific location. + * GLSL versions 4.0+ and ESSL versions 3.2+ support these functions. + * + * @return true if Interpolation Functions are supported - false otherwise. + */ + boolean supportedInterpolationFunctions(); + + /** + * Inverse Function returns the matrix that is the inverse of the given square matrix. + * GLSL versions 1.5+ and ESSL versions 3.0+ support this function. + * + * @return true if Inverse Function is supported - false otherwise. + */ boolean supportedInverse(); boolean supportedIsinf(); @@ -199,6 +255,12 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { boolean supportedNonSquareMatrices(); + /** + * OuterProduct Function does a linear algebraic matrix multiplication of two given vectors. + * GLSL versions 1.2+ and ESSL versions 3.0+ support this function. + * + * @return true if OuterProduct Function is supported - false otherwise. + */ boolean supportedOuterProduct(); boolean supportedPackHalf2x16(); @@ -219,6 +281,12 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { boolean supportedSwitchStmt(); + /** + * Transpose Function returns the transposed matrix of the given matrix. + * GLSL versions 1.2+ and ESSL versions 3.0+ support this function. + * + * @return true if Transpose Function is supported - false otherwise. + */ boolean supportedTranspose(); boolean supportedTrunc(); @@ -237,4 +305,28 @@ static ShadingLanguageVersion webGlFromVersionString(String versionString) { boolean supportedUnsigned(); + boolean supportedModf(); + + boolean supportedFrexp(); + + boolean supportedLdexp(); + + /** + * Angle and Trigonometric Functions are a set of built-in functions related to the calculation + * of an angle. For example, sin(angle) - computes the sine value of the angle provided. + * GLSL versions 1.1+ and ESSL versions 1.0+ support these functions. + * + * @return true if Angle and Trigonometric Functions are supported - false otherwise. + */ + boolean supportedAngleAndTrigonometricFunctions(); + + /** + * Hyperbolic Angle and Trigonometric Functions are a set of built-in functions that + * computes the hyperbolic trigonometric functions. For example, sinh() - calculate the + * hyperbolic sine function of the given value. + * GLSL versions 1.3+ and ESSL versions 3.0+ support these functions. + * + * @return true if Hyperbolic Angle and Trigonometric Functions are supported - false otherwise. + */ + boolean supportedHyperbolicAngleAndTrigonometricFunctions(); } diff --git a/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrint.java b/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrint.java index e4e73ca81..ec62c2c36 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrint.java +++ b/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrint.java @@ -44,11 +44,6 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .help("Target file name.") .type(String.class); - // Optional arguments - parser.addArgument("--glsl_version") - .help("Version of GLSL to target.") - .type(String.class); - return parser.parseArgs(args); } @@ -75,18 +70,9 @@ private static void prettyPrintShader(Namespace ns, TranslationUnit tu) throws FileNotFoundException { try (PrintStream stream = new PrintStream(new FileOutputStream(new File(ns.getString("output"))))) { - if (getGlslVersion(ns) != null) { - throw new RuntimeException(); - //Helper.emitDefines(stream, new ShadingLanguageVersion(getGlslVersion(ns), false), - // false); - } PrettyPrinterVisitor ppv = new PrettyPrinterVisitor(stream); ppv.visit(tu); } } - private static String getGlslVersion(Namespace ns) { - return ns.getString("glsl_version"); - } - } diff --git a/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java b/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java index 4a9032dcb..8be692ac3 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java +++ b/ast/src/main/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitor.java @@ -70,7 +70,10 @@ import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.type.VoidType; +import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.util.Constants; import java.io.ByteArrayOutputStream; @@ -128,10 +131,19 @@ public static void emitShader(TranslationUnit shader, Optional license, PrintStream stream, int indentationWidth, - Supplier newlineSupplier, - boolean emitGraphicsFuzzDefines) { + Supplier newlineSupplier) { + final boolean usesGraphicsFuzzDefines = new CheckPredicateVisitor() { + @Override + public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { + if (MacroNames.isGraphicsFuzzMacro(functionCallExpr)) { + predicateHolds(); + } + super.visitFunctionCallExpr(functionCallExpr); + } + }.test(shader); + new PrettyPrinterVisitor(stream, indentationWidth, newlineSupplier, - emitGraphicsFuzzDefines, license).visit(shader); + usesGraphicsFuzzDefines, license).visit(shader); } private String newLine() { @@ -163,10 +175,14 @@ public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) first = false; out.append(vdi.getName()); if (vdi.hasArrayInfo()) { - out.append("[" + vdi.getArrayInfo().getSize() + "]"); + out.append("["); + visit(vdi.getArrayInfo().getSizeExpr()); + out.append("]"); assert !(baseType instanceof ArrayType); } else if (baseType instanceof ArrayType) { - out.append("[" + ((ArrayType) baseType).getArrayInfo().getSize() + "]"); + out.append("["); + visit(((ArrayType) baseType).getArrayInfo().getSizeExpr()); + out.append("]"); } if (vdi.hasInitializer()) { out.append(" = "); @@ -210,7 +226,9 @@ public void visitParameterDecl(ParameterDecl parameterDecl) { out.append(" " + parameterDecl.getName()); } if (parameterDecl.getArrayInfo() != null) { - out.append("[" + parameterDecl.getArrayInfo().getSize() + "]"); + out.append("["); + visit(parameterDecl.getArrayInfo().getSizeExpr()); + out.append("]"); } } @@ -503,7 +521,9 @@ public void visitArrayType(ArrayType arrayType) { @Override public void visitArrayConstructorExpr(ArrayConstructorExpr arrayConstructorExpr) { visit(arrayConstructorExpr.getArrayType()); - out.append("[" + arrayConstructorExpr.getArrayType().getArrayInfo().getSize() + "]("); + out.append("["); + visit(arrayConstructorExpr.getArrayType().getArrayInfo().getSizeExpr()); + out.append("]("); boolean first = true; for (Expr e : arrayConstructorExpr.getArgs()) { if (!first) { @@ -541,9 +561,11 @@ private void processArrayInfo(Type type) { } ArrayType arrayType = (ArrayType) type.getWithoutQualifiers(); while (true) { - out.append("[" - + (arrayType.getArrayInfo().hasSize() ? arrayType.getArrayInfo().getSize() : "") - + "]"); + out.append("["); + if (arrayType.getArrayInfo().hasSizeExpr()) { + visit(arrayType.getArrayInfo().getSizeExpr()); + } + out.append("]"); if (!(arrayType.getBaseType().getWithoutQualifiers() instanceof ArrayType)) { break; } @@ -628,7 +650,7 @@ public void visitTranslationUnit(TranslationUnit translationUnit) { } if (emitGraphicsFuzzDefines) { - emitGraphicsFuzzDefines(out); + emitGraphicsFuzzDefines(out, translationUnit.getShadingLanguageVersion()); } super.visitTranslationUnit(translationUnit); @@ -684,20 +706,47 @@ public static String defaultIndent(int level) { return result.toString(); } - public static void emitGraphicsFuzzDefines(PrintStream out) { + public static void emitGraphicsFuzzDefines(PrintStream out, + ShadingLanguageVersion shadingLanguageVersion) { out.append("\n"); out.append("#ifndef REDUCER\n"); - out.append("#define " + Constants.GLF_ZERO + "(X, Y) (Y)\n"); - out.append("#define " + Constants.GLF_ONE + "(X, Y) (Y)\n"); - out.append("#define " + Constants.GLF_FALSE + "(X, Y) (Y)\n"); - out.append("#define " + Constants.GLF_TRUE + "(X, Y) (Y)\n"); - out.append("#define " + Constants.GLF_IDENTITY + "(X, Y) (Y)\n"); - out.append("#define " + Constants.GLF_DEAD + "(X) (X)\n"); - out.append("#define " + Constants.GLF_FUZZED + "(X) (X)\n"); - out.append("#define " + Constants.GLF_WRAPPED_LOOP + "(X) X\n"); - out.append("#define " + Constants.GLF_WRAPPED_IF_TRUE + "(X) X\n"); - out.append("#define " + Constants.GLF_WRAPPED_IF_FALSE + "(X) X\n"); - out.append("#define " + Constants.GLF_SWITCH + "(X) X\n"); + out.append("#define " + Constants.GLF_ZERO + "(X, Y) (Y)\n"); + out.append("#define " + Constants.GLF_ONE + "(X, Y) (Y)\n"); + out.append("#define " + Constants.GLF_FALSE + "(X, Y) (Y)\n"); + out.append("#define " + Constants.GLF_TRUE + "(X, Y) (Y)\n"); + out.append("#define " + Constants.GLF_IDENTITY + "(X, Y) (Y)\n"); + out.append("#define " + Constants.GLF_DEAD + "(X) (X)\n"); + out.append("#define " + Constants.GLF_FUZZED + "(X) (X)\n"); + out.append("#define " + Constants.GLF_WRAPPED_LOOP + "(X) X\n"); + out.append("#define " + Constants.GLF_WRAPPED_IF_TRUE + "(X) X\n"); + out.append("#define " + Constants.GLF_WRAPPED_IF_FALSE + "(X) X\n"); + out.append("#define " + Constants.GLF_SWITCH + "(X) X\n"); + + // The preferred way to make array accesses in bounds is to use 'clamp'. However, 'clamp' + // on integer types is not available in all shading language versions. When it is not + // available, we resort to a solution based on the ternary operator. This has the disadvantage + // of evaluating the array index multiple times, so that if it has side effects these will + // occur multiple times (potentially leading to the access ending up being out of bounds if the + // side effects change the index). If deemed worth prioritizing, this could be worked around + // by declaring appropriate 'clamp' functions (but with appropriately obfuscated names to avoid + // collisions with user-defined integer clamp functions). + + if (shadingLanguageVersion.supportedClampInt()) { + out.append("#define " + Constants.GLF_MAKE_IN_BOUNDS_INT + "(IDX, SZ) clamp(IDX, 0, SZ - 1)" + + "\n"); + } else { + out.append("#define " + Constants.GLF_MAKE_IN_BOUNDS_INT + "(IDX, SZ) ((IDX) < 0 ? 0 : (" + + "(IDX) >= SZ ? SZ - 1 : (IDX)))\n"); + } + + if (shadingLanguageVersion.supportedClampUint()) { + out.append("#define " + Constants.GLF_MAKE_IN_BOUNDS_UINT + "(IDX, SZ) clamp(IDX, 0u, SZ - " + + "1u)\n"); + } else { + out.append("#define " + Constants.GLF_MAKE_IN_BOUNDS_UINT + "(IDX, SZ) ((IDX) >= SZ ? SZ - " + + "1u : (IDX))\n"); + } + out.append("#endif\n"); out.append("\n"); out.append(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES + "\n"); diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitor.java b/ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitor.java new file mode 100644 index 000000000..b5022d677 --- /dev/null +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitor.java @@ -0,0 +1,306 @@ +/* + * Copyright 2018 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.common.typing; + +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; +import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.ParameterDecl; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.stmt.WhileStmt; +import com.graphicsfuzz.common.ast.type.ArrayType; +import com.graphicsfuzz.common.ast.type.StructDefinitionType; +import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; + +/** + * This class extends StandardVisitor to track details of what is in scope at each point of + * visitation. + */ +public abstract class ScopeTrackingVisitor extends StandardVisitor { + + // Tracks the scope at the current point of visitation. + private Scope currentScope; + + // Tracks the function, if any, enclosing the current point of visitation. If this field is null, + // this indicates that visitation is at global scope. + private FunctionDefinition enclosingFunction; + + // A stack of the blocks that enclose the current point of visitation. + private final Deque enclosingBlocks; + + // All of the function prototypes that have been encountered during visitation so far. + private final List encounteredFunctionPrototypes; + + // When we visit a function prototype, we want to add its parameters to the current scope if and + // only if it is the prototype component of a function definition. If it is a stand-alone + // function prototype then we do not want to add its parameters to the current scope. This field + // is used to determine whether or not we should record encountered parameters to the current + // scope. + private boolean addEncounteredParametersToScope; + + protected ScopeTrackingVisitor() { + this.currentScope = new Scope(null); + this.enclosingFunction = null; + this.enclosingBlocks = new LinkedList<>(); + this.encounteredFunctionPrototypes = new ArrayList<>(); + this.addEncounteredParametersToScope = false; + } + + // Over-ridden 'visit' methods; keep in alphabetical order: + + @Override + public void visitBlockStmt(BlockStmt stmt) { + enterBlockStmt(stmt); + super.visitBlockStmt(stmt); + leaveBlockStmt(stmt); + } + + @Override + public void visitForStmt(ForStmt forStmt) { + // A 'for' statement is special in that it, rather than its open-curly, starts a new scope. + // For example, this is illegal: + // + // for (int i = 0; i < 10; i++) { + // int i; + // } + // + // because it declares 'i' twice in the same scope. + // + // We thus have to push a new scope before traversing a 'for' statement and pop it afterwards. + pushScope(); + super.visitForStmt(forStmt); + popScope(); + } + + @Override + public void visitFunctionDefinition(FunctionDefinition functionDefinition) { + // Record the fact that we are in a function, and push a scope for that function. + assert enclosingFunction == null; + enclosingFunction = functionDefinition; + pushScope(); + + // As this is a function definition, we *do* want to add the function's parameters to the + // current scope when we visit the function prototpye. + addEncounteredParametersToScope = true; + visitFunctionPrototype(functionDefinition.getPrototype()); + addEncounteredParametersToScope = false; + + visit(functionDefinition.getBody()); + + // Get rid of the function's scope, and record that we are no longer in a function. + popScope(); + enclosingFunction = null; + } + + @Override + public void visitFunctionPrototype(FunctionPrototype functionPrototype) { + encounteredFunctionPrototypes.add(functionPrototype); + for (ParameterDecl p : functionPrototype.getParameters()) { + visitParameterDecl(p); + + // Only add the parameter to the current scope if we have decided to do so, and then only + // if it actually has a name. + if (!addEncounteredParametersToScope || p.getName() == null) { + continue; + } + currentScope.add(p.getName(), + p.getArrayInfo() == null ? p.getType() : new ArrayType(p.getType(), p.getArrayInfo()), + Optional.of(p)); + } + } + + @Override + public void visitStructDefinitionType(StructDefinitionType structDefinitionType) { + super.visitStructDefinitionType(structDefinitionType); + if (structDefinitionType.hasStructNameType()) { + // We add named structs to the current scope. + currentScope.addStructDefinition(structDefinitionType); + } + } + + @Override + public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) { + visit(variablesDeclaration.getBaseType()); + for (VariableDeclInfo declInfo : variablesDeclaration.getDeclInfos()) { + // Visit the declInfo both before and after adding it to the current scope, to give + // subclasses the flexibility to examine both cases. + visitVariableDeclInfo(declInfo); + currentScope.add(declInfo.getName(), + declInfo.getArrayInfo() == null + ? variablesDeclaration.getBaseType() + : new ArrayType(variablesDeclaration.getBaseType(), declInfo.getArrayInfo()), + Optional.empty(), + declInfo, variablesDeclaration); + visitVariableDeclInfoAfterAddedToScope(declInfo); + } + } + + @Override + public void visitWhileStmt(WhileStmt whileStmt) { + // A 'while' statement is special in that it, rather than its open-curly, starts a new scope. + // For example, this is illegal: + // + // while (bool b = true) { + // bool b; + // } + // + // because it declares 'b' twice in the same scope. + // + // We thus have to push a new scope before traversing a 'while' statement and pop it + // afterwards. + pushScope(); + super.visitWhileStmt(whileStmt); + popScope(); + } + + // Additional methods for use by subclasses: + + /** + * Tracks entry to a block, updating the current scope and the enclosing blocks. + * Subclasses can override this method to perform additional actions on block entry, but *must* + * invoke this superclass method, otherwise scopes will not be properly tracked. + * @param blockStmt A block statement that is about to be traversed + */ + protected void enterBlockStmt(BlockStmt blockStmt) { + enclosingBlocks.addFirst(blockStmt); + if (blockStmt.introducesNewScope()) { + pushScope(); + } + } + + /** + * Tracks exit from a block, updating the current scope and the enclosing blocks. + * Subclasses can override this method to perform additional actions on block exit, but *must* + * invoke this superclass method, otherwise scopes will not be properly tracked. + * @param blockStmt A block statement that has just been traversed + */ + protected void leaveBlockStmt(BlockStmt blockStmt) { + if (blockStmt.introducesNewScope()) { + popScope(); + } + enclosingBlocks.removeFirst(); + } + + /** + * Returns the closest block enclosing the current point of visitation. + * + * @return The closest block + */ + protected BlockStmt currentBlock() { + return enclosingBlocks.peekFirst(); + } + + /** + * Returns true if and only if visitation is in some block. + * + * @return Whether visitation is in a block + */ + protected boolean inSomeBlock() { + return !enclosingBlocks.isEmpty(); + } + + /** + * Because WebGL places limits on for loops it can be convenient to only visit the body of a + * for loop, skipping the header. Subclasses can call this method to invoke this behaviour + * during visitation, instead of calling super.visitForStmt(...). + * + * @param forStmt For statement whose body should be visited + */ + protected void visitForStmtBodyOnly(ForStmt forStmt) { + pushScope(); + visit(forStmt.getBody()); + popScope(); + } + + /** + * This is a hook for subclasses that need to perform an action after a declaration has been added + * to the current scope. + * + * @param declInfo The declaration info that was just added to the current scope + */ + protected void visitVariableDeclInfoAfterAddedToScope(VariableDeclInfo declInfo) { + // Deliberately empty - to be optionally overridden by subclasses. + } + + /** + * Replace the current scope with its parent. + */ + protected void popScope() { + currentScope = currentScope.getParent(); + } + + /** + * Replace the current scope with a new scope whose parent is the old current scope. + */ + protected void pushScope() { + Scope newScope = new Scope(currentScope); + currentScope = newScope; + } + + /** + * Yield the function prototypes that have been encountered during visitation so far. + * @return The function prototypes that have been encountered during visitation so far + */ + protected List getEncounteredFunctionPrototypes() { + return Collections.unmodifiableList(encounteredFunctionPrototypes); + } + + /** + * Determines whether visitation is at global scope, i.e. not in any function. + * @return true if and only if visitation is at global scope + */ + protected boolean atGlobalScope() { + return !currentScope.hasParent(); + } + + /** + * Yields the current scope. + * @return the current scope + */ + protected Scope getCurrentScope() { + return currentScope; + } + + /** + * Replaces the current scope with the given scope, returning the old current scope. This can + * be useful if one needs to temporarily change the visitor's view of what is in scope, + * @param newScope A scope to replace the current scope + * @return the previous current scope + */ + protected Scope swapCurrentScope(Scope newScope) { + final Scope result = currentScope; + currentScope = newScope; + return result; + } + + /** + * Yields the function enclosing the current point of visitation. + * @return the function enclosing the current point of visitation + */ + protected FunctionDefinition getEnclosingFunction() { + return enclosingFunction; + } + +} diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTreeBuilder.java b/ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTreeBuilder.java deleted file mode 100644 index df60c39af..000000000 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/ScopeTreeBuilder.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.common.typing; - -import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.FunctionPrototype; -import com.graphicsfuzz.common.ast.decl.ParameterDecl; -import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; -import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; -import com.graphicsfuzz.common.ast.stmt.BlockStmt; -import com.graphicsfuzz.common.ast.stmt.ForStmt; -import com.graphicsfuzz.common.ast.stmt.WhileStmt; -import com.graphicsfuzz.common.ast.type.ArrayType; -import com.graphicsfuzz.common.ast.type.StructDefinitionType; -import com.graphicsfuzz.common.ast.type.StructNameType; -import com.graphicsfuzz.common.ast.visitors.StandardVisitor; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -public abstract class ScopeTreeBuilder extends StandardVisitor { - - protected Scope currentScope; - private Deque enclosingBlocks; - private boolean addEncounteredParametersToScope; - protected FunctionDefinition enclosingFunction; - private List encounteredFunctionPrototypes; - - protected ScopeTreeBuilder() { - this.currentScope = new Scope(null); - this.enclosingBlocks = new LinkedList<>(); - this.addEncounteredParametersToScope = false; - this.enclosingFunction = null; - this.encounteredFunctionPrototypes = new ArrayList<>(); - } - - @Override - public void visitStructDefinitionType(StructDefinitionType structDefinitionType) { - super.visitStructDefinitionType(structDefinitionType); - if (structDefinitionType.hasStructNameType()) { - currentScope.addStructDefinition(structDefinitionType); - } - } - - @Override - public void visitBlockStmt(BlockStmt stmt) { - enterBlockStmt(stmt); - super.visitBlockStmt(stmt); - leaveBlockStmt(stmt); - } - - protected void enterBlockStmt(BlockStmt stmt) { - enclosingBlocks.addFirst(stmt); - if (stmt.introducesNewScope()) { - pushScope(); - } - } - - protected void leaveBlockStmt(BlockStmt stmt) { - if (stmt.introducesNewScope()) { - popScope(); - } - enclosingBlocks.removeFirst(); - } - - /** - * Returns the closest block enclosing the current point of visitation. - * - * @return The closest block - */ - protected BlockStmt currentBlock() { - return enclosingBlocks.peekFirst(); - } - - /** - * Returns true if and only if visitatin is in some block. - * - * @return Whether visitation is in a block - */ - protected boolean inBlock() { - return !enclosingBlocks.isEmpty(); - } - - @Override - public void visitForStmt(ForStmt forStmt) { - pushScope(); - super.visitForStmt(forStmt); - popScope(); - } - - /** - * Because WebGL places limits on for loops it can be convenient to only visit the body of a - * for loop, skipping the header. Subclasses can call this method to invoke this behaviour - * during visitation, instead of calling super.visitForStmt(...). - * - * @param forStmt For statement whose body should be visited - */ - protected void visitForStmtBodyOnly(ForStmt forStmt) { - pushScope(); - visit(forStmt.getBody()); - popScope(); - } - - - @Override - public void visitWhileStmt(WhileStmt whileStmt) { - pushScope(); - super.visitWhileStmt(whileStmt); - popScope(); - } - - @Override - public void visitFunctionDefinition(FunctionDefinition functionDefinition) { - assert enclosingFunction == null; - enclosingFunction = functionDefinition; - pushScope(); - - addEncounteredParametersToScope = true; - visitFunctionPrototype(functionDefinition.getPrototype()); - addEncounteredParametersToScope = false; - visit(functionDefinition.getBody()); - popScope(); - enclosingFunction = null; - } - - @Override - public void visitFunctionPrototype(FunctionPrototype functionPrototype) { - encounteredFunctionPrototypes.add(functionPrototype); - for (ParameterDecl p : functionPrototype.getParameters()) { - visitParameterDecl(p); - if (addEncounteredParametersToScope) { - if (p.getName() == null) { - continue; - } - currentScope.add(p.getName(), - p.getArrayInfo() == null ? p.getType() : new ArrayType(p.getType(), p.getArrayInfo()), - Optional.of(p)); - } - } - } - - @Override - public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) { - visit(variablesDeclaration.getBaseType()); - List children = new ArrayList<>(); - children.addAll(variablesDeclaration.getDeclInfos()); - for (VariableDeclInfo declInfo : children) { - visitVariableDeclInfo(declInfo); - currentScope.add(declInfo.getName(), - declInfo.getArrayInfo() == null - ? variablesDeclaration.getBaseType() - : new ArrayType(variablesDeclaration.getBaseType(), declInfo.getArrayInfo()), - Optional.empty(), - declInfo, variablesDeclaration); - visitVariableDeclInfoAfterAddedToScope(declInfo); - } - } - - /** - * This is a hook for subclasses that need to perform an action after a declaration has been added - * to the current scope. - * - * @param declInfo The declaration info that was just added to the current scope - */ - protected void visitVariableDeclInfoAfterAddedToScope(VariableDeclInfo declInfo) { - // Deliberately empty - to be optionally overridden by subclasses. - } - - protected void popScope() { - currentScope = currentScope.getParent(); - } - - protected void pushScope() { - Scope newScope = new Scope(currentScope); - currentScope = newScope; - } - - protected List getEncounteredFunctionPrototypes() { - return Collections.unmodifiableList(encounteredFunctionPrototypes); - } - - protected boolean atGlobalScope() { - return !currentScope.hasParent(); - } - - -} diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/SupportedTypes.java b/ast/src/main/java/com/graphicsfuzz/common/typing/SupportedTypes.java index 67a34e0f1..3441b6d17 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/SupportedTypes.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/SupportedTypes.java @@ -26,8 +26,7 @@ public static boolean supported(BasicType type, ShadingLanguageVersion shadingLa && !shadingLanguageVersion.supportedUnsigned()) { return false; } - if (BasicType.allMatrixTypes().contains(type) - && !BasicType.allSquareMatrixTypes().contains(type) + if (type.isMatrix() && !BasicType.allSquareMatrixTypes().contains(type) && !shadingLanguageVersion.supportedNonSquareMatrices()) { return false; } diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java b/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java index 0b1f78597..7f276871c 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/Typer.java @@ -16,7 +16,6 @@ package com.graphicsfuzz.common.typing; -import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; @@ -41,9 +40,9 @@ import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.type.TypeQualifier; -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.ast.visitors.UnsupportedLanguageFeatureException; import com.graphicsfuzz.common.util.OpenGlConstants; -import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -52,30 +51,22 @@ import java.util.Optional; import java.util.Set; -public class Typer extends ScopeTreeBuilder { +public class Typer extends ScopeTrackingVisitor { - private Map types; + private final TranslationUnit tu; - private Map> userDefinedFunctions; + private final Map types; - private Map structDeclarationMap; + private final Map> userDefinedFunctions; - private ShadingLanguageVersion shadingLanguageVersion; + private final Map structDeclarationMap; - public Map> getUserDefinedFunctions() { - return userDefinedFunctions; - } - - public Typer(IAstNode node, ShadingLanguageVersion shadingLanguageVersion) { + public Typer(TranslationUnit tu) { + this.tu = tu; this.types = new HashMap<>(); this.userDefinedFunctions = new HashMap<>(); this.structDeclarationMap = new HashMap<>(); - this.shadingLanguageVersion = shadingLanguageVersion; - visit(node); - } - - public Typer(TranslationUnit tu) { - this(tu, tu.getShadingLanguageVersion()); + visit(tu); } @Override @@ -111,26 +102,32 @@ public void visitFunctionDefinition(FunctionDefinition functionDefinition) { public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { super.visitFunctionCallExpr(functionCallExpr); - List candidateBuiltins = TyperHelper.getBuiltins(shadingLanguageVersion) - .get(functionCallExpr.getCallee()); - if (candidateBuiltins != null) { - for (FunctionPrototype prototype : candidateBuiltins) { - if (prototypeMatches(prototype, functionCallExpr)) { - types.put(functionCallExpr, prototype.getReturnType()); - } - } + // First, see if there is a builtin with a matching prototype. + final Optional maybeMatchingBuiltinFunctionReturn = + lookForMatchingFunction(functionCallExpr, + TyperHelper.getBuiltins(tu.getShadingLanguageVersion(), tu.getShaderKind()) + .get(functionCallExpr.getCallee())); + if (maybeMatchingBuiltinFunctionReturn.isPresent()) { + types.put(functionCallExpr, maybeMatchingBuiltinFunctionReturn.get()); + return; } - Set candidateUserDefined = - userDefinedFunctions.get(functionCallExpr.getCallee()); - if (candidateUserDefined != null) { - for (FunctionPrototype prototype : candidateUserDefined) { + // If there was no relevant builtin, see whether there is a user-defined type. + lookForMatchingFunction(functionCallExpr, + userDefinedFunctions.get(functionCallExpr.getCallee())) + .ifPresent(type -> types.put(functionCallExpr, type)); + } + + private Optional lookForMatchingFunction(FunctionCallExpr functionCallExpr, + Collection candidateFunctions) { + if (candidateFunctions != null) { + for (FunctionPrototype prototype : candidateFunctions) { if (prototypeMatches(prototype, functionCallExpr)) { - types.put(functionCallExpr, prototype.getReturnType()); + return Optional.of(prototype.getReturnType()); } } } - + return Optional.empty(); } /** @@ -155,8 +152,11 @@ public boolean prototypeMatches(FunctionPrototype prototype, FunctionCallExpr fu if (argType == null) { return false; } - // Not yet worked out how to deal with array info - assert prototype.getParameters().get(i).getArrayInfo() == null; + // TODO(https://github.com/google/graphicsfuzz/issues/784) Not yet worked out how to deal with + // array info + if (prototype.getParameters().get(i).getArrayInfo() != null) { + throw new UnsupportedLanguageFeatureException("Array parameters are not yet supported."); + } if (!argType.getWithoutQualifiers() .equals(prototype.getParameters().get(i).getType().getWithoutQualifiers())) { return false; @@ -168,7 +168,7 @@ public boolean prototypeMatches(FunctionPrototype prototype, FunctionCallExpr fu @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - Type type = currentScope.lookupType(variableIdentifierExpr.getName()); + Type type = getCurrentScope().lookupType(variableIdentifierExpr.getName()); if (type != null) { types.put(variableIdentifierExpr, type); return; @@ -345,7 +345,9 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { case LT: case LXOR: case NE: - types.put(binaryExpr, resolveBooleanResultType(lhsType, rhsType)); + // The above all yield 'bool', even if (as is allowed in the case of '==' and '!=' they are + // applied to vector types. + types.put(binaryExpr, BasicType.BOOL); return; case COMMA: // The type of "e1, e2" is the type of "e2". @@ -438,26 +440,6 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { } } - private Type resolveBooleanResultType(Type lhsType, Type rhsType) { - return maybeComputeBooleanVectorType(lhsType) - .orElse(maybeComputeBooleanVectorType(rhsType) - .orElse(BasicType.BOOL)); - } - - private Optional maybeComputeBooleanVectorType(Type lhsType) { - if (lhsType instanceof BasicType) { - final int numElements = ((BasicType) lhsType).getNumElements(); - if (1 < numElements && numElements <= 4) { - return Optional.of(BasicType.makeVectorType(BasicType.BOOL, numElements)); - } - } - return Optional.empty(); - } - - public Set getTypedExpressions() { - return Collections.unmodifiableSet(types.keySet()); - } - public Type lookupType(Expr expr) { return types.get(expr); } @@ -471,8 +453,10 @@ public Set getPrototypes(String name) { if (userDefinedFunctions.containsKey(name)) { result.addAll(userDefinedFunctions.get(name)); } - if (TyperHelper.getBuiltins(shadingLanguageVersion).containsKey(name)) { - result.addAll(TyperHelper.getBuiltins(shadingLanguageVersion).get(name)); + final Map> builtins = + TyperHelper.getBuiltins(tu.getShadingLanguageVersion(), tu.getShaderKind()); + if (builtins.containsKey(name)) { + result.addAll(builtins.get(name)); } return result; } diff --git a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java index d47575141..24ab430fc 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java +++ b/ast/src/main/java/com/graphicsfuzz/common/typing/TyperHelper.java @@ -18,8 +18,12 @@ import com.graphicsfuzz.common.ast.decl.FunctionPrototype; import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.QualifiedType; import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.common.ast.type.TypeQualifier; +import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.ShaderKind; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -28,6 +32,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; /** * This helper class factors out some context-independent parts of typechecking, @@ -35,8 +40,11 @@ */ public final class TyperHelper { + // Maps a (shading language version, shader kind) pair to a mapping from builtin names to builtin + // function prototypes. private static ConcurrentMap>> builtins = new ConcurrentHashMap<>(); + ConcurrentMap>>> builtins = + new ConcurrentHashMap<>(); private TyperHelper() { // Utility class @@ -280,111 +288,757 @@ public static Type resolveTypeOfMul(Type lhsType, Type rhsType) { return null; } + /** + * Yield the builtins available for the given shading language version and kind of shader. + * + * @param shadingLanguageVersion version of GLSL for which relevant builtins should be returned. + * @param shaderKind kind of shader (e.g. fragment or compute) for which relevant builtins + * should be returned. + * @return a mapping from name of builtin to sequence of function prototypes. + */ public static Map> getBuiltins(ShadingLanguageVersion - shadingLanguageVersion) { + shadingLanguageVersion, ShaderKind shaderKind) { + + assert shadingLanguageVersion != null; + assert shaderKind != null; + if (!builtins.containsKey(shadingLanguageVersion)) { - builtins.putIfAbsent(shadingLanguageVersion, - getBuiltinsForGlslVersion(shadingLanguageVersion)); + builtins.putIfAbsent(shadingLanguageVersion, new ConcurrentHashMap<>()); + } + if (!builtins.get(shadingLanguageVersion).containsKey(shaderKind)) { + builtins.get(shadingLanguageVersion).putIfAbsent(shaderKind, + getBuiltinsForGlslVersion(shadingLanguageVersion, shaderKind)); } - return Collections.unmodifiableMap(builtins.get(shadingLanguageVersion)); + return Collections.unmodifiableMap(builtins.get(shadingLanguageVersion).get(shaderKind)); } private static Map> getBuiltinsForGlslVersion( - ShadingLanguageVersion shadingLanguageVersion) { + ShadingLanguageVersion shadingLanguageVersion, ShaderKind shaderKind) { Map> builtinsForVersion = new HashMap<>(); - // Trigonometric Functions + // 8.1: Angle and Trigonometric Functions + + getBuiltinsForGlslVersionAngleAndTrigonometric(shadingLanguageVersion, + builtinsForVersion); + + // 8.2: Exponential Functions + + getBuiltinsForGlslVersionExponential(builtinsForVersion); + + // 8.3: Common Functions + + getBuiltinsForGlslVersionCommon(shadingLanguageVersion, builtinsForVersion); + + // 8.4: Floating-Point Pack and Unpack Functions + + getBuiltinsForGlslVersionFloatingPointPackAndUnpack(builtinsForVersion, shadingLanguageVersion); + + // 8.5: Geometric Functions + + getBuiltinsForGlslVersionGeometric(builtinsForVersion); + + // 8.6: Matrix Functions + + getBuiltinsForGlslVersionMatrix(builtinsForVersion, shadingLanguageVersion); + + // 8.7: Vector Relational Functions + + getBuiltinsForGlslVersionVectorRelational(builtinsForVersion, shadingLanguageVersion); + + // 8.8: Integer Functions + + getBuiltinsForGlslVersionInteger(builtinsForVersion, shadingLanguageVersion); + + // 8.13: Fragment Processing Functions (only available in fragment shaders) + + if (shaderKind == ShaderKind.FRAGMENT) { + getBuiltinsForGlslVersionFragmentProcessing(builtinsForVersion, shadingLanguageVersion); + } + + // 8.14: Noise Functions - deprecated, so we do not consider them + + return builtinsForVersion; + } + + /** + * Helper function to register built-in function prototypes for Angle and Trigonometric + * Functions, as specified in section 8.1 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionAngleAndTrigonometric( + ShadingLanguageVersion shadingLanguageVersion, + Map> builtinsForVersion) { + if (shadingLanguageVersion.supportedAngleAndTrigonometricFunctions()) { + { + final String name = "radians"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "degrees"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "sin"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "cos"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "tan"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "asin"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "acos"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "atan"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + } + + if (shadingLanguageVersion.supportedHyperbolicAngleAndTrigonometricFunctions()) { + { + final String name = "sinh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "cosh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "tanh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "asinh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "acosh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "atanh"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + } + } + + /** + * Helper function to register built-in function prototypes for Exponential Functions, as + * specified in section 8.2 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + */ + private static void getBuiltinsForGlslVersionExponential( + Map> builtinsForVersion) { { - final String name = "sin"; + final String name = "pow"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); + addBuiltin(builtinsForVersion, name, t, t, t); } } + { - final String name = "cos"; + final String name = "exp"; for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); } } + { - final String name = "asin"; + final String name = "log"; for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); } } + { - final String name = "acos"; + final String name = "exp2"; for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); } } + { - final String name = "tan"; + final String name = "log2"; for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); } } + { - final String name = "atan"; + final String name = "sqrt"; for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); } } + { - final String name = "atan"; + final String name = "inversesqrt"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t); + addBuiltin(builtinsForVersion, name, t, t); } } + } - //Exponential Functions + private static void getBuiltinsForGlslVersionVectorRelational( + Map> builtinsForVersion, + ShadingLanguageVersion shadingLanguageVersion) { + // We need these for every function, so instead of constantly calling the functions, + // we'll just cache them to reduce cruft. + final List genVectors = genType().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); + final List igenVectors = igenType().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); + final List ugenVectors = ugenType().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); + final List bgenVectors = bgenType().stream().filter( + item -> !BasicType.allScalarTypes().contains(item)).collect(Collectors.toList()); + final boolean supportsUnsigned = shadingLanguageVersion.supportedUnsigned(); { - final String name = "pow"; + final String name = "lessThan"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); + } + } + } + + { + final String name = "lessThanEqual"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); + } + } + } + + { + final String name = "greaterThan"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); + } + } + } + + { + final String name = "greaterThanEqual"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); + } + } + } + + { + final String name = "equal"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); + } + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), bgenVectors.get(i), + bgenVectors.get(i)); + } + } + + { + final String name = "notEqual"; + for (int i = 0; i < bgenVectors.size(); i++) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), genVectors.get(i), + genVectors.get(i)); + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), igenVectors.get(i), + igenVectors.get(i)); + if (supportsUnsigned) { + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), ugenVectors.get(i), + ugenVectors.get(i)); + } + addBuiltin(builtinsForVersion, name, bgenVectors.get(i), bgenVectors.get(i), + bgenVectors.get(i)); + } + } + + { + final String name = "any"; + for (Type t : bgenVectors) { + addBuiltin(builtinsForVersion, name, BasicType.BOOL, t); + } + } + + { + final String name = "all"; + for (Type t : bgenVectors) { + addBuiltin(builtinsForVersion, name, BasicType.BOOL, t); + } + } + + { + final String name = "not"; + for (Type t : bgenVectors) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + } + + + /** + * Helper function to register built-in function prototypes for Integer Functions, + * as specified in section 8.8 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionInteger( + Map> builtinsForVersion, + ShadingLanguageVersion shadingLanguageVersion) { + if (shadingLanguageVersion.supportedIntegerFunctions()) { + { + final String name = "uaddCarry"; + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, t, new QualifiedType(t, + Arrays.asList(TypeQualifier.OUT_PARAM))); + } + } + + { + final String name = "usubBorrow"; + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, t, new QualifiedType(t, + Arrays.asList(TypeQualifier.OUT_PARAM))); + } + } + + { + final String name = "umulExtended"; + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, VoidType.VOID, t, t, + new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM)), + new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM))); + } + } + + { + final String name = "imulExtended"; + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, VoidType.VOID, t, t, + new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM)), + new QualifiedType(t, Arrays.asList(TypeQualifier.OUT_PARAM))); + } + } + + { + final String name = "bitfieldExtract"; + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.INT, BasicType.INT); + } + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, BasicType.INT, BasicType.INT); + } + } + + { + final String name = "bitfieldInsert"; + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t, t, BasicType.INT, BasicType.INT); + } + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t, t, BasicType.INT, BasicType.INT); + } + } + + { + final String name = "bitfieldReverse"; + for (Type t : igenType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + for (Type t : ugenType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + // We need to use both igen and ugen types of the same size for these builtins, so we need a + // counting loop instead of an iterator to access both lists at the same time. + { + final String name = "bitCount"; + for (int i = 0; i < igenType().size(); i++) { + addBuiltin(builtinsForVersion, name, igenType().get(i), igenType().get(i)); + addBuiltin(builtinsForVersion, name, igenType().get(i), ugenType().get(i)); + } + } + + { + final String name = "findLSB"; + for (int i = 0; i < igenType().size(); i++) { + addBuiltin(builtinsForVersion, name, igenType().get(i), igenType().get(i)); + addBuiltin(builtinsForVersion, name, igenType().get(i), ugenType().get(i)); + } + } + + { + final String name = "findMSB"; + for (int i = 0; i < igenType().size(); i++) { + addBuiltin(builtinsForVersion, name, igenType().get(i), igenType().get(i)); + addBuiltin(builtinsForVersion, name, igenType().get(i), ugenType().get(i)); + } + } + } + } + + /** + * Helper function to register built-in function prototypes for Fragment Processing Functions, + * as specified in section 8.14 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionFragmentProcessing( + Map> builtinsForVersion, + ShadingLanguageVersion shadingLanguageVersion) { + // 8.14.1 Derivative Functions + if (shadingLanguageVersion.supportedDerivativeFunctions()) { + { + final String name = "dFdx"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "dFdy"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "fwidth"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + } + + if (shadingLanguageVersion.supportedExplicitDerivativeFunctions()) { + { + final String name = "dFdxFine"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "dFdxCoarse"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "dFdyFine"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "dFdyCoarse"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "fwidthFine"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + + { + final String name = "fwidthCoarse"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t); + } + } + } + + // 8.14.2 Interpolation Functions + // TODO(550): Support functions that take non-uniform shader input variables as parameters. + } + + /** + * Helper function to register built-in function prototypes for Matrix Functions, + * as specified in section 8.6 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionMatrix( + Map> builtinsForVersion, + ShadingLanguageVersion shadingLanguageVersion) { + { + final String name = "matrixCompMult"; + for (Type t : BasicType.allMatrixTypes()) { + if (BasicType.allSquareMatrixTypes().contains(t) + || shadingLanguageVersion.supportedMatrixCompMultNonSquare()) { + addBuiltin(builtinsForVersion, name, t, t, t); + } + } + } + + if (shadingLanguageVersion.supportedOuterProduct()) { + final String name = "outerProduct"; + addBuiltin(builtinsForVersion, name, BasicType.MAT2X2, BasicType.VEC2, BasicType.VEC2); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.VEC3, BasicType.VEC3); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.VEC4, BasicType.VEC4); + addBuiltin(builtinsForVersion, name, BasicType.MAT2X3, BasicType.VEC3, BasicType.VEC2); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X2, BasicType.VEC2, BasicType.VEC3); + addBuiltin(builtinsForVersion, name, BasicType.MAT2X4, BasicType.VEC4, BasicType.VEC2); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X2, BasicType.VEC2, BasicType.VEC4); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X4, BasicType.VEC4, BasicType.VEC3); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X3, BasicType.VEC3, BasicType.VEC4); + } + + if (shadingLanguageVersion.supportedTranspose()) { + final String name = "transpose"; + addBuiltin(builtinsForVersion, name, BasicType.MAT2X2, BasicType.MAT2X2); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X2, BasicType.MAT2X3); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X2, BasicType.MAT2X4); + addBuiltin(builtinsForVersion, name, BasicType.MAT2X3, BasicType.MAT3X2); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.MAT3X3); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X3, BasicType.MAT3X4); + addBuiltin(builtinsForVersion, name, BasicType.MAT2X4, BasicType.MAT4X2); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X4, BasicType.MAT4X3); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.MAT4X4); + } + + if (shadingLanguageVersion.supportedDeterminant()) { + final String name = "determinant"; + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.MAT2X2); + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.MAT3X3); + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.MAT4X4); + } + + if (shadingLanguageVersion.supportedInverse()) { + final String name = "inverse"; + addBuiltin(builtinsForVersion, name, BasicType.MAT2X2, BasicType.MAT2X2); + addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.MAT3X3); + addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.MAT4X4); + } + } + + /** + * Helper function to register built-in function prototypes for Geometric Functions, + * as specified in section 8.5 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + */ + private static void getBuiltinsForGlslVersionGeometric( + Map> builtinsForVersion) { + { + final String name = "length"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t); + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t); + } + } + + { + final String name = "distance"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t, t); } } + { - final String name = "exp"; + final String name = "dot"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t, t); } } + { - final String name = "log"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } + final String name = "cross"; + addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.VEC3, BasicType.VEC3); } + { - final String name = "exp2"; + final String name = "normalize"; for (Type t : genType()) { addBuiltin(builtinsForVersion, name, t, t); } } + { - final String name = "log2"; + final String name = "faceforward"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); + addBuiltin(builtinsForVersion, name, t, t, t, t); } } + { - final String name = "sqrt"; + final String name = "reflect"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); + addBuiltin(builtinsForVersion, name, t, t, t); } } + { - final String name = "inversesqrt"; + final String name = "refract"; for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); + addBuiltin(builtinsForVersion, name, t, t, t, BasicType.FLOAT); } } + } + + /** + * Helper function to register built-in function prototypes for Floating-Point Pack and + * Unpack Functions, as specified in section 8.4 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionFloatingPointPackAndUnpack( + Map> builtinsForVersion, + ShadingLanguageVersion shadingLanguageVersion) { + + if (shadingLanguageVersion.supportedPackUnorm2x16()) { + final String name = "packUnorm2x16"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); + } + if (shadingLanguageVersion.supportedPackSnorm2x16()) { + final String name = "packSnorm2x16"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); + } - // 8.3: Common Functions + if (shadingLanguageVersion.supportedPackUnorm4x8()) { + final String name = "packUnorm4x8"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC4); + } + + if (shadingLanguageVersion.supportedPackSnorm4x8()) { + final String name = "packSnorm4x8"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC4); + } + + if (shadingLanguageVersion.supportedUnpackUnorm2x16()) { + final String name = "unpackUnorm2x16"; + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); + } + + if (shadingLanguageVersion.supportedUnpackSnorm2x16()) { + final String name = "unpackSnorm2x16"; + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); + } + + if (shadingLanguageVersion.supportedUnpackUnorm4x8()) { + final String name = "unpackUnorm4x8"; + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UINT); + } + + if (shadingLanguageVersion.supportedUnpackSnorm4x8()) { + final String name = "unpackSnorm4x8"; + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UINT); + } + + if (shadingLanguageVersion.supportedPackHalf2x16()) { + final String name = "packHalf2x16"; + addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); + } + + if (shadingLanguageVersion.supportedUnpackHalf2x16()) { + final String name = "unpackHalf2x16"; + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); + } + } + /** + * Helper function to register built-in function prototypes for Common Functions, + * as specified in section 8.3 of the GLSL 4.6 and ESSL 3.2 specifications. + * + * @param builtinsForVersion the list of builtins to add prototypes to + * @param shadingLanguageVersion the version of GLSL in use + */ + private static void getBuiltinsForGlslVersionCommon( + ShadingLanguageVersion shadingLanguageVersion, + Map> builtinsForVersion) { { final String name = "abs"; for (Type t : genType()) { @@ -461,7 +1115,15 @@ private static Map> getBuiltinsForGlslVersion( } } - // TODO: genType modf(genType, out genType) + if (shadingLanguageVersion.supportedModf()) { + { + final String name = "modf"; + for (Type t : genType()) { + addBuiltin(builtinsForVersion, name, t, t, new QualifiedType(t, + Arrays.asList(TypeQualifier.OUT_PARAM))); + } + } + } { final String name = "min"; @@ -665,186 +1327,29 @@ private static Map> getBuiltinsForGlslVersion( } } - { - @SuppressWarnings("unused") - final String name = "frexp"; - // TODO: genType frexp(genType, out genIType) - } - - { - @SuppressWarnings("unused") - final String name = "ldexp"; - // TODO: genType frexp(genType, in genIType) - } - - // 8.4: Floating-Point Pack and Unpack Functions - - if (shadingLanguageVersion.supportedPackUnorm2x16()) { - final String name = "packUnorm2x16"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); - } - - if (shadingLanguageVersion.supportedPackSnorm2x16()) { - final String name = "packSnorm2x16"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); - } - - if (shadingLanguageVersion.supportedPackUnorm4x8()) { - final String name = "packUnorm4x8"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC4); - } - - if (shadingLanguageVersion.supportedPackSnorm4x8()) { - final String name = "packSnorm4x8"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC4); - } - - if (shadingLanguageVersion.supportedUnpackUnorm2x16()) { - final String name = "unpackUnorm2x16"; - addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); - } - - if (shadingLanguageVersion.supportedUnpackSnorm2x16()) { - final String name = "unpackSnorm2x16"; - addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); - } - - if (shadingLanguageVersion.supportedUnpackUnorm4x8()) { - final String name = "unpackUnorm4x8"; - addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UINT); - } - - if (shadingLanguageVersion.supportedUnpackSnorm4x8()) { - final String name = "unpackSnorm4x8"; - addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.UINT); - } - - if (shadingLanguageVersion.supportedPackHalf2x16()) { - final String name = "packHalf2x16"; - addBuiltin(builtinsForVersion, name, BasicType.UINT, BasicType.VEC2); - } - - if (shadingLanguageVersion.supportedUnpackHalf2x16()) { - final String name = "unpackHalf2x16"; - addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.UINT); - } - - // 8.5: Geometric Functions - - { - final String name = "length"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t); - } - } - - { - final String name = "distance"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t, t); - } - } - - { - final String name = "dot"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, t, t); - } - } - - { - final String name = "cross"; - addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.VEC3, BasicType.VEC3); - } - - { - final String name = "normalize"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t); - } - } - - { - final String name = "faceforward"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t, t); - } - } - - { - final String name = "reflect"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t); - } - } - - { - final String name = "refract"; - for (Type t : genType()) { - addBuiltin(builtinsForVersion, name, t, t, t, BasicType.FLOAT); + if (shadingLanguageVersion.supportedFrexp()) { + { + final String name = "frexp"; + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.FLOAT, + new QualifiedType(BasicType.INT, Arrays.asList(TypeQualifier.OUT_PARAM))); + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.VEC2, + new QualifiedType(BasicType.IVEC2, Arrays.asList(TypeQualifier.OUT_PARAM))); + addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.VEC3, + new QualifiedType(BasicType.IVEC3, Arrays.asList(TypeQualifier.OUT_PARAM))); + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.VEC4, + new QualifiedType(BasicType.IVEC4, Arrays.asList(TypeQualifier.OUT_PARAM))); } } - // 8.6: Matrix Functions - { - final String name = "matrixCompMult"; - for (Type t : BasicType.allMatrixTypes()) { - if (BasicType.allSquareMatrixTypes().contains(t) - || shadingLanguageVersion.supportedMatrixCompMultNonSquare()) { - addBuiltin(builtinsForVersion, name, t, t, t); - } + if (shadingLanguageVersion.supportedLdexp()) { + { + final String name = "ldexp"; + addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.FLOAT, BasicType.INT); + addBuiltin(builtinsForVersion, name, BasicType.VEC2, BasicType.VEC2, BasicType.IVEC2); + addBuiltin(builtinsForVersion, name, BasicType.VEC3, BasicType.VEC3, BasicType.IVEC3); + addBuiltin(builtinsForVersion, name, BasicType.VEC4, BasicType.VEC4, BasicType.IVEC4); } } - - if (shadingLanguageVersion.supportedOuterProduct()) { - final String name = "outerProduct"; - addBuiltin(builtinsForVersion, name, BasicType.MAT2X2, BasicType.VEC2, BasicType.VEC2); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.VEC3, BasicType.VEC3); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.VEC4, BasicType.VEC4); - addBuiltin(builtinsForVersion, name, BasicType.MAT2X3, BasicType.VEC3, BasicType.VEC2); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X2, BasicType.VEC2, BasicType.VEC3); - addBuiltin(builtinsForVersion, name, BasicType.MAT2X4, BasicType.VEC4, BasicType.VEC2); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X2, BasicType.VEC2, BasicType.VEC4); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X4, BasicType.VEC4, BasicType.VEC3); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X3, BasicType.VEC3, BasicType.VEC4); - } - - if (shadingLanguageVersion.supportedTranspose()) { - final String name = "transpose"; - addBuiltin(builtinsForVersion, name, BasicType.MAT2X2, BasicType.MAT2X2); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X2, BasicType.MAT2X3); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X2, BasicType.MAT2X4); - addBuiltin(builtinsForVersion, name, BasicType.MAT2X3, BasicType.MAT3X2); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.MAT3X3); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X3, BasicType.MAT3X4); - addBuiltin(builtinsForVersion, name, BasicType.MAT2X4, BasicType.MAT4X2); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X4, BasicType.MAT4X3); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.MAT4X4); - } - - if (shadingLanguageVersion.supportedDeterminant()) { - final String name = "determinant"; - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.MAT2X2); - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.MAT3X3); - addBuiltin(builtinsForVersion, name, BasicType.FLOAT, BasicType.MAT4X4); - } - - if (shadingLanguageVersion.supportedInverse()) { - final String name = "inverse"; - addBuiltin(builtinsForVersion, name, BasicType.MAT2X2, BasicType.MAT2X2); - addBuiltin(builtinsForVersion, name, BasicType.MAT3X3, BasicType.MAT3X3); - addBuiltin(builtinsForVersion, name, BasicType.MAT4X4, BasicType.MAT4X4); - } - - // 8.7: Vector Relational Functions - - // 8.8: Integer Functions - - // 8.13: Fragment Processing Functions (only available in fragment shaders) - - // 8.14: Noise Functions - deprecated, so we do not consider them - - return builtinsForVersion; } private static void addBuiltin(Map> builtinsForVersion, diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/MacroNames.java b/ast/src/main/java/com/graphicsfuzz/common/util/MacroNames.java similarity index 71% rename from reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/MacroNames.java rename to ast/src/main/java/com/graphicsfuzz/common/util/MacroNames.java index 459408b94..3ec58adc1 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/MacroNames.java +++ b/ast/src/main/java/com/graphicsfuzz/common/util/MacroNames.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.graphicsfuzz.reducer.reductionopportunities; +package com.graphicsfuzz.common.util; import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; @@ -22,6 +22,25 @@ public class MacroNames { + /** + * Determines whether the given expression is an invocation of a GraphicsFuzz macro. + * @param expr an expression to be tested. + * @return true if and only if the expression is an invocation of a GraphicsFuzz macro. + */ + public static boolean isGraphicsFuzzMacro(Expr expr) { + return isIdentity(expr) + || isZero(expr) + || isOne(expr) + || isFalse(expr) + || isTrue(expr) + || isFuzzed(expr) + || isDeadByConstruction(expr) + || isLoopWrapper(expr) + || isIfWrapperTrue(expr) + || isIfWrapperFalse(expr) + || isSwitch(expr); + } + public static boolean isIdentity(Expr expr) { return isCallToNamedFunction(expr, Constants.GLF_IDENTITY); } @@ -66,6 +85,14 @@ public static boolean isSwitch(Expr expr) { return isCallToNamedFunction(expr, Constants.GLF_SWITCH); } + public static boolean isMakeInBoundsInt(Expr expr) { + return isCallToNamedFunction(expr, Constants.GLF_MAKE_IN_BOUNDS_INT); + } + + public static boolean isMakeInBoundsUint(Expr expr) { + return isCallToNamedFunction(expr, Constants.GLF_MAKE_IN_BOUNDS_UINT); + } + private static boolean isCallToNamedFunction(Expr expr, String functionName) { return expr instanceof FunctionCallExpr && ((FunctionCallExpr) expr).getCallee() .equals(functionName); diff --git a/ast/src/main/java/com/graphicsfuzz/common/util/TruncateLoops.java b/ast/src/main/java/com/graphicsfuzz/common/util/TruncateLoops.java index 4751e8cd5..6bb96053e 100644 --- a/ast/src/main/java/com/graphicsfuzz/common/util/TruncateLoops.java +++ b/ast/src/main/java/com/graphicsfuzz/common/util/TruncateLoops.java @@ -18,15 +18,12 @@ import com.graphicsfuzz.common.ast.IParentMap; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; -import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; -import com.graphicsfuzz.common.ast.expr.Op; -import com.graphicsfuzz.common.ast.expr.ParenExpr; import com.graphicsfuzz.common.ast.expr.UnOp; import com.graphicsfuzz.common.ast.expr.UnaryExpr; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; @@ -45,23 +42,18 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Optional; -import org.apache.commons.lang3.tuple.ImmutablePair; public class TruncateLoops extends StandardVisitor { private final int limit; private final TranslationUnit tu; private final String prefix; - private final boolean ignoreShortRunningForLoops; private int counter; - public TruncateLoops(int limit, String prefix, TranslationUnit tu, - boolean ignoreShortRunningForLoops) { + public TruncateLoops(int limit, String prefix, TranslationUnit tu) { this.limit = limit; this.tu = tu; this.prefix = prefix; - this.ignoreShortRunningForLoops = ignoreShortRunningForLoops; counter = 0; visit(tu); } @@ -69,9 +61,7 @@ public TruncateLoops(int limit, String prefix, TranslationUnit tu, @Override public void visitForStmt(ForStmt forStmt) { super.visitForStmt(forStmt); - if (!ignoreShortRunningForLoops || maybeLongRunning(forStmt)) { - handleLoop(forStmt); - } + handleLoop(forStmt); } @Override @@ -94,7 +84,7 @@ private void handleLoop(LoopStmt loopStmt) { final DeclarationStmt limiterDeclaration = new DeclarationStmt( new VariablesDeclaration(BasicType.INT, new VariableDeclInfo(limiterName, null, - new ScalarInitializer(new IntConstantExpr("0"))))); + new Initializer(new IntConstantExpr("0"))))); final List limitCheckAndIncrement = Arrays.asList( new IfStmt( @@ -124,161 +114,4 @@ private void handleLoop(LoopStmt loopStmt) { parentMap.getParent(loopStmt).replaceChild(loopStmt, replacementBlock); } - private boolean maybeLongRunning(ForStmt forStmt) { - final Optional> initValueAndLoopCounterName - = getLoopCounterNameAndInitValue(forStmt.getInit()); - if (!initValueAndLoopCounterName.isPresent()) { - return false; - } - final String loopCounterName = initValueAndLoopCounterName.get().left; - final int initValue = initValueAndLoopCounterName.get().right; - - final Optional> condTestTypeAndLimitValue - = getCondTestTypeAndLimitValue(forStmt.getCondition(), loopCounterName); - if (!condTestTypeAndLimitValue.isPresent()) { - return false; - } - final BinOp condTestType = condTestTypeAndLimitValue.get().left; - final int condLimitValue = condTestTypeAndLimitValue.get().right; - if (condTestType.isSideEffecting()) { - return false; - } - - final Optional incrementValue - = getIncrementValue(forStmt.getIncrement(), loopCounterName); - if (!incrementValue.isPresent()) { - return false; - } - - return (((condTestType == BinOp.LT || condTestType == BinOp.LE) && incrementValue.get() <= 0) - || ((condTestType == BinOp.GT || condTestType == BinOp.GE) && incrementValue.get() >= 0) - || ((condLimitValue - initValue) / incrementValue.get() >= limit)); - } - - private Optional> getLoopCounterNameAndInitValue(Stmt init) { - String name = null; - Expr expr = null; - if (init instanceof ExprStmt - && ((ExprStmt) init).getExpr() instanceof BinaryExpr - && ((BinaryExpr)(((ExprStmt) init).getExpr())).getOp() == BinOp.ASSIGN - && ((BinaryExpr)(((ExprStmt) init).getExpr())).getLhs() - instanceof VariableIdentifierExpr) { - name = ((VariableIdentifierExpr) (((BinaryExpr)(((ExprStmt) init) - .getExpr())).getLhs())).getName(); - expr = ((BinaryExpr)(((ExprStmt) init) - .getExpr())).getRhs(); - } else if (init instanceof DeclarationStmt - && ((DeclarationStmt) init).getVariablesDeclaration().getNumDecls() == 1 - && ((DeclarationStmt) init).getVariablesDeclaration().getDeclInfo(0) - .getInitializer() instanceof ScalarInitializer) { - name = ((DeclarationStmt) init).getVariablesDeclaration() - .getDeclInfo(0).getName(); - expr = ((ScalarInitializer)((DeclarationStmt) init) - .getVariablesDeclaration().getDeclInfo(0) - .getInitializer()).getExpr(); - } - if (name == null || expr == null) { - return Optional.empty(); - } - Optional constant = getAsConstant(expr); - if (!constant.isPresent()) { - return Optional.empty(); - } - return Optional.of(new ImmutablePair<>(name, constant.get())); - } - - private Optional> getCondTestTypeAndLimitValue(Expr cond, - String loopCounterName) { - if (!(cond instanceof BinaryExpr)) { - return Optional.empty(); - } - final BinaryExpr binaryExprCond = (BinaryExpr) cond; - Optional condLimitValue = getAsConstant(binaryExprCond.getRhs()); - if (condLimitValue.isPresent() - && isSameVarIdentifier(binaryExprCond.getLhs(), loopCounterName)) { - return Optional.of(new ImmutablePair<>(binaryExprCond.getOp(), condLimitValue.get())); - } - condLimitValue = getAsConstant(binaryExprCond.getLhs()); - if (condLimitValue.isPresent() - && isSameVarIdentifier(binaryExprCond.getRhs(), loopCounterName)) { - return Optional.of(new ImmutablePair<>(switchCondTestType(binaryExprCond.getOp()), - condLimitValue.get())); - } - - return Optional.empty(); - } - - private Optional getIncrementValue(Expr incr, String loopCounterName) { - if (incr instanceof UnaryExpr - && isSameVarIdentifier(((UnaryExpr) incr).getExpr(), loopCounterName)) { - switch (((UnaryExpr) incr).getOp()) { - case POST_INC: - case PRE_INC: - return Optional.of(1); - case POST_DEC: - case PRE_DEC: - return Optional.of(-1); - default: - return Optional.empty(); - } - } - if (incr instanceof BinaryExpr - && isSameVarIdentifier(((BinaryExpr) incr).getLhs(), loopCounterName)) { - final Optional incrementValue = getAsConstant(((BinaryExpr) incr).getRhs()); - if (!incrementValue.isPresent()) { - return Optional.empty(); - } - switch (((BinaryExpr) incr).getOp()) { - case ADD_ASSIGN: - return incrementValue; - case SUB_ASSIGN: - return incrementValue.map(item -> -item); - default: - return Optional.empty(); - } - } - return Optional.empty(); - } - - private boolean isSameVarIdentifier(Expr expr, String loopCounterName) { - return expr instanceof VariableIdentifierExpr - && ((VariableIdentifierExpr)expr).getName().equals(loopCounterName); - } - - private Optional getAsConstant(Expr expr) { - if (expr instanceof ParenExpr) { - return getAsConstant(((ParenExpr)expr).getExpr()); - } - if (expr instanceof IntConstantExpr) { - return Optional.of(Integer.valueOf(((IntConstantExpr) expr).getValue())); - } - if (expr instanceof UnaryExpr) { - final UnaryExpr unaryExpr = (UnaryExpr) expr; - switch (unaryExpr.getOp()) { - case MINUS: - return getAsConstant(unaryExpr.getExpr()).map(item -> -item); - case PLUS: - return getAsConstant(unaryExpr.getExpr()); - default: - return Optional.empty(); - } - } - return Optional.empty(); - } - - private BinOp switchCondTestType(BinOp binOp) { - switch (binOp) { - case LT: - return BinOp.GT; - case GT: - return BinOp.LT; - case LE: - return BinOp.GE; - case GE: - return BinOp.LE; - default: - return binOp; - } - } - } diff --git a/ast/src/test/java/com/graphicsfuzz/common/ast/expr/ArrayConstructorExprTest.java b/ast/src/test/java/com/graphicsfuzz/common/ast/expr/ArrayConstructorExprTest.java index c7c312d14..fcc3e7ef6 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/ast/expr/ArrayConstructorExprTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/ast/expr/ArrayConstructorExprTest.java @@ -36,7 +36,7 @@ public void setup() { "vec4", new FloatConstantExpr("0.0")); arrayConstructor = new ArrayConstructorExpr(new ArrayType( BasicType.VEC4, - new ArrayInfo(3)), + new ArrayInfo(new IntConstantExpr("3"))), Arrays.asList(temp.clone(), temp.clone(), temp.clone()) ); } @@ -88,4 +88,4 @@ public void setChildBad() { arrayConstructor.setChild(3, new VariableIdentifierExpr("z")); } -} \ No newline at end of file +} diff --git a/ast/src/test/java/com/graphicsfuzz/common/ast/stmt/ForStmtTest.java b/ast/src/test/java/com/graphicsfuzz/common/ast/stmt/ForStmtTest.java index 3a021ee6b..7ec1adfbf 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/ast/stmt/ForStmtTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/ast/stmt/ForStmtTest.java @@ -17,7 +17,6 @@ package com.graphicsfuzz.common.ast.stmt; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import com.graphicsfuzz.common.ast.CompareAstsDuplicate; @@ -103,4 +102,69 @@ public void testCloneEmptyFor() throws Exception { CompareAstsDuplicate.assertEqualAsts(tu, tu2); } + @Test + public void testEmptyFor() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() { for (;;) ; }"); + final ForStmt stmt = + (ForStmt) ((FunctionDefinition) tu.getTopLevelDeclarations().get(0)).getBody() + .getStmt(0); + assertTrue(stmt.getInit() instanceof NullStmt); + assertFalse(stmt + .hasCondition()); + assertFalse(stmt + .hasIncrement()); + } + + @Test + public void testOnlyHasInit() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() { for (1; ; ) { } }"); + final ForStmt stmt = + (ForStmt) ((FunctionDefinition) tu.getTopLevelDeclarations().get(0)).getBody() + .getStmt(0); + assertFalse(stmt.getInit() instanceof NullStmt); + assertFalse(stmt + .hasCondition()); + assertFalse(stmt + .hasIncrement()); + } + + @Test + public void testOnlyHasCondition() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() { for (; 1; ) { } }"); + final ForStmt stmt = + (ForStmt) ((FunctionDefinition) tu.getTopLevelDeclarations().get(0)).getBody() + .getStmt(0); + assertTrue(stmt.getInit() instanceof NullStmt); + assertTrue(stmt + .hasCondition()); + assertFalse(stmt + .hasIncrement()); + } + + @Test + public void testOnlyHasIncrement() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() { for (; ; 1) { } }"); + final ForStmt stmt = + (ForStmt) ((FunctionDefinition) tu.getTopLevelDeclarations().get(0)).getBody() + .getStmt(0); + assertTrue(stmt.getInit() instanceof NullStmt); + assertFalse(stmt + .hasCondition()); + assertTrue(stmt + .hasIncrement()); + } + + @Test + public void testHasAllFields() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() { for (1; 1; 1) { } }"); + final ForStmt stmt = + (ForStmt) ((FunctionDefinition) tu.getTopLevelDeclarations().get(0)).getBody() + .getStmt(0); + assertFalse(stmt.getInit() instanceof NullStmt); + assertTrue(stmt + .hasCondition()); + assertTrue(stmt + .hasIncrement()); + } + } diff --git a/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java b/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java index dc9701743..1bee1fe00 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/tool/PrettyPrinterVisitorTest.java @@ -18,23 +18,45 @@ import com.graphicsfuzz.common.ast.CompareAstsDuplicate; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.util.Constants; import com.graphicsfuzz.util.ExecHelper; import com.graphicsfuzz.util.ToolHelper; +import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.IOException; +import java.io.PrintStream; import java.nio.charset.StandardCharsets; +import java.util.Optional; import org.apache.commons.io.FileUtils; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class PrettyPrinterVisitorTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test + public void testArraySizeExpression() throws Exception { + final String program = "" + + "void main()\n" + + "{\n" + + " int a[3 + 4];\n" + + "}\n"; + assertEquals(program, PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(program + ))); + } + @Test public void testParseAndPrint() throws Exception { final String program = "" @@ -124,7 +146,7 @@ public void testParseAndPrintComputeShader() throws Exception { final String program = "" + "layout(std430, binding = 2) buffer abuf {\n" + " int data[];\n" - + "} ;\n" + + "};\n" + "layout(local_size_x = 128, local_size_y = 1) in;\n" + "void main()\n" + "{\n" @@ -353,4 +375,92 @@ public void testSamplers() throws Exception { ))); } + /** + * To allow testing of the 'emitShader' method, this parses a shader from the given string, + * invokes 'emitShader' on the resulting parsed shader, and returns the result as a string. + */ + private String getStringViaEmitShader(String shader) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + PrettyPrinterVisitor.emitShader(ParseHelper.parse(shader), Optional.empty(), + new PrintStream(bytes), PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, + PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER); + return new String(bytes.toByteArray(), StandardCharsets.UTF_8); + } + + @Test + public void testNoMacrosUsedSoNoGraphicsFuzzHeader() throws Exception { + assertFalse(getStringViaEmitShader("void main() { }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testNoMacrosUsedSoNoGraphicsFuzzHeader2() throws Exception { + // Even though this uses a macro name, it doesn't use it as a function invocation. + assertFalse(getStringViaEmitShader("void main() { int " + Constants.GLF_FUZZED + "; }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToIdentityMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { " + Constants.GLF_IDENTITY + "(1, 1); }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToZeroMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { " + Constants.GLF_ZERO + "(0); }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToOneMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { " + Constants.GLF_ONE + "(1); }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToFalseMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { " + Constants.GLF_FALSE + "(false); }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToTrueMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { " + Constants.GLF_TRUE + "(true); }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToFuzzedMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { " + Constants.GLF_FUZZED + "(1234); }").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToDeadByConstructionMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { if(" + Constants.GLF_DEAD + "(false)) { } " + + "}").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToLoopWrapperMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { while(" + Constants.GLF_WRAPPED_LOOP + + "(false))" + + " {" + + " } " + + "}").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToIfFalseWrapperMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { if(" + Constants.GLF_WRAPPED_IF_FALSE + "(false)) { } " + + "}").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToIfTrueWrapperMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { if(" + Constants.GLF_WRAPPED_IF_TRUE + + "(true)) { } " + + "}").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + + @Test + public void testGraphicsFuzzMacrosDueToSwitchMacro() throws Exception { + assertTrue(getStringViaEmitShader("void main() { switch(" + Constants.GLF_SWITCH + + "(0)) { case 0: break; default: break; } " + + "}").contains(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)); + } + } diff --git a/ast/src/test/java/com/graphicsfuzz/common/typing/ScopeTreeBuilderTest.java b/ast/src/test/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitorTest.java similarity index 94% rename from ast/src/test/java/com/graphicsfuzz/common/typing/ScopeTreeBuilderTest.java rename to ast/src/test/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitorTest.java index 8e2d57032..6fd5a478f 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/typing/ScopeTreeBuilderTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/typing/ScopeTrackingVisitorTest.java @@ -20,7 +20,7 @@ import com.graphicsfuzz.common.util.ParseHelper; import org.junit.Test; -public class ScopeTreeBuilderTest { +public class ScopeTrackingVisitorTest { @Test public void testSwitch() throws Exception { @@ -35,9 +35,9 @@ public void testSwitch() throws Exception { + " }" + "}"; final TranslationUnit tu = ParseHelper.parse(program); - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { }.visit(tu); } -} \ No newline at end of file +} diff --git a/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java b/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java index 754b57b6d..40147618a 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/typing/TyperTest.java @@ -22,11 +22,10 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.ParameterDecl; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; @@ -38,16 +37,16 @@ import com.graphicsfuzz.common.ast.expr.ParenExpr; import com.graphicsfuzz.common.ast.expr.TernaryExpr; import com.graphicsfuzz.common.ast.expr.TypeConstructorExpr; -import com.graphicsfuzz.common.ast.expr.UIntConstantExpr; import com.graphicsfuzz.common.ast.expr.UnaryExpr; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.QualifiedType; import com.graphicsfuzz.common.ast.type.TypeQualifier; -import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; +import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.util.ExecHelper.RedirectType; import com.graphicsfuzz.util.ExecResult; import com.graphicsfuzz.common.util.OpenGlConstants; @@ -71,7 +70,8 @@ public class TyperTest { @Test public void visitMemberLookupExpr() throws Exception { - String prog = "struct S { float a; float b; };\n" + String prog = "#version 100\n" + + "struct S { float a; float b; };\n" + "struct T { S s; float c; };\n" + "void main() {\n" + " T myT = T(S(1.0, 2.0), 3.0);\n" @@ -81,7 +81,7 @@ public void visitMemberLookupExpr() throws Exception { TranslationUnit tu = ParseHelper.parse(prog); int actualCount = - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_100) { + new NullCheckTyper(tu) { private int count; @@ -104,7 +104,8 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void visitMemberLookupExprAnonymous() throws Exception { - String prog = "struct { float a; float b; } myStruct;\n" + String prog = "#version 100\n" + + "struct { float a; float b; } myStruct;\n" + "void main() {\n" + " myStruct.a = 2.0;\n" + " myStruct.b = 3.0;\n" @@ -114,7 +115,7 @@ public void visitMemberLookupExprAnonymous() throws Exception { TranslationUnit tu = ParseHelper.parse(prog); int actualCount = - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_100) { + new NullCheckTyper(tu) { private int count; @@ -136,14 +137,15 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void testTypeOfScalarConstructors() throws Exception { - String program = "void main() { float(1); int(1); uint(1); bool(1); }"; + String program = "#version 440\n" + + "void main() { float(1); int(1); uint(1); bool(1); }"; for (BasicType b : Arrays.asList(BasicType.FLOAT, BasicType.INT, BasicType.UINT, BasicType.BOOL)) { try { - new NullCheckTyper(ParseHelper.parse(program), ShadingLanguageVersion.GLSL_440) { + new NullCheckTyper(ParseHelper.parse(program)) { @Override public void visitTypeConstructorExpr(TypeConstructorExpr typeConstructorExpr) { @@ -170,14 +172,15 @@ public void visitTypeConstructorExpr(TypeConstructorExpr typeConstructorExpr) { @Test public void testMemberLookupTypeFloat() throws Exception { - final String program = "void main() { vec2 v2 = vec2(1.0);" + final String program = "#version 100\n" + + "void main() { vec2 v2 = vec2(1.0);" + " v2.x; v2.y;" + " vec3 v3 = vec3(1.0);" + " v3.x; v3.y; v3.z;" + " vec4 v4 = vec4(1.0);" + " v4.x; v4.y; v4.z; v4.w; }"; final TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_100) { + new NullCheckTyper(tu) { @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { super.visitMemberLookupExpr(memberLookupExpr); @@ -189,14 +192,15 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void testMemberLookupTypeInt() throws Exception { - final String program = "void main() { ivec2 v2 = ivec2(1);" + final String program = "#version 440\n" + + "void main() { ivec2 v2 = ivec2(1);" + " v2.x; v2.y;" + " ivec3 v3 = ivec3(1);" + " v3.x; v3.y; v3.z;" + " ivec4 v4 = ivec4(1);" + " v4.x; v4.y; v4.z; v4.w; }"; final TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440) { + new NullCheckTyper(tu) { @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { super.visitMemberLookupExpr(memberLookupExpr); @@ -208,14 +212,15 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void testMemberLookupTypeUint() throws Exception { - final String program = "void main() { uvec2 v2 = uvec2(1u);" + final String program = "#version 440\n" + + "void main() { uvec2 v2 = uvec2(1u);" + " v2.x; v2.y;" + " uvec3 v3 = uvec3(1u);" + " v3.x; v3.y; v3.z;" + " uvec4 v4 = uvec4(1u);" + " v4.x; v4.y; v4.z; v4.w; }"; final TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440) { + new NullCheckTyper(tu) { @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { super.visitMemberLookupExpr(memberLookupExpr); @@ -228,14 +233,15 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void testMemberLookupTypeBool() throws Exception { - final String program = "void main() { bvec2 v2 = bvec2(true);" + final String program = "#version 440\n" + + "void main() { bvec2 v2 = bvec2(true);" + " v2.x; v2.y;" + " bvec3 v3 = bvec3(true);" + " v3.x; v3.y; v3.z;" + " bvec4 v4 = bvec4(true);" + " v4.x; v4.y; v4.z; v4.w; }"; final TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440) { + new NullCheckTyper(tu) { @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { super.visitMemberLookupExpr(memberLookupExpr); @@ -244,37 +250,11 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { }; } - - @Test - public void testBooleanVectorType() throws Exception { - final String program = "void main() { vec3(1.0) > vec3(2.0); }"; - TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440) { - @Override - public void visitBinaryExpr(BinaryExpr binaryExpr) { - super.visitBinaryExpr(binaryExpr); - assertEquals(BasicType.BVEC3, lookupType(binaryExpr)); - } - }; - } - - @Test - public void testBooleanVectorType2() throws Exception { - final String program = "void main() { vec3(1.0) > 2.0; }"; - TranslationUnit tu = ParseHelper.parse(program); - new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440) { - @Override - public void visitBinaryExpr(BinaryExpr binaryExpr) { - super.visitBinaryExpr(binaryExpr); - assertEquals(BasicType.BVEC3, lookupType(binaryExpr)); - } - }; - } - + @Test public void testSwizzleTyped() throws Exception { TranslationUnit tu = ParseHelper.parse("void main() { vec2 v; v.xy = v.yx; }"); - Typer typer = new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_100); + Typer typer = new NullCheckTyper(tu); new StandardVisitor() { @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @@ -286,7 +266,8 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { @Test public void testAssignTyped() throws Exception { - TranslationUnit tu = ParseHelper.parse("void main() {" + TranslationUnit tu = ParseHelper.parse("#version 440\n" + + "void main() {" + "int x;" + "x = 2;" + "float f;" @@ -297,7 +278,7 @@ public void testAssignTyped() throws Exception { + "v3 /= v3;" + "ivec2 i2;" + "i2 -= i2; }"); - Typer typer = new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440); + Typer typer = new NullCheckTyper(tu); new StandardVisitor() { @Override public void visitBinaryExpr(BinaryExpr binaryExpr) { @@ -327,11 +308,12 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { @Test public void testCommaTyped() throws Exception { - TranslationUnit tu = ParseHelper.parse("void main() {" + TranslationUnit tu = ParseHelper.parse("#version 440\n" + + "void main() {" + "int x;" + "int y;" + "x, y; }"); - Typer typer = new NullCheckTyper(tu, ShadingLanguageVersion.GLSL_440); + Typer typer = new NullCheckTyper(tu); new StandardVisitor() { @Override public void visitBinaryExpr(BinaryExpr binaryExpr) { @@ -344,8 +326,10 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { @Test public void testGlPositionTyped() throws Exception { - TranslationUnit tu = ParseHelper.parse("void main() { gl_Position = vec4(0.0); }"); - Typer typer = new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_300); + TranslationUnit tu = ParseHelper.parse("#version 300 es\n" + + "void main() { gl_Position = vec4(0.0)" + + "; }"); + Typer typer = new NullCheckTyper(tu); new StandardVisitor() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { @@ -360,8 +344,9 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie @Test public void testGlPointSizeTyped() throws Exception { - TranslationUnit tu = ParseHelper.parse("void main() { gl_PointSize = 1.0; }"); - Typer typer = new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_300); + TranslationUnit tu = ParseHelper.parse("#version 300 es\n" + + "void main() { gl_PointSize = 1.0; }"); + Typer typer = new NullCheckTyper(tu); new StandardVisitor() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { @@ -433,11 +418,11 @@ public void testOctalIntLiteralTyped() throws Exception { + "void main() {" + " int x = 031;" + "}"); - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_300) { + new NullCheckTyper(tu) { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - assertSame(lookupType(scalarInitializer.getExpr()), BasicType.INT); + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + assertSame(lookupType(initializer.getExpr()), BasicType.INT); } }.visit(tu); } @@ -448,11 +433,11 @@ public void testHexIntLiteralTyped() throws Exception { + "void main() {" + " int x = 0xA03B;" + "}"); - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_300) { + new NullCheckTyper(tu) { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - assertSame(lookupType(scalarInitializer.getExpr()), BasicType.INT); + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + assertSame(lookupType(initializer.getExpr()), BasicType.INT); } }.visit(tu); } @@ -463,11 +448,11 @@ public void testOctalUnsignedIntLiteralTyped() throws Exception { + "void main() {" + " int x = 031u;" + "}"); - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_300) { + new NullCheckTyper(tu) { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - assertSame(lookupType(scalarInitializer.getExpr()), BasicType.UINT); + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + assertSame(lookupType(initializer.getExpr()), BasicType.UINT); } }.visit(tu); } @@ -478,11 +463,224 @@ public void testHexUnsignedIntLiteralTyped() throws Exception { + "void main() {" + " int x = 0xA03Bu;" + "}"); - new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_300) { + new NullCheckTyper(tu) { + @Override + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + assertSame(lookupType(initializer.getExpr()), BasicType.UINT); + } + }.visit(tu); + } + + @Test + public void testEqualityAndInequalityVectorsMatrices() throws Exception { + final TranslationUnit tu = ParseHelper.parse("#version 300 es\n" + + "void main() {\n" + + " float v1a, v1b;\n" + + " vec2 v2a, v2b;\n" + + " vec3 v3a, v3b;\n" + + " vec4 v4a, v4b;\n" + + " int i1a, i1b;\n" + + " ivec2 i2a, i2b;\n" + + " ivec3 i3a, i3b;\n" + + " ivec4 i4a, i4b;\n" + + " uint u1a, u1b;\n" + + " uvec2 u2a, u2b;\n" + + " uvec3 u3a, u3b;\n" + + " uvec4 u4a, u4b;\n" + + " bool b1a, b1b;\n" + + " bvec2 b2a, b2b;\n" + + " bvec3 b3a, b3b;\n" + + " bvec4 b4a, b4b;\n" + + " mat2x2 m22a, m22b;\n" + + " mat2x3 m23a, m23b;\n" + + " mat2x4 m24a, m24b;\n" + + " mat3x2 m32a, m32b;\n" + + " mat3x3 m33a, m33b;\n" + + " mat3x4 m34a, m34b;\n" + + " mat4x2 m42a, m42b;\n" + + " mat4x3 m43a, m43b;\n" + + " mat4x2 m44a, m44b;\n" + + " v1a == v1b;\n" + + " v2a == v2b;\n" + + " v3a == v3b;\n" + + " v4a == v4b;\n" + + " v1a != v1b;\n" + + " v2a != v2b;\n" + + " v3a != v3b;\n" + + " v4a != v4b;\n" + + " u1a == u1b;\n" + + " u2a == u2b;\n" + + " u3a == u3b;\n" + + " u4a == u4b;\n" + + " u1a != u1b;\n" + + " u2a != u2b;\n" + + " u3a != u3b;\n" + + " u4a != u4b;\n" + + " i1a == i1b;\n" + + " i2a == i2b;\n" + + " i3a == i3b;\n" + + " i4a == i4b;\n" + + " i1a != i1b;\n" + + " i2a != i2b;\n" + + " i3a != i3b;\n" + + " i4a != i4b;\n" + + " b1a == b1b;\n" + + " b2a == b2b;\n" + + " b3a == b3b;\n" + + " b4a == b4b;\n" + + " b1a != b1b;\n" + + " b2a != b2b;\n" + + " b3a != b3b;\n" + + " b4a != b4b;\n" + + " m22a == m22b;\n" + + " m22a != m22b;\n" + + " m23a == m23b;\n" + + " m23a != m23b;\n" + + " m24a == m24b;\n" + + " m24a != m24b;\n" + + " m32a == m32b;\n" + + " m32a != m32b;\n" + + " m33a == m33b;\n" + + " m33a != m33b;\n" + + " m34a == m34b;\n" + + " m34a != m34b;\n" + + " m42a == m42b;\n" + + " m42a != m42b;\n" + + " m43a == m43b;\n" + + " m43a != m43b;\n" + + " m44a == m44b;\n" + + " m44a != m44b;\n" + + "}\n"); + new NullCheckTyper(tu) { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - assertSame(lookupType(scalarInitializer.getExpr()), BasicType.UINT); + public void visitBinaryExpr(BinaryExpr binaryExpr) { + super.visitBinaryExpr(binaryExpr); + if (binaryExpr.getOp() == BinOp.EQ || binaryExpr.getOp() == BinOp.NE) { + assertSame(lookupType(binaryExpr), BasicType.BOOL); + } + } + }.visit(tu); + } + + @Test + public void testVectorRelational() throws Exception { + final TranslationUnit tu = ParseHelper.parse("#version 300 es\n" + + "void main() {\n" + + " vec2 v2a, v2b;\n" + + " vec3 v3a, v3b;\n" + + " vec4 v4a, v4b;\n" + + " ivec2 i2a, i2b;\n" + + " ivec3 i3a, i3b;\n" + + " ivec4 i4a, i4b;\n" + + " uvec2 u2a, u2b;\n" + + " uvec3 u3a, u3b;\n" + + " uvec4 u4a, u4b;\n" + + " bvec2 b2a, b2b;\n" + + " bvec3 b3a, b3b;\n" + + " bvec4 b4a, b4b;\n" + + "\n" + + " lessThan(v2a, v2b);\n" + + " lessThan(v3a, v3b);\n" + + " lessThan(v4a, v4b);\n" + + " lessThan(i2a, i2b);\n" + + " lessThan(i3a, i3b);\n" + + " lessThan(i4a, i4b);\n" + + " lessThan(u2a, u2b);\n" + + " lessThan(u3a, u3b);\n" + + " lessThan(u4a, u4b);\n" + + "\n" + + " lessThanEqual(v2a, v2b);\n" + + " lessThanEqual(v3a, v3b);\n" + + " lessThanEqual(v4a, v4b);\n" + + " lessThanEqual(i2a, i2b);\n" + + " lessThanEqual(i3a, i3b);\n" + + " lessThanEqual(i4a, i4b);\n" + + " lessThanEqual(u2a, u2b);\n" + + " lessThanEqual(u3a, u3b);\n" + + " lessThanEqual(u4a, u4b);\n" + + "\n" + + " greaterThan(v2a, v2b);\n" + + " greaterThan(v3a, v3b);\n" + + " greaterThan(v4a, v4b);\n" + + " greaterThan(i2a, i2b);\n" + + " greaterThan(i3a, i3b);\n" + + " greaterThan(i4a, i4b);\n" + + " greaterThan(u2a, u2b);\n" + + " greaterThan(u3a, u3b);\n" + + " greaterThan(u4a, u4b);\n" + + "\n" + + " greaterThanEqual(v2a, v2b);\n" + + " greaterThanEqual(v3a, v3b);\n" + + " greaterThanEqual(v4a, v4b);\n" + + " greaterThanEqual(i2a, i2b);\n" + + " greaterThanEqual(i3a, i3b);\n" + + " greaterThanEqual(i4a, i4b);\n" + + " greaterThanEqual(u2a, u2b);\n" + + " greaterThanEqual(u3a, u3b);\n" + + " greaterThanEqual(u4a, u4b);\n" + + "\n" + + " equal(v2a, v2b);\n" + + " equal(v3a, v3b);\n" + + " equal(v4a, v4b);\n" + + " equal(i2a, i2b);\n" + + " equal(i3a, i3b);\n" + + " equal(i4a, i4b);\n" + + " equal(u2a, u2b);\n" + + " equal(u3a, u3b);\n" + + " equal(u4a, u4b);\n" + + " equal(b2a, b2b);\n" + + " equal(b3a, b3b);\n" + + " equal(b4a, b4b);\n" + + "\n" + + " notEqual(v2a, v2b);\n" + + " notEqual(v3a, v3b);\n" + + " notEqual(v4a, v4b);\n" + + " notEqual(i2a, i2b);\n" + + " notEqual(i3a, i3b);\n" + + " notEqual(i4a, i4b);\n" + + " notEqual(u2a, u2b);\n" + + " notEqual(u3a, u3b);\n" + + " notEqual(u4a, u4b);\n" + + " notEqual(b2a, b2b);\n" + + " notEqual(b3a, b3b);\n" + + " notEqual(b4a, b4b);\n" + + "\n" + + " any(b2a);\n" + + " any(b3a);\n" + + " any(b4a);\n" + + "\n" + + " all(b2a);\n" + + " all(b3a);\n" + + " all(b4a);\n" + + "\n" + + " not(b2a);\n" + + " not(b3a);\n" + + " not(b4a);\n" + + "}\n"); + new NullCheckTyper(tu) { + @Override + public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { + super.visitFunctionCallExpr(functionCallExpr); + + // There must be a first argument type and it must be a vec type. + BasicType firstArgType = + (BasicType) lookupType(functionCallExpr.getArg(0)).getWithoutQualifiers(); + assertTrue(firstArgType.isVector()); + if (functionCallExpr.getNumArgs() > 1) { + // If there is a second argument type it must match the first argument type. + assertEquals(2, functionCallExpr.getNumArgs()); + assertEquals(firstArgType, lookupType(functionCallExpr.getArg(1)).getWithoutQualifiers()); + } + if (functionCallExpr.getCallee().equals("any") || functionCallExpr.getCallee().equals( + "all")) { + // 'any' and 'all' return 'bool' in all cases. + assertSame(BasicType.BOOL, lookupType(functionCallExpr)); + } else { + // If the first argument is a vector of length n, the result type must be 'bvecn'. + assertEquals(BasicType.makeVectorType(BasicType.BOOL, firstArgType.getNumElements()), + lookupType(functionCallExpr)); + } } }.visit(tu); } @@ -490,8 +688,9 @@ public void visitScalarInitializer(ScalarInitializer scalarInitializer) { private void checkComputeShaderBuiltin(String builtin, String builtinConstant, BasicType baseType, TypeQualifier qualifier) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { - TranslationUnit tu = ParseHelper.parse("void main() { " + builtin + "; }"); - Typer typer = new NullCheckTyper(tu, ShadingLanguageVersion.ESSL_310); + TranslationUnit tu = ParseHelper.parse("#version 310 es\n" + + "void main() { " + builtin + "; }"); + Typer typer = new NullCheckTyper(tu); new StandardVisitor() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { @@ -507,9 +706,8 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie class NullCheckTyper extends Typer { - public NullCheckTyper(IAstNode node, - ShadingLanguageVersion shadingLanguageVersion) { - super(node, shadingLanguageVersion); + NullCheckTyper(TranslationUnit tu) { + super(tu); } @Override @@ -683,24 +881,31 @@ public void testBuiltinsGlsl450() throws Exception { private void testBuiltins(ShadingLanguageVersion shadingLanguageVersion) throws IOException, InterruptedException { - final File tempFile = temporaryFolder.newFile("shader.frag"); - FileUtils.writeStringToFile( - tempFile, - makeBuiltinsProgram(shadingLanguageVersion).toString(), - StandardCharsets.UTF_8); - final ExecResult result = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, tempFile); - assertEquals(0, result.res); + for (ShaderKind shaderKind : ShaderKind.values()) { + if (shaderKind == ShaderKind.COMPUTE && !shadingLanguageVersion.supportedComputeShaders()) { + // Compute shaders are not supported for older GLSL versions. + continue; + } + final File tempFile = temporaryFolder.newFile("shader." + shaderKind.getFileExtension()); + FileUtils.writeStringToFile( + tempFile, + makeBuiltinsProgram(shadingLanguageVersion, shaderKind).toString(), + StandardCharsets.UTF_8); + final ExecResult result = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, tempFile); + assertEquals(0, result.res); + } } - private StringBuilder makeBuiltinsProgram(ShadingLanguageVersion shadingLanguageVersion) { + private StringBuilder makeBuiltinsProgram(ShadingLanguageVersion shadingLanguageVersion, + ShaderKind shaderKind) { StringBuilder result = new StringBuilder(); result.append("#version " + shadingLanguageVersion.getVersionString() + "\n"); result.append("#ifdef GL_ES\n"); result.append("precision mediump float;\n"); result.append("#endif\n"); int counter = 0; - for (String name : TyperHelper.getBuiltins(shadingLanguageVersion).keySet()) { - for (FunctionPrototype fp : TyperHelper.getBuiltins(shadingLanguageVersion).get(name)) { + for (String name : TyperHelper.getBuiltins(shadingLanguageVersion, shaderKind).keySet()) { + for (FunctionPrototype fp : TyperHelper.getBuiltins(shadingLanguageVersion, shaderKind).get(name)) { counter++; result.append(fp.getReturnType() + " test" + counter + "_" + fp.getName() + "("); boolean first = true; @@ -712,7 +917,11 @@ private StringBuilder makeBuiltinsProgram(ShadingLanguageVersion shadingLanguage result.append(decl.getType() + " " + decl.getName()); } result.append(") {\n"); - result.append(" return " + fp.getName() + "("); + result.append(" "); + if (fp.getReturnType() != VoidType.VOID) { + result.append("return "); + } + result.append(fp.getName() + "("); first = true; for (ParameterDecl decl : fp.getParameters()) { if (!first) { diff --git a/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java b/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java index ad65e1827..24a00207d 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/util/ParseHelperTest.java @@ -21,16 +21,19 @@ import com.graphicsfuzz.common.ast.decl.Declaration; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.FloatConstantExpr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.expr.UIntConstantExpr; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; import com.graphicsfuzz.common.ast.stmt.ReturnStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; +import com.graphicsfuzz.common.ast.visitors.UnsupportedLanguageFeatureException; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import java.io.BufferedWriter; @@ -104,7 +107,7 @@ private void checkTranslationUnit(TranslationUnit tu) { public void testParseCurrentHeaderNoMacros() throws Exception { File tempFile = testFolder.newFile("shader.frag"); PrintStream ps = new PrintStream(new FileOutputStream(tempFile)); - PrettyPrinterVisitor.emitGraphicsFuzzDefines(ps); + PrettyPrinterVisitor.emitGraphicsFuzzDefines(ps, ShadingLanguageVersion.ESSL_310); ps.println(TEST_PROGRAM); ps.close(); TranslationUnit tu = ParseHelper.parse(tempFile); @@ -115,7 +118,7 @@ public void testParseCurrentHeaderNoMacros() throws Exception { public void testParseCurrentHeaderWithMacros() throws Exception { File tempFile = testFolder.newFile("shader.frag"); PrintStream ps = new PrintStream(new FileOutputStream(tempFile)); - PrettyPrinterVisitor.emitGraphicsFuzzDefines(ps); + PrettyPrinterVisitor.emitGraphicsFuzzDefines(ps, ShadingLanguageVersion.ESSL_310); ps.println(TEST_PROGRAM); ps.close(); TranslationUnit tu = ParseHelper.parse(tempFile); @@ -381,10 +384,10 @@ public void testParseOctalIntLiteral() throws Exception { assertTrue( new CheckPredicateVisitor() { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - if (scalarInitializer.getExpr() instanceof IntConstantExpr - && ((IntConstantExpr) scalarInitializer.getExpr()).getValue().equals("031")) { + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + if (initializer.getExpr() instanceof IntConstantExpr + && ((IntConstantExpr) initializer.getExpr()).getValue().equals("031")) { predicateHolds(); } } @@ -401,10 +404,10 @@ public void testParseHexIntLiteral() throws Exception { assertTrue( new CheckPredicateVisitor() { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - if (scalarInitializer.getExpr() instanceof IntConstantExpr - && ((IntConstantExpr) scalarInitializer.getExpr()).getValue().equals("0xa03b")) { + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + if (initializer.getExpr() instanceof IntConstantExpr + && ((IntConstantExpr) initializer.getExpr()).getValue().equals("0xa03b")) { predicateHolds(); } } @@ -421,10 +424,10 @@ public void testParseOctalUnsignedIntLiteral() throws Exception { assertTrue( new CheckPredicateVisitor() { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - if (scalarInitializer.getExpr() instanceof UIntConstantExpr - && ((UIntConstantExpr) scalarInitializer.getExpr()).getValue().equals("031u")) { + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + if (initializer.getExpr() instanceof UIntConstantExpr + && ((UIntConstantExpr) initializer.getExpr()).getValue().equals("031u")) { predicateHolds(); } } @@ -441,10 +444,10 @@ public void testParseHexUnsignedIntLiteral() throws Exception { assertTrue( new CheckPredicateVisitor() { @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - if (scalarInitializer.getExpr() instanceof UIntConstantExpr - && ((UIntConstantExpr) scalarInitializer.getExpr()).getValue().equals("0xA03Bu")) { + public void visitInitializer(Initializer initializer) { + super.visitInitializer(initializer); + if (initializer.getExpr() instanceof UIntConstantExpr + && ((UIntConstantExpr) initializer.getExpr()).getValue().equals("0xA03Bu")) { predicateHolds(); } } @@ -517,8 +520,8 @@ public void testParseInPlacePrecisionAndFloatSuffix() throws Exception { variablesDeclaration.getBaseType(); assertTrue(baseType.hasQualifier(TypeQualifier.MEDIUMP)); assertEquals("1.00f", - ((FloatConstantExpr) ((ScalarInitializer) variablesDeclaration.getDeclInfo(0) - .getInitializer()).getExpr()).getValue()); + ((FloatConstantExpr) variablesDeclaration.getDeclInfo(0) + .getInitializer().getExpr()).getValue()); } @Test @@ -564,4 +567,217 @@ private String getStringFromInputStream(InputStream strippedIs) throws IOExcepti return writer.toString(); } + /** + * Validate folding in testSupportedArrayLength. We know that the array is the last statement + * in the function, but it may not be the first. + * @param shaderSource Source code for the shader to test + * @param expectedSize Expected size of the array + * @throws Exception Producing the AST may throw exceptions + */ + private void + validateFolding(String shaderSource, int expectedSize) throws Exception + { + final BlockStmt block = ParseHelper.parse(shaderSource) + .getMainFunction().getBody(); + assertTrue(((DeclarationStmt)block.getStmt(block.getNumStmts()-1)).getVariablesDeclaration() + .getDeclInfo(0).getArrayInfo().getConstantSize() == expectedSize); + } + + /** + * Test various forms of statements that may occur inside array size declaration + * @throws Exception Producing the AST may throw exceptions + */ + @Test + public void + testSupportedArrayLength() throws Exception { + try { + validateFolding("void main() {\n" + + " int A[3 + 4];\n" + + "}\n",7); + validateFolding("void main() {\n" + + " int A[4 - 3];\n" + + "}\n",1); + validateFolding("void main() {\n" + + " int A[3 * 4];\n" + + "}\n", 12); + validateFolding("void main() {\n" + + " int A[4 / 2];\n" + + "}\n", 2); + validateFolding("void main() {\n" + + " int A[3 << 2];\n" + + "}\n", 12); + validateFolding("void main() {\n" + + " int A[8 >> 2];\n" + + "}\n", 2); + validateFolding("void main() {\n" + + " int A[(3 + 4)];\n" + + "}\n", 7); + validateFolding("void main() {\n" + + " int A[3 && 4];\n" + + "}\n", 1); + validateFolding("void main() {\n" + + " int A[3 & 5];\n" + + "}\n", 1); + validateFolding("void main() {\n" + + " int A[3 || 4];\n" + + "}\n", 1); + validateFolding("void main() {\n" + + " int A[3 | 4];\n" + + "}\n", 7); + validateFolding("void main() {\n" + + " int A[3 ^^ 4];\n" + + "}\n", 0); + validateFolding("void main() {\n" + + " int A[3 ^ 4];\n" + + "}\n", 7); + validateFolding("void main() {\n" + + " int A[3 + 4 + 5];\n" + + "}\n", 12); + validateFolding("void main() {\n" + + " const int v = 5;\n" + + " int A[3 + v];\n" + + "}\n", 8); + validateFolding("void main() {\n" + + " const int v = 5;\n" + + " int A[(((3 + 4) * v) >> 2) && 7];\n" + + "}\n", 1); + } catch (UnsupportedLanguageFeatureException exception) { + fail(exception.getMessage()); + } + } + + @Test + public void testUnsupportedMultiDimensionalArrays() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("void main() {\n" + + " int A[3][4];\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Not yet supporting multi-dimensional arrays")); + } + } + + @Test + public void testUnsupportedArrayInBaseType() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("void main() {\n" + + " int[2] A, B[3];\n" + + " B[2][1] = 3;\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Array information specified at the base type")); + } + } + + @Test + public void testUnsupportedDeclarationInCondition() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("void main() {\n" + + " while(bool b = true) {\n" + + " if(b) {\n" + + " break;\n" + + " }\n" + + " }\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("We do not yet support the case where the " + + "condition of a 'for' or 'while' introduces a new variable")); + } + } + + @Test + public void testUnsupportedInitializerList() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("#version 440\n" + + "\n" + + "void main() {\n" + + "\n" + + " int A[4] = { 1, 2, 3, 4 };\n" + + "\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Initializer lists are not currently supported")); + } + } + + @Test + public void testUnsupportedMethodCall1() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("#version 310 es\n" + + "\n" + + "void main() {\n" + + "\n" + + " int A[4];\n" + + " A.length();\n" + + "\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Method calls are not currently supported")); + } + } + + @Test + public void testUnsupportedMethodCall2() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("#version 310 es\n" + + "\n" + + "void main() {\n" + + "\n" + + " int A[4];\n" + + " A.length(void);\n" + + "\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Method calls are not currently supported")); + } + } + + @Test + public void testUnsupportedMethodCall3() throws Exception { + + // Change this test to check for support if it is eventually introduced. + + try { + ParseHelper.parse("#version 320 es\n" + + "\n" + + "precision highp float;\n" + + "\n" + + "struct S { int A[4]; };\n" + + "\n" + + "void main() {\n" + + "\n" + + " S s = S(int[4](1,2,3,4));\n" + + " s.A.length();\n" + + "\n" + + "}\n"); + fail("Exception was expected"); + } catch (UnsupportedLanguageFeatureException exception) { + assertTrue(exception.getMessage().contains("Method calls are not currently supported")); + } + } + } diff --git a/ast/src/test/java/com/graphicsfuzz/common/util/TruncateLoopsTest.java b/ast/src/test/java/com/graphicsfuzz/common/util/TruncateLoopsTest.java index 17aaa6b17..8b47e072d 100644 --- a/ast/src/test/java/com/graphicsfuzz/common/util/TruncateLoopsTest.java +++ b/ast/src/test/java/com/graphicsfuzz/common/util/TruncateLoopsTest.java @@ -26,60 +26,6 @@ public class TruncateLoopsTest { - private static final String[] CONDITIONS = {"x < -(-20)", "20 > x", "x <= 20", "20 >= x", - "x > -2", "-2 < x","x >= -2", "-2 <= x"}; - private static final String[] INCREMETS = {"x++", "++x", "x += 1", "x += -(-1)", "x += 5", - "x--", "--x", "x -= 1", "x -= 5"}; - private static final String[] INIT_CONSTS = {"x = -1", "x = 0", "int x = 0", "x = -(+(-10))", - "int x = 10"}; - - @Test - public void intConstantsTest() throws Exception { - final String programBody = - "void main() {" - + "int u = 10;" - + "int x;" - + " for($INIT; $COND; $INCREMENT) {" - + " u = u * 2;" - + " }" - + "}"; - - for (int cond_index = 0; cond_index < CONDITIONS.length; ++cond_index) { - for (int incr_index = 0; incr_index < INCREMETS.length; ++incr_index) { - for (int init_index = 0; init_index < INIT_CONSTS.length; ++init_index) { - final boolean isSane = (cond_index < 4 && incr_index < 5) - || (4 <= cond_index && 5 <= incr_index); - testProgram(programBody - .replace("$INIT", INIT_CONSTS[init_index]) - .replace("$COND", CONDITIONS[cond_index]) - .replace("$INCREMENT", INCREMETS[incr_index]), - isSane); - - } - } - } - } - - private void testProgram(String program, boolean isSane) throws IOException, - ParseTimeoutException, InterruptedException, GlslParserException { - TranslationUnit tu = ParseHelper.parse(program); - new TruncateLoops(30, "webGL_", tu, true); - if(isSane) { - CompareAstsDuplicate.assertEqualAsts(program, tu); - } else { - assertProgramsNotEqual(program, tu); - } - tu = ParseHelper.parse(program); - new TruncateLoops(0, "webGL_", tu, true); - assertProgramsNotEqual(program, tu); - } - - private void assertProgramsNotEqual(String program, TranslationUnit otherProgram) - throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { - assert !PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(program)) - .equals(PrettyPrinterVisitor.prettyPrintAsString(otherProgram)); - } - @Test public void testTruncateLoops() throws Exception { final String program = "void main() {" @@ -132,7 +78,7 @@ public void testTruncateLoops() throws Exception { + " }\n" + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); - new TruncateLoops(3, "pre", tu, true); + new TruncateLoops(3, "pre", tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @@ -189,7 +135,7 @@ public void testTruncateLoops2() throws Exception { + " }\n" + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); - new TruncateLoops(3, "pre", tu, false); + new TruncateLoops(3, "pre", tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @@ -210,29 +156,43 @@ public void testTruncateLoops3() throws Exception { final String expected = "" + "void main() {\n" + " int x = 0;\n" - + " for (int i = 0; i < 2; i++) {\n" - + " int pre_looplimiter1 = 0;\n" - + " while (x < i) {\n" - + " if (pre_looplimiter1 >= 3) {\n" + + " {" + + " int pre_looplimiter3 = 0;\n" + + " for (int i = 0; i < 2; i++) {\n" + + " if (pre_looplimiter3 >= 3) {\n" + " break;\n" + " }\n" - + " pre_looplimiter1++;\n" - + " int pre_looplimiter0 = 0;\n" - + " do {\n" - + " if (pre_looplimiter0 >= 3) {\n" + + " pre_looplimiter3++;\n" + + " int pre_looplimiter2 = 0;\n" + + " while (x < i) {\n" + + " if (pre_looplimiter2 >= 3) {\n" + " break;\n" + " }\n" - + " pre_looplimiter0++;\n" - + " x++;\n" - + " for (int j = 0; j < 2; j++) {\n" - + " ;\n" - + " }\n" - + " } while (x > 0);\n" + + " pre_looplimiter2++;\n" + + " int pre_looplimiter1 = 0;\n" + + " do {\n" + + " if (pre_looplimiter1 >= 3) {\n" + + " break;\n" + + " }\n" + + " pre_looplimiter1++;\n" + + " x++;\n" + + " {\n" + + " int pre_looplimiter0 = 0;\n" + + " for (int j = 0; j < 2; j++) {\n" + + " if (pre_looplimiter0 >= 3) {\n" + + " break;\n" + + " }\n" + + " pre_looplimiter0++;\n" + + " ;\n" + + " }\n" + + " }\n" + + " } while (x > 0);\n" + + " }\n" + " }\n" + " }\n" + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); - new TruncateLoops(3, "pre", tu, true); + new TruncateLoops(3, "pre", tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } diff --git a/build/travis/check_headers.py b/build/travis/check_headers.py index d9ec51051..7ba3794e5 100644 --- a/build/travis/check_headers.py +++ b/build/travis/check_headers.py @@ -24,7 +24,9 @@ def exclude_dirname(f: str): - return f in [ + return ( + f.endswith(".egg-info") or + f in [ "target", ".git", ".idea", @@ -38,8 +40,9 @@ def exclude_dirname(f: str): "cmake-build-debug", "cmake-build-release", ".pytest_cache", - - ] + ".venv", + ] + ) def exclude_dirpath(f: str): @@ -110,6 +113,7 @@ def exclude_filename(f: str): f.endswith(".primitives") or \ f.endswith(".jar") or \ f.endswith(".spv") or \ + f.endswith(".dic") or \ f in [ ".editorconfig", ".gitmodules", @@ -128,13 +132,14 @@ def exclude_filename(f: str): "gradlew.bat", "dependency-reduced-pom.xml", "gradle-wrapper.properties", - + "Pipfile.lock", ] def go(): fail = False copyright_pattern = re.compile(r"Copyright (2018|2019) The GraphicsFuzz Project Authors") + generated_pattern = re.compile(r"(g|G)enerated") for (dirpath, dirnames, filenames) in os.walk(os.curdir, topdown=True): @@ -153,13 +158,13 @@ def go(): with io.open(file, "r") as fin: contents = fin.read() - # Generated files don't need a header. - if contents.split("\n")[0].find("generated") > -1: - # This file is OK. Continue to the next file. - continue first_lines = "\n".join(contents.split("\n")[:10]) - # Other files must contain a header for any year within the first few lines. + # OK if the generated pattern is found within the first few lines. + if generated_pattern.search(first_lines): + continue + + # Must contain a header for any year within the first few lines. if copyright_pattern.search(first_lines) is None: fail = True print("Missing license header " + file) diff --git a/build/travis/cmd-tests.sh b/build/travis/cmd-tests.sh index c0345ff5d..1bf943750 100755 --- a/build/travis/cmd-tests.sh +++ b/build/travis/cmd-tests.sh @@ -59,9 +59,9 @@ glsl-generate --seed 0 samples/300es samples/donors 10 family_300es work/shaderf glsl-generate --seed 0 samples/100 samples/donors 10 family_100 work/shaderfamilies >/dev/null glsl-generate --seed 0 --generate-uniform-bindings --max-uniforms 10 samples/310es samples/donors 10 family_vulkan work/shaderfamilies >/dev/null -test -d "work/shaderfamilies/family_100_bubblesort_flag" -test -d "work/shaderfamilies/family_300es_bubblesort_flag" -test -d "work/shaderfamilies/family_vulkan_bubblesort_flag" +test -d "work/shaderfamilies/family_100_stable_bubblesort_flag" +test -d "work/shaderfamilies/family_300es_stable_bubblesort_flag" +test -d "work/shaderfamilies/family_vulkan_stable_bubblesort_flag" ### Reduce examples. @@ -102,7 +102,7 @@ test "${EXIT_CODE}" -eq 143 ### Check some binaries. -SHADER=work/shaderfamilies/family_100_bubblesort_flag/reference.frag +SHADER=work/shaderfamilies/family_100_stable_bubblesort_flag/reference.frag glslangValidator "${SHADER}" shader_translator "${SHADER}" >/dev/null diff --git a/build/travis/install-github-release-tool.sh b/build/travis/install-github-release-tool.sh index 20e42a24d..0ca7513ec 100755 --- a/build/travis/install-github-release-tool.sh +++ b/build/travis/install-github-release-tool.sh @@ -20,8 +20,8 @@ set -u echo "Installing github-release ${GITHUB_RELEASE_TOOL_PLATFORM} (linux_amd64, darwin_amd64, windows_amd64, etc.) tool to $(pwd)." -GITHUB_RELEASE_TOOL_USER="c4milo" -GITHUB_RELEASE_TOOL_VERSION="v1.1.0" +GITHUB_RELEASE_TOOL_USER="paulthomson" +GITHUB_RELEASE_TOOL_VERSION="v1.1.0.1" GITHUB_RELEASE_TOOL_FILE="github-release_${GITHUB_RELEASE_TOOL_VERSION}_${GITHUB_RELEASE_TOOL_PLATFORM}.tar.gz" if test ! -f "${GITHUB_RELEASE_TOOL_FILE}.touch"; then diff --git a/build/travis/install-maven.sh b/build/travis/install-maven.sh index 553f1d0fa..01c137827 100755 --- a/build/travis/install-maven.sh +++ b/build/travis/install-maven.sh @@ -20,7 +20,7 @@ set -u MAVEN_VERSION="3.6.0" MAVEN_FILE="apache-maven-${MAVEN_VERSION}-bin.zip" -MAVEN_URL="https://www-us.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${MAVEN_FILE}" +MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${MAVEN_FILE}" echo "Installing maven ${MAVEN_VERSION} to $(pwd)/apache-maven-${MAVEN_VERSION}" diff --git a/build/travis/licenses.py b/build/travis/licenses.py index c7f6afbf6..01cf8a911 100644 --- a/build/travis/licenses.py +++ b/build/travis/licenses.py @@ -48,7 +48,7 @@ def get_extras(): 'comment': 'Used by angle', 'name': 'libpng', 'url': 'http://libpng.org/pub/png/libpng.html', - 'license_file': 'build/licenses/libpng-LICENSE.txt', + 'license_file': path('third_party', 'licenses', 'libpng-LICENSE.txt'), 'license_url': '', 'skipped': '', }, @@ -58,7 +58,16 @@ def get_extras(): 'name': 'minigbm', 'url': 'https://chromium.googlesource.com/chromiumos/platform/minigbm/+/master', 'license_url': '', - 'license_file': 'build/licenses/minigbm.txt', + 'license_file': path('third_party', 'licenses', 'minigbm.txt'), + 'skipped': '', + }, + + 'rapidjson': { + 'comment': 'Used by angle', + 'name': 'RapidJSON', + 'url': 'https://github.com/TenCent/rapidjson', + 'license_url': 'https://raw.githubusercontent.com/Tencent/rapidjson/6a6bed2759d42891f9e29a37b21315d3192890ed/license.txt', + 'license_file': '', 'skipped': '', }, @@ -107,14 +116,11 @@ def get_extras(): 'skipped': '', }, - 'vulkan-validationLayers': { + 'vulkan-validation-layers': { 'comment': 'Used by angle', 'name': 'Vulkan Validation Layers - Vulkan Ecosystem Components', 'url': 'https://github.com/KhronosGroup/Vulkan-ValidationLayers', - 'license_url': [ - 'http://www.apache.org/licenses/LICENSE-2.0.txt', - 'https://raw.githubusercontent.com/KhronosGroup/Vulkan-ValidationLayers/44b7b80e5e52f2962077346caa4ed175798d16df/COPYRIGHT.txt' - ], + 'license_url': 'https://raw.githubusercontent.com/KhronosGroup/Vulkan-ValidationLayers/b8d149f81be4496ef8df11097e17365268ea832f/LICENSE.txt', 'license_file': '', 'skipped': '', }, @@ -166,29 +172,38 @@ def get_extras(): 'skipped': '', }, - 'llvm': { + 'libbacktrace': { + 'comment': 'Used by swiftshader', + 'name': 'libbacktrace', + 'url': 'https://github.com/ianlancetaylor/libbacktrace', + 'license_url': 'https://raw.githubusercontent.com/ianlancetaylor/libbacktrace/5a99ff7fed66b8ea8f09c9805c138524a7035ece/LICENSE', + 'license_file': '', + 'skipped': '', + }, + + 'llvm-7.0': { 'comment': '', - 'name': 'LLVM', + 'name': 'LLVM 7.0', 'url': 'https://llvm.org/', - 'license_url': 'https://raw.githubusercontent.com/google/swiftshader/720aec1cd6ebf4c4d74326c5faaddd57ee351609/third_party/LLVM/LICENSE.TXT', + 'license_url': 'https://raw.githubusercontent.com/google/swiftshader/ae022faf53b9f648324874063d7147ba7b555417/third_party/llvm-7.0/llvm/LICENSE.TXT', 'license_file': '', 'skipped': '', }, - 'stlport-cpp11-extension': { + 'marl': { 'comment': '', - 'name': 'stlport-cpp11-extension', - 'url': 'https://github.com/google/swiftshader/tree/720aec1cd6ebf4c4d74326c5faaddd57ee351609/third_party/stlport-cpp11-extension', - 'license_url': 'http://www.apache.org/licenses/LICENSE-2.0.txt', + 'name': 'Marl', + 'url': 'https://github.com/google/marl', + 'license_url': 'https://raw.githubusercontent.com/google/swiftshader/ae022faf53b9f648324874063d7147ba7b555417/third_party/marl/LICENSE', 'license_file': '', 'skipped': '', }, - 'subzero': { + 'stlport-cpp11-extension': { 'comment': '', - 'name': 'Subzero from Chromium', - 'url': 'https://github.com/google/swiftshader/tree/720aec1cd6ebf4c4d74326c5faaddd57ee351609/third_party/subzero', - 'license_url': 'https://raw.githubusercontent.com/google/swiftshader/720aec1cd6ebf4c4d74326c5faaddd57ee351609/third_party/subzero/LICENSE.TXT', + 'name': 'stlport-cpp11-extension', + 'url': 'https://github.com/google/swiftshader/tree/ae022faf53b9f648324874063d7147ba7b555417/third_party/stlport-cpp11-extension', + 'license_url': 'http://www.apache.org/licenses/LICENSE-2.0.txt', 'license_file': '', 'skipped': '', }, @@ -199,7 +214,7 @@ def get_extras(): 'name': 'OpenGL headers', 'url': 'https://github.com/KhronosGroup/OpenGL-Registry', 'license_url': '', - 'license_file': 'build/licenses/opengl-headers.txt', + 'license_file': path('third_party', 'licenses', 'opengl-headers.txt'), 'skipped': '', }, @@ -208,7 +223,7 @@ def get_extras(): 'name': 'EGL headers', 'url': 'https://github.com/KhronosGroup/EGL-Registry', 'license_url': '', - 'license_file': 'build/licenses/opengl-headers.txt', + 'license_file': path('third_party', 'licenses', 'opengl-headers.txt'), 'skipped': '', }, @@ -240,7 +255,7 @@ def get_extras(): }, 'lodepng': { - 'comment': '', + 'comment': 'Used by Vulkan worker and Amber', 'name': 'LodePNG', 'url': 'https://github.com/lvandeve/lodepng', 'license_url': 'https://raw.githubusercontent.com/lvandeve/lodepng/941de186edfc68bca5ba1043423d0937b4baf3c6/LICENSE', @@ -269,7 +284,7 @@ def get_extras(): 'https://www.shadertoy.com/view/lsKXDW', ], 'license_url': '', - 'license_file': 'build/licenses/max-sills-shaders.txt', + 'license_file': path('third_party', 'licenses', 'max-sills-shaders.txt'), 'skipped': '', }, @@ -278,7 +293,7 @@ def get_extras(): 'name': 'Valters Mednis\' GLSL shaders', 'url':'https://www.shadertoy.com/view/ltVGWG', 'license_url': '', - 'license_file': 'build/licenses/valters-mednis-shaders.txt', + 'license_file': path('third_party', 'licenses', 'valters-mednis-shaders.txt'), 'skipped': '', }, @@ -307,7 +322,7 @@ def get_extras(): 'name': 'Android support libraries', 'url': 'https://source.android.com/', 'license_url': '', - 'license_file': 'build/licenses/android-support-libraries.txt', + 'license_file': path('third_party', 'licenses', 'android-support-libraries.txt'), 'skipped': '', }, @@ -360,7 +375,7 @@ def get_extras(): 'name': 'zt-process-killer', 'url': 'https://github.com/zeroturnaround/zt-process-killer', 'license_url': '', - 'license_file': 'build/licenses/zt-process-killer.txt', + 'license_file': path('third_party', 'licenses', 'zt-process-killer.txt'), 'skipped': '', }, @@ -385,6 +400,117 @@ def get_extras(): }, + 'amber': { + 'comment': '', + 'name': 'Amber', + 'url': 'https://github.com/google/amber', + 'license_url': 'https://raw.githubusercontent.com/google/amber/ab41eacc3fba9ea3365a57b1712837cb4ed6d2c4/LICENSE', + 'license_file': '', + 'skipped': '', + }, + + + + 'amdvlk': { + 'comment': '', + 'name': 'AMD Open Source Driver for Vulkan (AMDVLK)', + 'url': 'https://github.com/GPUOpen-Drivers/AMDVLK', + 'license_url': 'https://raw.githubusercontent.com/GPUOpen-Drivers/AMDVLK/808b6d5603653e6efa7a0da46b4f519f0c105b27/LICENSE.txt', + 'license_file': '', + 'skipped': '', + }, + + 'cwpack': { + 'comment': '', + 'name': 'CWPack', + 'url': 'https://github.com/clwi/CWPack', + 'license_url': 'https://raw.githubusercontent.com/clwi/CWPack/43583ff9abe6f5e68602eccb24d5f5c3aceac51c/LICENSE', + 'license_file': '', + 'skipped': '', + }, + + 'metrohash': { + 'comment': '', + 'name': 'MetroHash', + 'url': 'https://github.com/jandrewrogers/MetroHash', + 'license_url': 'https://raw.githubusercontent.com/jandrewrogers/MetroHash/690a521d9beb2e1050cc8f273fdabc13b31bf8f6/LICENSE', + 'license_file': '', + 'skipped': '', + }, + + 'llvm': { + 'comment': 'Used by AMDVLK', + 'name': 'LLVM', + 'url': 'https://llvm.org/', + 'license_url': 'https://raw.githubusercontent.com/GPUOpen-Drivers/llvm/21b028dc97ee9b0c6629ff83c2924b3b80c5d6e7/LICENSE.TXT', + 'license_file': '', + 'skipped': '', + }, + + 'llpc': { + 'comment': '', + 'name': 'LLVM-Based Pipeline Compiler (LLPC)', + 'url': 'https://github.com/GPUOpen-Drivers/llpc', + 'license_url': [ + 'https://raw.githubusercontent.com/GPUOpen-Drivers/llpc/8b8fc751408274f1f96064e183956d6fab1d54d1/LICENSE', + 'https://raw.githubusercontent.com/GPUOpen-Drivers/AMDVLK/808b6d5603653e6efa7a0da46b4f519f0c105b27/LICENSE.txt', + ], + 'license_file': '', + 'skipped': '', + }, + + 'pal': { + 'comment': '', + 'name': 'Platform Abstraction Library (PAL)', + 'url': 'https://github.com/GPUOpen-Drivers/pal', + 'license_url': 'https://raw.githubusercontent.com/GPUOpen-Drivers/pal/3e28ea331cf8e7b0e5733e5bd7f7d16582c603c5/LICENSE.txt', + 'license_file': '', + 'skipped': '', + }, + + 'spvgen': { + 'comment': '', + 'name': 'SPVGEN', + 'url': 'https://github.com/GPUOpen-Drivers/spvgen', + 'license_url': [ + 'https://raw.githubusercontent.com/GPUOpen-Drivers/spvgen/2f31d1170e8a12a66168b23235638c4bbc43ecdc/LICENSE', + 'https://raw.githubusercontent.com/GPUOpen-Drivers/AMDVLK/808b6d5603653e6efa7a0da46b4f519f0c105b27/LICENSE.txt', + ], + 'license_file': '', + 'skipped': '', + }, + + 'xgl': { + 'comment': '', + 'name': 'Vulkan API Layer (XGL)', + 'url': 'https://github.com/GPUOpen-Drivers/xgl', + 'license_url': 'https://raw.githubusercontent.com/GPUOpen-Drivers/xgl/3252b6223947f9fc67399e0798b1062983925fce/LICENSE.txt', + 'license_file': '', + 'skipped': '', + }, + + + 'gfauto': { + 'comment': '', + 'name': 'GraphicsFuzz auto (gfauto)', + 'url': 'https://github.com/google/graphicsfuzz/tree/dev/gfauto', + 'license_url': 'https://raw.githubusercontent.com/google/graphicsfuzz/143f46a1298a2a1e012b8b3ab31fc87c2075e82f/LICENSE', + 'license_file': '', + 'skipped': '', + }, + + + # Android stuff: + + 'bionic': { + 'comment': '', + 'name': 'Android bionic libc', + 'url': 'https://android.googlesource.com/platform/bionic/', + 'license_url': 'https://raw.githubusercontent.com/aosp-mirror/platform_bionic/48da33389b086d1a52524feee049d8219dc4a190/libc/NOTICE', + 'license_file': '', + 'skipped': '', + }, + } @@ -414,7 +540,7 @@ def get_maven_dependencies_populated(): 'name': 'The Alphanum Algorithm', 'url': 'http://www.davekoelle.com/alphanum.html', 'license_url': '', - 'license_file': 'third_party/alphanum-comparator/LICENSE', + 'license_file': path('third_party', 'alphanum-comparator', 'LICENSE'), 'skipped': '', }, 'com.graphicsfuzz:assembly-binaries': { @@ -470,7 +596,7 @@ def get_maven_dependencies_populated(): 'name': 'Animated GIF Writer', 'url': 'http://elliot.kroo.net/software/java/GifSequenceWriter/', 'license_url': '', - 'license_file': 'third_party/gif-sequence-writer/LICENSE', + 'license_file': path('third_party', 'gif-sequence-writer', 'LICENSE'), 'skipped': '', }, 'com.graphicsfuzz.thirdparty:jquery-js': { @@ -479,8 +605,8 @@ def get_maven_dependencies_populated(): 'url': 'https://jquery.org/', 'license_url': '', 'license_file': [ - 'third_party/jquery-js/LICENSE.txt', - 'third_party/jquery-js/AUTHORS.txt', + path('third_party', 'jquery-js', 'LICENSE.txt'), + path('third_party', 'jquery-js', 'AUTHORS.txt'), ], 'skipped': '', }, @@ -489,7 +615,7 @@ def get_maven_dependencies_populated(): 'name': 'six', 'url': 'https://pypi.org/project/six/', 'license_url': '', - 'license_file': 'third_party/python-six/LICENSE', + 'license_file': path('third_party', 'python-six', 'LICENSE'), 'skipped': '', }, 'com.graphicsfuzz:python': { @@ -513,7 +639,7 @@ def get_maven_dependencies_populated(): 'name': 'Semantic UI', 'url': 'https://github.com/semantic-org/semantic-ui/', 'license_url': '', - 'license_file': 'third_party/semantic-ui/LICENSE', + 'license_file': path('third_party', 'semantic-ui', 'LICENSE'), 'skipped': '', }, 'com.graphicsfuzz:server-static-public': { @@ -561,7 +687,7 @@ def get_maven_dependencies_populated(): 'name': 'Apache Thrift', 'url': 'https://thrift.apache.org/', 'license_url': '', - 'license_file': ['third_party/thrift-js/NOTICE', 'third_party/thrift-js/LICENSE'], + 'license_file': [path('third_party', 'thrift-js', 'NOTICE'), path('third_party', 'thrift-js', 'LICENSE')], 'skipped': '', }, 'com.graphicsfuzz.thirdparty:thrift-py': { @@ -622,6 +748,43 @@ def get_maven_dependencies_populated(): 'license_file': '', 'skipped': '', }, + 'org.apache.commons:commons-rng-core': { + 'comment': '', + 'name': 'Apache Commons RNG', + 'url': 'https://github.com/apache/commons-rng', + 'license_url': [ + 'https://raw.githubusercontent.com/apache/commons-rng' + '/838f60e09ce458ced96ea05bf09e576c4283136f/NOTICE.txt', + 'https://raw.githubusercontent.com/apache/commons-rng' + '/838f60e09ce458ced96ea05bf09e576c4283136f/LICENSE.txt' + ], + 'license_file': '', + 'skipped': '', + }, + 'org.apache.commons:commons-rng-simple': { + 'comment': '', + 'name': '', + 'url': '', + 'license_url': '', + 'license_file': '', + 'skipped': 'Same project as commons-rng-core.', + }, + 'org.apache.commons:commons-rng-sampling': { + 'comment': '', + 'name': '', + 'url': '', + 'license_url': '', + 'license_file': '', + 'skipped': 'Same project as commons-rng-core.', + }, + 'org.apache.commons:commons-rng-client-api': { + 'comment': '', + 'name': '', + 'url': '', + 'license_url': '', + 'license_file': '', + 'skipped': 'Same project as commons-rng-core.', + }, 'commons-logging:commons-logging': { 'comment': '', 'name': 'Apache Commons Logging', @@ -743,7 +906,7 @@ def get_maven_dependencies_populated(): 'name': 'JavaCPP', 'url': 'https://github.com/bytedeco/javacpp', 'license_url': '', - 'license_file': 'build/licenses/javacpp.txt', + 'license_file': path('third_party', 'licenses', 'javacpp.txt'), 'skipped': '', }, 'org.bytedeco.javacpp-presets:opencv': { @@ -751,7 +914,7 @@ def get_maven_dependencies_populated(): 'name': 'JavaCPP Presets (OpenCV)', 'url': 'https://github.com/bytedeco/javacpp-presets', 'license_url': '', - 'license_file': 'build/licenses/javacpp.txt', + 'license_file': path('third_party', 'licenses', 'javacpp.txt'), 'skipped': '', }, 'org.eclipse.jetty:jetty-http': { @@ -908,6 +1071,8 @@ def go(): print('Missing dependency license information ' + dep) sys.exit(1) + dependencies_populated_list = sorted(dependencies_populated.items(), key=lambda x: x[1]['name'].lower()) + # Write an OPEN_SOURCE_LICENSES.TXT file. with io.open( 'OPEN_SOURCE_LICENSES.TXT', @@ -917,9 +1082,11 @@ def go(): errors='ignore') as fout: fout.write('\n') + fout.write('Open source licenses for the GraphicsFuzz project.\n') + fout.write('https://github.com/google/graphicsfuzz\n\n') fout.write('Summary of projects:\n\n') - for (dep, details) in dependencies_populated.items(): + for (dep, details) in dependencies_populated_list: print('Dependency: ' + dep) if len(details['skipped']) > 0: print('Skipping (' + details['skipped'] + ')') @@ -938,7 +1105,7 @@ def go(): fout.write('\n') fout.write('All projects and licenses:\n') - for (dep, details) in dependencies_populated.items(): + for (dep, details) in dependencies_populated_list: print('Dependency: ' + dep) if len(details['skipped']) > 0: print('Skipping (' + details['skipped'] + ')') diff --git a/build/travis/python-launch.bat b/build/travis/python-launch.bat index 2d02e64a6..eb8d5908e 100644 --- a/build/travis/python-launch.bat +++ b/build/travis/python-launch.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 %* ) ELSE ( - python %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 %* + ) ELSE ( + python %* + ) ) ) diff --git a/build/travis/release.py b/build/travis/release.py index 51d9e0c74..5ca5e5cfa 100644 --- a/build/travis/release.py +++ b/build/travis/release.py @@ -51,7 +51,7 @@ def go(): subprocess.check_call([ "github-release", - "-prerelease", + "-draft", repo_name, tag_name, commit_hash, diff --git a/checkstyle-config/src/main/resources/graphicsfuzz/checkstyle.xml b/checkstyle-config/src/main/resources/graphicsfuzz/checkstyle.xml index 01f65e2d4..7bd1f641d 100644 --- a/checkstyle-config/src/main/resources/graphicsfuzz/checkstyle.xml +++ b/checkstyle-config/src/main/resources/graphicsfuzz/checkstyle.xml @@ -74,8 +74,8 @@ limitations under the License. - - + + diff --git a/client-tests/pom.xml b/client-tests/pom.xml index df0c2baa1..806bbfe04 100755 --- a/client-tests/pom.xml +++ b/client-tests/pom.xml @@ -28,7 +28,7 @@ limitations under the License. 1.0 ../parent-checkstyle/pom.xml - + diff --git a/common/pom.xml b/common/pom.xml index 5f08ed857..dcaf50db6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -55,6 +55,14 @@ limitations under the License. org.apache.commons commons-lang3 + + org.apache.commons + commons-rng-simple + + + org.apache.commons + commons-rng-client-api + com.google.code.gson gson diff --git a/common/src/main/java/com/graphicsfuzz/common/util/AddBraces.java b/common/src/main/java/com/graphicsfuzz/common/util/AddBraces.java index 2b235274a..123c42cdc 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/AddBraces.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/AddBraces.java @@ -98,10 +98,4 @@ public TranslationUnit addBraces(TranslationUnit tu) { } - public static void main(String[] args) throws IOException, ParseTimeoutException, - InterruptedException, GlslParserException { - TranslationUnit tu = ParseHelper.parse(new File(args[0])); - new PrettyPrinterVisitor(System.out).visit(transform(tu)); - } - } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/ApplySubstitution.java b/common/src/main/java/com/graphicsfuzz/common/util/ApplySubstitution.java index 2066d923f..b8e690af7 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/ApplySubstitution.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/ApplySubstitution.java @@ -18,10 +18,10 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import java.util.Map; -public class ApplySubstitution extends ScopeTreeBuilder { +public class ApplySubstitution extends ScopeTrackingVisitor { private Map substitution; @@ -33,7 +33,7 @@ public ApplySubstitution(Map substitution, IAstNode node) { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { String name = variableIdentifierExpr.getName(); - if (currentScope.lookupType(name) == null) { + if (getCurrentScope().lookupType(name) == null) { if (substitution.containsKey(name)) { variableIdentifierExpr.setName(substitution.get(name)); } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/CannedRandom.java b/common/src/main/java/com/graphicsfuzz/common/util/CannedRandom.java index 9d2c8b360..404038925 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/CannedRandom.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/CannedRandom.java @@ -70,6 +70,11 @@ public IRandom spawnChild() { return this; } + @Override + public String getDescription() { + return "CannedRandom"; + } + public boolean isExhausted() { return !items.hasNext(); } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/IRandom.java b/common/src/main/java/com/graphicsfuzz/common/util/IRandom.java index 0ad65814b..e3f196f71 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/IRandom.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/IRandom.java @@ -40,4 +40,10 @@ default int nextPositiveInt(int bound) { */ IRandom spawnChild(); + + /** + * @return A description of the random number generator, ideally including the seed. + */ + String getDescription(); + } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/Obfuscator.java b/common/src/main/java/com/graphicsfuzz/common/util/Obfuscator.java index 46fadb7cf..2b7c0c16e 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/Obfuscator.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/Obfuscator.java @@ -33,7 +33,7 @@ import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.typing.Typer; import com.graphicsfuzz.util.ExecHelper; import com.graphicsfuzz.util.ExecHelper.RedirectType; @@ -84,7 +84,7 @@ private ShaderJob obfuscate( clonedTus); } - private class TranslationUnitObfuscator extends ScopeTreeBuilder { + private class TranslationUnitObfuscator extends ScopeTrackingVisitor { private final Map functionRenaming; private final Map namedStructRenaming; private final List structDefinitionTypes; @@ -107,7 +107,7 @@ private TranslationUnitObfuscator() { private void obfuscateTranslationUnit(TranslationUnit tu) { - this.typer = new Typer(tu, tu.getShadingLanguageVersion()); + this.typer = new Typer(tu); visit(tu); for (VariableDeclInfo declInfo : varDeclMapping.keySet()) { assert varDeclMapping.containsKey(declInfo); @@ -241,7 +241,7 @@ private String applyFunctionNameMapping(String name) { } private String applyVariableNameMapping(String name) { - ScopeEntry scopeEntry = currentScope.lookupScopeEntry(name); + ScopeEntry scopeEntry = getCurrentScope().lookupScopeEntry(name); if (scopeEntry == null) { return name; } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/PipelineInfo.java b/common/src/main/java/com/graphicsfuzz/common/util/PipelineInfo.java index cc7c2cf39..b334f097a 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/PipelineInfo.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/PipelineInfo.java @@ -33,6 +33,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -52,9 +53,11 @@ public PipelineInfo() { this(new JsonObject()); } - public PipelineInfo(File file) throws FileNotFoundException { - dictionary = new Gson().fromJson(new FileReader(file), + public PipelineInfo(File file) throws IOException { + try (FileReader fr = new FileReader(file)) { + dictionary = new Gson().fromJson(fr, JsonObject.class); + } } public PipelineInfo(String string) { @@ -140,7 +143,7 @@ public void setUniforms(TranslationUnit tu, } int arrayLength; if (vdi.hasArrayInfo()) { - arrayLength = vdi.getArrayInfo().getSize(); + arrayLength = vdi.getArrayInfo().getConstantSize(); } else { arrayLength = 1; } @@ -164,7 +167,7 @@ public void setUniforms(TranslationUnit tu, private Optional maybeGetArrayCount(VariableDeclInfo vdi) { if (vdi.hasArrayInfo()) { - return Optional.of(vdi.getArrayInfo().getSize()); + return Optional.of(vdi.getArrayInfo().getConstantSize()); } return Optional.empty(); } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/PruneUniforms.java b/common/src/main/java/com/graphicsfuzz/common/util/PruneUniforms.java index a78cd74ac..7e243a63f 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/PruneUniforms.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/PruneUniforms.java @@ -20,7 +20,6 @@ import com.graphicsfuzz.common.ast.decl.ArrayInfo; import com.graphicsfuzz.common.ast.decl.Declaration; import com.graphicsfuzz.common.ast.decl.Initializer; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayConstructorExpr; @@ -124,18 +123,18 @@ private static Initializer makeInitializer(BasicType baseType, ArrayInfo arrayInfo, List args) { if (arrayInfo != null) { - assert arrayInfo.getSize() * baseType.getNumElements() == args.size(); + assert arrayInfo.getConstantSize() * baseType.getNumElements() == args.size(); List argExprs = new ArrayList<>(); - for (int index = 0; index < arrayInfo.getSize(); index++) { + for (int index = 0; index < arrayInfo.getConstantSize(); index++) { argExprs.add(getBasicTypeLiteralExpr(baseType, args.subList(index * baseType.getNumElements(), (index + 1) * baseType.getNumElements()))); } - return new ScalarInitializer(new ArrayConstructorExpr( + return new Initializer(new ArrayConstructorExpr( new ArrayType(baseType.getWithoutQualifiers(), arrayInfo.clone()), argExprs)); } - return new ScalarInitializer(getBasicTypeLiteralExpr(baseType, args)); + return new Initializer(getBasicTypeLiteralExpr(baseType, args)); } public static Expr getBasicTypeLiteralExpr(BasicType baseType, List args) { diff --git a/common/src/main/java/com/graphicsfuzz/common/util/RandomWrapper.java b/common/src/main/java/com/graphicsfuzz/common/util/RandomWrapper.java index 4d539c15d..95731a4bd 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/RandomWrapper.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/RandomWrapper.java @@ -16,46 +16,46 @@ package com.graphicsfuzz.common.util; -import java.util.Random; +import org.apache.commons.rng.UniformRandomProvider; +import org.apache.commons.rng.simple.RandomSource; /** - * Random generator that uses java.util.Random, to be used when genuine pseudo-random generation - * is required (as opposed to mocking for testing). + * Random generator to be used when genuine pseudo-random generation is required (as opposed to + * mocking for testing). */ public class RandomWrapper implements IRandom { - private final Random generator; + private final long seed; + private final UniformRandomProvider provider; - public RandomWrapper(int seed) { - this.generator = new Random(seed); - } - - public RandomWrapper() { - this.generator = new Random(); + public RandomWrapper(long seed) { + this.seed = seed; + this.provider = RandomSource.create(RandomSource.ISAAC, seed); } @Override public int nextInt(int bound) { - return generator.nextInt(bound); + return provider.nextInt(bound); } @Override public Float nextFloat() { - return generator.nextFloat(); + return provider.nextFloat(); } @Override public boolean nextBoolean() { - return generator.nextBoolean(); + return provider.nextBoolean(); } @Override public IRandom spawnChild() { - return new RandomWrapper(generator.nextInt()); + return new RandomWrapper(provider.nextLong()); } - public void setSeed(int seed) { - generator.setSeed(seed); + @Override + public String getDescription() { + return "RandomWrapper with seed: " + Long.toUnsignedString(seed); } } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/SameValueRandom.java b/common/src/main/java/com/graphicsfuzz/common/util/SameValueRandom.java index 608e6b98f..e6407914e 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/SameValueRandom.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/SameValueRandom.java @@ -46,4 +46,9 @@ public boolean nextBoolean() { public IRandom spawnChild() { throw new UnsupportedOperationException("Child spawning not available"); } + + @Override + public String getDescription() { + return "SameValueRandom: " + intValue + " " + boolValue; + } } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java index 2ec304b17..d20df620e 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/ShaderJobFileOperations.java @@ -37,6 +37,7 @@ import com.graphicsfuzz.util.ExecHelper; import com.graphicsfuzz.util.ExecResult; import com.graphicsfuzz.util.ToolHelper; +import com.graphicsfuzz.util.ToolPaths; import java.awt.image.BufferedImage; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -50,14 +51,12 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.NoSuchFileException; +import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import java.util.stream.Stream; import javax.imageio.ImageIO; import javax.imageio.stream.FileImageOutputStream; @@ -310,29 +309,6 @@ public boolean doesShaderJobResultFileExist(File shaderJobResultFile) { return shaderJobResultFile.isFile(); } - /** - * Determines whether the underlying shader files for the shader jobs use GraphicsFuzz defines. - * Assumes that if one shader does, they all do. - */ - public boolean doesShaderJobUseGraphicsFuzzDefines(File shaderJobFile) throws IOException { - for (ShaderKind shaderKind : ShaderKind.values()) { - //noinspection deprecation: fine inside this class. - final File shaderFile = getUnderlyingShaderFile(shaderJobFile, shaderKind); - if (!shaderFile.isFile()) { - continue; - } - try (BufferedReader br = new BufferedReader(new FileReader(shaderFile))) { - String line; - while ((line = br.readLine()) != null) { - if (line.trim().startsWith(ParseHelper.END_OF_GRAPHICSFUZZ_DEFINES)) { - return true; - } - } - } - } - return false; - } - /** * Does this shaderJobResultFile have an associated image result? * @@ -703,8 +679,7 @@ public void writeByteArrayToFile(File file, byte[] contents) throws IOException public void writeShaderJobFile( final ShaderJob shaderJob, - final File outputShaderJobFile, - final boolean emitGraphicsFuzzDefines) throws FileNotFoundException { + final File outputShaderJobFile) throws FileNotFoundException { assertIsShaderJobFile(outputShaderJobFile); @@ -714,8 +689,7 @@ public void writeShaderJobFile( writeShader( tu, shaderJob.getLicense(), - new File(outputFileNoExtension + "." + tu.getShaderKind().getFileExtension()), - emitGraphicsFuzzDefines + new File(outputFileNoExtension + "." + tu.getShaderKind().getFileExtension()) ); } @@ -726,12 +700,6 @@ public void writeShaderJobFile( shaderJob.getPipelineInfo().toString()); } - public void writeShaderJobFile( - final ShaderJob shaderJob, - final File outputShaderJobFile) throws FileNotFoundException { - writeShaderJobFile(shaderJob, outputShaderJobFile, true); - } - public void writeShaderJobFileFromImageJob( final ImageJob imageJob, final File outputShaderJobFile) throws IOException { @@ -786,7 +754,7 @@ public void writeShaderJobFileFromImageJob( public void writeShaderResultToFile( ImageJobResult shaderResult, File shaderResultFile, - Optional referenceShaderResultFile) throws IOException { + Optional referenceShaderResultFile) throws InterruptedException, IOException { writeShaderResultToFileHelper( shaderResult, @@ -874,8 +842,7 @@ private static PrintStream ps(File file) throws FileNotFoundException { private static void writeShader( TranslationUnit tu, Optional license, - File outputFile, - boolean emitGraphicsFuzzDefines + File outputFile ) throws FileNotFoundException { try (PrintStream stream = ps(outputFile)) { PrettyPrinterVisitor.emitShader( @@ -883,8 +850,7 @@ private static void writeShader( license, stream, PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, - emitGraphicsFuzzDefines + PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER ); } } @@ -1011,7 +977,13 @@ private String getMD5(File shaderJobFile) throws IOException { byte[] computeData = isFile(computeShaderFile) ? readFileToByteArray(computeShaderFile) : new byte[0]; - byte[] combinedData = new byte[vertexData.length + fragmentData.length + computeData.length]; + // This metadata is required in order to distinguish between shader jobs + // with identical shaders but different pipeline information. + byte[] metaData = isFile(shaderJobFile) + ? readFileToByteArray(shaderJobFile) + : new byte[0]; + byte[] combinedData = + new byte[vertexData.length + fragmentData.length + computeData.length + metaData.length ]; System.arraycopy( vertexData, 0, @@ -1030,6 +1002,12 @@ private String getMD5(File shaderJobFile) throws IOException { combinedData, vertexData.length + fragmentData.length, computeData.length); + System.arraycopy( + metaData, + 0, + combinedData, + vertexData.length + fragmentData.length + computeData.length, + metaData.length); return DigestUtils.md5Hex(combinedData); } @@ -1123,22 +1101,12 @@ private boolean shaderIsValidShaderTranslator( shaderFile, ShaderTranslatorShadingLanguageVersionSupport .getShaderTranslatorArgument(shadingLanguageVersion)); - if (isMemoryExhaustedError(shaderTranslatorResult)) { - return true; - } return checkValidationResult( shaderTranslatorResult, shaderFile.getName(), throwExceptionOnValidationError); } - // TODO(171): This is a workaround for an issue where shader_translator reports memory exhaustion. - // If the issue in shader_translator can be fixed, we should get rid of this check. - private boolean isMemoryExhaustedError(ExecResult shaderTranslatorResult) { - return shaderTranslatorResult.res != 0 - && shaderTranslatorResult.stdout.toString().contains("memory exhausted"); - } - private boolean checkValidationResult(ExecResult res, String filename, boolean throwExceptionOnValidationError) { if (res.res != 0) { @@ -1163,17 +1131,30 @@ protected static void writeShaderResultToFileHelper( ImageJobResult shaderResult, File shaderJobResultFile, ShaderJobFileOperations fileOps, - Optional referenceShaderResultFile) throws IOException { + Optional referenceShaderResultFile) throws InterruptedException, IOException { assertIsShaderJobResultFile(shaderJobResultFile); String shaderJobResultNoExtension = FileHelper.removeEnd(shaderJobResultFile.toString(), ".info.json"); - // Special case: compute shader job. + // Write the log component of the result to a text file, for easy viewing. + if (shaderResult.isSetLog()) { + fileOps.writeStringToFile( + new File(shaderJobResultNoExtension + ".txt"), + shaderResult.getLog()); + } + // Special case: compute shader job. if (shaderResult.isSetComputeOutputs()) { + // In addition to a status and log, results for a compute shader job have: + // - an "outputs" property, which maps to a dictionary representing the results + // that were obtained by running the shader; + // - (if reference result is present) a "comparison_with_reference" property + // describing whether or not the computed results exactly or nearly match those + // for the reference. + JsonObject infoJson = new JsonObject(); if (shaderResult.isSetStatus()) { infoJson.addProperty("status", shaderResult.getStatus().toString()); @@ -1186,8 +1167,64 @@ protected static void writeShaderResultToFileHelper( "outputs", new Gson().fromJson(shaderResult.getComputeOutputs(), JsonObject.class)); } + // We write out the .info.json file now, so that the Python tooling for diffing compute + // shader results can be invoked on it if needed. + fileOps.writeStringToFile( + shaderJobResultFile, + infoJson.toString()); + + if (referenceShaderResultFile.isPresent()) { + + // We have reference results, so can populate the "comparison_with_reference" property. + + // This maps to a dictionary with up to 4 keys: + // - "exact_match", true if and only if the results are identical + // - "exactdiff_output", populated only if "exact_match" is false, with the result of + // exact diffing + // - "fuzzy_match", present only if "exact_match" is false, and then true if and only if + // the results are similar + // - "fuzzydiff_output", present only if "fuzzy_diff" is set, with the result of + // fuzzy diffing. + + final JsonObject computeShaderComparisonWithReference = new JsonObject(); + + // Check whether the results exactly match those of the reference. + + final ExecResult exactDiffResult = + fileOps.runPythonDriver(ExecHelper.RedirectType.TO_BUFFER, + null, + "inspect-compute-results", + "exactdiff", + referenceShaderResultFile.get().getAbsolutePath(), + shaderJobResultFile.getAbsolutePath()); + computeShaderComparisonWithReference.addProperty("exact_match", + exactDiffResult.res == 0); + + if (exactDiffResult.res != 0) { + + // In the case that we do not have an exact match, store the output obtained by exact + // diffing (as it may be useful to inspect). + computeShaderComparisonWithReference.addProperty("exactdiff_output", + exactDiffResult.stderr.toString()); + + // Now perform a fuzzy diff. + final ExecResult fuzzyDiffResult = + fileOps.runPythonDriver(ExecHelper.RedirectType.TO_BUFFER, + null, + "inspect-compute-results", + "fuzzydiff", + referenceShaderResultFile.get().getAbsolutePath(), + shaderJobResultFile.getAbsolutePath()); + computeShaderComparisonWithReference.addProperty("fuzzy_match", + fuzzyDiffResult.res == 0); + computeShaderComparisonWithReference.addProperty("fuzzydiff_output", + fuzzyDiffResult.stderr.toString()); + } + infoJson.add("comparison_with_reference", computeShaderComparisonWithReference); + } + fileOps.writeStringToFile( - new File(shaderJobResultNoExtension + ".info.json"), + shaderJobResultFile, infoJson.toString()); return; @@ -1195,12 +1232,6 @@ protected static void writeShaderResultToFileHelper( final File outputImage = new File(shaderJobResultNoExtension + ".png"); - if (shaderResult.isSetLog()) { - fileOps.writeStringToFile( - new File(shaderJobResultNoExtension + ".txt"), - shaderResult.getLog()); - } - if (shaderResult.isSetPNG()) { fileOps.writeByteArrayToFile(outputImage, shaderResult.getPNG()); } @@ -1240,7 +1271,6 @@ protected static void writeShaderResultToFileHelper( gifOutput.close(); } catch (Exception err) { LOGGER.error("Error while creating GIF for nondet"); - err.printStackTrace(); } } @@ -1265,6 +1295,28 @@ protected static void writeShaderResultToFileHelper( JsonHelper.jsonToString(infoObject)); } + /** + * Runs a GraphicsFuzz Python driver script, from the python/drivers directory. + * @param redirectType Determines where output is redirected to. + * @param directory Working directory; set to null if current directory is fine. + * @param driverName Name of the Python driver, with no extension. + * @param driverArgs Arguments to be passed to the Python driver. + * @return the result of executing the Python driver. + * @throws IOException if something IO-related goes wrong. + * @throws InterruptedException if something goes wrong running the driver command. + */ + public ExecResult runPythonDriver(ExecHelper.RedirectType redirectType, File directory, + String driverName, String... driverArgs) throws IOException, + InterruptedException { + final String[] execArgs = new String[driverArgs.length + 1]; + execArgs[0] = Paths.get(ToolPaths.getPythonDriversDir(), driverName).toString() + + (System.getProperty("os.name").startsWith("Windows") ? ".bat" : ""); + System.arraycopy(driverArgs, 0, execArgs, 1, driverArgs.length); + return new ExecHelper().exec(redirectType, + directory, false, + execArgs); + } + /** * Stores data about an image; right now its file and histogram. * Could be extended in due course with e.g. PSNR diff --git a/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java b/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java index 9cac37cba..f8220bd9c 100755 --- a/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/SideEffectChecker.java @@ -17,6 +17,8 @@ package com.graphicsfuzz.common.util; import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.ParameterDecl; import com.graphicsfuzz.common.ast.expr.BinaryExpr; import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; @@ -28,6 +30,7 @@ import com.graphicsfuzz.common.ast.stmt.ExprCaseLabel; import com.graphicsfuzz.common.ast.stmt.ReturnStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.typing.TyperHelper; @@ -35,14 +38,30 @@ public class SideEffectChecker { private static boolean isSideEffectFreeVisitor(IAstNode node, - ShadingLanguageVersion shadingLanguageVersion) { + ShadingLanguageVersion shadingLanguageVersion, ShaderKind shaderKind) { return !new CheckPredicateVisitor() { @Override public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { - if (!TyperHelper.getBuiltins(shadingLanguageVersion) + if (TyperHelper.getBuiltins(shadingLanguageVersion, shaderKind) .containsKey(functionCallExpr.getCallee())) { - // Assume that any call to a non-builtin might have a side-effect. + for (FunctionPrototype p : + TyperHelper.getBuiltins(shadingLanguageVersion, shaderKind) + .get(functionCallExpr.getCallee())) { + // We check each argument of the built-in's prototypes to see if they require lvalues - + // if so, they can cause side effects. + // We could be more precise here by finding the specific overload of the function rather + // than checking every possible prototype for lvalue parameters. + for (ParameterDecl param : p.getParameters()) { + if (param.getType().hasQualifier(TypeQualifier.OUT_PARAM) + || param.getType().hasQualifier(TypeQualifier.INOUT_PARAM)) { + predicateHolds(); + } + } + } + } else if (!MacroNames.isGraphicsFuzzMacro(functionCallExpr)) { + // Assume that any call to a function that is not a GraphicsFuzz macro or builtin + // might have a side-effect. predicateHolds(); } super.visitFunctionCallExpr(functionCallExpr); @@ -97,12 +116,14 @@ public void visitDefaultCaseLabel(DefaultCaseLabel defaultCaseLabel) { }.test(node); } - public static boolean isSideEffectFree(Stmt stmt, ShadingLanguageVersion shadingLanguageVersion) { - return isSideEffectFreeVisitor(stmt, shadingLanguageVersion); + public static boolean isSideEffectFree(Stmt stmt, ShadingLanguageVersion shadingLanguageVersion, + ShaderKind shaderKind) { + return isSideEffectFreeVisitor(stmt, shadingLanguageVersion, shaderKind); } - public static boolean isSideEffectFree(Expr expr, ShadingLanguageVersion shadingLanguageVersion) { - return isSideEffectFreeVisitor(expr, shadingLanguageVersion); + public static boolean isSideEffectFree(Expr expr, ShadingLanguageVersion shadingLanguageVersion, + ShaderKind shaderKind) { + return isSideEffectFreeVisitor(expr, shadingLanguageVersion, shaderKind); } } diff --git a/common/src/main/java/com/graphicsfuzz/common/util/StripUnusedGlobals.java b/common/src/main/java/com/graphicsfuzz/common/util/StripUnusedGlobals.java index bbfce25f3..e1f0618a6 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/StripUnusedGlobals.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/StripUnusedGlobals.java @@ -26,14 +26,14 @@ import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.util.Constants; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; -public class StripUnusedGlobals extends ScopeTreeBuilder { +public class StripUnusedGlobals extends ScopeTrackingVisitor { public static void strip(TranslationUnit tu) { new StripUnusedGlobals(tu); @@ -65,7 +65,7 @@ public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) @Override public void visitStructNameType(StructNameType structNameType) { super.visitStructNameType(structNameType); - unusedStructs.remove(currentScope.lookupStructName(structNameType.getName())); + unusedStructs.remove(getCurrentScope().lookupStructName(structNameType.getName())); } @Override @@ -74,7 +74,7 @@ public void visitTypeConstructorExpr(TypeConstructorExpr typeConstructorExpr) { // If a struct name is used in a type constructor, the struct counts as being used. // The type constructor here might not be a struct type, e.g. it could be "float" or "vec2". // That's OK: lookupStructName will just return null so nothing will be removed. - unusedStructs.remove(currentScope.lookupStructName(typeConstructorExpr.getTypename())); + unusedStructs.remove(getCurrentScope().lookupStructName(typeConstructorExpr.getTypename())); } @Override @@ -89,7 +89,8 @@ public void visitVariableDeclInfo(VariableDeclInfo variableDeclInfo) { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - final ScopeEntry scopeEntry = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + final ScopeEntry scopeEntry = getCurrentScope().lookupScopeEntry(variableIdentifierExpr + .getName()); if (scopeEntry != null && scopeEntry.hasVariableDeclInfo()) { // If this is a global, mark it as used. unusedGlobals.remove(scopeEntry.getVariableDeclInfo()); diff --git a/common/src/main/java/com/graphicsfuzz/common/util/ZeroCannedRandom.java b/common/src/main/java/com/graphicsfuzz/common/util/ZeroCannedRandom.java index af7adbe41..112a1af1a 100644 --- a/common/src/main/java/com/graphicsfuzz/common/util/ZeroCannedRandom.java +++ b/common/src/main/java/com/graphicsfuzz/common/util/ZeroCannedRandom.java @@ -38,4 +38,9 @@ public IRandom spawnChild() { throw new UnsupportedOperationException(); } + @Override + public String getDescription() { + return "ZeroCannedRandom"; + } + } diff --git a/common/src/test/java/com/graphicsfuzz/common/util/AddBracesTest.java b/common/src/test/java/com/graphicsfuzz/common/util/AddBracesTest.java index 602d8b056..12f9b3b1d 100644 --- a/common/src/test/java/com/graphicsfuzz/common/util/AddBracesTest.java +++ b/common/src/test/java/com/graphicsfuzz/common/util/AddBracesTest.java @@ -61,17 +61,6 @@ public void loops() throws Exception { @Rule public TemporaryFolder testFolder = new TemporaryFolder(); - @Test - public void testMain() throws Exception { - File input = testFolder.newFile("input.frag"); - String program = "void main() { for (a; b; c) while (d) do e; while (f); }"; - BufferedWriter bw = new BufferedWriter(new FileWriter(input)); - bw.write(program); - bw.flush(); - bw.close(); - AddBraces.main(new String[] { input.getAbsolutePath() }); - } - @Test public void testIsUtilityClass() throws Exception { CheckUtilityClass.assertUtilityClassWellDefined(AddBraces.class); diff --git a/common/src/test/java/com/graphicsfuzz/common/util/PruneUniformsTest.java b/common/src/test/java/com/graphicsfuzz/common/util/PruneUniformsTest.java index 86800e805..422313110 100755 --- a/common/src/test/java/com/graphicsfuzz/common/util/PruneUniformsTest.java +++ b/common/src/test/java/com/graphicsfuzz/common/util/PruneUniformsTest.java @@ -16,16 +16,10 @@ package com.graphicsfuzz.common.util; -import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.common.transformreduce.GlslShaderJob; -import com.graphicsfuzz.util.ExecHelper.RedirectType; -import com.graphicsfuzz.util.ExecResult; -import com.graphicsfuzz.util.ToolHelper; +import com.graphicsfuzz.common.transformreduce.ShaderJob; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; @@ -183,39 +177,27 @@ private void doPruneTest(String program, String uniforms, String expectedProgram String expectedUniforms, List prefixList, int limit) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { - final File uniformsFile = temporaryFolder.newFile("uniforms.json"); - FileUtils.writeStringToFile(uniformsFile, uniforms, StandardCharsets.UTF_8); - final PipelineInfo pipelineInfo = new PipelineInfo(uniformsFile); - final TranslationUnit tu = ParseHelper.parse(program); + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), new PipelineInfo(uniforms), + ParseHelper.parse(program)); assertTrue(PruneUniforms.prune( - new GlslShaderJob(Optional.empty(), pipelineInfo, tu), + shaderJob, limit, prefixList)); - final File shaderFile = temporaryFolder.newFile("shader.frag"); - - try (PrintStream stream = new PrintStream(new FileOutputStream(shaderFile))) { - PrettyPrinterVisitor.emitShader( - tu, - Optional.empty(), - stream, - PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, - true - ); - } - final ExecResult execResult = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, shaderFile); - assertEquals(0, execResult.res); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + + fileOps.writeShaderJobFile(shaderJob, shaderJobFile); + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); final File expectedUniformsFile = temporaryFolder.newFile("expecteduniforms.json"); FileUtils.writeStringToFile(expectedUniformsFile, expectedUniforms, StandardCharsets.UTF_8); - assertEquals(PrettyPrinterVisitor.prettyPrintAsString( - ParseHelper.parse(expectedProgram)), - PrettyPrinterVisitor.prettyPrintAsString(tu)); - assertEquals(new PipelineInfo(expectedUniformsFile).toString(), - pipelineInfo.toString()); + CompareAsts.assertEqualAsts(expectedProgram, shaderJob.getFragmentShader().get()); + assertEquals(new PipelineInfo(expectedUniforms).toString(), + shaderJob.getPipelineInfo().toString()); } } diff --git a/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java b/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java index 00b5fe45e..17e894abe 100644 --- a/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java +++ b/common/src/test/java/com/graphicsfuzz/common/util/SideEffectCheckerTest.java @@ -16,8 +16,10 @@ package com.graphicsfuzz.common.util; +import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.expr.UnOp; import com.graphicsfuzz.common.ast.expr.UnaryExpr; @@ -25,14 +27,22 @@ import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.ExprStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; +import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import java.util.Collections; +import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import static org.junit.Assert.*; public class SideEffectCheckerTest { + @Rule + public TemporaryFolder testFolder = new TemporaryFolder(); + @Test public void testAssignmentHasSideEffects() throws Exception { assertFalse(SideEffectChecker.isSideEffectFree(new ExprStmt( @@ -40,7 +50,7 @@ public void testAssignmentHasSideEffects() throws Exception { new VariableIdentifierExpr("v"), new IntConstantExpr("12"), BinOp.ASSIGN - )), ShadingLanguageVersion.ESSL_310)); + )), ShadingLanguageVersion.ESSL_310, ShaderKind.FRAGMENT)); } @Test @@ -59,7 +69,81 @@ public void testCountingLoopHasSideEffects() throws Exception { new VariableIdentifierExpr("i"), UnOp.POST_INC ), new BlockStmt(Collections.emptyList(), false)), - ShadingLanguageVersion.ESSL_310)); + ShadingLanguageVersion.ESSL_310, ShaderKind.FRAGMENT)); + } + + @Test + public void testOutParamHasSideEffects() throws Exception { + final String shader = "void main() { " + + " uint out1;" + + " uvec2 out2;" + + " uvec3 out3;" + + " uvec4 out4;" + + " uaddCarry(uint(0), uint(0), out1);" + + " uaddCarry(uvec2(0, 0), uvec2(0, 0), out2);" + + " uaddCarry(uvec3(0, 0, 0), uvec3(0, 0, 0), out3);" + + " uaddCarry(uvec4(0, 0, 0, 0), uvec4(0, 0, 0, 0), out4);" + + "}"; + + final TranslationUnit tu = ParseHelper.parse(shader, ShaderKind.FRAGMENT); + tu.setShadingLanguageVersion(ShadingLanguageVersion.ESSL_310); + + new StandardVisitor() { + @Override + public void visitFunctionCallExpr(FunctionCallExpr expr) { + assertTrue(expr.getCallee().equals("uaddCarry")); + assertFalse(SideEffectChecker.isSideEffectFree(expr, ShadingLanguageVersion.ESSL_310, ShaderKind.FRAGMENT)); + } + }.visit(tu); } + @Test + public void testOutParamModfHasSideEffects() throws Exception { + final String shader = "void main() { " + + " float out1;" + + " vec2 out2;" + + " vec3 out3;" + + " vec4 out4;" + + " modf(0.0, out1);" + + " modf(vec2(0.0), out2);" + + " modf(vec3(0.0), out3);" + + " modf(vec4(0.0), out4);" + + "}"; + + final TranslationUnit tu = ParseHelper.parse(shader, ShaderKind.FRAGMENT); + tu.setShadingLanguageVersion(ShadingLanguageVersion.ESSL_310); + + new StandardVisitor() { + @Override + public void visitFunctionCallExpr(FunctionCallExpr expr) { + assertTrue(expr.getCallee().equals("modf")); + assertFalse(SideEffectChecker.isSideEffectFree(expr, ShadingLanguageVersion.ESSL_310, ShaderKind.FRAGMENT)); + } + }.visit(tu); + } + + @Test + public void testOutParamFrexpHasSideEffects() throws Exception { + final String shader = "void main() { " + + " int out1;" + + " ivec2 out2;" + + " ivec3 out3;" + + " ivec4 out4;" + + " frexp(0.0, out1);" + + " frexp(vec2(0.0), out2);" + + " frexp(vec3(0.0), out3);" + + " frexp(vec4(0.0), out4);" + + "}"; + + final TranslationUnit tu = ParseHelper.parse(shader, ShaderKind.FRAGMENT); + tu.setShadingLanguageVersion(ShadingLanguageVersion.ESSL_310); + + new StandardVisitor() { + @Override + public void visitFunctionCallExpr(FunctionCallExpr expr) { + assertTrue(expr.getCallee().equals("frexp")); + assertFalse(SideEffectChecker.isSideEffectFree(expr, ShadingLanguageVersion.ESSL_310, ShaderKind.FRAGMENT)); + } + }.visit(tu); + } } diff --git a/docs/piglit-worker.md b/docs/piglit-worker.md new file mode 100644 index 000000000..8e3748051 --- /dev/null +++ b/docs/piglit-worker.md @@ -0,0 +1,84 @@ +# GraphicsFuzz Piglit/shader_runner worker + +[Piglit](https://piglit.freedesktop.org/) is a test suite for OpenGL implementations that +is developed as part of the [Mesa](https://mesa.freedesktop.org/) open source graphics +driver project. Along with its huge test suite, Piglit provides a testing framework, +`piglit-gl-framework`, and an associated renderer implementation, `shader_runner`. +Our `piglit-worker` is a Python script that sends shader jobs to `shader_runner`, retrieves +the rendered image/crash logs, and sends the results back to the GraphicsFuzz webserver. + +## Using the worker + +To run the worker, you will first need to build the GraphicsFuzz project - see +[GraphicsFuzz: developer documentation](https://github.com/google/graphicsfuzz/blob/master/docs/glsl-fuzz-develop.md) +for more details. Ensure that `graphicsfuzz/target/graphicsfuzz/python/drivers` +is on your PATH environment variable - this is the final extraction directory of GraphicsFuzz's +Python scripts after building. + +Additionally, you will need to clone and build Piglit - see [Piglit's README.md](https://gitlab.freedesktop.org/mesa/piglit/blob/master/README.md) +for build instructions. Once Piglit is built, you **must also** add the build directory, +`/path/to/piglit/bin`, to your PATH environment variable. To check this, you can run + +```sh +$ shader_runner_gles3 +``` + +in your terminal - if you get +```sh +PIGLIT: {"result": "skip" } +``` +in the output, your PATH is set up properly. + +To start the worker, simply use + ```sh +$ piglit-worker (worker name) + ``` + +A server IP/URL can be specified with an optional --server argument: + ```sh +$ piglit-worker --server (IP:PORT) (worker name) + ``` + +## piglit-worker shader_runner arguments + +These are the arguments that piglit-worker runs `shader_runner_gles3` with: + +`-png`: Dumps the rendered image of the shader to the current working directory, + named `shader_runner_gles3000.png`. + +`-ignore-missing-uniforms`: Piglit prefers to fail quickly in the event of a potentially malformed +`.shader_test` file, causing a test to fail if it tries to load uniform data into a uniform variable +that has been optimized away by the compiler (e.g. if the uniform was an unused variable). This +argument makes `shader_runner_gles3` ignore loading a uniform if it can't be found in the shader, instead of failing. + +`-report_subtests`: Forces `shader_runner_gles3` to render out of screen VRAM, preventing issues +where garbage would be rendered whenever _GLF_color is undefined or a fragment is discarded. + +`-auto`: Prevents `shader_runner_gles3` from rendering to a window, to speed up rendering + and mass processing. + +## graphicsfuzz-piglit-converter + +A GraphicsFuzz test is composed two files - a 'shader job' JSON file that encodes uniform variables +and other data that needs to be supplied to the shader, and the shader source code itself. In +contrast, a `shader_runner` test is a `.shader_test` file that contains the required OpenGL/GLSL +versions for the shader, the shader source code, and a special `[test]` header that can be used to +load data, probe pixels for colors, clear the screen, and much more. + +To convert a GraphicsFuzz shader job to a `.shader_test` file for use with `shader_runner`, +`piglit-worker` uses a script, `graphicsfuzz-piglit-converter`, which formats the GraphicsFuzz test +into a working `.shader_test` file. + +`graphicsfuzz-piglit-converter` can be used standalone as well - once you have GraphicsFuzz built +and the Python scripts on your PATH, you can just use +```sh +$ graphicsfuzz-piglit-converter (GraphicsFuzz shader job JSON file) +``` +and the `.shader_test` file will be output in the same directory as the JSON file. + +To run the `.shader_test` file through shader_runner yourself: +```sh +$ shader_runner_gles3 (.shader_test file) -png +``` + The output will be dumped to a PNG file named +`shader_runner_gles3000.png` in the current working directory. diff --git a/docs/summer-of-code-ideas/2019-JiradetOunjai.md b/docs/summer-of-code-ideas/2019-JiradetOunjai.md new file mode 100644 index 000000000..57f1d8670 --- /dev/null +++ b/docs/summer-of-code-ideas/2019-JiradetOunjai.md @@ -0,0 +1,156 @@ +# Google Summer of Code 2019 Report - Jiradet Ounjai + +The Google Summer of Code program has turned out to be a great learning opportunity for me. I gain fresh knowledge from my fantastic mentors through the discussions and the intense code reviews we had. I would like to thank my mentors for the time and energy you put into helping me. + +I have made this document to summarize the work done for my Google Summer of Code project *Improve Shading Language Support in GraphicsFuzz*. + + +###### Personal Links +###### [Github](https://github.com/jiradeto) - [Linkedin](https://www.linkedin.com/in/jiradeto/) + + +###### Project Link +###### [Improve Shading Language Support in GraphicsFuzz](https://summerofcode.withgoogle.com/projects/#4859963594244096) + + +###### Mentors +###### Alastair Donaldson, Paul Thomson + + +## Deliverables +My project idea focused on enhancing a shading language support in GraphicsFuzz. To accomplish the goal, I have improved the tool in various aspects which could be elaborated as follow: + +- [Add support for GLSL Built-In Functions](#add-support-for-glsl-built-in-functions) +- [Add new ways to generate an opaque expression](#add-new-ways-to-generate-an-opaque-expression) +- [New sample shaders](#new-sample-shaders) +- [Enhancement for Reducer](#enhancement-for-reducer) +- [Use GraphicsFuzz to find bugs in other tools](#use-graphicsfuzz-to-find-bugs-in-other-tools) +- [New shader generator tool](#new-shader-generator-tool) + + +### Add support for GLSL Built-In Functions +GLSL built-in functions are functions that available for use in a shader and GraphicsFuzz donates these built-ins into the unreachable statements so that they will not be actually executed. By providing a support for the GLSL built-ins, we teach GraphicsFuzz how to call functions appropriately by ensuring that the required function arguments are matched with the GLSL specifications. Moreover, we have to check that the built-in function being injected into a shader is compatible with the given shading language version. The following PRs have provided a support for the GLSL built-in functions. + +[#554](https://github.com/google/graphicsfuzz/pull/554): Add support for GLSL built-in functions: Angle and Trigonometric Functions + +[#555](https://github.com/google/graphicsfuzz/pull/555): Add support for GLSL built-in functions: Matrix Functions + +[#581](https://github.com/google/graphicsfuzz/pull/581): Add support for GLSL built-in functions: Geometric Functions + +[#585](https://github.com/google/graphicsfuzz/pull/585): Add support for GLSL built-in functions: Floating-Point Pack and Unpack Functions + +[#592](https://github.com/google/graphicsfuzz/pull/592): Add support for GLSL built-in functions: Common Functions + + +### Add new ways to generate an opaque expression +Fuzzing a shader in GraphicsFuzz involves applying a number of transformations that might inject code fragment to the original shader aiming to generate a variant shader that is much larger than the original. Opaque expression, one of various expressions the generator employs when injecting code fragment, is an expression whose value is unknown by a compiler. Therefore, donating opaque expressions can help preventing the optimization performed by the compiler. In GraphicsFuzz, we have plenty of opaque expressions representing 0 and 1 however by introducing new opaque values we provide a new expression that is likely to trigger a new bug. The following PRs involve adding new opaque expressions to the GraphicsFuzz's generator. + + +[#413](https://github.com/google/graphicsfuzz/pull/413): Represent 1 as length of normalized vector + +[#421](https://github.com/google/graphicsfuzz/pull/421): Enlarge test in OpaqueExpressionGeneratorTest + +[#424](https://github.com/google/graphicsfuzz/pull/424): Represent 1 using cos() function + +[#428](https://github.com/google/graphicsfuzz/pull/428): Represent 1 using exp(0.0) + +[#429](https://github.com/google/graphicsfuzz/pull/429): Represent 0 using log(1.0) + +[#447](https://github.com/google/graphicsfuzz/pull/447): Seperate functions making opaque 1 and 0 + +[#459](https://github.com/google/graphicsfuzz/pull/459): Represent 0 and 1 using abs(opaque) + +[#472](https://github.com/google/graphicsfuzz/pull/472): Represent 0 as length of a zero vector + +[#475](https://github.com/google/graphicsfuzz/pull/475): Rename opaque factories variable + + +### New sample shaders + +In GraphicsFuzz, sample shaders play an important role in exposing bugs in shader compilers as they are the initial set of shaders that would be mutated whose final result are the variant shaders that might expose the potential compiler bugs. + +Having said that, we have only few sample shaders that come with GraphicsFuzz by default, the following PRs thus focus on adding a brand new set of 310 es sample shaders implementing different sorting and searching algorithms. + + +[#602](https://github.com/google/graphicsfuzz/pull/602): New 310es sample shaders - v1 + +| trigonometric_strip.frag | selection_sort_struct.frag | prefix_sum_checkers.frag | +| :---: | :---: | :---: | +| ![trigonometric_strip](./images/jiradet/shader_trigonometric_strip.png) | ![selection_sort_struct](./images/jiradet/shader_selection_sort_struct.png) | ![prefix_sum_checkers](./images/jiradet/shader_prefix_sum_checkers.png) | + +[#643](https://github.com/google/graphicsfuzz/pull/643): New 310es sample shaders - v2 + +| binarysearch_bw.frag | mergesort_mosaic.frag | +| :---: | :---: | +| ![binarysearch_bw](./images/jiradet/shader_binarysearch_bw.png) | ![mergesort_mosaic](./images/jiradet/shader_mergesort_mosaic.png) | + + +[#660](https://github.com/google/graphicsfuzz/pull/660): New 310es sample shaders - quicksort + +| quicksort_palette.frag | +| :---: | +| ![quicksort_palette](./images/jiradet/shader_quicksort_palette.png) | + + +[#676](https://github.com/google/graphicsfuzz/pull/676): New 310es sample shaders - binary search tree + +| binarysearch_tree.frag | +| :---: | +| ![binarysearch_tree](./images/jiradet/shader_binarysearch_tree.png) | + + +### Enhancement for Reducer +To remove code fragment injected by the generator, we use reducer to shrink the variant shader into a very small and simple shader that is still inducing a bug. GraphicsFuzz' reducer essentially finds the interesting code fragment to be removed based on the reducer opportunities. Currently, we have sufficiently large set of reducer opportunities that can significantly reduce a variant shader. However, there were areas need improvement. The following PRs involve adding new reducer opportunities to the GraphicsFuzz's reducer aiming to help reducer reducing GLSL shaders more efficient: + +[#477](https://github.com/google/graphicsfuzz/pull/477): Add tests for Simplify + +[#489](https://github.com/google/graphicsfuzz/pull/489): Remove parentheses after eliminating macros + +[#490](https://github.com/google/graphicsfuzz/pull/490): Reducer opportunity to remove redundant uniform metadata + +[#500](https://github.com/google/graphicsfuzz/pull/500): Add macro wrapper and fix space before comma + +[#508](https://github.com/google/graphicsfuzz/pull/508): Add a test for function call macro parentheses removal + +[#513](https://github.com/google/graphicsfuzz/pull/513): Add class and tests : reduce VariablesDeclaration with expression + +[#514](https://github.com/google/graphicsfuzz/pull/514): Add if_dead opportunities and fix fail tests + +[#529](https://github.com/google/graphicsfuzz/pull/529): Implementation: Reducer Opportunity to replace variable declaration with an expression + +[#538](https://github.com/google/graphicsfuzz/pull/538): Add test for const variable declaration + +[#561](https://github.com/google/graphicsfuzz/pull/561): Reducer: Add class skeleton and tests + +[#576](https://github.com/google/graphicsfuzz/pull/576): Reducer implementation: reduce global variable declarations to expression + +[#622](https://github.com/google/graphicsfuzz/pull/622): Add reducer opportunities + + +### Use GraphicsFuzz to find bugs in other tools +GraphicsFuzz's team has discovered and reported a bunch of bugs lying in many different GPU vendors. In fact, we constantly run a graphics compiler test since we would be delighted if we could help developers detecting uncovering bugs. Over the past few months, I had a chance to use GraphicsFuzz to help finding bugs in [SPIRV Cross](https://github.com/KhronosGroup/SPIRV-Cross). + +SPIRV-Cross is a very convenient tool that helps parsing and converting SPIR-V to other shader languages. Throughout GSoC program, I relied heavily on this tool since I am using [MoltenVK](https://github.com/KhronosGroup/MoltenVK) to run SPIR-V on Mac and MoltenVK internally calls SPIRV-Cross to convert SPIR-V into Apple's Metal Shading Language. + +To see all SPIRV-Cross issues I filed please check [here](https://github.com/KhronosGroup/SPIRV-Cross/issues?utf8=%E2%9C%93&q=+is%3Aissue+author%3Ajiradeto+). + + +### New shader generator tool +![new_shaders](./images/jiradet/shadergenerator_overview.png) + +GraphicsFuzz has now equipped with a new tool called Known Value Shader Generator which generates a shader job from the given RGBA colors. This tool mutates the numeric inputs by applying various transformations which eventually generates the mutated expressions that guarantee to produce the original input values. With the help of this tool, we have a brand new way to generate a variant shader just by simply providing the expected values. + +The following PRs involve implementing a new shader generator tool. + +[#625](https://github.com/google/graphicsfuzz/pull/625): Add a new tool: Known Value Shader Generator + +[#677](https://github.com/google/graphicsfuzz/pull/677): Expression Generator: add global initializer method + +[#681](https://github.com/google/graphicsfuzz/pull/681): Expression Generator: find numbers for addition based on known facts + +[#693](https://github.com/google/graphicsfuzz/pull/693): Expression Generator: introduce uniforms + +#### Future development +Currently, the shader generator tool has a limited number of transformations. The next steps for this tool involve extending transformations set and integrating this tool into the fuzzing chain of GraphicsFuzz. Afterward, it would also be interesting to apply new capabilities of GraphicsFuzz armed with the new tool to find bugs in shader compilers. + + diff --git a/docs/summer-of-code-ideas/images/jiradet/shader_binarysearch_bw.png b/docs/summer-of-code-ideas/images/jiradet/shader_binarysearch_bw.png new file mode 100644 index 000000000..d305eff76 Binary files /dev/null and b/docs/summer-of-code-ideas/images/jiradet/shader_binarysearch_bw.png differ diff --git a/docs/summer-of-code-ideas/images/jiradet/shader_binarysearch_tree.png b/docs/summer-of-code-ideas/images/jiradet/shader_binarysearch_tree.png new file mode 100644 index 000000000..a81a1dd54 Binary files /dev/null and b/docs/summer-of-code-ideas/images/jiradet/shader_binarysearch_tree.png differ diff --git a/docs/summer-of-code-ideas/images/jiradet/shader_mergesort_mosaic.png b/docs/summer-of-code-ideas/images/jiradet/shader_mergesort_mosaic.png new file mode 100644 index 000000000..24ae12019 Binary files /dev/null and b/docs/summer-of-code-ideas/images/jiradet/shader_mergesort_mosaic.png differ diff --git a/docs/summer-of-code-ideas/images/jiradet/shader_prefix_sum_checkers.png b/docs/summer-of-code-ideas/images/jiradet/shader_prefix_sum_checkers.png new file mode 100644 index 000000000..dc2d6ab78 Binary files /dev/null and b/docs/summer-of-code-ideas/images/jiradet/shader_prefix_sum_checkers.png differ diff --git a/docs/summer-of-code-ideas/images/jiradet/shader_quicksort_palette.png b/docs/summer-of-code-ideas/images/jiradet/shader_quicksort_palette.png new file mode 100644 index 000000000..8ec760b01 Binary files /dev/null and b/docs/summer-of-code-ideas/images/jiradet/shader_quicksort_palette.png differ diff --git a/docs/summer-of-code-ideas/images/jiradet/shader_selection_sort_struct.png b/docs/summer-of-code-ideas/images/jiradet/shader_selection_sort_struct.png new file mode 100644 index 000000000..34a290284 Binary files /dev/null and b/docs/summer-of-code-ideas/images/jiradet/shader_selection_sort_struct.png differ diff --git a/docs/summer-of-code-ideas/images/jiradet/shader_trigonometric_strip.png b/docs/summer-of-code-ideas/images/jiradet/shader_trigonometric_strip.png new file mode 100644 index 000000000..206094bc9 Binary files /dev/null and b/docs/summer-of-code-ideas/images/jiradet/shader_trigonometric_strip.png differ diff --git a/docs/summer-of-code-ideas/images/jiradet/shadergenerator_overview.png b/docs/summer-of-code-ideas/images/jiradet/shadergenerator_overview.png new file mode 100644 index 000000000..1e5ed2dd5 Binary files /dev/null and b/docs/summer-of-code-ideas/images/jiradet/shadergenerator_overview.png differ diff --git a/docs/summer-of-code-ideas/summer-of-code-reports/gsoc-2019-abelbriggs.md b/docs/summer-of-code-ideas/summer-of-code-reports/gsoc-2019-abelbriggs.md new file mode 100644 index 000000000..779b97580 --- /dev/null +++ b/docs/summer-of-code-ideas/summer-of-code-reports/gsoc-2019-abelbriggs.md @@ -0,0 +1,367 @@ +# Google Summer of Code 2019 Report - Abel Briggs + +###### Personal Links: +###### [Github](https://github.com/abelbriggs1) - [Linkedin](https://www.linkedin.com/in/abelbriggs/) + +###### Proposal: +###### [Enhancing Metamorphic Testing Tools For Graphics Drivers](https://summerofcode.withgoogle.com/projects/#6749896743321600) + +###### Mentors: +###### Alastair Donaldson, Hugues Evrard + +## Preface + +Before this report begins, I'd like to deliver a short spiel directed towards potential future +viewers (and likely GSoC hopefuls) of this document. + +While Google Summer of Code may look daunting to the prospective student applicant, I implore anyone +who even has a passing interest in the program to consider applying/proposing for an organization +that strikes your fancy (and do not be intimidated when looking at past GSoC projects!). Mentorship +is an invaluable experience to any engineer (especially junior engineers) and not all workplaces +have the budget and/or manpower to support it. The best feature of Google Summer of Code is that +this problem vanishes - you are paired up with mentors that have 'help my student succeed' as a +major goal, and this is a fundamental cornerstone of the program. I had little experience in +proper software collaboration and in working on large/domain-specific codebases before I began +submitting PRs to GraphicsFuzz - the code reviews and mentoring that I was given during the program +were instrumental in making me a better and more confident engineer. + +Thanks to Ally, Hugues and Paul for their continued mentorship and support over +the 2019 summer. + +With that said... + +## Deliverables + +During and throughout the summer of 2019, my mentors and I worked out the kinks in my original +proposal and developed some fairly solid objectives to work towards. This report is intended to +detail what those deliverables are, how they were delivered and what could still use work in the +future. + +These deliverables were: + + - [Enhance GraphicsFuzz's shader generator by making it more aware of OpenGL Shading Language's built-in functions.](#graphicsfuzz-shader-generator---glsl-built-in-support) + - [Enhance GraphicsFuzz's shader generator by adding additional ways to generate opaque values and new identity transformations.](#graphicsfuzz-shader-generator---identities-and-opaque-value-generation) + - [Add a new 'worker' program](#write-worker-script-that-uses-piglit-test-framework-to-render-images) + that takes GraphicsFuzz shader jobs from a server, renders the shader job via Mesa's open-source + OpenGL test framework Piglit, and sends the results back to the server. + - [Add new shaders to GraphicsFuzz's fuzz test set](#add-new-shaders-to-graphicsfuzzs-test-set) - brand new shaders as well as derivatives of the + original test set. + - [Apply GraphicsFuzz to the Mesa open-source graphics driver suite](#apply-graphicsfuzz-to-the-nouveau-open-source-graphics-driver), + specifically the NVIDIA reverse-engineered driver nouveau, the open-source driver that was easiest + for me to run tests on. + +### GraphicsFuzz shader generator - GLSL built-in support + +When GraphicsFuzz generates fuzzed/arbitrary expressions with the assumption that they won't be +executed (e.g. in `(true ? x : y)`, y will never be executed), it is able to generate calls to GLSL +built-in functions(think `abs()`, `sqrt()`, etc.) with arbitrary values as arguments. To do this, however, +GraphicsFuzz needs to know what types to fuzz expressions for, or it will cause type errors. + +The following PRs involve cross-checking built-in functions with the language version they were +introduced in, then adding their function prototypes to GraphicsFuzz. + +[#521](https://github.com/google/graphicsfuzz/pull/521): +Add built-in support for GLSL Integer Functions + +[#528](https://github.com/google/graphicsfuzz/pull/528): +Add built-in function support for GLSL Vector Relational Functions + +[#549](https://github.com/google/graphicsfuzz/pull/549): +Add built-in function support for GLSL 3.20 Fragment Processing Functions + +[#563](https://github.com/google/graphicsfuzz/pull/563): +Refactor exponential builtins into separate function + +Additionally, minor issues arose from adding support for these functions - some of these functions +involve using `out` parameters and modify said parameters during their execution. This would +potentially cause 'side effects' - lvalues being modified when not expected. The following PR +relates to mitigating this issue. + +[#533](https://github.com/google/graphicsfuzz/pull/533): +Check for side effects with lvalue parameters of built-in functions + +#### Leftovers + +The following issues are nits or minor problems related to built-ins that were left unfixed due to +time or knowledge constraints. + +[#550](https://github.com/google/graphicsfuzz/issues/550): +Ensure certain function prototypes can use non-uniform shader input variables + +[#570](https://github.com/google/graphicsfuzz/issues/570): +Be less conservative about when FunctionCallExprTemplates yield expressions that have side effects + +### GraphicsFuzz shader generator - identities and opaque value generation + +Among the various operations that GraphicsFuzz's generator can do to a shader when fuzzing, two of +the most important are identity transformations and opaque value generation. + +An identity transformation is a function that, given an expression `e`, transforms `e -> f`, where `f` is +an expression equivalent for all intents and purposes to `e` (*semantically equivalent*). GraphicsFuzz +is able to use various properties of GLSL data types and built-in functions to craft these identity +transformations, and can recursively apply them to create obscenely complicated expressions. + +For a toy example of an identity: Let `e` be a float of any valid value and let `identity(x) = x * 1.0`. +Then `e` is semantically equivalent to `identity(e)`. + +An opaque value in the context of GraphicsFuzz is an rvalue expression that is guaranteed to have +a certain value, yet the value is not obvious to a compiler. The generator introduces opaque zeroes +or opaque ones whenever required by other GraphicsFuzz transformations. Like identity transformations, +these opaque values exploit invariants and properties of GLSL data types and built-in functions, and +they can be recursively applied to complicate expressions. + +For a toy example of opaque values: Let `e = 0.0`. Notice that `0.0 == sqrt(0.0) == abs(0.0) == +float(0 >> 8) == injectionSwitch.x == abs(sqrt(float(0 >> 8)))`. Then `0.0` can be replaced with +any of these expressions. + +The following PRs directly involve adding new ways to generate opaque values or identity transformations. + +[#509](https://github.com/google/graphicsfuzz/pull/509): +Generate ternary identities for nonscalar types + +[#512](https://github.com/google/graphicsfuzz/pull/512): +Add new transformation for rewriting nonscalars as their constructors + +[#525](https://github.com/google/graphicsfuzz/pull/525): +Fix matrix constructors generating identities for the wrong base type + +[#527](https://github.com/google/graphicsfuzz/pull/527): +Enhance matrix constructor identity to potentially use all elements of matrix + +[#553](https://github.com/google/graphicsfuzz/pull/553): +Add new identities for integer types that use bitwise operations + +[#562](https://github.com/google/graphicsfuzz/pull/562): +Add new ways to generate opaque zero/one from bitwise operations + +[#580](https://github.com/google/graphicsfuzz/pull/580): +Ensure bitwise shift opaque one shifts left/right by same value + +[#641](https://github.com/google/graphicsfuzz/pull/641): +Refactor shift opaque to use identities instead of fuzzed clamped expressions + +[#642](https://github.com/google/graphicsfuzz/pull/642): +Add function to generate opaque zero/one from dot product of two vectors + +[#648](https://github.com/google/graphicsfuzz/pull/648): +Add function to make opaque zero/one from determinant of a matrix + +[#649](https://github.com/google/graphicsfuzz/pull/649): +Added identity to double transpose a matrix + +[#682](https://github.com/google/graphicsfuzz/pull/682): +Add function to make opaque zero vec3 from cross product of equal vec3s + +[#683](https://github.com/google/graphicsfuzz/pull/683): +Add identity to multiply rectangular matrix/vector by identity matrix + +[#695](https://github.com/google/graphicsfuzz/pull/695): +Fix opaque dot function inverted isZero indices + +[#704](https://github.com/google/graphicsfuzz/pull/704): +Add identity to put data in a larger type and pull it back out + +#### Leftovers + +The following issues are nits or minor problems related to opaque values/identity transformations +that were left unfixed due to time or knowledge constraints, or are issues out of the scope of +GraphicsFuzz. + +[#552](https://github.com/google/graphicsfuzz/issues/552): +Rewrite composite identity tries to index into shader output variables in GLES + +[#637](https://github.com/google/graphicsfuzz/issues/637): +Construct identity matrix by multiplying rectangular zero-one matrices + +[#653](https://github.com/google/graphicsfuzz/issues/653): +Be less conservative with matrix functions in constexpr contexts + +Related glslangValidator issue: https://github.com/KhronosGroup/glslang/issues/1865. + +### Write worker script that uses Piglit test framework to render images + +[Mesa](https://mesa.freedesktop.org/) is an open-source graphics driver suite that is able to power +a massive range of graphics hardware across the last twenty years of computing. To help deal with the +variance in hardware and driver implementations, contributors to Mesa wrote a graphics driver +test framework called [Piglit](https://piglit.freedesktop.org/). Among its huge suite of GLSL and +Vulkan tests, Piglit includes a rendering program, `shader_runner`, that can take user-supplied +`.shader_test` files, render the included shader programs, test images for correct color values, + and more. + +For running shaders, GraphicsFuzz uses a client-server model. In GraphicsFuzz's toolset, a webserver +and several 'workers', or clients, are supplied. These workers and the server communicate to each +other via an [Apache Thrift](https://thrift.apache.org/) specification. + +To run a test set, the user starts the webserver and a worker, then selects a test set to run +(via UI or otherwise). For each shader, the webserver sends the shader source code and a JSON file +that details any additional information the shader needs (such as uniform data) to the worker - the +worker then renders the shader in any way it sees fit and returns the rendered image +(if there was one), logs, etc. + +When reporting bugs in Mesa's drivers, it is most convenient for the Mesa developers to have a +`.shader_test` file that they can easily plug into Piglit as a test case. Additionally, the OpenGL ES +GraphicsFuzz worker is rather unwieldy to compile/use for desktop platforms, and so it was decided +that the development of a Piglit/`shader_runner` worker was worth pursuing, along with a script to +systematically convert GraphicsFuzz shader jobs into Piglit `.shader_test` files. + +The following PRs directly involve the Piglit converter script or worker. + +[#577](https://github.com/google/graphicsfuzz/pull/577): +Add script to convert a GraphicsFuzz shader job to a piglit shader test + +[#584](https://github.com/google/graphicsfuzz/pull/584): +Fix piglit converter script not inserting GL ES version + +[#613](https://github.com/google/graphicsfuzz/pull/613): +Fix piglit converter not accepting blank JSON + +[#617](https://github.com/google/graphicsfuzz/pull/617): +Add piglit shader_runner_gles3 worker script + +[#631](https://github.com/google/graphicsfuzz/pull/631): +docs: Add documentation for piglit worker + +[#632](https://github.com/google/graphicsfuzz/pull/632): +Piglit worker improvements + +[#638](https://github.com/google/graphicsfuzz/pull/638): +graphicsfuzz-piglit-converter: support more uniform types + +In addition, a contribution was made to Piglit upstream to allow `shader_runner` to handle a case +when unused uniform data was optimized out of a compiled shader. + +[shader_runner](https://gitlab.freedesktop.org/mesa/piglit/merge_requests/92): +Add command line option to ignore uniform if it isn't found in shader + +#### Leftovers + +The following issues are nits or minor problems related to the Piglit worker or converter script +that were left unfixed due to time or knowledge constraints. + +[#597](https://github.com/google/graphicsfuzz/issues/597): +Support compute shaders in graphicsfuzz_piglit_converter + +### Add new shaders to GraphicsFuzz's test set + +With the exception of compute shaders, GraphicsFuzz had a fairly limited set of five simple test +shaders for mutation. These shaders sufficed for finding simple bugs, but were limited to GLES 1.00 +features and were unable to test the full extent of GLSL even with significant additions +to the shader generator. + +The following PRs directly involve adding new shaders to GraphicsFuzz's test suite. + +[#605](https://github.com/google/graphicsfuzz/pull/605): +Add new versions of GraphicsFuzz shaders that use Integer Functions + +[#647](https://github.com/google/graphicsfuzz/pull/647): +New 310es shader: householder_lattice + +| householder_lattice.frag | +| ------------------------ | +| ![householder_lattice](https://i.imgur.com/ih3ORPH.png) | + +[#698](https://github.com/google/graphicsfuzz/pull/698): +New 310es shader - Muller's method + +| muller.frag | +| ----------- | +| ![muller](https://i.imgur.com/recmnjw.png) | + +### Apply GraphicsFuzz to the nouveau open-source graphics driver + +[nouveau](https://nouveau.freedesktop.org/wiki/) is a reverse-engineered open-source graphics driver +for NVIDIA graphics hardware that is developed under the umbrella of the Mesa driver suite. Because +of limitations on modern NVIDIA cards due to NVIDIA requiring signed firmware to access +key hardware on graphics cards past the [Maxwell](https://nouveau.freedesktop.org/wiki/CodeNames/#nv110familymaxwell) +generation, nouveau receives little use in comparison to other Mesa drivers or NVIDIA's closed-source +binaries, making it a prime target for fuzzing. + +The following are bugs reported to the Mesa bug tracker found via fuzzing. + +[Bug 110953](https://bugs.freedesktop.org/show_bug.cgi?id=110953): +Adding a redundant single-iteration do-while loop causes different image to be rendered + +| Reference | Variant | +| --------- | ------- | +| ![reference](https://i.imgur.com/PBY0tCP.png) | ![variant](https://i.imgur.com/BHbVc3x.png) + +[Bug 111006](https://bugs.freedesktop.org/show_bug.cgi?id=111006): +Adding a uniform-dependent if-statement in shader renders a different image + +| Reference | Variant | +| --------- | ------- | +| ![reference](https://i.imgur.com/0Q8dSRZ.png) | ![variant](https://i.imgur.com/uaZCU8t.png) + +[Bug 111167](https://bugs.freedesktop.org/show_bug.cgi?id=111167): +Dividing zero by a uniform in loop header causes segfault in nv50_ir::NVC0LegalizeSSA::handleDIV + +``` +for (int i = 1; 1 <= (0 / int(injectionSwitch.y)); 1) +{ +} +``` + +This snippet of shader code causes nouveau to crash when trying to handle the division. + +Stacktrace: +``` +#0 std::_Deque_iterator::_Deque_iterator ( + __x=, + this=) at /usr/include/c++/8/bits/stl_deque.h:1401 +#1 std::_Deque_iterator::operator+ (__n=0, this=0xb0) at /usr/include/c++/8/bits/stl_deque.h:230 +#2 std::_Deque_iterator::operator[] (__n=0, this=0xb0) at /usr/include/c++/8/bits/stl_deque.h:247 +#3 std::deque >::operator[] (__n=0, this=0xa0) at /usr/include/c++/8/bits/stl_deque.h:1404 +#4 nv50_ir::Instruction::getSrc (s=0, this=0x0) + at ../src/gallium/drivers/nouveau/codegen/nv50_ir.h:827 +#5 nv50_ir::NVC0LegalizeSSA::handleDIV (this=0x7ffd7753af60, i=0x55d2e1b132a0) + at ../src/gallium/drivers/nouveau/codegen/nv50_ir_lowering_nvc0.cpp:54 +#6 0x00007fc7191cb4b3 in nv50_ir::NVC0LegalizeSSA::visit ( + this=0x7ffd7753af60, bb=) + at ../src/gallium/drivers/nouveau/codegen/nv50_ir_lowering_nvc0.cpp:334 +#7 0x00007fc719111928 in nv50_ir::Pass::doRun (this=0x7ffd7753af60, + func=, ordered=, skipPhi=true) + at ../src/gallium/drivers/nouveau/codegen/nv50_ir_bb.cpp:500 +#8 0x00007fc7191119f4 in nv50_ir::Pass::doRun (this=0x7ffd7753af60, + prog=, ordered=false, skipPhi=true) + at ../src/gallium/drivers/nouveau/codegen/nv50_ir_inlines.h:413 +``` + +#### Leftovers + +[Bug 111006](https://bugs.freedesktop.org/show_bug.cgi?id=111006) involves a critical component of +GraphicsFuzz's fuzzing techniques - preventing a compiler from optimizing uni-directional +branches (like an if-statement that is always true) by using a variable that is unknown until runtime. +While this bug remains unfixed, it is difficult to continue fuzz testing without running into a +large number of potential duplicate issues. This issue prevented continued fuzzing over the second +half of GSoC 2019. + +### Miscellaneous + +While working on GraphicsFuzz for Google Summer of Code 2019, minor problems that were not directly +related to the detailed deliverables cropped up. + +The following PRs involve miscellaneous refactoring or changes to GraphicsFuzz. + +[#507](https://github.com/google/graphicsfuzz/pull/507): +Add additional tests for empty for statements + +[#532](https://github.com/google/graphicsfuzz/pull/532): +Refactor BasicType with documentation and exception improvements + +[#535](https://github.com/google/graphicsfuzz/pull/535): +Remove pointless static numColumns function + +[#537](https://github.com/google/graphicsfuzz/pull/537): +Refactor code to use isType functions when possible + +[#587](https://github.com/google/graphicsfuzz/pull/587): +Refactor common python script functions into one common lib + +[#668](https://github.com/google/graphicsfuzz/pull/668): +Refactor transposedMatrixType() to use makeMatrixType() + +[#670](https://github.com/google/graphicsfuzz/pull/670): +Fix making array accesses inbounds with uint index accesses + + + diff --git a/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/GenerateAndRunShaders.java b/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/GenerateAndRunShaders.java index 6def15710..60252eed9 100755 --- a/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/GenerateAndRunShaders.java +++ b/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/GenerateAndRunShaders.java @@ -16,12 +16,9 @@ package com.graphicsfuzz.generator; -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.generator.tool.Generate; -import com.graphicsfuzz.util.ArgsUtil; import java.io.File; import java.io.IOException; import java.util.HashSet; @@ -29,7 +26,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import net.sourceforge.argparse4j.ArgumentParsers; -import net.sourceforge.argparse4j.impl.Arguments; import net.sourceforge.argparse4j.inf.ArgumentParser; import net.sourceforge.argparse4j.inf.ArgumentParserException; import net.sourceforge.argparse4j.inf.Namespace; @@ -65,10 +61,6 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .help("Worker name.") .type(String.class); - parser.addArgument("glsl-version") - .help("Version of GLSL to target.") - .type(String.class); - // Optional arguments Generate.addGeneratorCommonArguments(parser); diff --git a/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/ShaderProducer.java b/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/ShaderProducer.java index f96466bc2..3068fc0b6 100755 --- a/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/ShaderProducer.java +++ b/generate-and-run-shaders/src/main/java/com/graphicsfuzz/generator/ShaderProducer.java @@ -89,8 +89,7 @@ public void run() { !ns.getBoolean("no_injection_switch") ); - final int outerSeed = ArgsUtil.getSeedArgument(ns); - final IRandom generator = new RandomWrapper(outerSeed); + final IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); int sent = 0; for (int counter = 0; sent < limit; counter++) { @@ -107,8 +106,9 @@ public void run() { new RemoveDiscardStatements(shaderJob.getFragmentShader().get()); // Create a variant. - Generate.generateVariant(shaderJob, generatorArguments, - generator.nextInt(Integer.MAX_VALUE)); + IRandom childRandom = generator.spawnChild(); + LOGGER.info("Calling generateVariant with RNG: {}", childRandom.getDescription()); + Generate.generateVariant(shaderJob, generatorArguments, childRandom); // Restrict the colors that the variant can emit. final float probabilityOfAddingNewColorWrite = 0.01f; diff --git a/generate-and-run-shaders/src/test/java/com/graphicsfuzz/generator/GenerateAndRunShadersTest.java b/generate-and-run-shaders/src/test/java/com/graphicsfuzz/generator/GenerateAndRunShadersTest.java index dbf1a7980..7fc689cbe 100755 --- a/generate-and-run-shaders/src/test/java/com/graphicsfuzz/generator/GenerateAndRunShadersTest.java +++ b/generate-and-run-shaders/src/test/java/com/graphicsfuzz/generator/GenerateAndRunShadersTest.java @@ -44,8 +44,7 @@ public void testNoReferences() throws Exception { donors.getAbsolutePath(), outputDir, "dummy_server", - "dummy_worker", - "100" + "dummy_worker" } ); throw new RuntimeException("Exception expected."); @@ -70,8 +69,7 @@ public void testNoDonors() throws Exception { donors.getAbsolutePath(), outputDir, "dummy_server", - "dummy_worker", - "100" + "dummy_worker" } ); throw new RuntimeException("Exception expected."); @@ -95,8 +93,7 @@ public void testMissingJson() throws Exception { donors.getAbsolutePath(), outputDir, "dummy_server", - "dummy_worker", - "100" + "dummy_worker" } ); throw new RuntimeException("Exception expected."); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Fuzzer.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Fuzzer.java index 17c73b361..814529344 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Fuzzer.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Fuzzer.java @@ -27,6 +27,7 @@ import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayConstructorExpr; import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.expr.TypeConstructorExpr; import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.BreakStmt; @@ -176,7 +177,7 @@ private Expr makeExpr(Type targetType, boolean isLValue, boolean constContext, f if (!shadingLanguageVersion.restrictedArrayIndexing()) { ArrayType arrayType = (ArrayType) targetType; List args = new ArrayList<>(); - for (int i = 0; i < arrayType.getArrayInfo().getSize(); i++) { + for (int i = 0; i < arrayType.getArrayInfo().getConstantSize(); i++) { args.add(makeExpr(arrayType.getBaseType(), isLValue, constContext, depth + 1)); } return new ArrayConstructorExpr((ArrayType) stripQualifiers(targetType), args); @@ -220,47 +221,21 @@ public static boolean isTooDeep(int depth, GenerationParams generationParams, IR private Stream availableTemplatesFromContext() { return Stream.concat(availableTemplatesFromScope(shadingLanguageVersion, + generationParams.getShaderKind(), fuzzingContext.getCurrentScope()), fuzzingContext.getFunctionPrototypes().stream().map(FunctionCallExprTemplate::new)); } public static Stream availableTemplatesFromScope( ShadingLanguageVersion shadingLanguageVersion, + ShaderKind shaderKind, Scope scope) { - return Stream.concat(Templates.get(shadingLanguageVersion).stream(), + return Stream.concat(Templates.get(shadingLanguageVersion, shaderKind).stream(), scope.namesOfAllVariablesInScope() .stream() .map(item -> new VariableIdentifierExprTemplate(item, scope.lookupType(item)))); } - public static void main(String[] args) { - try { - //testFuzzExpr(args); - showTemplates(args); - - } catch (Throwable throwable) { - throwable.printStackTrace(); - System.exit(1); - } - - } - - private static void showTemplates(String[] args) { - // Call this from main to produce a list of templates - List templates = new ArrayList<>(); - templates.addAll(Templates.get(ShadingLanguageVersion.fromVersionString(args[0]))); - Collections.sort(templates, new Comparator() { - @Override - public int compare(IExprTemplate t1, IExprTemplate t2) { - return t1.toString().compareTo(t2.toString()); - } - }); - - for (IExprTemplate t : templates) { - System.out.println(t); - } - } - public TranslationUnit fuzzTranslationUnit() { List decls = new ArrayList<>(); @@ -478,7 +453,9 @@ private DeclarationStmt fuzzDeclarationStmt() { final String name = createName("v"); ArrayInfo arrayInfo = null; if (generator.nextInt(10) < 3) { // TODO Hack for now, needs thought - arrayInfo = new ArrayInfo(generator.nextPositiveInt(MAX_ARRAY_SIZE)); + arrayInfo = + new ArrayInfo(new IntConstantExpr(Integer.toString( + generator.nextPositiveInt(MAX_ARRAY_SIZE)))); } fuzzingContext.addLocal(name, arrayInfo == null ? baseType : getType(baseType, arrayInfo)); decls.add(new VariableDeclInfo(name, arrayInfo, null)); // TODO: no initializer for now @@ -557,4 +534,33 @@ private Type stripQualifiers(Type type) { return type; } + public static void main(String[] args) { + try { + //testFuzzExpr(args); + showTemplates(args); + + } catch (Throwable throwable) { + throwable.printStackTrace(); + System.exit(1); + } + + } + + private static void showTemplates(String[] args) { + // Call this from main to produce a list of templates + List templates = new ArrayList<>(); + templates.addAll(Templates.get(ShadingLanguageVersion.fromVersionString(args[0]), + ShaderKind.fromExtension(args[1]))); + Collections.sort(templates, new Comparator() { + @Override + public int compare(IExprTemplate t1, IExprTemplate t2) { + return t1.toString().compareTo(t2.toString()); + } + }); + + for (IExprTemplate t : templates) { + System.out.println(t); + } + } + } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java index e7aabd060..586f2f867 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGenerator.java @@ -38,6 +38,7 @@ import com.graphicsfuzz.common.util.OpenGlConstants; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.common.util.SideEffectChecker; +import com.graphicsfuzz.generator.semanticschanging.LiteralFuzzer; import com.graphicsfuzz.generator.transformation.ExpressionIdentity; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.util.Constants; @@ -48,6 +49,7 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; public final class OpaqueExpressionGenerator { @@ -63,26 +65,40 @@ public OpaqueExpressionGenerator(IRandom generator, GenerationParams generationP ShadingLanguageVersion shadingLanguageVersion) { this.generator = generator; this.generationParams = generationParams; - // TODO: there are many more identities that we can easily play with here, e.g. bitwise and 1 - // for integer types this.expressionIdentities = new ArrayList<>(); this.shadingLanguageVersion = shadingLanguageVersion; + // TODO: there are many more identities that we can easily play with here expressionIdentities.add(new IdentityAddSubZero()); expressionIdentities.add(new IdentityMulDivOne()); expressionIdentities.add(new IdentityAndTrue()); expressionIdentities.add(new IdentityOrFalse()); - expressionIdentities.add(new IdentityNotNot()); + expressionIdentities.add(new IdentityLogicalNotNot()); expressionIdentities.add(new IdentityTernary()); + if (shadingLanguageVersion.supportedBitwiseOperations()) { + expressionIdentities.add(new IdentityBitwiseNotNot()); + expressionIdentities.add(new IdentityBitwiseOrSelf()); + expressionIdentities.add(new IdentityBitwiseOrZero()); + expressionIdentities.add(new IdentityBitwiseXorZero()); + expressionIdentities.add(new IdentityBitwiseShiftZero()); + } + expressionIdentities.add(new IdentityMin()); expressionIdentities.add(new IdentityMax()); expressionIdentities.add(new IdentityClamp()); + expressionIdentities.add(new IdentityRewriteComposite()); + expressionIdentities.add(new IdentityCompositeConstructorExpansion()); if (shadingLanguageVersion.supportedMixNonfloatBool()) { expressionIdentities.add(new IdentityMixBvec()); } + if (shadingLanguageVersion.supportedTranspose()) { + expressionIdentities.add(new IdentityDoubleTranspose()); + } + expressionIdentities.add(new IdentityMatrixMultIdentity()); + } private List waysToMakeZeroOrOne() { @@ -90,26 +106,39 @@ private List waysToMakeZeroOrOne() { this::opaqueZeroOrOneFromIdentityFunction, this::opaqueZeroOrOneFromInjectionSwitch, this::opaqueZeroOrOneSquareRoot, - this::opaqueZeroOrOneAbsolute + this::opaqueZeroOrOneAbsolute, + this::opaqueZeroOrOneDot, + this::opaqueZeroOrOneBitwiseShift, + this::opaqueZeroOrOneBitwiseOp, + this::opaqueZeroOrOneMatrixDet ); } - private List waysToMakeZero() { + /** + * This method has non-private visibility for purposes of testing only. + */ + List waysToMakeZero() { List opaqueZeroFactories = new ArrayList<>(); opaqueZeroFactories.addAll(waysToMakeZeroOrOne()); opaqueZeroFactories.add(this::opaqueZeroSin); opaqueZeroFactories.add(this::opaqueZeroLogarithm); opaqueZeroFactories.add(this::opaqueZeroTan); opaqueZeroFactories.add(this::opaqueZeroVectorLength); + opaqueZeroFactories.add(this::opaqueZeroVectorCross); return opaqueZeroFactories; } - private List waysToMakeOne() { + /** + * This method has non-private visibility for purposes of testing only. + */ + List waysToMakeOne() { List opaqueOneFactories = new ArrayList<>(); opaqueOneFactories.addAll(waysToMakeZeroOrOne()); opaqueOneFactories.add(this::opaqueOneExponential); opaqueOneFactories.add(this::opaqueOneCosine); - opaqueOneFactories.add(this::opaqueOneNormalizedVectorLength); + if (shadingLanguageVersion.supportedRound()) { + opaqueOneFactories.add(this::opaqueOneRoundedNormalizedVectorLength); + } return opaqueOneFactories; } @@ -155,6 +184,347 @@ private Optional opaqueZeroOrOneAbsolute(BasicType type, boolean constCont depth, fuzzer))); } + /** + * Function to generate an opaque zero or one from the determinant of a generated matrix. + * This function heavily relies on the fact that a triangular matrix - a matrix that only has + * values below the diagonal (lower triangular) or above the diagonal (upper triangular) - always + * has a determinant that is the product of its diagonal values. For example, + * [1 0 0] This is a lower triangular 3x3 matrix. Its determinant is the product of its + * [0 1 0] diagonals, so det(mat3) = 1 * 1 * 1 = 1. + * [1 0 1] + * -------------------------------------- + * [0 1 0 1] This is an upper triangular 4x4 matrix. Its determinant is the product of its + * [0 0 1 1] diagonals, so det(mat4) = 0 * 0 * 0 * 0 = 0. + * [0 0 0 1] + * [0 0 0 0] + * -------------------------------------- + * In general, a triangular matrix is of these forms for our purpose: + * [t x] [t 0] [t x x] [t 0 0] [t x x x] [t 0 0 0] + * [0 t] [x t] [0 t x] [x t 0] [0 t x x] [x t 0 0] + * [0 0 t] [x x t] [0 0 t x] [x x t 0] + * [0 0 0 t] [x x x t] + * where x is a "modifiable index" whose value is totally irrelevant for the determinant, + * and t is a "diagonal" that is zero if we're making an opaque zero, otherwise an opaque one. + * For simplicity, we choose not to consider the case where the diagonal has one zero and + * the rest ones (which would produce a determinant of zero). + * ------------------------------------- + * There are some other things to know about this algorithm: + * [0 1 2 3 ] - When constructing a square matrix in GLSL, the mat constructor takes + * [4 5 6 7 ] matrixDimension * matrixDimension number of arguments - this diagram shows + * [8 9 10 11] how the matrix is constructed from those arguments. + * [12 13 14 15] + * - Notice how in the above diagram, the diagonals for a 4x4 matrix are arguments 0, 5, 10, 15. + * This occurs similarly for 3x3 (0, 4, 8) and 2x2 (0, 3). The sequence of constructor indices + * that are diagonals: [0, (matrixDim + 1), 2 * (matrixDim + 1), 3 * (matrixDim + 1)...]. + * @param type - the base type of the opaque value being created. + * @param constContext - true if we're in a constant expression context, false otherwise. + * @param depth - how deep we are in the expression. + * @param fuzzer - the fuzzer object for generating fuzzed expressions. + * @param isZero - true if we are making an opaque zero, false otherwise. + * @return Optional.empty() if an opaque value can't be generated, otherwise an opaque value + * made from the determinant of a triangular matrix. + */ + private Optional opaqueZeroOrOneMatrixDet(BasicType type, boolean constContext, + final int depth, Fuzzer fuzzer, + boolean isZero) { + // TODO(https://github.com/KhronosGroup/glslang/issues/1865): Workaround for glslangvalidator. + // Remove when #653 is fixed. + if (constContext) { + return Optional.empty(); + } + if (type != BasicType.FLOAT) { + return Optional.empty(); + } + if (!shadingLanguageVersion.supportedDeterminant()) { + return Optional.empty(); + } + // If true, we will make an upper triangular matrix - otherwise it will be lower triangular. + final boolean isUpperTriangular = generator.nextBoolean(); + // Matrices sent to determinant() must be 2x2, 3x3, or 4x4. + final int matrixDimension = generator.nextInt(3) + 2; + // Indices of arguments in a matrix constructor that can be nonzero for triangular matrix, + // excluding the diagonal of the matrix. Zero-indexed. + List modifiableArgs; + switch (matrixDimension) { + case 2: + modifiableArgs = isUpperTriangular + ? Arrays.asList(1) // mat2, upper triangular + : Arrays.asList(2); // mat2, lower triangular + break; + case 3: + modifiableArgs = isUpperTriangular + ? Arrays.asList(1, 2, 5) // mat3, upper triangular + : Arrays.asList(3, 6, 7); // mat3, lower triangular + break; + case 4: + modifiableArgs = isUpperTriangular + ? Arrays.asList(1, 2, 3, 6, 7, 11) // mat4, upper triangular + : Arrays.asList(4, 8, 9, 12, 13, 14); // mat4, lower triangular + break; + default: + // Should not be reachable. + throw new UnsupportedOperationException("Generated matrix dimension was out of bounds?"); + } + final List matrixConstructorArgs = new ArrayList(); + // Diagonal indices are spaced by (matrixDimension + 1) - e.g. mat4 diagonals: 0, 5, 10, 15 + int nextDiagonalMatrixIndex = 0; + for (int i = 0; i < matrixDimension * matrixDimension; i++) { + if (i == nextDiagonalMatrixIndex) { + // We're in a diagonal, so the value depends on if we're making an opaque zero or one. + assert !modifiableArgs.contains(i); + matrixConstructorArgs.add(makeOpaqueZeroOrOne(isZero, type, constContext, depth, fuzzer)); + nextDiagonalMatrixIndex += matrixDimension + 1; + } else if (modifiableArgs.contains(i)) { + // We're in a modifiable index - the value doesn't matter for the determinant. + matrixConstructorArgs.add(makeOpaqueZeroOrOne( + generator.nextBoolean(), type, constContext, depth, fuzzer)); + } else { + // We're in a non-modifiable index - we need zero or we'll violate the triangular + // property of the matrix. + matrixConstructorArgs.add(makeOpaqueZero(type, constContext, depth, fuzzer)); + } + } + assert matrixConstructorArgs.size() == matrixDimension * matrixDimension; + return Optional.of( + new FunctionCallExpr("determinant", + new TypeConstructorExpr( + BasicType.makeMatrixType(matrixDimension, matrixDimension).toString(), + matrixConstructorArgs))); + } + + /** + * Function to generate an opaque zero or one by taking the dot product of two vectors. This + * function depends on two rules of the dot product. Let u and v be vectors of the same width, + * and let i be a valid index into those vectors. + * If u[i] != 0 <==> v[i] = 0 for all i, then dot(u, v) = 0. Example: u = (0, 1), v = (1, 0), + * then dot(u, v) = 1 * 0 + 0 * 1 = 0. + * If u[i] = v[i] = 0 for all i except for a single index, j, and if u[j] = v[j] = 1, then + * dot(u, v) = 0. Example: u = (0, 1), v = (0, 1), then dot(u, v) = 0 * 0 + 1 * 1 = 1. + * Because fuzzed expressions are not always well defined, we forgo those in favor of simply + * using opaque ones and zeroes when generating vectors for this function. + * @param type - the base type of the opaque value being created. + * @param constContext - true if we're in a constant expression context, false otherwise. + * @param depth - how deep we are in the expression. + * @param fuzzer - the fuzzer object for generating fuzzed expressions. + * @param isZero - true if we are making an opaque zero, false otherwise. + * @return Optional.empty() if an opaque value can't be generated, otherwise an opaque value + * made from the dot product of two vectors. + */ + private Optional opaqueZeroOrOneDot(BasicType type, boolean constContext, + final int depth, Fuzzer fuzzer, boolean isZero) { + if (type != BasicType.FLOAT) { + return Optional.empty(); + } + // If width is 1, type will be float - otherwise the type will be the corresponding vector type. + final int vectorWidth = generator.nextPositiveInt(BasicType.VEC4.getNumElements()) + 1; + final Expr dotProductExpr; + final List firstVectorArgs = new ArrayList(); + final List secondVectorArgs = new ArrayList(); + if (isZero) { + // We're basically producing inverse vectors: u[i] = 0 <==> v[i] != 0. + for (int i = 0; i < vectorWidth; i++) { + final boolean firstVectorArgIsZero = generator.nextBoolean(); + firstVectorArgs.add(makeOpaqueZeroOrOne( + firstVectorArgIsZero, type, constContext, depth, fuzzer)); + secondVectorArgs.add(makeOpaqueZeroOrOne( + !firstVectorArgIsZero, type, constContext, depth, fuzzer)); + } + } else { + // The two vectors will be exactly the same - all 0 except for one value, which will be 1. + final int oneIndex = generator.nextInt(vectorWidth); + for (int i = 0; i < vectorWidth; i++) { + firstVectorArgs.add( + makeOpaqueZeroOrOne(i != oneIndex, type, constContext, depth, fuzzer)); + secondVectorArgs.add( + makeOpaqueZeroOrOne(i != oneIndex, type, constContext, depth, fuzzer)); + } + } + assert firstVectorArgs.size() == vectorWidth && secondVectorArgs.size() == vectorWidth; + dotProductExpr = new FunctionCallExpr("dot", + new TypeConstructorExpr( + BasicType.makeVectorType(type, vectorWidth).toString(), firstVectorArgs), + new TypeConstructorExpr( + BasicType.makeVectorType(type, vectorWidth).toString(), secondVectorArgs)); + return Optional.of( + identityConstructor( + dotProductExpr, + applyIdentityFunction(dotProductExpr.clone(), type, constContext, depth, fuzzer))); + } + + /** + * Function to generate an opaque zero or one by bitwise shifting left or right by an amount of + * bits dependent on which opaque is being generated. + * The minimum precision for a lowp integer in GLSL is 9 bits, with one of those bits reserved as + * a sign bit if the integer is signed. The OpenGL specification does not define behavior for + * shifting an integer beyond its maximum size in bits. + * This influences the process of generating opaque values such that: + * If we're generating an opaque zero, we don't have to worry about losing bits, so our + * maximum shift value is 8 bits. + * If we're generating an opaque one, we have to make sure that we don't lose our 1 bit. + * This limits our maximum shift value to 7 bits (more on how we shift an opaque one below). + * Possibilities for generating an opaque zero include: + * Shifting an opaque zero by m bits: (opaque zero) >> n or (opaque zero) << n, where n is + * an integer in [0, 8]. + * Possibilities for generating an opaque zero include: + * Shifting an opaque one to the left by n bits, then shifting it to the right by n bits: + * ((opaque one) << n) >> n, where n is an integer in [0, 7]. + * + * @param type - the base type of the opaque value being created. + * @param constContext - true if we're in a constant expression context, false otherwise. + * @param depth - how deep we are in the expression. + * @param fuzzer - the fuzzer object for generating fuzzed expressions. + * @param isZero - true if we are making an opaque zero, false otherwise. + * @return Optional.empty() if an opaque value can't be generated, otherwise an opaque value + * made from bitwise shifting. + */ + private Optional opaqueZeroOrOneBitwiseShift(BasicType type, boolean constContext, + final int depth, Fuzzer fuzzer, + boolean isZero) { + if (!BasicType.allIntegerTypes().contains(type)) { + return Optional.empty(); + } + if (!shadingLanguageVersion.supportedBitwiseOperations()) { + return Optional.empty(); + } + // The minimum precision for a lowp integer in GLSL is 9 bits (with one reserved for a sign + // bit if the integer is signed) - we don't have to worry about losing information if we're + // shifting zero, but shifting more than 8 bits may result in undefined behavior. + final int minBitsForLowpInt = 9; + // While we still have a hard maximum bits to shift of 8 bits, if we're shifting one, then + // we can potentially lose information because GLSL bit shifting is not circular. To remedy + // that, we make sure not to shift our 1 bit out of the integer by limiting our maximum shift + // to 7 bits. + final int minBitsForLowpUnsignedInt = minBitsForLowpInt - 1; + final int shiftValueConstant = generator.nextInt( + isZero ? minBitsForLowpInt : minBitsForLowpUnsignedInt); + final Expr shiftValueConstructor = + new TypeConstructorExpr(type.toString(), + BasicType.allUnsignedTypes().contains(type) + ? new UIntConstantExpr(String.valueOf(shiftValueConstant) + 'u') + : new IntConstantExpr(String.valueOf(shiftValueConstant))); + final Expr shiftValueWithIdentityApplied = identityConstructor( + shiftValueConstructor, + applyIdentityFunction(shiftValueConstructor, type, constContext, depth, fuzzer)); + if (isZero) { + final BinOp operator = generator.nextBoolean() ? BinOp.SHL : BinOp.SHR; + return Optional.of( + new ParenExpr( + new BinaryExpr( + makeOpaqueZero(type, constContext, depth, fuzzer), + shiftValueWithIdentityApplied, + operator))); + } else { + // We're going to shift twice in opposite directions by the same value. + final Expr backShiftValueWithIdentityApplied = identityConstructor( + shiftValueConstructor.clone(), + applyIdentityFunction(shiftValueConstructor.clone(), type, constContext, depth, fuzzer)); + return Optional.of( + new ParenExpr( + new BinaryExpr( + new ParenExpr( + new BinaryExpr( + makeOpaqueOne(type, constContext, depth, fuzzer), + shiftValueWithIdentityApplied, + BinOp.SHL)), + backShiftValueWithIdentityApplied, + BinOp.SHR))); + } + } + + /** + * Function to generate an opaque value by performing a bitwise operation on an opaque zero or + * an opaque one. + * Possibilities for generating an opaque zero include: + * Bitwise ANDing a fuzzed expression with an opaque zero: (fuzzedexpr) & (opaque zero) + * Bitwise ORing an opaque zero with an opaque zero: (opaque zero) | (opaque zero) + * Bitwise XORing an opaque zero with an opaque zero or an opaque one with an opaque one: + * (opaque zero) ^ (opaque zero) or (opaque one) ^ (opaque one) + * Possibilities for generating an opaque one include: + * Bitwise ANDing an opaque one with an opaque one: (opaque one) & (opaque one) + * Bitwise ORing an opaque one with an opaque zero or one: (opaque one) | (opaque zero or one) + * Bitwise XORing an opaque zero with an opaque one or an opaque one with an opaque zero: + * (opaque zero) ^ (opaque one) or (opaque one) ^ (opaque zero) + * + * @param type - the base type of the opaque value being created. + * @param constContext - true if we're in a constant expression context, false otherwise. + * @param depth - how deep we are in the expression. + * @param fuzzer - the fuzzer object for generating fuzzed expressions. + * @param isZero - true if we are making an opaque zero, false otherwise. + * @return Optional.empty() if an opaque value can't be generated, otherwise an opaque value + * made from performing a bitwise operation on an opaque zero or opaque one. + */ + private Optional opaqueZeroOrOneBitwiseOp(BasicType type, boolean constContext, + final int depth, Fuzzer fuzzer, boolean isZero) { + if (!BasicType.allIntegerTypes().contains(type)) { + return Optional.empty(); + } + if (!shadingLanguageVersion.supportedBitwiseOperations()) { + return Optional.empty(); + } + final int numPossibleOperators = 3; + final int operator = generator.nextInt(numPossibleOperators); + Optional opaqueExpr; + switch (operator) { + case 0: + // Bitwise AND + if (isZero) { + // We pass true as constContext when fuzzing here because the expression will be + // evaluated, so we don't want any side effects. + final Expr fuzzedExpr = fuzzer.fuzzExpr(type, false, true, depth); + final Expr opaqueZero = makeOpaqueZero(type, constContext, depth, fuzzer); + opaqueExpr = Optional.of( + new ParenExpr( + generator.nextBoolean() + ? new BinaryExpr(fuzzedExpr, opaqueZero, BinOp.BAND) + : new BinaryExpr(opaqueZero, fuzzedExpr, BinOp.BAND))); + } else { + opaqueExpr = Optional.of( + new ParenExpr( + new BinaryExpr( + makeOpaqueOne(type, constContext, depth, fuzzer), + makeOpaqueOne(type, constContext, depth, fuzzer), + BinOp.BAND))); + } + break; + case 1: + // Bitwise OR + if (isZero) { + opaqueExpr = Optional.of( + new ParenExpr( + new BinaryExpr( + makeOpaqueZero(type, constContext, depth, fuzzer), + makeOpaqueZero(type, constContext, depth, fuzzer), + BinOp.BOR))); + } else { + final Expr opaqueOne = makeOpaqueOne(type, constContext, depth, fuzzer); + final Expr opaqueZeroOrOne = makeOpaqueZeroOrOne(generator.nextBoolean(), type, + constContext, depth, fuzzer); + opaqueExpr = Optional.of( + new ParenExpr( + generator.nextBoolean() + ? new BinaryExpr(opaqueOne, opaqueZeroOrOne, BinOp.BOR) + : new BinaryExpr(opaqueZeroOrOne, opaqueOne, BinOp.BOR))); + } + break; + default: + // Bitwise XOR + assert operator == numPossibleOperators - 1; + boolean useZeroOrOne = generator.nextBoolean(); + opaqueExpr = Optional.of( + new ParenExpr( + isZero + ? new BinaryExpr( + makeOpaqueZeroOrOne(useZeroOrOne, type, constContext, depth, fuzzer), + makeOpaqueZeroOrOne(useZeroOrOne, type, constContext, depth, fuzzer), + BinOp.BXOR) + : new BinaryExpr( + makeOpaqueZeroOrOne(useZeroOrOne, type, constContext, depth, fuzzer), + makeOpaqueZeroOrOne(!useZeroOrOne, type, constContext, depth, fuzzer), + BinOp.BXOR))); + } + return opaqueExpr; + } + private Optional opaqueZeroSin(BasicType type, boolean constContext, final int depth, Fuzzer fuzzer, boolean isZero) { // represent 0 as sin(opaqueZero) function, e.g. sin(0.0) @@ -193,13 +563,69 @@ private Optional opaqueZeroVectorLength(BasicType type, boolean constConte Fuzzer fuzzer, boolean isZero) { // represent 0 as the length of zero vector, e.g. length(opaqueZero). assert isZero; - if (!BasicType.allGenTypes().contains(type)) { + if (type != BasicType.FLOAT) { + // 'length' has return type 'float', so we can only create a scalar floating-point zero. return Optional.empty(); } - return Optional.of(new FunctionCallExpr("length", makeOpaqueZero(type, constContext, depth, + + // We can choose any vector size we wish for the vector whose length will be computed. + final BasicType vectorType = + BasicType.allGenTypes().get(generator.nextInt(BasicType.allGenTypes().size())); + + return Optional.of(new FunctionCallExpr("length", makeOpaqueZero(vectorType, constContext, + depth, fuzzer))); } + /** + * Function to create an opaque zero (specifically an opaque zero vec3) by taking the cross + * product of two equivalent vec3s. This opaque function relies on the fact that if u == v, where + * u and v are vec3, then cross(u, v) = (0, 0, 0). These equivalent vec3s are produced by + * fuzzing random float values and applying different identity functions to them per vector. + * @param type - the base type of the opaque value being created. + * @param constContext - true if we're in a constant expression context, false otherwise. + * @param depth - how deep we are in the expression. + * @param fuzzer - the fuzzer object for generating fuzzed expressions. + * @param isZero - true if we are making an opaque zero, false otherwise. + * @return Optional.empty() if an opaque value can't be generated, otherwise an opaque zero vec3 + * made from the cross product of two equal vec3s. + */ + private Optional opaqueZeroVectorCross(BasicType type, boolean constContext, + final int depth, + Fuzzer fuzzer, boolean isZero) { + assert isZero; + if (type != BasicType.VEC3) { + // Cross product only works on vec3 and can only produce vec3 as a return type. + return Optional.empty(); + } + // We only want literals, so we need to make our own LiteralFuzzer rather than use the supplied + // Fuzzer object. + final LiteralFuzzer litFuzzer = new LiteralFuzzer(generator); + // cross(vec3(firstVec3ConstructorArgs), vec3(secondVec3ConstructorArgs)) + final List firstVec3ConstructorArgs = new ArrayList(); + final List secondVec3ConstructorArgs = new ArrayList(); + for (int i = 0; i < type.getNumElements(); i++) { + final Optional maybeFuzzedFloatLiteral = litFuzzer.fuzz(type.getElementType()); + // Something went horribly wrong if we got an Optional.empty() - we are guaranteed to obtain a + // fuzzed expression, since the element type of a vec3 is a float, and it is always possible + // to fuzz a float literal. + assert maybeFuzzedFloatLiteral.isPresent(); + final Expr fuzzedFloatLiteral = maybeFuzzedFloatLiteral.get(); + // We apply different identities to the literals per vector - the two vectors are + // semantically the same vector (and so cross() should still return the zero vector). + firstVec3ConstructorArgs.add( + applyIdentityFunction( + fuzzedFloatLiteral, type.getElementType(), constContext, depth, fuzzer)); + secondVec3ConstructorArgs.add( + applyIdentityFunction( + fuzzedFloatLiteral.clone(), type.getElementType(), constContext, depth, fuzzer)); + } + return Optional.of( + new FunctionCallExpr("cross", + new TypeConstructorExpr(type.toString(), firstVec3ConstructorArgs), + new TypeConstructorExpr(type.toString(), secondVec3ConstructorArgs))); + } + private Optional opaqueOneExponential(BasicType type, boolean constContext, final int depth, Fuzzer fuzzer, boolean isZero) { // represent 1 as the exponential function of opaqueZero, e.g. exp(0.0) @@ -222,17 +648,29 @@ private Optional opaqueOneCosine(BasicType type, boolean constContext, fin fuzzer))); } - private Optional opaqueOneNormalizedVectorLength(BasicType type, boolean constContext, - final int depth, - Fuzzer fuzzer, boolean isZero) { - // represent 1 as the length of normalized vector + private Optional opaqueOneRoundedNormalizedVectorLength(BasicType type, + boolean constContext, + final int depth, + Fuzzer fuzzer, + boolean isZero) { + // represent 1 as the rounded length of a normalized vector assert !isZero; - if (!BasicType.allGenTypes().contains(type)) { + if (type != BasicType.FLOAT) { + // 'length' has return type 'float', so we can only create a scalar floating-point zero. return Optional.empty(); } - Expr normalizedExpr = new FunctionCallExpr("normalize", makeOpaqueZeroOrOne(isZero, - type, constContext, depth, fuzzer)); - return Optional.of(new FunctionCallExpr("length", normalizedExpr)); + + // We can choose any vector size we wish for the vector whose length will be computed. + final BasicType vectorType = + BasicType.allGenTypes().get(generator.nextInt(BasicType.allGenTypes().size())); + + // We create a vector of ones and normalize it, rounding the result to guard against the case + // where round-off leads to a result that is not quite one. Note that we could be more general + // here and normalize any non-zero vector. + return Optional.of(new FunctionCallExpr("round", + new FunctionCallExpr("length", + new FunctionCallExpr("normalize", + makeOpaqueZeroOrOne(false, vectorType, constContext, depth, fuzzer))))); } private List numericTypesOrGenTypesIfEssl100() { @@ -251,7 +689,7 @@ public Expr applyIdentityFunction(Expr expr, BasicType type, boolean constContex } final List availableTransformations = expressionIdentities.stream() - .filter(item -> item.preconditionHolds(expr, type)) + .filter(item -> item.preconditionHolds(expr, type, constContext)) .collect(Collectors.toList()); return availableTransformations.isEmpty() ? expr : availableTransformations.get(generator.nextInt(availableTransformations.size())) @@ -530,14 +968,15 @@ private abstract class AbstractIdentityTransformation implements ExpressionIdent } @Override - public final boolean preconditionHolds(Expr expr, BasicType basicType) { + public boolean preconditionHolds(Expr expr, BasicType basicType, boolean constContext) { if (!acceptableTypes.contains(basicType)) { return false; } if (!exprMustBeSideEffectFree) { return true; } - return SideEffectChecker.isSideEffectFree(expr, shadingLanguageVersion); + return SideEffectChecker.isSideEffectFree(expr, shadingLanguageVersion, + generationParams.getShaderKind()); } @@ -636,9 +1075,9 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, } - private class IdentityNotNot extends AbstractIdentityTransformation { + private class IdentityLogicalNotNot extends AbstractIdentityTransformation { - private IdentityNotNot() { + private IdentityLogicalNotNot() { super(Arrays.asList(BasicType.BOOL), false); } @@ -663,7 +1102,7 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, private class IdentityTernary extends AbstractIdentityTransformation { private IdentityTernary() { - super(BasicType.allScalarTypes(), false); + super(BasicType.allNumericTypes(), false); } @Override @@ -672,8 +1111,7 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, // (false ? whatever : expr) // or // (true ? expr : whatever) - assert BasicType.allScalarTypes().contains(type); - // Only generate ternary expressions for scalars; the vector case causes massive blow-up + assert BasicType.allNumericTypes().contains(type); Expr exprWithIdentityApplied = applyIdentityFunction(expr, type, constContext, depth, fuzzer); Expr something = fuzzedConstructor(fuzzer.fuzzExpr(type, false, constContext, depth)); if (generator.nextBoolean()) { @@ -690,6 +1128,124 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, } + /** + * Identity transformation for integer types (both unsigned and signed, and their vectors) that + * double bitwise inverts an integer, producing the same integer as output. When performed, + * transforms an expression, e, such that: + * e -> ~(~(expr)). + */ + private class IdentityBitwiseNotNot extends AbstractIdentityTransformation { + private IdentityBitwiseNotNot() { + super(BasicType.allIntegerTypes(), false); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert BasicType.allIntegerTypes().contains(type); + // Invert once + Expr result = new UnaryExpr(new ParenExpr(applyIdentityFunction(expr, type, + constContext, + depth, fuzzer)), UnOp.BNEG); + // Invert again + result = new UnaryExpr(new ParenExpr(applyIdentityFunction(result, type, constContext, + depth, fuzzer)), UnOp.BNEG); + return identityConstructor(expr, result); + } + } + + /** + * Identity transformation for integer types (both unsigned and signed, and their vectors) that + * ORs an integer with itself, producing the same integer as output. This identity requires + * expressions to be side effect free because the same mutated expression is evaluated twice. + * When performed, transforms an expression, e, such that: + * e -> (e) | (e). + */ + private class IdentityBitwiseOrSelf extends AbstractIdentityTransformation { + private IdentityBitwiseOrSelf() { + super(BasicType.allIntegerTypes(), true); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert BasicType.allIntegerTypes().contains(type); + // We use parentheses to prevent issues with order of operations in ternary expressions. + return identityConstructor( + expr, + new BinaryExpr( + new ParenExpr(applyIdentityFunction(expr.clone(), type, constContext, depth, fuzzer)), + new ParenExpr(applyIdentityFunction(expr.clone(), type, constContext, depth, fuzzer)), + BinOp.BOR)); + } + } + + /** + * Identity transformation for integer types (both unsigned and signed, and their vectors) that + * ORs an integer with zero, producing the same integer as output. + * When performed, transforms an expression, e, such that: + * e -> (e) | (opaque 0) or e -> (opaque 0) | (e). + */ + private class IdentityBitwiseOrZero extends AbstractIdentityTransformation { + private IdentityBitwiseOrZero() { + super(BasicType.allIntegerTypes(), false); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert BasicType.allIntegerTypes().contains(type); + return applyBinaryIdentityFunction( + expr.clone(), + makeOpaqueZero(type, constContext, depth, fuzzer), + BinOp.BOR, true, type, constContext, depth, fuzzer); + } + } + + /** + * Identity transformation for integer types (both unsigned and signed, and their vectors) that + * XORs an integer with zero, producing the same integer as output. When performed, transforms an + * expression, e, such that: + * e -> (e) ^ (opaque 0) or e -> (opaque 0) ^ (e). + */ + private class IdentityBitwiseXorZero extends AbstractIdentityTransformation { + private IdentityBitwiseXorZero() { + super(BasicType.allIntegerTypes(), false); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert BasicType.allIntegerTypes().contains(type); + return applyBinaryIdentityFunction( + expr.clone(), + makeOpaqueZero(type, constContext, depth, fuzzer), + BinOp.BXOR, true, type, constContext, depth, fuzzer); + } + } + + /** + * Identity transformation for integer types (both unsigned and signed, and their vectors) that + * shifts an integer by zero. When performed, transforms an expression, e, such that: + * e -> (e) >> (opaque 0) or e -> (e) << (opaque 0) + */ + private class IdentityBitwiseShiftZero extends AbstractIdentityTransformation { + private IdentityBitwiseShiftZero() { + super(BasicType.allIntegerTypes(), false); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert BasicType.allIntegerTypes().contains(type); + final BinOp operator = generator.nextBoolean() ? BinOp.SHL : BinOp.SHR; + return applyBinaryIdentityFunction( + expr.clone(), + makeOpaqueZero(type, constContext, depth, fuzzer), + operator, false, type, constContext, depth, fuzzer); + } + } + private class IdentityMin extends AbstractIdentityTransformation { private IdentityMin() { @@ -802,4 +1358,242 @@ public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, } + private class IdentityRewriteComposite extends AbstractIdentityTransformation { + private IdentityRewriteComposite() { + // all non-boolean vector/matrix types + super(BasicType.allNumericTypes().stream().filter( + item -> !item.isScalar()).collect(Collectors.toList()), + false); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + // v -> (true ? vecX(..., identity(v[Y]), ...) : _) + // v -> (false ? _ : vecX(..., identity(v[Y]), ...)) + // where v is a vector of size X, y is the random entry in v we want to apply + // identities to, and ... is the other entries in v that we don't change. + // Similarly for matrices. + + assert type.isVector() || type.isMatrix(); + assert expr instanceof VariableIdentifierExpr; + + final int numColumns = type.isVector() ? type.getNumElements() : type.getNumColumns(); + final int columnToFurtherTransform = generator.nextInt(numColumns); + final List typeConstructorArguments = new ArrayList<>(); + + if (type.isMatrix() && generator.nextBoolean()) { + // Further break down matrix constructor into single components, to increase diversity. + // For example, a 2x2 matrix may become mat2(m[0][0], m[0][1], IDENTITY(m[1][0]), m[1][1]). + // GLSL's matrix notation follows column-major indexing rules. + final int rowToFurtherTransform = generator.nextInt(type.getNumRows()); + for (int i = 0; i < numColumns; i++) { + for (int j = 0; j < type.getNumRows(); j++) { + // The inner ArrayIndexExpr is the column (first) index, while the outer ArrayIndexExpr + // is the row (second) index. + Expr argument = new ArrayIndexExpr(new ArrayIndexExpr(expr.clone(), + new IntConstantExpr(String.valueOf(i)).clone()), + new IntConstantExpr(String.valueOf(j))); + if (i == columnToFurtherTransform && j == rowToFurtherTransform) { + argument = applyIdentityFunction(argument, type.getElementType(), constContext, + depth, fuzzer); + } + typeConstructorArguments.add(argument); + } + } + } else { + // Create a constructor with columns only (or single entries, in the case of vectors). + // A 2x2 matrix may become mat2(m[0], IDENTITY(m[1])). + // A vec4 may become vec4(v[0], v[1], IDENTITY(v[2]), v[3]). + for (int i = 0; i < numColumns; i++) { + Expr argument = new ArrayIndexExpr(expr.clone(), new IntConstantExpr(String.valueOf(i))); + if (i == columnToFurtherTransform) { + argument = applyIdentityFunction(argument, + type.isVector() ? type.getElementType() : type.getColumnType(), + constContext, depth, fuzzer); + } + typeConstructorArguments.add(argument); + } + } + return identityConstructor(expr, + new TypeConstructorExpr(type.toString(), typeConstructorArguments)); + } + + @Override + public boolean preconditionHolds(Expr expr, BasicType basicType, boolean constContext) { + return super.preconditionHolds(expr, basicType, constContext) + && expr instanceof VariableIdentifierExpr; + } + } + + /** + * Identity transformation to insert an expression into a wider vector or matrix using a + * type constructor, then extract it back out again using another type constructor. When + * performed, transforms an expression e of type t to identity(t(identity(m(identity(e), ..)))), + * where m is a type of equal or greater width than t. + * The rules for this smaller -> larger -> smaller transformation are as follows: + * If the given type is a vector or scalar, the inner constructor type can be a vector of the + * same element type with equal/greater width than the given type, or (provided that the + * given type is a floating point genType) it can be a matrix with equal/greater column + * width than the given type. + * If the given type is a matrix, the inner constructor type can be a matrix with + * equal/greater column/row width than the given type. + */ + private class IdentityCompositeConstructorExpansion extends AbstractIdentityTransformation { + private IdentityCompositeConstructorExpansion() { + super(BasicType.allBasicTypes().stream() + .filter(item -> !Arrays.asList(BasicType.BVEC4, BasicType.IVEC4, BasicType.UVEC4, + BasicType.MAT4X4) + .contains(item)) + .collect(Collectors.toList()), true); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert !Arrays.asList(BasicType.BVEC4, BasicType.IVEC4, BasicType.UVEC4, BasicType.MAT4X4) + .contains(type); + final List innerConstructorTypes = new ArrayList(); + // Our inner constructor type will be equal or larger in width than our given type. The goal + // of the next set of conditionals is to populate this list with all types that can fit our + // given type. + final int maxVectorSize = 4; + if (!type.isMatrix()) { + for (int i = type.getNumElements(); i <= maxVectorSize; i++) { + innerConstructorTypes.add(BasicType.makeVectorType(type.getElementType(), i)); + } + if (type.getElementType() == BasicType.FLOAT) { + innerConstructorTypes.addAll(BasicType.allSquareMatrixTypes()); + if (shadingLanguageVersion.supportedNonSquareMatrices()) { + innerConstructorTypes.addAll(BasicType.allNonSquareMatrixTypes()); + } + } + } else { + for (BasicType constructorType : BasicType.allMatrixTypes()) { + if (type.getNumRows() <= constructorType.getNumRows() + && type.getNumColumns() <= constructorType.getNumColumns()) { + if (BasicType.allSquareMatrixTypes().contains(constructorType)) { + innerConstructorTypes.add(constructorType); + } else if (shadingLanguageVersion.supportedNonSquareMatrices()) { + innerConstructorTypes.add(constructorType); + } + } + } + } + assert !innerConstructorTypes.isEmpty(); + assert innerConstructorTypes.contains(type); + final BasicType randomInnerConstructorType = + innerConstructorTypes.get(generator.nextInt(innerConstructorTypes.size())); + final List innerConstructorArgs = new ArrayList(); + innerConstructorArgs.add( + applyIdentityFunction(expr.clone(), type, constContext, depth, fuzzer)); + // GLSL won't fill in the blanks of the inner constructor unless a matrix is being constructed + // from another matrix. + if (!type.isMatrix()) { + final int numExcessConstructorArgs = + randomInnerConstructorType.getNumElements() - type.getNumElements(); + for (int i = 0; i < numExcessConstructorArgs; i++) { + // We add a boolean if the element type is boolean, and a zero/one otherwise. + innerConstructorArgs.add( + type.getElementType() == BasicType.BOOL + ? makeOpaqueBoolean(generator.nextBoolean(), + BasicType.BOOL, + constContext, + depth, + fuzzer) + : makeOpaqueZeroOrOne(generator.nextBoolean(), + type.getElementType(), + constContext, + depth, + fuzzer)); + } + } + return identityConstructor( + expr, + applyIdentityFunction( + new TypeConstructorExpr( + type.toString(), + applyIdentityFunction( + new TypeConstructorExpr( + randomInnerConstructorType.toString(), + innerConstructorArgs), + randomInnerConstructorType, constContext, depth, fuzzer)), + type, constContext, depth, fuzzer)); + } + } + + /** + * Identity transformation to transpose a matrix twice. When performed, transforms an expression + * of a matrix m -> transpose(identity(transpose(identity(m)))). + */ + private class IdentityDoubleTranspose extends AbstractIdentityTransformation { + private IdentityDoubleTranspose() { + super(BasicType.allMatrixTypes(), true); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, + Fuzzer fuzzer) { + assert type.isMatrix(); + return identityConstructor( + expr, + new FunctionCallExpr("transpose", + applyIdentityFunction( + new FunctionCallExpr("transpose", + applyIdentityFunction(expr.clone(), type, constContext, depth, fuzzer)), + type.transposedMatrixType(), constContext, depth, fuzzer))); + } + + @Override + public boolean preconditionHolds(Expr expr, BasicType basicType, boolean constContext) { + // TODO(https://github.com/KhronosGroup/glslang/issues/1865): Workaround for glslangValidator + // issue, remove constContext check when fixed. + return super.preconditionHolds(expr, basicType, constContext) + && !constContext; + } + } + + /** + * Identity transformation to multiply a vector or a rectangular matrix P by an identity matrix I, + * producing the same vector/matrix P as output. This relies on two properties of the identity + * matrix: + * If P is a matrix of size n x m, where n is the number of columns and m the number of rows, + * and I is the identity matrix of size n x n, then P * I = P. + * If P is a matrix of size n x m, where n is the number of columns and m the number of rows, + * and I is the identity matrix of size m x m, then I * P = P. + * Note that matrices of size 1 x m or n x 1 are vecm or vecn, respectively. + * When performed, transforms an expression of a vector or rectangular matrix P -> + * identity(identityMatrix * P) or identity(P * identityMatrix). + */ + private class IdentityMatrixMultIdentity extends AbstractIdentityTransformation { + private IdentityMatrixMultIdentity() { + super(Stream.concat(BasicType.allNonSquareMatrixTypes().stream(), + Stream.of(BasicType.VEC2, BasicType.VEC3, BasicType.VEC4)) + .collect(Collectors.toList()), true); + } + + @Override + public Expr apply(Expr expr, BasicType type, boolean constContext, int depth, Fuzzer fuzzer) { + // We use isVector()/isMatrix() in this function for brevity and self-documentation instead of + // concatenating streams. + assert type.isVector() || type.isMatrix(); + final boolean usingRightMultiplication = generator.nextBoolean(); + final int identityMatrixSize = + type.isVector() + ? type.getNumElements() + : usingRightMultiplication + ? type.getNumRows() + : type.getNumColumns(); + final Expr identityMatrix = makeOpaqueIdentityMatrix( + BasicType.makeMatrixType(identityMatrixSize, identityMatrixSize), + constContext, depth, fuzzer); + // We wrap the original expr in parentheses to prevent issues with ternary operators. + final Expr binaryMultExpr = usingRightMultiplication + ? new BinaryExpr(identityMatrix, new ParenExpr(expr.clone()), BinOp.MUL) + : new BinaryExpr(new ParenExpr(expr.clone()), identityMatrix, BinOp.MUL); + return identityConstructor( + expr, + applyIdentityFunction(binaryMultExpr, type, constContext, depth, fuzzer)); + } + } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Templates.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Templates.java index a00ab5fc2..5eccce3e5 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Templates.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/Templates.java @@ -24,6 +24,7 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.typing.SupportedTypes; import com.graphicsfuzz.common.typing.TyperHelper; +import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.generator.fuzzer.templates.BinaryExprTemplate; import com.graphicsfuzz.generator.fuzzer.templates.ConstantExprTemplate; import com.graphicsfuzz.generator.fuzzer.templates.FunctionCallExprTemplate; @@ -46,21 +47,28 @@ public class Templates { - private static ConcurrentMap> templates + private static ConcurrentMap>> templates = new ConcurrentHashMap<>(); private Templates() { // Utility class } - public static List get(ShadingLanguageVersion shadingLanguageVersion) { + public static List get(ShadingLanguageVersion shadingLanguageVersion, + ShaderKind shaderKind) { if (!templates.containsKey(shadingLanguageVersion)) { - templates.putIfAbsent(shadingLanguageVersion, makeTemplates(shadingLanguageVersion)); + templates.putIfAbsent(shadingLanguageVersion, new ConcurrentHashMap<>()); } - return Collections.unmodifiableList(templates.get(shadingLanguageVersion)); + if (!templates.get(shadingLanguageVersion).containsKey(shaderKind)) { + templates.get(shadingLanguageVersion).putIfAbsent(shaderKind, + makeTemplates(shadingLanguageVersion, shaderKind)); + } + return Collections.unmodifiableList(templates.get(shadingLanguageVersion).get(shaderKind)); } - public static List makeTemplates(ShadingLanguageVersion shadingLanguageVersion) { + private static List makeTemplates(ShadingLanguageVersion shadingLanguageVersion, + ShaderKind shaderKind) { // TODO: assignment operators, array, vector and matrix lookups @@ -69,7 +77,7 @@ public static List makeTemplates(ShadingLanguageVersion shadingLa // Builtins { Map> builtins = TyperHelper.getBuiltins( - shadingLanguageVersion); + shadingLanguageVersion, shaderKind); List keys = builtins.keySet().stream().collect(Collectors.toList()); keys.sort(String::compareTo); for (String key : keys) { diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/FunctionCallExprTemplate.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/FunctionCallExprTemplate.java index f4952d1f3..6a5ee200b 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/FunctionCallExprTemplate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/FunctionCallExprTemplate.java @@ -21,6 +21,7 @@ import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.util.IRandom; import java.util.ArrayList; import java.util.Arrays; @@ -62,7 +63,8 @@ public List> getArgumentTypes() { @Override public boolean requiresLValueForArgument(int index) { assert index >= 0 && index < getNumArguments(); - return false; + return argTypes.get(index).hasQualifier(TypeQualifier.OUT_PARAM) + || argTypes.get(index).hasQualifier(TypeQualifier.INOUT_PARAM); } @Override diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/SwizzleExprTemplate.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/SwizzleExprTemplate.java index 82c29dc60..cab95ee32 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/SwizzleExprTemplate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/SwizzleExprTemplate.java @@ -32,9 +32,8 @@ public class SwizzleExprTemplate extends AbstractExprTemplate { private final boolean isLValue; public SwizzleExprTemplate(BasicType argType, BasicType resultType, boolean isLValue) { - assert BasicType.allVectorTypes().contains(argType); - assert BasicType.allVectorTypes().contains(resultType) || BasicType.allScalarTypes() - .contains(resultType); + assert argType.isVector(); + assert resultType.isVector() || resultType.isScalar(); if (isLValue) { assert resultType.getNumElements() <= argType.getNumElements(); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/VectorMatrixIndexExprTemplate.java b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/VectorMatrixIndexExprTemplate.java index 314e0fe2d..427884913 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/VectorMatrixIndexExprTemplate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/fuzzer/templates/VectorMatrixIndexExprTemplate.java @@ -33,8 +33,7 @@ public class VectorMatrixIndexExprTemplate extends AbstractExprTemplate { private final boolean isLValue; public VectorMatrixIndexExprTemplate(BasicType argType, boolean isLValue) { - assert BasicType.allVectorTypes().contains(argType) || BasicType.allMatrixTypes() - .contains(argType); + assert argType.isVector() || argType.isMatrix(); this.argType = argType; this.isLValue = isLValue; } @@ -44,11 +43,11 @@ public Expr generateExpr(IRandom generator, Expr... args) { assert args.length == getNumArguments(); int index; - if (BasicType.allVectorTypes().contains(argType)) { + if (argType.isVector()) { index = generator.nextInt(argType.getNumElements()); } else { - assert BasicType.allMatrixTypes().contains(argType); - index = generator.nextInt(BasicType.numColumns(argType)); + assert argType.isMatrix(); + index = generator.nextInt(argType.getNumColumns()); } return new ArrayIndexExpr(args[0], new IntConstantExpr(String.valueOf(index))); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/BooleanValue.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/BooleanValue.java new file mode 100644 index 000000000..98f6a03a5 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/BooleanValue.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.expr.BoolConstantExpr; +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.generator.semanticschanging.LiteralFuzzer; +import java.util.Objects; +import java.util.Optional; + +public class BooleanValue implements Value { + + private final Optional value; + + public BooleanValue(Optional value) { + this.value = value; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof BooleanValue)) { + return false; + } + + final BooleanValue thatBooleanValue = (BooleanValue) that; + return this.value.equals(thatBooleanValue.value); + } + + @Override + public int hashCode() { + return Objects.hash(getType(), value); + } + + @Override + public Type getType() { + return BasicType.BOOL; + } + + @Override + public boolean valueIsUnknown() { + return !value.isPresent(); + } + + public Optional getValue() { + return value; + } + + @Override + public Expr generateLiteral(LiteralFuzzer literalFuzzer) { + if (valueIsUnknown()) { + return literalFuzzer.fuzz(BasicType.BOOL).orElse(null); + } + return new BoolConstantExpr(value.get()); + } + + @Override + public String toString() { + return valueIsUnknown() ? "unknown_bool" : value.get().toString(); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/CompositeValue.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/CompositeValue.java new file mode 100644 index 000000000..06e7ff508 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/CompositeValue.java @@ -0,0 +1,98 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.TypeConstructorExpr; +import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.generator.semanticschanging.LiteralFuzzer; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +public class CompositeValue implements Value { + + private final Type type; + private final Optional> valueList; + + public CompositeValue(Type type, Optional> valueList) { + this.type = type; + this.valueList = valueList; + } + + public Optional> getValueList() { + return valueList; + } + + @Override + public Type getType() { + return type; + } + + @Override + public boolean valueIsUnknown() { + return !valueList.isPresent(); + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof CompositeValue)) { + return false; + } + + final CompositeValue thatCompositeValue = (CompositeValue) that; + return this.getType().equals(thatCompositeValue.getType()) + && this.getValueList().equals(thatCompositeValue.getValueList()); + } + + @Override + public int hashCode() { + return Objects.hash(type, valueList); + } + + @Override + public Expr generateLiteral(LiteralFuzzer literalFuzzer) { + if (valueIsUnknown()) { + return literalFuzzer.fuzz(type).orElse(null); + } + + if (type instanceof BasicType) { + final List args = valueList + .get() + .stream() + .map(item -> item.generateLiteral(literalFuzzer)) + .collect(Collectors.toList()); + return new TypeConstructorExpr(type.toString(), args); + } + + // TODO(https://github.com/google/graphicsfuzz/issues/664): we should also support array and + // struct types. + throw new RuntimeException("The given type is not supported"); + } + + @Override + public String toString() { + return valueIsUnknown() ? "unknown_composite" : valueList.get().toString(); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java new file mode 100644 index 000000000..846cc2e38 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ExpressionGenerator.java @@ -0,0 +1,795 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.Declaration; +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; +import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; +import com.graphicsfuzz.common.ast.decl.ParameterDecl; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.expr.BinOp; +import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; +import com.graphicsfuzz.common.ast.expr.UnOp; +import com.graphicsfuzz.common.ast.expr.UnaryExpr; +import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.stmt.NullStmt; +import com.graphicsfuzz.common.ast.stmt.ReturnStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.QualifiedType; +import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.common.ast.type.TypeQualifier; +import com.graphicsfuzz.common.ast.type.VoidType; +import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.PipelineInfo; +import com.graphicsfuzz.generator.semanticschanging.LiteralFuzzer; +import com.graphicsfuzz.util.Constants; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +public class ExpressionGenerator { + + private static final int MAX_FUNCTION_PARAMS = 5; + private static final int NUM_WAYS_TO_GENERATE_EXPR = 8; + private static final int MAX_DEPTH = 5; + // Theses boundaries are taken from the LiteralFuzzer, we may want to consider + // modifying these values. + private static final int INT_MIN = 0; + private static final int INT_MAX = 1 << 17; + + private static final String NEGATIVE = "_negative"; + private static final String ID = "_id"; + + private final TranslationUnit translationUnit; + private final PipelineInfo pipelineInfo; + private final IdGenerator idGenerator; + private final IRandom generator; + private int currentDepth; + private final LiteralFuzzer literalFuzzer; + private final FactManager globalFactManager; + + public ExpressionGenerator(TranslationUnit translationUnit, PipelineInfo pipelineInfo, + IRandom generator, + FactManager globalFactManager) { + this.translationUnit = translationUnit; + this.pipelineInfo = pipelineInfo; + this.idGenerator = new IdGenerator(); + this.generator = generator; + this.currentDepth = 0; + this.literalFuzzer = new LiteralFuzzer(this.generator); + this.globalFactManager = globalFactManager; + } + + private Expr generateLiteralValue(Value value) { + return value.generateLiteral(literalFuzzer); + } + + /** + * This function applies various transformations and returns a expression that guarantees to + * compute the given value. If the value is unknown, fact manager would generate any + * arbitrary values that have the correct type. + * + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param functionDefinition a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before. + * @param value the value that will be computed by the expression generated by + * this method. + * @return the expression whose value is known by the fact manager. + */ + public Expr generateExpr(FactManager factManager, + FunctionDefinition functionDefinition, + Stmt stmtToInsertBefore, + Value value) { + if (currentDepth > MAX_DEPTH) { + // When current depth has reached the maximum limit, we don't want generation to go deeper. + // Thus, we consider generating a new expression from non-recursive approaches only. + // To do so, we will choose to either generating the expression by making the literal + // value or calling the already declared variables retrieved from the fact manager. + Expr result; + do { + if (generator.nextBoolean()) { + result = generateLiteralValue(value); + } else { + result = generateKnownVariableFact(value, factManager); + } + } while (result == null); + return result; + } + + currentDepth++; + Expr result; + + while (true) { + switch (generator.nextInt(NUM_WAYS_TO_GENERATE_EXPR)) { + case 0: + result = generateLiteralValue(value); + break; + case 1: + result = generateVariableFact(value, factManager, functionDefinition, stmtToInsertBefore); + break; + case 2: + result = generateKnownVariableFact(value, factManager); + break; + case 3: + result = generateFunctionFact(value, factManager, functionDefinition, stmtToInsertBefore); + break; + case 4: + result = generateKnownFunctionFact(factManager, value, functionDefinition, + stmtToInsertBefore); + break; + case 5: + result = generateAdditionValue(value, factManager, functionDefinition, + stmtToInsertBefore); + break; + case 6: + result = generateForLoopValue(value, factManager, functionDefinition, stmtToInsertBefore); + break; + case 7: + result = generateUniformValue(value); + break; + default: + throw new RuntimeException("Should be unreachable as switch cases cover all cases"); + } + if (result != null) { + break; + } + } + currentDepth--; + return result; + } + + + private Expr generateUniformValue(Value value) { + if (!(value instanceof NumericValue) && !(value instanceof CompositeValue)) { + return null; + } + // TODO(https://github.com/google/graphicsfuzz/issues/692): we would be able to support + // matrix uniform once Amber has solved the matrix-related issues. + if (BasicType.allMatrixTypes().contains(value.getType())) { + return null; + } + final String uniformName = Constants.GLF_UNIFORM + "_" + + value.getType().toString() + + parseNameFromValue(value) + + freshId(); + + // We must assign a random value if the given value is empty since we are not allowed to + // inject unknown uniform variables. + if (value.valueIsUnknown()) { + value = fuzzValue(value.getType(), true); + } + + final List uniformValues = new ArrayList<>(); + if (value instanceof CompositeValue) { + CompositeValue compositeValue = (CompositeValue) value; + for (Value element : compositeValue.getValueList().get()) { + // It's also possible that an element of the given composite value is unknown, thus we have + // to initialize the element if it is empty. + final NumericValue elementValue = element.valueIsUnknown() + ? (NumericValue) fuzzValue(element.getType(), true) + : (NumericValue) element; + uniformValues.add(elementValue.getValue().get()); + } + } else { + assert value instanceof NumericValue; + uniformValues.add(((NumericValue) value).getValue().get()); + } + + pipelineInfo.addUniform(uniformName, (BasicType) value.getType(), Optional.empty(), + uniformValues); + // Declare a new variable declaration for the uniform we generated. + final VariableDeclInfo variableDeclInfo = new VariableDeclInfo(uniformName, null, null); + final VariablesDeclaration variablesDecl = new VariablesDeclaration( + new QualifiedType(value.getType(), Arrays.asList(TypeQualifier.UNIFORM)), variableDeclInfo + ); + translationUnit.addDeclaration(variablesDecl); + + // As uniform variables are available at the global scope, we create a new fact given a + // generated uniform and thus keep it at the global scope manager. + final VariableDeclFact variableDeclFact = new VariableDeclFact(variablesDecl, + variableDeclInfo, value); + globalFactManager.addVariableFact(value, variableDeclFact); + + return new VariableIdentifierExpr(uniformName); + } + + /** + * Assigns value to the newly-generated variable using a for loop. To do so, we first pick a + * random number (divisor) and add this number to the variable for each iteration. Then, we + * basically find a number of iterations by dividing the original value by the chosen divisor + * . We would also have to eventually add the remainder to the variable if needed. + * + *

For example, given a value x we can derive a for loop statement and a remainder from the + * equation x = (divisor * iterations) + remainder. + * + *

The code fragment below is an example result obtained by this method which returns an int 7. + * int _GLF_PRIMITIVE_int_7 = 0; + * // 3 is the number of iterations obtained by dividing the given value 7 by the random + * divisor 2. + * for(int i = 0; i < 3; i ++) { + * _GLF_PRIMITIVE_int_7 += 2; + * } + * _GLF_PRIMITIVE_int_7 += 1; // 1 is a remainder of 7 divided by 2. + * + * @param value the value that will be computed by the expression generated by + * this method. + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param currentFunction a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before. + * @return a variable identifier to the new variable generated by this method. + */ + private Expr generateForLoopValue(Value value, + FactManager factManager, + FunctionDefinition currentFunction, + Stmt stmtToInsertBefore) { + if (value.getType() != BasicType.INT && value.getType() != BasicType.UINT) { + return null; + } + // If the given fact manager is at global scope, the caller expects a expression generated by + // this method to be available at the global scope. However the new for loop statement + // generated by this method will be injected into the body of a given function making it + // available only in the local scope. Hence, if the global scope expression is requested, + // null is returned. + if (factManager.globalScope()) { + return null; + } + + // Declare and initialize a zero-value variable. + final String varName = genVarName(value, false); + final VariableDeclInfo variableDeclInfo = new VariableDeclInfo(varName, null, + new Initializer( + generateExpr(factManager, + currentFunction, + stmtToInsertBefore, + new NumericValue((BasicType) value.getType(), Optional.of(0))) + )); + final VariablesDeclaration variablesDecl = new VariablesDeclaration(value.getType(), + variableDeclInfo); + currentFunction.getBody().insertBefore(stmtToInsertBefore, + new DeclarationStmt(variablesDecl)); + + // Decide whether the loop should be incremented or decremented for each iteration. + final boolean isIncrement = generator.nextBoolean(); + // If value is unknown, we could generate and return any number. + int original = (int) ((NumericValue) value).getValue().orElse(generator.nextInt(INT_MAX)); + + // if the original value is zero, calling random wrapper will thrown an error illegal argument + // exception. + if (original < 1) { + return null; + } + // A random number that will be added to the variable on each iteration. + // We use max function here to prevent the division by zero error. + int divisor = Math.max(1, generator.nextInt(original)); + + // TODO(https://github.com/google/graphicsfuzz/issues/659): we should be able to set the max + // limit as we don't want to end up having a large number of iterations. + final int iterations = original / divisor; + // A left over number that will be added after for loop is executed. The binary + // expression responsible to add a remainder to the variable will be inserted after for + // loop statement. + final int remainder = original % divisor; + + // Values of numbers which will be used for the for loop. + final Value divisorValue = new NumericValue((BasicType) value.getType(), Optional.of(divisor)); + final Value iterationValue = new NumericValue(BasicType.INT, + Optional.of(isIncrement ? 0 : iterations)); + final Value conditionValue = new NumericValue(BasicType.INT, + Optional.of(isIncrement ? iterations : 0)); + + final Stmt init = new DeclarationStmt(new VariablesDeclaration(BasicType.INT, + new VariableDeclInfo("i", null, + new Initializer(generateExpr(factManager, currentFunction, stmtToInsertBefore, + iterationValue)))) + ); + final Expr condition = new BinaryExpr(new VariableIdentifierExpr("i"), + generateExpr(factManager, currentFunction, stmtToInsertBefore, conditionValue), + isIncrement ? BinOp.LT : BinOp.GE); + final Expr increment = new UnaryExpr(new VariableIdentifierExpr("i"), + isIncrement ? UnOp.POST_INC : UnOp.POST_DEC); + final Stmt body = new BlockStmt(Arrays.asList(new ExprStmt(new BinaryExpr( + new VariableIdentifierExpr(varName), + generateExpr( + factManager, + currentFunction, + stmtToInsertBefore, + divisorValue), + BinOp.ADD_ASSIGN + ))), false); + final ForStmt forStmt = new ForStmt(init, condition, increment, body); + currentFunction.getBody().insertBefore(stmtToInsertBefore, forStmt); + + if (remainder > 0) { + final Value remainderValue = new NumericValue((BasicType) value.getType(), + Optional.of(remainder)); + currentFunction.getBody().insertBefore(stmtToInsertBefore, new ExprStmt(new BinaryExpr( + new VariableIdentifierExpr(varName), + generateExpr(factManager, currentFunction, stmtToInsertBefore, remainderValue), + BinOp.ADD_ASSIGN + ))); + } + final VariableDeclFact variableDeclFact = new VariableDeclFact(variablesDecl, + variableDeclInfo, value); + factManager.addVariableFact(value, variableDeclFact); + return new VariableIdentifierExpr(varName); + } + + // A utility method performing the subtraction of the interface Number. + private Number subtractNumbers(Number firstOperand, Number secondOperand) { + if (firstOperand instanceof Float) { + assert secondOperand instanceof Float; + return firstOperand.floatValue() - secondOperand.floatValue(); + } + assert firstOperand instanceof Integer && secondOperand instanceof Integer; + return firstOperand.intValue() - secondOperand.intValue(); + } + + /** + * This method generates the expression that performs the addition of two values which result is + * equal to the given value. + * + * @param value the value that will be computed by the expression generated by + * this method. + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param currentFunction a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before. + * @return a binary expression performing the addition of two numbers whose sum is equal to Value. + */ + private Expr generateAdditionValue(Value value, + FactManager factManager, + FunctionDefinition currentFunction, + Stmt stmtToInsertBefore) { + if (!(value instanceof NumericValue)) { + return null; + } + if (value.getType() != BasicType.INT && value.getType() != BasicType.FLOAT + && value.getType() != BasicType.UINT) { + return null; + } + // If the given value is unknown, we are free to choose any arbitrary numbers for addition. + if (value.valueIsUnknown()) { + return new BinaryExpr( + generateExpr(factManager, currentFunction, stmtToInsertBefore, + new NumericValue((BasicType) value.getType(), Optional.empty())), + generateExpr(factManager, currentFunction, stmtToInsertBefore, + new NumericValue((BasicType) value.getType(), Optional.empty())), + BinOp.ADD); + } + + final Number expected = ((NumericValue) value).getValue().get(); + Number summandA = null; + // Given the expected type, we have to retrieve all values from the fact manager and filter only + // the facts that are known to compute particular values. + final List knownValues = factManager.getValuesFromType(value.getType()) + .stream().filter(item -> !item.valueIsUnknown()).collect(Collectors.toList()); + boolean genSummandsFromKnownValues = generator.nextBoolean(); + // If we are able to find any known values with the correct type, we then consider choosing + // summands based on such values. Otherwise, we will have to pick a random number to make an + // addition expression. + if (!knownValues.isEmpty() && genSummandsFromKnownValues) { + summandA = ((NumericValue) knownValues + .get(generator.nextInt(knownValues.size()))) + .getValue().get(); + } else { + if (value.getType() == BasicType.FLOAT) { + // TODO(https://github.com/google/graphicsfuzz/issues/688): range of numbers [-10, 10] is + // temporarily used here, we have to change how the summand is generated. + summandA = (float) generator.nextInt(21) - 10; + } else if (value.getType() == BasicType.UINT) { + // We pick a random number in a range of numbers [1- expectedValue] so that when + // subtracting the random summand with the expected value we would get the non-negative + // result. + summandA = generator.nextInt(Math.max(1, expected.intValue())); + } else if (value.getType() == BasicType.INT) { + summandA = generator.nextInt(INT_MAX); + } + } + + assert summandA != null; + // To get the second summand, we subtract original value by the the first summand. + final Number summandB = subtractNumbers(expected, summandA); + // We have a chance to have a summandA based on known fact that is greater than the expected + // value making the summandB obtained by subtraction negative. We thus have to ensure that + // the result of subtraction is non-negative number as since it is not legal to have singed + // number for uint type. + if (value.getType() == BasicType.UINT && String.valueOf(summandB).contains("-")) { + return null; + } + // Randomly decide whether summandA or summandB should be the first summand. + final boolean summandAFirst = generator.nextBoolean(); + final Number firstSummand = summandAFirst ? summandA : summandB; + final Number secondSummand = summandAFirst ? summandB : summandA; + + return new BinaryExpr( + generateExpr(factManager, currentFunction, stmtToInsertBefore, + new NumericValue((BasicType) value.getType(), Optional.of(firstSummand))), + generateExpr(factManager, currentFunction, stmtToInsertBefore, + new NumericValue((BasicType) value.getType(), Optional.of(secondSummand))), + BinOp.ADD); + } + + /** + * Retrieves known value from the function facts hold by fact manager and returns the + * function call to that function with the appropriate parameters if there is a function fact that + * represents the given value. + * + * @param value the value that will be computed by the expression generated by + * this method. + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param currentFunction a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before. + * @return a function call expression to the already declared function representing the Value. + */ + private Expr generateKnownFunctionFact(FactManager factManager, + Value value, + FunctionDefinition currentFunction, + Stmt stmtToInsertBefore) { + final List availableFacts = factManager.getFunctionFacts(value); + if (availableFacts.isEmpty()) { + return null; + } + final FunctionFact functionFact = availableFacts.get(generator.nextInt(availableFacts.size())); + final List argValues = functionFact.getArguments(); + final List args = generateArgsForFunctionCall(factManager, currentFunction, + stmtToInsertBefore, + argValues); + return new FunctionCallExpr(functionFact.getFunctionName(), args); + } + + /** + * Retrieves known value from the variable facts hold by fact manager and returns the + * variable identifier to that variable if there is a variable fact that represents the given + * value. + * + * @param value the value that will be computed by the expression generated by + * this method. + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @return a variable identifier expression to the already declared variable representing value. + */ + private Expr generateKnownVariableFact(Value value, FactManager factManager) { + final List availableFacts = factManager.getVariableFacts(value); + if (availableFacts.isEmpty()) { + return null; + } + final VariableFact variableFact = availableFacts.get(generator.nextInt(availableFacts.size())); + return new VariableIdentifierExpr(variableFact.getVariableName()); + } + + /** + * Utility function to generate a set of expressions used as the arguments of the function call + * expression. + * + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param functionDefinition a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before. + * @param argValues values of function arguments from which the new values being + * generated are derived. + * @return a list of function argument expressions. + */ + private List generateArgsForFunctionCall(FactManager factManager, + FunctionDefinition functionDefinition, + Stmt stmtToInsertBefore, + List argValues) { + return argValues.stream() + .map(item -> generateExpr( + factManager, + functionDefinition, + stmtToInsertBefore, + item + )).collect(Collectors.toList()); + } + + private String freshId() { + return ID + "_" + idGenerator.freshId(); + } + + private String genVarName(Value value, boolean isGlobal) { + // Provides name for a new variable based on the given value. + // For example: + // _GLF_PRIMITIVE_int_negative_103_0_id_18: a variable of a value -103. + // _GLF_PRIMITIVE_float_unknown_numeric_id_69: a variable that can be any value of float type. + return (isGlobal ? Constants.GLF_PRIMITIVE_GLOBAL : Constants.GLF_PRIMITIVE) + + "_" + value.getType().toString() + + parseNameFromValue(value) + + freshId(); + } + + private String genFunctionName(Value value) { + // Provides name for a new function based on the given value. + // For example: + // _GLF_COMPUTE_float_1_0_id_0: a function that returns a value 1.0 of float type. + // _GLF_COMPUTE_vec4_UNKNOWN_id_1: a function that returns randomly generated value of vec4 + // type. + return Constants.GLF_COMPUTE + + "_" + value.getType().toString() + + parseNameFromValue(value) + + freshId(); + } + + private String genParamName(Value value) { + // Provides name for a function arguments based on the given value. + // For example: + // _GLF_UNKNOWN_PARAM_vec4_id_1: a parameter of unknown value of vec4 type. + // _GLF_PARAM_int_id_62: a parameter of integer type. + return (value.valueIsUnknown() ? Constants.GLF_UNKNOWN_PARAM : Constants.GLF_PARAM) + + "_" + value.getType().toString() + + parseNameFromValue(value) + + freshId(); + } + + /** + * Utility function to parse name from the given value, for example, -0.45 will be parsed as + * negative_0_45. + * + * @param value value that will be converted to the name of variables or functions. + * @return a string derived from the given value which will be used as function or variable names. + */ + private String parseNameFromValue(Value value) { + if (value.valueIsUnknown()) { + return "_" + value.toString(); + } + final StringBuilder name = new StringBuilder(); + if (value instanceof NumericValue) { + final NumericValue numericValue = (NumericValue) value; + float floatValue = numericValue.getValue().get().floatValue(); + if (String.valueOf(floatValue).contains("-")) { + name.append(NEGATIVE); + floatValue = Math.abs(floatValue); + } + name.append("_"); + // Replace dot with underscore, i.e., 0.45 will be converted to 0_45. + name.append(Float.toString(floatValue).replace(".", "_")); + + } + if (value instanceof BooleanValue) { + name.append("_").append(value.toString()); + } + return name.toString(); + } + + /** + * Generates a new variable to which the expected value is assigned. The variable being + * generated is also randomly chosen whether it would be available at the local or global scope. + * + * @param value the value that will be computed by the expression generated by + * this method. + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param currentFunction a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before.* + * @return the variable identifier expression of the declared variable generated by this method. + */ + private Expr generateVariableFact(Value value, + FactManager factManager, + FunctionDefinition currentFunction, + Stmt stmtToInsertBefore) { + boolean atGlobalScope = factManager.globalScope() || generator.nextBoolean(); + final String varName = genVarName(value, atGlobalScope); + final Expr initializer = generateExpr(atGlobalScope ? globalFactManager : factManager, + currentFunction, stmtToInsertBefore, value); + // If generating global scope variables, initialisation would be performed later in a global + // initializer function. + final VariableDeclInfo variableDeclInfo = new VariableDeclInfo(varName, null, + atGlobalScope ? null : new Initializer(initializer)); + final VariablesDeclaration variablesDecl = new VariablesDeclaration(value.getType(), + variableDeclInfo); + final VariableDeclFact variableDeclFact = new VariableDeclFact(variablesDecl, + variableDeclInfo, value); + + if (atGlobalScope) { + // Search for an existing function for initialization of global variables. If not found, we + // have to create one and add a call to the top of 'main' to invoke it. + Optional maybeInitGlobalsFunction = + translationUnit.getTopLevelDeclarations() + .stream() + .filter(item -> item instanceof FunctionDefinition) + .map(item -> (FunctionDefinition) item) + .filter(item -> item.getPrototype().getName().equals(Constants.GLF_INIT_GLOBALS)) + .findFirst(); + + FunctionDefinition initGlobalsFunction; + if (maybeInitGlobalsFunction.isPresent()) { + initGlobalsFunction = maybeInitGlobalsFunction.get(); + } else { + // Function prototype of the globals initializer, this must be declared at the top level of + // the shader being generated. + final FunctionPrototype functionPrototype = + new FunctionPrototype(Constants.GLF_INIT_GLOBALS, VoidType.VOID, + new ArrayList<>()); + translationUnit.addDeclarationBefore(functionPrototype, translationUnit.getMainFunction()); + + // The invocation to the globals initializer which needs to be inserted as the first + // statement in main function. + translationUnit.getMainFunction().getBody().insertStmt(0, + new ExprStmt(new FunctionCallExpr(Constants.GLF_INIT_GLOBALS, new ArrayList<>()))); + + // A function into which the assignments of values for global variables are injected. + initGlobalsFunction = new FunctionDefinition(functionPrototype, + new BlockStmt(new ArrayList<>(), false)); + final List newTopLevelDeclarations = new ArrayList<>(); + newTopLevelDeclarations.addAll(translationUnit.getTopLevelDeclarations()); + newTopLevelDeclarations.add(initGlobalsFunction); + translationUnit.setTopLevelDeclarations(newTopLevelDeclarations); + } + + // Inject a statement assigning value to the global variable into function body. + initGlobalsFunction.getBody().addStmt(new ExprStmt(new BinaryExpr( + new VariableIdentifierExpr(varName), initializer, BinOp.ASSIGN + ))); + + translationUnit.addDeclarationBefore(variablesDecl, currentFunction); + globalFactManager.addVariableFact(value, variableDeclFact); + } else { + currentFunction.getBody().insertBefore(stmtToInsertBefore, + new DeclarationStmt(variablesDecl)); + factManager.addVariableFact(value, variableDeclFact); + } + return new VariableIdentifierExpr(varName); + } + + /** + * Generates a new function that guarantees to return the value representing the expected Value. + * A number and type of the function arguments for the function being generated are randomized. + * + * @param value the value that will be computed by the expression generated by + * this method. + * @param factManager manager class holding the value and its associated expression that + * guarantees to compute the given value. + * @param currentFunction a function into which the new expression will be injected. + * @param stmtToInsertBefore statement in the body of the given function where the generated + * expression will be inserted before. + * @return function call expression of the function generated by this method. + */ + private Expr generateFunctionFact(Value value, + FactManager factManager, + FunctionDefinition currentFunction, + Stmt stmtToInsertBefore) { + final String functionName = genFunctionName(value); + final FactManager newFunctionScope = globalFactManager.newScope(); + final List argumentValues = new ArrayList<>(); + final List parameterDecls = new ArrayList<>(); + + final int noOfParams = generator.nextInt(MAX_FUNCTION_PARAMS); + for (int i = 0; i < noOfParams; i++) { + final Type paramType = getAvailableTypes().get(generator.nextInt(getAvailableTypes().size())); + // Decide whether the value generated should be known by the fact manager. + // If the fact manager is generating an unknown parameter(value is Optional.empty), + // when calling this function the fact manager will generate any arbitrary value that + // matches the parameter type. + final Value paramValue = fuzzValue(paramType, false); + final String paramName = genParamName(paramValue); + + argumentValues.add(paramValue); + final ParameterDecl parameterDecl = new ParameterDecl( + paramName, + paramType, + null + ); + parameterDecls.add(parameterDecl); + newFunctionScope.addVariableFact(paramValue, new ParameterDeclFact(parameterDecl, + paramValue)); + } + final BlockStmt body = new BlockStmt(Collections.emptyList(), false); + final FunctionPrototype functionPrototype = new FunctionPrototype(functionName, + value.getType(), parameterDecls); + final FunctionDefinition newFunction = new FunctionDefinition(functionPrototype, + body); + translationUnit.addDeclarationBefore(newFunction, currentFunction); + + // Since the new function has an empty body, we first need to add a placeholder statement + // into the body to be the point in a function where the new expressions can be injected + // before. + final Stmt placeholderStmt = new NullStmt(); + body.addStmt(placeholderStmt); + + // We then replace the placeholder statement with a new return statement, returning a + // newly-generated expression. + body.replaceChild(placeholderStmt, new ReturnStmt( + generateExpr( + newFunctionScope, + newFunction, + placeholderStmt, + value + ) + )); + + globalFactManager.addFunctionFact(value, new FunctionFact(functionPrototype, argumentValues, + value)); + final List args = generateArgsForFunctionCall(factManager, currentFunction, + stmtToInsertBefore, + argumentValues); + return new FunctionCallExpr(functionName, args); + } + + private List getAvailableTypes() { + // We will have to consider supporting more types but at the moment as we are using + // LiteralFuzzer to generate literal expressions so we are unable to cover all basic type now. + // The following are types currently supported by LiteralFuzzer. + return Arrays.asList(BasicType.BOOL, BasicType.INT, BasicType.UINT, BasicType.FLOAT, + BasicType.VEC2, BasicType.VEC3, BasicType.VEC4); + } + + private Value fuzzValue(Type type, boolean forceGenerateKnownValue) { + // An Unknown value variable is the variable whose value could be anything. The fact manager + // could generate any arbitrary value but with the correct type. + final boolean isUnknown = generator.nextBoolean() && !forceGenerateKnownValue; + if (type == BasicType.BOOL) { + return new BooleanValue( + isUnknown ? Optional.empty() : Optional.of(generator.nextBoolean())); + } + if (BasicType.allScalarTypes().contains(type)) { + if (isUnknown) { + return new NumericValue((BasicType) type, Optional.empty()); + } + if (type == BasicType.INT) { + final String intString = String.valueOf(generator.nextInt(INT_MAX - INT_MIN) + INT_MIN); + return new NumericValue(BasicType.INT, Optional.of(Integer.valueOf(intString))); + } + if (type == BasicType.FLOAT) { + final String floatString = LiteralFuzzer.randomFloatString(generator); + return new NumericValue(BasicType.FLOAT, Optional.of(Float.valueOf(floatString))); + } + if (type == BasicType.UINT) { + final String intString = String.valueOf(generator.nextInt(INT_MAX - INT_MIN) + INT_MIN); + return new NumericValue(BasicType.UINT, Optional.of(Math.abs(Integer.valueOf(intString)))); + } + throw new RuntimeException("Not implemented yet!"); + } + + if (isUnknown) { + return new CompositeValue(type, Optional.empty()); + } + if (BasicType.allBasicTypes().contains(type)) { + final List values = new ArrayList<>(); + for (int i = 0; i < ((BasicType) type).getNumElements(); i++) { + values.add(fuzzValue(((BasicType) type).getElementType(), forceGenerateKnownValue)); + } + return new CompositeValue(type, Optional.of(values)); + } + + // TODO(https://github.com/google/graphicsfuzz/issues/664): we should also support array and + // struct types as well. + throw new RuntimeException("Not implemented yet!"); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FactManager.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FactManager.java new file mode 100644 index 000000000..e0565a12c --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FactManager.java @@ -0,0 +1,133 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.type.Type; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class FactManager { + + // A parent fact manager whose facts this fact manager extends. + private final FactManager prototype; + private final Map> variableFacts; + private final Map> functionFacts; + + public FactManager(FactManager prototype) { + this.prototype = prototype; + variableFacts = new HashMap>(); + functionFacts = new HashMap>(); + } + + /** + * Given an expected type, this method gets available function and variable facts hold by + * this fact manager. + * + * @param type type of the facts we are going to search for. + * @return a list of values that matches the given type. + */ + public List getValuesFromType(Type type) { + return Stream.concat(variableFacts.keySet().stream(), functionFacts.keySet().stream()) + .filter(value -> value.getType() == type) + .collect(Collectors.toList()); + } + + /** + * Creates a new fact manager where this fact manager is used as a prototype. We basically call + * this method only when generating a new function where the return fact manager is non global. + * + * @return new fact manager. + */ + public FactManager newScope() { + return new FactManager(this); + } + + /** + * Determines whether the fact manager is at the global scope. + * + * @return true if this manager has a prototype and false otherwise. + */ + public boolean globalScope() { + return prototype == null; + } + + /** + * Adds a new variable fact to the variable fact map of this fact manager. + * + * @param value value which the given variable fact is representing. + * @param variableFact a new variable fact that will be added into the map of variable facts. + */ + public void addVariableFact(Value value, VariableFact variableFact) { + final List newFacts = variableFacts.getOrDefault(value, new ArrayList<>()); + newFacts.add(variableFact); + variableFacts.put(value, newFacts); + } + + /** + * As function facts can only exist at the global scope, this method adds a new function + * fact into the map of the function facts of the global scope fact manager. + * + * @param value value which the given function fact is representing. + * @param functionFact a new function fact that will be added into the map of function facts. + */ + public void addFunctionFact(Value value, FunctionFact functionFact) { + if (globalScope()) { + final List newFacts = functionFacts.getOrDefault(value, new ArrayList<>()); + newFacts.add(functionFact); + functionFacts.put(value, newFacts); + } else { + prototype.addFunctionFact(value, functionFact); + } + } + + /** + * Retrieves a list of variable facts representing the value. + * + * @param value a value which the fact manager is asked to search in the variable fact map. + * @return if value does not exist in the variable fact map, an empty list is returned. Otherwise, + * returns a list of variable facts that guarantees to provide the given value. + */ + public List getVariableFacts(Value value) { + final List result = new ArrayList<>(); + result.addAll(variableFacts.getOrDefault(value, Collections.emptyList())); + if (!globalScope()) { + result.addAll(prototype.getVariableFacts(value)); + } + return result; + } + + /** + * Retrieves a list of function facts representing the value from the global scope fact + * manager. If the value does not exist in the map, an empty list is returned. + * + * @param value a value which the fact manager is asked to search in the function fact map. + * @return if value does not exist in the function fact map, an empty list is returned. + * Otherwise, returns a list of function facts that guarantees to provide the given value. + */ + public List getFunctionFacts(Value value) { + if (globalScope()) { + return functionFacts.getOrDefault(value, Collections.emptyList()); + } + return prototype.getFunctionFacts(value); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FunctionFact.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FunctionFact.java new file mode 100644 index 000000000..dfbdb15a5 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/FunctionFact.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import java.util.List; + +/** + * This class holds the information of the newly-generated function and its associated Value. + * Each time we generate a new function, we create a function fact and keep it in Fact Manager + * which later will be used by the Expression Generator when generating known value expression. + */ +public class FunctionFact { + + private final FunctionPrototype prototype; + private final List arguments; + private final Value value; + + public FunctionFact(FunctionPrototype prototype, List arguments, Value value) { + this.prototype = prototype; + this.arguments = arguments; + this.value = value; + } + + public String getFunctionName() { + return prototype.getName(); + } + + public List getArguments() { + return arguments; + } + + public Value getValue() { + return value; + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/NumericValue.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/NumericValue.java new file mode 100644 index 000000000..983a8451f --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/NumericValue.java @@ -0,0 +1,94 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.FloatConstantExpr; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; +import com.graphicsfuzz.common.ast.expr.UIntConstantExpr; +import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.generator.semanticschanging.LiteralFuzzer; +import java.util.Objects; +import java.util.Optional; + +public class NumericValue implements Value { + + private final BasicType basicType; + private final Optional value; + + public NumericValue(BasicType basicType, Optional value) { + this.basicType = basicType; + this.value = value; + } + + @Override + public Type getType() { + return basicType; + } + + @Override + public boolean valueIsUnknown() { + return !value.isPresent(); + } + + @Override + public Expr generateLiteral(LiteralFuzzer literalFuzzer) { + if (valueIsUnknown()) { + return literalFuzzer.fuzz(basicType).orElse(null); + } + if (basicType == BasicType.FLOAT) { + return new FloatConstantExpr(value.get().toString()); + } + if (basicType == BasicType.INT) { + return new IntConstantExpr(value.get().toString()); + } + if (basicType == BasicType.UINT) { + return new UIntConstantExpr(value.get().toString() + "u"); + } + throw new RuntimeException("Numeric value does not support the given type"); + } + + public Optional getValue() { + return value; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof NumericValue)) { + return false; + } + final NumericValue thatNumericValue = (NumericValue) that; + return this.basicType.equals(thatNumericValue.basicType) + && this.value.equals(thatNumericValue.getValue()); + } + + @Override + public int hashCode() { + return Objects.hash(basicType, value); + } + + @Override + public String toString() { + return valueIsUnknown() ? "unknown_numeric" : value.get().toString(); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ParameterDeclFact.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ParameterDeclFact.java new file mode 100644 index 000000000..f990d26ab --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/ParameterDeclFact.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.decl.ParameterDecl; + +/** + * This class holds the information of a parameter of generated functions and its associated value. + * For any generated function, we randomly decide how parameters should be and for each parameter + * we create a parameter declaration fact and keep it in Fact Manager which later will be used by + * the Expression Generator when generating known value expression. + */ +public class ParameterDeclFact extends VariableFact { + + private final ParameterDecl parameterDecl; + + public ParameterDeclFact(ParameterDecl parameterDecl, + Value value) { + super(value); + this.parameterDecl = parameterDecl; + } + + @Override + public String getVariableName() { + return parameterDecl.getName(); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/Value.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/Value.java new file mode 100644 index 000000000..da9197605 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/Value.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.generator.semanticschanging.LiteralFuzzer; + +/** + * This interface defines an expected Value used by {@link ExpressionGenerator} when + * generating an expression. It represents a possibly unknown value of some type. + */ +public interface Value { + + /** + * Indicates whether or not this is an unknown value. + * @return true if the value is unknown and false otherwise. + */ + boolean valueIsUnknown(); + + /** + * Gets the type of the underlying value. + * @return the basic type of the value. + */ + Type getType(); + + /** + * Provides a literal with the same type as the Value's type, such that all parts of the value + * that are known will have the expected values, and all other parts will be randomized. + * + * @param literalFuzzer a util class used to generate fuzzed expressions. + * @return the expression that represents the value. + */ + Expr generateLiteral(LiteralFuzzer literalFuzzer); + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableDeclFact.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableDeclFact.java new file mode 100644 index 000000000..8b49f1a5c --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableDeclFact.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; + +/** + * This class holds the information of the newly-generated variable and its associated Value. + * Each time we declare a new variable, we create a variable declaration fact and keep it in Fact + * Manager which later will be used by the Expression Generator when generating known value + * expression. + */ +public class VariableDeclFact extends VariableFact { + + private final VariablesDeclaration variablesDeclaration; + private final VariableDeclInfo variableDeclInfo; + + public VariableDeclFact(VariablesDeclaration variablesDeclaration, + VariableDeclInfo variableDeclInfo, + Value value) { + super(value); + this.variablesDeclaration = variablesDeclaration; + this.variableDeclInfo = variableDeclInfo; + } + + public String getVariableName() { + return variableDeclInfo.getName(); + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableFact.java b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableFact.java new file mode 100644 index 000000000..62b5c1e5d --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/knownvaluegeneration/VariableFact.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.knownvaluegeneration; + +public abstract class VariableFact { + + private final Value value; + + /** + * @param value which a new VariableDeclFact or ParameterDeclFact is representing. + */ + VariableFact(Value value) { + this.value = value; + } + + /** + * @return a known value of the variable fact. + */ + public Value getValue() { + return value; + } + + /** + * @return a variable name of this variable fact. This is used by the generator + * when it is generating a new variable identifier expression. + */ + public abstract String getVariableName(); + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/mutateapi/MutationFinderBase.java b/generator/src/main/java/com/graphicsfuzz/generator/mutateapi/MutationFinderBase.java index 5d8abfad9..a577bc996 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/mutateapi/MutationFinderBase.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/mutateapi/MutationFinderBase.java @@ -34,15 +34,13 @@ import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; -import com.graphicsfuzz.generator.mutateapi.Mutation; -import com.graphicsfuzz.generator.mutateapi.MutationFinder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import java.util.ArrayList; import java.util.Collections; import java.util.List; public abstract class MutationFinderBase - extends ScopeTreeBuilder implements MutationFinder { + extends ScopeTrackingVisitor implements MutationFinder { private final TranslationUnit tu; private final List mutations; diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/AddArrayMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/AddArrayMutation.java index 59097110a..ee052f257 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/AddArrayMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/AddArrayMutation.java @@ -20,6 +20,7 @@ import com.graphicsfuzz.common.ast.decl.ArrayInfo; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.generator.mutateapi.Mutation; @@ -40,7 +41,9 @@ public AddArrayMutation(TranslationUnit tu, String name, BasicType baseType, int @Override public void apply() { tu.addDeclaration(new VariablesDeclaration( - baseType, new VariableDeclInfo(name, new ArrayInfo(numElements), null))); + baseType, new VariableDeclInfo(name, + new ArrayInfo(new IntConstantExpr(Integer.toString(numElements))), + null))); } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/Expr2ArrayAccessMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/Expr2ArrayAccessMutationFinder.java index 66b610f64..744274264 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/Expr2ArrayAccessMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/Expr2ArrayAccessMutationFinder.java @@ -52,7 +52,7 @@ protected void visitVariableDeclInfoAfterAddedToScope(VariableDeclInfo declInfo) if (!atGlobalScope()) { return; } - ScopeEntry se = currentScope.lookupScopeEntry(declInfo.getName()); + ScopeEntry se = getCurrentScope().lookupScopeEntry(declInfo.getName()); if (se.getType() instanceof ArrayType) { globalArrays.put(declInfo.getName(), ((ArrayType) se.getType()).getBaseType() .getWithoutQualifiers()); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/InterchangeExprMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/InterchangeExprMutationFinder.java index 2680d1a6d..3ce173047 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/InterchangeExprMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/InterchangeExprMutationFinder.java @@ -63,7 +63,8 @@ protected void visitExpr(Expr expr) { private Expr applyInterchange(Expr expr, List argumentTypes) { final Type resultType = typer.lookupType(expr).getWithoutQualifiers(); List templates = Fuzzer.availableTemplatesFromScope( - getTranslationUnit().getShadingLanguageVersion(), currentScope) + getTranslationUnit().getShadingLanguageVersion(), getTranslationUnit().getShaderKind(), + getCurrentScope()) // Ignore templates that require l-values, so that invalidity is not too likely .filter(InterchangeExprMutationFinder::doesNotRequireLvalue) // Ignore the possibility of replacing a one-argument expression with parentheses, as it diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/LiteralFuzzer.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/LiteralFuzzer.java index 8a6c46de2..c97bbeb11 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/LiteralFuzzer.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/LiteralFuzzer.java @@ -21,6 +21,7 @@ import com.graphicsfuzz.common.ast.expr.FloatConstantExpr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.expr.TypeConstructorExpr; +import com.graphicsfuzz.common.ast.expr.UIntConstantExpr; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.util.IRandom; @@ -47,14 +48,17 @@ public Optional fuzz(Type type) { } if (type == BasicType.INT) { return Optional.of(new IntConstantExpr( - String.valueOf(generator.nextInt(INT_MAX - INT_MIN) + INT_MIN))); + String.valueOf(generator.nextInt(INT_MAX - INT_MIN) + INT_MIN))); + } + if (type == BasicType.UINT) { + return Optional.of(new UIntConstantExpr( + String.valueOf(Math.abs(generator.nextInt(INT_MAX - INT_MIN) + INT_MIN) + "u"))); } if (type == BasicType.FLOAT) { - return Optional.of(new FloatConstantExpr( - randomFloatString())); + return Optional.of(new FloatConstantExpr(LiteralFuzzer.randomFloatString(generator))); } if (type == BasicType.VEC2 || type == BasicType.VEC3 || type == BasicType.VEC4 - || BasicType.allMatrixTypes().contains(type)) { + || BasicType.allMatrixTypes().contains(type)) { final List args = new ArrayList<>(); for (int i = 0; i < ((BasicType) type).getNumElements(); i++) { args.add(fuzz(((BasicType) type).getElementType()).get()); @@ -64,7 +68,7 @@ public Optional fuzz(Type type) { return Optional.empty(); } - private String randomFloatString() { + public static String randomFloatString(IRandom generator) { final int maxDigitsEitherSide = 5; StringBuilder sb = new StringBuilder(); sb.append(generator.nextBoolean() ? "-" : ""); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinder.java index 04543c9f3..6cf21f153 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinder.java @@ -128,8 +128,9 @@ public void visitBlockStmt(BlockStmt block) { private Expr getSwitchCondition() { final List candidateVariables = new ArrayList<>(); - currentScope.namesOfAllVariablesInScope().stream() - .filter(item -> currentScope.lookupType(item).getWithoutQualifiers().equals(BasicType.INT)) + getCurrentScope().namesOfAllVariablesInScope().stream() + .filter(item -> getCurrentScope().lookupType(item).getWithoutQualifiers() + .equals(BasicType.INT)) .forEach(candidateVariables::add); if (candidateVariables.isEmpty()) { return new LiteralFuzzer(generator).fuzz(BasicType.INT).get(); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/SwapVariableIdentifiersMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/SwapVariableIdentifiersMutationFinder.java index 8ab4d39c2..d86574934 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/SwapVariableIdentifiersMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticschanging/SwapVariableIdentifiersMutationFinder.java @@ -44,11 +44,12 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie return; } - if (currentScope.lookupType(variableIdentifierExpr.getName()) == null) { + if (getCurrentScope().lookupType(variableIdentifierExpr.getName()) == null) { return; } - assert currentScope.lookupType(variableIdentifierExpr.getName()).getWithoutQualifiers() != null; + assert getCurrentScope().lookupType(variableIdentifierExpr.getName()).getWithoutQualifiers() + != null; final List candidateVariables = getCandidateVariables(variableIdentifierExpr.getName()); @@ -65,11 +66,11 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie } private List getCandidateVariables(String varIdentifier) { - return currentScope.namesOfAllVariablesInScope().stream() + return getCurrentScope().namesOfAllVariablesInScope().stream() .filter(item -> !item.equals(varIdentifier) - && currentScope.lookupType(item) != null - && currentScope.lookupType(varIdentifier).getWithoutQualifiers().equals( - currentScope.lookupType(item).getWithoutQualifiers())) + && getCurrentScope().lookupType(item) != null + && getCurrentScope().lookupType(varIdentifier).getWithoutQualifiers().equals( + getCurrentScope().lookupType(item).getWithoutQualifiers())) .collect(Collectors.toList()); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddLiveOutputWriteMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddLiveOutputWriteMutation.java index 4533afd39..4697e91db 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddLiveOutputWriteMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddLiveOutputWriteMutation.java @@ -30,6 +30,7 @@ import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.generator.fuzzer.Fuzzer; import com.graphicsfuzz.generator.fuzzer.FuzzingContext; import com.graphicsfuzz.generator.fuzzer.OpaqueExpressionGenerator; @@ -63,7 +64,8 @@ outputVariableType, new VariableDeclInfo(backupName, null, null)))); stmts.add(new ExprStmt(new BinaryExpr(new VariableIdentifierExpr(backupName), new VariableIdentifierExpr(outputVariableName), BinOp.ASSIGN))); - final Fuzzer fuzzer = new Fuzzer(new FuzzingContext(), shadingLanguageVersion, random, + final Fuzzer fuzzer = new Fuzzer(new FuzzingContext(), shadingLanguageVersion, + random, generationParams); stmts.add(new ExprStmt(new BinaryExpr( new VariableIdentifierExpr(outputVariableName), diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddWrappingConditionalMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddWrappingConditionalMutation.java index 6c5766b14..aae749311 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddWrappingConditionalMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/AddWrappingConditionalMutation.java @@ -16,7 +16,7 @@ package com.graphicsfuzz.generator.semanticspreserving; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -38,7 +38,6 @@ import com.graphicsfuzz.common.util.ContainsTopLevelBreak; import com.graphicsfuzz.common.util.ContainsTopLevelContinue; import com.graphicsfuzz.common.util.IRandom; -import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.generator.fuzzer.Fuzzer; import com.graphicsfuzz.generator.fuzzer.FuzzingContext; import com.graphicsfuzz.generator.fuzzer.OpaqueExpressionGenerator; @@ -167,7 +166,7 @@ private Stmt makeSingleIterationForStmt(Stmt stmt, : opaqueExpressionGenerator .makeOpaqueOne(BasicType.INT, loopBoundsMustBeConst, 0, fuzzer); DeclarationStmt init = new DeclarationStmt(new VariablesDeclaration(BasicType.INT, - new VariableDeclInfo(loopVariableName, null, new ScalarInitializer(start)))); + new VariableDeclInfo(loopVariableName, null, new Initializer(start)))); Expr end = up ? opaqueExpressionGenerator .makeOpaqueOne(BasicType.INT, loopBoundsMustBeConst, 0, fuzzer) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java index ff48e76ff..2a9f84724 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/IdentityMutationFinder.java @@ -17,9 +17,8 @@ package com.graphicsfuzz.generator.semanticspreserving; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ArrayInitializer; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; import com.graphicsfuzz.common.ast.expr.BinaryExpr; @@ -77,14 +76,13 @@ protected void visitExpr(Expr expr) { } final BasicType basicType = (BasicType) type; - final Scope clonedScope = currentScope.shallowClone(); + final Scope clonedScope = getCurrentScope().shallowClone(); if (getTranslationUnit().getShadingLanguageVersion().restrictedForLoops()) { for (Set iterators : forLoopIterators) { iterators.forEach(clonedScope::remove); } } - if (BasicType.allScalarTypes().contains(basicType) - || BasicType.allVectorTypes().contains(basicType) + if (basicType.isScalar() || basicType.isVector() || BasicType.allSquareMatrixTypes().contains(basicType)) { // TODO: add support for non-square matrices. addMutation(new Expr2ExprMutation(parentMap.getParent(expr), @@ -229,18 +227,10 @@ public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) } @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { + public void visitInitializer(Initializer initializer) { assert !inInitializer; inInitializer = true; - super.visitScalarInitializer(scalarInitializer); - inInitializer = false; - } - - @Override - public void visitArrayInitializer(ArrayInitializer arrayInitializer) { - assert !inInitializer; - inInitializer = true; - super.visitArrayInitializer(arrayInitializer); + super.visitInitializer(initializer); inInitializer = false; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/OutlineStatementMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/OutlineStatementMutationFinder.java index d62bf5306..c2634dd32 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/OutlineStatementMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/OutlineStatementMutationFinder.java @@ -27,19 +27,11 @@ import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.common.typing.Scope; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; -import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.IdGenerator; -import com.graphicsfuzz.generator.mutateapi.MutationFinder; import com.graphicsfuzz.generator.mutateapi.MutationFinderBase; -import com.graphicsfuzz.generator.util.TransformationProbabilities; import com.graphicsfuzz.util.Constants; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Set; -import java.util.stream.Collectors; public class OutlineStatementMutationFinder extends MutationFinderBase { @@ -74,11 +66,11 @@ public void visitExprStmt(ExprStmt exprStmt) { if (!OutlineStatementMutation.assignsDirectlyToVariable(be)) { return; } - if (referencesArray(be.getRhs(), currentScope)) { + if (referencesArray(be.getRhs(), getCurrentScope())) { return; } - addMutation(new OutlineStatementMutation(exprStmt, currentScope, getTranslationUnit(), - enclosingFunction, idGenerator)); + addMutation(new OutlineStatementMutation(exprStmt, getCurrentScope(), getTranslationUnit(), + getEnclosingFunction(), idGenerator)); } private boolean referencesArray(Expr expr, Scope enclosingScope) { diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/SplitForLoopMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/SplitForLoopMutation.java index ae356524f..e1a9da067 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/SplitForLoopMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/SplitForLoopMutation.java @@ -16,7 +16,7 @@ package com.graphicsfuzz.generator.semanticspreserving; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -102,7 +102,7 @@ private void adjustInitializer(ForStmt loop, int numIterationsToSplitAfter, + (loopSplitInfo.getIncreasing() ? 1 : -1) * numIterationsToSplitAfter; VariablesDeclaration varDecl = ((DeclarationStmt) loop.getInit()).getVariablesDeclaration(); - varDecl.getDeclInfo(0).setInitializer(new ScalarInitializer(new IntConstantExpr( + varDecl.getDeclInfo(0).setInitializer(new Initializer(new IntConstantExpr( String.valueOf(newStart)))); } @@ -179,12 +179,12 @@ private static Optional maybeGetLoopSplitInfo(ForStmt forStmt) { } // Now we grab the initial value, which needs to be an integer. - if (!(declInfo.getInitializer() instanceof ScalarInitializer)) { + if (!(declInfo.getInitializer() instanceof Initializer)) { return Optional.empty(); } // Now we get its integer value, if it has one - final Optional maybeStartValue = maybeGetIntegerValue(((ScalarInitializer) declInfo + final Optional maybeStartValue = maybeGetIntegerValue((declInfo .getInitializer()).getExpr()); if (!maybeStartValue.isPresent()) { return Optional.empty(); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutation.java index 9f1d74fd0..055903e12 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutation.java @@ -18,7 +18,7 @@ import com.graphicsfuzz.common.ast.IParentMap; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.Expr; @@ -33,7 +33,7 @@ import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.typing.SupportedTypes; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.IdGenerator; @@ -177,9 +177,9 @@ private void structifyDeclaration(String enclosingStructVariableName, declInfo.setName(enclosingStructVariableName); if (declInfo.hasInitializer()) { declInfo.setInitializer( - new ScalarInitializer( + new Initializer( makeInitializationExpr(enclosingStructType, - ((ScalarInitializer) declInfo.getInitializer()).getExpr()) + (declInfo.getInitializer()).getExpr()) ) ); } @@ -207,11 +207,11 @@ private void structifyBlock(Expr structifiedExpr) { final IParentMap parentMap = IParentMap.createParentMap(block); - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - ScopeEntry se = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + ScopeEntry se = getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()); if (se == null) { // We are traversing a block in isolation, so we won't have a scope entry for any variable // declared outside the block. diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutation.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutation.java index 43ef01c6d..fb3db2c74 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutation.java @@ -28,7 +28,7 @@ import com.graphicsfuzz.common.ast.stmt.ExprStmt; import com.graphicsfuzz.common.transformreduce.MergeSet; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import com.graphicsfuzz.generator.mutateapi.Mutation; import java.util.ArrayList; @@ -61,7 +61,7 @@ public void apply() { mergeSet.getMergedType(), new VariableDeclInfo(mergeSet.getMergedName(), null, null)))); } - private class VectorizerVisitor extends ScopeTreeBuilder { + private class VectorizerVisitor extends ScopeTrackingVisitor { private boolean inDeclarationOfTargetVariable = false; private final ScopeEntry currentComponent; @@ -96,19 +96,20 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie } private boolean isCurrentComponentVariable(String name) { - ScopeEntry entry = currentScope.lookupScopeEntry(name); + ScopeEntry entry = getCurrentScope().lookupScopeEntry(name); return entry != null && entry.hasVariableDeclInfo() && currentComponent.getVariableDeclInfo() == entry.getVariableDeclInfo(); } @Override public void visitDeclarationStmt(DeclarationStmt declarationStmt) { - List existingKeys = new ArrayList<>(); - existingKeys.addAll(currentScope.keys()); + List existingKeys = new ArrayList<>(getCurrentScope().keys()); super.visitDeclarationStmt(declarationStmt); - List newKeys = currentScope.keys().stream().filter(key -> !existingKeys.contains(key)) + List newKeys = getCurrentScope().keys() + .stream() + .filter(key -> !existingKeys.contains(key)) + .sorted(String::compareTo) .collect(Collectors.toList()); - newKeys.sort(String::compareTo); for (String newKey : newKeys) { if (isCurrentComponentVariable(newKey)) { final ExprStmt insertedStmt = new ExprStmt( diff --git a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutationFinder.java b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutationFinder.java index 513b1c3e0..b056590c4 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutationFinder.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/semanticspreserving/VectorizeMutationFinder.java @@ -26,12 +26,9 @@ import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.transformreduce.MergeSet; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; import com.graphicsfuzz.common.util.IRandom; -import com.graphicsfuzz.generator.mutateapi.MutationFinder; import com.graphicsfuzz.generator.mutateapi.MutationFinderBase; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -60,8 +57,8 @@ protected void popScope() { assert lastExitedBlock != null; if (!(parentMap.getParent(lastExitedBlock) instanceof SwitchStmt)) { List mergeSetsForThisScope = new ArrayList<>(); - for (String v : currentScope.keys()) { - ScopeEntry entry = currentScope.lookupScopeEntry(v); + for (String v : getCurrentScope().keys()) { + ScopeEntry entry = getCurrentScope().lookupScopeEntry(v); if (!isCandidateForMerging(entry)) { continue; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/CustomMutatorServer.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/CustomMutatorServer.java index 21b02eee6..a32b71997 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/CustomMutatorServer.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/CustomMutatorServer.java @@ -130,8 +130,7 @@ private static String mutate(String inputShader, int seed, boolean isFragment) { Optional.empty(), stream, INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, - false); + PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER); } final String outputShader = new String(byteArrayOutputStream.toByteArray(), StandardCharsets.UTF_8); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Fragment2Compute.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Fragment2Compute.java index a3ff3e972..a2b4da8a2 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Fragment2Compute.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Fragment2Compute.java @@ -45,7 +45,7 @@ import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.OpenGlConstants; @@ -56,6 +56,7 @@ import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.common.util.SsboFieldData; import com.graphicsfuzz.generator.util.RemoveDiscardStatements; +import com.graphicsfuzz.util.ArgsUtil; import java.io.File; import java.io.IOException; import java.util.Arrays; @@ -63,7 +64,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Optional; -import java.util.Random; import java.util.stream.Collectors; import net.sourceforge.argparse4j.ArgumentParsers; import net.sourceforge.argparse4j.impl.Arguments; @@ -99,8 +99,8 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .type(File.class); parser.addArgument("--seed") - .help("Seed for random number generator.") - .type(Integer.class); + .help("Seed (unsigned 64 bit long integer) for random number generator.") + .type(String.class); parser.addArgument("--generate-uniform-bindings") .help("Put all uniforms in uniform blocks and generate bindings; required for Vulkan " @@ -179,7 +179,7 @@ private static String demoteOutputVariable(TranslationUnit tu) { * Replace all references to gl_FragCoord with gl_GlobalInvocationID. */ private static void replaceFragCoordWithIdLookup(TranslationUnit computeTu) { - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { private IParentMap parentMap = IParentMap.createParentMap(computeTu); @@ -188,7 +188,7 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie super.visitVariableIdentifierExpr(variableIdentifierExpr); if (variableIdentifierExpr.getName().equals(OpenGlConstants.GL_FRAG_COORD)) { // It has the right name. - if (currentScope.lookupScopeEntry(OpenGlConstants.GL_FRAG_COORD) == null) { + if (getCurrentScope().lookupScopeEntry(OpenGlConstants.GL_FRAG_COORD) == null) { // It has no scope; i.e., it is built-in - so it is the real gl_FragCoord. // Replace it with something made from gl_GlobalInvocationID. parentMap.getParent(variableIdentifierExpr).replaceChild(variableIdentifierExpr, @@ -307,8 +307,7 @@ public static void mainHelper(String... args) throws ArgumentParserException, InterruptedException, GlslParserException, ParseTimeoutException, IOException { final Namespace ns = parse(args); final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - final IRandom generator = new RandomWrapper(ns.getInt("seed") == null - ? new Random().nextInt() : ns.getInt("seed")); + final IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final ShaderJob transformedShaderJob = transform( fileOps.readShaderJobFile(ns.get("fragment_json")), generator); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/FuzzShader.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/FuzzShader.java deleted file mode 100644 index 91e21e7b9..000000000 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/FuzzShader.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.generator.tool; - -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; -import com.graphicsfuzz.common.util.RandomWrapper; -import com.graphicsfuzz.common.util.ShaderKind; -import com.graphicsfuzz.generator.fuzzer.Fuzzer; -import com.graphicsfuzz.generator.fuzzer.FuzzingContext; -import com.graphicsfuzz.generator.util.GenerationParams; - -public class FuzzShader { - - public static void main(String[] args) { - - if (args.length != 1) { - System.err.print("Usage: seed"); - System.exit(1); - } - - new PrettyPrinterVisitor(System.out) - .visit(new Fuzzer(new FuzzingContext(), - ShadingLanguageVersion.GLSL_440, - new RandomWrapper(Integer.parseInt(args[0])), - GenerationParams.normal(ShaderKind.FRAGMENT, true)).fuzzTranslationUnit()); - - } - -} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java index dce9cc76c..5a40c701b 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Generate.java @@ -27,7 +27,6 @@ import com.graphicsfuzz.common.typing.Typer; import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.IRandom; -import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseTimeoutException; import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.PruneUniforms; @@ -54,6 +53,7 @@ import com.graphicsfuzz.generator.util.FloatLiteralReplacer; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.TransformationProbabilities; +import com.graphicsfuzz.util.ArgsUtil; import com.graphicsfuzz.util.Constants; import java.io.File; import java.io.IOException; @@ -61,7 +61,6 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; -import java.util.Random; import java.util.function.Supplier; import java.util.stream.Collectors; import net.sourceforge.argparse4j.ArgumentParsers; @@ -90,12 +89,8 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .help("Path of folder of donor shaders.") .type(File.class); - parser.addArgument("glsl-version") - .help("Version of GLSL to target.") - .type(String.class); - parser.addArgument("output") - .help("Output shader job file file (.json.") + .help("Output shader job file file (.json).") .type(File.class); addGeneratorCommonArguments(parser); @@ -106,8 +101,8 @@ private static Namespace parse(String[] args) throws ArgumentParserException { public static void addGeneratorCommonArguments(ArgumentParser parser) { parser.addArgument("--seed") - .help("Seed to initialize random number generator with.") - .type(Integer.class); + .help("Seed (unsigned 64 bit long integer) for the random number generator.") + .type(String.class); parser.addArgument("--small") .help("Try to generate small shaders.") @@ -169,9 +164,8 @@ public static void addGeneratorCommonArguments(ArgumentParser parser) { */ public static StringBuilder generateVariant(ShaderJob shaderJob, GeneratorArguments args, - int seed) { + IRandom random) { final StringBuilder result = new StringBuilder(); - final IRandom random = new RandomWrapper(seed); if (args.getAddInjectionSwitch()) { for (TranslationUnit shader : shaderJob.getShaders()) { @@ -211,7 +205,7 @@ public static StringBuilder generateVariant(ShaderJob shaderJob, * @param referenceShaderJobFile The shader job to be transformed. * @param outputShaderJobFile Output file for the variant. * @param generatorArguments Arguments to control generation. - * @param seed Seed for random number generation. + * @param random Random number generator. * @param writeProbabilities Records whether details about probabilities should be written. * @throws IOException if file reading or writing goes wrong. * @throws ParseTimeoutException if parsing takes too long. @@ -219,11 +213,12 @@ public static StringBuilder generateVariant(ShaderJob shaderJob, * preprocessor or validator. * @throws GlslParserException if a shader in the job fails to parse. */ + @SuppressWarnings("deprecation") public static void generateVariant(ShaderJobFileOperations fileOps, File referenceShaderJobFile, File outputShaderJobFile, GeneratorArguments generatorArguments, - int seed, + IRandom random, boolean writeProbabilities) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { // This is mutated into the variant. @@ -232,7 +227,7 @@ public static void generateVariant(ShaderJobFileOperations fileOps, final StringBuilder generationInfo = generateVariant( variantShaderJob, generatorArguments, - seed); + random); fileOps.writeShaderJobFile( variantShaderJob, @@ -253,7 +248,7 @@ private static StringBuilder transformShader(TranslationUnit shaderToTransform, GeneratorArguments args) { final ShaderKind shaderKind = shaderToTransform.getShaderKind(); StringBuilder result = new StringBuilder(); - result.append("======\n" + shaderKind + ":\n"); + result.append("======\n").append(shaderKind).append(":\n"); if (args.getReplaceFloatLiterals()) { FloatLiteralReplacer.replace( @@ -310,6 +305,9 @@ private static StringBuilder transformShader(TranslationUnit shaderToTransform, public static void main(String[] args) { try { mainHelper(args); + } catch (ArgumentParserException exception) { + exception.getParser().handleError(exception); + System.exit(1); } catch (Throwable exception) { LOGGER.error("", exception); System.exit(1); @@ -321,17 +319,14 @@ public static void mainHelper(String[] args) GlslParserException { final Namespace ns = parse(args); - Integer seed = ns.get("seed"); - if (seed == null) { - seed = new Random().nextInt(); - } + final IRandom random = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); generateVariant( new ShaderJobFileOperations(), ns.get("reference_json"), ns.get("output"), getGeneratorArguments(ns), - seed, + random, ns.getBoolean("write_probabilities")); } @@ -418,20 +413,20 @@ private static String applyTransformationsMultiPass(GeneratorArguments args, } List nextRoundTransformations = new ArrayList<>(); - String result = ""; + final StringBuilder result = new StringBuilder(); // Keep applying transformations until all transformations cease to be effective, or // we get a large enough shader. while (!transformations.isEmpty() && !shaderLargeEnough(reference, generator)) { ITransformation transformation = transformations.remove(generator.nextInt( transformations.size())); - result += transformation.getName() + "\n"; + result.append(transformation.getName()).append("\n"); if (transformation.apply(reference, probabilities, generator.spawnChild(), generationParams)) { // Keep the size down by stripping unused stuff. StripUnusedFunctions.strip(reference); StripUnusedGlobals.strip(reference); - assert canTypeCheckWithoutFailure(reference, reference.getShadingLanguageVersion()); + assert canTypeCheckWithoutFailure(reference); // Only if the transformation applied successfully (i.e., made a change), do we add it // to the list of transformations to be applied next round. @@ -442,14 +437,13 @@ private static String applyTransformationsMultiPass(GeneratorArguments args, nextRoundTransformations = new ArrayList<>(); } } - return result; + return result.toString(); } - private static boolean canTypeCheckWithoutFailure(TranslationUnit reference, - ShadingLanguageVersion shadingLanguageVersion) { + private static boolean canTypeCheckWithoutFailure(TranslationUnit reference) { // Debugging aid: fail early if we end up messing up the translation unit so that type checking // does not work. - new Typer(reference, shadingLanguageVersion); + new Typer(reference); return true; } @@ -591,7 +585,7 @@ private static String applyControlFlowComplication(GeneratorArguments args, public static void randomiseUnsetUniforms(TranslationUnit tu, PipelineInfo pipelineInfo, IRandom generator) { - final Supplier floatSupplier = () -> generator.nextFloat(); + final Supplier floatSupplier = generator::nextFloat; final Supplier intSupplier = () -> generator.nextInt(1 << 15); final Supplier uintSupplier = () -> generator.nextInt(1 << 15); final Supplier boolSupplier = () -> generator.nextInt(2); @@ -610,14 +604,14 @@ public static void addInjectionSwitchIfNotPresent(TranslationUnit tu) { private static boolean alreadyDeclaresInjectionSwitch(TranslationUnit tu) { return tu.getGlobalVarDeclInfos() .stream() - .map(item -> item.getName()) + .map(VariableDeclInfo::getName) .collect(Collectors.toList()) .contains(Constants.INJECTION_SWITCH); } public static void setInjectionSwitch(PipelineInfo pipelineInfo) { pipelineInfo.addUniform(Constants.INJECTION_SWITCH, BasicType.VEC2, Optional.empty(), - Arrays.asList(new Float(0.0), new Float(1.0))); + Arrays.asList(0.0f, 1.0f)); } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/GenerateShaderFamily.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/GenerateShaderFamily.java index 68f70f514..5900c31b4 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/GenerateShaderFamily.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/GenerateShaderFamily.java @@ -128,7 +128,7 @@ public static void mainHelper(String[] args) throws ArgumentParserException, final boolean writeProbabilities = ns.getBoolean("write_probabilities"); final boolean keepBadVariants = ns.getBoolean("keep_bad_variants"); final boolean stopOnFail = ns.getBoolean("stop_on_fail"); - final int seed = ArgsUtil.getSeedArgument(ns); + final IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final int numVariants = ns.getInt("num_variants"); Optional maxBytes = ns.get("max_bytes") == null ? Optional.empty() : Optional.of(ns.getInt("max_bytes")); @@ -137,7 +137,7 @@ public static void mainHelper(String[] args) throws ArgumentParserException, final GeneratorArguments generatorArguments = Generate.getGeneratorArguments(ns); if (verbose) { - LOGGER.info("Using seed " + seed); + LOGGER.info("Using random: " + generator.getDescription()); } if (!referenceShaderJob.isFile()) { @@ -207,8 +207,6 @@ public static void mainHelper(String[] args) throws ArgumentParserException, int generatedVariants = 0; int triedVariants = 0; - final IRandom generator = new RandomWrapper(seed); - // Main variant generation loop while (generatedVariants < numVariants) { if (verbose) { @@ -216,29 +214,28 @@ public static void mainHelper(String[] args) throws ArgumentParserException, + " of " + numVariants + ")"); } // Generate a variant - final int innerSeed = generator.nextInt(Integer.MAX_VALUE); - if (verbose) { - LOGGER.info("Generating variant with inner seed " + innerSeed); - } final File variantShaderJobFile = new File(outputDir, "variant_" + String.format("%03d", generatedVariants) + ".json"); + final IRandom childRandom = generator.spawnChild(); + if (verbose) { + LOGGER.info("Generating variant with inner random: " + childRandom.getDescription()); + } triedVariants++; try { Generate.generateVariant(fileOps, referenceShaderJob, variantShaderJobFile, - generatorArguments, innerSeed, writeProbabilities); + generatorArguments, childRandom, writeProbabilities); } catch (Exception exception) { - if (verbose) { - LOGGER.error("Failed generating variant: " + exception.getMessage() - + exception.getStackTrace() + /*if (verbose)*/ { + LOGGER.error("Failed generating variant: " + "\nGenerator arguments: " + generatorArguments + "\nReference shader job: " + referenceShaderJob - + "\nSeed: " + innerSeed); + + "\nRandom: " + childRandom.getDescription(), exception); } if (stopOnFail) { final String message = "Failed generating a variant, stopping."; LOGGER.info(message); - throw new RuntimeException(message); + throw new RuntimeException(message, exception); } continue; } @@ -283,7 +280,7 @@ public static void mainHelper(String[] args) throws ArgumentParserException, StandardCharsets.UTF_8) : "none"; infoLog.addProperty("git_hash", hashContents); infoLog.addProperty("args", String.join(" ", args)); - infoLog.addProperty("seed", seed); + infoLog.addProperty("seed", generator.getDescription()); // Pretty-print the info log. FileUtils.writeStringToFile(new File(outputDir,"infolog.json"), diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/GlslGenerate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/GlslGenerate.java index d7028ac21..46d602414 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/GlslGenerate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/GlslGenerate.java @@ -83,7 +83,7 @@ public static void mainHelper(String[] args, boolean failOnReferencePreparationE final String prefix = ns.get("prefix"); final int numVariants = ns.getInt("num_variants"); final boolean verbose = ns.getBoolean("verbose"); - final int seed = ArgsUtil.getSeedArgument(ns); + final IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); @@ -100,7 +100,6 @@ public static void mainHelper(String[] args, boolean failOnReferencePreparationE + " variant" + (numVariants == 1 ? "" : "s") + "."); } - final IRandom generator = new RandomWrapper(seed); int referenceCount = 0; for (File shaderJobFile : referenceShaderJobFiles) { LOGGER.info("Generating family " + referenceCount + " from reference " diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/KnownValueShaderGenerator.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/KnownValueShaderGenerator.java new file mode 100644 index 000000000..e6cf78c05 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/KnownValueShaderGenerator.java @@ -0,0 +1,196 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.tool; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; +import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.expr.BinOp; +import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.TypeConstructorExpr; +import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.NullStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.LayoutQualifierSequence; +import com.graphicsfuzz.common.ast.type.LocationLayoutQualifier; +import com.graphicsfuzz.common.ast.type.QualifiedType; +import com.graphicsfuzz.common.ast.type.TypeQualifier; +import com.graphicsfuzz.common.ast.type.VoidType; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.transformreduce.GlslShaderJob; +import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.PipelineInfo; +import com.graphicsfuzz.common.util.PruneUniforms; +import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderJobFileOperations; +import com.graphicsfuzz.generator.knownvaluegeneration.ExpressionGenerator; +import com.graphicsfuzz.generator.knownvaluegeneration.FactManager; +import com.graphicsfuzz.generator.knownvaluegeneration.NumericValue; +import com.graphicsfuzz.util.ArgsUtil; +import com.graphicsfuzz.util.Constants; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; +import java.util.stream.Stream; +import net.sourceforge.argparse4j.ArgumentParsers; +import net.sourceforge.argparse4j.impl.Arguments; +import net.sourceforge.argparse4j.inf.ArgumentParser; +import net.sourceforge.argparse4j.inf.ArgumentParserException; +import net.sourceforge.argparse4j.inf.Namespace; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KnownValueShaderGenerator { + + private static final Logger LOGGER = LoggerFactory.getLogger(KnownValueShaderGenerator.class); + + private static Namespace parse(String[] args) throws ArgumentParserException { + ArgumentParser parser = ArgumentParsers.newArgumentParser("KnownValueShaderGenerator") + .defaultHelp(true) + .description("Generate a known value shader with the desired RGBA color."); + + parser.addArgument("output") + .help("Output shader job file file (.json).") + .type(File.class); + + parser.addArgument("r") + .help("Floating point number of red value.") + .type(Float.class); + + parser.addArgument("g") + .help("Floating point number of green value.") + .type(Float.class); + + parser.addArgument("b") + .help("Floating point number of blue value.") + .type(Float.class); + + parser.addArgument("a") + .help("Floating point number of alpha value.") + .type(Float.class); + + parser.addArgument("--version") + .help("Shading language version of a generated fragment shader.") + .setDefault("410") + .type(String.class); + + parser.addArgument("--seed") + .help("Seed (unsigned 64 bit long integer) for the random number generator.") + .type(String.class); + + parser.addArgument("--max-uniforms") + .help("Ensure that generated shaders have no more than the given number of uniforms; " + + "required for Vulkan compatibility.") + .setDefault(0) + .type(Integer.class); + + parser.addArgument("--generate-uniform-bindings") + .help("Put all uniforms in uniform blocks and generate bindings; required for Vulkan " + + "compatibility.") + .action(Arguments.storeTrue()); + + return parser.parseArgs(args); + } + + public static void mainHelper(String[] args) throws ArgumentParserException, IOException { + final Namespace ns = parse(args); + final float rFloat = ns.getFloat("r"); + final float gFloat = ns.getFloat("g"); + final float bFloat = ns.getFloat("b"); + final float aFloat = ns.getFloat("a"); + final int maxUniforms = ns.getInt("max_uniforms"); + final boolean generateUniformBindings = ns.getBoolean("generate_uniform_bindings"); + final IRandom generator = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); + final String version = ns.getString("version"); + final File shaderJobFile = ns.get("output"); + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + + final TranslationUnit tu = + new TranslationUnit(Optional.of(ShadingLanguageVersion.fromVersionString(version)), + Arrays.asList( + new PrecisionDeclaration("precision highp float;"), + new PrecisionDeclaration("precision highp int;"), + new VariablesDeclaration(new QualifiedType(BasicType.VEC4, + Arrays.asList(new LayoutQualifierSequence( + new LocationLayoutQualifier(0)), TypeQualifier.SHADER_OUTPUT)), + new VariableDeclInfo("_GLF_color", null, null)), + new FunctionDefinition( + new FunctionPrototype("main", VoidType.VOID, Collections.emptyList()), + new BlockStmt(new ArrayList<>(), false)))); + + final PipelineInfo pipelineInfo = new PipelineInfo(); + final FactManager globalFactManager = new FactManager(null); + + // A placeholder statement for what will eventually be the color assignment. + final Stmt placeholderForColorAssignment = new NullStmt(); + tu.getMainFunction().getBody().addStmt(placeholderForColorAssignment); + + LOGGER.info("About to generate the known value fragment shader" + + " with the parameters R = " + rFloat + ", G = " + gFloat + ", B = " + bFloat + " and" + + " A = " + aFloat + "."); + + final ExpressionGenerator expressionGenerator = new + ExpressionGenerator(tu, pipelineInfo, generator, globalFactManager); + final FactManager mainFactManager = globalFactManager.newScope(); + final Expr[] rgbaExprs = Stream.of(rFloat, gFloat, bFloat, aFloat) + .map(item -> expressionGenerator.generateExpr( + mainFactManager, + tu.getMainFunction(), + placeholderForColorAssignment, + new NumericValue(BasicType.FLOAT, Optional.of(item)))).toArray(Expr[]::new); + + tu.getMainFunction().getBody().replaceChild(placeholderForColorAssignment, + new ExprStmt(new BinaryExpr(new VariableIdentifierExpr("_GLF_color"), + new TypeConstructorExpr("vec4", rgbaExprs), BinOp.ASSIGN))); + + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), pipelineInfo, tu); + + if (maxUniforms > 0) { + PruneUniforms.prune(shaderJob, maxUniforms, Arrays.asList(Constants.GLF_UNIFORM)); + } + if (generateUniformBindings) { + shaderJob.makeUniformBindings(); + } + + fileOps.writeShaderJobFile(shaderJob, shaderJobFile); + LOGGER.info("Generation complete."); + } + + public static void main(String[] args) { + try { + mainHelper(args); + } catch (ArgumentParserException exception) { + exception.getParser().handleError(exception); + System.exit(1); + } catch (IOException exception) { + exception.printStackTrace(); + System.exit(1); + } + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java index f839f09c1..31ff13355 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/Mutate.java @@ -18,12 +18,15 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.generator.mutateapi.Mutation; import com.graphicsfuzz.generator.mutateapi.MutationFinder; import com.graphicsfuzz.generator.semanticschanging.AddArrayMutationFinder; @@ -62,6 +65,7 @@ import net.sourceforge.argparse4j.inf.ArgumentParser; import net.sourceforge.argparse4j.inf.ArgumentParserException; import net.sourceforge.argparse4j.inf.Namespace; +import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,8 +88,8 @@ private static Namespace parse(String[] args) throws ArgumentParserException { .type(File.class); parser.addArgument("--seed") - .help("Seed to initialize random number generator with.") - .type(Integer.class); + .help("Seed (unsigned 64 bit long integer) for the random number generator.") + .type(String.class); return parser.parseArgs(args); } @@ -123,24 +127,21 @@ public static void mainHelper(String[] args) throws ArgumentParserException, IOE final File input = ns.get("input"); final File output = ns.get("output"); - final int seed = ArgsUtil.getSeedArgument(ns); + IRandom random = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final TranslationUnit tu = ParseHelper.parse(input); - LOGGER.info("Mutating from " + input + " to " + output + " with seed " + seed); + LOGGER.info("Mutating from " + input + " to " + output + " with RNG " + + random.getDescription()); - mutate(tu, new RandomWrapper(seed)); + mutate(tu, random); - try (PrintStream stream = new PrintStream(new FileOutputStream(output))) { - PrettyPrinterVisitor.emitShader( - tu, - Optional.empty(), - stream, - PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, - false - ); - } + final File shaderJobFile = new File(FilenameUtils.removeExtension(output.getName()) + ".json"); + + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo("{}"), + tu), shaderJobFile); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java b/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java index edf877f9b..8def96ba2 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/tool/PrepareReference.java @@ -18,6 +18,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.util.AddBraces; import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.ParseTimeoutException; import com.graphicsfuzz.common.util.PipelineInfo; @@ -150,6 +151,11 @@ private static void prepareReferenceShader(TranslationUnit tu, if (replaceFloatLiterals) { FloatLiteralReplacer.replace(tu, pipelineInfo); } + + // Ensure that all if-then-else statements have braces. This makes the reference easier to + // compare with a reduced variant. + AddBraces.transform(tu); + } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java index f25a18bca..0979e00dc 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateCodeTransformation.java @@ -22,9 +22,9 @@ import com.graphicsfuzz.common.ast.decl.Declaration; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.ParameterDecl; import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; @@ -33,7 +33,6 @@ import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.type.ArrayType; import com.graphicsfuzz.common.ast.type.BasicType; -import com.graphicsfuzz.common.ast.type.LayoutQualifier; import com.graphicsfuzz.common.ast.type.LayoutQualifierSequence; import com.graphicsfuzz.common.ast.type.QualifiedType; import com.graphicsfuzz.common.ast.type.StructDefinitionType; @@ -43,7 +42,7 @@ import com.graphicsfuzz.common.ast.visitors.StandardVisitor; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.typing.Scope; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.typing.Typer; import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.IRandom; @@ -68,24 +67,35 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; public abstract class DonateCodeTransformation implements ITransformation { - final Function probabilityOfDonation; + private final Function probabilityOfDonation; + // During a single donation pass, this is populated on demand with the donors that are used. private Map donorsToTranslationUnits; + private final List functionPrototypes; private final Map globalVariables; private final Set structNames; - private final List donorFiles; + + // 'donorFiles' contains those donors that have not yet been used for a code donation. Once a + // donor has been used, it is moved to 'usedDonorFiles' (unless it is found to be incompatible, + // in which case it is discarded). If 'donorFiles' becomes empty, it and 'usedDonorFiles' are + // swapped, so that previously used donors can be used again. + private List donorFiles; + private List usedDonorFiles; + final GenerationParams generationParams; private int translationUnitCount; @@ -96,13 +106,14 @@ public DonateCodeTransformation(Function probabilityOfDonation this.functionPrototypes = new ArrayList<>(); this.globalVariables = new HashMap<>(); this.structNames = new HashSet<>(); + assert donorsDirectory.exists(); this.donorFiles = new ArrayList<>(); + this.donorFiles.addAll(Arrays.asList(donorsDirectory.listFiles( + pathname -> pathname.getName().endsWith(".frag")))); + this.donorFiles.sort(Comparator.naturalOrder()); + this.usedDonorFiles = new ArrayList<>(); this.generationParams = generationParams; this.translationUnitCount = 0; - assert donorsDirectory.exists(); - donorFiles.addAll(Arrays.asList(donorsDirectory.listFiles( - pathname -> pathname.getName().endsWith(".frag")))); - donorFiles.sort(Comparator.naturalOrder()); } /** @@ -116,20 +127,19 @@ public DonateCodeTransformation(Function probabilityOfDonation private TranslationUnit prepareTranslationUnit(File donorFile, IRandom generator) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { - TranslationUnit tu = ParseHelper.parse(donorFile); + final TranslationUnit tu = ParseHelper.parse(donorFile); + final Typer typer = new Typer(tu); + // To avoid undefined behaviours, make all array access in bounds for every donor. + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); addPrefixes(tu, getDeclaredFunctionNames(tu)); // Add prefixed versions of these builtins, in case they are used. // Use explicit precision qualifier to avoid introducing errors if there are no float precision // qualifiers. - List coordQualifiers = new ArrayList<>(); - coordQualifiers.add(TypeQualifier.MEDIUMP); tu.addDeclaration(new VariablesDeclaration( - new QualifiedType(BasicType.VEC4, coordQualifiers), + new QualifiedType(BasicType.VEC4, Collections.singletonList(TypeQualifier.MEDIUMP)), new VariableDeclInfo(addPrefix(OpenGlConstants.GL_FRAG_COORD), null, null))); - List colorQualifiers = new ArrayList<>(); - colorQualifiers.add(TypeQualifier.MEDIUMP); tu.addDeclaration(new VariablesDeclaration( - new QualifiedType(BasicType.VEC4, colorQualifiers), + new QualifiedType(BasicType.VEC4, Collections.singletonList(TypeQualifier.MEDIUMP)), new VariableDeclInfo(addPrefix(OpenGlConstants.GL_FRAG_COLOR), null, null))); adaptTranslationUnitForSpecificDonation(tu, generator); translationUnitCount++; @@ -209,11 +219,19 @@ private Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, final int maxTries = 10; int tries = 0; while (true) { - DonationContext donationContext = new DonationContextFinder(chooseDonor(generator), generator) + final Optional maybeDonor = chooseDonor(generator); + if (!maybeDonor.isPresent()) { + // No compatible donors were found, thus we cannot do serious code donation here; + // we return a null statement instead. + return new NullStmt(); + } + DonationContext donationContext = new DonationContextFinder(maybeDonor.get(), generator) .getDonationContext(); if (incompatible(injectionPoint, donationContext, shadingLanguageVersion)) { tries++; if (tries == maxTries) { + // We have tried and tried to find something compatible to inject but not managed; + // return a null statement instead of a real piece of code to inject. return new NullStmt(); } } else { @@ -251,28 +269,25 @@ public boolean apply(TranslationUnit tu, } donateFunctionsAndGlobals(tu); eliminateUsedDonors(); - makeInjectedArrayAccessesInBounds(tu, injectedStmts, tu.getShadingLanguageVersion()); - return !injectionPoints.isEmpty(); } private void eliminateUsedDonors() { - // Having done donation using a particular donor, we remove it so that we do not use it - // again in a future pass. + // Having done donation using a particular donor, we move it to the list of used donors so that + // we only use it again in a future pass once all other donors have been tried. for (File donor : donorsToTranslationUnits.keySet()) { + assert donorFiles.contains(donor); donorFiles.remove(donor); + usedDonorFiles.add(donor); } - donorsToTranslationUnits = new HashMap<>(); - } - - private void makeInjectedArrayAccessesInBounds(TranslationUnit tu, - List injectedStmts, - ShadingLanguageVersion shadingLanguageVersion) { - Typer typer = new Typer(tu, shadingLanguageVersion); - for (Stmt stmt : injectedStmts) { - MakeArrayAccessesInBounds.makeInBounds(stmt, typer); + // If there are no donors left -- i.e., if all have been used, then recycle the list of used + // donors. + if (donorFiles.isEmpty()) { + donorFiles = usedDonorFiles; + usedDonorFiles = new ArrayList<>(); } + donorsToTranslationUnits = new HashMap<>(); } private boolean incompatible(IInjectionPoint injectionPoint, DonationContext donationContext, @@ -293,24 +308,24 @@ private boolean incompatible(IInjectionPoint injectionPoint, DonationContext don } private Map getGlobalVariablesFromShader(TranslationUnit shader) { - return new ScopeTreeBuilder() { + return new ScopeTrackingVisitor() { Map getGlobalsFromShader(TranslationUnit shader) { visit(shader); Map result = new HashMap<>(); - for (String globalName : currentScope.keys()) { - result.put(globalName, currentScope.lookupType(globalName)); + for (String globalName : getCurrentScope().keys()) { + result.put(globalName, getCurrentScope().lookupType(globalName)); } return result; } }.getGlobalsFromShader(shader); } - final ScalarInitializer getScalarInitializer(IInjectionPoint injectionPoint, - DonationContext donationContext, - Type type, - boolean restrictToConst, - IRandom generator, - ShadingLanguageVersion shadingLanguageVersion) { + final Initializer getInitializer(IInjectionPoint injectionPoint, + DonationContext donationContext, + Type type, + boolean restrictToConst, + IRandom generator, + ShadingLanguageVersion shadingLanguageVersion) { final boolean isConst = type.hasQualifier(TypeQualifier.CONST); try { @@ -323,7 +338,7 @@ final ScalarInitializer getScalarInitializer(IInjectionPoint injectionPoint, scopeForFuzzing.addStructDefinition(sdt); } - return new ScalarInitializer( + return new Initializer( new OpaqueExpressionGenerator(generator, generationParams, shadingLanguageVersion) .fuzzedConstructor( new Fuzzer(new FuzzingContext(scopeForFuzzing), @@ -413,14 +428,8 @@ private void donateFunctionsAndGlobals(TranslationUnit recipient) { newRecipientTopLevelDeclarations.addAll(recipient.getTopLevelDeclarations().stream() .filter(d -> d instanceof FunctionDefinition).collect(Collectors.toList())); - // Make sure we clone the top-level FunctionPrototypes that are added, otherwise each will - // exist twice in the AST: first as a top-level declaration, second as part of the - // FunctionDefinition. newRecipientTopLevelDeclarations = - addNecessaryForwardDeclarations(newRecipientTopLevelDeclarations) - .stream() - .map(Declaration::clone) - .collect(Collectors.toList()); + addNecessaryForwardDeclarations(newRecipientTopLevelDeclarations); recipient.setTopLevelDeclarations(newRecipientTopLevelDeclarations); @@ -480,12 +489,16 @@ private List addNecessaryForwardDeclarations(List decl Set calledFunctionNames = getCalledFunctions(decls.get(i)); - Set toDeclare = + List toDeclare = functionsDefinedAfterDecl.get(i).stream().filter(item -> calledFunctionNames.contains(item.getName())).filter(item -> !declared.stream().anyMatch(alreadyDeclared -> alreadyDeclared.matches(item))) - .collect(Collectors.toSet()); - result.addAll(toDeclare); + .collect(Collectors.toList()); + + // Make sure we clone the FunctionPrototypes that are added, otherwise each will exist + // twice in the AST: first as a top-level declaration, second as part of the + // FunctionDefinition. + result.addAll(toDeclare.stream().map(Declaration::clone).collect(Collectors.toList())); declared.addAll(toDeclare); } if (decls.get(i) instanceof FunctionDefinition) { @@ -584,25 +597,49 @@ private boolean prototypeMatches(FunctionPrototype fp, List f return !fs.stream().filter(item -> fp.matches(item)).collect(Collectors.toList()).isEmpty(); } - TranslationUnit chooseDonor(IRandom generator) { + Optional chooseDonor(IRandom generator) { + // The donors that we have previously selected during this donation pass are captured via + // 'donorsToTranslationUnits'. Furthermore, there is a maximum number of distinct donors we + // are allowed to use per donation pass. So first check whether the donors we have already + // used have hit this maximum. + + final String previouslyUsedDonorFoundToBeIncompatibleMessage = "A donor that was previously " + + "used has now been found to be incompatible. This should not occur."; + + if (donorsToTranslationUnits.size() >= generationParams.getMaxDonorsPerDonationPass()) { + // We have used the maximum number of donors we are allowed to use in this pass, so choose + // one of them again. + // The limit should be reached but never exceeded. + assert donorsToTranslationUnits.size() == generationParams.getMaxDonorsPerDonationPass(); + final List sortedKeys = new ArrayList<>(donorsToTranslationUnits.keySet()); + sortedKeys.sort(Comparator.naturalOrder()); + try { + return Optional.of(getDonorTranslationUnit(sortedKeys.get(generator.nextInt(sortedKeys + .size())), + generator)); + } catch (IncompatibleDonorException exception) { + throw new RuntimeException(previouslyUsedDonorFoundToBeIncompatibleMessage); + } + } + + // We have not reached the limit. So we can choose any one of the available donor files. It + // might turn out to be a donor we tried before (in which case it will already be a key to + // the 'donorsToTranslationUnits' map), or it may be new. while (!donorFiles.isEmpty()) { - int index = generator.nextInt(donorFiles.size()); - File donorFile = donorFiles.get(index); + final File candidateDonorFile = donorFiles + .get(generator.nextInt(donorFiles.size())); try { - if (donorsToTranslationUnits.keySet().size() >= generationParams.getMaxDonors() - && !donorsToTranslationUnits.containsKey(donorFile)) { - // We have reached the donor limit; try again until we find a donor that we've - // already used - throw new IncompatibleDonorException(); - } - return getDonorTranslationUnit(donorFile, generator); + return Optional.of(getDonorTranslationUnit(candidateDonorFile, generator)); } catch (IncompatibleDonorException exception) { - assert index >= 0; - assert index < donorFiles.size(); - donorFiles.remove(index); + if (donorsToTranslationUnits.containsKey(candidateDonorFile)) { + throw new RuntimeException(previouslyUsedDonorFoundToBeIncompatibleMessage); + } + donorFiles.remove(candidateDonorFile); } } - throw new RuntimeException("Could not find any compatible donors."); + // We did not manage to find any suitable donor. This happens if all donors prove to be + // incompatible. + return Optional.empty(); } private TranslationUnit getDonorTranslationUnit(File donorFile, IRandom generator) @@ -626,10 +663,11 @@ private TranslationUnit getDonorTranslationUnit(File donorFile, IRandom generato } private boolean compatibleDonor(TranslationUnit donor) { - List usedFunctionNames = functionPrototypes.stream().map(item -> item.getName()) + final List usedFunctionNames = functionPrototypes.stream() + .map(FunctionPrototype::getName) .collect(Collectors.toList()); - Set usedGlobalVariableNames = globalVariables.keySet(); - Set usedStructNames = structNames; + final Set usedGlobalVariableNames = globalVariables.keySet(); + final Set usedStructNames = structNames; for (FunctionPrototype donorPrototype : AstUtil.getFunctionPrototypesFromShader(donor)) { if (usedGlobalVariableNames.contains(donorPrototype.getName()) @@ -641,8 +679,8 @@ private boolean compatibleDonor(TranslationUnit donor) { } } for (Map.Entry global : getGlobalVariablesFromShader(donor).entrySet()) { - String name = global.getKey(); - Type type = global.getValue(); + final String name = global.getKey(); + final Type type = global.getValue(); if (usedFunctionNames.contains(name) || usedStructNames.contains(name)) { return false; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformation.java index c16d9a689..a66111e79 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformation.java @@ -18,7 +18,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.stmt.BlockStmt; @@ -39,7 +39,9 @@ import com.graphicsfuzz.generator.transformation.injection.IInjectionPoint; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.RemoveDiscardStatements; -import com.graphicsfuzz.generator.util.RemoveImmediateBreakAndContinueStatements; +import com.graphicsfuzz.generator.util.RemoveImmediateBreakStatements; +import com.graphicsfuzz.generator.util.RemoveImmediateCaseLabels; +import com.graphicsfuzz.generator.util.RemoveImmediateContinueStatements; import com.graphicsfuzz.generator.util.RemoveReturnStatements; import com.graphicsfuzz.generator.util.TransformationProbabilities; import com.graphicsfuzz.util.Constants; @@ -97,7 +99,7 @@ Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, DonationContext do String newName = "donor_replacement" + name; substitution.put(name, newName); - final ScalarInitializer initializer = getScalarInitializer( + final Initializer initializer = getInitializer( injectionPoint, donationContext, type, @@ -125,8 +127,24 @@ Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, DonationContext do shadingLanguageVersion, generator, generationParams)), new BlockStmt(donatedStmts, true), null); + // We cannot have 'case' and 'default' labels occurring in the donated statement, unless they + // are nested in their own 'switch' statement. In principle we could allow these if the + // injection point happens to be in a 'switch' already, but this would require some fiddly + // checks that do not seem worth doing. + new RemoveImmediateCaseLabels(donatedStmt); + + // If the injection point is in a loop then it's fine for the injected code to have continue + // statements (this is a dead code injection, so semantics will not be changed). But otherwise + // they would make the shader invalid. if (!injectionPoint.inLoop()) { - new RemoveImmediateBreakAndContinueStatements(donatedStmt); + new RemoveImmediateContinueStatements(donatedStmt); + } + + // If the injection point is in a loop or switch then it's fine for the injected code to have + // break statements (this is a dead code injection, so semantics will not be changed). But + // otherwise they would make the shader invalid. + if (!(injectionPoint.inLoop() || injectionPoint.inSwitch())) { + new RemoveImmediateBreakStatements(donatedStmt); } if (generationParams.getShaderKind() != ShaderKind.FRAGMENT) { diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java index 4618db351..db5cf1df5 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformation.java @@ -17,7 +17,7 @@ package com.graphicsfuzz.generator.transformation; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; @@ -33,7 +33,9 @@ import com.graphicsfuzz.generator.transformation.injection.IInjectionPoint; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.RemoveDiscardStatements; -import com.graphicsfuzz.generator.util.RemoveImmediateBreakAndContinueStatements; +import com.graphicsfuzz.generator.util.RemoveImmediateBreakStatements; +import com.graphicsfuzz.generator.util.RemoveImmediateCaseLabels; +import com.graphicsfuzz.generator.util.RemoveImmediateContinueStatements; import com.graphicsfuzz.generator.util.RemoveReturnStatements; import com.graphicsfuzz.generator.util.TransformationProbabilities; import com.graphicsfuzz.util.Constants; @@ -58,7 +60,7 @@ public DonateLiveCodeTransformation(Function probabilityOfDona } @Override - public Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, + Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, DonationContext donationContext, TransformationProbabilities probabilities, IRandom generator, ShadingLanguageVersion shadingLanguageVersion) { @@ -72,13 +74,13 @@ public Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, } type = dropQualifiersThatCannotBeUsedForLocalVariable(type); - ScalarInitializer initializer; + Initializer initializer; // We fuzz a const expression because we need to ensure we don't generate side-effects to // non-injected code if (isLoopLimiter(vars.getKey(), type.getWithoutQualifiers())) { - initializer = new ScalarInitializer(new IntConstantExpr("0")); + initializer = new Initializer(new IntConstantExpr("0")); } else { - initializer = getScalarInitializer(injectionPoint, donationContext, type, true, + initializer = getInitializer(injectionPoint, donationContext, type, true, generator, shadingLanguageVersion); } donatedStmts.add(new DeclarationStmt( @@ -87,7 +89,9 @@ public Stmt prepareStatementToDonate(IInjectionPoint injectionPoint, } donatedStmts.add(donationContext.getDonorFragment()); BlockStmt donatedStmt = new BlockStmt(donatedStmts, true); - new RemoveImmediateBreakAndContinueStatements(donatedStmt); + new RemoveImmediateBreakStatements(donatedStmt); + new RemoveImmediateContinueStatements(donatedStmt); + new RemoveImmediateCaseLabels(donatedStmt); new RemoveReturnStatements(donatedStmt); new RemoveDiscardStatements(donatedStmt); return donatedStmt; @@ -101,7 +105,7 @@ String getPrefix() { @Override void adaptTranslationUnitForSpecificDonation(TranslationUnit tu, IRandom generator) { if (!allowLongLoops) { - new TruncateLoops(3 + generator.nextInt(5), addPrefix(""), tu, false); + new TruncateLoops(3 + generator.nextInt(5), addPrefix(""), tu); } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/ExpressionIdentity.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/ExpressionIdentity.java index ec613a8b1..d3290aeae 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/ExpressionIdentity.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/ExpressionIdentity.java @@ -24,6 +24,6 @@ public interface ExpressionIdentity { Expr apply(Expr expr, BasicType type, boolean constContext, final int depth, Fuzzer fuzzer); - boolean preconditionHolds(Expr expr, BasicType basicType); + boolean preconditionHolds(Expr expr, BasicType basicType, boolean constContext); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/DonationContext.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/DonationContext.java index fa04733c5..eb05ff76a 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/DonationContext.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/DonationContext.java @@ -25,7 +25,7 @@ import com.graphicsfuzz.common.ast.type.StructDefinitionType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -88,7 +88,7 @@ public boolean indexesArrayUsingFreeVariable() { // Note: we don't use the freeVariables member here, because we want to account for // name shadowing. - return new ScopeTreeBuilder() { + return new ScopeTrackingVisitor() { private boolean found = false; @@ -105,7 +105,7 @@ public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); if (arrayIndexDepth > 0 - && currentScope.lookupScopeEntry(variableIdentifierExpr.getName()) == null) { + && getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()) == null) { // A free variable that appears under an array index found = true; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java index 672280532..6c8ad07fc 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBounds.java @@ -17,20 +17,26 @@ package com.graphicsfuzz.generator.transformation.donation; import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.expr.ParenExpr; import com.graphicsfuzz.common.ast.expr.TernaryExpr; +import com.graphicsfuzz.common.ast.expr.UIntConstantExpr; import com.graphicsfuzz.common.ast.type.ArrayType; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.typing.Typer; +import com.graphicsfuzz.util.Constants; -public class MakeArrayAccessesInBounds extends StandardVisitor { +public class MakeArrayAccessesInBounds extends ScopeTrackingVisitor { private Typer typer; @@ -38,12 +44,19 @@ private MakeArrayAccessesInBounds(Typer typer) { this.typer = typer; } - public static void makeInBounds(IAstNode node, Typer typer) { + /** + * Clamp array / vector / matrix accesses to ensure they will be in-bounds. + * @param node AST node under which accesses should be made in bound. + * @param typer Type info for the AST. + * @param tu The translation unit in which node is contained. + */ + public static void makeInBounds(IAstNode node, Typer typer, TranslationUnit tu) { new MakeArrayAccessesInBounds(typer).visit(node); } @Override public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { + super.visitArrayIndexExpr(arrayIndexExpr); Type type = typer.lookupType(arrayIndexExpr.getArray()); if (type == null) { return; @@ -51,28 +64,31 @@ public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { type = type.getWithoutQualifiers(); assert isArrayVectorOrMatrix(type); if (!staticallyInBounds(arrayIndexExpr.getIndex(), type)) { - arrayIndexExpr.setIndex(new TernaryExpr( - new BinaryExpr( - new BinaryExpr( - new ParenExpr(arrayIndexExpr.getIndex().clone()), - new IntConstantExpr("0"), - BinOp.GE), - new BinaryExpr( - new ParenExpr(arrayIndexExpr.getIndex().clone()), - new IntConstantExpr(getSize(type).toString()), - BinOp.LT), - BinOp.LAND), - arrayIndexExpr.getIndex(), - new IntConstantExpr("0")) - ); + Type indexType = typer.lookupType(arrayIndexExpr.getIndex()); + if (indexType == null) { + return; + } + indexType = indexType.getWithoutQualifiers(); + assert indexType == BasicType.INT || indexType == BasicType.UINT; + + final Expr arraySize = indexType == BasicType.INT + ? new IntConstantExpr(getSize(type).toString()) + : new UIntConstantExpr(getSize(type).toString() + "u"); + + final Expr clampedIndexExpr = + new FunctionCallExpr(indexType == BasicType.INT ? Constants.GLF_MAKE_IN_BOUNDS_INT : + Constants.GLF_MAKE_IN_BOUNDS_UINT, + arrayIndexExpr.getIndex(), + arraySize); + arrayIndexExpr.setIndex(clampedIndexExpr); + } - super.visitArrayIndexExpr(arrayIndexExpr); } private static Integer getSize(Type type) { assert isArrayVectorOrMatrix(type); if (type instanceof ArrayType) { - return ((ArrayType) type).getArrayInfo().getSize(); + return ((ArrayType) type).getArrayInfo().getConstantSize(); } if (BasicType.allVectorTypes().contains(type)) { return ((BasicType) type).getNumElements(); @@ -87,11 +103,17 @@ private static boolean isArrayVectorOrMatrix(Type type) { } private static boolean staticallyInBounds(Expr index, Type type) { - if (!(index instanceof IntConstantExpr)) { + if (!(index instanceof IntConstantExpr || index instanceof UIntConstantExpr)) { return false; } - Integer indexValue = Integer.parseInt(((IntConstantExpr) index).getValue()); - return indexValue >= 0 && indexValue < getSize(type); + Integer indexValue; + if (index instanceof IntConstantExpr) { + indexValue = ((IntConstantExpr) index).getNumericValue(); + return indexValue >= 0 && indexValue < getSize(type); + } else { // index instanceof UIntConstantExpr + indexValue = ((UIntConstantExpr) index).getNumericValue(); + return indexValue < getSize(type); + } } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/BlockInjectionPoint.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/BlockInjectionPoint.java index 614c7367c..206161f22 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/BlockInjectionPoint.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/BlockInjectionPoint.java @@ -27,8 +27,8 @@ public class BlockInjectionPoint extends InjectionPoint { private Stmt nextStmt; // null if there is no next statement public BlockInjectionPoint(BlockStmt blockStmt, Stmt nextStmt, - FunctionDefinition enclosingFunction, boolean inLoop, Scope scope) { - super(enclosingFunction, inLoop, scope); + FunctionDefinition enclosingFunction, boolean inLoop, boolean inSwitch, Scope scope) { + super(enclosingFunction, inLoop, inSwitch, scope); this.blockStmt = blockStmt; this.nextStmt = nextStmt; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IInjectionPoint.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IInjectionPoint.java index d22e201bc..58487e05b 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IInjectionPoint.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IInjectionPoint.java @@ -59,6 +59,13 @@ public interface IInjectionPoint { */ boolean inLoop(); + /** + * Determines whether the injection point is located inside a switch statement. + * + * @return true if and only if the injection point is inside a switch statement. + */ + boolean inSwitch(); + /** * Returns the function enclosing the injection point. * diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IfInjectionPoint.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IfInjectionPoint.java index 6fe24ab2a..788f781f9 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IfInjectionPoint.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/IfInjectionPoint.java @@ -30,8 +30,8 @@ public class IfInjectionPoint extends InjectionPoint { private boolean chooseThen; public IfInjectionPoint(IfStmt ifStmt, boolean chooseThen, FunctionDefinition enclosingFunction, - boolean inLoop, Scope scope) { - super(enclosingFunction, inLoop, scope); + boolean inLoop, boolean inSwitch, Scope scope) { + super(enclosingFunction, inLoop, inSwitch, scope); this.ifStmt = ifStmt; this.chooseThen = chooseThen; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoint.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoint.java index f4bb95412..b9dfd773b 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoint.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoint.java @@ -23,12 +23,15 @@ abstract class InjectionPoint implements IInjectionPoint { private final FunctionDefinition enclosingFunction; private final boolean inLoop; + private final boolean inSwitch; private final Scope scope; InjectionPoint(FunctionDefinition enclosingFunction, boolean inLoop, + boolean inSwitch, Scope scope) { this.enclosingFunction = enclosingFunction; this.inLoop = inLoop; + this.inSwitch = inSwitch; this.scope = scope.shallowClone(); } @@ -42,6 +45,11 @@ public final boolean inLoop() { return inLoop; } + @Override + public final boolean inSwitch() { + return inSwitch; + } + @Override public final Scope scopeAtInjectionPoint() { return scope; diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoints.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoints.java index de49199a5..d3a8863c2 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoints.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/InjectionPoints.java @@ -23,9 +23,11 @@ import com.graphicsfuzz.common.ast.stmt.DoStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; import com.graphicsfuzz.common.ast.stmt.IfStmt; +import com.graphicsfuzz.common.ast.stmt.LoopStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.ast.stmt.WhileStmt; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.IRandom; import java.util.ArrayList; import java.util.Collections; @@ -34,11 +36,12 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -public class InjectionPoints extends ScopeTreeBuilder { +public class InjectionPoints extends ScopeTrackingVisitor { private final List injectionPoints; private FunctionDefinition currentFunction; private int loopNestingDepth; + private int switchNestingDepth; private final IRandom generator; private final Predicate suitable; @@ -47,6 +50,7 @@ public InjectionPoints(TranslationUnit tu, IRandom generator, this.injectionPoints = new ArrayList<>(); this.currentFunction = null; this.loopNestingDepth = 0; + this.switchNestingDepth = 0; this.generator = generator; this.suitable = suitable; visit(tu); @@ -64,7 +68,8 @@ public void visitBlockStmt(BlockStmt stmt) { enterBlockStmt(stmt); // This setup ensures that variables declared inside the block are in scope by the time we // consider each statement as an injection point. - // It is a bit ugly because it has to mimic the "enter ... leave" structure of ScopeTreeBuilder + // It is a bit ugly because it has to mimic the "enter ... leave" structure of + // ScopeTrackingVisitor for (int i = 0; i < stmt.getNumStmts(); i++) { Stmt innerStmt = stmt.getStmt(i); if (i == 0 && innerStmt instanceof CaseLabel) { @@ -72,11 +77,13 @@ public void visitBlockStmt(BlockStmt stmt) { continue; } maybeAddInjectionPoint(new BlockInjectionPoint(stmt, innerStmt, currentFunction, inLoop(), - currentScope)); + inSwitch(), + getCurrentScope())); visit(innerStmt); } maybeAddInjectionPoint(new BlockInjectionPoint(stmt, null, currentFunction, inLoop(), - currentScope)); + inSwitch(), + getCurrentScope())); leaveBlockStmt(stmt); } @@ -84,12 +91,21 @@ private boolean inLoop() { return loopNestingDepth > 0; } + private boolean inSwitch() { + return switchNestingDepth > 0; + } + + private void considerLoopInjectionPoint(LoopStmt loopStmt) { + if (!(loopStmt.getBody() instanceof BlockStmt)) { + maybeAddInjectionPoint(new LoopInjectionPoint(loopStmt, currentFunction, + inSwitch(), + getCurrentScope())); + } + } + @Override public void visitDoStmt(DoStmt doStmt) { - if (!(doStmt.getBody() instanceof BlockStmt)) { - maybeAddInjectionPoint(new LoopInjectionPoint(doStmt, currentFunction, - currentScope)); - } + considerLoopInjectionPoint(doStmt); loopNestingDepth++; super.visitDoStmt(doStmt); loopNestingDepth--; @@ -97,39 +113,45 @@ public void visitDoStmt(DoStmt doStmt) { @Override public void visitForStmt(ForStmt forStmt) { - if (!(forStmt.getBody() instanceof BlockStmt)) { - maybeAddInjectionPoint(new LoopInjectionPoint(forStmt, currentFunction, - currentScope)); - } + considerLoopInjectionPoint(forStmt); loopNestingDepth++; super.visitForStmt(forStmt); loopNestingDepth--; } + @Override + public void visitWhileStmt(WhileStmt whileStmt) { + considerLoopInjectionPoint(whileStmt); + loopNestingDepth++; + super.visitWhileStmt(whileStmt); + loopNestingDepth--; + } + @Override public void visitIfStmt(IfStmt ifStmt) { if (!(ifStmt.getThenStmt() instanceof BlockStmt)) { - maybeAddInjectionPoint(new IfInjectionPoint(ifStmt, true, currentFunction, inLoop(), - currentScope)); + maybeAddInjectionPoint(new IfInjectionPoint(ifStmt, true, currentFunction, + inLoop(), + inSwitch(), + getCurrentScope())); } if (ifStmt.hasElseStmt()) { if (!(ifStmt.getElseStmt() instanceof BlockStmt)) { - maybeAddInjectionPoint(new IfInjectionPoint(ifStmt, false, currentFunction, inLoop(), - currentScope)); + maybeAddInjectionPoint(new IfInjectionPoint(ifStmt, false, currentFunction, + inLoop(), + inSwitch(), + getCurrentScope())); } } super.visitIfStmt(ifStmt); } @Override - public void visitWhileStmt(WhileStmt whileStmt) { - if (!(whileStmt.getBody() instanceof BlockStmt)) { - maybeAddInjectionPoint(new LoopInjectionPoint(whileStmt, currentFunction, - currentScope)); - } - loopNestingDepth++; - super.visitWhileStmt(whileStmt); - loopNestingDepth--; + public void visitSwitchStmt(SwitchStmt switchStmt) { + // We cannot inject immediately inside a switch statement. + switchNestingDepth++; + super.visitSwitchStmt(switchStmt); + switchNestingDepth--; } private void maybeAddInjectionPoint(IInjectionPoint injectionPoint) { diff --git a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/LoopInjectionPoint.java b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/LoopInjectionPoint.java index 3a66f757b..9a8eed652 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/LoopInjectionPoint.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/transformation/injection/LoopInjectionPoint.java @@ -29,8 +29,9 @@ public class LoopInjectionPoint extends InjectionPoint { private LoopStmt loopStmt; public LoopInjectionPoint(LoopStmt loopStmt, FunctionDefinition enclosingFunction, - Scope scope) { - super(enclosingFunction, true, scope); + boolean inSwitch, + Scope scope) { + super(enclosingFunction, true, inSwitch, scope); this.loopStmt = loopStmt; } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/AvailableStructsCollector.java b/generator/src/main/java/com/graphicsfuzz/generator/util/AvailableStructsCollector.java index 7066c39c1..4673a404e 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/AvailableStructsCollector.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/AvailableStructsCollector.java @@ -19,12 +19,12 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.type.StructDefinitionType; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import java.util.ArrayList; import java.util.Collections; import java.util.List; -public class AvailableStructsCollector extends ScopeTreeBuilder { +public class AvailableStructsCollector extends ScopeTrackingVisitor { private final List structDefinitionTypes; private final IAstNode donorFragment; @@ -39,8 +39,8 @@ public AvailableStructsCollector(TranslationUnit donor, IAstNode donorFragment) public void visit(IAstNode node) { if (node == donorFragment) { assert structDefinitionTypes.isEmpty(); - for (String structName : currentScope.namesOfAllStructDefinitionsInScope()) { - structDefinitionTypes.add(currentScope.lookupStructName(structName)); + for (String structName : getCurrentScope().namesOfAllStructDefinitionsInScope()) { + structDefinitionTypes.add(getCurrentScope().lookupStructName(structName)); } } super.visit(node); diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/ConstCleaner.java b/generator/src/main/java/com/graphicsfuzz/generator/util/ConstCleaner.java index 126bf1e69..f7a8299af 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/ConstCleaner.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/ConstCleaner.java @@ -18,7 +18,6 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -29,17 +28,16 @@ import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; /** * Attempts to minimally remove const qualifiers and move global declaration initializers into * main, to make a shader valid. */ -class ConstCleaner extends ScopeTreeBuilder { +class ConstCleaner extends ScopeTrackingVisitor { private boolean atGlobalScope; private Optional currentVariablesDeclaration; @@ -93,7 +91,7 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie } private boolean nonConst(VariableIdentifierExpr variableIdentifierExpr) { - final ScopeEntry se = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + final ScopeEntry se = getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()); return se != null && se.hasVariablesDeclaration() && !se.getVariablesDeclaration().getBaseType() .hasQualifier(TypeQualifier.CONST); } @@ -111,12 +109,9 @@ private void addGlobalInitializers(FunctionDefinition mainFunction) { for (int i = globalsToBeReInitialized.size() - 1; i >= 0; i--) { for (int j = globalsToBeReInitialized.get(i).getNumDecls() - 1; j >= 0; j--) { final VariableDeclInfo vdi = globalsToBeReInitialized.get(i).getDeclInfo(j); - if (!(vdi.getInitializer() instanceof ScalarInitializer)) { - throw new RuntimeException("Only know how to deal with scalar initializers at present."); - } mainFunction.getBody().insertStmt(0, new ExprStmt(new BinaryExpr(new VariableIdentifierExpr(vdi.getName()), - ((ScalarInitializer) vdi.getInitializer()).getExpr(), BinOp.ASSIGN))); + (vdi.getInitializer()).getExpr(), BinOp.ASSIGN))); vdi.setInitializer(null); } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/FloatLiteralReplacer.java b/generator/src/main/java/com/graphicsfuzz/generator/util/FloatLiteralReplacer.java index 1a1cfb2b0..aa6d29a8a 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/FloatLiteralReplacer.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/FloatLiteralReplacer.java @@ -79,7 +79,7 @@ private FloatLiteralReplacer(TranslationUnit tu) { tu.addDeclaration(new VariablesDeclaration( new QualifiedType(BasicType.FLOAT, Arrays.asList(TypeQualifier.UNIFORM)), new VariableDeclInfo(Constants.FLOAT_CONST, - new ArrayInfo(uniformIndex), null) + new ArrayInfo(new IntConstantExpr(Integer.toString(uniformIndex))), null) )); } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/FreeVariablesCollector.java b/generator/src/main/java/com/graphicsfuzz/generator/util/FreeVariablesCollector.java index fb0d6abf1..89634afae 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/FreeVariablesCollector.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/FreeVariablesCollector.java @@ -20,15 +20,14 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.ast.stmt.Stmt; -import com.graphicsfuzz.common.ast.type.StructDefinitionType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.typing.Scope; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.OpenGlConstants; import java.util.HashMap; import java.util.Map; -public class FreeVariablesCollector extends ScopeTreeBuilder { +public class FreeVariablesCollector extends ScopeTrackingVisitor { private final Stmt donorFragment; private Scope enclosingScope; @@ -48,14 +47,12 @@ public Map getFreeVariables() { @Override public void visit(IAstNode node) { if (node == donorFragment) { - enclosingScope = currentScope; - currentScope = new Scope(null); - super.visit(node); - // early exit now, if we implement that - currentScope = enclosingScope; + enclosingScope = swapCurrentScope(new Scope(null)); + } + super.visit(node); + if (node == donorFragment) { + swapCurrentScope(enclosingScope); enclosingScope = null; - } else { - super.visit(node); } } @@ -65,7 +62,7 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie if (isBuiltinVariable(name)) { return; } - if (enclosingScope != null && currentScope.lookupType(name) == null) { + if (enclosingScope != null && getCurrentScope().lookupType(name) == null) { Type type = enclosingScope.lookupType(name); if (type == null) { throw new RuntimeException( diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/GenerationParams.java b/generator/src/main/java/com/graphicsfuzz/generator/util/GenerationParams.java index f80b22c3a..df8c494e6 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/GenerationParams.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/GenerationParams.java @@ -29,7 +29,7 @@ public static GenerationParams small(ShaderKind shaderKind, boolean injectionSwi result.maxDepthForGeneratedExpr = 2; result.maxStructNestingDepth = 1; result.maxStructFields = 3; - result.maxDonors = 2; + result.maxDonorsPerDonationPass = 2; return result; } @@ -38,7 +38,7 @@ public static GenerationParams normal(ShaderKind shaderKind, boolean injectionSw result.maxDepthForGeneratedExpr = 3; result.maxStructNestingDepth = 2; result.maxStructFields = 5; - result.maxDonors = 4; + result.maxDonorsPerDonationPass = 4; return result; } @@ -47,7 +47,7 @@ public static GenerationParams large(ShaderKind shaderKind, boolean injectionSwi result.maxDepthForGeneratedExpr = 5; result.maxStructNestingDepth = 4; result.maxStructFields = 7; - result.maxDonors = 6; + result.maxDonorsPerDonationPass = 6; return result; } @@ -67,8 +67,8 @@ private GenerationParams(ShaderKind shaderKind, boolean injectionSwitchIsAvailab private int maxStructNestingDepth = 3; private int maxStructFields = 8; - // Donors - private int maxDonors = 5; + // The maximum number of distinct donors that can be used during one donation pass. + private int maxDonorsPerDonationPass = 5; private final boolean injectionSwitchIsAvailable; @@ -84,8 +84,8 @@ public int getMaxStructFields() { return maxStructFields; } - public int getMaxDonors() { - return maxDonors; + public int getMaxDonorsPerDonationPass() { + return maxDonorsPerDonationPass; } public ShaderKind getShaderKind() { diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveDiscardStatements.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveDiscardStatements.java index f96c4ceb1..d709fc2c8 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveDiscardStatements.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveDiscardStatements.java @@ -18,14 +18,12 @@ import com.graphicsfuzz.common.ast.IAstNode; import com.graphicsfuzz.common.ast.stmt.DiscardStmt; -import com.graphicsfuzz.common.ast.stmt.NullStmt; -import java.util.Optional; public class RemoveDiscardStatements extends RemoveStatements { public RemoveDiscardStatements(IAstNode node) { super(item -> item instanceof DiscardStmt, - item -> Optional.of(new NullStmt()), node); + item -> makeIntConstantExprStmt(), node); } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakStatements.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakStatements.java new file mode 100644 index 000000000..e96438a2b --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakStatements.java @@ -0,0 +1,55 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.util; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.BreakStmt; +import com.graphicsfuzz.common.ast.stmt.DoStmt; +import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; +import com.graphicsfuzz.common.ast.stmt.WhileStmt; + +/** + * This class removes break statements that are not nested inside loop or switch statements. + */ +public class RemoveImmediateBreakStatements extends RemoveStatements { + + public RemoveImmediateBreakStatements(IAstNode node) { + super(item -> item instanceof BreakStmt, + item -> makeIntConstantExprStmt(), node); + } + + @Override + public void visitDoStmt(DoStmt doStmt) { + // Block visitation: we don't want to remove break statements from inside a loop + } + + @Override + public void visitForStmt(ForStmt forStmt) { + // Block visitation: we don't want to remove break statements from inside a loop + } + + @Override + public void visitWhileStmt(WhileStmt whileStmt) { + // Block visitation: we don't want to remove break statements from inside a loop + } + + @Override + public void visitSwitchStmt(SwitchStmt switchStmt) { + // Block visitation: we don't want to remove break statements from inside a switch + } +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateCaseLabels.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateCaseLabels.java new file mode 100644 index 000000000..401266e08 --- /dev/null +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateCaseLabels.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.util; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.CaseLabel; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; + +/** + * This class removes case labels (including default) that are not nested inside switch statements. + */ +public class RemoveImmediateCaseLabels extends RemoveStatements { + + public RemoveImmediateCaseLabels(IAstNode node) { + super(item -> item instanceof CaseLabel, + item -> makeIntConstantExprStmt(), node); + } + + @Override + public void visitSwitchStmt(SwitchStmt switchStmt) { + // Block visitation: we don't want to remove labels from inside switch statements + } + +} diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakAndContinueStatements.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateContinueStatements.java similarity index 58% rename from generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakAndContinueStatements.java rename to generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateContinueStatements.java index f7dc4ac87..146f194fe 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateBreakAndContinueStatements.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveImmediateContinueStatements.java @@ -17,37 +17,34 @@ package com.graphicsfuzz.generator.util; import com.graphicsfuzz.common.ast.IAstNode; -import com.graphicsfuzz.common.ast.stmt.BreakStmt; import com.graphicsfuzz.common.ast.stmt.ContinueStmt; import com.graphicsfuzz.common.ast.stmt.DoStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; -import com.graphicsfuzz.common.ast.stmt.NullStmt; import com.graphicsfuzz.common.ast.stmt.WhileStmt; -import java.util.Optional; /** - * This class removes break and continue statements that are not nested inside loops. + * This class removes continue statements that are not nested inside loops. */ -public class RemoveImmediateBreakAndContinueStatements extends RemoveStatements { +public class RemoveImmediateContinueStatements extends RemoveStatements { - public RemoveImmediateBreakAndContinueStatements(IAstNode node) { - super(item -> item instanceof BreakStmt || item instanceof ContinueStmt, - item -> Optional.of(new NullStmt()), node); + public RemoveImmediateContinueStatements(IAstNode node) { + super(item -> item instanceof ContinueStmt, + item -> makeIntConstantExprStmt(), node); } @Override public void visitDoStmt(DoStmt doStmt) { - // Block visitation: we don't want to remove break and continue statements from inside a loop + // Block visitation: we don't want to remove continue statements from inside a loop } @Override public void visitForStmt(ForStmt forStmt) { - // Block visitation: we don't want to remove break and continue statements from inside a loop + // Block visitation: we don't want to remove continue statements from inside a loop } @Override public void visitWhileStmt(WhileStmt whileStmt) { - // Block visitation: we don't want to remove break and continue statements from inside a loop + // Block visitation: we don't want to remove continue statements from inside a loop } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveReturnStatements.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveReturnStatements.java index 1c0378c2e..cea3bdd84 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveReturnStatements.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveReturnStatements.java @@ -26,8 +26,8 @@ public class RemoveReturnStatements extends RemoveStatements { public RemoveReturnStatements(IAstNode node) { super(item -> item instanceof ReturnStmt, item -> ((ReturnStmt) item).hasExpr() - ? Optional.of(new ExprStmt(((ReturnStmt) item).getExpr())) - : Optional.empty(), + ? new ExprStmt(((ReturnStmt) item).getExpr()) + : makeIntConstantExprStmt(), node); } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveStatements.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveStatements.java index 5f54f9851..2d42bb1af 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveStatements.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RemoveStatements.java @@ -17,12 +17,13 @@ package com.graphicsfuzz.generator.util; import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.DoStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; import com.graphicsfuzz.common.ast.stmt.IfStmt; import com.graphicsfuzz.common.ast.stmt.LoopStmt; -import com.graphicsfuzz.common.ast.stmt.NullStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.stmt.WhileStmt; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; @@ -35,12 +36,12 @@ public abstract class RemoveStatements extends StandardVisitor { private final Predicate shouldRemove; - private final Function> maybeReplaceWith; + private final Function replaceWith; public RemoveStatements(Predicate shouldRemove, - Function> maybeReplaceWith, IAstNode node) { + Function replaceWith, IAstNode node) { this.shouldRemove = shouldRemove; - this.maybeReplaceWith = maybeReplaceWith; + this.replaceWith = replaceWith; visit(node); } @@ -49,10 +50,7 @@ public void visitBlockStmt(BlockStmt stmt) { List newStmts = new ArrayList<>(); for (Stmt s : stmt.getStmts()) { if (shouldRemove.test(s)) { - Optional possibleReplacement = maybeReplaceWith.apply(s); - if (possibleReplacement.isPresent()) { - newStmts.add(possibleReplacement.get()); - } + newStmts.add(getReplacementStmt(s)); continue; } newStmts.add(s); @@ -101,8 +99,14 @@ private void handleLoop(LoopStmt loop) { } private Stmt getReplacementStmt(Stmt stmt) { - Optional possibleReplacement = maybeReplaceWith.apply(stmt); - return possibleReplacement.isPresent() ? possibleReplacement.get() : new NullStmt(); + return replaceWith.apply(stmt); + } + + /** + * Provides a statement of the form "1;", which is what most statements are replaced with. + */ + static Stmt makeIntConstantExprStmt() { + return new ExprStmt(new IntConstantExpr("1")); } } diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/RestrictFragmentShaderColors.java b/generator/src/main/java/com/graphicsfuzz/generator/util/RestrictFragmentShaderColors.java index eed772dac..fd6036766 100644 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/RestrictFragmentShaderColors.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/RestrictFragmentShaderColors.java @@ -36,10 +36,11 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.typing.Scope; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.typing.Typer; import com.graphicsfuzz.common.typing.TyperHelper; import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.generator.fuzzer.Fuzzer; import com.graphicsfuzz.generator.fuzzer.FuzzingContext; import com.graphicsfuzz.generator.fuzzer.OpaqueExpressionGenerator; @@ -49,7 +50,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Set; public class RestrictFragmentShaderColors { @@ -156,15 +156,16 @@ private void addNewWrites() { private boolean adaptExistingWrites() { - final Typer typer = new Typer(shaderJob.getFragmentShader().get(), shadingLanguageVersion); + final Typer typer = new Typer(shaderJob.getFragmentShader().get()); - return new ScopeTreeBuilder() { + return new ScopeTrackingVisitor() { @Override public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { super.visitFunctionCallExpr(functionCallExpr); Set acceptableFunctionNames = new HashSet<>(); - acceptableFunctionNames.addAll(TyperHelper.getBuiltins(shadingLanguageVersion).keySet()); + acceptableFunctionNames.addAll(TyperHelper.getBuiltins(shadingLanguageVersion, + ShaderKind.FRAGMENT).keySet()); acceptableFunctionNames.add(Constants.GLF_FUZZED); acceptableFunctionNames.add(Constants.GLF_IDENTITY); if (acceptableFunctionNames.contains(functionCallExpr.getCallee()) @@ -172,7 +173,7 @@ public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { return; } for (Expr expr : functionCallExpr.getArgs()) { - if (isPartOfOutputVariable(expr, currentScope)) { + if (isPartOfOutputVariable(expr, getCurrentScope())) { throw new AbortVisitationException("We do not yet handle components of the output " + "variable being passed to functions, in case they are out parameters."); } @@ -186,7 +187,7 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { // Do nothing: the binary operator is not side-effecting. return; } - if (!isPartOfOutputVariable(binaryExpr.getLhs(), currentScope)) { + if (!isPartOfOutputVariable(binaryExpr.getLhs(), getCurrentScope())) { return; } final BasicType basicType = getBasicType(binaryExpr.getLhs()); @@ -196,9 +197,9 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { // Replace RHS by // (1.0 + 0.0 * (is_nan_or_inf(original_rhs) ? 0.0 : original_rhs)) binaryExpr.setRhs( - new ParenExpr(new BinaryExpr(opaqueOne(currentScope, basicType), - new BinaryExpr(opaqueZero(currentScope, basicType), - ifNanOrInfThenZeroElse(basicType, binaryExpr.getRhs(), currentScope), + new ParenExpr(new BinaryExpr(opaqueOne(getCurrentScope(), basicType), + new BinaryExpr(opaqueZero(getCurrentScope(), basicType), + ifNanOrInfThenZeroElse(basicType, binaryExpr.getRhs(), getCurrentScope()), BinOp.MUL), BinOp.ADD)) ); @@ -209,8 +210,8 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { binaryExpr.setRhs( new ParenExpr( new BinaryExpr( - opaqueZero(currentScope, basicType), - ifNanOrInfThenZeroElse(basicType, binaryExpr.getRhs(), currentScope), + opaqueZero(getCurrentScope(), basicType), + ifNanOrInfThenZeroElse(basicType, binaryExpr.getRhs(), getCurrentScope()), BinOp.MUL) ) ); @@ -220,9 +221,10 @@ public void visitBinaryExpr(BinaryExpr binaryExpr) { binaryExpr.setRhs( new ParenExpr( new BinaryExpr( - makeColorVector(basicType, currentScope), - new BinaryExpr(opaqueZero(currentScope, basicType), - ifNanOrInfThenZeroElse(basicType, binaryExpr.getRhs(), currentScope), + makeColorVector(basicType, getCurrentScope()), + new BinaryExpr(opaqueZero(getCurrentScope(), basicType), + ifNanOrInfThenZeroElse(basicType, binaryExpr.getRhs(), + getCurrentScope()), BinOp.MUL), BinOp.ADD ) diff --git a/generator/src/main/java/com/graphicsfuzz/generator/util/TransformationProbabilities.java b/generator/src/main/java/com/graphicsfuzz/generator/util/TransformationProbabilities.java index 006b626ff..154796b91 100755 --- a/generator/src/main/java/com/graphicsfuzz/generator/util/TransformationProbabilities.java +++ b/generator/src/main/java/com/graphicsfuzz/generator/util/TransformationProbabilities.java @@ -61,7 +61,7 @@ private TransformationProbabilities(int probSubstituteFreeVariable, int probDona this.probInjectDeadBarrierAtStmt = probInjectDeadBarrierAtStmt; } - public static final TransformationProbabilities DEFAULT_PROBABILITIES = + public static TransformationProbabilities DEFAULT_PROBABILITIES = new TransformationProbabilities( 80, 20, @@ -79,7 +79,7 @@ private TransformationProbabilities(int probSubstituteFreeVariable, int probDona 20 ); - public static final TransformationProbabilities SMALL_PROBABILITIES = + public static TransformationProbabilities SMALL_PROBABILITIES = new TransformationProbabilities( 80, 5, @@ -96,7 +96,7 @@ private TransformationProbabilities(int probSubstituteFreeVariable, int probDona 5, 5); - public static final TransformationProbabilities AGGRESSIVE_CONTROL_FLOW = + public static TransformationProbabilities AGGRESSIVE_CONTROL_FLOW = new TransformationProbabilities(DEFAULT_PROBABILITIES.probSubstituteFreeVariable, DEFAULT_PROBABILITIES.probDonateDeadCodeAtStmt, DEFAULT_PROBABILITIES.probDonateLiveCodeAtStmt, @@ -113,7 +113,7 @@ private TransformationProbabilities(int probSubstituteFreeVariable, int probDona 70); // Useful for testing; add similar for others when needed - public static final TransformationProbabilities ZERO = + public static TransformationProbabilities ZERO = new TransformationProbabilities( 0, 0, @@ -130,68 +130,68 @@ private TransformationProbabilities(int probSubstituteFreeVariable, int probDona 0, 0); - public static final TransformationProbabilities onlySplitLoops() { + public static TransformationProbabilities onlySplitLoops() { TransformationProbabilities result = ZERO; result.probSplitLoops = 100; return result; } - public static final TransformationProbabilities onlyStructify() { + public static TransformationProbabilities onlyStructify() { TransformationProbabilities result = ZERO; result.probStructify = 100; return result; } - public static final TransformationProbabilities onlyVectorize() { + public static TransformationProbabilities onlyVectorize() { TransformationProbabilities result = ZERO; result.probVectorizeStmts = 100; return result; } - public static final TransformationProbabilities onlyVectorizeAndMutate() { + public static TransformationProbabilities onlyVectorizeAndMutate() { TransformationProbabilities result = ZERO; result.probVectorizeStmts = 100; result.probMutatePoint = 100; return result; } - public static final TransformationProbabilities onlyOutlineStatements() { + public static TransformationProbabilities onlyOutlineStatements() { TransformationProbabilities result = ZERO; result.probOutline = 100; return result; } - public static final TransformationProbabilities onlyMutateExpressions() { + public static TransformationProbabilities onlyMutateExpressions() { TransformationProbabilities result = ZERO; result.probMutatePoint = 100; return result; } - public static final TransformationProbabilities onlyWrap() { + public static TransformationProbabilities onlyWrap() { TransformationProbabilities result = ZERO; result.probWrapStmtInConditional = 100; return result; } - public static final TransformationProbabilities onlyAddJumps() { + public static TransformationProbabilities onlyAddJumps() { TransformationProbabilities result = ZERO; result.probInjectJumpAtStmt = 100; return result; } - public static final TransformationProbabilities onlyAddDeadFragColorWrites() { + public static TransformationProbabilities onlyAddDeadFragColorWrites() { TransformationProbabilities result = ZERO; result.probAddDeadFragColorWrites = 100; return result; } - public static final TransformationProbabilities onlyAddLiveFragColorWrites() { + public static TransformationProbabilities onlyAddLiveFragColorWrites() { TransformationProbabilities result = ZERO; result.probAddLiveFragColorWrites = 100; return result; } - public static final TransformationProbabilities onlyLiveCodeAlwaysSubstitute() { + public static TransformationProbabilities onlyLiveCodeAlwaysSubstitute() { TransformationProbabilities result = ZERO; result.probDonateLiveCodeAtStmt = 100; result.probSubstituteFreeVariable = 100; @@ -202,14 +202,14 @@ public static TransformationProbabilities closeToDefaultProbabilities(IRandom ge return closeTo(generator, DEFAULT_PROBABILITIES); } - public static final TransformationProbabilities likelyDonateDeadCode() { + public static TransformationProbabilities likelyDonateDeadCode() { TransformationProbabilities result = ZERO; result.probSubstituteFreeVariable = 50; result.probDonateDeadCodeAtStmt = 60; return result; } - public static final TransformationProbabilities likelyDonateLiveCode() { + public static TransformationProbabilities likelyDonateLiveCode() { TransformationProbabilities result = ZERO; result.probSubstituteFreeVariable = 50; result.probDonateLiveCodeAtStmt = 60; diff --git a/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/FuzzerTest.java b/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/FuzzerTest.java index c4c63f9e4..7d5177a51 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/FuzzerTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/FuzzerTest.java @@ -22,7 +22,7 @@ import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.common.util.ZeroCannedRandom; @@ -52,12 +52,12 @@ public void testStructExprFuzzing() throws Exception { TranslationUnit tu = ParseHelper.parse(shader); - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); if (variableIdentifierExpr.getName().equals("doitWhenYouReachMyUse")) { - Expr expr = new Fuzzer(new FuzzingContext(currentScope), ShadingLanguageVersion.ESSL_100, + Expr expr = new Fuzzer(new FuzzingContext(getCurrentScope()), ShadingLanguageVersion.ESSL_100, new ZeroCannedRandom(), GenerationParams.normal(ShaderKind.FRAGMENT, true), "prefix") .fuzzExpr(new StructNameType("B"), false, false, 0); assertTrue(expr instanceof TypeConstructorExpr); @@ -75,4 +75,4 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie }.visit(tu); } -} \ No newline at end of file +} diff --git a/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java b/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java index fda14a1bc..b1e58d2f6 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/fuzzer/OpaqueExpressionGeneratorTest.java @@ -19,6 +19,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.PrecisionDeclaration; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; @@ -31,34 +32,33 @@ import com.graphicsfuzz.common.ast.stmt.ExprStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.type.BasicType; +import com.graphicsfuzz.common.ast.type.QualifiedType; +import com.graphicsfuzz.common.ast.type.Type; +import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.typing.Scope; import com.graphicsfuzz.common.typing.SupportedTypes; import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.common.util.ShaderKind; -import com.graphicsfuzz.generator.fuzzer.Fuzzer; -import com.graphicsfuzz.generator.fuzzer.FuzzingContext; -import com.graphicsfuzz.generator.fuzzer.OpaqueExpressionGenerator; import com.graphicsfuzz.generator.tool.Generate; import com.graphicsfuzz.generator.util.GenerationParams; -import com.graphicsfuzz.util.ExecHelper.RedirectType; -import com.graphicsfuzz.util.ExecResult; -import com.graphicsfuzz.util.ToolHelper; import java.io.File; -import java.io.FileOutputStream; -import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class OpaqueExpressionGeneratorTest { @@ -67,43 +67,45 @@ public class OpaqueExpressionGeneratorTest { @Test public void testGeneratesValidExpressions() throws Exception { + // We make a random number generator here and pass it into functions below in order to maximise + // diversity - even though there are differences between shading languages, there is still a + // good chance that until such a difference is hit, generating expressions for different + // shading languages but from the same seed will lead to identical results. + final IRandom generator = new RandomWrapper(0); + + // These are the shading language versions we support best, so test them thoroughly. for (ShadingLanguageVersion shadingLanguageVersion : Arrays.asList( + ShadingLanguageVersion.ESSL_100, ShadingLanguageVersion.ESSL_300, - ShadingLanguageVersion.ESSL_100)) { + ShadingLanguageVersion.ESSL_310, + ShadingLanguageVersion.ESSL_320)) { for (BasicType t : BasicType.allBasicTypes()) { if (!SupportedTypes.supported(t, shadingLanguageVersion)) { continue; } - final TranslationUnit tu = new TranslationUnit(Optional.of(ShadingLanguageVersion.ESSL_310), + final TranslationUnit tu = new TranslationUnit(Optional.of(shadingLanguageVersion), Arrays.asList( new PrecisionDeclaration("precision mediump float;"), new FunctionDefinition( - new FunctionPrototype("main", VoidType.VOID, new ArrayList<>()), + new FunctionPrototype("main", VoidType.VOID, Collections.emptyList()), new BlockStmt(makeMutatedExpressionAssignments(t, - shadingLanguageVersion, 1000), + shadingLanguageVersion, 1000, generator), false)))); Generate.addInjectionSwitchIfNotPresent(tu); - final File file = temporaryFolder.newFile("ex.frag"); - try (PrintStream stream = new PrintStream(new FileOutputStream(file))) { - PrettyPrinterVisitor.emitShader( - tu, - Optional.empty(), - stream, - PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, - true - ); - } - ExecResult execResult = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, file); - assertEquals(0, execResult.res); - file.delete(); + final File shaderJobFile = temporaryFolder.newFile("ex.json"); + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), + tu), shaderJobFile); + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); + fileOps.deleteShaderJobFile(shaderJobFile); } } } private List makeMutatedExpressionAssignments(BasicType basicType, ShadingLanguageVersion shadingLanguageVersion, - int numberOfAssignments) { + int numberOfAssignments, + IRandom generator) { // We declare only one variable and generate a number of assignments to this variable, // instead of making multiple declarations and assignments. // numberOfAssignments should be large enough to get high coverage, e.g. 1000. @@ -113,7 +115,6 @@ private List makeMutatedExpressionAssignments(BasicType basicType, new VariableDeclInfo("x", null, null)))); for (int i = 0; i < numberOfAssignments; i++) { - final IRandom generator = new RandomWrapper(i); final OpaqueExpressionGenerator opaqueExpressionGenerator = new OpaqueExpressionGenerator(generator, generationParams, shadingLanguageVersion); @@ -128,4 +129,80 @@ private List makeMutatedExpressionAssignments(BasicType basicType, } return newStmts; } + + @Test + public void testWaysToMakeZeroAndOne() throws Exception { + + final IRandom generator = new RandomWrapper(0); + final GenerationParams generationParams = GenerationParams.large(ShaderKind.FRAGMENT, true); + + // These are the shading language versions we support best, so test them thoroughly. + for (ShadingLanguageVersion shadingLanguageVersion : Arrays.asList( + ShadingLanguageVersion.ESSL_100, + ShadingLanguageVersion.ESSL_300, + ShadingLanguageVersion.ESSL_310, + ShadingLanguageVersion.ESSL_320)) { + final IdGenerator idGenerator = new IdGenerator(); + final List stmts = new ArrayList<>(); + final OpaqueExpressionGenerator opaqueExpressionGenerator = + new OpaqueExpressionGenerator(generator, + generationParams, shadingLanguageVersion); + for (BasicType basicType : BasicType.allNumericTypes()) { + if (!SupportedTypes.supported(basicType, shadingLanguageVersion)) { + continue; + } + for (boolean constContext : Arrays.asList(true, false)) { + stmts.addAll(makeStatementsFromFactories(generator, generationParams, + shadingLanguageVersion, + idGenerator, basicType, constContext, opaqueExpressionGenerator.waysToMakeZero(), true)); + // We do not allow making one for non-square matrices. + if (!BasicType.allNonSquareMatrixTypes().contains(basicType)) { + stmts.addAll(makeStatementsFromFactories(generator, generationParams, + shadingLanguageVersion, + idGenerator, basicType, constContext, opaqueExpressionGenerator.waysToMakeOne(), + false)); + } + } + } + final TranslationUnit tu = new TranslationUnit(Optional.of(shadingLanguageVersion), + Arrays.asList( + new PrecisionDeclaration("precision mediump float;"), + new FunctionDefinition( + new FunctionPrototype("main", VoidType.VOID, Collections.emptyList()), + new BlockStmt(stmts, + false)))); + Generate.addInjectionSwitchIfNotPresent(tu); + final File shaderJobFile = temporaryFolder.newFile("ex.json"); + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), + tu), shaderJobFile); + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); + fileOps.deleteShaderJobFile(shaderJobFile); + } + } + + private List makeStatementsFromFactories(IRandom generator, GenerationParams generationParams, ShadingLanguageVersion shadingLanguageVersion, IdGenerator idGenerator, BasicType typeToGenerate, boolean constContext, List factories, boolean makingZero) { + List result = new ArrayList<>(); + for (OpaqueZeroOneFactory factory : factories) { + final Optional expr = factory.tryMakeOpaque(typeToGenerate, constContext, 0, + new Fuzzer(new FuzzingContext(new Scope(null)), shadingLanguageVersion, + generator, generationParams), makingZero); + if (expr.isPresent()) { + final Type baseType = constContext ? new QualifiedType(typeToGenerate, + Collections.singletonList(TypeQualifier.CONST)) : typeToGenerate; + result.add(new DeclarationStmt( + new VariablesDeclaration( + baseType, + new VariableDeclInfo( + "v" + idGenerator.freshId(), + null, + new Initializer(expr.get()) + ) + ) + )); + } + } + return result; + } + } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/Expr2BinaryMutationFinderTest.java b/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/Expr2BinaryMutationFinderTest.java index 79dcb7d05..ade40d893 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/Expr2BinaryMutationFinderTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/Expr2BinaryMutationFinderTest.java @@ -96,6 +96,11 @@ public boolean nextBoolean() { public IRandom spawnChild() { throw new UnsupportedOperationException(); } + + @Override + public String getDescription() { + return "Just 2"; + } }; } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinderTest.java b/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinderTest.java index 0cb770a32..f1a285301 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinderTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/semanticschanging/ReplaceBlockStmtsWithSwitchMutationFinderTest.java @@ -61,55 +61,78 @@ public void testReplaceBlockStmtsWithSwitchMiner() throws Exception { + "}"; final String expected = "#version 300 es\n" - + "int foo(int x)" - + "{" - + " switch(x)" - + " {" - + " case 84:" - + " case 43:" - + " case 44:" - + " int a = 5;" - + " case 62:" - + " case 77:" - + " int b = 7;" - + " case 75:" - + " case 20:" - + " case 41:" - + " case 73:" - + " case 95:" - + " default:" - + " {" - + " switch(x)" - + " {" - + " case 84:" - + " case 43:" - + " default:" - + " {" - + " switch(0)" - + " {" - + " case 0:" - + " return x;" - + " }" - + " }" - + " }" - + " }" - + " }" - + "}" - + "void main()" - + "{" - + " int a = 0;" - + " int b = 0;" - + " int c = 0;" - + " for(" - + " int i = 0;" - + " i < 10;" - + " i ++" - + " )" - + " {" - + " a ++;" - + " b += 2;" - + " c += 3;" - + " }" + + "int foo(int x)\n" + + "{\n" + + " switch(x)\n" + + " {\n" + + " case 57:\n" + + " case 63:\n" + + " case 60:\n" + + " case 86:\n" + + " case 73:\n" + + " case 78:\n" + + " case 20:\n" + + " case 9:\n" + + " case 24:\n" + + " case 99:\n" + + " case 95:\n" + + " case 75:\n" + + " int a = 5;\n" + + " case 44:\n" + + " case 66:\n" + + " case 84:\n" + + " case 13:\n" + + " case 2:\n" + + " case 18:\n" + + " case 27:\n" + + " case 5:\n" + + " case 93:\n" + + " case 49:\n" + + " case 40:\n" + + " int b = 7;\n" + + " case 11:\n" + + " case 39:\n" + + " case 43:\n" + + " case 34:\n" + + " case 33:\n" + + " case 80:\n" + + " case 1:\n" + + " case 22:\n" + + " case 32:\n" + + " case 31:\n" + + " default:\n" + + " {\n" + + " switch(x)\n" + + " {\n" + + " case 84:\n" + + " case 43:\n" + + " default:\n" + + " {\n" + + " switch(0)\n" + + " {\n" + + " case 0:\n" + + " return x;\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + + "void main()\n" + + "{\n" + + " int a = 0;\n" + + " int b = 0;\n" + + " int c = 0;\n" + + " for(\n" + + " int i = 0;\n" + + " i < 10;\n" + + " i ++\n" + + " )\n" + + " {\n" + + " a ++;\n" + + " b += 2;\n" + + " c += 3;\n" + + " }\n" + "}"; @@ -158,33 +181,58 @@ public void testReplaceBlockStmtsWithSwitchMiner3() throws Exception { + "}"; final String expected = "#version 300 es\n" - + "int foo(int x) {" - + "switch(x)" - + " {" - + " case 84:" - + " case 43:" - + " case 44:" - + " case 62:" - + " case 77:" - + " case 75:" - + " case 20:" - + " case 41:" - + " case 73:" - + " case 95:" - + " default:" - + " {" - + " switch(x)" - + " {" - + " case 1:" - + " case 2:" - + " default:" - + " return 5;" - + " }" - + " }" - + " }" - + "}" - + "void main() {" - + "}"; + + "int foo(int x)\n" + + "{\n" + + " switch(x)\n" + + " {\n" + + " case 57:\n" + + " case 63:\n" + + " case 60:\n" + + " case 86:\n" + + " case 73:\n" + + " case 78:\n" + + " case 20:\n" + + " case 9:\n" + + " case 24:\n" + + " case 99:\n" + + " case 95:\n" + + " case 75:\n" + + " case 44:\n" + + " case 66:\n" + + " case 84:\n" + + " case 13:\n" + + " case 2:\n" + + " case 18:\n" + + " case 27:\n" + + " case 5:\n" + + " case 93:\n" + + " case 49:\n" + + " case 40:\n" + + " case 11:\n" + + " case 39:\n" + + " case 43:\n" + + " case 34:\n" + + " case 33:\n" + + " case 80:\n" + + " case 1:\n" + + " case 22:\n" + + " case 32:\n" + + " case 31:\n" + + " default:\n" + + " {\n" + + " switch(x)\n" + + " {\n" + + " case 1:\n" + + " case 2:\n" + + " default:\n" + + " return 5;\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + + "void main()\n" + + "{\n" + + "}\n"; TranslationUnit tu = ParseHelper.parse(program); diff --git a/generator/src/test/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutationTest.java index 20fd576ac..76c131f3e 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/semanticspreserving/StructificationMutationTest.java @@ -21,7 +21,7 @@ import static org.junit.Assert.assertTrue; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.type.StructDefinitionType; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; @@ -37,7 +37,6 @@ import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.generator.semanticspreserving.StructificationMutation; import com.graphicsfuzz.util.Constants; import com.graphicsfuzz.common.util.CannedRandom; import com.graphicsfuzz.common.util.IdGenerator; @@ -72,7 +71,7 @@ public void applyStructification() { new VariableDeclInfo( "v", null, - new ScalarInitializer( + new Initializer( new FloatConstantExpr("3.0"))))); // float v = 3.0; diff --git a/generator/src/test/java/com/graphicsfuzz/generator/tool/Fragment2ComputeTest.java b/generator/src/test/java/com/graphicsfuzz/generator/tool/Fragment2ComputeTest.java index dc7411564..b89203ffe 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/tool/Fragment2ComputeTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/tool/Fragment2ComputeTest.java @@ -21,11 +21,12 @@ import com.graphicsfuzz.util.ToolPaths; import java.io.File; import java.nio.file.Paths; +import java.util.Arrays; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; public class Fragment2ComputeTest { @@ -40,7 +41,11 @@ public void testTranslationToCompute() throws Exception { final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - for (File reference : referencesDir.listFiles((dir, name) -> name.endsWith(".json"))) { + String[] blacklist = {"trigonometric_strip", "mergesort_mosaic"}; + + for (File reference : + referencesDir.listFiles((dir, name) -> name.endsWith(".json") + && Arrays.stream(blacklist).noneMatch(s -> name.contains(s)))) { File outputShaderJob = temporaryFolder.newFile(reference.getName()); Fragment2Compute.mainHelper(reference.getAbsolutePath(), outputShaderJob.getAbsolutePath()); assertTrue(fileOps.getUnderlyingShaderFile(outputShaderJob, ShaderKind.COMPUTE) diff --git a/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateShaderFamilyTest.java b/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateShaderFamilyTest.java index 5b28930a4..6a2fa282c 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateShaderFamilyTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateShaderFamilyTest.java @@ -42,7 +42,7 @@ public class GenerateShaderFamilyTest { @Test public void testGenerateSmall100ShaderFamily() throws Exception { final String samplesSubdir = "100"; - final String referenceShaderName = "bubblesort_flag"; + final String referenceShaderName = "stable_bubblesort_flag"; final int numVariants = 3; int seed = 0; checkShaderFamilyGeneration(samplesSubdir, referenceShaderName, numVariants, @@ -62,7 +62,7 @@ public void testGenerateSmallWebGL1ShaderFamily() throws Exception { @Test public void testGenerateSmall300esShaderFamily() throws Exception { final String samplesSubdir = "300es"; - final String referenceShaderName = "mandelbrot_blurry"; + final String referenceShaderName = "mandelbrot_zoom"; final int numVariants = 3; int seed = 2; checkShaderFamilyGeneration(samplesSubdir, referenceShaderName, numVariants, @@ -136,7 +136,7 @@ public void testGenerateSmallWebGL2ShaderFamilyMultiPass() throws Exception { @Test public void testGenerateSmallVulkanShaderFamilyMultiPass() throws Exception { final String samplesSubdir = "310es"; - final String referenceShaderName = "bubblesort_flag"; + final String referenceShaderName = "stable_bubblesort_flag"; final int numVariants = 3; int seed = 9; checkShaderFamilyGeneration(samplesSubdir, referenceShaderName, numVariants, diff --git a/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateTest.java b/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateTest.java index 3eb3a079c..581b5fd78 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/tool/GenerateTest.java @@ -17,26 +17,28 @@ package com.graphicsfuzz.generator.tool; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.ListConcat; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.generator.transformation.DonateLiveCodeTransformation; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.TransformationProbabilities; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Optional; import net.sourceforge.argparse4j.inf.ArgumentParserException; import org.junit.Rule; import org.junit.Test; @@ -48,7 +50,7 @@ public class GenerateTest { - public static final String A_VERTEX_SHADER = "#version 300 es\n" + + private static final String A_VERTEX_SHADER = "#version 300 es\n" + "layout(location=0) in highp vec4 a_position;" + "float foo(float b) {" + " for (int i = 0; i < 10; i++) {" + @@ -76,8 +78,7 @@ public class GenerateTest { @Test public void testSynthetic() throws Exception { - ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - // TODO: Use fileOps more. + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); final String program = "#version 300 es\n" + "precision highp float;\n" @@ -137,21 +138,18 @@ public void testSynthetic() throws Exception { + " }\n" + "}"; - File shaderFile = temporaryFolder.newFile("shader.frag"); - File jsonFile = temporaryFolder.newFile("shader.json"); - fileOps.writeStringToFile(shaderFile, program); - fileOps.writeStringToFile(jsonFile, json); - - File outputDir = temporaryFolder.getRoot(); - - File outputShaderJobFile = new File(outputDir, "output.json"); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), + ParseHelper.parse(program, ShaderKind.FRAGMENT)), + shaderJobFile); - File donors = temporaryFolder.newFolder("donors"); + final File outputDir = temporaryFolder.getRoot(); + final File outputShaderJobFile = new File(outputDir, "output.json"); + final File donors = temporaryFolder.newFolder("donors"); Generate.mainHelper(new String[]{"--seed", "0", - jsonFile.toString(), + shaderJobFile.toString(), donors.toString(), - "300 es", outputShaderJobFile.toString(), }); @@ -161,17 +159,16 @@ public void testSynthetic() throws Exception { @Test public void testStructDonation() throws Exception { + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); final String usesStruct = "struct A { int x; }; void main() { { A a = A(1); a.x = 2; } }"; - File donorsFolder = temporaryFolder.newFolder(); + final File donorsFolder = temporaryFolder.newFolder(); for (int i = 0; i < 10; i++) { - File donor = new File( + final File donor = new File( Paths.get(donorsFolder.getAbsolutePath(), "donor" + i + ".frag").toString()); - BufferedWriter bw = new BufferedWriter(new FileWriter(donor)); - bw.write(usesStruct); - bw.close(); + fileOps.writeStringToFile(donor, usesStruct); } - String reference = "void main() { ; { ; ; ; }; ; { ; ; ; }; ; ; ; ; ; ; }"; - TranslationUnit tu = ParseHelper.parse(reference); + final String reference = "void main() { ; { ; ; ; }; ; { ; ; ; }; ; ; ; ; ; ; }"; + final TranslationUnit tu = ParseHelper.parse(reference); new DonateLiveCodeTransformation(TransformationProbabilities.likelyDonateLiveCode()::donateLiveCodeAtStmt, donorsFolder, GenerationParams.normal(ShaderKind.FRAGMENT, true), false) .apply(tu, @@ -183,35 +180,25 @@ public void testStructDonation() throws Exception { @Test public void testFragVertAndUniformsPassedThrough() throws Exception { - ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - // TODO: Use fileOps more. + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - final String dummyFragment = "precision mediump float;\n" + final TranslationUnit fragmentShader = ParseHelper.parse("precision mediump float;\n" + "void main()\n" + "{\n" + " float iAmAFragmentShader;" - + "}\n"; + + "}\n", ShaderKind.FRAGMENT); - final String dummyVertex = "void main()\n" + final TranslationUnit vertexShader = ParseHelper.parse("void main()\n" + "{\n" + " float iAmAVertexShader;" - + "}\n"; + + "}\n", ShaderKind.VERTEX); - final String json = "{\n" - + "}"; + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), new PipelineInfo(), + vertexShader, fragmentShader); + + final File shaderJobFile = temporaryFolder.newFile("shader.json"); - File fragmentShaderFile = temporaryFolder.newFile("shader.frag"); - File vertexShaderFile = temporaryFolder.newFile("shader.vert"); - File jsonFile = temporaryFolder.newFile("shader.json"); - BufferedWriter bw = new BufferedWriter(new FileWriter(fragmentShaderFile)); - bw.write(dummyFragment); - bw.close(); - bw = new BufferedWriter(new FileWriter(vertexShaderFile)); - bw.write(dummyVertex); - bw.close(); - bw = new BufferedWriter(new FileWriter(jsonFile)); - bw.write(json); - bw.close(); + fileOps.writeShaderJobFile(shaderJob, shaderJobFile); File outputDir = temporaryFolder.getRoot(); @@ -220,9 +207,8 @@ public void testFragVertAndUniformsPassedThrough() throws Exception { File donors = temporaryFolder.newFolder("donors"); Generate.mainHelper(new String[]{"--seed", "0", - jsonFile.toString(), + shaderJobFile.toString(), donors.toString(), - "100", outputShaderJobFile.toString() }); @@ -254,13 +240,13 @@ public void testValidityOfVertexShaderJumpTransformations() throws Exception { private void testValidityOfVertexShaderTransformations(List extraArgs, int repeatCount) throws IOException, InterruptedException, ParseTimeoutException, ArgumentParserException, GlslParserException { - ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - // TODO: Use fileOps more. + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + new PipelineInfo(), ParseHelper.parse(A_VERTEX_SHADER, ShaderKind.VERTEX)); - File vertexShaderFile = temporaryFolder.newFile("shader.vert"); - File jsonFile = temporaryFolder.newFile("shader.json"); - fileOps.writeStringToFile(vertexShaderFile, A_VERTEX_SHADER); - fileOps.writeStringToFile(jsonFile, EMPTY_JSON); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + fileOps.writeShaderJobFile(shaderJob, shaderJobFile); final File outputDir = temporaryFolder.getRoot(); final File outputShaderJobFile = new File(outputDir, "output.json"); @@ -271,9 +257,8 @@ private void testValidityOfVertexShaderTransformations(List extraArgs, i args.addAll( Arrays.asList( "--seed", Integer.toString(seed), - jsonFile.toString(), + shaderJobFile.toString(), donors.toString(), - "300 es", outputShaderJobFile.toString() ) ); @@ -303,31 +288,30 @@ public void testInjectionSwitchAddedByDefault() throws Exception { " x = x + i;" + " }" + "}"; - final String uniforms = "{}"; - final File json = temporaryFolder.newFile("shader.json"); - final File frag = temporaryFolder.newFile("shader.frag"); - fileOps.writeStringToFile(frag, program); - fileOps.writeStringToFile(json, uniforms); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), + new PipelineInfo(), ParseHelper.parse(program, ShaderKind.FRAGMENT)), + shaderJobFile); final File donors = temporaryFolder.newFolder(); final File output = temporaryFolder.newFile("output.json"); - Generate.mainHelper(new String[] { json.getAbsolutePath(), donors.getAbsolutePath(), "100", + Generate.mainHelper(new String[] { shaderJobFile.getAbsolutePath(), donors.getAbsolutePath(), output.getAbsolutePath(), "--seed", "0" }); - final ShaderJob shaderJob = fileOps.readShaderJobFile(output); + final ShaderJob outputShaderJob = fileOps.readShaderJobFile(output); - assertTrue(shaderJob.getPipelineInfo().hasUniform("injectionSwitch")); + assertTrue(outputShaderJob.getPipelineInfo().hasUniform("injectionSwitch")); assertTrue(fileOps.areShadersValid(output, false)); - assertTrue(shaderJob.getShaders().get(0).getTopLevelDeclarations() + assertTrue(outputShaderJob.getShaders().get(0).getTopLevelDeclarations() .stream() .filter(item -> item instanceof VariablesDeclaration) .map(item -> ((VariablesDeclaration) item).getDeclInfos()) .reduce(new ArrayList<>(), ListConcat::concatenate) .stream() - .map(item -> item.getName()) + .map(VariableDeclInfo::getName) .anyMatch(item -> item.equals("injectionSwitch"))); } @@ -343,31 +327,30 @@ public void testNoInjectionSwitchIfDisabled() throws Exception { " x = x + i;" + " }" + "}"; - final String uniforms = "{}"; - final File json = temporaryFolder.newFile("shader.json"); - final File frag = temporaryFolder.newFile("shader.frag"); - fileOps.writeStringToFile(frag, program); - fileOps.writeStringToFile(json, uniforms); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), + ParseHelper.parse(program, ShaderKind.FRAGMENT)), + shaderJobFile); final File donors = temporaryFolder.newFolder(); final File output = temporaryFolder.newFile("output.json"); - Generate.mainHelper(new String[] { json.getAbsolutePath(), donors.getAbsolutePath(), "100", + Generate.mainHelper(new String[] { shaderJobFile.getAbsolutePath(), donors.getAbsolutePath(), output.getAbsolutePath(), "--no-injection-switch", "--seed", "0" }); - final ShaderJob shaderJob = fileOps.readShaderJobFile(output); + final ShaderJob outputShaderJob = fileOps.readShaderJobFile(output); - assertFalse(shaderJob.getPipelineInfo().hasUniform("injectionSwitch")); + assertFalse(outputShaderJob.getPipelineInfo().hasUniform("injectionSwitch")); assertTrue(fileOps.areShadersValid(output, false)); - assertFalse(shaderJob.getShaders().get(0).getTopLevelDeclarations() + assertFalse(outputShaderJob.getShaders().get(0).getTopLevelDeclarations() .stream() .filter(item -> item instanceof VariablesDeclaration) .map(item -> ((VariablesDeclaration) item).getDeclInfos()) .reduce(new ArrayList<>(), ListConcat::concatenate) .stream() - .map(item -> item.getName()) + .map(VariableDeclInfo::getName) .anyMatch(item -> item.equals("injectionSwitch"))); } @@ -377,18 +360,16 @@ public void testBeRobustWhenNoDonors() throws Exception { final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); final String program = "#version 100\n" + "void main() { }"; - final String uniforms = "{}"; - - final File json = temporaryFolder.newFile("shader.json"); - final File frag = temporaryFolder.newFile("shader.frag"); - fileOps.writeStringToFile(frag, program); - fileOps.writeStringToFile(json, uniforms); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), + ParseHelper.parse(program, ShaderKind.FRAGMENT)), + shaderJobFile); final File donors = new File(temporaryFolder.getRoot(), "does_not_exist"); final File output = temporaryFolder.newFile("output.json"); try { - Generate.mainHelper(new String[]{json.getAbsolutePath(), donors.getAbsolutePath(), "100", + Generate.mainHelper(new String[]{shaderJobFile.getAbsolutePath(), donors.getAbsolutePath(), output.getAbsolutePath(), "--seed", "0"}); fail("An exception should have been thrown."); } catch (RuntimeException runtimeException) { @@ -397,6 +378,7 @@ public void testBeRobustWhenNoDonors() throws Exception { } } + @Test public void testStructUniform() throws Exception { // Checks that the generator does not fall over when presented with struct uniforms. final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); @@ -405,16 +387,15 @@ public void testStructUniform() throws Exception { + "uniform S myS;" + "uniform struct T { int y; } myT;" + "void main() {}"; - final String uniforms = "{}"; - final File json = temporaryFolder.newFile("shader.json"); - final File frag = temporaryFolder.newFile("shader.frag"); - fileOps.writeStringToFile(frag, program); - fileOps.writeStringToFile(json, uniforms); + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), + ParseHelper.parse(program, ShaderKind.FRAGMENT)), + shaderJobFile); final File donors = temporaryFolder.newFolder(); final File output = temporaryFolder.newFile("output.json"); - Generate.mainHelper(new String[] { json.getAbsolutePath(), donors.getAbsolutePath(), - "310 es", output.getAbsolutePath(), "--seed", "0" }); + Generate.mainHelper(new String[] { shaderJobFile.getAbsolutePath(), donors.getAbsolutePath(), + output.getAbsolutePath(), "--seed", "0" }); } } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/tool/GlslGenerateTest.java b/generator/src/test/java/com/graphicsfuzz/generator/tool/GlslGenerateTest.java index a3a176165..679cbf4ea 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/tool/GlslGenerateTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/tool/GlslGenerateTest.java @@ -150,8 +150,8 @@ private void checkFragmentShaderFamilyGeneration(String references, generateShaderFamily(references, donors, numVariants, prefix, outputDir, seed, extraArgs, true); - for (String reference : Arrays.asList("bubblesort_flag", "colorgrid_modulo", - "mandelbrot_blurry", "prefix_sum", "squares")) { + for (String reference : Arrays.asList("stable_bubblesort_flag", "colorgrid_modulo", + "mandelbrot_zoom", "prefix_sum", "squares")) { final File expectedOutputDirectory = new File(temporaryFolder.getRoot(), prefix + "_" + reference); assertTrue(expectedOutputDirectory.isDirectory()); diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformationTest.java new file mode 100644 index 000000000..d92425aed --- /dev/null +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateDeadCodeTransformationTest.java @@ -0,0 +1,375 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.generator.transformation; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.stmt.BreakStmt; +import com.graphicsfuzz.common.ast.stmt.ContinueStmt; +import com.graphicsfuzz.common.ast.stmt.DefaultCaseLabel; +import com.graphicsfuzz.common.ast.stmt.ExprCaseLabel; +import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.stmt.IfStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; +import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; +import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderKind; +import com.graphicsfuzz.generator.transformation.donation.DonationContext; +import com.graphicsfuzz.generator.transformation.injection.IInjectionPoint; +import com.graphicsfuzz.generator.transformation.injection.InjectionPoints; +import com.graphicsfuzz.generator.util.GenerationParams; +import com.graphicsfuzz.generator.util.TransformationProbabilities; +import java.util.ArrayList; +import java.util.HashMap; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class DonateDeadCodeTransformationTest { + + @Rule + public TemporaryFolder testFolder = new TemporaryFolder(); + + private DonateDeadCodeTransformation getDummyTransformationObject() { + return new DonateDeadCodeTransformation(IRandom::nextBoolean, testFolder.getRoot(), + GenerationParams.normal(ShaderKind.FRAGMENT, true)); + } + + @Test + public void prepareStatementToDonateTopLevelBreakRemovedWhenNecessary() throws Exception { + + // Checks that a top-level 'break' gets removed, unless injecting into a loop or switch. + + final DonateDeadCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) break;\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " for(int i = 0; i < 100; i++) {\n" + + " switch (i) {\n" + + " case 0:\n" + + " i++;\n" + + " default:\n" + + " i++;\n" + + " }\n" + + " }" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + final Stmt toDonate = ((ForStmt) donor.getMainFunction().getBody().getStmt(0)).getBody() + .clone(); + assert toDonate instanceof IfStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_310); + final boolean containsBreak = new CheckPredicateVisitor() { + @Override + public void visitBreakStmt(BreakStmt breakStmt) { + predicateHolds(); + } + }.test(donated); + assertEquals(containsBreak, injectionPoint.inLoop() || injectionPoint.inSwitch()); + } + } + + @Test + public void prepareStatementToDonateTopLevelContinueRemovedWhenNecessary() throws Exception { + + // Checks that a top-level 'continue' gets removed, unless injecting into a loop. + + final DonateDeadCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) continue;\n" + + "\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " switch(0) {\n" + + " case 1:\n" + + " break;\n" + + " default:\n" + + " 1;\n" + + " }\n" + + " for(int i = 0; i < 100; i++) {\n" + + " ;\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = ((ForStmt) donor.getMainFunction().getBody().getStmt(0)).getBody() + .clone(); + assert toDonate instanceof IfStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_100); + + final boolean containsContinue = new CheckPredicateVisitor() { + @Override + public void visitContinueStmt(ContinueStmt continueStmt) { + predicateHolds(); + } + }.test(donated); + assertEquals(containsContinue, injectionPoint.inLoop()); + } + + } + + @Test + public void prepareStatementToDonateTopLevelCaseAndDefaultRemoved() throws Exception { + // Checks that top-level 'case' and 'default' labels get removed, even when injecting into + // a switch. + + final DonateDeadCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " int x = 3;\n" + + " switch (x) {\n" + + " case 0:\n" + + " x++;\n" + + " default:\n" + + " x++;\n" + + " }\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " switch (0) {\n" + + " case 1:\n" + + " 1;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = ((SwitchStmt) donor.getMainFunction().getBody().getStmt(1)).getBody() + .clone(); + + DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_310); + + new StandardVisitor() { + @Override + public void visitDefaultCaseLabel(DefaultCaseLabel defaultCaseLabel) { + // 'default' labels should have been removed. + fail(); + } + + @Override + public void visitExprCaseLabel(ExprCaseLabel exprCaseLabel) { + // 'case' labels should have been removed. + fail(); + } + }.visit(donated); + } + } + + @Test + public void prepareStatementToDonateBreakFromLoopKept() throws Exception { + // Checks that a 'break' in a loop gets kept if the whole loop is donated. + + final DonateDeadCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) break;\n" + + "\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " for(int i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = donor.getMainFunction().getBody().getStmt(0).clone(); + assert toDonate instanceof ForStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_100); + assertTrue(new CheckPredicateVisitor() { + @Override + public void visitBreakStmt(BreakStmt breakStmt) { + predicateHolds(); + } + }.test(donated)); + } + } + + @Test + public void prepareStatementToDonateSwitchWithBreakAndDefaultKept() throws Exception { + // Checks that 'case', 'default' and 'break' occurring in a switch are kept if the whole + // switch statement is donated. + + final DonateDeadCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " switch (0) {\n" + + " case 0:\n" + + " 1;\n" + + " break;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " switch (0) {\n" + + " case 1:\n" + + " 1;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = donor.getMainFunction().getBody().getStmt(0).clone(); + assert toDonate instanceof SwitchStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_310); + + // Check that the donated statement contains exactly one each of 'break', 'case' and + // 'default'. + new StandardVisitor() { + + private boolean foundBreak = false; + private boolean foundCase = false; + private boolean foundDefault = false; + + @Override + public void visitBreakStmt(BreakStmt breakStmt) { + assertFalse(foundBreak); + foundBreak = true; + } + + @Override + public void visitExprCaseLabel(ExprCaseLabel exprCaseLabel) { + assertFalse(foundCase); + foundCase = true; + } + + @Override + public void visitDefaultCaseLabel(DefaultCaseLabel defaultCaseLabel) { + assertFalse(foundDefault); + foundDefault = true; + } + + private void check(Stmt stmt) { + visit(stmt); + assertTrue(foundBreak); + assertTrue(foundCase); + assertTrue(foundDefault); + } + }; + } + } + + @Test + public void prepareStatementToDonateContinueInLoopKept() throws Exception { + // Checks that a 'continue' in a loop gets kept if the whole loop is donated. + + final DonateDeadCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) continue;\n" + + "\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " for(int i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = donor.getMainFunction().getBody().getStmt(0).clone(); + assert toDonate instanceof ForStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_100); + + // Check that the 'continue' statement is retained. + assertTrue(new CheckPredicateVisitor() { + @Override + public void visitContinueStmt(ContinueStmt continueStmt) { + predicateHolds(); + } + }.test(donated)); + } + } + +} diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java index ebe1ff952..2cad5ca76 100755 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/DonateLiveCodeTransformationTest.java @@ -20,23 +20,29 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; import com.graphicsfuzz.common.ast.stmt.DiscardStmt; +import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.stmt.IfStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeEntry; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; @@ -44,6 +50,8 @@ import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.generator.transformation.donation.DonationContext; import com.graphicsfuzz.generator.transformation.injection.BlockInjectionPoint; +import com.graphicsfuzz.generator.transformation.injection.IInjectionPoint; +import com.graphicsfuzz.generator.transformation.injection.InjectionPoints; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.TransformationProbabilities; import com.graphicsfuzz.server.thrift.ImageJob; @@ -56,19 +64,25 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; public class DonateLiveCodeTransformationTest { @Rule public TemporaryFolder testFolder = new TemporaryFolder(); + private DonateLiveCodeTransformation getDummyTransformationObject() { + return new DonateLiveCodeTransformation(IRandom::nextBoolean, testFolder.getRoot(), GenerationParams.normal(ShaderKind.FRAGMENT, true), + false); + } + @Test - public void prepareStatementToDonate() throws Exception { + public void prepareStatementToDonateDiscardRemoved() throws Exception { - final DonateLiveCodeTransformation dlc = - new DonateLiveCodeTransformation(IRandom::nextBoolean, testFolder.getRoot(), GenerationParams.normal(ShaderKind.FRAGMENT, true), - false); + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); DonationContext dc = new DonationContext(new DiscardStmt(), new HashMap<>(), new ArrayList<>(), null); @@ -84,6 +98,279 @@ public void visitDiscardStmt(DiscardStmt discardStmt) { } + @Test + public void prepareStatementToDonateTopLevelBreakRemoved() throws Exception { + + // Checks that a top-level 'break' gets removed, even when injecting into a loop or + // switch. + + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) break;\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for(int i = 0; i < 100; i++) {\n" + + " switch (i) {\n" + + " case 0:\n" + + " i++;\n" + + " default:\n" + + " i++;\n" + + " }\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = ((ForStmt) donor.getMainFunction().getBody().getStmt(0)).getBody() + .clone(); + assert toDonate instanceof IfStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_310); + assertEquals("{\n" + + " if(i > 5)\n" + + " 1;\n" + + "}\n", donated.getText()); + } + + } + + @Test + public void prepareStatementToDonateTopLevelContinueRemoved() throws Exception { + + // Checks that a top-level 'continue' gets removed, even when injecting into a loop. + + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) continue;\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for(int i = 0; i < 100; i++) {\n" + + " 1;\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + final Stmt toDonate = ((ForStmt) donor.getMainFunction().getBody().getStmt(0)).getBody() + .clone(); + assert toDonate instanceof IfStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_100); + assertEquals("{\n" + + " if(i > 5)\n" + + " 1;\n" + + "}\n", donated.getText()); + } + + } + + @Test + public void prepareStatementToDonateTopLevelCaseAndDefaultRemoved() throws Exception { + // Checks that top-level 'case' and 'default' labels get removed, even when injecting into + // a switch. + + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " int x = 3;\n" + + " switch (x) {\n" + + " case 0:\n" + + " x++;\n" + + " default:\n" + + " x++;\n" + + " }\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " switch (0) {\n" + + " case 1:\n" + + " 1;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = ((SwitchStmt) donor.getMainFunction().getBody().getStmt(1)).getBody() + .clone(); + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_310); + assertEquals("{\n" + + " {\n" + + " 1;\n" + + " x ++;\n" + + " 1;\n" + + " x ++;\n" + + " }\n" + + "}\n", donated.getText()); + } + } + + @Test + public void prepareStatementToDonateBreakFromLoopKept() throws Exception { + // Checks that a 'break' in a loop gets kept if the whole loop is donated. + + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) break;\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " for(int i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = donor.getMainFunction().getBody().getStmt(0) + .clone(); + assert toDonate instanceof ForStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_100); + assertEquals("{\n" + + " for(\n" + + " int i = 0;\n" + + " i < 10;\n" + + " i ++\n" + + " )\n" + + " if(i > 5)\n" + + " break;\n" + + "}\n", donated.getText()); + } + } + + @Test + public void prepareStatementToDonateSwitchWithBreakAndDefaultKept() throws Exception { + // Checks that 'case', 'default' and 'break' occurring in a switch are kept if the whole + // switch statement is donated. + + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " switch (0) {\n" + + " case 0:\n" + + " 1;\n" + + " break;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " switch (0) {\n" + + " case 1:\n" + + " 1;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = donor.getMainFunction().getBody().getStmt(0).clone(); + assert toDonate instanceof SwitchStmt; + + final DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_310); + assertEquals("{\n" + + " switch(0)\n" + + " {\n" + + " case 0:\n" + + " 1;\n" + + " break;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n", donated.getText()); + } + } + + @Test + public void prepareStatementToDonateContinueInLoopKept() throws Exception { + // Checks that a 'continue' in a loop gets kept if the whole loop is donated. + + final DonateLiveCodeTransformation dlc = getDummyTransformationObject(); + final TranslationUnit donor = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " for (int i = 0; i < 10; i ++)\n" + + " if (i > 5) continue;\n" + + "}\n"); + + final TranslationUnit reference = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " ;\n" + + " for(int i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"); + + for (IInjectionPoint injectionPoint : new InjectionPoints(reference, new RandomWrapper(0), + item -> true).getAllInjectionPoints()) { + + final Stmt toDonate = donor.getMainFunction().getBody().getStmt(0).clone(); + assert toDonate instanceof ForStmt; + + DonationContext dc = new DonationContext(toDonate, new HashMap<>(), + new ArrayList<>(), donor.getMainFunction()); + + final Stmt donated = dlc.prepareStatementToDonate(injectionPoint, dc, + TransformationProbabilities.DEFAULT_PROBABILITIES, new RandomWrapper(0), + ShadingLanguageVersion.ESSL_100); + assertEquals("{\n" + + " for(\n" + + " int i = 0;\n" + + " i < 10;\n" + + " i ++\n" + + " )\n" + + " if(i > 5)\n" + + " continue;\n" + + "}\n", donated.getText()); + } + } + @Test public void checkMutateSpecialCase() throws Exception { @@ -91,12 +378,12 @@ public void checkMutateSpecialCase() throws Exception { // here in the spirit of "why delete a test?" final String reference = "#version 300 es\n" - + "void main() {" - + " int t;" - + " {" - + " }" - + " gl_FragColor = vec4(float(t));" - + "}"; + + "void main() {\n" + + " int t;\n" + + " {\n" + + " }\n" + + " gl_FragColor = vec4(float(t));\n" + + "}\n"; final IRandom generator = new RandomWrapper(0); @@ -110,7 +397,7 @@ public void checkMutateSpecialCase() throws Exception { BlockInjectionPoint blockInjectionPoint = - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { BlockInjectionPoint blockInjectionPoint; @@ -118,8 +405,8 @@ public void checkMutateSpecialCase() throws Exception { public void visitBlockStmt(BlockStmt stmt) { super.visitBlockStmt(stmt); if (stmt.getNumStmts() == 0) { - blockInjectionPoint = new BlockInjectionPoint(stmt, null, enclosingFunction, - false, currentScope); + blockInjectionPoint = new BlockInjectionPoint(stmt, null, getEnclosingFunction(), + false, false, getCurrentScope()); } } @@ -133,7 +420,7 @@ BlockInjectionPoint getBlockInjectionPoint(TranslationUnit tu) { new VariablesDeclaration( BasicType.INT, new VariableDeclInfo("a", null, - new ScalarInitializer( + new Initializer( new BinaryExpr( new BinaryExpr( new BinaryExpr( @@ -224,9 +511,8 @@ public void checkFunctionDeclsUnique() throws Exception { { final String referenceSource = "#version 300 es\n" - + "void main() {" - + " " - + "}"; + + "void main() {\n" + + "}\n"; fileOps.writeShaderJobFileFromImageJob( new ImageJob() @@ -291,9 +577,8 @@ public void verySimpleDonorAndSourceNoPrecision() throws Exception { { final String referenceSource = "#version 300 es\n" - + "void main() {" - + " " - + "}"; + + "void main() {\n" + + "}\n"; fileOps.writeShaderJobFileFromImageJob( new ImageJob() @@ -328,4 +613,122 @@ public void verySimpleDonorAndSourceNoPrecision() throws Exception { IParentMap.createParentMap(referenceShaderJob.getFragmentShader().get()); } + @Test + public void testArrayAccessesAreInBounds() throws Exception { + // This checks that array accesses are correctly made in-bounds when injecting live code. + + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + + final File donors = testFolder.newFolder("donors"); + final File referenceFile = testFolder.newFile("reference.json"); + + { + // This donor is designed to have a high chance of leading to an array access getting injected + // such that the array indexing expression will be a free variable for which a fuzzed initial + // value will be created. + final String donorSource = + "#version 300 es\n" + + "void main() {\n" + + " int x = 0;\n" + + " {\n" + + " int A[1];\n" + + " A[x] = 42;\n" + + " {\n" + + " int B[1];\n" + + " B[x] = 42;\n" + + " {\n" + + " int C[1];\n" + + " C[x] = 42;\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + + fileOps.writeShaderJobFileFromImageJob( + new ImageJob() + .setFragmentSource(donorSource) + .setUniformsInfo("{}"), + new File(donors, "donor.json") + ); + } + + { + final String referenceSource = "#version 300 es\n" + + "void main() {\n" + + "}\n"; + + fileOps.writeShaderJobFileFromImageJob( + new ImageJob() + .setFragmentSource(referenceSource) + .setUniformsInfo("{}"), + referenceFile + + ); + } + + int noCodeDonatedCount = 0; + + // Try the following a few times, so that there is a good chance of triggering the issue + // this test was used to catch, should it return: + for (int seed = 0; seed < 15; seed++) { + + final ShaderJob referenceShaderJob = fileOps.readShaderJobFile(referenceFile); + + // Do live code donation. + DonateLiveCodeTransformation transformation = + new DonateLiveCodeTransformation(IRandom::nextBoolean, donors, + GenerationParams.normal(ShaderKind.FRAGMENT, true), false); + + assert referenceShaderJob.getFragmentShader().isPresent(); + + boolean result = transformation.apply( + referenceShaderJob.getFragmentShader().get(), + TransformationProbabilities.onlyLiveCodeAlwaysSubstitute(), + new RandomWrapper(seed), + GenerationParams.normal(ShaderKind.FRAGMENT, true) + ); + + if (!result) { + ++noCodeDonatedCount; + continue; + } + + // An array access injected into the shader must either be (1) already in bounds, or + // (2) made in bounds. Only in the former case can the array index be a variable identifier + // expression, and in that case the expression cannot realistically be statically in bounds + // if the initializer for that expression is under a _GLF_FUZZED macro. (There is a tiny + // chance that the fuzzed expression might statically evaluate to 0, but currently a + // _GLF_FUZZED macro will be treated as not statically in bounds, so the access would be + // made in bounds in that case.) + // + // The following thus checks that if an array is indexed directly by a variable reference, + // the initializer for that variable is not a function call expression. + new ScopeTrackingVisitor() { + + @Override + public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { + super.visitArrayIndexExpr(arrayIndexExpr); + if (arrayIndexExpr.getIndex() instanceof VariableIdentifierExpr) { + final ScopeEntry scopeEntry = getCurrentScope().lookupScopeEntry( + ((VariableIdentifierExpr) arrayIndexExpr.getIndex()).getName()); + assertTrue(scopeEntry.hasVariableDeclInfo()); + assertNotNull(scopeEntry.getVariableDeclInfo().getInitializer()); + assertFalse((scopeEntry.getVariableDeclInfo().getInitializer()) + .getExpr() instanceof FunctionCallExpr); + } + } + + }.visit(referenceShaderJob.getFragmentShader().get()); + + } + // The above code tests donation of live code, but there is still a chance that no code will + // be donated. We assert that this happens < 10 times to ensure that we get some test + // coverage, but this could fail due to bad luck. + Assert.assertTrue( + "Donation failure count should be < 10, " + noCodeDonatedCount, + noCodeDonatedCount < 10 + ); + + } + } diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java index 86e91d3f3..5cddc4694 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/SplitForLoopTransformationTest.java @@ -19,8 +19,8 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.decl.ParameterDecl; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; import com.graphicsfuzz.common.ast.expr.BinOp; @@ -38,13 +38,11 @@ import com.graphicsfuzz.common.ast.type.TypeQualifier; import com.graphicsfuzz.common.ast.type.VoidType; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.typing.Scope; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.generator.semanticspreserving.SplitForLoopMutation; -import com.graphicsfuzz.generator.transformation.SplitForLoopTransformation; import com.graphicsfuzz.generator.transformation.injection.IInjectionPoint; import com.graphicsfuzz.generator.transformation.injection.InjectionPoints; import com.graphicsfuzz.generator.util.GenerationParams; @@ -95,6 +93,11 @@ public boolean inLoop() { throw new RuntimeException(); } + @Override + public boolean inSwitch() { + throw new RuntimeException(); + } + @Override public FunctionDefinition getEnclosingFunction() { throw new RuntimeException(); @@ -117,7 +120,7 @@ public void checkSimpleLoopSplit() { TranslationUnit tu = makeExampleTranslationUnit(); - List ips = new InjectionPoints(tu, new RandomWrapper(), + List ips = new InjectionPoints(tu, new RandomWrapper(0), SplitForLoopMutation::suitableForSplitting).getInjectionPoints( TransformationProbabilities.onlySplitLoops()::splitLoops); @@ -135,7 +138,7 @@ private static TranslationUnit makeExampleTranslationUnit() { new DeclarationStmt( new VariablesDeclaration( new QualifiedType(BasicType.INT, - new ArrayList()), new VariableDeclInfo("i", null, new ScalarInitializer(new IntConstantExpr("0"))))), + new ArrayList()), new VariableDeclInfo("i", null, new Initializer(new IntConstantExpr("0"))))), new BinaryExpr( new VariableIdentifierExpr("i"), new IntConstantExpr("10"), diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java index 3fe3d3d87..150363c02 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/donation/MakeArrayAccessesInBoundsTest.java @@ -20,7 +20,10 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.common.typing.Typer; +import com.graphicsfuzz.common.util.CompareAsts; import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.util.Constants; +import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -29,32 +32,34 @@ public class MakeArrayAccessesInBoundsTest { @Test public void testBasic() throws Exception { - final String shader = "void main() { int A[5]; int x = 17; A[x] = 2; }"; - final String expected = "void main() { int A[5]; int x = 17; A[(x) >= 0 && (x) < 5 ? x : 0] = 2; }"; + final String shader = "#version 300 es\nvoid main() { int A[5]; int x = 17; A[x] = 2; }"; + final String expected = "#version 300 es\nvoid main() { int A[5]; int x = 17; A[" + + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 5)] = 2; }"; final TranslationUnit tu = ParseHelper.parse(shader); - final Typer typer = new Typer(tu, ShadingLanguageVersion.ESSL_300); - MakeArrayAccessesInBounds.makeInBounds(tu, typer); + final Typer typer = new Typer(tu); + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @Test public void testMatrixVector() throws Exception { - final String shader = "void main() { mat4x2 As[5]; int x = 17; int y = -22; int z = 100; As[x][y][z] = 2.0; }"; - final String expected = "void main() { mat4x2 As[5]; int x = 17; int y = -22; int z = 100;" - + "As[(x) >= 0 && (x) < 5 ? x : 0]" - + " /* column */ [(y) >= 0 && (y) < 4 ? y : 0]" - + " /* row */ [(z) >= 0 && (z) < 2 ? z : 0] = 2.0; }"; + final String shader = "#version 300 es\nvoid main() { mat4x2 As[5]; int x = 17; int y = -22; int z = 100; As[x][y][z] = 2.0; }"; + final String expected = "#version 300 es\nvoid main() { mat4x2 As[5]; int x = 17; int y = -22; int z = 100;" + + "As[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 5)]" + + " /* column */ [" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(y, 4)]" + + " /* row */ [" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(z, 2)] = 2.0; }"; final TranslationUnit tu = ParseHelper.parse(shader); - final Typer typer = new Typer(tu, ShadingLanguageVersion.ESSL_300); - MakeArrayAccessesInBounds.makeInBounds(tu, typer); + final Typer typer = new Typer(tu); + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @Test public void testMatrixVector2() throws Exception { - final String shader = "void main() { mat3x4 As[5];" + final String shader = "#version 300 es\n" + + "void main() { mat3x4 As[5];" + " int x = 17;" + " int y = -22;" + " int z = 100;" @@ -64,21 +69,184 @@ public void testMatrixVector2() throws Exception { + " float f;" + " f = v[z];" + "}"; - final String expected = "void main() { mat3x4 As[5];" + final String expected = "#version 300 es\n" + + "void main() { mat3x4 As[5];" + " int x = 17;" + " int y = -22;" + " int z = 100;" - + " mat3x4 A = As[(x) >= 0 && (x) < 5 ? x : 0];" + + " mat3x4 A = As[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 5)];" + " vec4 v;" - + " v = A[(y) >= 0 && (y) < 3 ? y : 0];" + + " v = A[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(y, 3)];" + " float f;" - + " f = v[(z) >= 0 && (z) < 4 ? z : 0];" + + " f = v[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(z, 4)];" + "}"; final TranslationUnit tu = ParseHelper.parse(shader); - final Typer typer = new Typer(tu, ShadingLanguageVersion.ESSL_300); - MakeArrayAccessesInBounds.makeInBounds(tu, typer); + final Typer typer = new Typer(tu); + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } -} \ No newline at end of file + @Test + public void testUIntConstantExprIndex() throws Exception { + final String shader = "#version 300 es\n" + + "void main() { vec3 stuff[16];" + + " uint x = 19u;" + + " vec3 f = stuff[x];" + + "}"; + final String expected = "#version 300 es\n" + + "void main() { vec3 stuff[16];" + + " uint x = 19u;" + + " vec3 f = stuff[" + Constants.GLF_MAKE_IN_BOUNDS_UINT + "(x, 16u)];" + + "}"; + final TranslationUnit tu = ParseHelper.parse(shader); + final Typer typer = new Typer(tu); + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); + assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), + PrettyPrinterVisitor.prettyPrintAsString(tu)); + } + + @Test + public void testUIntFunctionCallReturnIndex() throws Exception { + final String shader = "#version 310 es\n" + + "void main() { vec3 stuff[16];" + + " uint uselessOut;" + + " vec3 f = stuff[uaddCarry(19u, 15u, uselessOut)];" + + "}"; + final String expected = "#version 310 es\n" + + "void main() { vec3 stuff[16];" + + " uint uselessOut;" + + " vec3 f = stuff[" + Constants.GLF_MAKE_IN_BOUNDS_UINT + "(uaddCarry(19u, 15u, " + + "uselessOut), 16u)];" + + "}"; + final TranslationUnit tu = ParseHelper.parse(shader); + final Typer typer = new Typer(tu); + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); + assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), + PrettyPrinterVisitor.prettyPrintAsString(tu)); + } + + @Test + public void testUIntStaticallyInBounds() throws Exception { + final String shader = "#version 310 es\n" + + "void main() { float stuff[16];" + + " stuff[3u] = 1.0;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(shader); + final Typer typer = new Typer(tu); + MakeArrayAccessesInBounds.makeInBounds(tu, typer, tu); + assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(shader)), + PrettyPrinterVisitor.prettyPrintAsString(tu)); + } + + @Test + public void testMakeStructMemberArrayAccessInBounds1() throws Exception { + final String shader = "#version 310 es\n" + + "struct S {\n" + + " int A[3];\n" + + "};\n" + + "void main() {\n" + + " int x;\n" + + " x = 22;\n" + + " S myS;\n" + + " myS.A[x] = 6;\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "struct S {\n" + + " int A[3];\n" + + "};\n" + + "void main() {\n" + + " int x;\n" + + " x = 22;\n" + + " S myS;\n" + + " myS.A[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 3)] = 6;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + MakeArrayAccessesInBounds.makeInBounds(tu, new Typer(tu), tu); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testMakeStructMemberArrayAccessInBounds2() throws Exception { + final String shader = "#version 310 es\n" + + "const int N = 3;\n" + + "struct S {\n" + + " int A[N];\n" + + "};\n" + + "void main() {\n" + + " int x;\n" + + " x = 22;\n" + + " S myS;\n" + + " myS.A[x] = 6;\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "const int N = 3;\n" + + "struct S {\n" + + " int A[N];\n" + + "};\n" + + "void main() {\n" + + " int x;\n" + + " x = 22;\n" + + " S myS;\n" + + " myS.A[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 3)] = 6;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + MakeArrayAccessesInBounds.makeInBounds(tu, new Typer(tu), tu); + CompareAsts.assertEqualAsts(expected, tu); + } + + // TODO(https://github.com/google/graphicsfuzz/issues/784) Enable once array parameter support is + // overhauled. + @Ignore + @Test + public void testMakeArrayParameterAccessInBounds1() throws Exception { + final String shader = "#version 310 es\n" + + "int foo(int A[3], int x) {\n" + + " return A[x];\n" + + "}\n" + + "void main() {\n" + + " int A[3];\n" + + " foo(A, 7);\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "int foo(int A[3], int x) {\n" + + " return A[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 3)];\n" + + "}\n" + + "void main() {\n" + + " int A[3];\n" + + " foo(A, 7);\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + MakeArrayAccessesInBounds.makeInBounds(tu, new Typer(tu), tu); + CompareAsts.assertEqualAsts(expected, tu); + } + + // TODO(https://github.com/google/graphicsfuzz/issues/784) Enable once array parameter support is + // overhauled. + @Ignore + @Test + public void testMakeArrayParameterAccessInBounds2() throws Exception { + final String shader = "#version 310 es\n" + + "const int N = 3;\n" + + "int foo(int A[N], int x) {\n" + + " return A[x];\n" + + "}\n" + + "void main() {\n" + + " int A[N];\n" + + " foo(A, 7);\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "const int N = 3;\n" + + "int foo(int A[N], int x) {\n" + + " return A[" + Constants.GLF_MAKE_IN_BOUNDS_INT + "(x, 3)];\n" + + "}\n" + + "void main() {\n" + + " int A[N];\n" + + " foo(A, 7);\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + MakeArrayAccessesInBounds.makeInBounds(tu, new Typer(tu), tu); + CompareAsts.assertEqualAsts(expected, tu); + } + +} diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/InjectionPointTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/InjectionPointTest.java index 4caeba47c..6b0f0033d 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/InjectionPointTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/InjectionPointTest.java @@ -34,7 +34,7 @@ public void testScopeAtInjectionPoint() { final Scope scope = new Scope(null); assertNotNull( - new InjectionPoint(null, false, scope) { + new InjectionPoint(null, false, false, scope) { @Override public void inject(Stmt stmt) { @@ -64,7 +64,7 @@ public void replaceNext(Stmt stmt) { public void testThatScopeIsCloned() { Scope s = new Scope(null); s.add("v", BasicType.INT, Optional.empty()); - IInjectionPoint injectionPoint = new InjectionPoint(null, false, s) { + IInjectionPoint injectionPoint = new InjectionPoint(null, false, false, s) { @Override public void inject(Stmt stmt) { throw new RuntimeException(); @@ -92,4 +92,4 @@ public void replaceNext(Stmt stmt) { } -} \ No newline at end of file +} diff --git a/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/RemoveReturnStatementsTest.java b/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/RemoveReturnStatementsTest.java index 56fe718c7..6830c6343 100644 --- a/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/RemoveReturnStatementsTest.java +++ b/generator/src/test/java/com/graphicsfuzz/generator/transformation/injection/RemoveReturnStatementsTest.java @@ -29,7 +29,7 @@ public class RemoveReturnStatementsTest { @Test public void testDo() throws Exception { final String prog = "void main() { do return; while(true); }"; - final String expectedProg = "void main() { do ; while(true); }"; + final String expectedProg = "void main() { do 1; while(true); }"; TranslationUnit tu = ParseHelper.parse(prog); new RemoveReturnStatements(tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expectedProg)), @@ -39,7 +39,7 @@ public void testDo() throws Exception { @Test public void testWhile() throws Exception { final String prog = "void main() { while(true) return; }"; - final String expectedProg = "void main() { while(true) ; }"; + final String expectedProg = "void main() { while(true) 1; }"; TranslationUnit tu = ParseHelper.parse(prog); new RemoveReturnStatements(tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expectedProg)), @@ -49,7 +49,7 @@ public void testWhile() throws Exception { @Test public void testFor() throws Exception { final String prog = "void main() { for(int i = 0; i < 100; i++) return; }"; - final String expectedProg = "void main() { for(int i = 0; i < 100; i++) ; }"; + final String expectedProg = "void main() { for(int i = 0; i < 100; i++) 1; }"; TranslationUnit tu = ParseHelper.parse(prog); new RemoveReturnStatements(tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expectedProg)), @@ -59,7 +59,27 @@ public void testFor() throws Exception { @Test public void testIf() throws Exception { final String prog = "void main() { if(true) return; else return; }"; - final String expectedProg = "void main() { if(true) ; else ; }"; + final String expectedProg = "void main() { if(true) 1; else 1; }"; + TranslationUnit tu = ParseHelper.parse(prog); + new RemoveReturnStatements(tu); + assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expectedProg)), + PrettyPrinterVisitor.prettyPrintAsString(tu)); + } + + @Test + public void testSwitch() throws Exception { + final String prog = "void main() {\n" + + " switch(0) {\n" + + " case 0:\n" + + " return;\n" + + " }\n" + + "}\n"; + final String expectedProg = "void main() {\n" + + " switch(0) {\n" + + " case 0:\n" + + " 1;\n" + + " }\n" + + "}\n"; TranslationUnit tu = ParseHelper.parse(prog); new RemoveReturnStatements(tu); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expectedProg)), diff --git a/gfauto/.editorconfig b/gfauto/.editorconfig new file mode 100644 index 000000000..d9eb82f9b --- /dev/null +++ b/gfauto/.editorconfig @@ -0,0 +1,20 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true +insert_final_newline = true +tab_width = 2 +# IDEA specific. +continuation_indent_size = 4 + +[*.py] +indent_size = 4 +tab_width = 4 +# IDEA specific. +continuation_indent_size = 4 diff --git a/gfauto/.flake8 b/gfauto/.flake8 new file mode 100644 index 000000000..a081794ee --- /dev/null +++ b/gfauto/.flake8 @@ -0,0 +1,81 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is not a TOML file; no quoting. + +[flake8] +filename = */gfauto/*.py, */gfautotests/*.py +exclude = *_pb2.py* +max-line-length = 88 +ignore = + E203 + W503 + E501 + + # Missing doc strings. + D100 + D101 + D103 + D104 + D105 + D107 + D102 + + # Audit url open for permitted schemes. + S310 + + # Standard pseudo-random generators are not suitable for security/cryptographic purposes. + S311 + + # Use of subprocess call and run. + S404 + S603 + + # Warning: First line should be in imperative mood. + # This is the opposite of the Google style. + D401 + + # Missing trailing comma. + C812 + + # Missing trailing comma. + S101 + + # Allow unindexed string format parameters + P101 + P102 + P103 + + # Consider possible security implications associated with pickle module + S403 + # Pickle and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue. + S301 + +per-file-ignores = + # Ignore spelling errors in signatures. + gfauto/signature_util.py:SC100,SC200 + + # Ignore use of print in coverage code. + gfauto/cov_*.py:T001 + +# For flake8-quotes: +inline-quotes = " + +# Use the formatter provided by flake8_formatter_abspath. +# This will output the full path of files with warnings so they will be clickable from PyCharm's terminal. +format = abspath + +# For flake8-spellcheck. Default is whitelist.txt but using whitelist.dic means +# we can add the file to PyCharm/IntelliJ. +whitelist=whitelist.dic diff --git a/gfauto/.gitignore b/gfauto/.gitignore new file mode 100644 index 000000000..39ec0a6a5 --- /dev/null +++ b/gfauto/.gitignore @@ -0,0 +1,18 @@ +/dev_shell.sh + +.idea/misc.xml +.idea/modules.xml +.idea/vcs.xml +.idea/workspace.xml +.idea/dictionaries/* + +.venv/ +*.egg-info/ +__pycache__/ +.mypy_cache/ + + +.pytest_cache/ +.vscode/ + +pip-wheel-metadata/ diff --git a/gfauto/.idea/inspectionProfiles/gfauto_inspections.xml b/gfauto/.idea/inspectionProfiles/gfauto_inspections.xml new file mode 100644 index 000000000..35f2f7b1d --- /dev/null +++ b/gfauto/.idea/inspectionProfiles/gfauto_inspections.xml @@ -0,0 +1,17 @@ + + + + \ No newline at end of file diff --git a/gfauto/.idea/inspectionProfiles/profiles_settings.xml b/gfauto/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 000000000..4e0cac616 --- /dev/null +++ b/gfauto/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/gfauto/.idea/mypy.xml b/gfauto/.idea/mypy.xml new file mode 100644 index 000000000..877b548bc --- /dev/null +++ b/gfauto/.idea/mypy.xml @@ -0,0 +1,8 @@ + + + + + \ No newline at end of file diff --git a/gfauto/.idea/runConfigurations/Run_fuzz.xml b/gfauto/.idea/runConfigurations/Run_fuzz.xml new file mode 100644 index 000000000..34423a980 --- /dev/null +++ b/gfauto/.idea/runConfigurations/Run_fuzz.xml @@ -0,0 +1,24 @@ + + + + + \ No newline at end of file diff --git a/gfauto/.idea/runConfigurations/pytest.xml b/gfauto/.idea/runConfigurations/pytest.xml new file mode 100644 index 000000000..e4a9cac18 --- /dev/null +++ b/gfauto/.idea/runConfigurations/pytest.xml @@ -0,0 +1,18 @@ + + + + + \ No newline at end of file diff --git a/gfauto/.idea/scopes/python_files.xml b/gfauto/.idea/scopes/python_files.xml new file mode 100644 index 000000000..44ba6e31b --- /dev/null +++ b/gfauto/.idea/scopes/python_files.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/gfauto/.idea/watcherTasks.xml b/gfauto/.idea/watcherTasks.xml new file mode 100644 index 000000000..5f6740146 --- /dev/null +++ b/gfauto/.idea/watcherTasks.xml @@ -0,0 +1,25 @@ + + + + + + + + \ No newline at end of file diff --git a/gfauto/.isort.cfg b/gfauto/.isort.cfg new file mode 100644 index 000000000..d0ad0b236 --- /dev/null +++ b/gfauto/.isort.cfg @@ -0,0 +1,23 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is not a TOML file; no quoting. + +[isort] +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 0 +use_parentheses = True +line_length = 88 +skip_glob = *_pb2.py* diff --git a/gfauto/Pipfile b/gfauto/Pipfile new file mode 100644 index 000000000..085a37f0e --- /dev/null +++ b/gfauto/Pipfile @@ -0,0 +1,86 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[packages] +# Add "this" package as a dependency; this ensures all packages listed in +# setup.py are installed into the venv, as well as gfauto itself. +gfauto = {path = ".", editable = true} + +[dev-packages] +# autoflake = "*" # Tool to auto-remove unused imports; does not seem to work +# well. + +# iPython shell is useful for modifying artifacts interactively. +ipython = "*" +jedi = "*" + +# Code formatter. Explicit version given because it is a pre-release and +# otherwise we get errors. +black = "==19.10b0" + +# Plugin to protoc that outputs .pyi files (type information). +mypy-protobuf = "*" + +# Provides protoc. +grpcio-tools = "*" + +# Type checking. +mypy = "*" + +# Testing. +pytest = "*" + +# PyLint linter. +pylint = "*" + +# Flake8 linter. +flake8 = "*" +flake8_formatter_abspath = "*" +# cohesion = "*" # A tool for measuring Python class cohesion +# flake8-alfred = "*" # Can be used to blacklist symbols. +# flake8-copyright = "*" # Not maintained. +# flake8-return = "*" # Does not look that good. +# flake8-if-expr = "*" # Disallows ternary expressions. +# flake8-strict = "*" # Just contains two redundant checks. +# flake8-eradicate = "*" # Disallows commented out code, but has false-positives. +flake8-bandit = "*" +flake8-black = "==0.1.0" # Fix to 0.1.0 because otherwise it requires black =>19.3b0 (pre-release) which messes up dependency resolution for some reason. +flake8-breakpoint = "*" +flake8-broken-line = "*" +flake8-bugbear = "*" +flake8-builtins = "*" +flake8-coding = "*" # Checks for a utf-8 comment at top of every file. +flake8-comprehensions = "*" +flake8-commas = "*" +flake8-debugger = "*" +flake8-docstrings = "*" +flake8-isort = "*" +flake8-logging-format = "*" +flake8-mock = "*" +flake8-mutable = "*" +# flake8-mypy = "*" # We run the full mypy; this plugin gives false-positives. +flake8-pep3101 = "*" +flake8-print = "*" +flake8-quotes = "*" +flake8-spellcheck = "*" +flake8-string-format = "*" +flake8-type-annotations = "*" +flake8-variables-names = "*" +mccabe = "*" +pep8-naming = "*" diff --git a/gfauto/Pipfile.lock b/gfauto/Pipfile.lock new file mode 100644 index 000000000..9eb587289 --- /dev/null +++ b/gfauto/Pipfile.lock @@ -0,0 +1,878 @@ +{ + "_meta": { + "hash": { + "sha256": "dfa4f6c61ae08c5c770b0a1eb05ecf76bb31bb1d4013710c2206d26b54717c56" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" + ], + "version": "==2019.11.28" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "gfauto": { + "editable": true, + "path": "." + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "protobuf": { + "hashes": [ + "sha256:0265379852b9e1f76af6d3d3fe4b3c383a595cc937594bda8565cf69a96baabd", + "sha256:29bd1ed46b2536ad8959401a2f02d2d7b5a309f8e97518e4f92ca6c5ba74dbed", + "sha256:3175d45698edb9a07c1a78a1a4850e674ce8988f20596580158b1d0921d0f057", + "sha256:34a7270940f86da7a28be466ac541c89b6dbf144a6348b9cf7ac6f56b71006ce", + "sha256:38cbc830a4a5ba9956763b0f37090bfd14dd74e72762be6225de2ceac55f4d03", + "sha256:665194f5ad386511ac8d8a0bd57b9ab37b8dd2cd71969458777318e774b9cd46", + "sha256:839bad7d115c77cdff29b488fae6a3ab503ce9a4192bd4c42302a6ea8e5d0f33", + "sha256:934a9869a7f3b0d84eca460e386fba1f7ba2a0c1a120a2648bc41fadf50efd1c", + "sha256:aecdf12ef6dc7fd91713a6da93a86c2f2a8fe54840a3b1670853a2b7402e77c9", + "sha256:c4e90bc27c0691c76e09b5dc506133451e52caee1472b8b3c741b7c912ce43ef", + "sha256:c65d135ea2d85d40309e268106dab02d3bea723db2db21c23ecad4163ced210b", + "sha256:c98dea04a1ff41a70aff2489610f280004831798cb36a068013eed04c698903d", + "sha256:d9049aa194378a426f0b2c784e2054565bf6f754d20fcafdee7102a6250556e8", + "sha256:e028fee51c96de4e81924484c77111dfdea14010ecfc906ea5b252209b0c4de6", + "sha256:e84ad26fb50091b1ea676403c0dd2bd47663099454aa6d88000b1dafecab0941", + "sha256:e88a924b591b06d0191620e9c8aa75297b3111066bb09d49a24bae1054a10c13" + ], + "version": "==3.11.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", + "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" + ], + "version": "==2.8.1" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "version": "==2.22.0" + }, + "six": { + "hashes": [ + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + ], + "version": "==1.13.0" + }, + "urllib3": { + "hashes": [ + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" + ], + "version": "==1.25.7" + } + }, + "develop": { + "appdirs": { + "hashes": [ + "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", + "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e" + ], + "version": "==1.4.3" + }, + "astroid": { + "hashes": [ + "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", + "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42" + ], + "version": "==2.3.3" + }, + "attrs": { + "hashes": [ + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + ], + "version": "==19.3.0" + }, + "backcall": { + "hashes": [ + "sha256:38ecd85be2c1e78f77fd91700c76e14667dc21e2713b63876c0eb901196e01e4", + "sha256:bbbf4b1e5cd2bdb08f915895b51081c041bac22394fdfcfdfbe9f14b77c08bf2" + ], + "version": "==0.1.0" + }, + "bandit": { + "hashes": [ + "sha256:336620e220cf2d3115877685e264477ff9d9abaeb0afe3dc7264f55fa17a3952", + "sha256:41e75315853507aa145d62a78a2a6c5e3240fe14ee7c601459d0df9418196065" + ], + "version": "==1.6.2" + }, + "black": { + "hashes": [ + "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b", + "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539" + ], + "index": "pypi", + "version": "==19.10b0" + }, + "click": { + "hashes": [ + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + ], + "version": "==7.0" + }, + "decorator": { + "hashes": [ + "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce", + "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d" + ], + "version": "==4.4.1" + }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "version": "==0.3" + }, + "flake8": { + "hashes": [ + "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", + "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" + ], + "index": "pypi", + "version": "==3.7.9" + }, + "flake8-bandit": { + "hashes": [ + "sha256:687fc8da2e4a239b206af2e54a90093572a60d0954f3054e23690739b0b0de3b" + ], + "index": "pypi", + "version": "==2.1.2" + }, + "flake8-black": { + "hashes": [ + "sha256:6b5fe2a609fa750170da8d5b1ed7c11029bceaff025660be7f19307ec6fa0c35" + ], + "index": "pypi", + "version": "==0.1.0" + }, + "flake8-breakpoint": { + "hashes": [ + "sha256:27e0cb132647f9ef348b4a3c3126e7350bedbb22e8e221cd11712a223855ea0b", + "sha256:5bc70d478f0437a3655d094e1d2fca81ddacabaa84d99db45ad3630bf2004064" + ], + "index": "pypi", + "version": "==1.1.0" + }, + "flake8-broken-line": { + "hashes": [ + "sha256:30378a3749911e453d0a9e03204156cbbd35bcc03fb89f12e6a5206e5baf3537", + "sha256:7721725dce3aeee1df371a252822f1fcecfaf2766dcf5bac54ee1b3f779ee9d1" + ], + "index": "pypi", + "version": "==0.1.1" + }, + "flake8-bugbear": { + "hashes": [ + "sha256:d8c466ea79d5020cb20bf9f11cf349026e09517a42264f313d3f6fddb83e0571", + "sha256:ded4d282778969b5ab5530ceba7aa1a9f1b86fa7618fc96a19a1d512331640f8" + ], + "index": "pypi", + "version": "==19.8.0" + }, + "flake8-builtins": { + "hashes": [ + "sha256:8d806360767947c0035feada4ddef3ede32f0a586ef457e62d811b8456ad9a51", + "sha256:cd7b1b7fec4905386a3643b59f9ca8e305768da14a49a7efb31fe9364f33cd04" + ], + "index": "pypi", + "version": "==1.4.1" + }, + "flake8-coding": { + "hashes": [ + "sha256:79704112c44d09d4ab6c8965e76a20c3f7073d52146db60303bce777d9612260", + "sha256:b8f4d5157a8f74670e6cfea732c3d9f4291a4e994c8701d2c55f787c6e6cb741" + ], + "index": "pypi", + "version": "==1.3.2" + }, + "flake8-commas": { + "hashes": [ + "sha256:d3005899466f51380387df7151fb59afec666a0f4f4a2c6a8995b975de0f44b7", + "sha256:ee2141a3495ef9789a3894ed8802d03eff1eaaf98ce6d8653a7c573ef101935e" + ], + "index": "pypi", + "version": "==2.0.0" + }, + "flake8-comprehensions": { + "hashes": [ + "sha256:6de428c3ac67d3614d527456469c8a3d6638960e9ad7e1222358526a2507400a", + "sha256:e3a8350a35d7bc71f8a1f64cf3c633a418a26b0edace2219d26ea4dd78ac21f3" + ], + "index": "pypi", + "version": "==3.1.4" + }, + "flake8-debugger": { + "hashes": [ + "sha256:712d7c1ff69ddf3f0130e94cc88c2519e720760bce45e8c330bfdcb61ab4090d" + ], + "index": "pypi", + "version": "==3.2.1" + }, + "flake8-docstrings": { + "hashes": [ + "sha256:3d5a31c7ec6b7367ea6506a87ec293b94a0a46c0bce2bb4975b7f1d09b6f3717", + "sha256:a256ba91bc52307bef1de59e2a009c3cf61c3d0952dbe035d6ff7208940c2edc" + ], + "index": "pypi", + "version": "==1.5.0" + }, + "flake8-formatter-abspath": { + "hashes": [ + "sha256:694d0874d5d047ed57c82a10213f75604475e4525ee8bbaad53417a7d6f8442c", + "sha256:7ff3d9186eb61b4c78a118acf70cd6ef26fb5e323d9a927b47f062b5e8bf31ed" + ], + "index": "pypi", + "version": "==1.0.1" + }, + "flake8-isort": { + "hashes": [ + "sha256:1e67b6b90a9b980ac3ff73782087752d406ce0a729ed928b92797f9fa188917e", + "sha256:81a8495eefed3f2f63f26cd2d766c7b1191e923a15b9106e6233724056572c68" + ], + "index": "pypi", + "version": "==2.7.0" + }, + "flake8-logging-format": { + "hashes": [ + "sha256:ca5f2b7fc31c3474a0aa77d227e022890f641a025f0ba664418797d979a779f8" + ], + "index": "pypi", + "version": "==0.6.0" + }, + "flake8-mock": { + "hashes": [ + "sha256:2fa775e7589f4e1ad74f35d60953eb20937f5d7355235e54bf852c6837f2bede" + ], + "index": "pypi", + "version": "==0.3" + }, + "flake8-mutable": { + "hashes": [ + "sha256:38fd9dadcbcda6550a916197bc40ed76908119dabb37fbcca30873666c31d2d5", + "sha256:ee9b77111b867d845177bbc289d87d541445ffcc6029a0c5c65865b42b18c6a6" + ], + "index": "pypi", + "version": "==1.2.0" + }, + "flake8-pep3101": { + "hashes": [ + "sha256:493821d6bdd083794eb0691ebe5b68e5c520b622b269d60e54308fb97440e21a", + "sha256:b661ab718df42b87743dde266ef5de4f9e900b56c67dbccd45d24cf527545553" + ], + "index": "pypi", + "version": "==1.2.1" + }, + "flake8-plugin-utils": { + "hashes": [ + "sha256:1ac5eb19773d5c7fdde60b0d901ae86be9c751bf697c61fdb6609b86872f3c6e", + "sha256:24b4a3b216ad588951d3d7adef4645dcb3b32a33b878e03baa790b5a66bf3a73" + ], + "version": "==1.0.0" + }, + "flake8-polyfill": { + "hashes": [ + "sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9", + "sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda" + ], + "version": "==1.0.2" + }, + "flake8-print": { + "hashes": [ + "sha256:324f9e59a522518daa2461bacd7f82da3c34eb26a4314c2a54bd493f8b394a68" + ], + "index": "pypi", + "version": "==3.1.4" + }, + "flake8-quotes": { + "hashes": [ + "sha256:11a15d30c92ca5f04c2791bd7019cf62b6f9d3053eb050d02a135557eb118bfc" + ], + "index": "pypi", + "version": "==2.1.1" + }, + "flake8-spellcheck": { + "hashes": [ + "sha256:a82429d1e479f4efbd20870be1963ed0d9b389ddc3b087ff2261eb137ab8988a", + "sha256:f92e6da46fd30d805c28e22f705b5da2a719b9dca8a9abd669bdcd9cc6158e09" + ], + "index": "pypi", + "version": "==0.9.1" + }, + "flake8-string-format": { + "hashes": [ + "sha256:68ea72a1a5b75e7018cae44d14f32473c798cf73d75cbaed86c6a9a907b770b2", + "sha256:774d56103d9242ed968897455ef49b7d6de272000cfa83de5814273a868832f1" + ], + "index": "pypi", + "version": "==0.2.3" + }, + "flake8-type-annotations": { + "hashes": [ + "sha256:88775455792ad7bbd63a71bc94e8a077deb5608eacb5add7e5a7a648c7636426", + "sha256:de64de5efef3277d7b6012e8618c37d35b21465fb16292e46e6eec5b87e47a8c" + ], + "index": "pypi", + "version": "==0.1.0" + }, + "flake8-variables-names": { + "hashes": [ + "sha256:d109f5a8fe8c20d64e165287330f1b0160b442d7f96e1527124ba1b63c438347" + ], + "index": "pypi", + "version": "==0.0.3" + }, + "gitdb2": { + "hashes": [ + "sha256:1b6df1433567a51a4a9c1a5a0de977aa351a405cc56d7d35f3388bad1f630350", + "sha256:96bbb507d765a7f51eb802554a9cfe194a174582f772e0d89f4e87288c288b7b" + ], + "version": "==2.0.6" + }, + "gitpython": { + "hashes": [ + "sha256:9c2398ffc3dcb3c40b27324b316f08a4f93ad646d5a6328cafbb871aa79f5e42", + "sha256:c155c6a2653593ccb300462f6ef533583a913e17857cfef8fc617c246b6dc245" + ], + "version": "==3.0.5" + }, + "grpcio": { + "hashes": [ + "sha256:0419ae5a45f49c7c40d9ae77ae4de9442431b7822851dfbbe56ee0eacb5e5654", + "sha256:1e8631eeee0fb0b4230aeb135e4890035f6ef9159c2a3555fa184468e325691a", + "sha256:24db2fa5438f3815a4edb7a189035051760ca6aa2b0b70a6a948b28bfc63c76b", + "sha256:2adb1cdb7d33e91069517b41249622710a94a1faece1fed31cd36904e4201cde", + "sha256:2cd51f35692b551aeb1fdeb7a256c7c558f6d78fcddff00640942d42f7aeba5f", + "sha256:3247834d24964589f8c2b121b40cd61319b3c2e8d744a6a82008643ef8a378b1", + "sha256:3433cb848b4209717722b62392e575a77a52a34d67c6730138102abc0a441685", + "sha256:39671b7ff77a962bd745746d9d2292c8ed227c5748f16598d16d8631d17dd7e5", + "sha256:40a0b8b2e6f6dd630f8b267eede2f40a848963d0f3c40b1b1f453a4a870f679e", + "sha256:40f9a74c7aa210b3e76eb1c9d56aa8d08722b73426a77626967019df9bbac287", + "sha256:423f76aa504c84cb94594fb88b8a24027c887f1c488cf58f2173f22f4fbd046c", + "sha256:43bd04cec72281a96eb361e1b0232f0f542b46da50bcfe72ef7e5a1b41d00cb3", + "sha256:43e38762635c09e24885d15e3a8e374b72d105d4178ee2cc9491855a8da9c380", + "sha256:4413b11c2385180d7de03add6c8845dd66692b148d36e27ec8c9ef537b2553a1", + "sha256:4450352a87094fd58daf468b04c65a9fa19ad11a0ac8ac7b7ff17d46f873cbc1", + "sha256:49ffda04a6e44de028b3b786278ac9a70043e7905c3eea29eed88b6524d53a29", + "sha256:4a38c4dde4c9120deef43aaabaa44f19186c98659ce554c29788c4071ab2f0a4", + "sha256:50b1febdfd21e2144b56a9aa226829e93a79c354ef22a4e5b013d9965e1ec0ed", + "sha256:559b1a3a8be7395ded2943ea6c2135d096f8cc7039d6d12127110b6496f251fe", + "sha256:5de86c182667ec68cf84019aa0d8ceccf01d352cdca19bf9e373725204bdbf50", + "sha256:5fc069bb481fe3fad0ba24d3baaf69e22dfa6cc1b63290e6dfeaf4ac1e996fb7", + "sha256:6a19d654da49516296515d6f65de4bbcbd734bc57913b21a610cfc45e6df3ff1", + "sha256:7535b3e52f498270e7877dde1c8944d6b7720e93e2e66b89c82a11447b5818f5", + "sha256:7c4e495bcabc308198b8962e60ca12f53b27eb8f03a21ac1d2d711d6dd9ecfca", + "sha256:8a8fc4a0220367cb8370cedac02272d574079ccc32bffbb34d53aaf9e38b5060", + "sha256:8b008515e067232838daca020d1af628bf6520c8cc338bf383284efe6d8bd083", + "sha256:8d1684258e1385e459418f3429e107eec5fb3d75e1f5a8c52e5946b3f329d6ea", + "sha256:8eb5d54b87fb561dc2e00a5c5226c33ffe8dbc13f2e4033a412bafb7b37b194d", + "sha256:94cdef0c61bd014bb7af495e21a1c3a369dd0399c3cd1965b1502043f5c88d94", + "sha256:9d9f3be69c7a5e84c3549a8c4403fa9ac7672da456863d21e390b2bbf45ccad1", + "sha256:9fb6fb5975a448169756da2d124a1beb38c0924ff6c0306d883b6848a9980f38", + "sha256:a5eaae8700b87144d7dfb475aa4675e500ff707292caba3deff41609ddc5b845", + "sha256:aaeac2d552772b76d24eaff67a5d2325bc5205c74c0d4f9fbe71685d4a971db2", + "sha256:bb611e447559b3b5665e12a7da5160c0de6876097f62bf1d23ba66911564868e", + "sha256:bc0d41f4eb07da8b8d3ea85e50b62f6491ab313834db86ae2345be07536a4e5a", + "sha256:bf51051c129b847d1bb63a9b0826346b5f52fb821b15fe5e0d5ef86f268510f5", + "sha256:c948c034d8997526011960db54f512756fb0b4be1b81140a15b4ef094c6594a4", + "sha256:d435a01334157c3b126b4ee5141401d44bdc8440993b18b05e2f267a6647f92d", + "sha256:d46c1f95672b73288e08cdca181e14e84c6229b5879561b7b8cfd48374e09287", + "sha256:d5d58309b42064228b16b0311ff715d6c6e20230e81b35e8d0c8cfa1bbdecad8", + "sha256:dc6e2e91365a1dd6314d615d80291159c7981928b88a4c65654e3fefac83a836", + "sha256:e0dfb5f7a39029a6cbec23affa923b22a2c02207960fd66f109e01d6f632c1eb", + "sha256:eb4bf58d381b1373bd21d50837a53953d625d1693f1b58fed12743c75d3dd321", + "sha256:ebb211a85248dbc396b29320273c1ffde484b898852432613e8df0164c091006", + "sha256:ec759ece4786ae993a5b7dc3b3dead6e9375d89a6c65dfd6860076d2eb2abe7b", + "sha256:f55108397a8fa164268238c3e69cc134e945d1f693572a2f05a028b8d0d2b837", + "sha256:f6c706866d424ff285b85a02de7bbe5ed0ace227766b2c42cbe12f3d9ea5a8aa", + "sha256:f8370ad332b36fbad117440faf0dd4b910e80b9c49db5648afd337abdde9a1b6" + ], + "version": "==1.25.0" + }, + "grpcio-tools": { + "hashes": [ + "sha256:007c075eb9611379fa8f520a1865b9afd850469495b0e4a46e1349b2dc1744ce", + "sha256:02ae9708bdd3f329b1abe1ee16b1d768b2dd7a036a8a57e342d08ee8ca054cec", + "sha256:2f10226bfea4f947de355008b14fb4711c85fc1121570833a96f0e2cd8de580f", + "sha256:314354c7321c84a6e176a99afe1945c933b8a38b4f837255c8decfef8d07f24e", + "sha256:406b530c283a2bb804a10ee97928290b0b60788cd114ddfce0faa681cccfe4b8", + "sha256:49e7682e505e6a1d35459dae1d8a616a08d5cfa6f05de00235aff2e15786af14", + "sha256:4a5c2b38078fc4b949e4e70f7e25cb80443d1ee9a648ce4223aa3c040a0d3b9b", + "sha256:4b40291d67a1fecb5170ed9ec32016e2ae07908a8fa143d2d37311b2bcbeb2c5", + "sha256:4b72b04cba6ecd1940d6eda07886f80fe71fb2e669f1095ebab58b1eb17a53fa", + "sha256:4cc95d5fddebb9348fafcc4c0147745882794ded7cfd5282b2aa158596c77a8a", + "sha256:4ce0261bd4426482a96467ed9ad8411417a6932c331a5bb35aa1907f618f34f6", + "sha256:5226371a2b569c62be0d0590ccff7bbb9566762f243933efbd4b695f9f108cd5", + "sha256:52aab4cbab10683f8830420c0b55ccdc6344702b4a0940913d71fe928dd731c9", + "sha256:532a19419535a92a1b621222f70d6da7624151fe69afa4a1063be56e7a2b884a", + "sha256:5a8d44add097e0a3a7c27e66a8ed0aa2fd561cda77381e818cf7862d4ad0f629", + "sha256:64f6027887e32a938f00b2344c337c6d4f7c4cf157ec2e84b1dd6b6fddad8e50", + "sha256:651b0441e8d8f302b44fb50397fe73dcd5e61b790533438e690055abdef3b234", + "sha256:67d12ec4548dd2b1f15c9e3a953c8f48d8c3441c2d8bd143fc3af95a1c041c2b", + "sha256:6c029341132a0e64cbd2dba1dda9a125e06a798b9ec864569afdecce626dd5d5", + "sha256:6e64214709f37b347875ac83cfed4e9cfd287f255dab2836521f591620412c40", + "sha256:6f70fc9a82a0145296358720cf24f83a657a745e8b51ec9564f4c9e678c5b872", + "sha256:6fb4739eb5eef051945b16b3c434d08653ea05f0313cf88495ced5d9db641745", + "sha256:79b5b1c172dafb0e76aa95bf572d4c7afc0bf97a1669b2228a0bc151071c4666", + "sha256:7d02755480cec3c0222f35397e810bfaf4cf9f2bf2e626f7f6efc1d40fffb7fa", + "sha256:818f2b8168760cf16e66fe85894a37afcff5378a64939549663a371216618498", + "sha256:834564c2fba02c31179af081bd80aada8dfdcca52c80e241353f6063b6154bd2", + "sha256:8b17347a90a14386641ffe57743bbb01a16a7149c95905364d3c8091ad377bd8", + "sha256:902e13dbaca9733e4668928967b301526197ecffacb8c7a0acc0c7045de8836f", + "sha256:988014c714ca654b3b7ca9f4dabfe487b00e023bfdd9eaf1bb0fed82bf8c4255", + "sha256:9a83d39e198cbed5d093f43790b92945ab74140357ec00e53ae13b421489ffb7", + "sha256:ac7649cff7354d2f04ebe2872f786a1d07547deded61f3d39036ebb569de91bc", + "sha256:b013d93bc6dc5c7bf3642bf30e673daee46f9a4984fbd9588a9cda1071278414", + "sha256:b02701d40f1ccf16bc8c46f56bdbf89e03110bd8fd570c854e72299ce2920c35", + "sha256:b0ef0da2eec959def8ba508b2a763c492f1fb989446a422d1456ac17dc1b19f4", + "sha256:bb8264ccf8ff904a1a396dc757ac1560b24f270b90e7dabb0ae3f637cb351bb3", + "sha256:bbfb58f5c0aa27b599141bb5eacaf8116b55ad89bc5a2c3afd5e965d840ad341", + "sha256:c1a482fdd8952a7f0098f78161a4deef8a500e54babef302548cd9f1e326d42c", + "sha256:c40efc662fa037898488e31756242af68a8ab5729f939bc8c9ba259bc32e7d6a", + "sha256:c5ad07adae3fe62761bc662c554c2734203f0f700616fc58138b852a7ef5e40e", + "sha256:c765512cb5cb4afaf652837b8cc69229dee14c8e92f15a6ea0f4dfd646902dd2", + "sha256:c871f5a89012ae44d9233305d74dfdd2059a78f0cb0303d38a4b6a562c6f9ba7", + "sha256:cc950fb17c1172d0c0129e8c6e787206e7ef8c24a8e39005f8cc297e9faa4f9a", + "sha256:d3619b43009a5c82cb7ef11847518236140d7ffdcc6600e1a151b8b49350693a", + "sha256:dc17a8a8b39cb37380d927d4669882af4ccc7d3ee298a15a3004f4b18ecd2ac3", + "sha256:eab3684ce9dec3a934a36ba79e8435210d07c50906425ab157eeb4b14503a925", + "sha256:f258b32dffd27ef1eb5f5f01ebb115dfad07677b0510b41f786c511a62ded033", + "sha256:f550c94728b67a7eeddc35b03c99552f2d7aac09c52935ad4b0552d0843fd03c", + "sha256:f7fc690a517c8f3765796ed005bb3273895a985a8593977291bad24568e018e3" + ], + "index": "pypi", + "version": "==1.25.0" + }, + "importlib-metadata": { + "hashes": [ + "sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21", + "sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742" + ], + "markers": "python_version < '3.8'", + "version": "==1.1.0" + }, + "ipython": { + "hashes": [ + "sha256:c66c7e27239855828a764b1e8fc72c24a6f4498a2637572094a78c5551fb9d51", + "sha256:f186b01b36609e0c5d0de27c7ef8e80c990c70478f8c880863004b3489a9030e" + ], + "index": "pypi", + "version": "==7.10.1" + }, + "ipython-genutils": { + "hashes": [ + "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8", + "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8" + ], + "version": "==0.2.0" + }, + "isort": { + "hashes": [ + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" + ], + "version": "==4.3.21" + }, + "jedi": { + "hashes": [ + "sha256:786b6c3d80e2f06fd77162a07fed81b8baa22dde5d62896a790a331d6ac21a27", + "sha256:ba859c74fa3c966a22f2aeebe1b74ee27e2a462f56d3f5f7ca4a59af61bfe42e" + ], + "index": "pypi", + "version": "==0.15.1" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", + "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", + "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", + "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", + "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", + "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", + "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", + "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", + "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", + "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", + "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", + "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", + "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", + "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", + "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", + "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", + "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", + "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", + "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", + "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", + "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" + ], + "version": "==1.4.3" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "index": "pypi", + "version": "==0.6.1" + }, + "more-itertools": { + "hashes": [ + "sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2", + "sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45" + ], + "version": "==8.0.0" + }, + "mypy": { + "hashes": [ + "sha256:02d9bdd3398b636723ecb6c5cfe9773025a9ab7f34612c1cde5c7f2292e2d768", + "sha256:088f758a50af31cf8b42688118077292370c90c89232c783ba7979f39ea16646", + "sha256:28e9fbc96d13397a7ddb7fad7b14f373f91b5cff538e0772e77c270468df083c", + "sha256:30e123b24931f02c5d99307406658ac8f9cd6746f0d45a3dcac2fe5fbdd60939", + "sha256:3294821b5840d51a3cd7a2bb63b40fc3f901f6a3cfb3c6046570749c4c7ef279", + "sha256:41696a7d912ce16fdc7c141d87e8db5144d4be664a0c699a2b417d393994b0c2", + "sha256:4f42675fa278f3913340bb8c3371d191319704437758d7c4a8440346c293ecb2", + "sha256:54d205ccce6ed930a8a2ccf48404896d456e8b87812e491cb907a355b1a9c640", + "sha256:6992133c95a2847d309b4b0c899d7054adc60481df6f6b52bb7dee3d5fd157f7", + "sha256:6ecbd0e8e371333027abca0922b0c2c632a5b4739a0c61ffbd0733391e39144c", + "sha256:83fa87f556e60782c0fc3df1b37b7b4a840314ba1ac27f3e1a1e10cb37c89c17", + "sha256:c87ac7233c629f305602f563db07f5221950fe34fe30af072ac838fa85395f78", + "sha256:de9ec8dba773b78c49e7bec9a35c9b6fc5235682ad1fc2105752ae7c22f4b931", + "sha256:f385a0accf353ca1bca4bbf473b9d83ed18d923fdb809d3a70a385da23e25b6a" + ], + "index": "pypi", + "version": "==0.750" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" + }, + "mypy-protobuf": { + "hashes": [ + "sha256:4c18a300054abdd6d635b02059d36bc165a166dab2bde4b899d76e1118f63d9a", + "sha256:72ab724299aebd930b88476f6545587bff5bf480697c016097bd188841a56276" + ], + "index": "pypi", + "version": "==1.16" + }, + "packaging": { + "hashes": [ + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" + ], + "version": "==19.2" + }, + "parso": { + "hashes": [ + "sha256:63854233e1fadb5da97f2744b6b24346d2750b85965e7e399bec1620232797dc", + "sha256:666b0ee4a7a1220f65d367617f2cd3ffddff3e205f3f16a0284df30e774c2a9c" + ], + "version": "==0.5.1" + }, + "pathspec": { + "hashes": [ + "sha256:e285ccc8b0785beadd4c18e5708b12bb8fcf529a1e61215b3feff1d1e559ea5c" + ], + "version": "==0.6.0" + }, + "pbr": { + "hashes": [ + "sha256:139d2625547dbfa5fb0b81daebb39601c478c21956dc57e2e07b74450a8c506b", + "sha256:61aa52a0f18b71c5cc58232d2cf8f8d09cd67fcad60b742a60124cb8d6951488" + ], + "version": "==5.4.4" + }, + "pep8-naming": { + "hashes": [ + "sha256:45f330db8fcfb0fba57458c77385e288e7a3be1d01e8ea4268263ef677ceea5f", + "sha256:a33d38177056321a167decd6ba70b890856ba5025f0a8eca6a3eda607da93caf" + ], + "index": "pypi", + "version": "==0.9.1" + }, + "pexpect": { + "hashes": [ + "sha256:2094eefdfcf37a1fdbfb9aa090862c1a4878e5c7e0e7e7088bdb511c558e5cd1", + "sha256:9e2c1fd0e6ee3a49b28f95d4b33bc389c89b20af6a1255906e90ff1262ce62eb" + ], + "markers": "sys_platform != 'win32'", + "version": "==4.7.0" + }, + "pickleshare": { + "hashes": [ + "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca", + "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56" + ], + "version": "==0.7.5" + }, + "pluggy": { + "hashes": [ + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" + ], + "version": "==0.13.1" + }, + "prompt-toolkit": { + "hashes": [ + "sha256:0278d2f51b5ceba6ea8da39f76d15684e84c996b325475f6e5720edc584326a7", + "sha256:63daee79aa8366c8f1c637f1a4876b890da5fc92a19ebd2f7080ebacb901e990" + ], + "version": "==3.0.2" + }, + "protobuf": { + "hashes": [ + "sha256:0265379852b9e1f76af6d3d3fe4b3c383a595cc937594bda8565cf69a96baabd", + "sha256:29bd1ed46b2536ad8959401a2f02d2d7b5a309f8e97518e4f92ca6c5ba74dbed", + "sha256:3175d45698edb9a07c1a78a1a4850e674ce8988f20596580158b1d0921d0f057", + "sha256:34a7270940f86da7a28be466ac541c89b6dbf144a6348b9cf7ac6f56b71006ce", + "sha256:38cbc830a4a5ba9956763b0f37090bfd14dd74e72762be6225de2ceac55f4d03", + "sha256:665194f5ad386511ac8d8a0bd57b9ab37b8dd2cd71969458777318e774b9cd46", + "sha256:839bad7d115c77cdff29b488fae6a3ab503ce9a4192bd4c42302a6ea8e5d0f33", + "sha256:934a9869a7f3b0d84eca460e386fba1f7ba2a0c1a120a2648bc41fadf50efd1c", + "sha256:aecdf12ef6dc7fd91713a6da93a86c2f2a8fe54840a3b1670853a2b7402e77c9", + "sha256:c4e90bc27c0691c76e09b5dc506133451e52caee1472b8b3c741b7c912ce43ef", + "sha256:c65d135ea2d85d40309e268106dab02d3bea723db2db21c23ecad4163ced210b", + "sha256:c98dea04a1ff41a70aff2489610f280004831798cb36a068013eed04c698903d", + "sha256:d9049aa194378a426f0b2c784e2054565bf6f754d20fcafdee7102a6250556e8", + "sha256:e028fee51c96de4e81924484c77111dfdea14010ecfc906ea5b252209b0c4de6", + "sha256:e84ad26fb50091b1ea676403c0dd2bd47663099454aa6d88000b1dafecab0941", + "sha256:e88a924b591b06d0191620e9c8aa75297b3111066bb09d49a24bae1054a10c13" + ], + "version": "==3.11.1" + }, + "ptyprocess": { + "hashes": [ + "sha256:923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0", + "sha256:d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f" + ], + "version": "==0.6.0" + }, + "py": { + "hashes": [ + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + ], + "version": "==1.8.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" + ], + "version": "==2.5.0" + }, + "pydocstyle": { + "hashes": [ + "sha256:04c84e034ebb56eb6396c820442b8c4499ac5eb94a3bda88951ac3dc519b6058", + "sha256:66aff87ffe34b1e49bff2dd03a88ce6843be2f3346b0c9814410d34987fbab59" + ], + "version": "==4.0.1" + }, + "pyflakes": { + "hashes": [ + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" + ], + "version": "==2.1.1" + }, + "pygments": { + "hashes": [ + "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b", + "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe" + ], + "version": "==2.5.2" + }, + "pylint": { + "hashes": [ + "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd", + "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4" + ], + "index": "pypi", + "version": "==2.4.4" + }, + "pyparsing": { + "hashes": [ + "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", + "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" + ], + "version": "==2.4.5" + }, + "pytest": { + "hashes": [ + "sha256:63344a2e3bce2e4d522fd62b4fdebb647c019f1f9e4ca075debbd13219db4418", + "sha256:f67403f33b2b1d25a6756184077394167fe5e2f9d8bdaab30707d19ccec35427" + ], + "index": "pypi", + "version": "==5.3.1" + }, + "pyyaml": { + "hashes": [ + "sha256:0e7f69397d53155e55d10ff68fdfb2cf630a35e6daf65cf0bdeaf04f127c09dc", + "sha256:2e9f0b7c5914367b0916c3c104a024bb68f269a486b9d04a2e8ac6f6597b7803", + "sha256:35ace9b4147848cafac3db142795ee42deebe9d0dad885ce643928e88daebdcc", + "sha256:38a4f0d114101c58c0f3a88aeaa44d63efd588845c5a2df5290b73db8f246d15", + "sha256:483eb6a33b671408c8529106df3707270bfacb2447bf8ad856a4b4f57f6e3075", + "sha256:4b6be5edb9f6bb73680f5bf4ee08ff25416d1400fbd4535fe0069b2994da07cd", + "sha256:7f38e35c00e160db592091751d385cd7b3046d6d51f578b29943225178257b31", + "sha256:8100c896ecb361794d8bfdb9c11fce618c7cf83d624d73d5ab38aef3bc82d43f", + "sha256:c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c", + "sha256:e4c015484ff0ff197564917b4b4246ca03f411b9bd7f16e02a2f586eb48b6d04", + "sha256:ebc4ed52dcc93eeebeae5cf5deb2ae4347b3a81c3fa12b0b8c976544829396a4" + ], + "version": "==5.2" + }, + "regex": { + "hashes": [ + "sha256:15454b37c5a278f46f7aa2d9339bda450c300617ca2fca6558d05d870245edc7", + "sha256:1ad40708c255943a227e778b022c6497c129ad614bb7a2a2f916e12e8a359ee7", + "sha256:5e00f65cc507d13ab4dfa92c1232d004fa202c1d43a32a13940ab8a5afe2fb96", + "sha256:604dc563a02a74d70ae1f55208ddc9bfb6d9f470f6d1a5054c4bd5ae58744ab1", + "sha256:720e34a539a76a1fedcebe4397290604cc2bdf6f81eca44adb9fb2ea071c0c69", + "sha256:7caf47e4a9ac6ef08cabd3442cc4ca3386db141fb3c8b2a7e202d0470028e910", + "sha256:7faf534c1841c09d8fefa60ccde7b9903c9b528853ecf41628689793290ca143", + "sha256:b4e0406d822aa4993ac45072a584d57aa4931cf8288b5455bbf30c1d59dbad59", + "sha256:c31eaf28c6fe75ea329add0022efeed249e37861c19681960f99bbc7db981fb2", + "sha256:c7393597191fc2043c744db021643549061e12abe0b3ff5c429d806de7b93b66", + "sha256:d2b302f8cdd82c8f48e9de749d1d17f85ce9a0f082880b9a4859f66b07037dc6", + "sha256:e3d8dd0ec0ea280cf89026b0898971f5750a7bd92cb62c51af5a52abd020054a", + "sha256:ec032cbfed59bd5a4b8eab943c310acfaaa81394e14f44454ad5c9eba4f24a74" + ], + "version": "==2019.11.1" + }, + "six": { + "hashes": [ + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + ], + "version": "==1.13.0" + }, + "smmap2": { + "hashes": [ + "sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde", + "sha256:29a9ffa0497e7f2be94ca0ed1ca1aa3cd4cf25a1f6b4f5f87f74b46ed91d609a" + ], + "version": "==2.0.5" + }, + "snowballstemmer": { + "hashes": [ + "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0", + "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52" + ], + "version": "==2.0.0" + }, + "stevedore": { + "hashes": [ + "sha256:01d9f4beecf0fbd070ddb18e5efb10567801ba7ef3ddab0074f54e3cd4e91730", + "sha256:e0739f9739a681c7a1fda76a102b65295e96a144ccdb552f2ae03c5f0abe8a14" + ], + "version": "==1.31.0" + }, + "testfixtures": { + "hashes": [ + "sha256:8f22100d4fb841b958f64e71c8820a32dc46f57d4d7e077777b932acd87b7327", + "sha256:9334f64d4210b734d04abff516d6ddaab7328306a0c4c1268ce4624df51c4f6d" + ], + "version": "==6.10.3" + }, + "toml": { + "hashes": [ + "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + ], + "version": "==0.10.0" + }, + "traitlets": { + "hashes": [ + "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44", + "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7" + ], + "version": "==4.3.3" + }, + "typed-ast": { + "hashes": [ + "sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161", + "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", + "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", + "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", + "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", + "sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47", + "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", + "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", + "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", + "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", + "sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2", + "sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e", + "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", + "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", + "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", + "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", + "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", + "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", + "sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66", + "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" + ], + "markers": "implementation_name == 'cpython' and python_version < '3.8'", + "version": "==1.4.0" + }, + "typing-extensions": { + "hashes": [ + "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2", + "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d", + "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575" + ], + "version": "==3.7.4.1" + }, + "wcwidth": { + "hashes": [ + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + ], + "version": "==0.1.7" + }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" + }, + "zipp": { + "hashes": [ + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" + ], + "version": "==0.6.0" + } + } +} diff --git a/gfauto/README.md b/gfauto/README.md new file mode 100644 index 000000000..b54810c22 --- /dev/null +++ b/gfauto/README.md @@ -0,0 +1,210 @@ +# GraphicsFuzz auto + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Build Status](https://paulthomson.visualstudio.com/gfauto/_apis/build/status/google.graphicsfuzz?branchName=master)](https://paulthomson.visualstudio.com/gfauto/_build/latest?definitionId=2&branchName=master) + + +## GraphicsFuzz auto is a set of Python scripts for running GraphicsFuzz + +[GraphicsFuzz](https://github.com/google/graphicsfuzz) provides tools that automatically find and simplify bugs in graphics shader compilers. +GraphicsFuzz auto (this project) provides scripts for running these tools with minimal interaction. + +## Development setup + +> Optional: if you have just done `git pull` to get a more recent version of GraphicsFuzz auto, consider deleting `.venv/` to start from a fresh virtual environment. This is rarely needed. + +> On Windows, you can use the Git Bash shell, or adapt the commands (including those inside `dev_shell.sh.template`) to the Windows command prompt. + +Execute `./dev_shell.sh.template`. If the default settings don't work, make a copy of the file called `dev_shell.sh` and modify according to the comments before executing. `pip` must be installed for the version of Python you wish to use. Note that you can do e.g. `export PYTHON=python3.6.8` to set your preferred Python binary. We currently target Python 3.6. + +> Pip for Python 3.6 may be broken on certain Debian distributions. +> You can just use the newer Python 3.7+ version provided by your +> distribution. +> See "Installing Python" below if you want to use Python 3.6. + +The script generates and activates a Python virtual environment (located at `.venv/`) with all dependencies installed. + +* Execute `./check_all.sh` to run various presubmit checks, linters, etc. +* Execute `./fix_all.sh` to automatically fix certain issues, such as formatting. + + +### PyCharm + +Use PyCharm to open the top-level `gfauto` directory. +It should pick up the Python virtual environment (at `.venv/`) automatically +for both the code +and when you open a `Terminal` or `Python Console` tab. + +Install and configure plugins: + +* Protobuf Support +* File Watchers (may already be installed) + * The watcher task should already be under version control. +* Mypy: the built-in PyCharm type checking uses Mypy behind-the-scenes, but this plugin enhances it by using the latest version and allowing the use of stricter settings, matching the settings used by the `./check_all.sh` script. + +Add `whitelist.dic` as a custom dictionary (search for "Spelling" in Actions). Do not add words via PyCharm's "Quick Fixes" feature, as the word will only be added to your personal dictionary. Instead, manually add the word to `whitelist.dic`. + +## Imports + +We use the `black` Python code formatter and `isort` for sorting imports. + +We also use the following import style, which is not automatically checked: use the package name (e.g. `gfauto`) and only import modules, not functions. For example: + + +```python +# Good: importing a module +from gfauto import binaries_util + +binaries_util.add_common_tags_from_platform_suffix(...) # using a function +b = binaries_util.BinaryManager() # using a class +d = binaries_util.DEFAULT_BINARIES # using a variable + + +# Bad: directly importing a function, class, variable +from gfauto.binaries_util import add_common_tags_from_platform_suffix, BinaryManager, DEFAULT_BINARIES + +add_common_tags_from_platform_suffix(...) +b = BinaryManager() +d = DEFAULT_BINARIES + + +# Bad: importing from "." +from . import binaries_util + + +# Bad: import from "." AND importing a function +from .binaries_util import add_common_tags_from_platform_suffix + + +# OK: importing "check" and "log" functions directly +from gfauto.gflogging import log +from gfauto.util import check + +log("Running") +check(1 + 1 == 2, AssertionError("1 + 1 should be 2")) + + +# OK: importing types for type annotations +from typing import Dict, List, Optional, Union + +def prepend_catchsegv_if_available(cmd: List[str]) -> List[str]: + ... + + +# OK: importing Path +from pathlib import Path + +def tool_path(tool: str) -> Path: + return Path(tool) + + +# OK: importing generated protobuf types +from gfauto.settings_pb2 import Settings + +DEFAULT_SETTINGS = Settings() +``` + +## Symlinking other scripts + +GraphicsFuzz auto moves fast and so it is useful to add symlinks to other repositories that contain Python scripts that depend on GraphicsFuzz auto. This allows you to search for all references before changing a function. A `temp/` directory exists for this purpose. For example: + +```sh +cd temp +ln -s /path/to/shader-generation shader-generation +``` + +Now any scripts in the `shader-generation` repository are visible in PyCharm. + +You can execute scripts in this repository by opening a Terminal in PyCharm. + +## Terminal + +The `Terminal` tab in PyCharm is useful and will use the project's Python virtual environment. In any other terminal, use: + +* `source .venv/bin/activate` (on Linux) +* `source .venv/Scripts/activate` (on Windows with the Git Bash shell) +* `.venv/Scripts/activate.bat` (on Windows with cmd) + +You can alternatively execute the `./dev_shell.sh` script, but this is fairly slow as it checks and reinstalls all dependencies + +## Fuzzing + +To start fuzzing, create and change to a directory outside the `gfauto/` directory. E.g. `/data/temp/gfauto_fuzzing/2019_06_24`. From here, create `references/` and `donors/` directories containing GLSL shader jobs as used by GraphicsFuzz. +You can get some samples from the GraphicsFuzz project. + +```sh +mkdir references/ donors/ +cp /data/graphicsfuzz_zip/samples/310es/* references/ +cp /data/graphicsfuzz_zip/samples/310es/* donors/ +``` + +Now run the fuzzer. + +```sh +gfauto_fuzz +``` + +It will fail because there is no settings file, but a default `settings.json` file will be created for you. +Review this file. +All plugged Android devices should have been added to the list of devices. +The `active_device_names` list should be modified to include only the unique devices that you care about: + +* Including multiple, identical devices (with the same GPU and drivers) is not recommended, as it will currently result in redundant testing. +* `host_preprocessor` should always be included first, as this virtual device detects failures in tools such as `glslangValidator` and `spirv-opt`, and will ensure such failures will not be logged for real devices. +* You can use `--settings SETTINGS.json` to use a settings file other than `settings.json` (the default). In this way, you can run multiple instances of `gfauto_fuzz` in parallel to test *different* devices. + +You can generate a space-separated list of seeds as follows: + +```python +import secrets + +" ".join([str(secrets.randbits(256)) for i in range(0, 1000)]) +``` + +Assuming you saved those to `../seeds.txt`, you can run parallel instances of `gfauto_fuzz` using: + +```sh +parallel -j 32 gfauto_fuzz --iteration_seed -- $(cat ../seeds.txt) +``` + +This is probably only suitable for testing the `host_preprocessor` and `swift_shader` virtual devices; running parallel tests on actual hardware is likely to give unreliable results. + +You can run parallel instances of gfauto (just for increased throughput, not with fixed seeds) using: + +```sh +parallel -j 32 -i gfauto_fuzz -- $(seq 100) +``` + +# Installing Python + +To manually install Python on your Linux distribution, you can use `pyenv`. + +https://github.com/pyenv/pyenv#basic-github-checkout + +In summary: + +* Install the required packages recommended [here](https://github.com/pyenv/pyenv/wiki/Common-build-problems). + +* Then: + +```sh +git clone https://github.com/pyenv/pyenv.git ~/.pyenv + +# Add the following two lines to your ~/.bashrc file. +export PYENV_ROOT="$HOME/.pyenv" +export PATH="$PYENV_ROOT/bin:$PATH" + +# In a new terminal: +eval "$(pyenv init -)" +pyenv install 3.6.9 +pyenv global 3.6.9 + +# Now execute the development shell script, as usual. +export PYTHON="python" +./dev_shell.sh.template +``` + +You can reactivate the development shell later, +as explained above, using +`source .venv/bin/activate`, +without having to re-execute the above `pyenv` commands. diff --git a/gfauto/azure-pipelines.yml b/gfauto/azure-pipelines.yml new file mode 100644 index 000000000..b19f52479 --- /dev/null +++ b/gfauto/azure-pipelines.yml @@ -0,0 +1,44 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +trigger: +- gfauto + +pr: +- dev +- master + +pool: + vmImage: 'ubuntu-latest' +strategy: + matrix: + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + +- script: | + cd gfauto + export PYTHON=python + export SKIP_SHELL=1 + ./dev_shell.sh.template + source .venv/bin/activate + ./check_all.sh + displayName: 'Install dependencies' diff --git a/gfauto/check_all.sh b/gfauto/check_all.sh new file mode 100755 index 000000000..0255d73a1 --- /dev/null +++ b/gfauto/check_all.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e +set -u + +if [ -z ${VIRTUAL_ENV+x} ]; then + source .venv/bin/activate +fi + +mypy --strict --show-absolute-path gfauto gfautotests +pylint gfauto gfautotests +# Flake checks formatting via black. +flake8 . +pytest gfautotests diff --git a/gfauto/dev_shell.sh.template b/gfauto/dev_shell.sh.template new file mode 100755 index 000000000..2374e0b93 --- /dev/null +++ b/gfauto/dev_shell.sh.template @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e +set -u + +# Check for some known files for sanity. +test -f ./Pipfile +test -f ./dev_shell.sh.template + +# Sets PYTHON to python3.6, unless already defined. +# Modify if needed; this should be a Python 3.6 binary. +# E.g. PYTHON=python3.6 +# Or, do `export PYTHON=python3.6` before executing this script. +PYTHON=${PYTHON-python3.6} + +# Upgrade/install pip and pipenv if needed. +"${PYTHON}" -m pip install --upgrade --user --no-warn-script-location 'pip>=19.2.3' 'pipenv>=2018.11.26' + +# Place the virtual environment at `gfauto/.venv`. +export PIPENV_VENV_IN_PROJECT=1 + +# Use the hard-coded versions of packages in Pipfile.lock. +export PIPENV_IGNORE_PIPFILE=1 + +# Install project dependencies, including development dependencies, into the +# virtual environment using pipenv. +"${PYTHON}" -m pipenv install --dev + +if [ -z ${SKIP_SHELL+x} ]; then + # Enter the virtual environment, unless SKIP_SHELL is defined. + # `python` should now point to the correct version of Python. + "${PYTHON}" -m pipenv shell +fi diff --git a/gfauto/docs/coverage.md b/gfauto/docs/coverage.md new file mode 100644 index 000000000..fe3c8a872 --- /dev/null +++ b/gfauto/docs/coverage.md @@ -0,0 +1,104 @@ +# Coverage + +```sh +# Make sure gfauto_* is available. +# E.g. +# source /path/to/gfauto/.venv/activate +# gfauto_cov_from_gcov -h + +COV_ROOT=$(pwd) + + +### Build SwiftShader ### + +git clone https://swiftshader.googlesource.com/SwiftShader +cd SwiftShader + +mkdir -p out/build +cd out/build + +BUILD_DIR=$(pwd) + +export CFLAGS=--coverage +export CXXFLAGS=--coverage +export LDFLAGS=--coverage + +cmake -G Ninja ../.. -DCMAKE_BUILD_TYPE=Debug +cmake --build . --config Debug + +unset CFLAGS +unset CXXFLAGS +unset LDFLAGS + +cd $COV_ROOT + +export VK_ICD_FILENAMES=$BUILD_DIR/Linux/vk_swiftshader_icd.json + + +### Build dEQP ### +DEQP_USERNAME=paulthomson +git clone ssh://$DEQP_USERNAME@gerrit.khronos.org:29418/vk-gl-cts && scp -p -P 29418 $DEQP_USERNAME@gerrit.khronos.org:hooks/commit-msg vk-gl-cts/.git/hooks/ + +cd vk-gl-cts +python external/fetch_sources.py +cd .. + +mkdir vk-gl-cts-build +cd vk-gl-cts-build +cmake -G Ninja ../vk-gl-cts -DCMAKE_BUILD_TYPE=Release DCMAKE_C_FLAGS=-m64 -DCMAKE_CXX_FLAGS=-m64 +cmake --build . --config Release --target deqp-vk + +cd $COV_ROOT + + +### Run gfauto ### + +export GCOV_PREFIX=$COV_ROOT/prefix_gfauto/PROC_ID + +mkdir gfauto_run +cd gfauto_run + +# Run gfauto. See ../README.md. +# In settings.json, set `active_devices` to `host`. +# Of course, `host` will actually be your built version of SwiftShader +# because of VK_ICD_FILENAMES. +# In settings.json, set `reduce_*` to false. E.g. "reduce_bad_images": false. +# Once `gfauto_fuzz` seems to work, you can try running e.g. 32 instances in parallel: + +parallel -j 32 -i gfauto_fuzz -- $(seq 100) +# Output coverage files are in: $COV_ROOT/prefix_gfauto/*/ + +cd $COV_ROOT + + +### Run dEQP ### + +export GCOV_PREFIX=$COV_ROOT/prefix_deqp/PROC_ID + +# TODO: where do we find rundeqp.go? + +go run rundeqpvk.go -deqp-vk $COV_ROOT/vk-gl-cts-build/external/vulkancts/modules/vulkan/deqp-vk -num-threads 32 + +cd $COV_ROOT + + +### Process coverage info ### + +gfauto_cov_from_gcov -h +# Check help text. +# You may need to add --gcov_uses_json depending on your version of GCC. +# This flag has not been tested yet. + +gfauto_cov_from_gcov --out deqp.cov $BUILD_DIR $COV_ROOT/prefix_deqp/PROC_ID --num_threads 32 + +gfauto_cov_from_gcov --out gfauto.cov $BUILD_DIR $COV_ROOT/prefix_gfauto/PROC_ID --num_threads 32 + +gfauto_cov_new deqp.cov gfauto.cov new.cov + +gfauto_cov_to_source --coverage_out new_cov --zero_coverage_out new_cov_zero --cov new.cov $BUILD_DIR + +# Note: Install meld if needed. + +meld new_cov_zero/ new_cov/ + +``` diff --git a/gfauto/fix_all.sh b/gfauto/fix_all.sh new file mode 100755 index 000000000..681d8d04e --- /dev/null +++ b/gfauto/fix_all.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e +set -u + +if [ -z ${VIRTUAL_ENV+x} ]; then + source .venv/bin/activate +fi + +isort -rc gfauto gfautotests +black gfauto gfautotests diff --git a/gfauto/gfauto/__init__.py b/gfauto/gfauto/__init__.py new file mode 100644 index 000000000..f7bad95d3 --- /dev/null +++ b/gfauto/gfauto/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/gfauto/gfauto/add_amber_tests_to_cts.py b/gfauto/gfauto/add_amber_tests_to_cts.py new file mode 100644 index 000000000..83f845ecb --- /dev/null +++ b/gfauto/gfauto/add_amber_tests_to_cts.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add amber tests to CTS. + +This module/script adds Amber test files to the VK-GL-CTS project. +This file is self-contained so it can be provided alongside Amber test files. +""" + +import argparse +import os +import re +import shutil +import sys +from typing import Pattern, TextIO, cast + +SHORT_DESCRIPTION_LINE_PREFIX = "# Short description: " + +MUST_PASS_PATHS = [ + os.path.join("android", "cts", "master", "vk-master.txt"), + os.path.join( + "external", "vulkancts", "mustpass", "master", "vk-default-no-waivers.txt" + ), + os.path.join("external", "vulkancts", "mustpass", "master", "vk-default.txt"), +] + +TEST_NAME_IN_LINE_PATTERN: Pattern[str] = re.compile(r"\"(.*?)\.amber\"") + + +def check(condition: bool, exception: Exception) -> None: + if not condition: + raise exception + + +def log(message: str = "") -> None: + print(message, flush=True) # noqa: T001 + + +def remove_start(string: str, start: str) -> str: + check( + string.startswith(start), AssertionError("|string| does not start with |start|") + ) + + return string[len(start) :] + + +def open_helper(file: str, mode: str) -> TextIO: # noqa VNE002 + check("b" not in mode, AssertionError(f"|mode|(=={mode}) should not contain 'b'")) + return cast(TextIO, open(file, mode, encoding="utf-8", errors="ignore")) + + +def check_dir_exists(directory: str) -> None: + log("Checking that directory {} exists".format(directory)) + if not os.path.isdir(directory): + raise FileNotFoundError("Directory not found: {}".format(directory)) + + +def check_file_exists(file: str) -> None: # noqa VNE002 + log("Checking that file {} exists".format(file)) + if not os.path.isfile(file): + raise FileNotFoundError("File not found: {}".format(file)) + + +def get_amber_test_file_path(vk_gl_cts: str, amber_test_name: str) -> str: + return os.path.join( + vk_gl_cts, + "external", + "vulkancts", + "data", + "vulkan", + "amber", + "graphicsfuzz", + amber_test_name + ".amber", + ) + + +def get_graphics_fuzz_tests_index_file_path(vk_gl_cts: str) -> str: + return os.path.join( + vk_gl_cts, + "external", + "vulkancts", + "data", + "vulkan", + "amber", + "graphicsfuzz", + "index.txt", + ) + + +def get_amber_test_short_description(amber_test_file_path: str) -> str: + with open_helper(amber_test_file_path, "r") as f: + for line in f: + if line.startswith(SHORT_DESCRIPTION_LINE_PREFIX): + line = remove_start(line, SHORT_DESCRIPTION_LINE_PREFIX) + # Remove \n + line = line[:-1] + return line + return "" + + +def check_and_add_tabs( + line: str, string_name: str, string_value: str, field_index: int, tab_size: int +) -> str: + + # Field index starts at 1. Change it to start at 0. + field_index -= 1 + + check( + len(line.expandtabs(tab_size)) <= field_index, + AssertionError('{} "{}" is too long!'.format(string_name, string_value)), + ) + + while len(line.expandtabs(tab_size)) < field_index: + line += "\t" + + check( + len(line.expandtabs(tab_size)) == field_index, + AssertionError( + "Field index {} is incorrect; Python script needs fixing".format( + field_index + ) + ), + ) + + return line + + +def get_index_line_to_write(amber_test_name: str, short_description: str) -> str: + + # A test line has the following form, except with tabs aligning each field. + # { "name.amber", "name", "description" }, + # | | | | + # 1 2 3 4 + + # 1 + test_file_name_start_index = 5 + # 2 + test_name_start_index = 53 + # 3 + test_description_start_index = 93 + # 4 + test_close_bracket_index = 181 + + tab_size = 4 + + line = "{" + + line = check_and_add_tabs( + line, "internal", "internal", test_file_name_start_index, tab_size + ) + + line += '"{}.amber",'.format(amber_test_name) + + line = check_and_add_tabs( + line, "amber test name", amber_test_name, test_name_start_index, tab_size + ) + + line += '"{}",'.format(amber_test_name) + + line = check_and_add_tabs( + line, "amber test name", amber_test_name, test_description_start_index, tab_size + ) + + line += '"{}"'.format(short_description) + + line = check_and_add_tabs( + line, "short description", short_description, test_close_bracket_index, tab_size + ) + + line += "},\n" + + return line + + +def add_amber_test_to_index_file( + vk_gl_cts: str, amber_test_name: str, input_amber_test_file_path: str +) -> None: + log("Adding Amber test to the index file.") + + short_description = get_amber_test_short_description(input_amber_test_file_path) + + index_file = get_graphics_fuzz_tests_index_file_path(vk_gl_cts) + index_file_bak = index_file + ".bak" + + copyfile(index_file, index_file_bak) + + line_to_write = get_index_line_to_write(amber_test_name, short_description) + + log("Writing from {} to {}.".format(index_file_bak, index_file)) + + with open_helper(index_file_bak, "r") as index_file_in: + with open_helper(index_file, "w") as index_file_out: + + # { "continue-and-merge.amber",... + + # Get to the point where we should insert our line. + line = "" + for line in index_file_in: + if line.startswith("{"): + result = TEST_NAME_IN_LINE_PATTERN.search(line) + if result: + # Group 1 is the test name. + test_name_in_line = result.group(1) + if test_name_in_line >= amber_test_name: + break + index_file_out.write(line) + + # Write our line and then the previously read line. + + # Don't write the line if it already exists; idempotent. + if line != line_to_write: + log("Writing line: {}".format(line_to_write[:-1])) + index_file_out.write(line_to_write) + else: + log("Line already exists.") + index_file_out.write(line) + + # Write the remaining lines. + for line in index_file_in: + index_file_out.write(line) + + remove(index_file_bak) + + +def add_amber_test_to_must_pass(amber_test_name: str, must_pass_file_path: str) -> None: + log("Adding the Amber test to {}".format(must_pass_file_path)) + + must_pass_file_path_bak = must_pass_file_path + ".bak" + copyfile(must_pass_file_path, must_pass_file_path_bak) + + line_to_write = "dEQP-VK.graphicsfuzz.{}\n".format(amber_test_name) + + log("Writing from {} to {}.".format(must_pass_file_path_bak, must_pass_file_path)) + + with open_helper(must_pass_file_path_bak, "r") as pass_in: + with open_helper(must_pass_file_path, "w") as pass_out: + # Get to just before the first GraphicsFuzz test. + line = "" + for line in pass_in: + if line.startswith("dEQP-VK.graphicsfuzz."): + break + pass_out.write(line) + + # |line| contains an unwritten line. + # Get to the point where we need to write line_to_write. + while True: + if (not len) or line >= line_to_write: + break + pass_out.write(line) + line = pass_in.readline() + + # Don't write the line if it already exists; idempotent. + if line != line_to_write: + log("Writing line: {}".format(line_to_write[:-1])) + pass_out.write(line_to_write) + else: + log("Line already exists.") + pass_out.write(line) + + # Write remaining lines. + for line in pass_in: + pass_out.write(line) + + remove(must_pass_file_path_bak) + + +def copyfile(source: str, dest: str) -> None: + log("Copying {} to {}".format(source, dest)) + shutil.copyfile(source, dest) + + +def remove(file: str) -> None: # noqa VNE002 + log("Deleting {}".format(file)) + os.remove(file) + + +def copy_amber_test_file( + vk_gl_cts: str, amber_test_name: str, input_amber_test_file_path: str +) -> None: + log("Copying Amber test file") + + amber_test_file_path = get_amber_test_file_path(vk_gl_cts, amber_test_name) + + check_dir_exists(os.path.dirname(amber_test_file_path)) + + copyfile(input_amber_test_file_path, amber_test_file_path) + + +def add_amber_test(input_amber_test_file_path: str, vk_gl_cts: str) -> None: + log('Adding Amber test "{}" to "{}"'.format(input_amber_test_file_path, vk_gl_cts)) + # E.g. "continue-and-merge" + amber_test_name = os.path.basename(input_amber_test_file_path) + amber_test_name = os.path.splitext(amber_test_name)[0] + + log('Using test name "{}"'.format(amber_test_name)) + + add_amber_test_to_index_file(vk_gl_cts, amber_test_name, input_amber_test_file_path) + + copy_amber_test_file(vk_gl_cts, amber_test_name, input_amber_test_file_path) + + for must_pass_file_path in MUST_PASS_PATHS: + add_amber_test_to_must_pass( + amber_test_name, os.path.join(vk_gl_cts, must_pass_file_path) + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="A script to add Amber tests to the CTS." + ) + + parser.add_argument("vk_gl_cts", help="Path to a checkout of VK-GL-CTS") + + parser.add_argument( + "amber_files", + help="One or more Amber test files (often ending in .amber_script, .amber, .vkscript)", + nargs="+", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + vk_gl_cts = parsed_args.vk_gl_cts + amber_files = parsed_args.amber_files + + check_dir_exists(vk_gl_cts) + check_file_exists(get_graphics_fuzz_tests_index_file_path(vk_gl_cts)) + + for amber_file in amber_files: + add_amber_test(amber_file, vk_gl_cts) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/gfauto/gfauto/amber_converter.py b/gfauto/gfauto/amber_converter.py new file mode 100644 index 000000000..ac6fefbf1 --- /dev/null +++ b/gfauto/gfauto/amber_converter.py @@ -0,0 +1,860 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Amber shader job converter module. + +Converts a SPIR-V assembly shader job (all shaders are already disassembled) to an Amber script file. +""" + +import itertools +import json +import pathlib +import re +from copy import copy +from enum import Enum +from pathlib import Path +from typing import Dict, List, Match, Optional + +import attr + +from gfauto import binaries_util, shader_job_util, subprocess_util, util +from gfauto.gflogging import log +from gfauto.util import check + +AMBER_FENCE_TIMEOUT_MS = 60000 + + +@attr.dataclass +class AmberfySettings: # pylint: disable=too-many-instance-attributes + copyright_header_text: Optional[str] = None + add_generated_comment: bool = False + add_graphics_fuzz_comment: bool = False + short_description: Optional[str] = None + comment_text: Optional[str] = None + use_default_fence_timeout: bool = False + extra_commands: Optional[str] = None + spirv_opt_args: Optional[List[str]] = None + spirv_opt_hash: Optional[str] = None + + def copy(self: "AmberfySettings") -> "AmberfySettings": + # A shallow copy is adequate. + return copy(self) + + +def get_spirv_opt_args_comment( + spirv_opt_args: List[str], spirv_opt_hash: Optional[str] +) -> str: + if not spirv_opt_args: + return "" + result = "# Optimized using spirv-opt with the following arguments:\n" + args = [f"# '{arg}'" for arg in spirv_opt_args] + result += "\n".join(args) + if spirv_opt_hash: + result += f"\n# spirv-opt commit hash: {spirv_opt_hash}" + result += "\n\n" + return result + + +def get_text_as_comment(text: str) -> str: + lines = text.split("\n") + + # Remove empty lines from start and end. + while not lines[0]: + lines.pop(0) + while not lines[-1]: + lines.pop() + + lines = [("# " + line).rstrip() for line in lines] + return "\n".join(lines) + + +def amberscript_comp_buff_def(comp_json: str, make_empty_buffer: bool = False) -> str: + """ + Returns a string (template) containing AmberScript commands for defining the initial in/out buffer. + + Only the "$compute" key is read. + + { + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, + + "$compute": { + "num_groups": [12, 13, 14]; + "buffer": { + "binding": 123, + "fields": + [ + { "type": "int", "data": [ 0 ] }, + { "type": "int", "data": [ 1, 2 ] }, + ] + } + } + + } + + becomes: + + BUFFER {} DATA_TYPE int DATA + 0 1 2 + END + + Or, if |make_empty_buffer| is True: + + BUFFER {} DATA_TYPE int SIZE 3 0 + + + :param comp_json: The shader job JSON as a string. + :param make_empty_buffer: If true, an "empty" buffer is created that is of the same size and type as the normal + in/out buffer; the empty buffer can be used to store the contents of the in/out buffer via the Amber COPY command. + The only difference is the "empty" buffer is initially filled with just one value, which avoids redundantly + listing hundreds of values that will just be overwritten, and makes it clear(er) for those reading the AmberScript + file that the initial state of the buffer is unused. + """ + ssbo_types = { + "int": "int32", + "ivec2": "vec2", + "ivec3": "vec3", + "ivec4": "vec4", + "uint": "uint32", + "float": "float", + "vec2": "vec2", + "vec3": "vec3", + "vec4": "vec4", + } + + comp = json.loads(comp_json) + + check( + "$compute" in comp.keys(), + AssertionError("Cannot find '$compute' key in JSON file"), + ) + + compute_info = comp["$compute"] + + check( + len(compute_info["buffer"]["fields"]) > 0, + AssertionError("Compute shader test with empty SSBO"), + ) + + field_types_set = {field["type"] for field in compute_info["buffer"]["fields"]} + + check(len(field_types_set) == 1, AssertionError("All field types must be the same")) + + ssbo_type = compute_info["buffer"]["fields"][0]["type"] + if ssbo_type not in ssbo_types.keys(): + raise ValueError(f"Unsupported SSBO datum type: {ssbo_type}") + ssbo_type_amber = ssbo_types[ssbo_type] + + # E.g. [[0, 0], [5], [1, 2, 3]] + field_data = [field["data"] for field in compute_info["buffer"]["fields"]] + + # E.g. [0, 0, 5, 1, 2, 3] + # |*| unpacks the list so each element is passed as an argument. + # |chain| takes a list of iterables and concatenates them. + field_data_flattened = itertools.chain(*field_data) + + # E.g. ["0", "0", "5", "1", "2", "3"] + field_data_flattened_str = [str(field) for field in field_data_flattened] + + result = "" + if make_empty_buffer: + # We just use the first value to initialize every element of the "empty" buffer. + result += f"BUFFER {{}} DATA_TYPE {ssbo_type_amber} SIZE {len(field_data_flattened_str)} {field_data_flattened_str[0]}\n" + else: + result += f"BUFFER {{}} DATA_TYPE {ssbo_type_amber} DATA\n" + result += f" {' '.join(field_data_flattened_str)}\n" + result += "END\n" + + return result + + +def amberscript_comp_num_groups_def(json_contents: str) -> str: + shader_job_info = json.loads(json_contents) + num_groups = shader_job_info["$compute"]["num_groups"] + num_groups_str = [str(dimension) for dimension in num_groups] + return " ".join(num_groups_str) + + +def amberscript_uniform_buffer_bind(uniform_json: str, prefix: str) -> str: + """ + Returns AmberScript commands for uniform binding. + + Skips the special '$compute' key, if present. + + { + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, + "$compute": { ... will be ignored ... } + } + + becomes: + + BIND BUFFER {prefix}_myuniform AS uniform DESCRIPTOR_SET 0 BINDING 3 + """ + result = "" + uniforms = json.loads(uniform_json) + for name, entry in uniforms.items(): + if name == "$compute": + continue + result += f" BIND BUFFER {prefix}_{name} AS uniform DESCRIPTOR_SET 0 BINDING {entry['binding']}\n" + return result + + +def amberscript_uniform_buffer_def(uniform_json_contents: str, prefix: str) -> str: + """ + Returns the string representing AmberScript version of uniform definitions. + + Skips the special '$compute' key, if present. + + { + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, + "$compute": { ... will be ignored ... } + } + + becomes: + + # uniforms for {prefix} + + # myuniform + BUFFER {prefix}_myuniform DATA_TYPE float DATA + 42.0 + END + + :param uniform_json_contents: + :param prefix: E.g. "reference" or "variant". The buffer names will include this prefix to avoid name + clashes. + """ + uniform_types: Dict[str, str] = { + "glUniform1i": "int32", + "glUniform2i": "vec2", + "glUniform3i": "vec3", + "glUniform4i": "vec4", + "glUniform1f": "float", + "glUniform2f": "vec2", + "glUniform3f": "vec3", + "glUniform4f": "vec4", + "glUniformMatrix2fv": "mat2x2", + "glUniformMatrix2x3fv": "mat2x3", + "glUniformMatrix2x4fv": "mat2x4", + "glUniformMatrix3x2fv": "mat3x2", + "glUniformMatrix3fv": "mat3x3", + "glUniformMatrix3x4fv": "mat3x4", + "glUniformMatrix4x2fv": "mat4x2", + "glUniformMatrix4x3fv": "mat4x3", + "glUniformMatrix4fv": "mat4x4", + } + + uniforms = json.loads(uniform_json_contents) + + # If there are no uniforms, do not generate anything. + if not uniforms: + return "" + + result = f"# uniforms for {prefix}\n" + + result += "\n" + + for name, entry in uniforms.items(): + + if name == "$compute": + continue + + func = entry["func"] + if func not in uniform_types.keys(): + raise ValueError("Error: unknown uniform type for function: " + func) + uniform_type = uniform_types[func] + + result += f"# {name}\n" + result += f"BUFFER {prefix}_{name} DATA_TYPE {uniform_type} DATA\n" + for arg in entry["args"]: + result += f" {arg}" + result += "\n" + result += "END\n" + + return result + + +def translate_type_for_amber(type_name: str) -> str: + if type_name == "bool": + return "uint" + return type_name + + +def is_compute_job(input_asm_spirv_job_json_path: pathlib.Path) -> bool: + comp_files = shader_job_util.get_related_files( + input_asm_spirv_job_json_path, + [shader_job_util.EXT_COMP], + [shader_job_util.SUFFIX_ASM_SPIRV], + ) + check( + len(comp_files) <= 1, + AssertionError(f"Expected 1 or 0 compute shader files: {comp_files}"), + ) + return len(comp_files) == 1 + + +class ShaderType(Enum): + FRAGMENT = "fragment" + VERTEX = "vertex" + COMPUTE = "compute" + + +@attr.dataclass +class Shader: + shader_type: ShaderType + shader_spirv_asm: Optional[str] + shader_source: Optional[str] + processing_info: str # E.g. "optimized with spirv-opt -O" + + +@attr.dataclass +class ShaderJob: + name_prefix: str # Can be used to create unique ssbo buffer names; uniform names are already unique. + uniform_definitions: str # E.g. BUFFER reference_resolution DATA_TYPE vec2 DATA 256.0 256.0 END ... + uniform_bindings: str # E.g. BIND BUFFER reference_resolution AS uniform DESCRIPTOR_SET 0 BINDING 2 ... + + +@attr.dataclass +class ComputeShaderJob(ShaderJob): + + compute_shader: Shader + + # String containing AmberScript command(s) for defining a buffer containing the initial data for the input/output + # buffer that will be used by the compute shader. + # This string is a template (use with .format()) where the template argument is the name of buffer. + # E.g. BUFFER {} DATA_TYPE vec4 DATA 0.0 0.0 END + initial_buffer_definition_template: str + + # Same as above, but defines an empty buffer with the same size and type as the initial buffer. + # E.g. BUFFER {name} DATA_TYPE vec4 SIZE 2 0.0 + empty_buffer_definition_template: str + + # String specifying the number of groups to run when calling the Amber RUN command. E.g. "7 3 4". + num_groups_def: str + + +@attr.dataclass +class GraphicsShaderJob(ShaderJob): + vertex_shader: Shader + fragment_shader: Shader + + +@attr.dataclass +class ShaderJobFile: + name_prefix: str # Uniform names will be prefixed with this name to ensure they are unique. E.g. "reference". + asm_spirv_shader_job_json: Path + glsl_source_json: Optional[Path] + processing_info: str # E.g. "optimized with spirv-opt -O" + + def to_shader_job(self) -> ShaderJob: + json_contents = util.file_read_text(self.asm_spirv_shader_job_json) + + if is_compute_job(self.asm_spirv_shader_job_json): + glsl_comp_contents = None + if self.glsl_source_json: + glsl_comp_contents = shader_job_util.get_shader_contents( + self.glsl_source_json, shader_job_util.EXT_COMP + ) + comp_asm_contents = shader_job_util.get_shader_contents( + self.asm_spirv_shader_job_json, + shader_job_util.EXT_COMP, + shader_job_util.SUFFIX_ASM_SPIRV, + must_exist=True, + ) + + # Guaranteed + assert comp_asm_contents # noqa + + return ComputeShaderJob( + self.name_prefix, + amberscript_uniform_buffer_def(json_contents, self.name_prefix), + amberscript_uniform_buffer_bind(json_contents, self.name_prefix), + Shader( + ShaderType.COMPUTE, + comp_asm_contents, + glsl_comp_contents, + self.processing_info, + ), + amberscript_comp_buff_def(json_contents), + amberscript_comp_buff_def(json_contents, make_empty_buffer=True), + amberscript_comp_num_groups_def(json_contents), + ) + + # Get GLSL contents + glsl_vert_contents = None + glsl_frag_contents = None + if self.glsl_source_json: + glsl_vert_contents = shader_job_util.get_shader_contents( + self.glsl_source_json, shader_job_util.EXT_VERT + ) + glsl_frag_contents = shader_job_util.get_shader_contents( + self.glsl_source_json, shader_job_util.EXT_FRAG + ) + + # Get spirv asm contents + vert_contents = shader_job_util.get_shader_contents( + self.asm_spirv_shader_job_json, + shader_job_util.EXT_VERT, + shader_job_util.SUFFIX_ASM_SPIRV, + ) + + frag_contents = shader_job_util.get_shader_contents( + self.asm_spirv_shader_job_json, + shader_job_util.EXT_FRAG, + shader_job_util.SUFFIX_ASM_SPIRV, + must_exist=True, + ) + + return GraphicsShaderJob( + self.name_prefix, + amberscript_uniform_buffer_def(json_contents, self.name_prefix), + amberscript_uniform_buffer_bind(json_contents, self.name_prefix), + Shader( + ShaderType.VERTEX, + vert_contents, + glsl_vert_contents, + self.processing_info, + ), + Shader( + ShaderType.FRAGMENT, + frag_contents, + glsl_frag_contents, + self.processing_info, + ), + ) + + +@attr.dataclass +class ShaderJobBasedAmberTest: + reference: Optional[ShaderJob] + variants: List[ShaderJob] + + +@attr.dataclass +class ShaderJobFileBasedAmberTest: + reference_asm_spirv_job: Optional[ShaderJobFile] + variants_asm_spirv_job: List[ShaderJobFile] + + def to_shader_job_based(self) -> ShaderJobBasedAmberTest: + variants = [ + variant_asm_spirv_job.to_shader_job() + for variant_asm_spirv_job in self.variants_asm_spirv_job + ] + return ShaderJobBasedAmberTest( + self.reference_asm_spirv_job.to_shader_job() + if self.reference_asm_spirv_job + else None, + variants, + ) + + +def get_amber_script_header(amberfy_settings: AmberfySettings) -> str: + result = "#!amber\n" + + if amberfy_settings.copyright_header_text: + result += "\n" + result += get_text_as_comment(amberfy_settings.copyright_header_text) + result += "\n\n" + + if amberfy_settings.add_generated_comment: + result += "\n# Generated.\n\n" + + if amberfy_settings.add_graphics_fuzz_comment: + result += "\n# A test for a bug found by GraphicsFuzz.\n" + + if amberfy_settings.short_description: + result += f"\n# Short description: {amberfy_settings.short_description}\n" + + if amberfy_settings.comment_text: + result += f"\n{get_text_as_comment(amberfy_settings.comment_text)}\n" + + if amberfy_settings.spirv_opt_args: + result += "\n" + result += get_spirv_opt_args_comment( + amberfy_settings.spirv_opt_args, amberfy_settings.spirv_opt_hash + ) + result += "\n" + + if not amberfy_settings.use_default_fence_timeout: + result += f"\nSET ENGINE_DATA fence_timeout_ms {AMBER_FENCE_TIMEOUT_MS}\n" + + return result + + +def get_amber_script_shader_def(shader: Shader, name: str) -> str: + result = "" + if shader.shader_source: + result += f"\n# {name} is derived from the following GLSL:\n" + result += get_text_as_comment(shader.shader_source) + if shader.shader_spirv_asm: + result += f"\nSHADER {str(shader.shader_type.value)} {name} SPIRV-ASM\n" + result += shader.shader_spirv_asm + result += "END\n" + else: + result += f"\nSHADER {str(shader.shader_type.value)} {name} PASSTHROUGH\n" + + return result + + +# noinspection DuplicatedCode +def graphics_shader_job_amber_test_to_amber_script( + shader_job_amber_test: ShaderJobBasedAmberTest, amberfy_settings: AmberfySettings +) -> str: + + result = get_amber_script_header(amberfy_settings) + + jobs = shader_job_amber_test.variants.copy() + + if shader_job_amber_test.reference: + assert isinstance(shader_job_amber_test.reference, GraphicsShaderJob) # noqa + jobs.insert(0, shader_job_amber_test.reference) + + for job in jobs: + # Guaranteed, and needed for type checker. + assert isinstance(job, GraphicsShaderJob) # noqa + + prefix = job.name_prefix + + vertex_shader_name = f"{prefix}_vertex_shader" + fragment_shader_name = f"{prefix}_fragment_shader" + + # Define shaders. + + result += get_amber_script_shader_def(job.vertex_shader, vertex_shader_name) + + result += get_amber_script_shader_def(job.fragment_shader, fragment_shader_name) + + # Define uniforms for shader job. + + result += "\n" + result += job.uniform_definitions + + result += f"\nBUFFER {prefix}_framebuffer FORMAT B8G8R8A8_UNORM\n" + + # Create a pipeline. + + result += f"\nPIPELINE graphics {prefix}_pipeline\n" + result += f" ATTACH {vertex_shader_name}\n" + result += f" ATTACH {fragment_shader_name}\n" + result += " FRAMEBUFFER_SIZE 256 256\n" + result += f" BIND BUFFER {prefix}_framebuffer AS color LOCATION 0\n" + result += job.uniform_bindings + result += "END\n" + result += f"CLEAR_COLOR {prefix}_pipeline 0 0 0 255\n" + + # Run the pipeline. + + result += f"\nCLEAR {prefix}_pipeline\n" + result += f"RUN {prefix}_pipeline DRAW_RECT POS 0 0 SIZE 256 256\n" + result += "\n" + + # Add fuzzy compare of framebuffers if there's more than one pipeline. + + for pipeline_index in range(1, len(jobs)): + prefix_0 = jobs[0].name_prefix + prefix_1 = jobs[pipeline_index].name_prefix + result += f"EXPECT {prefix_0}_framebuffer EQ_HISTOGRAM_EMD_BUFFER {prefix_1}_framebuffer TOLERANCE 0.005" + result += "\n" + + if amberfy_settings.extra_commands: + result += amberfy_settings.extra_commands + + return result + + +# noinspection DuplicatedCode +def compute_shader_job_amber_test_to_amber_script( + shader_job_amber_test: ShaderJobBasedAmberTest, amberfy_settings: AmberfySettings +) -> str: + + jobs = shader_job_amber_test.variants.copy() + + if shader_job_amber_test.reference: + assert isinstance(shader_job_amber_test.reference, ComputeShaderJob) # noqa + jobs.insert(0, shader_job_amber_test.reference) + + result = get_amber_script_header(amberfy_settings) + + for job in jobs: + # Guaranteed, and needed for type checker. + assert isinstance(job, ComputeShaderJob) # noqa + + prefix = job.name_prefix + + compute_shader_name = f"{prefix}_compute_shader" + ssbo_name = f"{prefix}_ssbo" + + # Define shaders. + + result += get_amber_script_shader_def(job.compute_shader, compute_shader_name) + + # Define uniforms for variant shader job. + + result += "\n" + result += job.uniform_definitions + + # Define in/out buffer for variant shader job. + # Note that |initial_buffer_definition_template| is a string template that takes the buffer name as an argument. + + result += "\n" + result += job.initial_buffer_definition_template.format(ssbo_name) + + # Create a pipeline that uses the variant compute shader and binds |variant_ssbo_name|. + + result += f"\nPIPELINE compute {prefix}_pipeline\n" + result += f" ATTACH {compute_shader_name}\n" + result += f" BIND BUFFER {ssbo_name} AS storage DESCRIPTOR_SET 0 BINDING 0\n" + result += job.uniform_bindings + result += "END\n" + + # Run the pipeline. + + result += f"\nRUN {prefix}_pipeline {job.num_groups_def}\n\n" + + # Add fuzzy compare of result SSBOs if there's more than one pipeline. + + for pipeline_index in range(1, len(jobs)): + prefix_0 = jobs[0].name_prefix + prefix_1 = jobs[pipeline_index].name_prefix + result += f"EXPECT {prefix_0}_ssbo RMSE_BUFFER {prefix_1}_ssbo TOLERANCE 7\n" + + if amberfy_settings.extra_commands: + result += amberfy_settings.extra_commands + + return result + + +def spirv_asm_shader_job_to_amber_script( + shader_job_file_amber_test: ShaderJobFileBasedAmberTest, + output_amber_script_file_path: Path, + amberfy_settings: AmberfySettings, +) -> Path: + + log( + f"Amberfy: {[str(variant.asm_spirv_shader_job_json) for variant in shader_job_file_amber_test.variants_asm_spirv_job]} " + + ( + f"with reference {str(shader_job_file_amber_test.reference_asm_spirv_job.asm_spirv_shader_job_json)} " + if shader_job_file_amber_test.reference_asm_spirv_job + else "" + ) + + f"to {str(output_amber_script_file_path)}" + ) + + shader_job_amber_test = shader_job_file_amber_test.to_shader_job_based() + + if isinstance(shader_job_amber_test.variants[0], GraphicsShaderJob): + result = graphics_shader_job_amber_test_to_amber_script( + shader_job_amber_test, amberfy_settings + ) + + elif isinstance(shader_job_amber_test.variants[0], ComputeShaderJob): + result = compute_shader_job_amber_test_to_amber_script( + shader_job_amber_test, amberfy_settings + ) + else: + raise AssertionError( + f"Unknown shader job type: {shader_job_amber_test.variants[0]}" + ) + + util.file_write_text(output_amber_script_file_path, result) + return output_amber_script_file_path + + +def write_shader( + shader_asm: str, + amber_file: Path, + output_dir: Path, + shader_type: str, + shader_name: str, + binaries: binaries_util.BinaryManager, +) -> List[Path]: + + files_written: List[Path] = [] + + shader_type_to_suffix = { + "fragment": shader_job_util.EXT_FRAG, + "vertex": shader_job_util.EXT_VERT, + "compute": shader_job_util.EXT_COMP, + } + + shader_type_suffix = shader_type_to_suffix[shader_type] + + # E.g. ifs-and-whiles.variant_fragment_shader.frag.asm + shader_asm_file_path = output_dir / ( + f"{amber_file.stem}.{shader_name}{shader_type_suffix}{shader_job_util.SUFFIX_ASM_SPIRV}" + ) + + # E.g. ifs-and-whiles.variant_fragment_shader.frag.spv + shader_spirv_file_path = output_dir / ( + f"{amber_file.stem}.{shader_name}{shader_type_suffix}{shader_job_util.SUFFIX_SPIRV}" + ) + + # E.g. dEQP-VK.graphicsfuzz.ifs-and-whiles.variant_fragment_shader.spvas + # These files can be added to the llpc repo as a shader test. + shader_llpc_asm_test_file_path = output_dir / ( + f"dEQP-VK.graphicsfuzz.{amber_file.stem}.{shader_name}.spvas" + ) + + util.file_write_text(shader_asm_file_path, shader_asm) + files_written.append(shader_asm_file_path) + + spirv_as_path = binaries.get_binary_path_by_name("spirv-as").path + + subprocess_util.run( + [ + str(spirv_as_path), + "-o", + str(shader_spirv_file_path), + str(shader_asm_file_path), + "--target-env", + "spv1.0", + ], + verbose=True, + ) + + files_written.append(shader_spirv_file_path) + + util.file_write_text( + shader_llpc_asm_test_file_path, + """; BEGIN_SHADERTEST +; RUN: amdllpc -verify-ir -spvgen-dir=%spvgendir% -v %gfxip %s | FileCheck -check-prefix=SHADERTEST %s +; SHADERTEST-LABEL: {{^// LLPC.*}} SPIRV-to-LLVM translation results +; SHADERTEST: AMDLLPC SUCCESS +; END_SHADERTEST +; +""" + + f"; Based on dEQP-VK.graphicsfuzz.{amber_file.stem}\n\n" + + shader_asm, + ) + files_written.append(shader_llpc_asm_test_file_path) + + return files_written + + +def extract_shaders_amber_script( + amber_file: Path, + lines: List[str], + output_dir: Path, + binaries: binaries_util.BinaryManager, +) -> List[Path]: + files_written: List[Path] = [] + i = -1 + while i < len(lines) - 1: + i += 1 + line = lines[i] + if not line.strip().startswith("SHADER"): + continue + parts = line.strip().split() + shader_type = parts[1] + shader_name = parts[2] + shader_language = parts[3] + if shader_language == "PASSTHROUGH": + continue + check( + shader_language == "SPIRV-ASM", + AssertionError( + f"For {str(amber_file)}: unsupported shader language: {shader_language}" + ), + ) + i += 1 + shader_asm = "" + while not lines[i].strip().startswith("END"): + shader_asm += lines[i] + i += 1 + + files_written += write_shader( + shader_asm=shader_asm, + amber_file=amber_file, + output_dir=output_dir, + shader_type=shader_type, + shader_name=shader_name, + binaries=binaries, + ) + + return files_written + + +# E.g. [compute shader spirv] +VK_SCRIPT_SHADER_REGEX = re.compile(r"\[(compute|fragment|vertex) shader (\w*)\]") + + +def extract_shaders_vkscript( + amber_file: Path, + lines: List[str], + output_dir: Path, + binaries: binaries_util.BinaryManager, +) -> List[Path]: + files_written: List[Path] = [] + i = -1 + while i < len(lines) - 1: + i += 1 + line = lines[i] + match: Optional[Match[str]] = re.match(VK_SCRIPT_SHADER_REGEX, line.strip()) + if not match: + continue + shader_type = match.group(1) + shader_language = match.group(2) + if shader_language == "passthrough": + continue + check( + shader_language == "spirv", + AssertionError( + f"For {str(amber_file)}: unsupported shader language: {shader_language}" + ), + ) + i += 1 + shader_asm = "" + while not lines[i].strip().startswith("["): + shader_asm += lines[i] + i += 1 + files_written += write_shader( + shader_asm=shader_asm, + amber_file=amber_file, + output_dir=output_dir, + shader_type=shader_type, + shader_name="shader", + binaries=binaries, + ) + return files_written + + +def extract_shaders( + amber_file: Path, output_dir: Path, binaries: binaries_util.BinaryManager +) -> List[Path]: + files_written: List[Path] = [] + with util.file_open_text(amber_file, "r") as file_handle: + lines = file_handle.readlines() + if lines[0].startswith("#!amber"): + files_written += extract_shaders_amber_script( + amber_file, lines, output_dir, binaries + ) + else: + log(f"Skipping VkScript file {str(amber_file)} for now.") + files_written += extract_shaders_vkscript( + amber_file, lines, output_dir, binaries + ) + + return files_written diff --git a/gfauto/gfauto/android_device.py b/gfauto/gfauto/android_device.py new file mode 100644 index 000000000..db45a5767 --- /dev/null +++ b/gfauto/gfauto/android_device.py @@ -0,0 +1,411 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Android devices. + +Provides functions for interacting with Android devices. +""" + +import os +import shutil +import subprocess +import time +from pathlib import Path +from typing import List, Optional + +from gfauto import ( + devices_util, + fuzz, + gflogging, + result_util, + subprocess_util, + types, + util, +) +from gfauto.device_pb2 import Device, DeviceAndroid +from gfauto.gflogging import log +from gfauto.util import check, file_open_text, file_write_text + +ANDROID_DEVICE_DIR = "/data/local/tmp" +ANDROID_AMBER_NDK = "amber_ndk" +ANDROID_DEVICE_AMBER = ANDROID_DEVICE_DIR + "/" + ANDROID_AMBER_NDK + +ANDROID_DEVICE_GRAPHICSFUZZ_DIR = ANDROID_DEVICE_DIR + "/graphicsfuzz" +ANDROID_DEVICE_RESULT_DIR = ANDROID_DEVICE_GRAPHICSFUZZ_DIR + "/result" +ANDROID_DEVICE_AMBER_SCRIPT_FILE = ANDROID_DEVICE_GRAPHICSFUZZ_DIR + "/test.amber" + +BUSY_WAIT_SLEEP_SLOW = 1.0 + +WAIT_AFTER_BOOT_ANIMATION = 10 + +WAIT_AFTER_BOOT_AND_UNLOCK = 20 + +ADB_DEFAULT_TIME_LIMIT = 30 + +ADB_SHORT_LOGCAT_TIME_LIMIT = 3 + + +def adb_path() -> Path: + if "ANDROID_HOME" in os.environ: + platform_tools_path = Path(os.environ["ANDROID_HOME"]) / "platform-tools" + adb = shutil.which("adb", path=str(platform_tools_path)) + if adb: + return Path(adb) + return util.tool_on_path("adb") + + +def adb_helper( + serial: Optional[str], + adb_args: List[str], + check_exit_code: bool, + verbose: bool = False, + timeout: Optional[int] = ADB_DEFAULT_TIME_LIMIT, +) -> types.CompletedProcess: + + adb_cmd = [str(adb_path())] + if serial: + adb_cmd.append("-s") + adb_cmd.append(serial) + + adb_cmd.extend(adb_args) + + return subprocess_util.run( + adb_cmd, check_exit_code=check_exit_code, timeout=timeout, verbose=verbose + ) + + +def adb_check( + serial: Optional[str], + adb_args: List[str], + verbose: bool = False, + timeout: Optional[int] = ADB_DEFAULT_TIME_LIMIT, +) -> types.CompletedProcess: + + return adb_helper( + serial, adb_args, check_exit_code=True, verbose=verbose, timeout=timeout + ) + + +def adb_can_fail( + serial: Optional[str], + adb_args: List[str], + verbose: bool = False, + timeout: Optional[int] = ADB_DEFAULT_TIME_LIMIT, +) -> types.CompletedProcess: + + return adb_helper( + serial, adb_args, check_exit_code=False, verbose=verbose, timeout=timeout + ) + + +def stay_awake_warning(serial: Optional[str] = None) -> None: + try: + res = adb_check( + serial, ["shell", "settings get global stay_on_while_plugged_in"] + ) + if str(res.stdout).strip() == "0": + log('\nWARNING: please enable "Stay Awake" from developer settings\n') + except subprocess.CalledProcessError: + log( + "Failed to check Stay Awake setting. This can happen if the device has just booted." + ) + + +def is_screen_off_or_locked(serial: Optional[str] = None) -> bool: + """:return: True: the screen is off or locked. False: unknown.""" + res = adb_can_fail(serial, ["shell", "dumpsys nfc"]) + if res.returncode != 0: + log("Failed to run dumpsys.") + return False + + stdout = str(res.stdout) + # You will often find "mScreenState=OFF_LOCKED", but this catches OFF too, which is good. + if "mScreenState=OFF" in stdout: + return True + if "mScreenState=ON_LOCKED" in stdout: + return True + + return False + + +def get_device_driver_details(serial: str) -> str: + prepare_device(wait_for_screen=True, serial=serial) + cmd = [ + "shell", + # The following is a single string: + f"cd {ANDROID_DEVICE_RESULT_DIR} && " + # -d disables Vulkan validation layers. + f"{ANDROID_DEVICE_AMBER} -d -V", + ] + try: + result = adb_check(serial, cmd, verbose=True) + + except subprocess.SubprocessError as ex: + raise devices_util.GetDeviceDetailsError() from ex + + match = devices_util.AMBER_DEVICE_DETAILS_PATTERN.search(result.stdout) + + if not match: + raise devices_util.GetDeviceDetailsError( + "Could not find device details in stdout: " + result.stdout + ) + + return match.group(1) + + +def get_all_android_devices() -> List[Device]: + result: List[Device] = [] + + log("Getting the list of connected Android devices via adb\n") + + adb_devices = adb_check(None, ["devices", "-l"], verbose=True) + stdout: str = adb_devices.stdout + lines: List[str] = stdout.splitlines() + # Remove empty lines. + lines = [l for l in lines if l] + check( + lines[0].startswith("List of devices"), + AssertionError("Could find list of devices from 'adb devices'"), + ) + for i in range(1, len(lines)): + fields = lines[i].split() + device_serial = fields[0] + device_state = fields[1] + if device_state != "device": + log( + f'Skipping adb device with serial {device_serial} as its state "{device_state}" is not "device".' + ) + # Set a simple model name, but then try to find the actual model name. + device_model = "android_device" + for field_index in range(2, len(fields)): + if fields[field_index].startswith("model:"): + device_model = util.remove_start(fields[field_index], "model:") + break + + build_fingerprint: str = "" + try: + adb_fingerprint_result = adb_check( + device_serial, ["adb", "shell", "getprop ro.build.fingerprint"] + ) + build_fingerprint = adb_fingerprint_result.stdout + build_fingerprint = build_fingerprint.strip() + except subprocess.CalledProcessError: + log("Failed to get device fingerprint") + + driver_details = "" + try: + driver_details = get_device_driver_details(device_serial) + except devices_util.GetDeviceDetailsError as ex: + log(f"WARNING: Failed to get device driver details: {ex}") + + device = Device( + name=f"{device_model}_{device_serial}", + android=DeviceAndroid( + serial=device_serial, + model=device_model, + build_fingerprint=build_fingerprint, + ), + device_properties=driver_details, + ) + log(f"Found Android device:\n{str(device)}") + result.append(device) + + return result + + +def prepare_device(wait_for_screen: bool, serial: Optional[str] = None) -> None: + device_was_booting_or_locked = False + + res = adb_can_fail(serial, ["get-state"]) + if res.returncode != 0 or res.stdout.strip() != "device": + device_was_booting_or_locked = True + + adb_check(serial, ["wait-for-device"], timeout=None) + + # Conservatively check that booting has finished. + while True: + + log("Checking if boot animation has finished.") + + res_bootanim = adb_can_fail(serial, ["shell", "getprop init.svc.bootanim"]) + res_bootanim_exit = adb_can_fail( + serial, ["shell", "getprop service.bootanim.exit"] + ) + if res_bootanim.returncode != 0 and res_bootanim_exit != 0: + # Both commands failed so there is no point in trying to use either result. + log("Could not check boot animation; continuing.") + break + if ( + res_bootanim.stdout.strip() != "running" + and res_bootanim_exit.stdout.strip() != "0" + ): + # Both commands suggest the boot animation is NOT running. + # This may include one or both commands returning nothing because the property doesn't exist on this device. + log("Boot animation is not running.") + break + + # If either command suggests the boot animation is running, we assume it is accurate, so we wait. + device_was_booting_or_locked = True + time.sleep(BUSY_WAIT_SLEEP_SLOW) + + if device_was_booting_or_locked: + log( + f"Device appeared to be booting previously, so waiting a further {WAIT_AFTER_BOOT_ANIMATION} seconds." + ) + time.sleep(WAIT_AFTER_BOOT_ANIMATION) + + if wait_for_screen: + stay_awake_warning(serial) + # We cannot reliably know if the screen is on, but this function definitely knows if it is + # off or locked. So we wait here while we definitely know there is an issue. + count = 0 + while is_screen_off_or_locked(serial): + log( + "\nWARNING: The screen appears to be off or locked. Please unlock the device and ensure 'Stay Awake' is enabled in developer settings.\n" + ) + device_was_booting_or_locked = True + time.sleep(BUSY_WAIT_SLEEP_SLOW) + count += 1 + if count > 1 and (count % 3) == 0: + log("Pressing the menu key.") + adb_can_fail(serial, ["shell", "input keyevent 82"], verbose=True) + + if device_was_booting_or_locked: + log( + f"Device appeared to be booting previously, so waiting a further {WAIT_AFTER_BOOT_AND_UNLOCK} seconds." + ) + time.sleep(WAIT_AFTER_BOOT_AND_UNLOCK) + + log("Clearing logcat.") + adb_check(serial, ["logcat", "-c"]) + + res = adb_can_fail(serial, ["shell", "test -e " + ANDROID_DEVICE_AMBER]) + check( + res.returncode == 0, + AssertionError("Failed to find amber on device at: " + ANDROID_DEVICE_AMBER), + ) + + adb_check( + serial, + [ + "shell", + # One string: + # Remove graphicsfuzz directory. + f"rm -rf {ANDROID_DEVICE_GRAPHICSFUZZ_DIR} && " + # Make result directory. + f"mkdir -p {ANDROID_DEVICE_RESULT_DIR}", + ], + ) + + +def run_amber_on_device( + amber_script_file: Path, + output_dir: Path, + dump_image: bool, + dump_buffer: bool, + skip_render: bool = False, + serial: Optional[str] = None, +) -> Path: + + with file_open_text(result_util.get_amber_log_path(output_dir), "w") as log_file: + try: + gflogging.push_stream_for_logging(log_file) + + run_amber_on_device_helper( + amber_script_file, + output_dir, + dump_image, + dump_buffer, + skip_render, + serial, + ) + finally: + gflogging.pop_stream_for_logging() + + return output_dir + + +def run_amber_on_device_helper( + amber_script_file: Path, + output_dir: Path, + dump_image: bool, + dump_buffer: bool, + skip_render: bool = False, + serial: Optional[str] = None, +) -> Path: + + prepare_device(wait_for_screen=True, serial=serial) + + adb_check( + serial, ["push", str(amber_script_file), ANDROID_DEVICE_AMBER_SCRIPT_FILE] + ) + + amber_flags = "--log-graphics-calls-time --disable-spirv-val" + if skip_render: + # -ps tells amber to stop after pipeline creation + amber_flags += " -ps" + else: + if dump_image: + amber_flags += f" -I variant_framebuffer -i {fuzz.VARIANT_IMAGE_FILE_NAME} -I reference_framebuffer -i {fuzz.REFERENCE_IMAGE_FILE_NAME}" + if dump_buffer: + amber_flags += f" -b {fuzz.BUFFER_FILE_NAME} -B 0" + + cmd = [ + "shell", + # The following is a single string: + f"cd {ANDROID_DEVICE_RESULT_DIR} && " + # -d disables Vulkan validation layers. + f"{ANDROID_DEVICE_AMBER} -d {ANDROID_DEVICE_AMBER_SCRIPT_FILE} {amber_flags}", + ] + + status = "UNEXPECTED_ERROR" + + result: Optional[types.CompletedProcess] = None + + try: + result = adb_can_fail( + serial, cmd, verbose=True, timeout=fuzz.AMBER_RUN_TIME_LIMIT + ) + except subprocess.TimeoutExpired: + status = fuzz.STATUS_TIMEOUT + + try: + if result: + if result.returncode != 0: + status = fuzz.STATUS_CRASH + else: + status = fuzz.STATUS_SUCCESS + + if not skip_render: + adb_check( + serial, + # The /. syntax means the contents of the results directory will be copied into output_dir. + ["pull", ANDROID_DEVICE_RESULT_DIR + "/.", str(output_dir)], + ) + + # Grab log. Use a short time limit to increase the chance of detecting a device reboot. + adb_check( + serial, ["logcat", "-d"], verbose=True, timeout=ADB_SHORT_LOGCAT_TIME_LIMIT + ) + except subprocess.SubprocessError: + # If we fail in getting the results directory or log, assume the device has rebooted. + status = fuzz.STATUS_UNRESPONSIVE + + log("\nSTATUS " + status + "\n") + + file_write_text(result_util.get_status_path(output_dir), status) + + return output_dir diff --git a/gfauto/gfauto/artifact.proto b/gfauto/gfauto/artifact.proto new file mode 100644 index 000000000..a76fcfffe --- /dev/null +++ b/gfauto/gfauto/artifact.proto @@ -0,0 +1,54 @@ +// Copyright 2019 The GraphicsFuzz Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package gfauto; + +import "gfauto/common.proto"; + + +// An artifact is a directory containing an ArtifactMetadata proto stored in "artifact.json" and/or a Recipe proto +// stored in "recipe.json", plus any other files and directories within. The recipe can be executed to produce the files +// that make up the artifact, which must include the "artifact.json" file. An artifact is referred to via an +// "artifact path" that starts from a directory containing a file named "ROOT" (the contents is not used), written +// as "//", followed by the path to the directory, using "/" as the directory separator. E.g. +// +// //binaries/my_binary +// +// Artifacts were planned to be used more frequently, but are now mainly used just for downloading and extracting +// binaries. +// See recipe.proto. +message ArtifactMetadata { + + // The artifact's type-specific data. + Data data = 1; + + // A list of artifact paths from which this artifact is derived. Typically, this is similar to + // following the input artifacts in the recipe transitively, but some artifacts may not have a + // recipe because they were created manually so it is still useful to have this field. + repeated string derived_from = 2; + + string comment = 3; + + message Data { + oneof data { + ArtifactMetadataExtractedArchiveSet extracted_archive_set = 1; + } + } +} + +message ArtifactMetadataExtractedArchiveSet { + ArchiveSet archive_set = 1; +} diff --git a/gfauto/gfauto/artifact_pb2.py b/gfauto/gfauto/artifact_pb2.py new file mode 100644 index 000000000..5b8b9a396 --- /dev/null +++ b/gfauto/gfauto/artifact_pb2.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gfauto/artifact.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from gfauto import common_pb2 as gfauto_dot_common__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='gfauto/artifact.proto', + package='gfauto', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x15gfauto/artifact.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\"\xc4\x01\n\x10\x41rtifactMetadata\x12+\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32\x1d.gfauto.ArtifactMetadata.Data\x12\x14\n\x0c\x64\x65rived_from\x18\x02 \x03(\t\x12\x0f\n\x07\x63omment\x18\x03 \x01(\t\x1a\\\n\x04\x44\x61ta\x12L\n\x15\x65xtracted_archive_set\x18\x01 \x01(\x0b\x32+.gfauto.ArtifactMetadataExtractedArchiveSetH\x00\x42\x06\n\x04\x64\x61ta\"N\n#ArtifactMetadataExtractedArchiveSet\x12\'\n\x0b\x61rchive_set\x18\x01 \x01(\x0b\x32\x12.gfauto.ArchiveSetb\x06proto3') + , + dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,]) + + + + +_ARTIFACTMETADATA_DATA = _descriptor.Descriptor( + name='Data', + full_name='gfauto.ArtifactMetadata.Data', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='extracted_archive_set', full_name='gfauto.ArtifactMetadata.Data.extracted_archive_set', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='data', full_name='gfauto.ArtifactMetadata.Data.data', + index=0, containing_type=None, fields=[]), + ], + serialized_start=159, + serialized_end=251, +) + +_ARTIFACTMETADATA = _descriptor.Descriptor( + name='ArtifactMetadata', + full_name='gfauto.ArtifactMetadata', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='data', full_name='gfauto.ArtifactMetadata.data', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='derived_from', full_name='gfauto.ArtifactMetadata.derived_from', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='comment', full_name='gfauto.ArtifactMetadata.comment', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_ARTIFACTMETADATA_DATA, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=55, + serialized_end=251, +) + + +_ARTIFACTMETADATAEXTRACTEDARCHIVESET = _descriptor.Descriptor( + name='ArtifactMetadataExtractedArchiveSet', + full_name='gfauto.ArtifactMetadataExtractedArchiveSet', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='archive_set', full_name='gfauto.ArtifactMetadataExtractedArchiveSet.archive_set', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=253, + serialized_end=331, +) + +_ARTIFACTMETADATA_DATA.fields_by_name['extracted_archive_set'].message_type = _ARTIFACTMETADATAEXTRACTEDARCHIVESET +_ARTIFACTMETADATA_DATA.containing_type = _ARTIFACTMETADATA +_ARTIFACTMETADATA_DATA.oneofs_by_name['data'].fields.append( + _ARTIFACTMETADATA_DATA.fields_by_name['extracted_archive_set']) +_ARTIFACTMETADATA_DATA.fields_by_name['extracted_archive_set'].containing_oneof = _ARTIFACTMETADATA_DATA.oneofs_by_name['data'] +_ARTIFACTMETADATA.fields_by_name['data'].message_type = _ARTIFACTMETADATA_DATA +_ARTIFACTMETADATAEXTRACTEDARCHIVESET.fields_by_name['archive_set'].message_type = gfauto_dot_common__pb2._ARCHIVESET +DESCRIPTOR.message_types_by_name['ArtifactMetadata'] = _ARTIFACTMETADATA +DESCRIPTOR.message_types_by_name['ArtifactMetadataExtractedArchiveSet'] = _ARTIFACTMETADATAEXTRACTEDARCHIVESET +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +ArtifactMetadata = _reflection.GeneratedProtocolMessageType('ArtifactMetadata', (_message.Message,), { + + 'Data' : _reflection.GeneratedProtocolMessageType('Data', (_message.Message,), { + 'DESCRIPTOR' : _ARTIFACTMETADATA_DATA, + '__module__' : 'gfauto.artifact_pb2' + # @@protoc_insertion_point(class_scope:gfauto.ArtifactMetadata.Data) + }) + , + 'DESCRIPTOR' : _ARTIFACTMETADATA, + '__module__' : 'gfauto.artifact_pb2' + # @@protoc_insertion_point(class_scope:gfauto.ArtifactMetadata) + }) +_sym_db.RegisterMessage(ArtifactMetadata) +_sym_db.RegisterMessage(ArtifactMetadata.Data) + +ArtifactMetadataExtractedArchiveSet = _reflection.GeneratedProtocolMessageType('ArtifactMetadataExtractedArchiveSet', (_message.Message,), { + 'DESCRIPTOR' : _ARTIFACTMETADATAEXTRACTEDARCHIVESET, + '__module__' : 'gfauto.artifact_pb2' + # @@protoc_insertion_point(class_scope:gfauto.ArtifactMetadataExtractedArchiveSet) + }) +_sym_db.RegisterMessage(ArtifactMetadataExtractedArchiveSet) + + +# @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/artifact_pb2.pyi b/gfauto/gfauto/artifact_pb2.pyi new file mode 100644 index 000000000..373b6e3b1 --- /dev/null +++ b/gfauto/gfauto/artifact_pb2.pyi @@ -0,0 +1,96 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from gfauto.common_pb2 import ( + ArchiveSet as gfauto___common_pb2___ArchiveSet, +) + +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, +) + +from google.protobuf.internal.containers import ( + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class ArtifactMetadata(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + class Data(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + + @property + def extracted_archive_set(self) -> ArtifactMetadataExtractedArchiveSet: ... + + def __init__(self, + *, + extracted_archive_set : typing___Optional[ArtifactMetadataExtractedArchiveSet] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ArtifactMetadata.Data: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"data",u"extracted_archive_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"data",u"extracted_archive_set"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"data",b"data",u"extracted_archive_set",b"extracted_archive_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"data",b"data",u"extracted_archive_set",b"extracted_archive_set"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"data",b"data"]) -> typing_extensions___Literal["extracted_archive_set"]: ... + + derived_from = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + comment = ... # type: typing___Text + + @property + def data(self) -> ArtifactMetadata.Data: ... + + def __init__(self, + *, + data : typing___Optional[ArtifactMetadata.Data] = None, + derived_from : typing___Optional[typing___Iterable[typing___Text]] = None, + comment : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ArtifactMetadata: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"data"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"comment",u"data",u"derived_from"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"data",b"data"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"comment",b"comment",u"data",b"data",u"derived_from",b"derived_from"]) -> None: ... + +class ArtifactMetadataExtractedArchiveSet(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + + @property + def archive_set(self) -> gfauto___common_pb2___ArchiveSet: ... + + def __init__(self, + *, + archive_set : typing___Optional[gfauto___common_pb2___ArchiveSet] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ArtifactMetadataExtractedArchiveSet: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"archive_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"archive_set"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"archive_set",b"archive_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"archive_set",b"archive_set"]) -> None: ... diff --git a/gfauto/gfauto/artifact_util.py b/gfauto/gfauto/artifact_util.py new file mode 100644 index 000000000..9159811be --- /dev/null +++ b/gfauto/gfauto/artifact_util.py @@ -0,0 +1,320 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Artifacts utility module. + +See artifact.proto for information about artifacts. +""" + +import pathlib +import time +from typing import Dict, List, Optional, Tuple + +from gfauto import gflogging, proto_util, recipe_download_and_extract_archive_set, util +from gfauto.artifact_pb2 import ArtifactMetadata +from gfauto.common_pb2 import ArchiveSet +from gfauto.gflogging import log +from gfauto.recipe_pb2 import Recipe +from gfauto.util import check + +ARTIFACT_METADATA_FILE_NAME = "artifact.json" +ARTIFACT_RECIPE_FILE_NAME = "recipe.json" +ARTIFACT_RECIPE_LOG_FILE_NAME = "recipe.log" +ARTIFACT_ROOT_FILE_NAME = "ROOT" +ARTIFACT_EXECUTING_LOCK_FILE_NAME = "EXECUTING_LOCK" + +BUSY_WAIT_IN_SECONDS = 1 + +RecipeMap = Dict[str, Recipe] + + +class ArtifactWrap: + def __init__(self, path: str, metadata: Optional[ArtifactMetadata] = None): + self.path = path + self.metadata = ( + metadata if metadata is not None else artifact_read_metadata(path) + ) + + +def binary_artifacts_find(artifact_path_prefix: str) -> List[Tuple[ArchiveSet, str]]: + artifact_paths = artifacts_find(artifact_path_prefix) + result: List[Tuple[ArchiveSet, str]] = [] + for artifact_path in artifact_paths: + archive_set = None + if artifact_get_metadata_file_path(artifact_path).exists(): + metadata = artifact_read_metadata(artifact_path) + if metadata.data.HasField("extracted_archive_set"): + archive_set = metadata.data.extracted_archive_set.archive_set + elif artifact_get_recipe_file_path(artifact_path).exists(): + recipe = artifact_read_recipe(artifact_path) + if recipe.HasField("download_and_extract_archive_set"): + archive_set = recipe.download_and_extract_archive_set.archive_set + if archive_set: + result.append((archive_set, artifact_path)) + + return result + + +def artifacts_find(artifact_path_prefix: str) -> List[str]: + path_prefix = artifact_get_directory_path(artifact_path_prefix) + + metadata_files = path_prefix.rglob("*.json") + metadata_files = ( + file + for file in metadata_files + if file.name in (ARTIFACT_METADATA_FILE_NAME, ARTIFACT_RECIPE_FILE_NAME) + ) + artifact_paths = {path_to_artifact_path(file.parent) for file in metadata_files} + return sorted(artifact_paths) + + +def artifact_path_get_root() -> pathlib.Path: + root_file_suffix = pathlib.Path(ARTIFACT_ROOT_FILE_NAME) + fake_file = util.norm_path(pathlib.Path("fake").absolute()) + for parent in fake_file.parents: + if (parent / root_file_suffix).exists(): + return parent + raise FileNotFoundError( + "Could not find root file {}".format(ARTIFACT_ROOT_FILE_NAME) + ) + + +def path_to_artifact_path(path: pathlib.Path) -> str: + from_root = path.relative_to(artifact_path_get_root()).as_posix() + return "//" + from_root + + +def artifact_path_absolute(artifact_path: str) -> str: + """ + Returns the absolute |artifact_path| starting with '//'. + + Artifact paths should almost always begin with '//', but for convenience it can be useful to + use relative paths, especially when calling functions from an IPython shell. + :param artifact_path: An artifact path. + :return: absolute |artifact_path| starting with '//'. + """ + if artifact_path.startswith("//"): + return artifact_path + path = util.norm_path(pathlib.Path(artifact_path)).absolute() + return path_to_artifact_path(path) + + +def artifact_path_to_path(artifact_path: str) -> pathlib.Path: + """ + Returns |artifact_path| converted to an OS specific path. + + Artifact paths always use '/' to separate paths. + Artifact paths usually begin with '//' which is the artifact root directory, marked via a ROOT + file. + + :param artifact_path: an artifact path. + :return: + """ + if artifact_path.startswith("//"): + return util.norm_path( + artifact_path_get_root() / pathlib.Path(artifact_path[2:]) + ) + + return util.norm_path(pathlib.Path(artifact_path)) + + +def artifact_get_directory_path(artifact_path: str = "") -> pathlib.Path: + return artifact_path_to_path(artifact_path) + + +def artifact_get_recipe_file_path(artifact_path: str = "") -> pathlib.Path: + return artifact_get_inner_file_path(ARTIFACT_RECIPE_FILE_NAME, artifact_path) + + +def artifact_get_metadata_file_path(artifact_path: str = "") -> pathlib.Path: + return artifact_get_inner_file_path(ARTIFACT_METADATA_FILE_NAME, artifact_path) + + +def artifact_get_recipe_log_file_path(artifact_path: str) -> pathlib.Path: + return artifact_get_inner_file_path(ARTIFACT_RECIPE_LOG_FILE_NAME, artifact_path) + + +def artifact_write_recipe_and_execute(recipe: Recipe, artifact_path: str = "") -> str: + artifact_path_full = artifact_path_absolute(artifact_path) + + artifact_write_recipe(recipe, artifact_path_full) + artifact_execute_recipe(artifact_path_full) + return artifact_path_full + + +def artifact_write_recipe( + recipe: Optional[Recipe] = None, artifact_path: str = "" +) -> str: + if recipe is None: + recipe = Recipe() + + artifact_path = artifact_path_absolute(artifact_path) + + json_text = proto_util.message_to_json(recipe, including_default_value_fields=True) + json_file_path = artifact_get_recipe_file_path(artifact_path) + util.file_write_text_atomic(json_file_path, json_text) + return artifact_path + + +def artifact_read_recipe(artifact_path: str = "") -> Recipe: + recipe = Recipe() + json_file_path = artifact_get_recipe_file_path(artifact_path) + json_text = util.file_read_text(json_file_path) + proto_util.json_to_message(json_text, recipe) + return recipe + + +def artifact_write_metadata( + artifact_metadata: ArtifactMetadata, artifact_path: str +) -> str: + artifact_path = artifact_path_absolute(artifact_path) + + json_text = proto_util.message_to_json( + artifact_metadata, including_default_value_fields=True + ) + json_file_path = artifact_get_metadata_file_path(artifact_path) + util.file_write_text_atomic(json_file_path, json_text) + return artifact_path + + +def artifact_read_metadata(artifact_path: str = "") -> ArtifactMetadata: + artifact_metadata = ArtifactMetadata() + json_file_path = artifact_get_metadata_file_path(artifact_path) + json_contents = util.file_read_text(json_file_path) + proto_util.json_to_message(json_contents, artifact_metadata) + return artifact_metadata + + +def artifact_execute_recipe_if_needed( + artifact_path: str = "", built_in_recipes: Optional[RecipeMap] = None +) -> None: + artifact_execute_recipe( + artifact_path, + only_if_artifact_json_missing=True, + built_in_recipes=built_in_recipes, + ) + + +def artifact_execute_recipe( # pylint: disable=too-many-branches; + artifact_path: str = "", + only_if_artifact_json_missing: bool = False, + built_in_recipes: Optional[RecipeMap] = None, +) -> None: + artifact_path = artifact_path_absolute(artifact_path) + executing_lock_file_path = artifact_get_inner_file_path( + ARTIFACT_EXECUTING_LOCK_FILE_NAME, artifact_path + ) + + busy_waiting = False + first_wait = True + + # We may have to retry if another process appears to be executing this recipe. + while True: + if busy_waiting: + time.sleep(BUSY_WAIT_IN_SECONDS) + if first_wait: + log( + f"Waiting for {artifact_path} due to lock file {executing_lock_file_path}" + ) + first_wait = False + + if executing_lock_file_path.exists(): + # Retry. + busy_waiting = True + continue + + # Several processes can still execute here concurrently; the above check is just an optimization. + + # The metadata file should be written atomically once the artifact is ready for use, so if it exists, we can + # just return. + if ( + only_if_artifact_json_missing + and artifact_get_metadata_file_path(artifact_path).exists() + ): + return + + # The recipe file should be written atomically. If it exists, we are fine to continue. If not and if we have + # the recipe in |built_in_recipes|, more than one process might write it, but the final rename from TEMP_FILE -> + # RECIPE.json is atomic, so *some* process will succeed and the contents will be valid. Thus, we should be fine + # to continue once we have written the recipe. + if ( + not artifact_get_recipe_file_path(artifact_path).exists() + and built_in_recipes + ): + built_in_recipe = built_in_recipes[artifact_path] + if not built_in_recipe: + raise FileNotFoundError( + str(artifact_get_recipe_file_path(artifact_path)) + ) + # This is atomic; should not fail. + artifact_write_recipe(built_in_recipe, artifact_path) + + recipe = artifact_read_recipe(artifact_path) + + # Create EXECUTING_LOCK file. The "x" means exclusive creation. This will fail if the file already exists; + # i.e. another process won the race and is executing the recipe; if so, we retry from the beginning of this + # function (and will return early). Otherwise, we can continue. We don't need to keep the file open; the file + # is not opened with exclusive access, just created exclusively. + try: + with util.file_open_text(executing_lock_file_path, "x") as lock_file: + lock_file.write("locked") + except FileExistsError: + # Retry. + busy_waiting = True + continue + + # If we fail here (e.g. KeyboardInterrupt), we won't remove the lock file. But any alternative will either have + # the same problem (interrupts can happen at almost any time) or could end up accidentally removing the lock + # file made by another process, so this is the safest approach. Users can manually delete lock files if needed; + # the log output indicates the file on which we are blocked. + + try: + with util.file_open_text( + artifact_get_recipe_log_file_path(artifact_path), "w" + ) as f: + gflogging.push_stream_for_logging(f) + try: + if recipe.HasField("download_and_extract_archive_set"): + recipe_download_and_extract_archive_set.recipe_download_and_extract_archive_set( + recipe.download_and_extract_archive_set, artifact_path + ) + else: + raise NotImplementedError( + "Artifact {} has recipe type {} and this is not implemented".format( + artifact_path, recipe.WhichOneof("recipe") + ) + ) + finally: + gflogging.pop_stream_for_logging() + finally: + # Delete the lock file when we have finished. Ignore errors. + try: + executing_lock_file_path.unlink() + except OSError: + log(f"WARNING: failed to delete: {str(executing_lock_file_path)}") + + +def artifact_get_inner_file_path(inner_file: str, artifact_path: str) -> pathlib.Path: + check( + not inner_file.startswith("//"), + AssertionError( + "bad inner_file argument passed to artifact_get_inner_file_path" + ), + ) + # TODO: Consider absolute paths that we might want to support for quick hacks. + return util.norm_path( + artifact_get_directory_path(artifact_path) / pathlib.Path(inner_file) + ) diff --git a/gfauto/gfauto/binaries_util.py b/gfauto/gfauto/binaries_util.py new file mode 100644 index 000000000..34632b249 --- /dev/null +++ b/gfauto/gfauto/binaries_util.py @@ -0,0 +1,895 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Binary utilities module. + +Defines the latest binaries (name and version) that will be used by default for new fuzzing sessions. +Defines the recipes (see recipe.proto) for all built-in binaries, including old versions of binaries. +Defines BinaryManager; see below. +""" + +import abc +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import attr +import requests + +from gfauto import artifact_util, recipe_wrap, test_util, util +from gfauto.common_pb2 import Archive, ArchiveSet, Binary +from gfauto.gflogging import log +from gfauto.recipe_pb2 import Recipe, RecipeDownloadAndExtractArchiveSet +from gfauto.settings_pb2 import Settings +from gfauto.util import check + +BINARY_RECIPES_PREFIX = "//binaries" + +LATEST_GRAPHICSFUZZ_ARTIFACT = f"{BINARY_RECIPES_PREFIX}/graphicsfuzz_v1.2.1" + +GLSLANG_VALIDATOR_NAME = "glslangValidator" +SPIRV_OPT_NAME = "spirv-opt" +SPIRV_VAL_NAME = "spirv-val" +SPIRV_DIS_NAME = "spirv-dis" +SWIFT_SHADER_NAME = "swift_shader_icd" +AMBER_NAME = "amber" + +SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG = "no-validate-after-all" + +BUILT_IN_BINARY_RECIPES_PATH_PREFIX = f"{BINARY_RECIPES_PREFIX}/built_in" + +CUSTOM_BINARY_RECIPES_PATH_PREFIX = f"{BINARY_RECIPES_PREFIX}/custom" + +PLATFORMS = ["Linux", "Mac", "Windows"] + +PLATFORMS_SET = set(PLATFORMS) + +CONFIGS = ["Release", "Debug"] + +CONFIGS_SET = set(CONFIGS) + +PLATFORM_SUFFIXES_DEBUG = ["Linux_x64_Debug", "Windows_x64_Debug", "Mac_x64_Debug"] +PLATFORM_SUFFIXES_RELEASE = [ + "Linux_x64_Release", + "Windows_x64_Release", + "Mac_x64_Release", +] +PLATFORM_SUFFIXES_RELWITHDEBINFO = [ + "Linux_x64_RelWithDebInfo", + "Windows_x64_RelWithDebInfo", + "Mac_x64_RelWithDebInfo", +] + +DEFAULT_SPIRV_TOOLS_VERSION = "f1e5cd73f658abcc23ee96d78f2dc27c4b7028c1" + +DEFAULT_BINARIES = [ + Binary( + name="glslangValidator", + tags=["Debug"], + version="18d6b6b63e9adc2aa2cce1ce85d1c348f9475118", + ), + Binary(name="spirv-opt", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), + Binary(name="spirv-dis", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), + Binary(name="spirv-as", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), + Binary(name="spirv-val", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), + Binary(name="spirv-fuzz", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), + Binary(name="spirv-reduce", tags=["Debug"], version=DEFAULT_SPIRV_TOOLS_VERSION), + Binary( + name="swift_shader_icd", + tags=["Debug"], + version="aaa64b76c0b40c2958a18cfdc623157c8c6e1b7d", + ), + Binary( + name="amber", tags=["Debug"], version="2bade8f0a3608872962c0e9e451ccdd63c3332f9" + ), + Binary( + name="graphicsfuzz-tool", + tags=[], + version="2584a469d121aa0b1304115a8640cc4e7aedfcf4", + ), + Binary( + name="amdllpc", + tags=["Debug"], + version="06e4d24336a16ed10d305931804d75a4104dce35", + ), +] + + +@attr.dataclass +class BinaryPathAndInfo: + path: Path + binary: Binary + + +class BinaryGetter(abc.ABC): + def get_binary_path_by_name(self, name: str) -> BinaryPathAndInfo: + pass + + +class BinaryNotFound(Exception): + pass + + +class BinaryPathNotFound(Exception): + def __init__(self, binary: Binary): + super().__init__(f"Could not find binary path for binary: \n{binary}") + + +@attr.dataclass +class ToolNameAndPath: + name: str + subpath: str + add_exe_on_windows: bool = True + + +def get_platform_from_platform_suffix(platform_suffix: str) -> str: + platforms = ("Linux", "Mac", "Windows") + for platform in platforms: + if platform in platform_suffix: + return platform + raise AssertionError(f"Could not guess platform of {platform_suffix}") + + +def add_common_tags_from_platform_suffix(tags: List[str], platform_suffix: str) -> None: + platform = get_platform_from_platform_suffix(platform_suffix) + tags.append(platform) + common_tags = ["Release", "Debug", "RelWithDebInfo", "x64"] + for common_tag in common_tags: + if common_tag in platform_suffix: + tags.append(common_tag) + + +def _get_built_in_binary_recipe_from_build_github_repo( + project_name: str, + version_hash: str, + build_version_hash: str, + platform_suffixes: List[str], + tools: List[ToolNameAndPath], +) -> List[recipe_wrap.RecipeWrap]: + + result: List[recipe_wrap.RecipeWrap] = [] + + for platform_suffix in platform_suffixes: + tags: List[str] = [] + add_common_tags_from_platform_suffix(tags, platform_suffix) + binaries = [ + Binary( + name=binary.name, + tags=tags, + path=( + f"{project_name}/{(binary.subpath + '.exe') if 'Windows' in tags and binary.add_exe_on_windows else binary.subpath}" + ), + version=version_hash, + ) + for binary in tools + ] + + result.append( + recipe_wrap.RecipeWrap( + f"{BUILT_IN_BINARY_RECIPES_PATH_PREFIX}/{project_name}_{version_hash}_{platform_suffix}", + Recipe( + download_and_extract_archive_set=RecipeDownloadAndExtractArchiveSet( + archive_set=ArchiveSet( + archives=[ + Archive( + url=f"https://github.com/paulthomson/build-{project_name}/releases/download/github/paulthomson/build-{project_name}/{build_version_hash}/build-{project_name}-{build_version_hash}-{platform_suffix}.zip", + output_file=f"{project_name}.zip", + output_directory=project_name, + ) + ], + binaries=binaries, + ) + ) + ), + ) + ) + + return result + + +def _get_built_in_swift_shader_version( + version_hash: str, build_version_hash: str +) -> List[recipe_wrap.RecipeWrap]: + return _get_built_in_binary_recipe_from_build_github_repo( + project_name="swiftshader", + version_hash=version_hash, + build_version_hash=build_version_hash, + platform_suffixes=PLATFORM_SUFFIXES_RELEASE + + PLATFORM_SUFFIXES_DEBUG + + PLATFORM_SUFFIXES_RELWITHDEBINFO, + tools=[ + ToolNameAndPath( + name="swift_shader_icd", + subpath="lib/vk_swiftshader_icd.json", + add_exe_on_windows=False, + ) + ], + ) + + +def _get_built_in_spirv_tools_version( + version_hash: str, build_version_hash: str, includes_spirv_fuzz: bool = True +) -> List[recipe_wrap.RecipeWrap]: + return _get_built_in_binary_recipe_from_build_github_repo( + project_name="SPIRV-Tools", + version_hash=version_hash, + build_version_hash=build_version_hash, + platform_suffixes=PLATFORM_SUFFIXES_RELEASE + PLATFORM_SUFFIXES_DEBUG, + tools=[ + ToolNameAndPath(name="spirv-as", subpath="bin/spirv-as"), + ToolNameAndPath(name="spirv-dis", subpath="bin/spirv-dis"), + ToolNameAndPath(name="spirv-opt", subpath="bin/spirv-opt"), + ToolNameAndPath(name="spirv-val", subpath="bin/spirv-val"), + ] + + ( + [ToolNameAndPath(name="spirv-fuzz", subpath="bin/spirv-fuzz")] + if includes_spirv_fuzz + else [] + ), + ) + + +def _get_built_in_glslang_version( + version_hash: str, build_version_hash: str +) -> List[recipe_wrap.RecipeWrap]: + return _get_built_in_binary_recipe_from_build_github_repo( + project_name="glslang", + version_hash=version_hash, + build_version_hash=build_version_hash, + platform_suffixes=PLATFORM_SUFFIXES_RELEASE + PLATFORM_SUFFIXES_DEBUG, + tools=[ + ToolNameAndPath(name="glslangValidator", subpath="bin/glslangValidator") + ], + ) + + +def get_graphics_fuzz_121() -> List[recipe_wrap.RecipeWrap]: + return [ + recipe_wrap.RecipeWrap( + f"{BUILT_IN_BINARY_RECIPES_PATH_PREFIX}/graphicsfuzz_v1.2.1", + Recipe( + download_and_extract_archive_set=RecipeDownloadAndExtractArchiveSet( + archive_set=ArchiveSet( + archives=[ + Archive( + url="https://github.com/google/graphicsfuzz/releases/download/v1.2.1/graphicsfuzz.zip", + output_file="graphicsfuzz.zip", + output_directory="graphicsfuzz", + ) + ], + binaries=[ + # + # glslangValidator + Binary( + name="glslangValidator", + tags=["Linux", "x64", "Release"], + path="graphicsfuzz/bin/Linux/glslangValidator", + version="40c16ec0b3ad03fc170f1369a58e7bbe662d82cd", + ), + Binary( + name="glslangValidator", + tags=["Windows", "x64", "Release"], + path="graphicsfuzz/bin/Windows/glslangValidator.exe", + version="40c16ec0b3ad03fc170f1369a58e7bbe662d82cd", + ), + Binary( + name="glslangValidator", + tags=["Mac", "x64", "Release"], + path="graphicsfuzz/bin/Mac/glslangValidator", + version="40c16ec0b3ad03fc170f1369a58e7bbe662d82cd", + ), + # + # spirv-opt + Binary( + name="spirv-opt", + tags=[ + "Linux", + "x64", + "Release", + SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG, + ], + path="graphicsfuzz/bin/Linux/spirv-opt", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-opt", + tags=[ + "Windows", + "x64", + "Release", + SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG, + ], + path="graphicsfuzz/bin/Windows/spirv-opt.exe", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-opt", + tags=[ + "Mac", + "x64", + "Release", + SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG, + ], + path="graphicsfuzz/bin/Mac/spirv-opt", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + # + # spirv-dis + Binary( + name="spirv-dis", + tags=["Linux", "x64", "Release"], + path="graphicsfuzz/bin/Linux/spirv-dis", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-dis", + tags=["Windows", "x64", "Release"], + path="graphicsfuzz/bin/Windows/spirv-dis.exe", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-dis", + tags=["Mac", "x64", "Release"], + path="graphicsfuzz/bin/Mac/spirv-dis", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + # + # spirv-as + Binary( + name="spirv-as", + tags=["Linux", "x64", "Release"], + path="graphicsfuzz/bin/Linux/spirv-as", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-as", + tags=["Windows", "x64", "Release"], + path="graphicsfuzz/bin/Windows/spirv-as.exe", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-as", + tags=["Mac", "x64", "Release"], + path="graphicsfuzz/bin/Mac/spirv-as", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + # + # spirv-val + Binary( + name="spirv-val", + tags=["Linux", "x64", "Release"], + path="graphicsfuzz/bin/Linux/spirv-val", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-val", + tags=["Windows", "x64", "Release"], + path="graphicsfuzz/bin/Windows/spirv-val.exe", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + Binary( + name="spirv-val", + tags=["Mac", "x64", "Release"], + path="graphicsfuzz/bin/Mac/spirv-val", + version="a2ef7be242bcacaa9127a3ce011602ec54b2c9ed", + ), + ], + ) + ) + ), + ) + ] + + +BUILT_IN_BINARY_RECIPES: List[recipe_wrap.RecipeWrap] = ( + _get_built_in_spirv_tools_version( + version_hash="4a00a80c40484a6f6f72f48c9d34943cf8f180d4", + build_version_hash="422f2fe0f0f32494fa687a12ba343d24863b330a", + includes_spirv_fuzz=False, + ) + + _get_built_in_glslang_version( + version_hash="9866ad9195cec8f266f16191fb4ec2ce4896e5c0", + build_version_hash="1586e566f4949b1957e7c32454cbf27e501ed632", + ) + + _get_built_in_swift_shader_version( + version_hash="a0b3a02601da8c48012a4259d335be04d00818da", + build_version_hash="08fb8d429272ef8eedb4d610943b9fe59d336dc6", + ) + + get_graphics_fuzz_121() + + _get_built_in_spirv_tools_version( + version_hash="1c1e749f0b51603032ed573acb5ee4cd6fee8d01", + build_version_hash="7663d620a7fbdccb330d2baec138d0e3e096457c", + includes_spirv_fuzz=False, + ) + + _get_built_in_spirv_tools_version( + version_hash="55adf4cf707bb12c29fc12f784ebeaa29a819e9b", + build_version_hash="f2170cc791d0eaa5789ec7528862ae00b984b3b8", + includes_spirv_fuzz=False, + ) + + _get_built_in_glslang_version( + version_hash="e383c5f55defdb884a77820483d3360617391d78", + build_version_hash="f3df04d4f582af6b54989d7da86f58f8f38423ba", + ) + + _get_built_in_spirv_tools_version( + version_hash="230c9e437146e48ec58adb4433890403c23c98fa", + build_version_hash="288b0f57443e221df530b705085df59f2da93843", + includes_spirv_fuzz=False, + ) + + _get_built_in_spirv_tools_version( + version_hash="76b75c40a1e27939957e6a598292e9f32b4e98d4", + build_version_hash="9debf645007ef2807ba68f4497d50638c4c57878", + includes_spirv_fuzz=False, + ) + + _get_built_in_spirv_tools_version( + version_hash="9559cdbdf011c487f67f89e2d694bd4a18d5c1e0", + build_version_hash="693b9805d162d5a49592912f6b9bb2d0b4868ec8", + ) + + _get_built_in_glslang_version( + version_hash="f04f1f93a70f4608ffa9903b20bfb95f20a063f5", + build_version_hash="211afd921a2b354ee579cd4b60f761bfe27c1003", + ) + + _get_built_in_swift_shader_version( + version_hash="fa0175c0988dd542f008257232207a8b87ad6c63", + build_version_hash="ea3b929604da6873ace48988b8d4651bbcd2e573", + ) + + _get_built_in_swift_shader_version( + version_hash="f25a1c68473b868ce61e97fe5b830c0cdd7e8181", + build_version_hash="ad0a59319c4a3e23db2688c593a1e0459a99340d", + ) + + _get_built_in_spirv_tools_version( + version_hash="06407250a169c6a03b3765e86619075af1a8c187", + build_version_hash="04b2b8e2543b4643c533b20ca1a9d88c72fea370", + ) + + _get_built_in_glslang_version( + version_hash="fe0b2bd694bb07004a2db859c5714c321c26b751", + build_version_hash="0f167ce7125795df62ae5893f553e5608c9652f4", + ) + + _get_built_in_spirv_tools_version( + version_hash="ad7f2c5c4c7f51360e9e079109a9217aa5ba5cc0", + build_version_hash="b97215064186d731eac68adcc5ade4c7b96b265b", + ) + + _get_built_in_spirv_tools_version( + version_hash="6b072126595dd8c2448eb1fda616251c5e6d7079", + build_version_hash="74886e02e26453ee1dcba4290157e9c8a5e8d07e", + ) + + _get_built_in_swift_shader_version( + version_hash="b6fa949c45397bd1fbfda769a104b9e8884f343e", + build_version_hash="70e8d53b94227fed094975771d96f240f7d00911", + ) +) + +BUILT_IN_BINARY_RECIPES_MAP: Dict[str, Recipe] = { + recipe_wrap.path: recipe_wrap.recipe for recipe_wrap in BUILT_IN_BINARY_RECIPES +} + + +def get_platform_from_binary(binary: Binary) -> str: + tags = list(binary.tags) + platforms = [p for p in tags if p in PLATFORMS_SET] + if platforms: + check( + len(platforms) == 1, AssertionError(f"More than one platform in: {binary}") + ) + platform = platforms[0] + else: + platform = util.get_platform() + return platform + + +def get_config_from_binary(binary: Binary) -> str: + tags = list(binary.tags) + configs = [c for c in tags if c in CONFIGS_SET] + if not configs: + raise AssertionError(f"Could not find a config in tags: {tags}") + check(len(configs) == 1, AssertionError(f"More than one config in: {binary}")) + config = configs[0] + return config + + +def binary_name_to_project_name(binary_name: str) -> str: + if binary_name == "glslangValidator": + project_name = "glslang" + elif binary_name in ( + "spirv-opt", + "spirv-as", + "spirv-dis", + "spirv-val", + "spirv-fuzz", + "spirv-reduce", + ): + project_name = "SPIRV-Tools" + elif binary_name == "swift_shader_icd": + project_name = "swiftshader" + elif binary_name == "amber": + project_name = "amber" + elif binary_name == "graphicsfuzz-tool": + project_name = "graphicsfuzz" + elif binary_name == "amdllpc": + project_name = "llpc" + else: + raise AssertionError( + f"Could not find {binary_name}. Could not map {binary_name} to a gfbuild- repo." + ) + + return project_name + + +def get_github_release_recipe( # pylint: disable=too-many-branches; + binary: Binary, +) -> recipe_wrap.RecipeWrap: + + project_name = binary_name_to_project_name(binary.name) + + if project_name == "graphicsfuzz": + # Special case: + platform = util.get_platform() + tags = PLATFORMS[:] + repo_name = f"gfbuild-{project_name}" + version = binary.version + artifact_name = f"gfbuild-{project_name}-{version}" + else: + # Normal case: + platform = get_platform_from_binary(binary) + config = get_config_from_binary(binary) + arch = "x64" + + tags = [platform, config, arch] + + repo_name = f"gfbuild-{project_name}" + version = binary.version + artifact_name = f"gfbuild-{project_name}-{version}-{platform}_{arch}_{config}" + + recipe = recipe_wrap.RecipeWrap( + path=f"{BUILT_IN_BINARY_RECIPES_PATH_PREFIX}/{artifact_name}", + recipe=Recipe( + download_and_extract_archive_set=RecipeDownloadAndExtractArchiveSet( + archive_set=ArchiveSet( + archives=[ + Archive( + url=f"https://github.com/google/{repo_name}/releases/download/github/google/{repo_name}/{version}/{artifact_name}.zip", + output_file=f"{project_name}.zip", + output_directory=f"{project_name}", + ) + ], + binaries=[], + ) + ) + ), + ) + + executable_suffix = ".exe" if platform == "Windows" else "" + + if project_name == "glslang": + binaries = [ + Binary( + name="glslangValidator", + tags=tags, + path=f"{project_name}/bin/glslangValidator{executable_suffix}", + version=version, + ) + ] + elif project_name == "SPIRV-Tools": + binaries = [ + Binary( + name="spirv-opt", + tags=tags, + path=f"{project_name}/bin/spirv-opt{executable_suffix}", + version=version, + ), + Binary( + name="spirv-as", + tags=tags, + path=f"{project_name}/bin/spirv-as{executable_suffix}", + version=version, + ), + Binary( + name="spirv-dis", + tags=tags, + path=f"{project_name}/bin/spirv-dis{executable_suffix}", + version=version, + ), + Binary( + name="spirv-val", + tags=tags, + path=f"{project_name}/bin/spirv-val{executable_suffix}", + version=version, + ), + Binary( + name="spirv-fuzz", + tags=tags, + path=f"{project_name}/bin/spirv-fuzz{executable_suffix}", + version=version, + ), + Binary( + name="spirv-reduce", + tags=tags, + path=f"{project_name}/bin/spirv-reduce{executable_suffix}", + version=version, + ), + ] + elif project_name == "swiftshader": + binaries = [ + Binary( + name="swift_shader_icd", + tags=tags, + path=f"{project_name}/lib/vk_swiftshader_icd.json", + version=version, + ) + ] + elif project_name == "amber": + binaries = [ + Binary( + name="amber", + tags=tags, + path=f"{project_name}/bin/amber{executable_suffix}", + version=version, + ) + ] + elif project_name == "graphicsfuzz": + binaries = [ + Binary( + name="graphicsfuzz-tool", + tags=tags, + path=f"{project_name}/python/drivers/graphicsfuzz-tool", + version=version, + ) + ] + elif project_name == "llpc": + if platform != "Linux": + raise AssertionError("amdllpc is only available on Linux") + binaries = [ + Binary( + name="amdllpc", + tags=tags, + path=f"{project_name}/bin/amdllpc{executable_suffix}", + version=version, + ) + ] + else: + raise AssertionError(f"Unknown project name: {project_name}") + + recipe.recipe.download_and_extract_archive_set.archive_set.binaries.extend(binaries) + return recipe + + +class BinaryManager(BinaryGetter): + """ + Implements BinaryGetter. + + An instance of BinaryManager is the main way that code accesses binaries. BinaryManger allows certain tests and/or + devices to override binaries by passing a list of binary versions that take priority, so the correct versions are + always used. Plus, the current platform will be used when deciding which binary to download and return. + + See the Binary proto. + + _binary_list: A list of Binary with name, version, configuration. This is used to map a binary name to a Binary. + _resolved_paths: A map containing: Binary (serialized bytes) -> Path + _binary_artifacts: A list of all available binary artifacts/recipes stored as tuples: (ArchiveSet, artifact_path). + _built_in_binary_recipes: This is needed to pass to artifact_util.artifact_execute_recipe_if_needed() so that the + recipe file can be written on-demand from our in-memory list of built-in recipes. + """ + + _binary_list: List[Binary] + _resolved_paths: Dict[bytes, Path] + _binary_artifacts: List[Tuple[ArchiveSet, str]] + _built_in_binary_recipes: artifact_util.RecipeMap + + def __init__( + self, + binary_list: Optional[List[Binary]] = None, + platform: Optional[str] = None, + built_in_binary_recipes: Optional[Dict[str, Recipe]] = None, + ): + self._binary_list = binary_list or DEFAULT_BINARIES + self._resolved_paths = {} + self._platform = platform or util.get_platform() + self._binary_artifacts = [] + self._built_in_binary_recipes = {} + + self._binary_artifacts.extend( + artifact_util.binary_artifacts_find(BINARY_RECIPES_PREFIX) + ) + + # When changing this constructor, check self.get_child_binary_manager(). + + if built_in_binary_recipes: + self._built_in_binary_recipes = built_in_binary_recipes + # For each recipe, add a tuple (ArchiveSet, artifact_path) to self._binary_artifacts. + for (artifact_path, recipe) in self._built_in_binary_recipes.items(): + check( + recipe.HasField("download_and_extract_archive_set"), + AssertionError(f"Bad built-in recipe: {recipe}"), + ) + archive_set: RecipeDownloadAndExtractArchiveSet = recipe.download_and_extract_archive_set + self._binary_artifacts.append((archive_set.archive_set, artifact_path)) + + @staticmethod + def get_binary_list_from_test_metadata(test_json_path: Path) -> List[Binary]: + test_metadata = test_util.metadata_read_from_path(test_json_path) + result: List[Binary] = [] + if test_metadata.device: + result.extend(test_metadata.device.binaries) + result.extend(test_metadata.binaries) + return result + + def _get_binary_path_from_binary_artifacts(self, binary: Binary) -> Optional[Path]: + binary_tags = set(binary.tags) + binary_tags.add(self._platform) + for (archive_set, artifact_path) in self._binary_artifacts: + for artifact_binary in archive_set.binaries: # type: Binary + if artifact_binary.name != binary.name: + continue + if artifact_binary.version != binary.version: + continue + recipe_binary_tags = set(artifact_binary.tags) + if not binary_tags.issubset(recipe_binary_tags): + continue + artifact_util.artifact_execute_recipe_if_needed( + artifact_path, self._built_in_binary_recipes + ) + result = artifact_util.artifact_get_inner_file_path( + artifact_binary.path, artifact_path + ) + self._resolved_paths[binary.SerializePartialToString()] = result + return result + return None + + def get_binary_path(self, binary: Binary) -> Path: + # Try resolved cache first. + result = self._resolved_paths.get(binary.SerializePartialToString()) + if result: + return result + log(f"Finding path of binary:\n{binary}") + + # Try list (cache) of binary artifacts on disk. + result = self._get_binary_path_from_binary_artifacts(binary) + if result: + return result + + # Try online. + wrapped_recipe = get_github_release_recipe(binary) + # Execute the recipe to download the binaries. + artifact_util.artifact_execute_recipe_if_needed( + wrapped_recipe.path, {wrapped_recipe.path: wrapped_recipe.recipe} + ) + # Add to binary artifacts list (cache). + self._binary_artifacts.append( + ( + wrapped_recipe.recipe.download_and_extract_archive_set.archive_set, + wrapped_recipe.path, + ) + ) + # Now we should be able to find it in the binary artifacts list. + result = self._get_binary_path_from_binary_artifacts(binary) + check( + bool(result), + AssertionError( + f"Could not find:\n{binary} even though we just added it:\n{wrapped_recipe}" + ), + ) + assert result # noqa + return result + + @staticmethod + def get_binary_by_name_from_list(name: str, binary_list: List[Binary]) -> Binary: + for binary in binary_list: + if binary.name == name: + return binary + raise BinaryNotFound( + f"Could not find binary named {name} in list:\n{binary_list}" + ) + + def get_binary_path_by_name(self, name: str) -> BinaryPathAndInfo: + binary = self.get_binary_by_name(name) + return BinaryPathAndInfo(self.get_binary_path(binary), binary) + + def get_binary_by_name(self, name: str) -> Binary: + return self.get_binary_by_name_from_list(name, self._binary_list) + + def get_child_binary_manager( + self, binary_list: List[Binary], prepend: bool = False + ) -> "BinaryManager": + child_binary_list = binary_list + if prepend: + child_binary_list += self._binary_list + result = BinaryManager(child_binary_list, self._platform) + # pylint: disable=protected-access; This is fine since |result| is a BinaryManager. + result._resolved_paths = self._resolved_paths + # pylint: disable=protected-access; This is fine since |result| is a BinaryManager. + result._binary_artifacts = self._binary_artifacts + # pylint: disable=protected-access; This is fine since |result| is a BinaryManager. + result._built_in_binary_recipes = self._built_in_binary_recipes + return result + + +def get_default_binary_manager(settings: Settings) -> BinaryManager: + """ + Gets the default binary manager. + + :param settings: Passing just "Settings()" will use the hardcoded (slightly out-of-date) default binary_list, which + may be fine, especially if you plan to use specific versions anyway by immediately overriding the binary_list using + get_child_binary_manager(). + :return: + """ + return BinaryManager( + binary_list=list(settings.latest_binary_versions) or DEFAULT_BINARIES, + built_in_binary_recipes=BUILT_IN_BINARY_RECIPES_MAP, + ) + + +class DownloadVersionError(Exception): + pass + + +def _download_latest_version_number(project_name: str) -> str: + + url = f"https://api.github.com/repos/google/gfbuild-{project_name}/releases" + log(f"Checking: {url}") + response = requests.get(url) + if not response: + raise DownloadVersionError(f"Failed to find version of {project_name}") + + result = response.json() + + expected_num_assets_map = { + "amber": 17, + "glslang": 15, + "SPIRV-Tools": 15, + "swiftshader": 15, + "graphicsfuzz": 5, + "llpc": 7, + } + + expected_num_assets = expected_num_assets_map[project_name] + + for release in result: + assets = release["assets"] + if len(assets) != expected_num_assets: + log( + f"SKIPPING a release of {project_name} with {len(assets)} assets (expected {expected_num_assets})" + ) + continue + + tag_name: str = release["tag_name"] + last_slash = tag_name.rfind("/") + if last_slash == -1: + raise DownloadVersionError( + f"Failed to find version of {project_name}; tag name: {tag_name}" + ) + version = tag_name[last_slash + 1 :] + log(f"Found {project_name} version {version}") + return version + + raise DownloadVersionError( + f"Failed to find version of {project_name} with {expected_num_assets} assets" + ) + + +def download_latest_binary_version_numbers() -> List[Binary]: + log("Downloading the latest binary version numbers...") + + # Deep copy of DEFAULT_BINARIES. + binaries: List[Binary] = [] + for binary in DEFAULT_BINARIES: + new_binary = Binary() + new_binary.CopyFrom(binary) + binaries.append(new_binary) + + # Update version numbers. + for binary in binaries: + project_name = binary_name_to_project_name(binary.name) + binary.version = _download_latest_version_number(project_name) + + return binaries diff --git a/gfauto/gfauto/common.proto b/gfauto/gfauto/common.proto new file mode 100644 index 000000000..91b97509a --- /dev/null +++ b/gfauto/gfauto/common.proto @@ -0,0 +1,61 @@ +// Copyright 2019 The GraphicsFuzz Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package gfauto; + + +// Defines a binary (or really, any file). +message Binary { + + // E.g. glslangValidator. + string name = 1; + + // This should be a list of strings (tags) containing the platform and other details. + // E.g. "Linux", "x64", "Debug", "malloc", "msan". + // When being used to specify how to *find* a binary, the tags must be found in the binary definition. + // E.g. "Debug", "msan". + // Typically, the current platform will also need to be found (e.g. "Linux") before the Binary can be chosen for use. + repeated string tags = 2; + + // Used when defining a binary within a recipe or artifact. Gives the path (always using "/" as the directory + // separator) relative from the artifact.json/recipe.json file to the binary, after the ArchiveSet has been extracted. + // Should be empty in Settings. + // Also used internally (but never written to disk) to store the resolved path of a binary. + string path = 3; + + // This should typically be a hash of some kind. + string version = 4; +} + +message Archive { + // The URL from which to download the archive. + string url = 1; + + // The output location for the downloaded archive. + string output_file = 2; + + // The directory in which the archive will be extracted. + string output_directory = 3; +} + +message ArchiveSet { + // A list of Archives; all of these will be downloaded and extracted when used in a recipe. + // See RecipeDownloadAndExtractArchiveSet proto. + repeated Archive archives = 1; + + // A list of Binaries that can be found in the artifact. See ArtifactMetadataExtractedArchiveSet proto. + repeated Binary binaries = 2; +} diff --git a/gfauto/gfauto/common_pb2.py b/gfauto/gfauto/common_pb2.py new file mode 100644 index 000000000..d1e59c91d --- /dev/null +++ b/gfauto/gfauto/common_pb2.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gfauto/common.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='gfauto/common.proto', + package='gfauto', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x13gfauto/common.proto\x12\x06gfauto\"C\n\x06\x42inary\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04tags\x18\x02 \x03(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\t\"E\n\x07\x41rchive\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x13\n\x0boutput_file\x18\x02 \x01(\t\x12\x18\n\x10output_directory\x18\x03 \x01(\t\"Q\n\nArchiveSet\x12!\n\x08\x61rchives\x18\x01 \x03(\x0b\x32\x0f.gfauto.Archive\x12 \n\x08\x62inaries\x18\x02 \x03(\x0b\x32\x0e.gfauto.Binaryb\x06proto3') +) + + + + +_BINARY = _descriptor.Descriptor( + name='Binary', + full_name='gfauto.Binary', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='gfauto.Binary.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tags', full_name='gfauto.Binary.tags', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='path', full_name='gfauto.Binary.path', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='version', full_name='gfauto.Binary.version', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=31, + serialized_end=98, +) + + +_ARCHIVE = _descriptor.Descriptor( + name='Archive', + full_name='gfauto.Archive', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='url', full_name='gfauto.Archive.url', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='output_file', full_name='gfauto.Archive.output_file', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='output_directory', full_name='gfauto.Archive.output_directory', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=100, + serialized_end=169, +) + + +_ARCHIVESET = _descriptor.Descriptor( + name='ArchiveSet', + full_name='gfauto.ArchiveSet', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='archives', full_name='gfauto.ArchiveSet.archives', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='binaries', full_name='gfauto.ArchiveSet.binaries', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=171, + serialized_end=252, +) + +_ARCHIVESET.fields_by_name['archives'].message_type = _ARCHIVE +_ARCHIVESET.fields_by_name['binaries'].message_type = _BINARY +DESCRIPTOR.message_types_by_name['Binary'] = _BINARY +DESCRIPTOR.message_types_by_name['Archive'] = _ARCHIVE +DESCRIPTOR.message_types_by_name['ArchiveSet'] = _ARCHIVESET +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Binary = _reflection.GeneratedProtocolMessageType('Binary', (_message.Message,), { + 'DESCRIPTOR' : _BINARY, + '__module__' : 'gfauto.common_pb2' + # @@protoc_insertion_point(class_scope:gfauto.Binary) + }) +_sym_db.RegisterMessage(Binary) + +Archive = _reflection.GeneratedProtocolMessageType('Archive', (_message.Message,), { + 'DESCRIPTOR' : _ARCHIVE, + '__module__' : 'gfauto.common_pb2' + # @@protoc_insertion_point(class_scope:gfauto.Archive) + }) +_sym_db.RegisterMessage(Archive) + +ArchiveSet = _reflection.GeneratedProtocolMessageType('ArchiveSet', (_message.Message,), { + 'DESCRIPTOR' : _ARCHIVESET, + '__module__' : 'gfauto.common_pb2' + # @@protoc_insertion_point(class_scope:gfauto.ArchiveSet) + }) +_sym_db.RegisterMessage(ArchiveSet) + + +# @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/common_pb2.pyi b/gfauto/gfauto/common_pb2.pyi new file mode 100644 index 000000000..cb7667b84 --- /dev/null +++ b/gfauto/gfauto/common_pb2.pyi @@ -0,0 +1,92 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class Binary(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + name = ... # type: typing___Text + tags = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + path = ... # type: typing___Text + version = ... # type: typing___Text + + def __init__(self, + *, + name : typing___Optional[typing___Text] = None, + tags : typing___Optional[typing___Iterable[typing___Text]] = None, + path : typing___Optional[typing___Text] = None, + version : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Binary: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"name",u"path",u"tags",u"version"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"path",b"path",u"tags",b"tags",u"version",b"version"]) -> None: ... + +class Archive(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + url = ... # type: typing___Text + output_file = ... # type: typing___Text + output_directory = ... # type: typing___Text + + def __init__(self, + *, + url : typing___Optional[typing___Text] = None, + output_file : typing___Optional[typing___Text] = None, + output_directory : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Archive: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"output_directory",u"output_file",u"url"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[u"output_directory",b"output_directory",u"output_file",b"output_file",u"url",b"url"]) -> None: ... + +class ArchiveSet(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + + @property + def archives(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[Archive]: ... + + @property + def binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[Binary]: ... + + def __init__(self, + *, + archives : typing___Optional[typing___Iterable[Archive]] = None, + binaries : typing___Optional[typing___Iterable[Binary]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> ArchiveSet: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"archives",u"binaries"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[u"archives",b"archives",u"binaries",b"binaries"]) -> None: ... diff --git a/gfauto/gfauto/cov_from_gcov.py b/gfauto/gfauto/cov_from_gcov.py new file mode 100644 index 000000000..29184553b --- /dev/null +++ b/gfauto/gfauto/cov_from_gcov.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Processes .gcda and .gcno files to get a .cov file. + +Unlike most code in gfauto, we use str instead of pathlib.Path because the increased speed is probably worthwhile. +""" + +import argparse +import os +import pickle +import shutil +import sys + +from gfauto import cov_util + + +def main() -> None: + + parser = argparse.ArgumentParser( + description="Processes .gcda and .gcno files to get a .cov file that contains the line coverage data.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "--gcov_path", type=str, help="Path to gcov", default=shutil.which("gcov") + ) + + parser.add_argument( + "--gcov_uses_json", + help="Pass to indicate that your gcov version is 9+ and so uses the newer JSON intermediate format. " + "This is faster, but you must have gcc 9+.", + action="store_true", + ) + + parser.add_argument( + "--num_threads", + type=int, + help="Thead pool size for running gcov in parallel.", + default=8, + ) + + parser.add_argument( + "--out", type=str, help="The output .cov file", default="output.cov" + ) + + parser.add_argument( + "build_dir", + type=str, + help="The build directory where the compiler was invoked.", + ) + + parser.add_argument( + "gcov_prefix_dir", + type=str, + help="The GCOV_PREFIX directory that was used when running the target application. " + 'If the directory ends with "PROC_ID" then "PROC_ID" will be replaced with each directory that exists ' + 'and the coverage results will be merged. E.g. Given "--gcov_prefix_dir /cov/PROC_ID", the results from ' + "/cov/001 /cov/002 /cov/blah etc. will be computed and the results will be merged.", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + if not parsed_args.gcov_path: + parser.error("Please provide gcov_path") + + gcov_path: str = parsed_args.gcov_path + + build_dir = parsed_args.build_dir + build_dir = os.path.abspath(build_dir) + build_dir = os.path.normpath(build_dir) + + gcov_prefix_dir = parsed_args.gcov_prefix_dir + gcov_prefix_dir = os.path.abspath(gcov_prefix_dir) + gcov_prefix_dir = os.path.normpath(gcov_prefix_dir) + + data = cov_util.GetLineCountsData( + gcov_path=gcov_path, + gcov_uses_json_output=parsed_args.gcov_uses_json, + build_dir=build_dir, + gcov_prefix_dir=gcov_prefix_dir, + num_threads=parsed_args.num_threads, + ) + + output_coverage_path: str = parsed_args.out + + # Special case for "PROC_ID". + if "PROC_ID" in gcov_prefix_dir: + print("Detected PROC_ID in gcov_prefix_dir") + if os.path.exists(gcov_prefix_dir): + raise AssertionError(f"Unexpected file/directory: {gcov_prefix_dir}.") + gcov_prefix_prefix = os.path.dirname(gcov_prefix_dir) + if "PROC_ID" in gcov_prefix_prefix: + raise AssertionError( + f"Can only handle PROC_ID as the last component of the path: {gcov_prefix_dir}" + ) + for proc_dir in os.listdir(gcov_prefix_prefix): + proc_dir = os.path.join(gcov_prefix_prefix, proc_dir) + if not os.path.isdir(proc_dir): + continue + data.gcov_prefix_dir = proc_dir + print(f"Consuming {data.gcov_prefix_dir}") + cov_util.get_line_counts(data) + else: + print(f"Consuming {data.gcov_prefix_dir}") + cov_util.get_line_counts(data) + + with open(output_coverage_path, mode="wb") as f: + pickle.dump(data.line_counts, f, protocol=pickle.HIGHEST_PROTOCOL) + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/cov_merge.py b/gfauto/gfauto/cov_merge.py new file mode 100644 index 000000000..6cc4fe4c1 --- /dev/null +++ b/gfauto/gfauto/cov_merge.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Outputs merged coverage data (merged.cov) given a list of .cov files. + +Unlike most code in gfauto, we use str instead of pathlib.Path because the increased speed is probably worthwhile. +""" + +import argparse +import pickle +import sys +from typing import List + +from gfauto import cov_util + + +def main() -> None: + + parser = argparse.ArgumentParser( + description="Outputs merged coverage data given a list of .cov files.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "--out", type=str, default="merged.cov", help="The output merged coverage file." + ) + parser.add_argument( + "coverage_files", + metavar="coverage_files", + type=str, + nargs="*", + help="The .cov files to merge.", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + output_file: str = parsed_args.out + input_files: List[str] = parsed_args.coverage_files + + output_line_counts: cov_util.LineCounts = {} + + for input_file in input_files: + with open(input_file, mode="rb") as f: + input_line_counts: cov_util.LineCounts = pickle.load(f) + for file_path, line_counts in input_line_counts.items(): + if file_path not in line_counts: + output_line_counts[file_path] = line_counts + else: + output_line_counts[file_path].update(line_counts) + + with open(output_file, mode="wb") as f: + pickle.dump(output_line_counts, f, protocol=pickle.HIGHEST_PROTOCOL) + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/cov_new.py b/gfauto/gfauto/cov_new.py new file mode 100644 index 000000000..75d1ed614 --- /dev/null +++ b/gfauto/gfauto/cov_new.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Outputs the "new coverage" given A.cov (baseline) and B.cov. + +Unlike most code in gfauto, we use str instead of pathlib.Path because the increased speed is probably worthwhile. +""" + +import argparse +import pickle +import sys + +from gfauto import cov_util + + +def main() -> None: + + parser = argparse.ArgumentParser( + description='Outputs the "new coverage" from A.cov (baseline) to B.cov. ' + "The output coverage file will only include the counts from B.cov for newly-covered lines; " + "all other lines will have a count of zero.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument("a_coverage", type=str, help="The baseline A.cov file.") + parser.add_argument("b_coverage", type=str, help="The B.cov file.") + parser.add_argument("output_coverage", type=str, help="The output .cov file.") + + parsed_args = parser.parse_args(sys.argv[1:]) + + a_coverage: str = parsed_args.a_coverage + b_coverage: str = parsed_args.b_coverage + output_coverage: str = parsed_args.output_coverage + + with open(a_coverage, mode="rb") as f: + a_line_counts: cov_util.LineCounts = pickle.load(f) + + with open(b_coverage, mode="rb") as f: + b_line_counts: cov_util.LineCounts = pickle.load(f) + + # We modify b_line_counts so that lines already covered by A are set to 0. + # Note that line counts appear to be able to overflow, so we use "!= 0" instead of "> 0". + for source_file_path, b_counts in b_line_counts.items(): + if source_file_path in a_line_counts: + a_counts = a_line_counts[source_file_path] + for line_number, b_count in b_counts.items(): + if b_count != 0: + # Defaults to 0 if not present. + if a_counts[line_number] != 0: + b_counts[line_number] = 0 + + with open(output_coverage, mode="wb") as f: + pickle.dump(b_line_counts, f, protocol=pickle.HIGHEST_PROTOCOL) + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/cov_to_source.py b/gfauto/gfauto/cov_to_source.py new file mode 100644 index 000000000..579ace126 --- /dev/null +++ b/gfauto/gfauto/cov_to_source.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Takes a .cov file and outputs annotated source files. + +Unlike most code in gfauto, we use str instead of pathlib.Path because the increased speed is probably worthwhile. +""" + +import argparse +import pickle +import sys + +from gfauto import cov_util + + +def main() -> None: + + parser = argparse.ArgumentParser( + description="Takes a .cov file and outputs annotated source files.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "--coverage_out", + type=str, + help="The output directory for source files annotated with line coverage.", + default="", + ) + + parser.add_argument( + "--zero_coverage_out", + type=str, + help="The output directory for source files annotated with line coverage, assuming zero coverage.", + default="", + ) + + parser.add_argument("--cov", type=str, help="The .cov file.", default="output.cov") + + parser.add_argument( + "build_dir", + type=str, + help="The build directory where the compiler was invoked.", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + coverage_out: str = parsed_args.coverage_out + zero_coverage_out: str = parsed_args.zero_coverage_out + coverage_file: str = parsed_args.cov + build_dir: str = parsed_args.build_dir + + with open(coverage_file, mode="rb") as f: + line_counts: cov_util.LineCounts = pickle.load(f) + + if coverage_out: + cov_util.output_source_files(build_dir, coverage_out, line_counts) + + if zero_coverage_out: + cov_util.output_source_files( + build_dir, zero_coverage_out, line_counts, force_zero_coverage=True + ) + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/cov_util.py b/gfauto/gfauto/cov_util.py new file mode 100644 index 000000000..472603d51 --- /dev/null +++ b/gfauto/gfauto/cov_util.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Processes coverage files.""" + +import io +import os +import subprocess +import threading +import typing +from collections import Counter +from queue import Queue +from typing import Dict, List, Tuple + +from attr import dataclass + +# Type: +from gfauto import util + +LineCounts = Dict[str, typing.Counter[int]] + +DirAndItsFiles = Tuple[str, List[str]] + +DirAndItsOutput = Tuple[str, str] + +IGNORED_MISSING_FILES = ["CMakeCXXCompilerId.cpp", "CMakeCCompilerId.c"] + + +@dataclass # pylint: disable=too-many-instance-attributes; +class GetLineCountsData: + gcov_path: str + gcov_uses_json_output: bool + build_dir: str + gcov_prefix_dir: str + num_threads: int + gcda_files_queue: "Queue[DirAndItsFiles]" = Queue() + stdout_queue: "Queue[DirAndItsOutput]" = Queue() + line_counts: LineCounts = {} + + +def _thread_gcov(data: GetLineCountsData) -> None: + # Keep getting files and processing until the special "done" ("", []) message. + + while True: + root, files = data.gcda_files_queue.get() + if not root: + # This is the special "done" message. + break + cmd = [data.gcov_path, "-i"] + if data.gcov_uses_json_output: + cmd.append("-t") + cmd.extend(files) + # I.e.: cd $root && gcov -i -t file_1.gcda file_2.gcda ... + result = subprocess.run( + cmd, + encoding="utf-8", + errors="ignore", + check=True, + cwd=root, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if data.gcov_uses_json_output: + data.stdout_queue.put((root, result.stdout)) + else: + gcov_files = [file + ".gcov" for file in files] + gcov_contents = [] + for gcov_file in gcov_files: + with open( + os.path.join(root, gcov_file), + "r", + encoding="utf-8", + errors="ignore", + ) as f: + gcov_contents.append(f.read()) + gcov_contents_combined = "\n".join(gcov_contents) + data.stdout_queue.put((root, gcov_contents_combined)) + + +def _thread_gcovs(data: GetLineCountsData) -> None: + threads = [ + threading.Thread(target=_thread_gcov, args=(data,)) + for _ in range(data.num_threads) + ] + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + + # send "done" message to adder thread. + data.stdout_queue.put(("", "")) + + +def _process_text_lines(data: GetLineCountsData, lines: typing.TextIO) -> None: + current_file = "" + current_file_line_counts: typing.Counter[int] = Counter() + for line in lines: + line = line.strip() + # file:file_path + if line.startswith("file:"): + current_file = line[5:] + current_file_line_counts = data.line_counts.setdefault( + current_file, Counter() + ) + continue + # lcount:line number,execution_count,has_unexecuted_block + if line.startswith("lcount:"): + assert current_file + line = line[7:] + parts = line.split(sep=",") + line_number = int(parts[0]) + count = int(parts[1]) + # Confusingly, |update| adds counts. + current_file_line_counts.update({line_number: count}) + continue + + +def _process_json_lines(data: GetLineCountsData, lines: typing.TextIO) -> None: + current_file = "" + current_file_line_counts: typing.Counter[int] = Counter() + # The count value given by the previous line, which may or may not be used depending on the next line. + current_count = -1 + for line in lines: + line = line.strip() + # "file": "file_name", + if line.startswith('"file": "'): + assert current_count < 0 + current_file = line[9:-2] + current_file_line_counts = data.line_counts.setdefault( + current_file, Counter() + ) + continue + # "count": count, + if line.startswith('"count": '): + assert current_file + current_count = int(line[9:-1]) + continue + # "line_number": line_number, + if line.startswith('"line_number": '): + assert current_file + assert current_count >= 0 + line_number = int(line[15:-1]) + # Confusingly, |update| adds counts. + current_file_line_counts.update({line_number: current_count}) + # Fallthrough. + + # Reset current_count; the "count" field can occur in a few places, but we only want to use the count that + # is immediately followed by "line_number". + current_count = -1 + + +def _thread_adder(data: GetLineCountsData) -> None: + # Keep processing stdout entries until we get the special "done" message. + + while True: + root, stdout = data.stdout_queue.get() + if not root: + # This is the special "done" message. + break + lines = io.StringIO(stdout) + + if data.gcov_uses_json_output: + _process_json_lines(data, lines) + else: + _process_text_lines(data, lines) + + +def get_line_counts(data: GetLineCountsData) -> None: + + root: str + files: List[str] + + # In gcov_prefix_dir, add symlinks to build_dir .gcno files. + print("Adding symlinks.") + + # If the symlinks already exist, we will get an error. + # os.symlink does not provide an option to overwrite. + # We instead create the symlink with a randomized name and then rename it using os.replace, which atomically + # overwrites any existing file. + random_text = util.get_random_name()[:10] + + for root, _, files in os.walk(data.build_dir): + gcno_files = [f for f in files if f.endswith(".gcno")] + if gcno_files: + root_rel = strip_root(root) + os.makedirs(os.path.join(data.gcov_prefix_dir, root_rel), exist_ok=True) + for file_name in gcno_files: + source = os.path.join(root, file_name) + dest = os.path.join(data.gcov_prefix_dir, root_rel, file_name) + temp = dest + random_text + os.symlink(source, temp) + os.replace(temp, dest) + + print("Done.") + + print("Processing .gcno files.") + + gcovs_thread = threading.Thread(target=_thread_gcovs, args=(data,)) + adder_thread = threading.Thread(target=_thread_adder, args=(data,)) + + gcovs_thread.start() + adder_thread.start() + + for root, _, files in os.walk( + os.path.join(data.gcov_prefix_dir, strip_root(data.build_dir)) + ): + gcno_files = [f for f in files if f.endswith(".gcno")] + # TODO: Could split further if necessary. + if gcno_files: + data.gcda_files_queue.put((os.path.join(root), gcno_files)) + + # Send a "done" message for each thread. + for _ in range(data.num_threads): + data.gcda_files_queue.put(("", [])) + + # wait for threads. + gcovs_thread.join() + adder_thread.join() + + print("Done.") + + +INDENT = 8 + + +def strip_root(path: str) -> str: + path_stripped = path + if os.path.isabs(path_stripped): + # Most of this coverage code only works on Linux, so we assume Linux here. + # If we had Windows paths that could be on different drives etc., we would need to be more careful. + util.check( + path_stripped.startswith("/"), + AssertionError(f"Non-posix absolute file path? {path}"), + ) + path_stripped = path_stripped[1:] + util.check( + not path_stripped.startswith("/"), + AssertionError( + f"Internal error trying to make a relative path: {path_stripped}" + ), + ) + return path_stripped + + +def output_source_files( + build_dir: str, + output_dir: str, + line_counts: LineCounts, + force_zero_coverage: bool = False, +) -> None: + build_dir = os.path.abspath(build_dir) + + for source_path, counts in line_counts.items(): + source_path = os.path.join(build_dir, source_path) + source_path = os.path.normpath(source_path) + + if not os.path.isfile(source_path): + if not os.path.basename(source_path) in IGNORED_MISSING_FILES: + print(f"WARNING: Could not find source file: {source_path}") + continue + + dest_path = os.path.join(output_dir, strip_root(source_path)) + + with open(source_path, "r", encoding="utf-8", errors="ignore") as source_file: + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + with open(dest_path, "w", encoding="utf-8", errors="ignore") as dest_file: + line_number = 1 + while True: + line = source_file.readline() + if not line: + break + # .get() returns None if the line is not executable + line_count = counts.get(line_number) + if line_count: + if force_zero_coverage: + line_count = 0 + line_count_str = str(line_count) + " " + else: + line_count_str = " " + line_count_str = ( + " " * (INDENT - len(line_count_str)) + line_count_str + ) + line = line_count_str + line + dest_file.write(line) + line_number += 1 diff --git a/gfauto/gfauto/device.proto b/gfauto/gfauto/device.proto new file mode 100644 index 000000000..e75e8df71 --- /dev/null +++ b/gfauto/gfauto/device.proto @@ -0,0 +1,72 @@ +// Copyright 2019 The GraphicsFuzz Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package gfauto; + +import "gfauto/common.proto"; + + +// A list of devices. Also includes the |active_device_names|, which are the ones that will be used when fuzzing +// (so this is really more than just a list). +message DeviceList { + repeated string active_device_names = 1; + repeated Device devices = 2; +} + +// A device, such as an Android device or the host machine with its GPU and drivers. +message Device { + string name = 1; + oneof device { + DevicePreprocess preprocess = 2; + DeviceSwiftShader swift_shader = 3; + DeviceHost host = 4; + DeviceAndroid android = 5; + DeviceShaderCompiler shader_compiler = 7; + } + // A device can specify/override some binaries. E.g. you may want several SwiftShader "devices" with different + // versions or configurations of SwiftShader. + repeated Binary binaries = 6; + + // The Vulkan device properties, most likely obtained by running "amber -d -V". + string device_properties = 9; +} + +message DeviceSwiftShader { +} + +// A virtual device that just executes all steps up until the actual graphics hardware or simulator is needed. This +// helps to find bugs in, say, spirv-opt. +message DevicePreprocess { + +} + +// The host machine on which we are running gfauto. +message DeviceHost { + +} + +message DeviceAndroid { + string serial = 1; + string model = 2; + string build_fingerprint = 3; + +} + +// A virtual device that just executes an offline shader compiler. +message DeviceShaderCompiler { + string binary = 1; + repeated string args = 2; +} diff --git a/gfauto/gfauto/device_pb2.py b/gfauto/gfauto/device_pb2.py new file mode 100644 index 000000000..b9db15aca --- /dev/null +++ b/gfauto/gfauto/device_pb2.py @@ -0,0 +1,387 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gfauto/device.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from gfauto import common_pb2 as gfauto_dot_common__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='gfauto/device.proto', + package='gfauto', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x13gfauto/device.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\"J\n\nDeviceList\x12\x1b\n\x13\x61\x63tive_device_names\x18\x01 \x03(\t\x12\x1f\n\x07\x64\x65vices\x18\x02 \x03(\x0b\x32\x0e.gfauto.Device\"\xc7\x02\n\x06\x44\x65vice\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\npreprocess\x18\x02 \x01(\x0b\x32\x18.gfauto.DevicePreprocessH\x00\x12\x31\n\x0cswift_shader\x18\x03 \x01(\x0b\x32\x19.gfauto.DeviceSwiftShaderH\x00\x12\"\n\x04host\x18\x04 \x01(\x0b\x32\x12.gfauto.DeviceHostH\x00\x12(\n\x07\x61ndroid\x18\x05 \x01(\x0b\x32\x15.gfauto.DeviceAndroidH\x00\x12\x37\n\x0fshader_compiler\x18\x07 \x01(\x0b\x32\x1c.gfauto.DeviceShaderCompilerH\x00\x12 \n\x08\x62inaries\x18\x06 \x03(\x0b\x32\x0e.gfauto.Binary\x12\x19\n\x11\x64\x65vice_properties\x18\t \x01(\tB\x08\n\x06\x64\x65vice\"\x13\n\x11\x44\x65viceSwiftShader\"\x12\n\x10\x44\x65vicePreprocess\"\x0c\n\nDeviceHost\"I\n\rDeviceAndroid\x12\x0e\n\x06serial\x18\x01 \x01(\t\x12\r\n\x05model\x18\x02 \x01(\t\x12\x19\n\x11\x62uild_fingerprint\x18\x03 \x01(\t\"4\n\x14\x44\x65viceShaderCompiler\x12\x0e\n\x06\x62inary\x18\x01 \x01(\t\x12\x0c\n\x04\x61rgs\x18\x02 \x03(\tb\x06proto3') + , + dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,]) + + + + +_DEVICELIST = _descriptor.Descriptor( + name='DeviceList', + full_name='gfauto.DeviceList', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='active_device_names', full_name='gfauto.DeviceList.active_device_names', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='devices', full_name='gfauto.DeviceList.devices', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=52, + serialized_end=126, +) + + +_DEVICE = _descriptor.Descriptor( + name='Device', + full_name='gfauto.Device', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='gfauto.Device.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='preprocess', full_name='gfauto.Device.preprocess', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='swift_shader', full_name='gfauto.Device.swift_shader', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='host', full_name='gfauto.Device.host', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='android', full_name='gfauto.Device.android', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='shader_compiler', full_name='gfauto.Device.shader_compiler', index=5, + number=7, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='binaries', full_name='gfauto.Device.binaries', index=6, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='device_properties', full_name='gfauto.Device.device_properties', index=7, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='device', full_name='gfauto.Device.device', + index=0, containing_type=None, fields=[]), + ], + serialized_start=129, + serialized_end=456, +) + + +_DEVICESWIFTSHADER = _descriptor.Descriptor( + name='DeviceSwiftShader', + full_name='gfauto.DeviceSwiftShader', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=458, + serialized_end=477, +) + + +_DEVICEPREPROCESS = _descriptor.Descriptor( + name='DevicePreprocess', + full_name='gfauto.DevicePreprocess', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=479, + serialized_end=497, +) + + +_DEVICEHOST = _descriptor.Descriptor( + name='DeviceHost', + full_name='gfauto.DeviceHost', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=499, + serialized_end=511, +) + + +_DEVICEANDROID = _descriptor.Descriptor( + name='DeviceAndroid', + full_name='gfauto.DeviceAndroid', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='serial', full_name='gfauto.DeviceAndroid.serial', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='model', full_name='gfauto.DeviceAndroid.model', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='build_fingerprint', full_name='gfauto.DeviceAndroid.build_fingerprint', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=513, + serialized_end=586, +) + + +_DEVICESHADERCOMPILER = _descriptor.Descriptor( + name='DeviceShaderCompiler', + full_name='gfauto.DeviceShaderCompiler', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='binary', full_name='gfauto.DeviceShaderCompiler.binary', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='args', full_name='gfauto.DeviceShaderCompiler.args', index=1, + number=2, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=588, + serialized_end=640, +) + +_DEVICELIST.fields_by_name['devices'].message_type = _DEVICE +_DEVICE.fields_by_name['preprocess'].message_type = _DEVICEPREPROCESS +_DEVICE.fields_by_name['swift_shader'].message_type = _DEVICESWIFTSHADER +_DEVICE.fields_by_name['host'].message_type = _DEVICEHOST +_DEVICE.fields_by_name['android'].message_type = _DEVICEANDROID +_DEVICE.fields_by_name['shader_compiler'].message_type = _DEVICESHADERCOMPILER +_DEVICE.fields_by_name['binaries'].message_type = gfauto_dot_common__pb2._BINARY +_DEVICE.oneofs_by_name['device'].fields.append( + _DEVICE.fields_by_name['preprocess']) +_DEVICE.fields_by_name['preprocess'].containing_oneof = _DEVICE.oneofs_by_name['device'] +_DEVICE.oneofs_by_name['device'].fields.append( + _DEVICE.fields_by_name['swift_shader']) +_DEVICE.fields_by_name['swift_shader'].containing_oneof = _DEVICE.oneofs_by_name['device'] +_DEVICE.oneofs_by_name['device'].fields.append( + _DEVICE.fields_by_name['host']) +_DEVICE.fields_by_name['host'].containing_oneof = _DEVICE.oneofs_by_name['device'] +_DEVICE.oneofs_by_name['device'].fields.append( + _DEVICE.fields_by_name['android']) +_DEVICE.fields_by_name['android'].containing_oneof = _DEVICE.oneofs_by_name['device'] +_DEVICE.oneofs_by_name['device'].fields.append( + _DEVICE.fields_by_name['shader_compiler']) +_DEVICE.fields_by_name['shader_compiler'].containing_oneof = _DEVICE.oneofs_by_name['device'] +DESCRIPTOR.message_types_by_name['DeviceList'] = _DEVICELIST +DESCRIPTOR.message_types_by_name['Device'] = _DEVICE +DESCRIPTOR.message_types_by_name['DeviceSwiftShader'] = _DEVICESWIFTSHADER +DESCRIPTOR.message_types_by_name['DevicePreprocess'] = _DEVICEPREPROCESS +DESCRIPTOR.message_types_by_name['DeviceHost'] = _DEVICEHOST +DESCRIPTOR.message_types_by_name['DeviceAndroid'] = _DEVICEANDROID +DESCRIPTOR.message_types_by_name['DeviceShaderCompiler'] = _DEVICESHADERCOMPILER +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +DeviceList = _reflection.GeneratedProtocolMessageType('DeviceList', (_message.Message,), { + 'DESCRIPTOR' : _DEVICELIST, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.DeviceList) + }) +_sym_db.RegisterMessage(DeviceList) + +Device = _reflection.GeneratedProtocolMessageType('Device', (_message.Message,), { + 'DESCRIPTOR' : _DEVICE, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.Device) + }) +_sym_db.RegisterMessage(Device) + +DeviceSwiftShader = _reflection.GeneratedProtocolMessageType('DeviceSwiftShader', (_message.Message,), { + 'DESCRIPTOR' : _DEVICESWIFTSHADER, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.DeviceSwiftShader) + }) +_sym_db.RegisterMessage(DeviceSwiftShader) + +DevicePreprocess = _reflection.GeneratedProtocolMessageType('DevicePreprocess', (_message.Message,), { + 'DESCRIPTOR' : _DEVICEPREPROCESS, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.DevicePreprocess) + }) +_sym_db.RegisterMessage(DevicePreprocess) + +DeviceHost = _reflection.GeneratedProtocolMessageType('DeviceHost', (_message.Message,), { + 'DESCRIPTOR' : _DEVICEHOST, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.DeviceHost) + }) +_sym_db.RegisterMessage(DeviceHost) + +DeviceAndroid = _reflection.GeneratedProtocolMessageType('DeviceAndroid', (_message.Message,), { + 'DESCRIPTOR' : _DEVICEANDROID, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.DeviceAndroid) + }) +_sym_db.RegisterMessage(DeviceAndroid) + +DeviceShaderCompiler = _reflection.GeneratedProtocolMessageType('DeviceShaderCompiler', (_message.Message,), { + 'DESCRIPTOR' : _DEVICESHADERCOMPILER, + '__module__' : 'gfauto.device_pb2' + # @@protoc_insertion_point(class_scope:gfauto.DeviceShaderCompiler) + }) +_sym_db.RegisterMessage(DeviceShaderCompiler) + + +# @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/device_pb2.pyi b/gfauto/gfauto/device_pb2.pyi new file mode 100644 index 000000000..dedd5c86b --- /dev/null +++ b/gfauto/gfauto/device_pb2.pyi @@ -0,0 +1,166 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from gfauto.common_pb2 import ( + Binary as gfauto___common_pb2___Binary, +) + +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class DeviceList(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + active_device_names = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + @property + def devices(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[Device]: ... + + def __init__(self, + *, + active_device_names : typing___Optional[typing___Iterable[typing___Text]] = None, + devices : typing___Optional[typing___Iterable[Device]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DeviceList: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"active_device_names",u"devices"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[u"active_device_names",b"active_device_names",u"devices",b"devices"]) -> None: ... + +class Device(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + name = ... # type: typing___Text + device_properties = ... # type: typing___Text + + @property + def preprocess(self) -> DevicePreprocess: ... + + @property + def swift_shader(self) -> DeviceSwiftShader: ... + + @property + def host(self) -> DeviceHost: ... + + @property + def android(self) -> DeviceAndroid: ... + + @property + def shader_compiler(self) -> DeviceShaderCompiler: ... + + @property + def binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[gfauto___common_pb2___Binary]: ... + + def __init__(self, + *, + name : typing___Optional[typing___Text] = None, + preprocess : typing___Optional[DevicePreprocess] = None, + swift_shader : typing___Optional[DeviceSwiftShader] = None, + host : typing___Optional[DeviceHost] = None, + android : typing___Optional[DeviceAndroid] = None, + shader_compiler : typing___Optional[DeviceShaderCompiler] = None, + binaries : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, + device_properties : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Device: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"android",u"device",u"host",u"preprocess",u"shader_compiler",u"swift_shader"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"android",u"binaries",u"device",u"device_properties",u"host",u"name",u"preprocess",u"shader_compiler",u"swift_shader"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"android",b"android",u"device",b"device",u"host",b"host",u"preprocess",b"preprocess",u"shader_compiler",b"shader_compiler",u"swift_shader",b"swift_shader"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"android",b"android",u"binaries",b"binaries",u"device",b"device",u"device_properties",b"device_properties",u"host",b"host",u"name",b"name",u"preprocess",b"preprocess",u"shader_compiler",b"shader_compiler",u"swift_shader",b"swift_shader"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"device",b"device"]) -> typing_extensions___Literal["preprocess","swift_shader","host","android","shader_compiler"]: ... + +class DeviceSwiftShader(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + + def __init__(self, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DeviceSwiftShader: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + +class DevicePreprocess(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + + def __init__(self, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DevicePreprocess: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + +class DeviceHost(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + + def __init__(self, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DeviceHost: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + +class DeviceAndroid(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + serial = ... # type: typing___Text + model = ... # type: typing___Text + build_fingerprint = ... # type: typing___Text + + def __init__(self, + *, + serial : typing___Optional[typing___Text] = None, + model : typing___Optional[typing___Text] = None, + build_fingerprint : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DeviceAndroid: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"build_fingerprint",u"model",u"serial"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[u"build_fingerprint",b"build_fingerprint",u"model",b"model",u"serial",b"serial"]) -> None: ... + +class DeviceShaderCompiler(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + binary = ... # type: typing___Text + args = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + def __init__(self, + *, + binary : typing___Optional[typing___Text] = None, + args : typing___Optional[typing___Iterable[typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> DeviceShaderCompiler: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"args",u"binary"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[u"args",b"args",u"binary",b"binary"]) -> None: ... diff --git a/gfauto/gfauto/devices_util.py b/gfauto/gfauto/devices_util.py new file mode 100644 index 000000000..9a5b33b6b --- /dev/null +++ b/gfauto/gfauto/devices_util.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Devices utility module. + +Used to enumerate the available devices and work with device lists. +""" +import re +from typing import Dict, List, Optional, Pattern + +from gfauto import android_device, binaries_util, host_device_util +from gfauto.device_pb2 import ( + Device, + DeviceHost, + DeviceList, + DevicePreprocess, + DeviceShaderCompiler, + DeviceSwiftShader, +) +from gfauto.gflogging import log +from gfauto.util import ToolNotOnPathError + +# [\s\S] matches anything, including newlines. +# *? is non-greedy. + +AMBER_DEVICE_DETAILS_PATTERN: Pattern[str] = re.compile( + r"Physical device properties:\n([\s\S]*?)End of physical device properties." +) + + +class GetDeviceDetailsError(Exception): + pass + + +def swift_shader_device(binary_manager: binaries_util.BinaryManager) -> Device: + + amber_path = binary_manager.get_binary_path_by_name(binaries_util.AMBER_NAME).path + swift_shader_binary_and_path = binary_manager.get_binary_path_by_name( + binaries_util.SWIFT_SHADER_NAME + ) + + driver_details = "" + try: + driver_details = host_device_util.get_driver_details( + amber_path, swift_shader_binary_and_path.path + ) + except GetDeviceDetailsError as ex: + log(f"WARNING: Failed to get device driver details: {ex}") + + device = Device( + name="swift_shader", + swift_shader=DeviceSwiftShader(), + binaries=[swift_shader_binary_and_path.binary], + device_properties=driver_details, + ) + + return device + + +def device_preprocessor() -> Device: + # TODO: think about the fact that the versions of glslang, spirv-opt etc. are stored in the test and not the device. + # This might make it rather strange if you want to test many different versions of, say, spirv-opt. + # We could store version hashes in both the test and the "preprocessor device", and let the "preprocessor device" + # have higher priority. + return Device(name="host_preprocessor", preprocess=DevicePreprocess()) + + +def device_host(binary_manager: binaries_util.BinaryManager) -> Device: + amber_path = binary_manager.get_binary_path_by_name(binaries_util.AMBER_NAME).path + + driver_details = "" + try: + driver_details = host_device_util.get_driver_details(amber_path) + except GetDeviceDetailsError as ex: + log(f"WARNING: Failed to get device driver details: {ex}") + + return Device(name="host", host=DeviceHost(), device_properties=driver_details) + + +def get_device_list( + binary_manager: binaries_util.BinaryManager, + device_list: Optional[DeviceList] = None, +) -> DeviceList: + + if not device_list: + device_list = DeviceList() + + # We use |extend| below (instead of |append|) because you cannot append to a list of non-scalars in protobuf. + # |extend| copies the elements from the list and appends them. + + # Host preprocessor. + device = device_preprocessor() + device_list.devices.extend([device]) + device_list.active_device_names.append(device.name) + + # SwiftShader. + device = swift_shader_device(binary_manager) + device_list.devices.extend([device]) + device_list.active_device_names.append(device.name) + + # Host device. + device = device_host(binary_manager) + device_list.devices.extend([device]) + device_list.active_device_names.append(device.name) + + try: + # Android devices. + android_devices = android_device.get_all_android_devices() + device_list.devices.extend(android_devices) + device_list.active_device_names.extend([d.name for d in android_devices]) + except ToolNotOnPathError: + log( + "WARNING: adb was not found on PATH nor was ANDROID_HOME set; " + "Android devices will not be added to settings.json" + ) + + # Offline compiler. + device = Device( + name="amdllpc", + shader_compiler=DeviceShaderCompiler( + binary="amdllpc", args=["-gfxip=9.0.0", "-verify-ir", "-auto-layout-desc"] + ), + binaries=[binary_manager.get_binary_path_by_name("amdllpc").binary], + ) + device_list.devices.extend([device]) + # Don't add to active devices, since this is mostly just an example. + + return device_list + + +def get_active_devices(device_list: DeviceList) -> List[Device]: + device_map: Dict[str, Device] = {} + for device in device_list.devices: + device_map[device.name] = device + return [device_map[device] for device in device_list.active_device_names] diff --git a/gfauto/gfauto/download_cts_gf_tests.py b/gfauto/gfauto/download_cts_gf_tests.py new file mode 100644 index 000000000..0c712ebe8 --- /dev/null +++ b/gfauto/gfauto/download_cts_gf_tests.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Downloads the latest GraphicsFuzz AmberScript tests from vk-gl-cts. + +Downloads the latest tests, including those that are pending. +""" + +import argparse +import sys +from pathlib import Path + +from gfauto import ( + amber_converter, + binaries_util, + fuzz, + gerrit_util, + settings_util, + subprocess_util, + util, +) + + +def download_cts_graphicsfuzz_tests( # pylint: disable=too-many-locals; + git_tool: Path, cookie: str, binaries: binaries_util.BinaryManager +) -> None: + work_dir = Path() / "temp" / ("cts_" + fuzz.get_random_name()) + + latest_change = gerrit_util.get_latest_deqp_change(cookie) + latest_change_number = latest_change["_number"] + latest_change_details = gerrit_util.get_gerrit_change_details( + change_number=latest_change_number, cookie=cookie + ) + current_revision = latest_change_details["current_revision"] + cts_archive_path = gerrit_util.download_gerrit_revision( + output_path=work_dir / "cts.tgz", + change_number=latest_change_number, + revision=current_revision, + download_type=gerrit_util.DownloadType.Archive, + cookie=cookie, + ) + + cts_dir_name = "cts_temp" + cts_out = util.extract_archive(cts_archive_path, work_dir / cts_dir_name) + + pending_graphicsfuzz_changes = gerrit_util.get_deqp_graphicsfuzz_pending_changes( + cookie + ) + + for pending_change in pending_graphicsfuzz_changes: + change_number = pending_change["_number"] + change_details = gerrit_util.get_gerrit_change_details( + change_number=change_number, cookie=cookie + ) + current_revision = change_details["current_revision"] + patch_zip = gerrit_util.download_gerrit_revision( + output_path=work_dir / f"{change_number}.zip", + change_number=change_number, + revision=current_revision, + download_type=gerrit_util.DownloadType.Patch, + cookie=cookie, + ) + util.extract_archive(patch_zip, work_dir) + + # Create a dummy git repo in the work directory, otherwise "git apply" can fail silently. + # --unsafe-paths is possibly supposed to address this, but it doesn't seem to work if we + # are already in a git repo. + subprocess_util.run( + [str(git_tool), "init", "."], verbose=True, working_dir=work_dir + ) + + cmd = [str(git_tool), "apply"] + + patch_names = [p.name for p in work_dir.glob("*.diff")] + + cmd += patch_names + + # Use unix-style path for git. + cmd += [ + "--verbose", + "--unsafe-paths", + f"--directory={cts_dir_name}", + f"--include={cts_dir_name}/external/vulkancts/data/vulkan/amber/graphicsfuzz/*", + ] + + subprocess_util.run(cmd, verbose=True, working_dir=work_dir) + + shader_dir = util.copy_dir( + cts_out + / "external" + / "vulkancts" + / "data" + / "vulkan" + / "amber" + / "graphicsfuzz", + Path() / "graphicsfuzz", + ) + + for amber_file in shader_dir.glob("*.amber"): + amber_converter.extract_shaders( + amber_file, output_dir=amber_file.parent, binaries=binaries + ) + + zip_files = [ + util.ZipEntry(f, Path(f.name)) + for f in sorted(shader_dir.glob(f"{amber_file.stem}.*")) + ] + + util.create_zip(amber_file.with_suffix(".zip"), zip_files) + + +GERRIT_COOKIE_ARGUMENT_DESCRIPTION = ( + "The Gerrit cookie used for authentication. To get this, log in to the Khronos Gerrit page in your " + "browser and paste the following into the JavaScript console (F12) to copy the cookie to your clipboard: " + "copy(document.cookie.match(/GerritAccount=([^;]*)/)[1])" +) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Downloads the latest GraphicsFuzz AmberScript tests from vk-gl-cts, " + "including those in pending CLs. " + "Requires Git." + ) + + parser.add_argument("gerrit_cookie", help=GERRIT_COOKIE_ARGUMENT_DESCRIPTION) + + parser.add_argument( + "--settings", + help="Path to the settings JSON file for this instance.", + default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + cookie: str = parsed_args.gerrit_cookie + settings_path: Path = Path(parsed_args.settings) + + # Need git. + git_tool = util.tool_on_path("git") + + settings = settings_util.read_or_create(settings_path) + + binaries = binaries_util.get_default_binary_manager(settings=settings) + + download_cts_graphicsfuzz_tests(git_tool, cookie, binaries) + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/fuzz.py b/gfauto/gfauto/fuzz.py new file mode 100644 index 000000000..24edb6476 --- /dev/null +++ b/gfauto/gfauto/fuzz.py @@ -0,0 +1,302 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fuzzing module. + +The main entry point to GraphicsFuzz Auto. +""" + +import argparse +import random +import secrets +import shutil +import sys +from pathlib import Path +from typing import List, Optional + +from gfauto import ( + artifact_util, + binaries_util, + devices_util, + fuzz_glsl_test, + fuzz_spirv_test, + gflogging, + settings_util, + shader_job_util, + test_util, + util, +) +from gfauto.gflogging import log +from gfauto.util import check_dir_exists + +# Root: +# - donors/ (contains GLSL shader jobs) +# - temp/ (contains staging directories with random names) +# - reports/ (contains reports) + +# Staging directory. +# - source_template/ (source directory but with no test metadata yet) +# - test_1/, test_2/, etc. (test directories) + +# Test directory: +# - source/ (source directory, with test.json and other files) +# - results/ (results) + +# Report: a test directory with a reduction for a specific device. +# E.g. v signature v device name added +# reports/crashes/Null_point/123_opt_pixel/ +# - source/ +# - results/ +# - laptop/ +# - reference/ variant/ +# - ... (see below for a more detailed example) +# - pixel/ +# - reference/ variant/ +# - ... (see below for a more detailed example) +# - reductions/ (since this is a report for a pixel device, we have reductions) +# - ... (see below for a more detailed example) + + +# - temp/123/ (a staging directory; not a proper test_dir, as it only has "source_template", not "source".) +# - source_template/ +# - --test.json-- this will NOT be present because this is just a source template directory. +# - reference/ variant/ +# - shader.json, shader.{comp,frag} +# - 123_no_opt/ 123_opt_O/ 123_opt_Os/ 123_opt_rand_1/ etc. (Proper test_dirs, as they have "source". These may be +# copied to become a report if a bug is found.) +# - source/ (same as source_template, but with test.json) +# - results/ +# - pixel/ other_phone/ laptop/ etc. +# - reference/ variant/ +# - test.amber +# - image.png +# - STATUS +# - log.txt +# - (all other result files and intermediate files for running the shader on the device) +# - reductions/ (reductions are only added once the staging directory is copied to the reports directory) +# - reduction_1/ reduction_blah/ etc. (reduction name; also a test_dir) +# - source/ (same as other source dirs, but with the final reduced shader source) +# - reduction_work/ +# - reference/ variant/ +# - shader.json, shader_reduction_001_success.json, +# shader_reduction_002_failed.json, etc., shader_reduced_final.json +# - shader/ shader_reduction_001/ +# (these are the result directories for each step, containing STATUS, etc.) +# + + +REFERENCE_IMAGE_FILE_NAME = "reference.png" +VARIANT_IMAGE_FILE_NAME = "variant.png" +BUFFER_FILE_NAME = "buffer.bin" + +BEST_REDUCTION_NAME = "best" + +AMBER_RUN_TIME_LIMIT = 30 + +STATUS_TOOL_CRASH = "TOOL_CRASH" +STATUS_CRASH = "CRASH" +STATUS_UNRESPONSIVE = "UNRESPONSIVE" +STATUS_TOOL_TIMEOUT = "TOOL_TIMEOUT" +STATUS_TIMEOUT = "TIMEOUT" +STATUS_SUCCESS = "SUCCESS" + +# Number of bits for seeding the RNG. +# Python normally uses 256 bits internally when seeding its RNG, hence this choice. +ITERATION_SEED_BITS = 256 + +FUZZ_FAILURES_DIR_NAME = "fuzz_failures" + + +def get_random_name() -> str: + # TODO: could change to human-readable random name or the date. + return util.get_random_name() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fuzz") + + parser.add_argument( + "--settings", + help="Path to the settings JSON file for this fuzzing instance.", + default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), + ) + + parser.add_argument( + "--iteration_seed", + help="The seed to use for one fuzzing iteration (useful for reproducing an issue).", + ) + + parser.add_argument( + "--use_spirv_fuzz", + help="Do fuzzing using spirv-fuzz, which must be on your PATH.", + action="store_true", + ) + + parser.add_argument( + "--force_no_stack_traces", + help="Continue even if we cannot get stack traces (using catchsegv or cdb).", + action="store_true", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + settings_path = Path(parsed_args.settings) + iteration_seed: Optional[int] = None if parsed_args.iteration_seed is None else int( + parsed_args.iteration_seed + ) + use_spirv_fuzz: bool = parsed_args.use_spirv_fuzz + force_no_stack_traces: bool = parsed_args.force_no_stack_traces + + with util.file_open_text(Path(f"log_{get_random_name()}.txt"), "w") as log_file: + gflogging.push_stream_for_logging(log_file) + try: + main_helper( + settings_path, iteration_seed, use_spirv_fuzz, force_no_stack_traces + ) + except settings_util.NoSettingsFile as exception: + log(str(exception)) + finally: + gflogging.pop_stream_for_logging() + + +def main_helper( # pylint: disable=too-many-locals, too-many-branches, too-many-statements; + settings_path: Path, + iteration_seed_override: Optional[int], + use_spirv_fuzz: bool, + force_no_stack_traces: bool, +) -> None: + + util.update_gcov_environment_variable_if_needed() + + try: + artifact_util.artifact_path_get_root() + except FileNotFoundError: + log( + "Could not find ROOT file (in the current directory or above) to mark where binaries should be stored. " + "Creating a ROOT file in the current directory." + ) + util.file_write_text(Path(artifact_util.ARTIFACT_ROOT_FILE_NAME), "") + + settings = settings_util.read_or_create(settings_path) + + active_devices = devices_util.get_active_devices(settings.device_list) + + reports_dir = Path() / "reports" + fuzz_failures_dir = reports_dir / FUZZ_FAILURES_DIR_NAME + temp_dir = Path() / "temp" + references_dir = Path() / "references" + donors_dir = Path() / "donors" + spirv_fuzz_shaders_dir = Path() / "spirv_fuzz_shaders" + + # Log a warning if there is no tool on the PATH for printing stack traces. + prepended = util.prepend_catchsegv_if_available([], log_warning=True) + if not force_no_stack_traces and not prepended: + raise AssertionError("Stopping because we cannot get stack traces.") + + spirv_fuzz_shaders: List[Path] = [] + references: List[Path] = [] + + if use_spirv_fuzz: + check_dir_exists(spirv_fuzz_shaders_dir) + spirv_fuzz_shaders = sorted(spirv_fuzz_shaders_dir.rglob("*.json")) + else: + check_dir_exists(references_dir) + check_dir_exists(donors_dir) + # TODO: make GraphicsFuzz find donors recursively. + references = sorted(references_dir.rglob("*.json")) + # Filter to only include .json files that have at least one shader (.frag, .vert, .comp) file. + references = [ + ref for ref in references if shader_job_util.get_related_files(ref) + ] + + binary_manager = binaries_util.get_default_binary_manager( + settings=settings + ).get_child_binary_manager(list(settings.custom_binaries), prepend=True) + + while True: + + # We have to use "is not None" because the seed could be 0. + if iteration_seed_override is not None: + iteration_seed = iteration_seed_override + else: + iteration_seed = secrets.randbits(ITERATION_SEED_BITS) + + log(f"Iteration seed: {iteration_seed}") + random.seed(iteration_seed) + + staging_name = get_random_name()[:8] + staging_dir = temp_dir / staging_name + + try: + util.mkdir_p_new(staging_dir) + except FileExistsError: + if iteration_seed_override is not None: + raise + log(f"Staging directory already exists: {str(staging_dir)}") + log(f"Starting new iteration.") + continue + + # Pseudocode: + # - Create test_dir(s) in staging directory. + # - Run test_dir(s) on all active devices (stop early if appropriate). + # - For each test failure on each device, copy the test to reports_dir, adding the device and crash signature. + # - Reduce each report (on the given device). + # - Produce a summary for each report. + + if use_spirv_fuzz: + fuzz_spirv_test.fuzz_spirv( + staging_dir, + reports_dir, + fuzz_failures_dir, + active_devices, + spirv_fuzz_shaders, + settings, + binary_manager, + ) + else: + fuzz_glsl_test.fuzz_glsl( + staging_dir, + reports_dir, + fuzz_failures_dir, + active_devices, + references, + donors_dir, + settings, + binary_manager, + ) + + shutil.rmtree(staging_dir) + + if iteration_seed_override is not None: + log("Stopping due to iteration_seed") + break + + +def create_summary_and_reproduce( + test_dir: Path, binary_manager: binaries_util.BinaryManager +) -> None: + util.mkdirs_p(test_dir / "summary") + test_metadata = test_util.metadata_read(test_dir) + + # noinspection PyTypeChecker + if test_metadata.HasField("glsl") or test_metadata.HasField("spirv_fuzz"): + fuzz_glsl_test.create_summary_and_reproduce(test_dir, binary_manager) + else: + raise AssertionError("Unrecognized test type") + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/fuzz_glsl_test.py b/gfauto/gfauto/fuzz_glsl_test.py new file mode 100644 index 000000000..eeeb185fe --- /dev/null +++ b/gfauto/gfauto/fuzz_glsl_test.py @@ -0,0 +1,892 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GLSL fuzzing module. + +Functions for handling GLSL shader job tests. +""" + +import random +import subprocess +from pathlib import Path +from typing import Iterable, List, Optional + +from gfauto import ( + amber_converter, + android_device, + binaries_util, + fuzz, + gflogging, + glsl_generate_util, + host_device_util, + result_util, + shader_compiler_util, + shader_job_util, + signature_util, + spirv_opt_util, + subprocess_util, + test_util, + tool, + util, +) +from gfauto.device_pb2 import Device +from gfauto.gflogging import log +from gfauto.settings_pb2 import Settings +from gfauto.test_pb2 import Test, TestGlsl +from gfauto.util import check + + +class ReductionFailedError(Exception): + def __init__(self, message: str, reduction_name: str, reduction_work_dir: Path): + super().__init__(message) + self.reduction_name = reduction_name + self.reduction_work_dir = reduction_work_dir + + +def fuzz_glsl( + staging_dir: Path, + reports_dir: Path, + fuzz_failures_dir: Path, + active_devices: List[Device], + references: List[Path], + donors_dir: Path, + settings: Settings, + binary_manager: binaries_util.BinaryManager, +) -> None: + staging_name = staging_dir.name + template_source_dir = staging_dir / "source_template" + + # Pick a randomly chosen reference. + unprepared_reference_shader_job = random.choice(references) + + # The "graphicsfuzz-tool" tool is designed to be on your PATH so that e.g. ".bat" will be appended on Windows. + # So we use tool_on_path with a custom PATH to get the actual file we want to execute. + graphicsfuzz_tool_path = util.tool_on_path( + "graphicsfuzz-tool", + str(binary_manager.get_binary_path_by_name("graphicsfuzz-tool").path.parent), + ) + + try: + with util.file_open_text(staging_dir / "log.txt", "w") as log_file: + try: + gflogging.push_stream_for_logging(log_file) + + # Create the prepared (for Vulkan GLSL) reference. + glsl_generate_util.run_prepare_reference( + graphicsfuzz_tool_path, + unprepared_reference_shader_job, + template_source_dir + / test_util.REFERENCE_DIR + / test_util.SHADER_JOB, + ) + + # Generate the variant (GraphicsFuzz requires the unprepared reference as input). + glsl_generate_util.run_generate( + graphicsfuzz_tool_path, + unprepared_reference_shader_job, + donors_dir, + template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, + seed=str(random.getrandbits(glsl_generate_util.GENERATE_SEED_BITS)), + ) + finally: + gflogging.pop_stream_for_logging() + except subprocess.CalledProcessError: + util.mkdirs_p(fuzz_failures_dir) + if len(list(fuzz_failures_dir.iterdir())) < settings.maximum_fuzz_failures: + util.copy_dir(staging_dir, fuzz_failures_dir / staging_dir.name) + return + + test_dirs = [ + make_test( + template_source_dir, + staging_dir / f"{staging_name}_no_opt_test", + spirv_opt_args=None, + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_O_test", + spirv_opt_args=["-O"], + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_Os_test", + spirv_opt_args=["-Os"], + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_rand1_test", + spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_rand2_test", + spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_rand3_test", + spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), + binary_manager=binary_manager, + ), + ] + + for test_dir in test_dirs: + if handle_test(test_dir, reports_dir, active_devices, binary_manager, settings): + # If we generated a report, don't bother trying other optimization combinations. + break + + +def make_test( + base_source_dir: Path, + subtest_dir: Path, + spirv_opt_args: Optional[List[str]], + binary_manager: binaries_util.BinaryManager, +) -> Path: + # Create the subtest by copying the base source. + util.copy_dir(base_source_dir, test_util.get_source_dir(subtest_dir)) + + test = Test(glsl=TestGlsl(spirv_opt_args=spirv_opt_args)) + + test.binaries.extend([binary_manager.get_binary_by_name(name="glslangValidator")]) + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-dis")]) + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-val")]) + if spirv_opt_args: + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-opt")]) + + # Write the test metadata. + test_util.metadata_write(test, subtest_dir) + + return subtest_dir + + +def run( + test_dir: Path, + binary_manager: binaries_util.BinaryManager, + device: Optional[Device] = None, +) -> str: + + test: Test = test_util.metadata_read(test_dir) + if not device: + device = test.device + + result_output_dir = run_shader_job( + source_dir=test_util.get_source_dir(test_dir), + output_dir=test_util.get_results_directory(test_dir, device.name), + binary_manager=binary_manager, + device=device, + ) + + return result_util.get_status(result_output_dir) + + +def maybe_add_report( # pylint: disable=too-many-locals; + test_dir: Path, reports_dir: Path, device: Device, settings: Settings +) -> Optional[Path]: + + result_output_dir = test_util.get_results_directory(test_dir, device.name) + + status = result_util.get_status(result_output_dir) + + report_subdirectory_name = "" + + if status == fuzz.STATUS_CRASH: + report_subdirectory_name = "crashes" + elif status == fuzz.STATUS_TOOL_CRASH: + report_subdirectory_name = "tool_crashes" + elif status == fuzz.STATUS_UNRESPONSIVE: + report_subdirectory_name = "unresponsive" + + if not report_subdirectory_name: + return None + log_path = result_util.get_log_path(result_output_dir) + + log_contents = util.file_read_text(log_path) + signature = signature_util.get_signature_from_log_contents(log_contents) + + signature_dir = reports_dir / report_subdirectory_name / signature + + util.mkdirs_p(signature_dir) + + # If the signature_dir contains a NOT_INTERESTING file, then don't bother creating a report. + if (signature_dir / "NOT_INTERESTING").exists(): + return None + + if signature != signature_util.BAD_IMAGE_SIGNATURE: + # If we have reached the maximum number of crashes per signature for this device, don't create a report. + num_duplicates = [ + report_dir + for report_dir in signature_dir.iterdir() + if report_dir.is_dir() and report_dir.name.endswith(f"_{device.name}") + ] + if len(num_duplicates) >= settings.maximum_duplicate_crashes: + return None + + # We include the device name in the directory name because it is possible that this test crashes on two + # different devices but gives the same crash signature in both cases (e.g. for generic signatures + # like "compile_error"). This would lead to two test copies having the same path. + # It also means we can limit duplicates per device using the directory name. + test_dir_in_reports = signature_dir / f"{test_dir.name}_{device.name}" + + util.copy_dir(test_dir, test_dir_in_reports) + + if signature != signature_util.BAD_IMAGE_SIGNATURE: + + # If we found a crash, rename the directories for all shaders other than the variant. Thus, only the variant + # shader will run. + + bad_shader_name = result_util.get_status_bad_shader_name( + test_util.get_results_directory(test_dir_in_reports, device.name) + ) + + # TODO: Could possibly improve this. Could try scanning the Amber log to figure out which shader failed? + + if not bad_shader_name: + log("WARNING: assuming that the bad shader is the variant") + bad_shader_name = test_util.VARIANT_DIR + + shader_jobs = tool.get_shader_jobs( + test_util.get_source_dir(test_dir_in_reports) + ) + found_bad_shader = False + for shader_job in shader_jobs: + if shader_job.name == bad_shader_name: + found_bad_shader = True + else: + shader_job.shader_job.parent.rename( + shader_job.shader_job.parent.parent / f"_{shader_job.name}" + ) + check( + found_bad_shader, + AssertionError( + f"Could not find bad shader at: {test_util.get_source_dir(test_dir_in_reports) / bad_shader_name}" + ), + ) + + test_metadata = test_util.metadata_read(test_dir_in_reports) + test_metadata.crash_signature = signature + test_metadata.device.CopyFrom(device) + test_metadata.expected_status = status + test_util.metadata_write(test_metadata, test_dir_in_reports) + + return test_dir_in_reports + + +def should_reduce_report(settings: Settings, test_dir: Path) -> bool: + test = test_util.metadata_read(test_dir) + status = test.expected_status + signature = test.crash_signature + + if not settings.reduce_tool_crashes and status == fuzz.STATUS_TOOL_CRASH: + return False + if ( + not settings.reduce_crashes + and status == fuzz.STATUS_CRASH + and signature != signature_util.BAD_IMAGE_SIGNATURE + ): + return False + if ( + not settings.reduce_bad_images + and status == fuzz.STATUS_CRASH + and signature == signature_util.BAD_IMAGE_SIGNATURE + ): + return False + + return True + + +def run_reduction_on_report( + test_dir: Path, reports_dir: Path, binary_manager: binaries_util.BinaryManager +) -> None: + test = test_util.metadata_read(test_dir) + + try: + reduced_test = test_dir + reduced_test = run_reduction( + test_dir_reduction_output=test_dir, + test_dir_to_reduce=reduced_test, + preserve_semantics=True, + binary_manager=binary_manager, + reduction_name="1", + ) + + if test.crash_signature != signature_util.BAD_IMAGE_SIGNATURE: + reduced_test = run_reduction( + test_dir_reduction_output=test_dir, + test_dir_to_reduce=reduced_test, + preserve_semantics=False, + binary_manager=binary_manager, + reduction_name="2", + ) + + device_name = test.device.name + + # Create a symlink to the "best" reduction. + best_reduced_test_link = test_util.get_reduced_test_dir( + test_dir, device_name, fuzz.BEST_REDUCTION_NAME + ) + util.make_directory_symlink( + new_symlink_file_path=best_reduced_test_link, existing_dir=reduced_test + ) + except ReductionFailedError as ex: + # Create a symlink to the failed reduction so it is easy to investigate failed reductions. + link_to_failed_reduction_path = ( + reports_dir / "failed_reductions" / f"{test_dir.name}_{ex.reduction_name}" + ) + util.make_directory_symlink( + new_symlink_file_path=link_to_failed_reduction_path, + existing_dir=ex.reduction_work_dir, + ) + + +def handle_test( + test_dir: Path, + reports_dir: Path, + active_devices: List[Device], + binary_manager: binaries_util.BinaryManager, + settings: Settings, +) -> bool: + + report_paths: List[Path] = [] + issue_found = False + + # Run on all devices. + for device in active_devices: + status = run(test_dir, binary_manager, device) + if status in ( + fuzz.STATUS_CRASH, + fuzz.STATUS_TOOL_CRASH, + fuzz.STATUS_UNRESPONSIVE, + ): + issue_found = True + if status == fuzz.STATUS_TOOL_CRASH: + # No need to run further on real devices if the pre-processing step failed. + break + + # For each device that saw a crash, copy the test to reports_dir, adding the signature and device info to the test + # metadata. + for device in active_devices: + report_dir = maybe_add_report(test_dir, reports_dir, device, settings) + if report_dir: + report_paths.append(report_dir) + + # For each report, run a reduction on the target device with the device-specific crash signature. + for test_dir_in_reports in report_paths: + if should_reduce_report(settings, test_dir_in_reports): + run_reduction_on_report(test_dir_in_reports, reports_dir, binary_manager) + else: + log("Skipping reduction due to settings.") + + # For each report, create a summary and reproduce the bug. + for test_dir_in_reports in report_paths: + fuzz.create_summary_and_reproduce(test_dir_in_reports, binary_manager) + + return issue_found + + +def run_shader_job( # pylint: disable=too-many-return-statements,too-many-branches, too-many-locals, too-many-statements; + source_dir: Path, + output_dir: Path, + binary_manager: binaries_util.BinaryManager, + test: Optional[Test] = None, + device: Optional[Device] = None, + ignore_test_and_device_binaries: bool = False, + shader_job_overrides: Iterable[tool.NameAndShaderJob] = (), + shader_job_shader_overrides: Optional[ + tool.ShaderJobNameToShaderOverridesMap + ] = None, +) -> Path: + + if not shader_job_shader_overrides: + shader_job_shader_overrides = {} + + with util.file_open_text(output_dir / "log.txt", "w") as log_file: + try: + gflogging.push_stream_for_logging(log_file) + + # TODO: Find amber path. NDK or host. + + # TODO: If Amber is going to be used, check if Amber can use Vulkan debug layers now, and if not, pass that + # info down via a bool. + + if not test: + test = test_util.metadata_read_from_path( + source_dir / test_util.TEST_METADATA + ) + + if not device: + device = test.device + + log(f"Running test on device:\n{device.name}") + + # We will create a binary_manager child with a restricted set of binaries so that we only use the binaries + # specified in the test and by the device; if some required binaries are not specified by the test nor the + # device, there will be an error instead of falling back to our default binaries. But we keep a reference to + # the parent so we can still access certain "test-independent" binaries like Amber. + + binary_manager_parent = binary_manager + + if not ignore_test_and_device_binaries: + binary_manager = binary_manager.get_child_binary_manager( + list(device.binaries) + list(test.binaries) + ) + + spirv_opt_hash: Optional[str] = None + spirv_opt_args: Optional[List[str]] = None + if test.glsl.spirv_opt_args or test.spirv_fuzz.spirv_opt_args: + spirv_opt_hash = binary_manager.get_binary_by_name( + binaries_util.SPIRV_OPT_NAME + ).version + spirv_opt_args = ( + list(test.glsl.spirv_opt_args) + if test.glsl.spirv_opt_args + else list(test.spirv_fuzz.spirv_opt_args) + ) + + shader_jobs = tool.get_shader_jobs( + source_dir, overrides=shader_job_overrides + ) + + combined_spirv_shader_jobs: List[tool.SpirvCombinedShaderJob] = [] + + for shader_job in shader_jobs: + try: + shader_overrides = shader_job_shader_overrides.get( + shader_job.name, None + ) + combined_spirv_shader_jobs.append( + tool.compile_shader_job( + name=shader_job.name, + input_json=shader_job.shader_job, + work_dir=output_dir / shader_job.name, + binary_paths=binary_manager, + spirv_opt_args=spirv_opt_args, + shader_overrides=shader_overrides, + ) + ) + except subprocess.CalledProcessError: + result_util.write_status( + output_dir, fuzz.STATUS_TOOL_CRASH, shader_job.name + ) + return output_dir + except subprocess.TimeoutExpired: + result_util.write_status( + output_dir, fuzz.STATUS_TOOL_TIMEOUT, shader_job.name + ) + return output_dir + + # Device types: |preprocess| and |shader_compiler| don't need an AmberScript file. + + # noinspection PyTypeChecker + if device.HasField("preprocess"): + # The "preprocess" device type just needs to get this far, so this is a success. + result_util.write_status(output_dir, fuzz.STATUS_SUCCESS) + return output_dir + + # noinspection PyTypeChecker + if device.HasField("shader_compiler"): + for combined_spirv_shader_job in combined_spirv_shader_jobs: + try: + shader_compiler_util.run_shader_job( + device.shader_compiler, + combined_spirv_shader_job.spirv_shader_job, + output_dir, + binary_manager=binary_manager, + ) + except subprocess.CalledProcessError: + result_util.write_status( + output_dir, + fuzz.STATUS_CRASH, + combined_spirv_shader_job.name, + ) + return output_dir + except subprocess.TimeoutExpired: + result_util.write_status( + output_dir, + fuzz.STATUS_TIMEOUT, + combined_spirv_shader_job.name, + ) + return output_dir + + # The shader compiler succeeded on all files; this is a success. + result_util.write_status(output_dir, fuzz.STATUS_SUCCESS) + return output_dir + + # Other device types need an AmberScript file. + + amber_converter_shader_job_files = [ + amber_converter.ShaderJobFile( + name_prefix=combined_spirv_shader_job.name, + asm_spirv_shader_job_json=combined_spirv_shader_job.spirv_asm_shader_job, + glsl_source_json=combined_spirv_shader_job.glsl_source_shader_job, + processing_info="", + ) + for combined_spirv_shader_job in combined_spirv_shader_jobs + ] + + # Check if the first is the reference shader; if so, pull it out into its own variable. + + reference: Optional[amber_converter.ShaderJobFile] = None + variants = amber_converter_shader_job_files + + if ( + amber_converter_shader_job_files[0].name_prefix + == test_util.REFERENCE_DIR + ): + reference = amber_converter_shader_job_files[0] + variants = variants[1:] + elif len(variants) > 1: + raise AssertionError( + "More than one variant, but no reference. This is unexpected." + ) + + amber_script_file = amber_converter.spirv_asm_shader_job_to_amber_script( + shader_job_file_amber_test=amber_converter.ShaderJobFileBasedAmberTest( + reference_asm_spirv_job=reference, variants_asm_spirv_job=variants + ), + output_amber_script_file_path=output_dir / "test.amber", + amberfy_settings=amber_converter.AmberfySettings( + spirv_opt_args=spirv_opt_args, spirv_opt_hash=spirv_opt_hash + ), + ) + + is_compute = bool( + shader_job_util.get_related_files( + combined_spirv_shader_jobs[0].spirv_shader_job, + [shader_job_util.EXT_COMP], + ) + ) + + # noinspection PyTypeChecker + if device.HasField("host") or device.HasField("swift_shader"): + icd: Optional[Path] = None + + # noinspection PyTypeChecker + if device.HasField("swift_shader"): + icd = binary_manager.get_binary_path_by_name( + binaries_util.SWIFT_SHADER_NAME + ).path + + # Run the test on the host using Amber. + host_device_util.run_amber( + amber_script_file, + output_dir, + amber_path=binary_manager_parent.get_binary_path_by_name( + binaries_util.AMBER_NAME + ).path, + dump_image=(not is_compute), + dump_buffer=is_compute, + icd=icd, + ) + return output_dir + + # noinspection PyTypeChecker + if device.HasField("android"): + + android_device.run_amber_on_device( + amber_script_file, + output_dir, + dump_image=(not is_compute), + dump_buffer=is_compute, + serial=device.android.serial, + ) + return output_dir + + # TODO: For a remote device (which we will probably need to support), use log_a_file to output the + # "amber_log.txt" file. + + raise AssertionError(f"Unhandled device type:\n{str(device)}") + + finally: + gflogging.pop_stream_for_logging() + + +def get_final_reduced_shader_job_path(reduction_work_shader_dir: Path) -> Path: + return reduction_work_shader_dir / "shader_reduced_final.json" + + +def run_reduction( + test_dir_reduction_output: Path, + test_dir_to_reduce: Path, + preserve_semantics: bool, + binary_manager: binaries_util.BinaryManager, + reduction_name: str = "reduction1", +) -> Path: + test = test_util.metadata_read(test_dir_to_reduce) + + if not test.device or not test.device.name: + raise AssertionError( + f"Cannot reduce {str(test_dir_to_reduce)}; " + f"device must be specified in {str(test_util.get_metadata_path(test_dir_to_reduce))}" + ) + + if not test.crash_signature: + raise AssertionError( + f"Cannot reduce {str(test_dir_to_reduce)} because there is no crash string specified." + ) + + # E.g. reports/crashes/no_signature/d50c96e8_opt_rand2_test_phone_ABC/results/phone_ABC/reductions/1 + # Will contain work/ and source/ + reduced_test_dir = test_util.get_reduced_test_dir( + test_dir_reduction_output, test.device.name, reduction_name + ) + + source_dir = test_util.get_source_dir(test_dir_to_reduce) + + shader_jobs = tool.get_shader_jobs(source_dir) + + # TODO: if needed, this could become a parameter to this function. + name_of_shader_to_reduce = shader_jobs[0].name + + if len(shader_jobs) > 1: + check( + len(shader_jobs) == 2 and shader_jobs[1].name == test_util.VARIANT_DIR, + AssertionError( + "Can only reduce tests with shader jobs reference and variant, or just variant." + ), + ) + name_of_shader_to_reduce = shader_jobs[1].name + + reduction_work_variant_dir = run_glsl_reduce( + source_dir=source_dir, + name_of_shader_to_reduce=name_of_shader_to_reduce, + output_dir=test_util.get_reduction_work_directory( + reduced_test_dir, name_of_shader_to_reduce + ), + binary_manager=binary_manager, + preserve_semantics=preserve_semantics, + ) + + final_reduced_shader_job_path = get_final_reduced_shader_job_path( + reduction_work_variant_dir + ) + + check( + final_reduced_shader_job_path.exists(), + ReductionFailedError( + "Reduction failed.", reduction_name, reduction_work_variant_dir + ), + ) + + # Finally, create the source_dir so the returned directory can be used as a test_dir. + + util.copy_dir(source_dir, test_util.get_source_dir(reduced_test_dir)) + + shader_job_util.copy( + final_reduced_shader_job_path, + test_util.get_shader_job_path(reduced_test_dir, name_of_shader_to_reduce), + ) + + return reduced_test_dir + + +def run_glsl_reduce( + source_dir: Path, + name_of_shader_to_reduce: str, + output_dir: Path, + binary_manager: binaries_util.BinaryManager, + preserve_semantics: bool = False, +) -> Path: + + input_shader_job = source_dir / name_of_shader_to_reduce / test_util.SHADER_JOB + + glsl_reduce_path = util.tool_on_path( + "glsl-reduce", + str(binary_manager.get_binary_path_by_name("graphicsfuzz-tool").path.parent), + ) + + cmd = [ + str(glsl_reduce_path), + str(input_shader_job), + "--output", + str(output_dir), + # This ensures the arguments that follow are all positional arguments. + "--", + "gfauto_interestingness_test", + str(source_dir), + # --override_shader_job requires two parameters to follow; the second will be added by glsl-reduce (the shader.json file). + "--override_shader_job", + str(name_of_shader_to_reduce), + ] + + if preserve_semantics: + cmd.insert(1, "--preserve-semantics") + + # Log the reduction. + with util.file_open_text(output_dir / "command.log", "w") as f: + gflogging.push_stream_for_logging(f) + try: + # The reducer can fail, but it will typically output an exception file, so we can ignore the exit code. + subprocess_util.run(cmd, verbose=True, check_exit_code=False) + finally: + gflogging.pop_stream_for_logging() + + return output_dir + + +def create_summary_and_reproduce( + test_dir: Path, binary_manager: binaries_util.BinaryManager +) -> None: + test_metadata = test_util.metadata_read(test_dir) + + summary_dir = test_dir / "summary" + + unreduced = util.copy_dir( + test_util.get_source_dir(test_dir), summary_dir / "unreduced" + ) + + reduced_test_dir = test_util.get_reduced_test_dir( + test_dir, test_metadata.device.name, fuzz.BEST_REDUCTION_NAME + ) + reduced_source_dir = test_util.get_source_dir(reduced_test_dir) + reduced: Optional[Path] = None + if reduced_source_dir.exists(): + reduced = util.copy_dir(reduced_source_dir, summary_dir / "reduced") + + run_shader_job( + source_dir=unreduced, + output_dir=(summary_dir / "unreduced_result"), + binary_manager=binary_manager, + ) + + variant_reduced_glsl_result: Optional[Path] = None + if reduced: + variant_reduced_glsl_result = run_shader_job( + source_dir=reduced, + output_dir=(summary_dir / "reduced_result"), + binary_manager=binary_manager, + ) + + # Some post-processing for common error types. + + if variant_reduced_glsl_result: + status = result_util.get_status(variant_reduced_glsl_result) + if status == fuzz.STATUS_TOOL_CRASH: + tool_crash_summary_bug_report_dir( + reduced_source_dir, + variant_reduced_glsl_result, + summary_dir, + binary_manager, + ) + + +def tool_crash_summary_bug_report_dir( # pylint: disable=too-many-locals; + reduced_glsl_source_dir: Path, + variant_reduced_glsl_result_dir: Path, + output_dir: Path, + binary_manager: binaries_util.BinaryManager, +) -> Optional[Path]: + # Create a simple script and README. + + shader_job = reduced_glsl_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB + + if not shader_job.is_file(): + return None + + test_metadata: Test = test_util.metadata_read_from_path( + reduced_glsl_source_dir / test_util.TEST_METADATA + ) + + shader_files = shader_job_util.get_related_files( + shader_job, + shader_job_util.EXT_ALL, + (shader_job_util.SUFFIX_GLSL, shader_job_util.SUFFIX_SPIRV), + ) + check( + len(shader_files) > 0, + AssertionError(f"Need at least one shader for {shader_job}"), + ) + + shader_extension = shader_files[0].suffix + + bug_report_dir = util.copy_dir( + variant_reduced_glsl_result_dir, output_dir / "bug_report" + ) + + shader_files = sorted(bug_report_dir.rglob("shader.*")) + + glsl_files = [ + shader_file + for shader_file in shader_files + if shader_file.suffix == shader_extension + ] + + asm_files = [ + shader_file + for shader_file in shader_files + if shader_file.name.endswith( + shader_extension + shader_job_util.SUFFIX_ASM_SPIRV + ) + ] + + spv_files = [ + shader_file + for shader_file in shader_files + if shader_file.name.endswith(shader_extension + shader_job_util.SUFFIX_SPIRV) + ] + + readme = "\n\n" + readme += ( + "Issue found using [GraphicsFuzz](https://github.com/google/graphicsfuzz).\n\n" + ) + readme += "Tool versions:\n\n" + + # noinspection PyTypeChecker + if test_metadata.HasField("glsl"): + readme += f"* glslangValidator commit hash: {binary_manager.get_binary_by_name(binaries_util.GLSLANG_VALIDATOR_NAME).version}\n" + + if test_metadata.glsl.spirv_opt_args or test_metadata.spirv_fuzz.spirv_opt_args: + readme += f"* spirv-opt commit hash: {binary_manager.get_binary_by_name(binaries_util.SPIRV_OPT_NAME).version}\n" + + readme += "\nTo reproduce:\n\n" + readme += f"`glslangValidator -V shader{shader_extension} -o shader{shader_extension}.spv`\n\n" + + if ( + test_metadata.HasField("glsl") + and spv_files + and not test_metadata.glsl.spirv_opt_args + ): + # GLSL was converted to SPIR-V, and spirv-opt was not run, so indicate that we should validate the SPIR-V. + readme += f"`spirv-val shader{shader_extension}.spv`\n\n" + + if test_metadata.glsl.spirv_opt_args or test_metadata.spirv_fuzz.spirv_opt_args: + readme += f"`spirv-opt shader{shader_extension}.spv -o temp.spv --validate-after-all {' '.join(test_metadata.glsl.spirv_opt_args)}`\n\n" + + files_to_list = glsl_files + spv_files + asm_files + files_to_list.sort() + + files_to_show = glsl_files + asm_files + files_to_show.sort() + + readme += "The following shader files are included in the attached archive, some of which are also shown inline below:\n\n" + + for file_to_list in files_to_list: + short_path = file_to_list.relative_to(bug_report_dir).as_posix() + readme += f"* {short_path}\n" + + for file_to_show in files_to_show: + short_path = file_to_show.relative_to(bug_report_dir).as_posix() + file_contents = util.file_read_text(file_to_show) + readme += f"\n{short_path}:\n\n" + readme += f"```\n{file_contents}\n```\n" + + util.file_write_text(output_dir / "README.md", readme) + + return bug_report_dir diff --git a/gfauto/gfauto/fuzz_spirv_test.py b/gfauto/gfauto/fuzz_spirv_test.py new file mode 100644 index 000000000..8d2379efa --- /dev/null +++ b/gfauto/gfauto/fuzz_spirv_test.py @@ -0,0 +1,487 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SPIR-V fuzzing module. + +Functions for handling spirv-fuzz generated tests. +""" + +import random +import subprocess +from pathlib import Path +from typing import List, Optional + +from gfauto import ( + binaries_util, + fuzz, + fuzz_glsl_test, + gflogging, + result_util, + shader_job_util, + signature_util, + spirv_fuzz_util, + spirv_opt_util, + subprocess_util, + test_util, + tool, + util, +) +from gfauto.device_pb2 import Device +from gfauto.fuzz_glsl_test import ReductionFailedError +from gfauto.gflogging import log +from gfauto.settings_pb2 import Settings +from gfauto.test_pb2 import Test, TestSpirvFuzz +from gfauto.util import check + + +def make_test( + base_source_dir: Path, + subtest_dir: Path, + spirv_opt_args: Optional[List[str]], + binary_manager: binaries_util.BinaryManager, +) -> Path: + # Create the subtest by copying the base source. + util.copy_dir(base_source_dir, test_util.get_source_dir(subtest_dir)) + + test = Test(spirv_fuzz=TestSpirvFuzz(spirv_opt_args=spirv_opt_args)) + + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-dis")]) + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-val")]) + if spirv_opt_args: + test.binaries.extend([binary_manager.get_binary_by_name(name="spirv-opt")]) + + # Write the test metadata. + test_util.metadata_write(test, subtest_dir) + + return subtest_dir + + +def run( + test_dir: Path, + binary_manager: binaries_util.BinaryManager, + device: Optional[Device] = None, +) -> str: + + test: Test = test_util.metadata_read(test_dir) + if not device: + device = test.device + + result_output_dir = fuzz_glsl_test.run_shader_job( + source_dir=test_util.get_source_dir(test_dir), + output_dir=test_util.get_results_directory(test_dir, device.name), + binary_manager=binary_manager, + device=device, + ) + + return result_util.get_status(result_output_dir) + + +def run_spirv_reduce_or_shrink( + source_dir: Path, + name_of_shader_job_to_reduce: str, + extension_to_reduce: str, + output_dir: Path, + preserve_semantics: bool, + binary_manager: binaries_util.BinaryManager, +) -> Path: + input_shader_job = source_dir / name_of_shader_job_to_reduce / test_util.SHADER_JOB + + original_spirv_file = input_shader_job.with_suffix( + extension_to_reduce + shader_job_util.SUFFIX_SPIRV_ORIG + ) + + transformed_spirv_file = input_shader_job.with_suffix( + extension_to_reduce + shader_job_util.SUFFIX_SPIRV + ) + transformations_file = input_shader_job.with_suffix( + extension_to_reduce + shader_job_util.SUFFIX_TRANSFORMATIONS + ) + + util.mkdirs_p(output_dir) + + final_shader = output_dir / "final.spv" + + # E.g. transformation_suffix_to_reduce == ".frag.transformations" + + # E.g. ".frag.??" -> ".frag.spv" + shader_suffix_to_override = extension_to_reduce + shader_job_util.SUFFIX_SPIRV + + if preserve_semantics: + cmd = [ + str(binary_manager.get_binary_path_by_name("spirv-fuzz").path), + str(original_spirv_file), + "-o", + str(final_shader), + f"--shrink={str(transformations_file)}", + f"--shrinker-temp-file-prefix={str(output_dir / 'temp_')}", + # This ensures the arguments that follow are all positional arguments. + "--", + "gfauto_interestingness_test", + str(source_dir), + # --override_shader requires three parameters to follow; the third will be added by spirv-fuzz (the shader.spv file). + "--override_shader", + name_of_shader_job_to_reduce, + shader_suffix_to_override, + ] + else: + cmd = [ + str(binary_manager.get_binary_path_by_name("spirv-reduce").path), + str(transformed_spirv_file), + "-o", + str(final_shader), + f"--temp-file-prefix={str(output_dir / 'temp_')}", + # This ensures the arguments that follow are all positional arguments. + "--", + "gfauto_interestingness_test", + str(source_dir), + # --override_shader requires three parameters to follow; the third will be added by spirv-reduce (the shader.spv file). + "--override_shader", + name_of_shader_job_to_reduce, + shader_suffix_to_override, + ] + + # Log the reduction. + with util.file_open_text(output_dir / "command.log", "w") as f: + gflogging.push_stream_for_logging(f) + try: + # The reducer can fail, but it will typically output an exception file, so we can ignore the exit code. + subprocess_util.run(cmd, verbose=True, check_exit_code=False) + finally: + gflogging.pop_stream_for_logging() + + return final_shader + + +def run_reduction( + test_dir_reduction_output: Path, + test_dir_to_reduce: Path, + shader_job_name_to_reduce: str, + extension_to_reduce: str, + preserve_semantics: bool, + binary_manager: binaries_util.BinaryManager, + reduction_name: str = "reduction1", +) -> Path: + test = test_util.metadata_read(test_dir_to_reduce) + + check( + bool(test.device and test.device.name), + AssertionError( + f"Cannot reduce {str(test_dir_to_reduce)}; " + f"device must be specified in {str(test_util.get_metadata_path(test_dir_to_reduce))}" + ), + ) + + check( + bool(test.crash_signature), + AssertionError( + f"Cannot reduce {str(test_dir_to_reduce)} because there is no crash string specified." + ), + ) + + # E.g. reports/crashes/no_signature/d50c96e8_opt_rand2_test_phone_ABC/results/phone_ABC/reductions/1 + # Will contain work/ and source/ + reduced_test_dir = test_util.get_reduced_test_dir( + test_dir_reduction_output, test.device.name, reduction_name + ) + + source_dir = test_util.get_source_dir(test_dir_to_reduce) + output_dir = test_util.get_reduction_work_directory( + reduced_test_dir, shader_job_name_to_reduce + ) + if preserve_semantics: + final_shader_path = run_spirv_reduce_or_shrink( + source_dir=source_dir, + name_of_shader_job_to_reduce=shader_job_name_to_reduce, + extension_to_reduce=extension_to_reduce, + output_dir=output_dir, + preserve_semantics=preserve_semantics, + binary_manager=binary_manager, + ) + else: + final_shader_path = run_spirv_reduce_or_shrink( + source_dir=source_dir, + name_of_shader_job_to_reduce=shader_job_name_to_reduce, + extension_to_reduce=extension_to_reduce, + output_dir=output_dir, + preserve_semantics=preserve_semantics, + binary_manager=binary_manager, + ) + + check( + final_shader_path.exists(), + ReductionFailedError("Reduction failed.", reduction_name, output_dir), + ) + + # Finally, create the source_dir so the returned directory can be used as a test_dir. + + # Copy the original source directory. + util.copy_dir(source_dir, test_util.get_source_dir(reduced_test_dir)) + + # And then replace the shader. + + # Destination file. E.g. reductions/source/variant/shader.frag.spv + output_shader_prefix = ( + test_util.get_source_dir(reduced_test_dir) + / shader_job_name_to_reduce + / test_util.SHADER_JOB + ).with_suffix(extension_to_reduce + shader_job_util.SUFFIX_SPIRV) + + util.copy_file( + final_shader_path.with_suffix(shader_job_util.SUFFIX_SPIRV), + output_shader_prefix.with_suffix(shader_job_util.SUFFIX_SPIRV), + ) + + if preserve_semantics: + util.copy_file( + final_shader_path.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS), + output_shader_prefix.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS), + ) + + util.copy_file( + final_shader_path.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS_JSON), + output_shader_prefix.with_suffix( + shader_job_util.SUFFIX_TRANSFORMATIONS_JSON + ), + ) + + return reduced_test_dir + + +def run_reduction_on_report( # pylint: disable=too-many-locals; + test_dir: Path, reports_dir: Path, binary_manager: binaries_util.BinaryManager +) -> None: + test = test_util.metadata_read(test_dir) + + check( + bool(test.device and test.device.name), + AssertionError( + f"Cannot reduce {str(test_dir)}; " + f"device must be specified in {str(test_util.get_metadata_path(test_dir))}" + ), + ) + + check( + bool(test.crash_signature), + AssertionError( + f"Cannot reduce {str(test_dir)} because there is no crash string specified." + ), + ) + + source_dir = test_util.get_source_dir(test_dir) + + shader_jobs = tool.get_shader_jobs(source_dir) + + # TODO: if needed, this could become a parameter to this function. + shader_job_to_reduce = shader_jobs[0] + + if len(shader_jobs) > 1: + check( + len(shader_jobs) == 2 and shader_jobs[1].name == test_util.VARIANT_DIR, + AssertionError( + "Can only reduce tests with shader jobs reference and variant, or just variant." + ), + ) + shader_job_to_reduce = shader_jobs[1] + + shader_transformation_suffixes = shader_job_util.get_related_suffixes_that_exist( + shader_job_to_reduce.shader_job, + language_suffix=(shader_job_util.SUFFIX_TRANSFORMATIONS,), + ) + + shader_spv_suffixes = shader_job_util.get_related_suffixes_that_exist( + shader_job_to_reduce.shader_job, language_suffix=(shader_job_util.SUFFIX_SPIRV,) + ) + + try: + reduced_test = test_dir + + for index, suffix in enumerate(shader_transformation_suffixes): + # E.g. .frag.transformations -> .frag + extension_to_reduce = str(Path(suffix).with_suffix("")) + reduced_test = run_reduction( + test_dir_reduction_output=test_dir, + test_dir_to_reduce=reduced_test, + shader_job_name_to_reduce=shader_job_to_reduce.name, + extension_to_reduce=extension_to_reduce, + preserve_semantics=True, + binary_manager=binary_manager, + reduction_name=f"0_{index}_{suffix.split('.')[1]}", + ) + + if test.crash_signature != signature_util.BAD_IMAGE_SIGNATURE: + for index, suffix in enumerate(shader_spv_suffixes): + # E.g. .frag.spv -> .frag + extension_to_reduce = str(Path(suffix).with_suffix("")) + reduced_test = run_reduction( + test_dir_reduction_output=test_dir, + test_dir_to_reduce=reduced_test, + shader_job_name_to_reduce=shader_job_to_reduce.name, + extension_to_reduce=extension_to_reduce, + preserve_semantics=False, + binary_manager=binary_manager, + reduction_name=f"1_{index}_{suffix.split('.')[1]}", + ) + + device_name = test.device.name + + # Create a symlink to the "best" reduction. + best_reduced_test_link = test_util.get_reduced_test_dir( + test_dir, device_name, fuzz.BEST_REDUCTION_NAME + ) + util.make_directory_symlink( + new_symlink_file_path=best_reduced_test_link, existing_dir=reduced_test + ) + except ReductionFailedError as ex: + # Create a symlink to the failed reduction so it is easy to investigate failed reductions. + link_to_failed_reduction_path = ( + reports_dir / "failed_reductions" / f"{test_dir.name}_{ex.reduction_name}" + ) + util.make_directory_symlink( + new_symlink_file_path=link_to_failed_reduction_path, + existing_dir=ex.reduction_work_dir, + ) + + +def handle_test( + test_dir: Path, + reports_dir: Path, + active_devices: List[Device], + binary_manager: binaries_util.BinaryManager, + settings: Settings, +) -> bool: + report_paths: List[Path] = [] + issue_found = False + + # Run on all devices. + for device in active_devices: + status = run(test_dir, binary_manager, device) + if status in ( + fuzz.STATUS_CRASH, + fuzz.STATUS_TOOL_CRASH, + fuzz.STATUS_UNRESPONSIVE, + ): + issue_found = True + if status == fuzz.STATUS_TOOL_CRASH: + # No need to run further on real devices if the pre-processing step failed. + break + + # For each device that saw a crash, copy the test to reports_dir, adding the signature and device info to the test + # metadata. + for device in active_devices: + report_dir = fuzz_glsl_test.maybe_add_report( + test_dir, reports_dir, device, settings + ) + if report_dir: + report_paths.append(report_dir) + + # For each report, run a reduction on the target device with the device-specific crash signature. + for test_dir_in_reports in report_paths: + if fuzz_glsl_test.should_reduce_report(settings, test_dir_in_reports): + run_reduction_on_report( + test_dir_in_reports, reports_dir, binary_manager=binary_manager + ) + else: + log("Skipping reduction due to settings.") + + # For each report, create a summary and reproduce the bug. + for test_dir_in_reports in report_paths: + fuzz.create_summary_and_reproduce(test_dir_in_reports, binary_manager) + + return issue_found + + +def fuzz_spirv( + staging_dir: Path, + reports_dir: Path, + fuzz_failures_dir: Path, + active_devices: List[Device], + spirv_fuzz_shaders: List[Path], + settings: Settings, + binary_manager: binaries_util.BinaryManager, +) -> None: + + staging_name = staging_dir.name + template_source_dir = staging_dir / "source_template" + + # Copy in a randomly chosen reference. + reference_spirv_shader_job = shader_job_util.copy( + random.choice(spirv_fuzz_shaders), + template_source_dir / test_util.REFERENCE_DIR / test_util.SHADER_JOB, + language_suffix=shader_job_util.SUFFIXES_SPIRV_FUZZ_INPUT, + ) + + # TODO: Allow using downloaded spirv-fuzz. + try: + with util.file_open_text(staging_dir / "log.txt", "w") as log_file: + try: + gflogging.push_stream_for_logging(log_file) + spirv_fuzz_util.run_generate_on_shader_job( + binary_manager.get_binary_path_by_name("spirv-fuzz").path, + reference_spirv_shader_job, + template_source_dir / test_util.VARIANT_DIR / test_util.SHADER_JOB, + seed=str(random.getrandbits(spirv_fuzz_util.GENERATE_SEED_BITS)), + ) + finally: + gflogging.pop_stream_for_logging() + except subprocess.CalledProcessError: + util.mkdirs_p(fuzz_failures_dir) + if len(list(fuzz_failures_dir.iterdir())) < settings.maximum_fuzz_failures: + util.copy_dir(staging_dir, fuzz_failures_dir / staging_dir.name) + return + + test_dirs = [ + make_test( + template_source_dir, + staging_dir / f"{staging_name}_no_opt_test", + spirv_opt_args=None, + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_O_test", + spirv_opt_args=["-O"], + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_Os_test", + spirv_opt_args=["-Os"], + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_rand1_test", + spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_rand2_test", + spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), + binary_manager=binary_manager, + ), + make_test( + template_source_dir, + staging_dir / f"{staging_name}_opt_rand3_test", + spirv_opt_args=spirv_opt_util.random_spirv_opt_args(), + binary_manager=binary_manager, + ), + ] + + for test_dir in test_dirs: + if handle_test(test_dir, reports_dir, active_devices, binary_manager, settings): + # If we generated a report, don't bother trying other optimization combinations. + break diff --git a/gfauto/gfauto/generate_cts_test_template.py b/gfauto/gfauto/generate_cts_test_template.py new file mode 100644 index 000000000..59c56d488 --- /dev/null +++ b/gfauto/gfauto/generate_cts_test_template.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate a CTS test. + +This module/script is copied next to a specific test in your repository of bugs +to generate an Amber script test suitable for adding to the CTS. +In particular, the Amber script test is suitable for use with |add_amber_tests_to_cts.py|. +""" + +import sys +from pathlib import Path + +from gfauto import tool, util + + +def main() -> None: + + # Checklist: + # - check output_amber + # - check short_description + # - check comment_text + # - check copyright_year + # - check extra_commands + + bug_dir = util.norm_path(Path(__file__).absolute()).parent + + tool.glsl_shader_job_crash_to_amber_script_for_google_cts( + source_dir=bug_dir / "reduced_manual", + output_amber=bug_dir / "name-of-test-TODO.amber", + work_dir=bug_dir / "work", + # One sentence, 58 characters max., no period, no line breaks. + short_description="A fragment shader with TODO", + comment_text="""The test passes because TODO""", + copyright_year="2019", + extra_commands=tool.AMBER_COMMAND_EXPECT_RED, + ) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/gfauto/gfauto/gerrit_util.py b/gfauto/gfauto/gerrit_util.py new file mode 100644 index 000000000..a176fd8e7 --- /dev/null +++ b/gfauto/gfauto/gerrit_util.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gerrit utility module. + +Provides functions for interacting with Gerrit via its REST API. +""" + +import json +from enum import Enum +from pathlib import Path +from typing import Any, Dict, Optional + +import requests +from dateutil.parser import parse as parse_date +from requests import Response + +from gfauto import util +from gfauto.gflogging import log +from gfauto.util import check + +KHRONOS_GERRIT_URL = "https://gerrit.khronos.org" + +KHRONOS_GERRIT_LOGIN_PAGE_START = "" + +RESPONSE_PREFIX = ")]}'\n" + + +class BadCookieError(Exception): + pass + + +def gerrit_get_stream( + url: str, path: str, params: Optional[Dict[str, str]], cookie: str +) -> Response: + return requests.get( + url + path, params=params, stream=True, cookies={"GerritAccount": cookie} + ) + + +def gerrit_get(url: str, path: str, params: Dict[str, str], cookie: str) -> Any: + response = requests.get( + url + path, params=params, cookies={"GerritAccount": cookie} + ).text + + check( + response.startswith(RESPONSE_PREFIX), + AssertionError(f"Unexpected response from Gerrit: {response}"), + ) + response = util.remove_start(response, RESPONSE_PREFIX) + return json.loads(response) + + +def find_latest_change(changes: Any) -> Any: + check( + len(changes) > 0, AssertionError(f"Expected at least one CL but got: {changes}") + ) + + # Find the latest submit date (the default order is based on when the CL was last updated). + + latest_change = changes[0] + latest_date = parse_date(latest_change["submitted"]) + + for i in range(1, len(changes)): + change = changes[i] + submitted_date = parse_date(change["submitted"]) + if submitted_date > latest_date: + latest_change = change + latest_date = submitted_date + + return latest_change + + +def get_latest_deqp_change(cookie: str) -> Any: + log("Getting latest deqp change") + changes = gerrit_get( + KHRONOS_GERRIT_URL, + "/changes/", + params={"q": "project:vk-gl-cts status:merged branch:master", "n": "1000"}, + cookie=cookie, + ) + + return find_latest_change(changes) + + +def get_gerrit_change_details(change_number: str, cookie: str) -> Any: + log(f"Getting change details for change number: {change_number}") + return gerrit_get( + KHRONOS_GERRIT_URL, + f"/changes/{change_number}/detail", + params={"O": "10004"}, + cookie=cookie, + ) + + +class DownloadType(Enum): + """For download_gerrit_revision, specifies what to download.""" + + # Downloads the entire repo as a .tgz file. + Archive = "archive" + + # Downloads the patch in a .zip file. + Patch = "patch" + + +def download_gerrit_revision( + output_path: Path, + change_number: str, + revision: str, + download_type: DownloadType, + cookie: str, +) -> Path: + path = f"/changes/{change_number}/revisions/{revision}/{download_type.value}" + log(f"Downloading revision from: {path}\n to: {str(output_path)}") + + params = {"format": "tgz"} if download_type == DownloadType.Archive else {"zip": ""} + + response = gerrit_get_stream(KHRONOS_GERRIT_URL, path, params=params, cookie=cookie) + + counter = 0 + with util.file_open_binary(output_path, "wb") as output_stream: + for chunk in response.iter_content(chunk_size=None): + log(".", skip_newline=True) + counter += 1 + if counter > 80: + counter = 0 + log("") # new line + output_stream.write(chunk) + log("") # new line + + with util.file_open_text(output_path, "r") as input_stream: + line = input_stream.readline(len(KHRONOS_GERRIT_LOGIN_PAGE_START) * 2) + if line.startswith(KHRONOS_GERRIT_LOGIN_PAGE_START) or line.startswith( + "Not found" + ): + raise BadCookieError() + + return output_path + + +def get_deqp_graphicsfuzz_pending_changes(cookie: str) -> Any: + return gerrit_get( + KHRONOS_GERRIT_URL, + "/changes/", + params={ + "q": "project:vk-gl-cts status:pending branch:master dEQP-VK.graphicsfuzz.", + "n": "1000", + }, + cookie=cookie, + ) diff --git a/gfauto/gfauto/gfauto_interestingness_test.py b/gfauto/gfauto/gfauto_interestingness_test.py new file mode 100644 index 000000000..ba48f1f6e --- /dev/null +++ b/gfauto/gfauto/gfauto_interestingness_test.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Interestingness test for gfauto. + +When running a reducer, this module is used as the interestingness test. +An interestingness test runs a test on a device and returns 0 if the test +still exposes the behavior of interest (e.g. a crash, an incorrectly rendered image). + +See |setup.py| to see how this module is added to the entry_points/console_scripts. +""" + +import argparse +import sys +from pathlib import Path +from typing import List, Optional, Tuple + +from gfauto import ( + binaries_util, + fuzz, + fuzz_glsl_test, + result_util, + settings_util, + signature_util, + test_util, + tool, + util, +) +from gfauto.gflogging import log +from gfauto.settings_pb2 import Settings + +# TODO: Could we make the interestingness test the only way of running a shader job (or indeed, any test)? +# We would want to pass the output directory (default is one will be created, as is currently the case), the test_json, +# no crash signature nor device (we can get that from the test_json), although perhaps these could be overridden? +# A device (or test?) could then even specify a custom interestingness command, although the default one would probably +# be the same for all devices and it would look at the device info in the test_json? + +# TODO: Maybe add helper method and throw exceptions instead of calling sys.exit. + + +def main() -> None: # pylint: disable=too-many-statements, too-many-locals, too-many-branches; + parser = argparse.ArgumentParser( + description="Interestingness test that runs a test using Amber, " + "calculates the crash signature based on the result, and returns 0 " + "if the signature matches the expected crash signature." + ) + + parser.add_argument( + "source_dir", + help="The source directory containing the shaders and the test.json file that describes how to run the test.", + ) + parser.add_argument( + "--override_shader_job", + nargs=2, + metavar=("shader_job_name", "shader_job_json"), + help='Override one of the shader jobs. E.g.: "--override_shader_job variant temp/variant.json". Note that ' + "the output directory will be set to shader_job_json/ (with the .json extension removed) by default in this case. ", + ) + + parser.add_argument( + "--override_shader", + nargs=3, + metavar=("shader_name", "suffix", "shader_path"), + help='Override one of the shaders. E.g.: "--override_shader variant .frag.spv temp/my_shader.spv". Note that ' + "the output directory will be set to shader_path/ (with the .spv extension removed) by default in this case. ", + ) + + parser.add_argument( + "--use_default_binaries", + help="Use the latest binaries, ignoring those defined in the test.json. " + "Implies --fallback_binaries. Passing --settings is recommended to ensure the latest binaries are used.", + action="store_true", + ) + + parser.add_argument( + "--fallback_binaries", + help="Fallback to the latest binaries if they are not defined in the test.json.", + action="store_true", + ) + + parser.add_argument( + "--output", + help="Output directory. Required unless --override_shader[_job] is used; see --override_shader[_job] for details.", + default=None, + ) + + parser.add_argument( + "--settings", + help="Path to a settings JSON file for this instance. " + "Unlike with gfauto_fuzz, the default value is an empty string, which is ignored. " + "You only need to use a settings file if you pass --use_default_binaries and you want to use the latest binary versions. " + 'In this case, use e.g. "--settings settings.json" so that a default settings file is generated with the latest binary version numbers ' + "and then run gfauto_interestingness_test again to use those latest binaries.", + default="", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + source_dir: Path = Path(parsed_args.source_dir) + override_shader_job: Optional[Tuple[str, str]] = parsed_args.override_shader_job + override_shader: Optional[Tuple[str, str, str]] = parsed_args.override_shader + settings_str: str = parsed_args.settings + + settings = Settings() + if settings_str: + settings = settings_util.read_or_create(Path(settings_str)) + + use_default_binaries: bool = parsed_args.use_default_binaries + fallback_binaries: bool = parsed_args.fallback_binaries or use_default_binaries + output: Path + if parsed_args.output: + output = Path(parsed_args.output) + elif override_shader_job: + output = Path(override_shader_job[1]).with_suffix("") + elif override_shader: + output = Path(override_shader[2]).with_suffix("") + else: + raise AssertionError("Need --output or --override_shader[_job] parameter.") + + binary_manager = binaries_util.get_default_binary_manager(settings=settings) + + if not fallback_binaries: + binary_manager = binary_manager.get_child_binary_manager(binary_list=[]) + + shader_job_overrides: List[tool.NameAndShaderJob] = [] + + if override_shader_job: + shader_job_overrides.append( + tool.NameAndShaderJob( + name=override_shader_job[0], shader_job=Path(override_shader_job[1]) + ) + ) + + shader_overrides: tool.ShaderJobNameToShaderOverridesMap = {} + + if override_shader: + override = tool.ShaderPathWithNameAndSuffix( + name=override_shader[0], + suffix=override_shader[1], + path=Path(override_shader[2]), + ) + shader_overrides[override.name] = {override.suffix: override} + + # E.g. shader_overrides == + # { + # "variant": { + # ".frag.spv": ShaderPathWithNameAndSuffix("variant", ".frag.spv", Path("path/to/shader.frag.spv")) + # } + # } + + # We don't need to read this to run the shader, but we need it afterwards anyway. + test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) + + output_dir = fuzz_glsl_test.run_shader_job( + source_dir=source_dir, + output_dir=output, + binary_manager=binary_manager, + test=test, + ignore_test_and_device_binaries=use_default_binaries, + shader_job_overrides=shader_job_overrides, + shader_job_shader_overrides=shader_overrides, + ) + + log( + f"gfauto_interestingness_test: finished running {str(source_dir)} in {str(output_dir)}." + ) + + if override_shader_job: + log( + f"The {override_shader_job[0]} shader was overridden with {override_shader_job[1]}" + ) + + status = result_util.get_status(output_dir) + if test.expected_status: + log("") + log(f"Expected status: {test.expected_status}") + log(f"Actual status: {status}") + + log_contents = util.file_read_text(result_util.get_log_path(output_dir)) + signature = signature_util.get_signature_from_log_contents(log_contents) + + log("") + log(f"Expected signature: {test.crash_signature}") + log(f"Actual signature: {signature}") + + log("") + + if test.expected_status: + if status != test.expected_status: + log("status != expected_status; not interesting") + sys.exit(1) + else: + # There is no expected status given, so just assume it needs to be one of the "bad" statuses: + if status not in ( + fuzz.STATUS_CRASH, + fuzz.STATUS_TOOL_CRASH, + fuzz.STATUS_UNRESPONSIVE, + ): + log("shader run did not fail; not interesting.") + sys.exit(1) + + if signature != test.crash_signature: + log("signature != crash_signature; not interesting") + sys.exit(1) + + log("Interesting!") + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/gfauto/gfauto/gflogging.py b/gfauto/gfauto/gflogging.py new file mode 100644 index 000000000..e181d10d5 --- /dev/null +++ b/gfauto/gfauto/gflogging.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Logging utility for gfauto. + +Use to log to stdout and/or to a file. Print statements are banned in gfauto. +""" + +from pathlib import Path +from typing import List, TextIO + +from gfauto import util + +_LOG_TO_STDOUT = True +_LOG_TO_STREAM: List[TextIO] = [] + + +def push_stream_for_logging(stream: TextIO) -> None: + _LOG_TO_STREAM.append(stream) + + +def pop_stream_for_logging() -> None: + _LOG_TO_STREAM.pop() + + +def log(message: str, skip_newline: bool = False) -> None: + if _LOG_TO_STDOUT: + if not skip_newline: + print(message, flush=True) # noqa: T001 + else: + print(message, end="", flush=True) # noqa: T001 + for stream in _LOG_TO_STREAM: + stream.write(message) + if not skip_newline: + stream.write("\n") + stream.flush() + + +def log_a_file(log_file: Path) -> None: + log(f"Logging the contents of {str(log_file)}") + try: + log(util.file_read_text(log_file)) + except IOError: + log(f"Failed to read {str(log_file)}") diff --git a/gfauto/gfauto/glsl_generate_util.py b/gfauto/gfauto/glsl_generate_util.py new file mode 100644 index 000000000..5153be344 --- /dev/null +++ b/gfauto/gfauto/glsl_generate_util.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""glsl-generate utility module. + +Generates GLSL shaders using GraphicsFuzz. +""" + +import pathlib +from pathlib import Path +from typing import List, Optional + +from gfauto import subprocess_util, util + +GENERATE_SEED_BITS = 64 + + +def run_prepare_reference( + graphicsfuzz_tool_path: Path, + input_reference_shader_json: Path, + output_reference_shader_json: Path, +) -> Path: + util.file_mkdirs_parent(output_reference_shader_json) + cmd = [ + str(graphicsfuzz_tool_path), + "com.graphicsfuzz.generator.tool.PrepareReference", + "--generate-uniform-bindings", + "--max-uniforms", + "10", + str(input_reference_shader_json), + str(output_reference_shader_json), + ] + + subprocess_util.run(cmd) + + return output_reference_shader_json + + +def run_generate( + graphicsfuzz_tool_path: Path, + reference_shader_json: pathlib.Path, + donors_path: pathlib.Path, + output_shader_json: pathlib.Path, + seed: Optional[str] = None, + other_args: Optional[List[str]] = None, +) -> pathlib.Path: + util.file_mkdirs_parent(output_shader_json) + cmd = [ + str(graphicsfuzz_tool_path), + "com.graphicsfuzz.generator.tool.Generate", + str(reference_shader_json), + str(donors_path), + str(output_shader_json), + "--generate-uniform-bindings", + "--max-uniforms", + "10", + ] + + if seed: + cmd.append(f"--seed={seed}") + + if other_args: + cmd.extend(other_args) + + subprocess_util.run(cmd) + + return output_shader_json diff --git a/gfauto/gfauto/glslang_validator_util.py b/gfauto/gfauto/glslang_validator_util.py new file mode 100644 index 000000000..7407f4305 --- /dev/null +++ b/gfauto/gfauto/glslang_validator_util.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""glslangValidator utility module. + +Runs glslangValidator on GLSL shader jobs to get SPIR-V shader jobs. +""" + +import pathlib +from typing import Optional + +from gfauto import binaries_util, shader_job_util, subprocess_util, util + +GLSLANG_DEFAULT_TIME_LIMIT = 120 + + +def run_glslang_glsl_shader_to_spirv_shader( + glsl_shader_path: pathlib.Path, + output_dir_path: pathlib.Path, + glslang_validator_file_path: Optional[pathlib.Path] = None, + time_limit: int = GLSLANG_DEFAULT_TIME_LIMIT, +) -> pathlib.Path: + + if not glslang_validator_file_path: + glslang_validator_file_path = util.tool_on_path( + binaries_util.GLSLANG_VALIDATOR_NAME + ) + + output_spirv_file_path = output_dir_path / (glsl_shader_path.name + ".spv") + + util.file_mkdirs_parent(output_spirv_file_path) + + subprocess_util.run( + util.prepend_catchsegv_if_available( + [ + str(glslang_validator_file_path), + "-V", + "-o", + str(output_spirv_file_path), + str(glsl_shader_path), + ] + ), + timeout=time_limit, + ) + + return output_spirv_file_path + + +def run_glslang_glsl_to_spirv_job( + glsl_shader_job_json_file_path: pathlib.Path, + spirv_shader_job_json_file_path: pathlib.Path, + glslang_validator_file_path: Optional[pathlib.Path] = None, +) -> pathlib.Path: + + if not glslang_validator_file_path: + glslang_validator_file_path = util.tool_on_path( + binaries_util.GLSLANG_VALIDATOR_NAME + ) + + glsl_shader_files = shader_job_util.get_related_files( + glsl_shader_job_json_file_path + ) + + util.copy_file(glsl_shader_job_json_file_path, spirv_shader_job_json_file_path) + + for glsl_shader_file in glsl_shader_files: + run_glslang_glsl_shader_to_spirv_shader( + glsl_shader_file, + spirv_shader_job_json_file_path.parent, + glslang_validator_file_path, + ) + + return spirv_shader_job_json_file_path diff --git a/gfauto/gfauto/host_device_util.py b/gfauto/gfauto/host_device_util.py new file mode 100644 index 000000000..ab6c2f8c9 --- /dev/null +++ b/gfauto/gfauto/host_device_util.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Host device utility module. + +Used to run Amber on the host machine. +""" + +import subprocess +from pathlib import Path +from typing import Dict, Optional + +from gfauto import ( + devices_util, + fuzz, + gflogging, + result_util, + subprocess_util, + types, + util, +) +from gfauto.gflogging import log + + +def get_driver_details(amber_path: Path, icd: Optional[Path] = None) -> str: + + env: Optional[Dict[str, str]] = None + + if icd: + env = {"VK_ICD_FILENAMES": str(icd)} + + try: + result = subprocess_util.run( + [str(amber_path), "-d", "-V"], + timeout=fuzz.AMBER_RUN_TIME_LIMIT, + verbose=True, + env=env, + ) + except subprocess.SubprocessError as ex: + raise devices_util.GetDeviceDetailsError() from ex + + match = devices_util.AMBER_DEVICE_DETAILS_PATTERN.search(result.stdout) + + if not match: + raise devices_util.GetDeviceDetailsError( + "Could not find device details in stdout: " + result.stdout + ) + + return match.group(1) + + +def run_amber( + amber_script_file: Path, + output_dir: Path, + dump_image: bool, + dump_buffer: bool, + amber_path: Path, + skip_render: bool = False, + debug_layers: bool = False, + icd: Optional[Path] = None, +) -> Path: + with util.file_open_text( + result_util.get_amber_log_path(output_dir), "w" + ) as log_file: + try: + gflogging.push_stream_for_logging(log_file) + + run_amber_helper( + amber_script_file, + output_dir, + dump_image, + dump_buffer, + amber_path, + skip_render, + debug_layers, + icd, + ) + finally: + gflogging.pop_stream_for_logging() + + return output_dir + + +def run_amber_helper( + amber_script_file: Path, + output_dir: Path, + dump_image: bool, + dump_buffer: bool, + amber_path: Path, + skip_render: bool = False, + debug_layers: bool = False, + icd: Optional[Path] = None, +) -> Path: + + variant_image_file = output_dir / fuzz.VARIANT_IMAGE_FILE_NAME + reference_image_file = output_dir / fuzz.REFERENCE_IMAGE_FILE_NAME + buffer_file = output_dir / fuzz.BUFFER_FILE_NAME + + cmd = [ + str(amber_path), + str(amber_script_file), + "--log-graphics-calls-time", + "--disable-spirv-val", + ] + + if not debug_layers: + cmd.append("-d") + + if skip_render: + # -ps tells amber to stop after pipeline creation + cmd.append("-ps") + else: + if dump_image: + cmd.append("-I") + cmd.append("variant_framebuffer") + cmd.append("-i") + cmd.append(str(variant_image_file)) + cmd.append("-I") + cmd.append("reference_framebuffer") + cmd.append("-i") + cmd.append(str(reference_image_file)) + if dump_buffer: + cmd.append("-b") + cmd.append(str(buffer_file)) + cmd.append("-B") + cmd.append("0") + + cmd = util.prepend_catchsegv_if_available(cmd) + + status = "UNEXPECTED_ERROR" + + result: Optional[types.CompletedProcess] = None + env: Optional[Dict[str, str]] = None + + if icd: + env = {"VK_ICD_FILENAMES": str(icd)} + + try: + result = subprocess_util.run( + cmd, + check_exit_code=False, + timeout=fuzz.AMBER_RUN_TIME_LIMIT, + verbose=True, + env=env, + ) + except subprocess.TimeoutExpired: + status = fuzz.STATUS_TIMEOUT + + if result: + if result.returncode != 0: + status = fuzz.STATUS_CRASH + else: + status = fuzz.STATUS_SUCCESS + + log("\nSTATUS " + status + "\n") + + util.file_write_text(result_util.get_status_path(output_dir), status) + + return output_dir diff --git a/gfauto/gfauto/proto_util.py b/gfauto/gfauto/proto_util.py new file mode 100644 index 000000000..cf352eb73 --- /dev/null +++ b/gfauto/gfauto/proto_util.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Protobuf utility module. + +Used to read/write protobuf data structures from/to JSON text and from/to JSON files. +""" + +from pathlib import Path +from typing import TypeVar + +from google.protobuf import json_format +from google.protobuf.message import Message + +from gfauto import util + +# pylint: disable=invalid-name; Generic type variable names are usually one letter. +M = TypeVar("M", bound=Message) # noqa + + +def json_to_message(json: str, message: M) -> M: + json_format.Parse(json, message, ignore_unknown_fields=True) + return message + + +def message_to_json( + message: Message, including_default_value_fields: bool = True +) -> str: + return json_format.MessageToJson( + message, + including_default_value_fields=including_default_value_fields, + preserving_proto_field_name=True, + sort_keys=True, + ) + + +def message_to_file( + message: Message, + output_file_path: Path, + including_default_value_fields: bool = True, +) -> Path: + message_json = message_to_json(message, including_default_value_fields) + util.file_write_text(output_file_path, message_json) + return output_file_path + + +def file_to_message(input_file_path: Path, message: M) -> M: + message_json = util.file_read_text(input_file_path) + return json_to_message(message_json, message) diff --git a/gfauto/gfauto/recipe.proto b/gfauto/gfauto/recipe.proto new file mode 100644 index 000000000..274c29679 --- /dev/null +++ b/gfauto/gfauto/recipe.proto @@ -0,0 +1,36 @@ +// Copyright 2019 The GraphicsFuzz Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package gfauto; + +import "gfauto/common.proto"; + + +// An artifact is a directory containing an ArtifactMetadata proto stored in "artifact.json" and/or a Recipe proto +// stored in "recipe.json", plus any other files and directories within. The recipe is executed to produce the files +// that make up the artifact, which must include the "artifact.json" file. +// See artifact.proto. +message Recipe { + + oneof recipe { + RecipeDownloadAndExtractArchiveSet download_and_extract_archive_set = 1; + } +} + +// Downloads and extracts one or more archives and makes the binaries available. +message RecipeDownloadAndExtractArchiveSet { + ArchiveSet archive_set = 1; +} diff --git a/gfauto/gfauto/recipe_download_and_extract_archive_set.py b/gfauto/gfauto/recipe_download_and_extract_archive_set.py new file mode 100644 index 000000000..fc50b8f0a --- /dev/null +++ b/gfauto/gfauto/recipe_download_and_extract_archive_set.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Download and extract archive set recipe. + +Implements the RecipeDownloadAndExtractArchiveSet recipe. +See recipe.proto. +""" + +import os +import shutil +import stat +import urllib.request +from zipfile import ZipFile, ZipInfo + +from gfauto import artifact_util, util +from gfauto.artifact_pb2 import ArtifactMetadata +from gfauto.common_pb2 import Archive +from gfauto.gflogging import log +from gfauto.recipe_pb2 import RecipeDownloadAndExtractArchiveSet + +ALL_EXECUTABLE_PERMISSION_BITS = ( + stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH # noqa: SC200 +) + + +def recipe_download_and_extract_archive_set( + recipe: RecipeDownloadAndExtractArchiveSet, output_artifact_path: str +) -> None: + + for archive in recipe.archive_set.archives: # type: Archive + util.check_field_truthy(archive.url, "url") + util.check_field_truthy(archive.output_file, "output_file") + util.check_field_truthy(archive.output_directory, "output_directory") + + output_file_path = artifact_util.artifact_get_inner_file_path( + archive.output_file, output_artifact_path + ) + + output_directory_path = artifact_util.artifact_get_inner_file_path( + archive.output_directory, output_artifact_path + ) + + log(f"Downloading {archive.url} to {str(output_file_path)}") + urllib.request.urlretrieve(archive.url, str(output_file_path)) + + if output_file_path.name.lower().endswith(".zip"): + with ZipFile(str(output_file_path), "r") as zip_file: + for info in zip_file.infolist(): # type: ZipInfo + extracted_file = zip_file.extract(info, str(output_directory_path)) + # If the file was created on a UNIX-y system: + if info.create_system == 3: + # Shift away first 2 bytes to get permission bits. + zip_file_exec_bits = info.external_attr >> 16 + # Just consider the executable bits. + zip_file_exec_bits = ( + zip_file_exec_bits & ALL_EXECUTABLE_PERMISSION_BITS + ) + current_attribute_bits = os.stat(extracted_file).st_mode + if ( + current_attribute_bits | zip_file_exec_bits + ) != current_attribute_bits: + os.chmod( + extracted_file, + current_attribute_bits | zip_file_exec_bits, + ) + else: + shutil.unpack_archive(str(output_file_path), str(output_directory_path)) + + output_metadata = ArtifactMetadata() + output_metadata.data.extracted_archive_set.archive_set.CopyFrom(recipe.archive_set) + + artifact_util.artifact_write_metadata(output_metadata, output_artifact_path) diff --git a/gfauto/gfauto/recipe_pb2.py b/gfauto/gfauto/recipe_pb2.py new file mode 100644 index 000000000..f962362c1 --- /dev/null +++ b/gfauto/gfauto/recipe_pb2.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gfauto/recipe.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from gfauto import common_pb2 as gfauto_dot_common__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='gfauto/recipe.proto', + package='gfauto', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x13gfauto/recipe.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\"j\n\x06Recipe\x12V\n download_and_extract_archive_set\x18\x01 \x01(\x0b\x32*.gfauto.RecipeDownloadAndExtractArchiveSetH\x00\x42\x08\n\x06recipe\"M\n\"RecipeDownloadAndExtractArchiveSet\x12\'\n\x0b\x61rchive_set\x18\x01 \x01(\x0b\x32\x12.gfauto.ArchiveSetb\x06proto3') + , + dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,]) + + + + +_RECIPE = _descriptor.Descriptor( + name='Recipe', + full_name='gfauto.Recipe', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='download_and_extract_archive_set', full_name='gfauto.Recipe.download_and_extract_archive_set', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='recipe', full_name='gfauto.Recipe.recipe', + index=0, containing_type=None, fields=[]), + ], + serialized_start=52, + serialized_end=158, +) + + +_RECIPEDOWNLOADANDEXTRACTARCHIVESET = _descriptor.Descriptor( + name='RecipeDownloadAndExtractArchiveSet', + full_name='gfauto.RecipeDownloadAndExtractArchiveSet', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='archive_set', full_name='gfauto.RecipeDownloadAndExtractArchiveSet.archive_set', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=160, + serialized_end=237, +) + +_RECIPE.fields_by_name['download_and_extract_archive_set'].message_type = _RECIPEDOWNLOADANDEXTRACTARCHIVESET +_RECIPE.oneofs_by_name['recipe'].fields.append( + _RECIPE.fields_by_name['download_and_extract_archive_set']) +_RECIPE.fields_by_name['download_and_extract_archive_set'].containing_oneof = _RECIPE.oneofs_by_name['recipe'] +_RECIPEDOWNLOADANDEXTRACTARCHIVESET.fields_by_name['archive_set'].message_type = gfauto_dot_common__pb2._ARCHIVESET +DESCRIPTOR.message_types_by_name['Recipe'] = _RECIPE +DESCRIPTOR.message_types_by_name['RecipeDownloadAndExtractArchiveSet'] = _RECIPEDOWNLOADANDEXTRACTARCHIVESET +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Recipe = _reflection.GeneratedProtocolMessageType('Recipe', (_message.Message,), { + 'DESCRIPTOR' : _RECIPE, + '__module__' : 'gfauto.recipe_pb2' + # @@protoc_insertion_point(class_scope:gfauto.Recipe) + }) +_sym_db.RegisterMessage(Recipe) + +RecipeDownloadAndExtractArchiveSet = _reflection.GeneratedProtocolMessageType('RecipeDownloadAndExtractArchiveSet', (_message.Message,), { + 'DESCRIPTOR' : _RECIPEDOWNLOADANDEXTRACTARCHIVESET, + '__module__' : 'gfauto.recipe_pb2' + # @@protoc_insertion_point(class_scope:gfauto.RecipeDownloadAndExtractArchiveSet) + }) +_sym_db.RegisterMessage(RecipeDownloadAndExtractArchiveSet) + + +# @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/recipe_pb2.pyi b/gfauto/gfauto/recipe_pb2.pyi new file mode 100644 index 000000000..654eee0ad --- /dev/null +++ b/gfauto/gfauto/recipe_pb2.pyi @@ -0,0 +1,65 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from gfauto.common_pb2 import ( + ArchiveSet as gfauto___common_pb2___ArchiveSet, +) + +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Optional as typing___Optional, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class Recipe(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + + @property + def download_and_extract_archive_set(self) -> RecipeDownloadAndExtractArchiveSet: ... + + def __init__(self, + *, + download_and_extract_archive_set : typing___Optional[RecipeDownloadAndExtractArchiveSet] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Recipe: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"download_and_extract_archive_set",u"recipe"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"download_and_extract_archive_set",u"recipe"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"download_and_extract_archive_set",b"download_and_extract_archive_set",u"recipe",b"recipe"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"download_and_extract_archive_set",b"download_and_extract_archive_set",u"recipe",b"recipe"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"recipe",b"recipe"]) -> typing_extensions___Literal["download_and_extract_archive_set"]: ... + +class RecipeDownloadAndExtractArchiveSet(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + + @property + def archive_set(self) -> gfauto___common_pb2___ArchiveSet: ... + + def __init__(self, + *, + archive_set : typing___Optional[gfauto___common_pb2___ArchiveSet] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> RecipeDownloadAndExtractArchiveSet: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"archive_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"archive_set"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"archive_set",b"archive_set"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"archive_set",b"archive_set"]) -> None: ... diff --git a/gfauto/gfauto/recipe_wrap.py b/gfauto/gfauto/recipe_wrap.py new file mode 100644 index 000000000..0031c15af --- /dev/null +++ b/gfauto/gfauto/recipe_wrap.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""RecipeWrap class.""" + +import attr + +from gfauto import artifact_util +from gfauto.recipe_pb2 import Recipe + + +@attr.dataclass +class RecipeWrap: + """Wraps a Recipe proto with its path for convenience.""" + + path: str + recipe: Recipe + + def write(self) -> str: + return artifact_util.artifact_write_recipe(self.recipe, self.path) diff --git a/gfauto/gfauto/result_util.py b/gfauto/gfauto/result_util.py new file mode 100644 index 000000000..ff2fecbb3 --- /dev/null +++ b/gfauto/gfauto/result_util.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Result utility module. + +When we run Amber, we write certain files to indicate the result. +This module contains functions for getting the paths of these files. +""" + +from pathlib import Path +from typing import Optional + +from gfauto import util + + +def write_status( + result_output_dir: Path, status: str, bad_shader_name: Optional[str] = None +) -> None: + util.file_write_text(get_status_path(result_output_dir), status) + if bad_shader_name: + util.file_write_text( + get_status_bad_shader_name_path(result_output_dir), bad_shader_name + ) + + +def get_status_path(result_output_dir: Path) -> Path: + return result_output_dir / "STATUS" + + +def get_status_bad_shader_name(result_output_dir: Path) -> str: + bad_shader_name_path = get_status_bad_shader_name_path(result_output_dir) + return util.file_read_text_or_else(bad_shader_name_path, "") + + +def get_status_bad_shader_name_path(result_output_dir: Path) -> Path: + return result_output_dir / "BAD_SHADER" + + +def get_status(result_output_dir: Path) -> str: + status_file = get_status_path(result_output_dir) + return util.file_read_text_or_else(status_file, "UNEXPECTED_ERROR") + + +def get_log_path(result_output_dir: Path) -> Path: + return result_output_dir / "log.txt" + + +def get_amber_log_path(result_dir: Path) -> Path: + return result_dir / "amber_log.txt" diff --git a/gfauto/gfauto/run_bin.py b/gfauto/gfauto/run_bin.py new file mode 100644 index 000000000..cea48e916 --- /dev/null +++ b/gfauto/gfauto/run_bin.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runs a binary from the given binary name and settings file.""" + +import argparse +import subprocess +import sys +from pathlib import Path +from typing import List + +from gfauto import binaries_util, settings_util + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Runs a binary given the binary name and settings.json file." + ) + + parser.add_argument( + "--settings", + help="Path to the settings JSON file for this instance.", + default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), + ) + + parser.add_argument( + "binary_name", + help="The name of the binary to run. E.g. spirv-opt, glslangValidator", + type=str, + ) + + parser.add_argument( + "arguments", + metavar="arguments", + type=str, + nargs="*", + help="The arguments to pass to the binary", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + # Args. + settings_path: Path = Path(parsed_args.settings) + binary_name: str = parsed_args.binary_name + arguments: List[str] = parsed_args.arguments + + settings = settings_util.read_or_create(settings_path) + binary_manager = binaries_util.get_default_binary_manager(settings=settings) + + cmd = [str(binary_manager.get_binary_path_by_name(binary_name).path)] + cmd.extend(arguments) + return subprocess.run(cmd, check=False).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/gfauto/gfauto/run_cts_gf_tests.py b/gfauto/gfauto/run_cts_gf_tests.py new file mode 100644 index 000000000..164d07e46 --- /dev/null +++ b/gfauto/gfauto/run_cts_gf_tests.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runs GraphicsFuzz AmberScript tests.""" + +import argparse +import sys +from pathlib import Path +from subprocess import CalledProcessError, TimeoutExpired +from typing import Optional + +from gfauto import ( + android_device, + binaries_util, + devices_util, + fuzz, + host_device_util, + result_util, + settings_util, + shader_compiler_util, + spirv_opt_util, + util, +) + +DEFAULT_TIMEOUT = 30 + + +def main() -> None: # pylint: disable=too-many-locals,too-many-branches,too-many-statements; + parser = argparse.ArgumentParser( + description="Runs GraphicsFuzz AmberScript tests on the active devices listed in " + "the settings.json file." + ) + + parser.add_argument( + "--settings", + help="Path to the settings JSON file for this instance.", + default=str(settings_util.DEFAULT_SETTINGS_FILE_PATH), + ) + + parser.add_argument( + "--tests", + help="Path to the directory of AmberScript tests with shaders extracted.", + default=str("graphicsfuzz"), + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + # Args. + tests_dir: Path = Path(parsed_args.tests) + settings_path: Path = Path(parsed_args.settings) + + # Settings and devices. + settings = settings_util.read_or_create(settings_path) + active_devices = devices_util.get_active_devices(settings.device_list) + + # Binaries. + binaries = binaries_util.get_default_binary_manager(settings=settings) + + work_dir = Path() / "temp" / f"cts_run_{fuzz.get_random_name()[:8]}" + + util.mkdirs_p(work_dir) + + with util.file_open_text(Path("results.txt"), "w") as log_handle: + + def write_entry(entry: str) -> None: + log_handle.write(entry) + log_handle.write(", ") + log_handle.flush() + + def write_newline() -> None: + log_handle.write("\n") + log_handle.flush() + + spirv_opt_path: Optional[Path] = None + swift_shader_path: Optional[Path] = None + amber_path: Optional[Path] = None + + # Enumerate active devices, writing their name and storing binary paths if needed. + write_entry("test") + for device in active_devices: + + if device.name == "host_preprocessor": + # We are actually just running spirv-opt on the SPIR-V shaders. + write_entry("spirv-opt") + else: + write_entry(device.name) + + if device.HasField("preprocess"): + spirv_opt_path = binaries.get_binary_path_by_name( + binaries_util.SPIRV_OPT_NAME + ).path + + if device.HasField("swift_shader"): + swift_shader_path = binaries.get_binary_path_by_name( + binaries_util.SWIFT_SHADER_NAME + ).path + + if device.HasField("swift_shader") or device.HasField("host"): + amber_path = binaries.get_binary_path_by_name( + binaries_util.AMBER_NAME + ).path + + write_newline() + + # Enumerate tests and devices, writing the results. + + for test in sorted(tests_dir.glob("*.amber")): + test_name = util.remove_end(test.name, ".amber") + write_entry(test_name) + spirv_shaders = sorted( + tests_dir.glob(util.remove_end(test.name, "amber") + "*.spv") + ) + for device in active_devices: + test_run_dir = work_dir / f"{test_name}_{device.name}" + util.mkdirs_p(test_run_dir) + try: + # Confusingly, some functions below will raise on an error; others will write e.g. CRASH to the + # STATUS file in the output directory. In the latter case, we update |status|. We check |status| at + # the end of this if-else chain and raise fake exceptions if appropriate. + status = fuzz.STATUS_SUCCESS + + if device.HasField("preprocess"): + # This just means spirv-op for now. + + assert spirv_opt_path # noqa + for spirv_shader in spirv_shaders: + spirv_opt_util.run_spirv_opt_on_spirv_shader( + spirv_shader, test_run_dir, ["-O"], spirv_opt_path + ) + elif device.HasField("shader_compiler"): + for spirv_shader in spirv_shaders: + shader_compiler_util.run_shader( + shader_compiler_device=device.shader_compiler, + shader_path=spirv_shader, + output_dir=test_run_dir, + compiler_path=binaries.get_binary_path_by_name( + device.shader_compiler.binary + ).path, + timeout=DEFAULT_TIMEOUT, + ) + elif device.HasField("swift_shader"): + assert swift_shader_path # noqa + assert amber_path # noqa + host_device_util.run_amber( + test, + test_run_dir, + amber_path=amber_path, + dump_image=False, + dump_buffer=False, + icd=swift_shader_path, + ) + status = result_util.get_status(test_run_dir) + elif device.HasField("host"): + assert amber_path # noqa + host_device_util.run_amber( + test, + test_run_dir, + amber_path=amber_path, + dump_image=False, + dump_buffer=False, + ) + status = result_util.get_status(test_run_dir) + elif device.HasField("android"): + android_device.run_amber_on_device( + test, + test_run_dir, + dump_image=False, + dump_buffer=False, + serial=device.android.serial, + ) + status = result_util.get_status(test_run_dir) + else: + raise AssertionError(f"Unsupported device {device.name}") + + if status in (fuzz.STATUS_CRASH, fuzz.STATUS_TOOL_CRASH): + raise CalledProcessError(1, "??") + if status != fuzz.STATUS_SUCCESS: + raise TimeoutExpired("??", fuzz.AMBER_RUN_TIME_LIMIT) + + write_entry("P") + except CalledProcessError: + write_entry("F") + except TimeoutExpired: + write_entry("T") + write_newline() + + +if __name__ == "__main__": + main() diff --git a/gfauto/gfauto/settings.proto b/gfauto/gfauto/settings.proto new file mode 100644 index 000000000..e2c8f6416 --- /dev/null +++ b/gfauto/gfauto/settings.proto @@ -0,0 +1,53 @@ +// Copyright 2019 The GraphicsFuzz Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package gfauto; + +import "gfauto/common.proto"; +import "gfauto/device.proto"; + + +// A fuzzing session requires a Settings proto stored in "settings.json". +// Default values are defined in settings_util.py. +message Settings { + // A list of devices, including the list of active devices names that will actually be used in this session. + DeviceList device_list = 1; + + // A list of custom binary versions; this should rarely be set, but can be used, for example, if you want to override + // the default versions of some binaries for all devices. It is probably more common to specify binaries for certain + // devices. + repeated Binary custom_binaries = 2; + + // Allow at most this many duplicate crashes per device. + // If 0 (the protobuf default value), a different default value will actually be used, as determined by gfauto. + uint32 maximum_duplicate_crashes = 3; + + // Store at most this many fuzzing failure files. + uint32 maximum_fuzz_failures = 4; + + // Reduce tool crashes. E.g. spirv-opt or glslangValidator crashes. You can disable this if you would rather focus on + // bugs in your actual test device. + bool reduce_tool_crashes = 5; + + // Reduce crashes. E.g. shader compilation errors, shader compilation crashes, runtime crashes. + bool reduce_crashes = 6; + + // Reduce bad images. E.g. your test device renders an incorrect image. + bool reduce_bad_images = 7; + + // A list of built-in binary versions that will be generated automatically. + repeated Binary latest_binary_versions = 8; +} diff --git a/gfauto/gfauto/settings_pb2.py b/gfauto/gfauto/settings_pb2.py new file mode 100644 index 000000000..9697d091f --- /dev/null +++ b/gfauto/gfauto/settings_pb2.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gfauto/settings.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from gfauto import common_pb2 as gfauto_dot_common__pb2 +from gfauto import device_pb2 as gfauto_dot_device__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='gfauto/settings.proto', + package='gfauto', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x15gfauto/settings.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\x9e\x02\n\x08Settings\x12\'\n\x0b\x64\x65vice_list\x18\x01 \x01(\x0b\x32\x12.gfauto.DeviceList\x12\'\n\x0f\x63ustom_binaries\x18\x02 \x03(\x0b\x32\x0e.gfauto.Binary\x12!\n\x19maximum_duplicate_crashes\x18\x03 \x01(\r\x12\x1d\n\x15maximum_fuzz_failures\x18\x04 \x01(\r\x12\x1b\n\x13reduce_tool_crashes\x18\x05 \x01(\x08\x12\x16\n\x0ereduce_crashes\x18\x06 \x01(\x08\x12\x19\n\x11reduce_bad_images\x18\x07 \x01(\x08\x12.\n\x16latest_binary_versions\x18\x08 \x03(\x0b\x32\x0e.gfauto.Binaryb\x06proto3') + , + dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,gfauto_dot_device__pb2.DESCRIPTOR,]) + + + + +_SETTINGS = _descriptor.Descriptor( + name='Settings', + full_name='gfauto.Settings', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='device_list', full_name='gfauto.Settings.device_list', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='custom_binaries', full_name='gfauto.Settings.custom_binaries', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='maximum_duplicate_crashes', full_name='gfauto.Settings.maximum_duplicate_crashes', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='maximum_fuzz_failures', full_name='gfauto.Settings.maximum_fuzz_failures', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='reduce_tool_crashes', full_name='gfauto.Settings.reduce_tool_crashes', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='reduce_crashes', full_name='gfauto.Settings.reduce_crashes', index=5, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='reduce_bad_images', full_name='gfauto.Settings.reduce_bad_images', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='latest_binary_versions', full_name='gfauto.Settings.latest_binary_versions', index=7, + number=8, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=76, + serialized_end=362, +) + +_SETTINGS.fields_by_name['device_list'].message_type = gfauto_dot_device__pb2._DEVICELIST +_SETTINGS.fields_by_name['custom_binaries'].message_type = gfauto_dot_common__pb2._BINARY +_SETTINGS.fields_by_name['latest_binary_versions'].message_type = gfauto_dot_common__pb2._BINARY +DESCRIPTOR.message_types_by_name['Settings'] = _SETTINGS +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Settings = _reflection.GeneratedProtocolMessageType('Settings', (_message.Message,), { + 'DESCRIPTOR' : _SETTINGS, + '__module__' : 'gfauto.settings_pb2' + # @@protoc_insertion_point(class_scope:gfauto.Settings) + }) +_sym_db.RegisterMessage(Settings) + + +# @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/settings_pb2.pyi b/gfauto/gfauto/settings_pb2.pyi new file mode 100644 index 000000000..099633771 --- /dev/null +++ b/gfauto/gfauto/settings_pb2.pyi @@ -0,0 +1,70 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from gfauto.common_pb2 import ( + Binary as gfauto___common_pb2___Binary, +) + +from gfauto.device_pb2 import ( + DeviceList as gfauto___device_pb2___DeviceList, +) + +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class Settings(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + maximum_duplicate_crashes = ... # type: int + maximum_fuzz_failures = ... # type: int + reduce_tool_crashes = ... # type: bool + reduce_crashes = ... # type: bool + reduce_bad_images = ... # type: bool + + @property + def device_list(self) -> gfauto___device_pb2___DeviceList: ... + + @property + def custom_binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[gfauto___common_pb2___Binary]: ... + + @property + def latest_binary_versions(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[gfauto___common_pb2___Binary]: ... + + def __init__(self, + *, + device_list : typing___Optional[gfauto___device_pb2___DeviceList] = None, + custom_binaries : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, + maximum_duplicate_crashes : typing___Optional[int] = None, + maximum_fuzz_failures : typing___Optional[int] = None, + reduce_tool_crashes : typing___Optional[bool] = None, + reduce_crashes : typing___Optional[bool] = None, + reduce_bad_images : typing___Optional[bool] = None, + latest_binary_versions : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Settings: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"device_list"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",u"device_list",u"latest_binary_versions",u"maximum_duplicate_crashes",u"maximum_fuzz_failures",u"reduce_bad_images",u"reduce_crashes",u"reduce_tool_crashes"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"device_list",b"device_list"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"custom_binaries",b"custom_binaries",u"device_list",b"device_list",u"latest_binary_versions",b"latest_binary_versions",u"maximum_duplicate_crashes",b"maximum_duplicate_crashes",u"maximum_fuzz_failures",b"maximum_fuzz_failures",u"reduce_bad_images",b"reduce_bad_images",u"reduce_crashes",b"reduce_crashes",u"reduce_tool_crashes",b"reduce_tool_crashes"]) -> None: ... diff --git a/gfauto/gfauto/settings_util.py b/gfauto/gfauto/settings_util.py new file mode 100644 index 000000000..1aa862655 --- /dev/null +++ b/gfauto/gfauto/settings_util.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Settings utility module. + +Used to read and write the Settings proto. +See settings.proto. +""" +import traceback +from pathlib import Path + +from gfauto import binaries_util, devices_util, proto_util +from gfauto.gflogging import log +from gfauto.settings_pb2 import Settings + +DEFAULT_SETTINGS_FILE_PATH = Path("settings.json") + +DEFAULT_SETTINGS = Settings( + maximum_duplicate_crashes=3, + maximum_fuzz_failures=10, + reduce_tool_crashes=True, + reduce_crashes=True, + reduce_bad_images=True, +) + + +class NoSettingsFile(Exception): + pass + + +def read_or_create(settings_path: Path) -> Settings: + try: + return read(settings_path) + except FileNotFoundError as exception: + if settings_path.exists(): + raise + log( + f'\ngfauto could not find "{settings_path}" so one will be created for you\n' + ) + write_default(settings_path) + raise NoSettingsFile( + f'\ngfauto could not find "{settings_path}" so one was created for you. Please review "{settings_path}" and try again.\n' + ) from exception + + +def read(settings_path: Path) -> Settings: + result = proto_util.file_to_message(settings_path, Settings()) + return result + + +def write(settings: Settings, settings_path: Path) -> Path: + return proto_util.message_to_file(settings, settings_path) + + +def write_default(settings_path: Path) -> Path: + settings = Settings() + settings.CopyFrom(DEFAULT_SETTINGS) + + # noinspection PyBroadException + try: + + settings.latest_binary_versions.extend( + binaries_util.download_latest_binary_version_numbers() + ) + except Exception: # pylint: disable=broad-except; + message = "WARNING: Could not download the latest binary version numbers. We will just use the (older) hardcoded versions." + details = traceback.format_exc() # noqa: SC100, SC200 (spelling of exc) + + log(message) + log(f"\nDetails:\n{details}\n\n") + log(message) + + settings.latest_binary_versions.extend(binaries_util.DEFAULT_BINARIES) + + binary_manager = binaries_util.get_default_binary_manager(settings=settings) + + devices_util.get_device_list(binary_manager, settings.device_list) + + return write(settings, settings_path) diff --git a/gfauto/gfauto/shader_compiler_util.py b/gfauto/gfauto/shader_compiler_util.py new file mode 100644 index 000000000..0ff2df2fa --- /dev/null +++ b/gfauto/gfauto/shader_compiler_util.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""(Offline) Shader compiler utility module. + +Used to run offline shader compilers on all shaders in a shader job. +""" + +from pathlib import Path +from typing import List + +from gfauto import binaries_util, shader_job_util, subprocess_util, util +from gfauto.device_pb2 import DeviceShaderCompiler +from gfauto.gflogging import log + +DEFAULT_TIMEOUT = 600 + + +def run_shader( + shader_compiler_device: DeviceShaderCompiler, + shader_path: Path, + output_dir: Path, + compiler_path: Path, + timeout: int = DEFAULT_TIMEOUT, +) -> Path: + output_file_path = output_dir / (shader_path.name + ".out") + + cmd = [str(compiler_path)] + cmd += list(shader_compiler_device.args) + cmd += [str(shader_path), "-o", str(output_file_path)] + cmd = util.prepend_catchsegv_if_available(cmd) + subprocess_util.run(cmd, verbose=True, timeout=timeout) + return output_file_path + + +def run_shader_job( + shader_compiler_device: DeviceShaderCompiler, + spirv_shader_job_path: Path, + output_dir: Path, + binary_manager: binaries_util.BinaryManager, +) -> List[Path]: + compiler_path = binary_manager.get_binary_path_by_name( + shader_compiler_device.binary + ).path + + log(f"Running {str(compiler_path)} on shader job {str(spirv_shader_job_path)}") + + shader_paths = shader_job_util.get_related_files( + spirv_shader_job_path, language_suffix=[shader_job_util.SUFFIX_SPIRV] + ) + + log(f"Running {str(compiler_path)} on shaders: {shader_paths}") + + result = [] + + for shader_path in shader_paths: + result.append( + run_shader( + shader_compiler_device, + compiler_path=compiler_path, + shader_path=shader_path, + output_dir=output_dir, + ) + ) + + return result diff --git a/gfauto/gfauto/shader_job_util.py b/gfauto/gfauto/shader_job_util.py new file mode 100644 index 000000000..5acfccbec --- /dev/null +++ b/gfauto/gfauto/shader_job_util.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shader job utility module. + +Used to copy shader jobs and get the related files of a shader job. +""" +import itertools +import pathlib +from pathlib import Path +from typing import List, Optional, Tuple, Union + +from gfauto import util + +EXT_FRAG = ".frag" +EXT_VERT = ".vert" +EXT_COMP = ".comp" + +EXT_ALL = (EXT_FRAG, EXT_VERT, EXT_COMP) + +SUFFIX_GLSL = "" +SUFFIX_SPIRV = ".spv" +SUFFIX_ASM_SPIRV = ".asm" +SUFFIX_FACTS = ".facts" +SUFFIX_TRANSFORMATIONS = ".transformations" +SUFFIX_TRANSFORMATIONS_JSON = ".transformations_json" + +SUFFIX_SPIRV_ORIG = ".spv_orig" + +SUFFIXES_SPIRV_FUZZ_INPUT = (SUFFIX_SPIRV, SUFFIX_FACTS) + +SUFFIXES_SPIRV_FUZZ = ( + SUFFIX_SPIRV, + SUFFIX_SPIRV_ORIG, + SUFFIX_FACTS, + SUFFIX_TRANSFORMATIONS, + SUFFIX_TRANSFORMATIONS_JSON, +) + + +def get_shader_contents( + shader_job_file_path: pathlib.Path, + extension: str, + language_suffix: str = SUFFIX_GLSL, + must_exist: bool = False, +) -> Optional[str]: + shader_file = shader_job_file_path.with_suffix(extension + language_suffix) + if shader_file.exists(): + return util.file_read_text(shader_file) + if must_exist: + raise AssertionError(f"could not read {shader_file}") + + return None + + +def get_related_suffixes_that_exist( + shader_job_file_path: Path, + extensions: Union[Tuple[str, ...], List[str]] = EXT_ALL, + language_suffix: Union[Tuple[str, ...], List[str]] = (SUFFIX_GLSL,), +) -> List[str]: + # Cross product of extensions and suffixes as tuples. + # (".frag", ".comp", ...) x (".facts", ".spv") + suffixes_as_tuples = itertools.product(extensions, language_suffix) + + # Same as above but as str. E.g. ".frag.facts", ".frag.spv" + suffixes = [file_parts[0] + file_parts[1] for file_parts in suffixes_as_tuples] + + # Filter + suffixes_that_exist = [ + s for s in suffixes if shader_job_file_path.with_suffix(s).is_file() + ] + + return suffixes_that_exist + + +def get_related_files( + shader_job_file_path: pathlib.Path, + extensions: Union[Tuple[str, ...], List[str]] = EXT_ALL, + language_suffix: Union[Tuple[str, ...], List[str]] = (SUFFIX_GLSL,), +) -> List[pathlib.Path]: + + suffixes_that_exist = get_related_suffixes_that_exist( + shader_job_file_path, extensions, language_suffix + ) + + # variant_001.frag.spv, ... + paths = [shader_job_file_path.with_suffix(s) for s in suffixes_that_exist] + + return paths + + +def copy( + shader_job_file_path: pathlib.Path, + output_shader_job_file_path: pathlib.Path, + extensions: Union[Tuple[str, ...], List[str]] = EXT_ALL, + language_suffix: Union[Tuple[str, ...], List[str]] = (SUFFIX_GLSL,), +) -> pathlib.Path: + + util.copy_file(shader_job_file_path, output_shader_job_file_path) + + suffixes_that_exist = get_related_suffixes_that_exist( + shader_job_file_path, extensions, language_suffix + ) + + for suffix in suffixes_that_exist: + util.copy_file( + shader_job_file_path.with_suffix(suffix), + output_shader_job_file_path.with_suffix(suffix), + ) + + return output_shader_job_file_path diff --git a/gfauto/gfauto/signature_util.py b/gfauto/gfauto/signature_util.py new file mode 100644 index 000000000..b27b3995c --- /dev/null +++ b/gfauto/gfauto/signature_util.py @@ -0,0 +1,362 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bug signature module. + +Used to compute the "signature" of a bug, typically using the error message or the function name from the top of the +stack trace. +""" + + +import re +from pathlib import Path +from typing import Match, Optional, Pattern + +from gfauto import subprocess_util, util + +# .* does not match newlines +# (?: ) non-group parentheses + +HEX_LIKE = r"(?:0x)?[0-9a-fA-F]" + +# 06-15 21:17:00.039 7517 7517 F DEBUG : #00 pc 00000000009d9c34 /my/library.so ((anonymous namespace)::Bar::Baz(aaa::MyInstr*, void* (*)(unsigned int))+456) +# Another example of the function signature: /my/library.so (myFunction+372) +# Another example of the function signature: /my/library.so (myFunction(...)+372) + +# Just look for anything that contains "word(" or "word+" from after the (hex-like) PC address. +PATTERN_ANDROID_BACKTRACE_FUNCTION = re.compile( + r"\n.*#00 pc " + HEX_LIKE + r"+ (.*[\w\d_]+[(+].*)" +) + +ANDROID_BACKTRACE_COMMON_TEXT_TO_REMOVE = re.compile( + r"(?:" + r"vendor/" + r"|hw/" + r"|data/local/(?:tmp/)?" + r"|system/(?:lib(?:64)?/)?" + r"|lib(?:64)?/" + r"|apex/" + r"|bionic/" + r"|com.android.runtime(?:/lib(?:64)?)?/" + r"|anonymous namespace" + r"|\(BuildId: " + HEX_LIKE + r"+\)" + r")" +) + +PATTERN_CDB_CALL_SITE = re.compile( + fr"\n{HEX_LIKE}+`{HEX_LIKE}+ {HEX_LIKE}+`{HEX_LIKE}+ (.*)" +) + +PATTERN_ANDROID_BACKTRACE_CATCHALL = re.compile(r"\n.*#00 pc " + HEX_LIKE + r"+ (.*)") + +# E.g. ERROR: temp/.../variant/shader.frag:549: 'variable indexing fragment shader output array' : not supported with this profile: es +# variable indexing fragment shader output array <-- group 1 +# E.g. ERROR: reports/.../part_1_preserve_semantics/reduction_work/variant/shader_reduced_0173/0_glsl/shader_reduced_0173.frag:456: '=' : cannot convert from ' const 3-component vector of bool' to ' temp bool' +PATTERN_GLSLANG_ERROR = re.compile(r"ERROR: .*?:\d+: (.*)") + + +# E.g. +# glslangValidator: ../glslang/MachineIndependent/ParseHelper.cpp:2212: void glslang::TParseContext::nonOpBuiltInCheck(const glslang::TSourceLoc&, const glslang::TFunction&, glslang::TIntermAggregate&): Assertion `PureOperatorBuiltins == false' failed. + +PATTERN_ASSERTION_FAILURE = re.compile(r"\n.*?:\d+: (.*? [Aa]ssert(?:ion)?)") + + +# Only used if "0 pass, 1 fail" is found. +# E.g. /data/local/tmp/graphicsfuzz/test.amber: 256: probe ssbo format does not match buffer format +# probe ssbo format does not match buffer format +PATTERN_AMBER_ERROR = re.compile(r"\n.*?\w: \d+: (.*)") + +# E.g. error: line 0: Module contains unreachable blocks during merge return. Run dead branch elimination before merge return. +# error: line 0: Module contains unreachable blocks during merge return. Run dead branch elimination before merge return. +# Module contains unreachable blocks during merge return. Run dead branch elimination before merge return. +PATTERN_SPIRV_OPT_ERROR: Pattern[str] = re.compile(r"error: line \d+: (.*)") + +# E.g. +# Backtrace: +# /data/git/graphicsfuzz/graphicsfuzz/target/graphicsfuzz/bin/Linux/spirv-opt(_ZN8spvtools3opt21StructuredCFGAnalysis16SwitchMergeBlockEj+0x369)[0x5bd6d9] +PATTERN_CATCHSEGV_STACK_FRAME = re.compile(r"Backtrace:\n.*/([^/(]*\([^)+\[]+)\+") + +# E.g. +# Backtrace: +# /data/git/graphicsfuzz/gfauto/temp/june_20/binaries/swiftshader_vulkan/Linux/libvk_swiftshader.so(+0x1d537d)[0x7f51ebd1237d] +# /data/git/graphicsfuzz/gfauto/temp/june_20/binaries/swiftshader_vulkan/Linux/libvk_swiftshader.so 0x1d537d +# ^ group 1 ^ group 2 +PATTERN_CATCHSEGV_STACK_FRAME_ADDRESS = re.compile( + r"Backtrace:\n(.*)\(\+([x\da-fA-F]+)+\)\[" +) + +PATTERN_SWIFT_SHADER_ABORT = re.compile(r":\d+ ABORT:(.*)") + +PATTERN_SWIFT_SHADER_WARNING = re.compile(r":\d+ WARNING:(.*)") + +PATTERN_CATCH_ALL_ERROR = re.compile(r"\nERROR: (.*)") + +# [\s\S] matches anything, including newlines. +PATTERN_LLVM_FATAL_ERROR = re.compile( + r"LLVM FATAL ERROR:Broken function found, compilation aborted![\s\S]*STDERR:\n(.*)" +) + +PATTERN_LLVM_MACHINE_CODE_ERROR = re.compile( + r"ERROR: LLVM FATAL ERROR:Found .* machine code error[\s\S]*Bad machine code: (.*)" +) + +PATTERN_LLVM_ERROR_DIAGNOSIS = re.compile(r"ERROR: LLVM DIAGNOSIS INFO: (.*)") + + +PATTERN_AMBER_TOLERANCE_ERROR = re.compile( + r"is greater th[ae]n tolerance|Buffers have different values" +) + +BAD_IMAGE_SIGNATURE = "bad_image" + + +def remove_hex_like(string: str) -> str: + temp = string + # Remove hex like chunks of 4 or more. + temp = re.sub(HEX_LIKE + r"{4,}", "", temp) + return temp + + +def clean_up(string: str, remove_numbers: bool = True) -> str: + temp: str = string + # Remove numbers. + if remove_numbers: + temp = re.sub(r"\d+", "", temp) + # Replace spaces with _. + temp = re.sub(r" ", "_", temp) + # Remove non-word, non-_ characters. + temp = re.sub(r"[^\w_]", "", temp) + # Replace multiple _ with _. + temp = re.sub(r"__+", "_", temp) + # Strip _ + temp = temp.strip("_") + return temp + + +def reduce_length(string: str) -> str: + return string[:50] + + +def basic_match(pattern: Pattern[str], log_contents: str) -> Optional[str]: + match: Optional[Match[str]] = re.search(pattern, log_contents) + if not match: + return None + group = match.group(1) + group = clean_up(group) + group = reduce_length(group) + return group + + +def get_signature_from_log_contents( # pylint: disable=too-many-return-statements, too-many-branches, too-many-statements; + log_contents: str, +) -> str: + + # noinspection PyUnusedLocal + match: Optional[Match[str]] + # noinspection PyUnusedLocal + group: Optional[str] + + # LLVM FATAL ERROR (special override). + group = basic_match(PATTERN_LLVM_FATAL_ERROR, log_contents) + if group: + return group + + # LLVM MACHINE CODE ERROR (special override). + group = basic_match(PATTERN_LLVM_MACHINE_CODE_ERROR, log_contents) + if group: + return group + + # LLVM ERROR DIAGNOSIS: should come before PATTERN_ASSERTION_FAILURE. + group = basic_match(PATTERN_LLVM_ERROR_DIAGNOSIS, log_contents) + if group: + return group + + # glslang error. + group = basic_match(PATTERN_GLSLANG_ERROR, log_contents) + if group: + return group + + # Assertion error pattern, used by glslang. + group = basic_match(PATTERN_ASSERTION_FAILURE, log_contents) + if group: + return group + + # Spirv-opt error. + group = basic_match(PATTERN_SPIRV_OPT_ERROR, log_contents) + if group: + return group + + # ABORT message from SwiftShader. + group = basic_match(PATTERN_SWIFT_SHADER_ABORT, log_contents) + if group: + return group + + # WARNING message from SwiftShader. + group = basic_match(PATTERN_SWIFT_SHADER_WARNING, log_contents) + if group: + return group + + # Amber error. + if "0 pass, 1 fail" in log_contents: + group = basic_match(PATTERN_AMBER_ERROR, log_contents) + if group: + return group + + # Cdb stack trace + cdb_call_site = re.search( + PATTERN_CDB_CALL_SITE, log_contents + ) # type: Optional[Match[str]] + if cdb_call_site: + site = cdb_call_site.group(1) + if "!" in site: + # We probably have symbols, so remove the address and everything after e.g. "+0x111 [file/path @ 123]" + site = re.sub(rf"\+{HEX_LIKE}+.*", "", site) + site = clean_up(site) + else: + # We don't have symbols so we may as well keep offsets around; don't remove numbers. + site = clean_up(site, remove_numbers=False) + site = reduce_length(site) + return site + + # Android stack traces. + if "#00 pc" in log_contents: + lines = log_contents.split("\n") + for line in lines: + pc_pos = line.find("#00 pc") + if pc_pos == -1: + continue + line = line[pc_pos:] + + if "/amber_ndk" in line: + return "amber_ndk" + break + + # Check for stack line with libc alloc. + if re.search(r"\n.*#\d+ pc .*libc\.so \(\w?alloc", log_contents): + # Find the first stack frame without libc.so and replace the log_contents with that frame. + # We do this because the error is better identified by this line and because out of memory errors + # often occur at a nondeterministic location within libc. + for line in lines: + if ( + re.search(r" #\d+ pc ", line) + and "libc.so" not in line + and "operator new" not in line + ): + # Replace the stack frame number so it looks like the 0th frame. + line = re.sub(r" #\d+ ", " #00 ", line) + log_contents = f"\n{line}\n" + break + + match = re.search(PATTERN_ANDROID_BACKTRACE_FUNCTION, log_contents) + if match: + group = match.group(1) + # Remove common text. + group = re.sub(ANDROID_BACKTRACE_COMMON_TEXT_TO_REMOVE, "", group) + group = clean_up(group) + group = reduce_length(group) + return group + + # TODO: Maybe more. + + # If we get here, we found #00 pc, but nothing else. + # This regex essentially matches the entire line after the hex-like PC address. + match = re.search(PATTERN_ANDROID_BACKTRACE_CATCHALL, log_contents) + if match: + group = match.group(1) + # Remove common text. + group = re.sub(ANDROID_BACKTRACE_COMMON_TEXT_TO_REMOVE, "", group) + # Remove hex-like chunks. + group = remove_hex_like(group) + group = clean_up(group) + group = reduce_length(group) + return group + + # catchsegv "Backtrace:" with source code info. + group = basic_match(PATTERN_CATCHSEGV_STACK_FRAME, log_contents) + if group: + return group + + # catchsegv "Backtrace:" with addresses. + if "Backtrace:" in log_contents: + result = get_signature_from_catchsegv_frame_address(log_contents) + if result: + return result + + group = basic_match(PATTERN_CATCH_ALL_ERROR, log_contents) + if group: + return group + + if "Shader compilation failed" in log_contents: + return "compile_error" + + if "Failed to link shaders" in log_contents: + return "link_error" + + if "Calling vkCreateGraphicsPipelines Fail" in log_contents: + return "pipeline_failure" + + # TODO: Check for Amber fence failure. + + if "Resource deadlock would occur" in log_contents: + return "Resource_deadlock_would_occur" + + match = re.search(PATTERN_AMBER_TOLERANCE_ERROR, log_contents) + if match: + return BAD_IMAGE_SIGNATURE + + return "no_signature" + + +def get_signature_from_catchsegv_frame_address(log_contents: str) -> Optional[str]: + match = re.search(PATTERN_CATCHSEGV_STACK_FRAME_ADDRESS, log_contents) + if not match: + return None + module = Path(match.group(1)) + if not module.exists(): + return None + address = match.group(2) + function_signature = get_function_signature_from_address(module, address) + if not function_signature: + # As a last resort, we can use the module name + offset as the signature. + return get_hex_signature_from_frame(module, address) + function_signature = clean_up(function_signature) + function_signature = reduce_length(function_signature) + return function_signature + + +def get_hex_signature_from_frame(module: Path, address: str) -> str: + signature = f"{module.name}+{address}" + signature = clean_up(signature) + signature = reduce_length(signature) + return signature + + +def get_function_signature_from_address(module: Path, address: str) -> Optional[str]: + try: + address_tool = util.tool_on_path("addr2line") + result = subprocess_util.run( + [str(address_tool), "-e", str(module), address, "-f", "-C"], + check_exit_code=False, + ) + if result.returncode != 0: + return None + stdout: str = result.stdout + lines = stdout.splitlines() + if not lines: + return None + return lines[0] + except util.ToolNotOnPathError: + return None diff --git a/gfauto/gfauto/spirv_dis_util.py b/gfauto/gfauto/spirv_dis_util.py new file mode 100644 index 000000000..77cd7d30b --- /dev/null +++ b/gfauto/gfauto/spirv_dis_util.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""spirv-dis utility module. + +Converts a SPIR-V shader job to a SPIR-V assembly shader job (the shaders are disassembled). +""" + +import pathlib +from typing import Optional + +from gfauto import binaries_util, shader_job_util, subprocess_util, util + + +def run_spirv_dis_on_spirv_shader( + input_spirv_file_path: pathlib.Path, + output_dir_path: pathlib.Path, + spirv_dis_file_path: Optional[pathlib.Path] = None, +) -> pathlib.Path: + if not spirv_dis_file_path: + spirv_dis_file_path = util.tool_on_path(binaries_util.SPIRV_DIS_NAME) + + output_spirv_file_path = output_dir_path / ( + util.remove_end(input_spirv_file_path.name, ".spv") + ".asm" + ) + + util.file_mkdirs_parent(output_spirv_file_path) + + subprocess_util.run( + util.prepend_catchsegv_if_available( + [ + str(spirv_dis_file_path), + str(input_spirv_file_path), + "-o", + str(output_spirv_file_path), + "--raw-id", + ] + ) + ) + + return output_spirv_file_path + + +def run_spirv_shader_job_to_spirv_asm_shader_job( + input_spirv_job_json_file_path: pathlib.Path, + output_spirv_job_json_file_path: pathlib.Path, + spirv_dis_file_path: Optional[pathlib.Path] = None, +) -> pathlib.Path: + + if not spirv_dis_file_path: + spirv_dis_file_path = util.tool_on_path(binaries_util.SPIRV_DIS_NAME) + + shader_files = shader_job_util.get_related_files( + input_spirv_job_json_file_path, language_suffix=[shader_job_util.SUFFIX_SPIRV] + ) + + util.copy_file(input_spirv_job_json_file_path, output_spirv_job_json_file_path) + + for shader_file in shader_files: + run_spirv_dis_on_spirv_shader( + shader_file, output_spirv_job_json_file_path.parent, spirv_dis_file_path + ) + + return output_spirv_job_json_file_path diff --git a/gfauto/gfauto/spirv_fuzz_util.py b/gfauto/gfauto/spirv_fuzz_util.py new file mode 100644 index 000000000..b12fea034 --- /dev/null +++ b/gfauto/gfauto/spirv_fuzz_util.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""spirv-fuzz utility module. + +Runs spirv-fuzz to generate a variant SPIR-V shader job. +""" + +from pathlib import Path +from typing import List, Optional + +from gfauto import shader_job_util, subprocess_util, util + +# TODO: Make this 64 bits. + +GENERATE_SEED_BITS = 32 + + +def run_generate( + spirv_fuzz_path: Path, + reference_shader_spv: Path, + output_shader_spv: Path, + seed: Optional[str] = None, + other_args: Optional[List[str]] = None, +) -> Path: + + util.check( + output_shader_spv.suffix == shader_job_util.SUFFIX_SPIRV, + AssertionError(f"Expected {str(output_shader_spv)} to end with .spv"), + ) + + util.file_mkdirs_parent(output_shader_spv) + cmd = [ + str(spirv_fuzz_path), + str(reference_shader_spv), + "-o", + str(output_shader_spv), + ] + + if seed: + cmd.append(f"--seed={seed}") + + if other_args: + cmd.extend(other_args) + + subprocess_util.run(cmd) + + # reference.spv -> output.spv_orig + + util.copy_file( + reference_shader_spv, + output_shader_spv.with_suffix(shader_job_util.SUFFIX_SPIRV_ORIG), + ) + + # reference.spv.facts -> output.spv.facts + + source_facts_path = reference_shader_spv.with_suffix(shader_job_util.SUFFIX_FACTS) + dest_facts_path = output_shader_spv.with_suffix(shader_job_util.SUFFIX_FACTS) + + if source_facts_path.exists(): + util.copy_file(source_facts_path, dest_facts_path) + + return output_shader_spv + + +def run_generate_on_shader_job( + spirv_fuzz_path: Path, + reference_shader_json: Path, + output_shader_json: Path, + seed: Optional[str] = None, + other_args: Optional[List[str]] = None, +) -> Path: + + util.copy_file(reference_shader_json, output_shader_json) + + suffixes_that_exist = shader_job_util.get_related_suffixes_that_exist( + reference_shader_json, shader_job_util.EXT_ALL, [shader_job_util.SUFFIX_SPIRV] + ) + + for suffix in suffixes_that_exist: + run_generate( + spirv_fuzz_path, + reference_shader_json.with_suffix(suffix), + output_shader_json.with_suffix(suffix), + seed, + other_args, + ) + + return output_shader_json diff --git a/gfauto/gfauto/spirv_opt_util.py b/gfauto/gfauto/spirv_opt_util.py new file mode 100644 index 000000000..dc5bbfc5f --- /dev/null +++ b/gfauto/gfauto/spirv_opt_util.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""spirv-opt utility module. + +Runs spirv-opt on a SPIR-V shader job to get a new SPIR-V shader job where the shaders have been optimized. +""" + +import pathlib +import random +from typing import List, Optional + +from gfauto import binaries_util, shader_job_util, subprocess_util, util + +SPIRV_OPT_DEFAULT_TIME_LIMIT = 120 + +OPT_OPTIONS: List[str] = [ + "--ccp", + "--combine-access-chains", + "--convert-local-access-chains", + "--copy-propagate-arrays", + "--eliminate-dead-branches", + "--eliminate-dead-code-aggressive", + "--eliminate-dead-inserts", + "--eliminate-local-multi-store", + "--eliminate-local-single-block", + "--eliminate-local-single-store", + "--if-conversion", + "--inline-entry-points-exhaustive", + "--merge-blocks", + "--merge-return", + "--private-to-local", + "--reduce-load-size", + "--redundancy-elimination", + "--scalar-replacement=100", + "--simplify-instructions", + "--vector-dce", +] + + +def random_spirv_opt_args(max_num_args: int = 30) -> List[str]: + result: List[str] = [] + num_args = random.randint(1, max_num_args) + for _ in range(0, num_args): + arg = random.choice(OPT_OPTIONS) + # --merge-return relies on there not being unreachable code, so we always invoke dead branch + # elimination before --merge-return. + if arg in ("--merge-return", "--merge-blocks"): + result.append("--eliminate-dead-branches") + result.append(arg) + return result + + +def run_spirv_opt_on_spirv_shader( + input_spirv_file_path: pathlib.Path, + output_dir_path: pathlib.Path, + spirv_opt_args: List[str], + spirv_opt_file_path: Optional[pathlib.Path] = None, + spirv_opt_no_validate_after_all: bool = False, + time_limit: int = SPIRV_OPT_DEFAULT_TIME_LIMIT, +) -> pathlib.Path: + + if not spirv_opt_file_path: + spirv_opt_file_path = util.tool_on_path(binaries_util.SPIRV_OPT_NAME) + + output_spirv_file_path = output_dir_path / input_spirv_file_path.name + + util.file_mkdirs_parent(output_spirv_file_path) + + cmd = [ + str(spirv_opt_file_path), + str(input_spirv_file_path), + "-o", + str(output_spirv_file_path), + ] + + if not spirv_opt_no_validate_after_all: + cmd.append("--validate-after-all") + + cmd += spirv_opt_args + + cmd = util.prepend_catchsegv_if_available(cmd) + + subprocess_util.run(cmd, timeout=time_limit) + + return output_spirv_file_path + + +def run_spirv_opt_on_spirv_shader_job( + input_spirv_shader_job_json_file_path: pathlib.Path, + output_spirv_shader_job_json_file_path: pathlib.Path, + spirv_opt_args: List[str], + spirv_opt_file_path: Optional[pathlib.Path] = None, + spirv_opt_no_validate_after_all: bool = False, +) -> pathlib.Path: + + if not spirv_opt_file_path: + spirv_opt_file_path = util.tool_on_path(binaries_util.SPIRV_OPT_NAME) + + shader_files = shader_job_util.get_related_files( + input_spirv_shader_job_json_file_path, + language_suffix=[shader_job_util.SUFFIX_SPIRV], + ) + + util.copy_file( + input_spirv_shader_job_json_file_path, output_spirv_shader_job_json_file_path + ) + + for shader_file in shader_files: + run_spirv_opt_on_spirv_shader( + shader_file, + output_spirv_shader_job_json_file_path.parent, + spirv_opt_args, + spirv_opt_file_path, + spirv_opt_no_validate_after_all, + ) + + return output_spirv_shader_job_json_file_path diff --git a/gfauto/gfauto/subprocess_util.py b/gfauto/gfauto/subprocess_util.py new file mode 100644 index 000000000..a4bace436 --- /dev/null +++ b/gfauto/gfauto/subprocess_util.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Subprocess utility module. + +Used to execute a process. In particular, the stdout and stderr are captured, and logged on failure. +The entire process group is killed on timeout, when this feature is available; this can prevent hangs, +especially when using catchsegv. +""" + +import os +import signal +import subprocess +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +from gfauto import types +from gfauto.gflogging import log +from gfauto.util import check + +LOG_COMMAND_FAILED_PREFIX = "Command failed: " +LOG_COMMAND_TIMED_OUT_PREFIX = "Command timed out: " + + +def log_stdout_stderr_helper(stdout: str, stderr: str) -> None: + + log("STDOUT:") + log(stdout) + log("") + + log("STDERR:") + log(stderr) + log("") + + +def log_stdout_stderr( + result: Union[ + subprocess.CalledProcessError, + types.CompletedProcess, + subprocess.TimeoutExpired, + ], +) -> None: + log_stdout_stderr_helper(result.stdout, result.stderr) + + +def log_returncode_helper(returncode: int) -> None: + log(f"RETURNCODE: {str(returncode)}") + + +def log_returncode( + result: Union[subprocess.CalledProcessError, types.CompletedProcess, types.Popen], +) -> None: + log_returncode_helper(result.returncode) + + +def posix_kill_group(process: types.Popen) -> None: + # Work around type warnings that will only show up on Windows: + os_alias: Any = os + os_alias.killpg(process.pid, signal.SIGTERM) + time.sleep(1) + os_alias.killpg(process.pid, signal.SIGKILL) + + +def run_helper( + cmd: List[str], + check_exit_code: bool = True, + timeout: Optional[float] = None, + env: Optional[Dict[str, str]] = None, + working_dir: Optional[Path] = None, +) -> types.CompletedProcess: + check( + bool(cmd) and cmd[0] is not None and isinstance(cmd[0], str), + AssertionError("run takes a list of str, not a str"), + ) + + env_child: Optional[Dict[str, str]] = None + if env: + log(f"Extra environment variables are: {env}") + env_child = os.environ.copy() + env_child.update(env) + + with subprocess.Popen( + cmd, + encoding="utf-8", + errors="ignore", + start_new_session=True, + env=env_child, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=working_dir, + ) as process: + try: + stdout, stderr = process.communicate(input=None, timeout=timeout) + except subprocess.TimeoutExpired: + try: + posix_kill_group(process) + except AttributeError: + process.kill() + stdout, stderr = process.communicate() + assert timeout # noqa + raise subprocess.TimeoutExpired(process.args, timeout, stdout, stderr) + except: # noqa + try: + posix_kill_group(process) + except AttributeError: + process.kill() + raise + + exit_code = process.poll() + if check_exit_code and exit_code != 0: + raise subprocess.CalledProcessError(exit_code, process.args, stdout, stderr) + return subprocess.CompletedProcess(process.args, exit_code, stdout, stderr) + + +def run( + cmd: List[str], + check_exit_code: bool = True, + timeout: Optional[float] = None, + verbose: bool = False, + env: Optional[Dict[str, str]] = None, + working_dir: Optional[Path] = None, +) -> types.CompletedProcess: + log("Exec" + (" (verbose):" if verbose else ":") + str(cmd)) + try: + result = run_helper(cmd, check_exit_code, timeout, env, working_dir) + except subprocess.TimeoutExpired as ex: + log(LOG_COMMAND_TIMED_OUT_PREFIX + str(cmd)) + # no return code to log in case of timeout + log_stdout_stderr(ex) + raise ex + except subprocess.CalledProcessError as ex: + log(LOG_COMMAND_FAILED_PREFIX + str(cmd)) + log_returncode(ex) + log_stdout_stderr(ex) + raise ex + + log_returncode(result) + if verbose: + log_stdout_stderr(result) + + return result diff --git a/gfauto/gfauto/test.proto b/gfauto/gfauto/test.proto new file mode 100644 index 000000000..5c1dce29e --- /dev/null +++ b/gfauto/gfauto/test.proto @@ -0,0 +1,44 @@ +// Copyright 2019 The GraphicsFuzz Project Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package gfauto; + +import "gfauto/common.proto"; +import "gfauto/device.proto"; + +// A test directory (test_dir) contains a Test proto in "test.json", plus the reference and variant shader jobs. +// A Test proto contains all the information needed to execute a test on a specific device, plus the crash signature +// for detecting if the result is interesting (i.e. the bug reproduces). +message Test { + oneof test { + TestGlsl glsl = 1; + TestSpirvFuzz spirv_fuzz = 5; + } + string crash_signature = 2; + Device device = 3; + repeated Binary binaries = 4; + // The expected status when running the test. E.g. CRASH, TOOL_CRASH, TIMEOUT. See fuzz.py. + string expected_status = 6; +} + +message TestGlsl { + repeated string spirv_opt_args = 1; +} + +// Spirv-fuzz generated spirv test. +message TestSpirvFuzz { + repeated string spirv_opt_args = 1; +} diff --git a/gfauto/gfauto/test_create_readme.py b/gfauto/gfauto/test_create_readme.py new file mode 100644 index 000000000..24d7fb1a4 --- /dev/null +++ b/gfauto/gfauto/test_create_readme.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create readme module. + +A script that creates a README and bug_report directory for a test and its result_dir. +""" + +import argparse +import sys +from pathlib import Path + +from gfauto import binaries_util, fuzz_glsl_test, test_util +from gfauto.settings_pb2 import Settings +from gfauto.util import check, check_dir_exists + + +def main() -> None: + parser = argparse.ArgumentParser( + description="A script that creates a README and bug_report directory for a test and its result_dir." + ) + + parser.add_argument( + "source_dir", help="Source directory containing test.json and shaders." + ) + + parser.add_argument( + "result_dir", + help="Path to the result_dir of a test containing e.g. the intermediate shader files, log.txt, etc.", + ) + + parser.add_argument( + "--output_dir", + help="Output directory where the README and bug_report directory will be written.", + default=".", + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + source_dir = Path(parsed_args.source_dir) + result_dir = Path(parsed_args.result_dir) + output_dir = Path(parsed_args.output_dir) + + check_dir_exists(source_dir) + check_dir_exists(result_dir) + + test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) + + binary_manager = binaries_util.get_default_binary_manager( + settings=Settings() + ).get_child_binary_manager( + binary_list=list(test.binaries) + list(test.device.binaries) + ) + + check(test.HasField("glsl"), AssertionError("Only glsl tests currently supported")) + + check( + test.device.HasField("preprocess"), + AssertionError("Only preprocess device tests currently supported"), + ) + + fuzz_glsl_test.tool_crash_summary_bug_report_dir( + source_dir, result_dir, output_dir, binary_manager + ) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/gfauto/gfauto/test_pb2.py b/gfauto/gfauto/test_pb2.py new file mode 100644 index 000000000..bbec8bdbd --- /dev/null +++ b/gfauto/gfauto/test_pb2.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: gfauto/test.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from gfauto import common_pb2 as gfauto_dot_common__pb2 +from gfauto import device_pb2 as gfauto_dot_device__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='gfauto/test.proto', + package='gfauto', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x11gfauto/test.proto\x12\x06gfauto\x1a\x13gfauto/common.proto\x1a\x13gfauto/device.proto\"\xd1\x01\n\x04Test\x12 \n\x04glsl\x18\x01 \x01(\x0b\x32\x10.gfauto.TestGlslH\x00\x12+\n\nspirv_fuzz\x18\x05 \x01(\x0b\x32\x15.gfauto.TestSpirvFuzzH\x00\x12\x17\n\x0f\x63rash_signature\x18\x02 \x01(\t\x12\x1e\n\x06\x64\x65vice\x18\x03 \x01(\x0b\x32\x0e.gfauto.Device\x12 \n\x08\x62inaries\x18\x04 \x03(\x0b\x32\x0e.gfauto.Binary\x12\x17\n\x0f\x65xpected_status\x18\x06 \x01(\tB\x06\n\x04test\"\"\n\x08TestGlsl\x12\x16\n\x0espirv_opt_args\x18\x01 \x03(\t\"\'\n\rTestSpirvFuzz\x12\x16\n\x0espirv_opt_args\x18\x01 \x03(\tb\x06proto3') + , + dependencies=[gfauto_dot_common__pb2.DESCRIPTOR,gfauto_dot_device__pb2.DESCRIPTOR,]) + + + + +_TEST = _descriptor.Descriptor( + name='Test', + full_name='gfauto.Test', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='glsl', full_name='gfauto.Test.glsl', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='spirv_fuzz', full_name='gfauto.Test.spirv_fuzz', index=1, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='crash_signature', full_name='gfauto.Test.crash_signature', index=2, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='device', full_name='gfauto.Test.device', index=3, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='binaries', full_name='gfauto.Test.binaries', index=4, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='expected_status', full_name='gfauto.Test.expected_status', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='test', full_name='gfauto.Test.test', + index=0, containing_type=None, fields=[]), + ], + serialized_start=72, + serialized_end=281, +) + + +_TESTGLSL = _descriptor.Descriptor( + name='TestGlsl', + full_name='gfauto.TestGlsl', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='spirv_opt_args', full_name='gfauto.TestGlsl.spirv_opt_args', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=283, + serialized_end=317, +) + + +_TESTSPIRVFUZZ = _descriptor.Descriptor( + name='TestSpirvFuzz', + full_name='gfauto.TestSpirvFuzz', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='spirv_opt_args', full_name='gfauto.TestSpirvFuzz.spirv_opt_args', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=319, + serialized_end=358, +) + +_TEST.fields_by_name['glsl'].message_type = _TESTGLSL +_TEST.fields_by_name['spirv_fuzz'].message_type = _TESTSPIRVFUZZ +_TEST.fields_by_name['device'].message_type = gfauto_dot_device__pb2._DEVICE +_TEST.fields_by_name['binaries'].message_type = gfauto_dot_common__pb2._BINARY +_TEST.oneofs_by_name['test'].fields.append( + _TEST.fields_by_name['glsl']) +_TEST.fields_by_name['glsl'].containing_oneof = _TEST.oneofs_by_name['test'] +_TEST.oneofs_by_name['test'].fields.append( + _TEST.fields_by_name['spirv_fuzz']) +_TEST.fields_by_name['spirv_fuzz'].containing_oneof = _TEST.oneofs_by_name['test'] +DESCRIPTOR.message_types_by_name['Test'] = _TEST +DESCRIPTOR.message_types_by_name['TestGlsl'] = _TESTGLSL +DESCRIPTOR.message_types_by_name['TestSpirvFuzz'] = _TESTSPIRVFUZZ +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Test = _reflection.GeneratedProtocolMessageType('Test', (_message.Message,), { + 'DESCRIPTOR' : _TEST, + '__module__' : 'gfauto.test_pb2' + # @@protoc_insertion_point(class_scope:gfauto.Test) + }) +_sym_db.RegisterMessage(Test) + +TestGlsl = _reflection.GeneratedProtocolMessageType('TestGlsl', (_message.Message,), { + 'DESCRIPTOR' : _TESTGLSL, + '__module__' : 'gfauto.test_pb2' + # @@protoc_insertion_point(class_scope:gfauto.TestGlsl) + }) +_sym_db.RegisterMessage(TestGlsl) + +TestSpirvFuzz = _reflection.GeneratedProtocolMessageType('TestSpirvFuzz', (_message.Message,), { + 'DESCRIPTOR' : _TESTSPIRVFUZZ, + '__module__' : 'gfauto.test_pb2' + # @@protoc_insertion_point(class_scope:gfauto.TestSpirvFuzz) + }) +_sym_db.RegisterMessage(TestSpirvFuzz) + + +# @@protoc_insertion_point(module_scope) diff --git a/gfauto/gfauto/test_pb2.pyi b/gfauto/gfauto/test_pb2.pyi new file mode 100644 index 000000000..a2065bf09 --- /dev/null +++ b/gfauto/gfauto/test_pb2.pyi @@ -0,0 +1,105 @@ +# @generated by generate_proto_mypy_stubs.py. Do not edit! +import sys +from gfauto.common_pb2 import ( + Binary as gfauto___common_pb2___Binary, +) + +from gfauto.device_pb2 import ( + Device as gfauto___device_pb2___Device, +) + +from google.protobuf.descriptor import ( + Descriptor as google___protobuf___descriptor___Descriptor, +) + +from google.protobuf.internal.containers import ( + RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer, + RepeatedScalarFieldContainer as google___protobuf___internal___containers___RepeatedScalarFieldContainer, +) + +from google.protobuf.message import ( + Message as google___protobuf___message___Message, +) + +from typing import ( + Iterable as typing___Iterable, + Optional as typing___Optional, + Text as typing___Text, +) + +from typing_extensions import ( + Literal as typing_extensions___Literal, +) + + +class Test(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + crash_signature = ... # type: typing___Text + expected_status = ... # type: typing___Text + + @property + def glsl(self) -> TestGlsl: ... + + @property + def spirv_fuzz(self) -> TestSpirvFuzz: ... + + @property + def device(self) -> gfauto___device_pb2___Device: ... + + @property + def binaries(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[gfauto___common_pb2___Binary]: ... + + def __init__(self, + *, + glsl : typing___Optional[TestGlsl] = None, + spirv_fuzz : typing___Optional[TestSpirvFuzz] = None, + crash_signature : typing___Optional[typing___Text] = None, + device : typing___Optional[gfauto___device_pb2___Device] = None, + binaries : typing___Optional[typing___Iterable[gfauto___common_pb2___Binary]] = None, + expected_status : typing___Optional[typing___Text] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> Test: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def HasField(self, field_name: typing_extensions___Literal[u"device",u"glsl",u"spirv_fuzz",u"test"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"binaries",u"crash_signature",u"device",u"expected_status",u"glsl",u"spirv_fuzz",u"test"]) -> None: ... + else: + def HasField(self, field_name: typing_extensions___Literal[u"device",b"device",u"glsl",b"glsl",u"spirv_fuzz",b"spirv_fuzz",u"test",b"test"]) -> bool: ... + def ClearField(self, field_name: typing_extensions___Literal[u"binaries",b"binaries",u"crash_signature",b"crash_signature",u"device",b"device",u"expected_status",b"expected_status",u"glsl",b"glsl",u"spirv_fuzz",b"spirv_fuzz",u"test",b"test"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions___Literal[u"test",b"test"]) -> typing_extensions___Literal["glsl","spirv_fuzz"]: ... + +class TestGlsl(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + spirv_opt_args = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + def __init__(self, + *, + spirv_opt_args : typing___Optional[typing___Iterable[typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> TestGlsl: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"spirv_opt_args"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[u"spirv_opt_args",b"spirv_opt_args"]) -> None: ... + +class TestSpirvFuzz(google___protobuf___message___Message): + DESCRIPTOR: google___protobuf___descriptor___Descriptor = ... + spirv_opt_args = ... # type: google___protobuf___internal___containers___RepeatedScalarFieldContainer[typing___Text] + + def __init__(self, + *, + spirv_opt_args : typing___Optional[typing___Iterable[typing___Text]] = None, + ) -> None: ... + @classmethod + def FromString(cls, s: bytes) -> TestSpirvFuzz: ... + def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... + if sys.version_info >= (3,): + def ClearField(self, field_name: typing_extensions___Literal[u"spirv_opt_args"]) -> None: ... + else: + def ClearField(self, field_name: typing_extensions___Literal[u"spirv_opt_args",b"spirv_opt_args"]) -> None: ... diff --git a/gfauto/gfauto/test_update_binaries.py b/gfauto/gfauto/test_update_binaries.py new file mode 100644 index 000000000..5a3049e0e --- /dev/null +++ b/gfauto/gfauto/test_update_binaries.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Update binaries module. + +A script that updates the binaries in a test.json file. +""" + +import argparse +import itertools +import sys +from pathlib import Path +from typing import List + +from gfauto import binaries_util, test_util +from gfauto.common_pb2 import Binary +from gfauto.gflogging import log +from gfauto.util import check_file_exists + + +def update_test_json(test_json: Path) -> Path: + test = test_util.metadata_read_from_path(test_json) + + for test_binary in itertools.chain( + test.binaries, test.device.binaries + ): # type: Binary + for default_binary in binaries_util.DEFAULT_BINARIES: + if ( + test_binary.name == default_binary.name + and test_binary.version != default_binary.version + ): + log( + f"Updating version: {test_binary.version} -> {default_binary.version}" + ) + test_binary.version = default_binary.version + break + + return test_util.metadata_write_to_path(test, test_json) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="A script that updates the binaries in a test.json file." + ) + + parser.add_argument( + "test_json", help="Paths to one or more test.json files.", nargs="*" + ) + + parsed_args = parser.parse_args(sys.argv[1:]) + + test_jsons: List[Path] = [Path(json_path) for json_path in parsed_args.test_json] + + for test_json in test_jsons: + check_file_exists(test_json) + + for test_json in test_jsons: + update_test_json(test_json) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/gfauto/gfauto/test_util.py b/gfauto/gfauto/test_util.py new file mode 100644 index 000000000..275ce5aca --- /dev/null +++ b/gfauto/gfauto/test_util.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test utility module. + +A test directory contains a Test proto stored in "source/test.json", the reference and variant shader jobs, and various +other files, including results. +This module is used to read Test proto files and get various paths that exist in test directories. +""" + +from pathlib import Path + +from gfauto import proto_util, util +from gfauto.test_pb2 import Test + +TEST_METADATA = "test.json" +REFERENCE_DIR = "reference" +VARIANT_DIR = "variant" +SHADER_JOB = "shader.json" +SHADER_JOB_RESULT = "shader.info.json" + + +def get_source_dir(test_dir: Path) -> Path: + return test_dir / "source" + + +def get_metadata_path(test_dir: Path) -> Path: + return get_source_dir(test_dir) / TEST_METADATA + + +def metadata_write(metadata: Test, test_dir: Path) -> Path: + metadata_write_to_path(metadata, get_metadata_path(test_dir)) + return test_dir + + +def metadata_read(test_dir: Path) -> Test: + return metadata_read_from_path(get_metadata_path(test_dir)) + + +def metadata_read_from_path(test_metadata_path: Path) -> Test: + text = util.file_read_text(test_metadata_path) + result = Test() + proto_util.json_to_message(text, result) + return result + + +def metadata_write_to_path(metadata: Test, test_metadata_path: Path) -> Path: + text = proto_util.message_to_json(metadata) + util.file_write_text(test_metadata_path, text) + return test_metadata_path + + +def get_shader_job_path(test_dir: Path, shader_name: str) -> Path: + return test_dir / "source" / shader_name / SHADER_JOB + + +def get_device_directory(test_dir: Path, device_name: str) -> Path: + return test_dir / "results" / device_name + + +def get_results_directory(test_dir: Path, device_name: str) -> Path: + return get_device_directory(test_dir, device_name) / "result" + + +def get_reduced_test_dir(test_dir: Path, device_name: str, reduction_name: str) -> Path: + return get_device_directory(test_dir, device_name) / "reductions" / reduction_name + + +def get_reduction_work_directory(reduced_test_dir: Path, name_of_shader: str) -> Path: + return reduced_test_dir / "reduction_work" / name_of_shader diff --git a/gfauto/gfauto/tool.py b/gfauto/gfauto/tool.py new file mode 100644 index 000000000..74d44b497 --- /dev/null +++ b/gfauto/gfauto/tool.py @@ -0,0 +1,431 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tool module. + +Used to convert shader jobs to Amber script tests that are suitable for adding to the VK-GL-CTS project. +""" + +from pathlib import Path +from typing import Dict, Iterable, Iterator, List, Optional + +from attr import dataclass + +from gfauto import ( + amber_converter, + binaries_util, + glslang_validator_util, + shader_job_util, + spirv_dis_util, + spirv_opt_util, + subprocess_util, + test_util, + util, +) +from gfauto.settings_pb2 import Settings +from gfauto.util import check + +AMBER_COMMAND_PROBE_TOP_LEFT_RED = "probe rgba (0, 0) (1, 0, 0, 1)\n" + +AMBER_COMMAND_PROBE_TOP_LEFT_WHITE = "probe rgba (0, 0) (1, 1, 1, 1)\n" + +AMBER_COMMAND_EXPECT_RED = ( + "EXPECT variant_framebuffer IDX 0 0 SIZE 256 256 EQ_RGBA 255 0 0 255\n" +) + +AMBER_COMMAND_EXPECT_BLACK = ( + "EXPECT variant_framebuffer IDX 0 0 SIZE 256 256 EQ_RGBA 0 0 0 255\n" +) + + +@dataclass +class NameAndShaderJob: + name: str + shader_job: Path + + +@dataclass +class ShaderPathWithNameAndSuffix: + name: str + suffix: str + path: Path + + +ShaderSuffixToShaderOverride = Dict[str, ShaderPathWithNameAndSuffix] + +ShaderJobNameToShaderOverridesMap = Dict[str, ShaderSuffixToShaderOverride] + + +def get_copyright_header_google(year: str) -> str: + return f"""Copyright {year} Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +def amberfy( + input_json: Path, + output_amber: Path, + amberfy_settings: amber_converter.AmberfySettings, + input_glsl_source_json_path: Optional[Path] = None, +) -> Path: + + shader_job_file_amber_test = amber_converter.ShaderJobFileBasedAmberTest( + reference_asm_spirv_job=None, + variants_asm_spirv_job=[ + amber_converter.ShaderJobFile( + "variant", input_json, input_glsl_source_json_path, "" + ) + ], + ) + return amber_converter.spirv_asm_shader_job_to_amber_script( + shader_job_file_amber_test, output_amber, amberfy_settings + ) + + +def spirv_dis_shader_job( + input_json: Path, output_json: Path, binary_paths: binaries_util.BinaryGetter +) -> Path: + return spirv_dis_util.run_spirv_shader_job_to_spirv_asm_shader_job( + input_json, + output_json, + binary_paths.get_binary_path_by_name(binaries_util.SPIRV_DIS_NAME).path, + ) + + +def spirv_opt_shader_job( + input_json: Path, + spirv_opt_args: List[str], + output_json: Path, + binary_paths: binaries_util.BinaryGetter, +) -> Path: + spirv_opt_binary = binary_paths.get_binary_path_by_name( + binaries_util.SPIRV_OPT_NAME + ) + return spirv_opt_util.run_spirv_opt_on_spirv_shader_job( + input_json, + output_json, + spirv_opt_args, + spirv_opt_binary.path, + binaries_util.SPIRV_OPT_NO_VALIDATE_AFTER_ALL_TAG + in spirv_opt_binary.binary.tags, + ) + + +def glslang_glsl_shader_job_to_spirv( + input_json: Path, output_json: Path, binary_paths: binaries_util.BinaryGetter +) -> Path: + return glslang_validator_util.run_glslang_glsl_to_spirv_job( + input_json, + output_json, + binary_paths.get_binary_path_by_name(binaries_util.GLSLANG_VALIDATOR_NAME).path, + ) + + +def run_spirv_val_on_shader(shader_path: Path, spirv_val_path: Path) -> None: + subprocess_util.run( + util.prepend_catchsegv_if_available([str(spirv_val_path), str(shader_path)]) + ) + + +def validate_spirv_shader_job_helper(input_json: Path, spirv_val_path: Path) -> None: + shader_paths = shader_job_util.get_related_files( + input_json, shader_job_util.EXT_ALL, [shader_job_util.SUFFIX_SPIRV] + ) + for shader_path in shader_paths: + run_spirv_val_on_shader(shader_path, spirv_val_path) + + +def validate_spirv_shader_job( + input_json: Path, binary_paths: binaries_util.BinaryGetter +) -> None: + validate_spirv_shader_job_helper( + input_json, + binary_paths.get_binary_path_by_name(binaries_util.SPIRV_VAL_NAME).path, + ) + + +@dataclass +class SpirvCombinedShaderJob: + name: str + spirv_asm_shader_job: Path + spirv_shader_job: Path + glsl_source_shader_job: Optional[Path] + + def __iter__(self) -> Iterator[Path]: + return iter((self.spirv_asm_shader_job, self.spirv_shader_job)) + + +def compile_shader_job( + name: str, + input_json: Path, + work_dir: Path, + binary_paths: binaries_util.BinaryGetter, + spirv_opt_args: Optional[List[str]] = None, + shader_overrides: Optional[ShaderSuffixToShaderOverride] = None, +) -> SpirvCombinedShaderJob: + + result = input_json + + glsl_source_shader_job: Optional[Path] = None + + glsl_suffixes = shader_job_util.get_related_suffixes_that_exist( + result, language_suffix=(shader_job_util.SUFFIX_GLSL,) + ) + + spirv_suffixes = shader_job_util.get_related_suffixes_that_exist( + result, language_suffix=[shader_job_util.SUFFIX_SPIRV] + ) + + # If GLSL: + if glsl_suffixes: + glsl_source_shader_job = result + + result = shader_job_util.copy(result, work_dir / "0_glsl" / result.name) + + if shader_overrides: + raise AssertionError("Shader overrides are not supported for GLSL") + + result = glslang_glsl_shader_job_to_spirv( + result, work_dir / "1_spirv" / result.name, binary_paths + ) + # If SPIR-V: + elif spirv_suffixes: + + result = shader_job_util.copy( + result, + work_dir / "1_spirv" / result.name, + # Copy all spirv-fuzz related files too: + language_suffix=shader_job_util.SUFFIXES_SPIRV_FUZZ, + ) + + if shader_overrides: + for suffix in spirv_suffixes: + shader_override = shader_overrides.get(suffix) + if shader_override: + check( + name == shader_override.name, + AssertionError( + f"shader job name {name} does not match shader override job name {shader_override.name}" + ), + ) + check( + shader_override.suffix == suffix, + AssertionError( + f"shader suffix {suffix} does not match shader override suffix {shader_override.suffix}" + ), + ) + + # These will be used as prefixes via .with_suffix(). + # E.g. path/to/temp.spv + source_prefix = shader_override.path + # E.g. path/to/shader.json -> path/to/shader.frag.spv + dest_prefix = result.with_suffix(suffix) + + util.copy_file_if_exists(source_prefix, dest_prefix) + util.copy_file_if_exists( + source_prefix.with_suffix( + shader_job_util.SUFFIX_TRANSFORMATIONS + ), + dest_prefix.with_suffix(shader_job_util.SUFFIX_TRANSFORMATIONS), + ) + util.copy_file_if_exists( + source_prefix.with_suffix( + shader_job_util.SUFFIX_TRANSFORMATIONS_JSON + ), + dest_prefix.with_suffix( + shader_job_util.SUFFIX_TRANSFORMATIONS_JSON + ), + ) + else: + # result has not changed, which means nothing was executed above. + raise AssertionError(f"Unrecognized shader job type: {str(input_json)}") + + result_spirv = result + + result = spirv_dis_shader_job( + result, work_dir / "1_spirv_asm" / result.name, binary_paths + ) + + validate_spirv_shader_job(result_spirv, binary_paths) + + if spirv_opt_args: + result = result_spirv + result = spirv_opt_shader_job( + result, spirv_opt_args, work_dir / "2_spirv_opt" / result.name, binary_paths + ) + result_spirv = result + result = spirv_dis_shader_job( + result, work_dir / "2_spirv_opt_asm" / result.name, binary_paths + ) + + validate_spirv_shader_job(result_spirv, binary_paths) + + return SpirvCombinedShaderJob( + name=name, + spirv_asm_shader_job=result, + spirv_shader_job=result_spirv, + glsl_source_shader_job=glsl_source_shader_job, + ) + + +def glsl_shader_job_crash_to_amber_script_for_google_cts( + source_dir: Path, + output_amber: Path, + work_dir: Path, + short_description: str, + comment_text: str, + copyright_year: str, + extra_commands: str, +) -> Path: + """Converts a GLSL shader job to an Amber script suitable for adding to the CTS.""" + return glsl_shader_job_wrong_image_to_amber_script_for_google_cts( + source_dir=source_dir, + output_amber=output_amber, + work_dir=work_dir, + short_description=short_description, + comment_text=comment_text, + copyright_year=copyright_year, + extra_commands=extra_commands, + ) + + +# +# @dataclass +# class Shader: +# suffix: str +# path: Path +# +# +# @dataclass +# class ShaderJob: +# name: str +# path: Path +# shader_files: Dict[str, Shader] +# +# +# @dataclass +# class SourceDirFiles: +# test_metadata: Path +# shader_jobs: Dict[str, ShaderJob] + + +def get_shader_jobs( + source_dir: Path, overrides: Iterable[NameAndShaderJob] = () +) -> List[NameAndShaderJob]: + shader_job_dir_names = [test_util.REFERENCE_DIR] + [ + shader_directory.name + for shader_directory in source_dir.glob(test_util.VARIANT_DIR + "*") + if shader_directory.is_dir() + ] + + # name -> Path + shader_jobs: Dict[str, Path] = { + shader_job_dir_name: source_dir / shader_job_dir_name / test_util.SHADER_JOB + for shader_job_dir_name in shader_job_dir_names + if (source_dir / shader_job_dir_name / test_util.SHADER_JOB).is_file() + } + + for override in overrides: + shader_jobs[override.name] = override.shader_job + + shader_jobs_list = sorted( + ( + NameAndShaderJob(name, Path(shader_job)) + for (name, shader_job) in shader_jobs.items() + ), + key=lambda x: x.name, + ) + + return shader_jobs_list + + +def glsl_shader_job_wrong_image_to_amber_script_for_google_cts( + source_dir: Path, + output_amber: Path, + work_dir: Path, + short_description: str, + comment_text: str, + copyright_year: str, + extra_commands: str, +) -> Path: + """Converts a GLSL shader job of a wrong image case to an Amber script suitable for adding to the CTS.""" + shader_jobs = get_shader_jobs(source_dir) + + test = test_util.metadata_read_from_path(source_dir / test_util.TEST_METADATA) + binary_manager = binaries_util.get_default_binary_manager( + settings=Settings() + ).get_child_binary_manager( + binary_list=list(test.device.binaries) + list(test.binaries) + ) + + spirv_opt_args = list(test.glsl.spirv_opt_args) or None + spirv_opt_hash: Optional[str] = None + if spirv_opt_args: + spirv_opt_hash = binary_manager.get_binary_by_name( + binaries_util.SPIRV_OPT_NAME + ).version + + # Compile all shader jobs + shader_job_files = [ + amber_converter.ShaderJobFile( + shader_job.name, + compile_shader_job( + shader_job.name, + shader_job.shader_job, + work_dir / shader_job.name, + binary_manager, + spirv_opt_args=spirv_opt_args, + ).spirv_asm_shader_job, + shader_job.shader_job, + "", + ) + for shader_job in shader_jobs + ] + + reference_asm_spirv_job: Optional[amber_converter.ShaderJobFile] = None + + if shader_job_files[0].name_prefix == test_util.REFERENCE_DIR: + reference_asm_spirv_job = shader_job_files[0] + del shader_job_files[0] + + return amber_converter.spirv_asm_shader_job_to_amber_script( + amber_converter.ShaderJobFileBasedAmberTest( + reference_asm_spirv_job=reference_asm_spirv_job, + variants_asm_spirv_job=shader_job_files, + ), + output_amber, + amber_converter.AmberfySettings( + copyright_header_text=get_copyright_header_google(copyright_year), + add_graphics_fuzz_comment=True, + short_description=short_description, + comment_text=comment_text, + use_default_fence_timeout=True, + spirv_opt_args=spirv_opt_args, + spirv_opt_hash=spirv_opt_hash, + extra_commands=extra_commands, + ), + ) diff --git a/gfauto/gfauto/types.py b/gfauto/gfauto/types.py new file mode 100644 index 000000000..ccdb6280a --- /dev/null +++ b/gfauto/gfauto/types.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Types module. + +Defines types that cause issues with the type checker and/or other error checking. +""" + +import subprocess +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + CompletedProcess = subprocess.CompletedProcess[Any] # pylint: disable=invalid-name; + Popen = subprocess.Popen[Any] # pylint: disable=invalid-name; +else: + CompletedProcess = subprocess.CompletedProcess # pylint: disable=invalid-name; + Popen = subprocess.Popen # pylint: disable=invalid-name; diff --git a/gfauto/gfauto/util.py b/gfauto/gfauto/util.py new file mode 100644 index 000000000..64a89c6cf --- /dev/null +++ b/gfauto/gfauto/util.py @@ -0,0 +1,350 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""General utility module. + +Used for accessing files, file system operations like creating directories, copying, moving, etc., getting the full path +of a tool on the PATH, removing the beginning and/or end of string, and custom assert functions like check, +check_check_field_truthy, check_file_exists, etc. +""" + +import os +import pathlib +import platform +import shutil +import uuid +import zipfile +from contextlib import contextmanager +from pathlib import Path +from typing import Any, BinaryIO, Iterator, List, Optional, TextIO, cast + +import attr + +from gfauto import gflogging + +MIN_SIGNED_INT_32 = -pow(2, 31) +MAX_SIGNED_INT_32 = pow(2, 31) - 1 + +# Note: Could use the built-in |file.open| and |file.write_text|, etc. + + +def file_open_binary(file: pathlib.Path, mode: str) -> BinaryIO: # noqa VNE002 + check("b" in mode, AssertionError(f"|mode|(=={mode}) should contain 'b'")) + if "w" in mode or "x" in mode: + file_mkdirs_parent(file) + # Type hint (no runtime check). + result = cast(BinaryIO, open(str(file), mode)) + return result + + +def file_open_text(file: pathlib.Path, mode: str) -> TextIO: # noqa VNE002 + check("b" not in mode, AssertionError(f"|mode|(=={mode}) should not contain 'b'")) + if "w" in mode or "x" in mode: + file_mkdirs_parent(file) + # Type hint (no runtime check). + result = cast(TextIO, open(str(file), mode, encoding="utf-8", errors="ignore")) + return result + + +def file_read_text_or_else(text_file: pathlib.Path, or_else: str) -> str: + try: + return file_read_text(text_file) + except IOError: + return or_else + + +def file_read_text(file: pathlib.Path) -> str: # noqa VNE002 + with file_open_text(file, "r") as f: + return f.read() + + +def file_read_lines(file: pathlib.Path) -> List[str]: # noqa VNE002 + with file_open_text(file, "r") as f: + return f.readlines() + + +def file_write_text_atomic(file: Path, text: str) -> Path: # noqa VNE002 + """ + Writes |text| to |file| atomically by first writing to a temporary file alongside |file| then renaming the + temporary file to |file|. + + :param file: + :param text: + :return: + """ + temp_file: Path = file.parent / get_random_name() + with file_open_text(temp_file, "x") as f: + f.write(text) + f.flush() + os.fsync(f.fileno()) + # Will not fail if dest already exists; will just silently replace dest. + os.replace(str(temp_file), str(file)) + + return file + + +def file_write_text(file: pathlib.Path, text: str) -> Path: # noqa VNE002 + with file_open_text(file, "w") as f: + f.write(text) + return file + + +def mkdir_p_new(path: Path) -> Path: # noqa VNE002 + """ + Creates a directory (and all parents, if needed), but fails if the directory already exists. + + :param path: the path to create + :return: the created path + """ + + file_mkdirs_parent(path) + path.mkdir() + return path + + +def mkdirs_p(path: pathlib.Path) -> Path: # noqa VNE002 + # Use os.makedirs, as this is more likely to be atomic. + os.makedirs(str(path), exist_ok=True) + return path + + +def file_mkdirs_parent(file: pathlib.Path) -> None: # noqa VNE002 + mkdirs_p(file.parent) + + +class ToolNotOnPathError(Exception): + pass + + +def tool_on_path(tool: str, path: Optional[str] = None) -> pathlib.Path: # noqa VNE002 + result = shutil.which(tool, path=path) + if result is None: + raise ToolNotOnPathError( + "Could not find {} on PATH. Please add to PATH.".format(tool) + ) + return pathlib.Path(result) + + +def prepend_catchsegv_if_available( + cmd: List[str], log_warning: bool = False +) -> List[str]: + try: + return [str(tool_on_path("catchsegv"))] + cmd + except ToolNotOnPathError: + pass + + try: + # cdb is the command line version of WinDbg. + return [ + str(tool_on_path("cdb")), + "-g", + "-G", + "-lines", + "-nosqm", + "-o", + "-x", + "-c", + "kp;q", + ] + cmd + except ToolNotOnPathError: + pass + + if log_warning: + gflogging.log( + "WARNING: Could not find catchsegv (Linux) or cdb (Windows) on your PATH; you will not be able to get " + "stack traces from tools or the host driver." + ) + + return cmd + + +def copy_file_if_exists(source_file_path: Path, dest_file_path: Path) -> Optional[Path]: + if not source_file_path.is_file(): + return None + return copy_file(source_file_path, dest_file_path) + + +def copy_file( + source_file_path: pathlib.Path, dest_file_path: pathlib.Path +) -> pathlib.Path: + file_mkdirs_parent(dest_file_path) + gflogging.log(f"Copying {str(source_file_path)} to {str(dest_file_path)}") + shutil.copy(str(source_file_path), str(dest_file_path)) + return dest_file_path + + +def copy_dir( + source_dir_path: pathlib.Path, dest_dir_path: pathlib.Path +) -> pathlib.Path: + file_mkdirs_parent(dest_dir_path) + shutil.copytree(source_dir_path, dest_dir_path) + return dest_dir_path + + +def move_file(source_path: Path, dest_path: Path) -> Path: + check_file_exists(source_path) + check( + not dest_path.is_dir(), + AssertionError( + f"Tried to move {str(source_path)} to a directory {str(dest_path)}" + ), + ) + file_mkdirs_parent(dest_path) + gflogging.log(f"Move file {str(source_path)} to {str(dest_path)}") + source_path.replace(dest_path) + return dest_path + + +def move_dir( + source_dir_path: pathlib.Path, dest_dir_path: pathlib.Path +) -> pathlib.Path: + file_mkdirs_parent(dest_dir_path) + shutil.move(source_dir_path, dest_dir_path) + return dest_dir_path + + +def make_directory_symlink(new_symlink_file_path: Path, existing_dir: Path) -> Path: + gflogging.log(f"symlink: from {str(new_symlink_file_path)} to {str(existing_dir)}") + check(existing_dir.is_dir(), AssertionError(f"Not a directory: {existing_dir}")) + file_mkdirs_parent(new_symlink_file_path) + + # symlink_to takes a path relative to the location of the new file (or an absolute path, but we avoid this). + symlink_contents = os.path.relpath( + str(existing_dir), start=str(new_symlink_file_path.parent) + ) + try: + new_symlink_file_path.symlink_to(symlink_contents, target_is_directory=True) + except OSError: + if get_platform() != "Windows": + raise + # Retry using junctions under Windows. + try: + # noinspection PyUnresolvedReferences + import _winapi # pylint: disable=import-error,import-outside-toplevel; + + # Unlike symlink_to, CreateJunction takes a path relative to the current directory. + _winapi.CreateJunction(str(existing_dir), str(new_symlink_file_path)) + return new_symlink_file_path + except ModuleNotFoundError: + pass + raise + + return new_symlink_file_path + + +def remove_start(string: str, start: str) -> str: + check( + string.startswith(start), AssertionError("|string| does not start with |start|") + ) + + return string[len(start) :] + + +def remove_end(str_in: str, str_end: str) -> str: + check( + str_in.endswith(str_end), + AssertionError(f"|str_in|(=={str_in}) should end with |str_end|(=={str_end})"), + ) + return str_in[: -len(str_end)] + + +def norm_path(path: pathlib.Path) -> pathlib.Path: # noqa VNE002 + return pathlib.Path(os.path.normpath(str(path))) + + +@contextmanager +def pushd(path: pathlib.Path) -> Iterator[None]: # noqa VNE002 + current_dir = pathlib.Path().resolve() + os.chdir(str(path)) + try: + yield + finally: + os.chdir(str(current_dir)) + + +def check(condition: bool, exception: Exception) -> None: + if not condition: + raise exception + + +def check_field_truthy(field: Any, field_name: str) -> None: + if not field: + raise ValueError(f"{field_name}(={str(field)}) must be filled") + + +def check_file_exists(path: Path) -> None: + check(path.is_file(), FileNotFoundError(f'Could not find file "{str(path)}"')) + + +def check_dir_exists(path: Path) -> None: + check(path.is_dir(), FileNotFoundError(f'Could not find directory "{str(path)}"')) + + +def get_platform() -> str: + host = platform.system() + if host in ("Linux", "Windows"): + return host + if host == "Darwin": + return "Mac" + raise AssertionError("Unsupported platform: {}".format(host)) + + +def extract_archive(archive_file: Path, output_dir: Path) -> Path: + """ + Extract/unpack an archive. + + :return: output_dir + """ + gflogging.log(f"Extracting {str(archive_file)} to {str(output_dir)}") + shutil.unpack_archive(str(archive_file), extract_dir=str(output_dir)) + gflogging.log("Done") + return output_dir + + +@attr.dataclass +class ZipEntry: + path: Path + path_in_archive: Optional[Path] = None + + +def create_zip(output_file_path: Path, entries: List[ZipEntry]) -> Path: + gflogging.log(f"Creating zip {str(output_file_path)}:") + with zipfile.ZipFile( + output_file_path, "w", compression=zipfile.ZIP_DEFLATED + ) as file_handle: + for entry in entries: + file_handle.write(entry.path, entry.path_in_archive) + gflogging.log(f"Adding: {entry.path} {entry.path_in_archive or ''}") + return output_file_path + + +def get_random_name() -> str: + return uuid.uuid4().hex + + +def update_gcov_environment_variable_if_needed() -> None: + if "GCOV_PREFIX" in os.environ.keys(): + gcov_prefix: str = os.environ["GCOV_PREFIX"] + if "PROC_ID" in gcov_prefix: + pid = str(os.getpid()) + check( + bool(pid), + AssertionError( + "Failed to get process ID to replace PROC_ID in GCOV_PREFIX environment variable." + ), + ) + gcov_prefix = gcov_prefix.replace("PROC_ID", pid) + os.environ["GCOV_PREFIX"] = gcov_prefix diff --git a/gfauto/gfautotests/__init__.py b/gfauto/gfautotests/__init__.py new file mode 100644 index 000000000..f7bad95d3 --- /dev/null +++ b/gfauto/gfautotests/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/gfauto/gfautotests/test_gerrit_util.py b/gfauto/gfautotests/test_gerrit_util.py new file mode 100644 index 000000000..dfa81debf --- /dev/null +++ b/gfauto/gfautotests/test_gerrit_util.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from gfauto import gerrit_util + + +def test_get_latest_change() -> None: + changes = [ + {"submitted": "2018-09-08 05:54:26.000000000"}, + {"submitted": "2019-06-10 11:54:26.000000000"}, + {"submitted": "2019-09-09 05:54:26.000000000"}, + {"submitted": "2019-09-10 05:54:26.000000000"}, # This one is the latest. + {"submitted": "2019-09-10 05:53:26.000000000"}, # This one is similar. + {"submitted": "2018-09-08 05:54:26.000000000"}, + ] + latest_change = gerrit_util.find_latest_change(changes) + assert latest_change == changes[3] + assert latest_change != changes[2] diff --git a/gfauto/gfautotests/test_signature_util.py b/gfauto/gfautotests/test_signature_util.py new file mode 100644 index 000000000..351957ae8 --- /dev/null +++ b/gfauto/gfautotests/test_signature_util.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from gfauto import signature_util + +# Disable spell-checking for this file. +# flake8: noqa: SC100 + + +def test_glslang_assertion() -> None: + log = """ +glslangValidator: ../glslang/MachineIndependent/ParseHelper.cpp:2212: void glslang::TParseContext::nonOpBuiltInCheck(const glslang::TSourceLoc&, const glslang::TFunction&, glslang::TIntermAggregate&): Assertion `PureOperatorBuiltins == false' failed. +""" + signature = signature_util.get_signature_from_log_contents(log) + assert signature == "void_glslangTParseContextnonOpBuiltInCheckconst_gl" + + +def test_glslang_error_1() -> None: + log = """ +ERROR: temp/.../variant/shader.frag:549: 'variable indexing fragment shader output array' : not supported with this profile: es +""" + signature = signature_util.get_signature_from_log_contents(log) + assert signature == "variable_indexing_fragment_shader_output_array_not" + + +def test_glslang_error_2() -> None: + log = """ +ERROR: reports/.../part_1_preserve_semantics/reduction_work/variant/shader_reduced_0173/0_glsl/shader_reduced_0173.frag:456: '=' : cannot convert from ' const 3-component vector of bool' to ' temp bool' +""" + signature = signature_util.get_signature_from_log_contents(log) + assert signature == "cannot_convert_from_const_component_vector_of_bool" diff --git a/gfauto/mypy.ini b/gfauto/mypy.ini new file mode 100644 index 000000000..f71b7730b --- /dev/null +++ b/gfauto/mypy.ini @@ -0,0 +1,16 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[mypy] +python_version = 3.6 diff --git a/gfauto/pylintrc b/gfauto/pylintrc new file mode 100644 index 000000000..153828859 --- /dev/null +++ b/gfauto/pylintrc @@ -0,0 +1,520 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns=.*_pb2.* + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=C0330, + C0111, + W0511, # Allow to-dos. + E1101, # Missing member; mypy (type checking) should cover this. + R0913, # Too many arguments; could re-enable this. + C0301, # Line too long; we use an auto formatter. + R0903, # too-few-public-methods; good, but I want to store data in classes in case I later add methods. + ungrouped-imports, # We use an imports formatter. + cyclic-import, # We allow cyclic imports, but we should import modules, not functions and variables. + duplicate-code, # Unfortunately, disabling this on a case-by-case basis is broken: https://github.com/PyCQA/pylint/issues/214 + unsubscriptable-object, # False-positives for type hints. E.g. CompletedProcess[Any]. Mypy should cover this. + + + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template='{abspath}:{line:d}:{column}: {obj}: [{msg_id} {symbol}] {msg}' + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package.. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[LOGGING] + +# Format style used to check logging format string. `old` means using % +# formatting, while `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma, + dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[STRING] + +# This flag controls whether the implicit-str-concat-in-sequence should +# generate a warning on implicit string concatenation in sequences defined over +# several lines. +check-str-concat-over-line-jumps=no + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=yes # This helps. + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _, + f # Good for file handles. + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement. +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception diff --git a/gfauto/pyproject.toml b/gfauto/pyproject.toml new file mode 100644 index 000000000..4b2af62fb --- /dev/null +++ b/gfauto/pyproject.toml @@ -0,0 +1,29 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +requires = ['wheel', 'setuptools'] +build-backend = 'setuptools.build_meta' + +[tool.black] +line-length = 88 +target-version = ['py36', 'py37', 'py38'] + +# Black currently matches against full paths, and can be quite slow at +# filtering files. Thus, we just use the following filters and always +# specify the "gfauto" directory when running. +# https://github.com/python/black/issues/712 + +include = '.*[.]py$' +exclude = '__pycache__|_pb2[.]py' diff --git a/gfauto/run_protoc.sh b/gfauto/run_protoc.sh new file mode 100755 index 000000000..c945f0ef3 --- /dev/null +++ b/gfauto/run_protoc.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e +set -u + +if [ -z ${VIRTUAL_ENV+x} ]; then + source .venv/bin/activate +fi + +python -m grpc.tools.protoc --python_out=. --proto_path=. --mypy_out=. gfauto/*.proto + +# protoc gfauto/artifact.proto gfauto/recipe.proto --python_out=. --plugin=protoc-gen-mypy=github/mypy-protobuf/python/protoc-gen-mypy --mypy_out=. --proto_path=. diff --git a/gfauto/setup.cfg b/gfauto/setup.cfg new file mode 100644 index 000000000..a26d25184 --- /dev/null +++ b/gfauto/setup.cfg @@ -0,0 +1,26 @@ +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[metadata] +description-file = README.md + +# This affects pep8.py, which in turn affects PyCharm/IntelliJ PEP8 warnings. +[pep8] +ignore = E203,W503,E501 + +# pycodestyle is the new name and version of pep8.py. I'm not sure if these are +# actually used by flake8 or PyCharm/IntelliJ, but they should be kept in sync +# with [pep8] above and the config in .flake8. +[pycodestyle] +ignore = E203,W503,E501 diff --git a/gfauto/setup.py b/gfauto/setup.py new file mode 100644 index 000000000..1199bc9d0 --- /dev/null +++ b/gfauto/setup.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + from setuptools import setup, Extension +except Exception: + from distutils.core import setup, Extension + + +setup( + name="gfauto", + version=0.9, + description="GraphicsFuzz auto.", + keywords="GraphicsFuzz fuzzing GLSL SPIRV", + author="The GraphicsFuzz Project Authors", + author_email="android-graphics-tools-team@google.com", + url="https://github.com/google/graphicsfuzz", + license="Apache License 2.0", + packages=["gfauto"], + python_requires=">=3.6", + install_requires=["protobuf", "requests", "python-dateutil"], + package_data={"gfauto": ["*.proto", "*.pyi"]}, + classifiers=[ + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved::Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3 :: Only", + ], + entry_points={"console_scripts": [ + "gfauto_fuzz = gfauto.fuzz:main", + "gfauto_interestingness_test = gfauto.gfauto_interestingness_test:main", + "gfauto_write_device_file = gfauto.devices_util:write_device_file", + "add_amber_tests_to_cts = gfauto.add_amber_tests_to_cts:main", + "gfauto_test_update_binaries = gfauto.test_update_binaries:main", + "gfauto_test_create_readme = gfauto.test_create_readme:main", + "gfauto_download_cts_gf_tests = gfauto.download_cts_gf_tests:main", + "gfauto_run_cts_gf_tests = gfauto.run_cts_gf_tests:main", + "gfauto_run_bin = gfauto.run_bin:main", + "gfauto_cov_merge = gfauto.cov_merge:main", + "gfauto_cov_new = gfauto.cov_new:main", + "gfauto_cov_to_source = gfauto.cov_to_source:main", + "gfauto_cov_from_gcov = gfauto.cov_from_gcov:main", + ]}, +) diff --git a/gfauto/temp/.gitignore b/gfauto/temp/.gitignore new file mode 100644 index 000000000..563d85214 --- /dev/null +++ b/gfauto/temp/.gitignore @@ -0,0 +1,3 @@ + +* +!/.gitignore diff --git a/gfauto/whitelist.dic b/gfauto/whitelist.dic new file mode 100644 index 000000000..49df03a0b --- /dev/null +++ b/gfauto/whitelist.dic @@ -0,0 +1,229 @@ + +Interm +Builtins +Loc +8spvtools3opt21 +Analysis16 +0x369 +0x5bd6d9 +00000000009d9c34 +0x1d537d +0x7f51ebd1237d +0x1d537d + + +'00000000009d9c34' +'variable +array' +inux +tructured +nalysis16 +erge +false' +creen +ull + + +amberscript +arg +argparse +argv +asm +bak +basename +catchsegv +chdir +cmd +contextlib +contextmanager +copyfile +cpp +dest +dirname +expandtabs +func +gflogging +gl +glsl +glslang +isdir +isfile +isort +mkdir +mkdirs +nargs +noqa +normpath +Oneof +pathlib +pb2 +Popen +posix +proto +python3 +readline +RETURNCODE +returncode +rstrip +shader +shaders +shutil +spirv +spitext +splitext +spv +stderr +stdout +subn +subprocess +usr +util +validator +vk +runtime +readlines +pylint +num +uuid +uuid4 +truthy +graphicsfuzz +randint +amberfy +zipfile +urlretrieve +infolist +attr +chmod +amberfy +amberscriptify +ssbo +SSBO +subtest +interestingness +vulkancts +mustpass +dumpsys +logcat +Vulkan +gfauto +protobuf +SPIR +paulthomson +msan +spvtools +swiftshader +libvk +RELWITHDEBINFO +vkscript +subpath +myuniform +ivec +subdata +fbsize +passthrough +pushd +reinstalls +icd +Getter +prepend +splitlines +adb +namespace +Baz +const +tmp +Backtrace +noinspection +Preprocess +preprocessor +dirs +png +txt +rglob +copytree +symlink +relpath +issubset +exe +repo +environ +subdirectory +killpg +iterdir +unreduced +Pseudocode +ssert +nondeterministic +keyevent +libc +alloc +0th +getrandbits +randbits +jsons +spvt +enum +Enum +P102 +iterables +unindexed +vec2 +vec4 +framebuffer +framebuffers +gfautotests +cdb +Dbg +mklink +fullmatch +winapi +iter +Gerrit +khronos +cwd +params +deqp +tgz +bootanim +metavar +d50c96e8 +rand2 +makedirs +unlink +fsync +fileno +Debian +prepended +config +configs +amdllpc +protos +llpc +rfind +spvas +gcov +gcovs +gcda +gcno +mScreenState +SIGTERM +SIGKILL +cov +formatter +Formatter +TODO +setdefault +lcount +unexecuted +Fallthrough +symlinks +T001 +isabs +SC200 +SC100 +PROC +listdir +RNG +toplevel +getpid diff --git a/graphicsfuzz/pom.xml b/graphicsfuzz/pom.xml index a61db5ab4..655c114a9 100644 --- a/graphicsfuzz/pom.xml +++ b/graphicsfuzz/pom.xml @@ -29,6 +29,23 @@ limitations under the License. ../parent-all/pom.xml + + **/*.pdb + + false + + + + + lite + + **/*.pdb,**/swiftshader/**,**/angle/** + **/opencv-4* + true + + + + @@ -210,6 +227,10 @@ limitations under the License. + + + + @@ -246,8 +267,8 @@ limitations under the License. - - + + @@ -260,7 +281,7 @@ limitations under the License. - + diff --git a/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/fake_compiler.bat b/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/fake_compiler.bat index cafd86cce..8d598fb54 100644 --- a/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/fake_compiler.bat +++ b/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/fake_compiler.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/interestingness_test.bat b/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/interestingness_test.bat index cafd86cce..8d598fb54 100644 --- a/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/interestingness_test.bat +++ b/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/interestingness_test.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/weak_interestingness_test.bat b/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/weak_interestingness_test.bat index cafd86cce..8d598fb54 100644 --- a/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/weak_interestingness_test.bat +++ b/graphicsfuzz/src/main/scripts/examples/glsl-reduce-walkthrough/weak_interestingness_test.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/parent-all/pom.xml b/parent-all/pom.xml index f740038eb..c9f156544 100644 --- a/parent-all/pom.xml +++ b/parent-all/pom.xml @@ -42,7 +42,7 @@ limitations under the License. 1.8 1.8 - 9.4.12.v20180830 + 9.4.18.v20190429 @@ -123,12 +123,12 @@ limitations under the License. commons-io commons-io - 2.5 + 2.6 org.apache.commons commons-lang3 - 3.5 + 3.9 org.apache.commons @@ -224,7 +224,22 @@ limitations under the License. commons-codec commons-codec - 1.9 + 1.12 + + + org.apache.commons + commons-rng-simple + 1.2 + + + org.apache.commons + commons-rng-sampling + 1.2 + + + org.apache.commons + commons-rng-client-api + 1.2 @@ -251,11 +266,6 @@ limitations under the License. assembly-desktop-client 1.0 - - com.graphicsfuzz - assembly-public - 1.0 - com.graphicsfuzz assembly-webglfuzz @@ -266,11 +276,6 @@ limitations under the License. ast 1.0 - - com.graphicsfuzz - astfuzzer - 1.0 - com.graphicsfuzz checkstyle-config @@ -482,7 +487,7 @@ limitations under the License. org.apache.maven.plugins maven-compiler-plugin - 3.6.1 + 3.8.1 @@ -500,7 +505,7 @@ limitations under the License. org.apache.maven.plugins maven-jar-plugin - 3.0.2 + 3.1.2 org.apache.maven.plugins @@ -530,7 +535,7 @@ limitations under the License. com.puppycrawl.tools checkstyle - 8.14 + 8.20 diff --git a/parent-checkstyle/pom.xml b/parent-checkstyle/pom.xml index 7b8a165c2..ca77ffb17 100644 --- a/parent-checkstyle/pom.xml +++ b/parent-checkstyle/pom.xml @@ -39,6 +39,10 @@ limitations under the License. + + ${project.build.sourceDirectory} + + graphicsfuzz/checkstyle.xml UTF-8 true diff --git a/pom.xml b/pom.xml index 84707bc4a..9b1e2ed68 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,6 @@ limitations under the License. assembly-binaries - assembly-public ast checkstyle-config client-tests diff --git a/python/src/main/python/drivers/backtrace-summary.bat b/python/src/main/python/drivers/backtrace-summary.bat index aded83e44..4860a636c 100644 --- a/python/src/main/python/drivers/backtrace-summary.bat +++ b/python/src/main/python/drivers/backtrace-summary.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/convert-shaders-to-spirv-fuzz-form.py b/python/src/main/python/drivers/convert-shaders-to-spirv-fuzz-form.py new file mode 100644 index 000000000..4aa05e335 --- /dev/null +++ b/python/src/main/python/drivers/convert-shaders-to-spirv-fuzz-form.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import glob +import os +import shutil +import sys + +import runspv +import graphicsfuzz_tool +import shader_job_uniforms_to_spirv_fuzz_facts + + +HERE = os.path.abspath(__file__) + + +def main_helper(args): + description = ( + 'Convert a directory of GLSL shader jobs to shader jobs suitable as inputs to spirv-fuzz.') + + parser = argparse.ArgumentParser(description=description) + + # Required arguments + parser.add_argument('input_dir', help="A directory of GLSL shader jobs.") + parser.add_argument('output_dir', help="The name of a directory that will be created and that " + "will contain the resulting spirv-fuzz shader jobs.") + + args = parser.parse_args(args) + + if not os.path.isdir(args.input_dir): + raise ValueError("Input directory " + "'" + args.input_dir + "' not found.") + + # Make the output directory if it does not yet exist. + if not os.path.isdir(args.output_dir): + os.makedirs(args.output_dir) + + for shader_job in glob.glob(os.path.join(args.input_dir, "*.json")): + print(shader_job) + shader_job_prefix = os.path.splitext(os.path.basename(shader_job))[0] + json_in_output_dir = os.path.join(args.output_dir, shader_job_prefix + ".json") + graphicsfuzz_tool.main_helper(["com.graphicsfuzz.generator.tool.PrepareReference", + "--generate-uniform-bindings", "--max-uniforms", "10", + shader_job, json_in_output_dir]) + shader_job_uniforms_to_spirv_fuzz_facts.main_helper([json_in_output_dir, + os.path.join(args.output_dir, + shader_job_prefix + + ".frag.facts")]) + runspv.convert_glsl_to_spv(os.path.join(args.output_dir, shader_job_prefix + ".frag"), + os.path.join(args.output_dir, shader_job_prefix + ".frag.spv")) + + for opt_settings in ["-O", "-Os"]: + shutil.copyfile(os.path.join(args.output_dir, shader_job_prefix + ".json"), + os.path.join(args.output_dir, shader_job_prefix + opt_settings + + ".json")) + shutil.copyfile(os.path.join(args.output_dir, shader_job_prefix + ".frag.facts"), + os.path.join(args.output_dir, shader_job_prefix + opt_settings + + ".frag.facts")) + runspv.run_spirv_opt(os.path.join(args.output_dir, shader_job_prefix + ".frag.spv"), + [opt_settings], + os.path.join(args.output_dir, shader_job_prefix + opt_settings + + ".frag.spv")) + + +if __name__ == '__main__': + main_helper(sys.argv[1:]) diff --git a/python/src/main/python/drivers/gapidfuzz.bat b/python/src/main/python/drivers/gapidfuzz.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/gapidfuzz.bat +++ b/python/src/main/python/drivers/gapidfuzz.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/gapidfuzz.py b/python/src/main/python/drivers/gapidfuzz.py index fed3d2426..7cca0e474 100755 --- a/python/src/main/python/drivers/gapidfuzz.py +++ b/python/src/main/python/drivers/gapidfuzz.py @@ -22,6 +22,8 @@ import typing import io import pprint + +import gfuzz_common glsl_generate = __import__("glsl-generate") # Types: @@ -75,16 +77,6 @@ def nz(s): return s -def remove_end(s: str, end: str): - assert s.endswith(end) - return s[:-len(end)] - - -def remove_start(s: str, start: str): - assert s.startswith(start) - return s[len(start):] - - def call(args: List[str], cwd=None): print(" ".join(args)) res = subprocess.run( @@ -223,7 +215,7 @@ def create_traces(params: Params): handle_to_variant_list = {} # type: typing.Dict[str, typing.List[str]] for family in Path(params.families_dir).iterdir(): # type: Path if family.name.startswith("family_"): - shader_handle = remove_start(family.name, "family_") + shader_handle = gfuzz_common.remove_start(family.name, "family_") if params.specific_handle is not None and params.specific_handle != shader_handle: continue variant_shaders = sorted(list(family.glob("variant_*.frag"))) diff --git a/python/src/main/python/drivers/gfuzz_common.py b/python/src/main/python/drivers/gfuzz_common.py new file mode 100644 index 000000000..9de9119e6 --- /dev/null +++ b/python/src/main/python/drivers/gfuzz_common.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import subprocess +from typing import List + +import runspv + + +def open_helper(file: str, mode: str): + """ + Helper function to open a file with UTF-8 format, ignoring errors. + Wraps around runspv.open_helper(), which wraps open() in turn. + :param file: the file to open - same as the file argument of open(). + :param mode: the mode to open the file in - same as the mode argument of open(). + :return: a File object opened in UTF-8 format and ignoring errors. + """ + return runspv.open_helper(file, mode) + + +def open_bin_helper(file: str, mode: str): + """ + Helper function to open a file in binary mode. Wraps around runspv.open_bin_helper(). + Will cause an assertion failure if 'mode' does not contain the open() binary symbol 'b'. + :param file: the file to open - same as the file argument of open(). + :param mode: the mode to open the file in - same as the mode argument of open(). + :return: a File object opened in binary mode. + """ + return runspv.open_bin_helper(file, mode) + + +def get_platform() -> str: + """ + Helper function to determine the current platform in use. Wraps around runspv.get_platform(). + :return: the platform in use. Throws AssertionError if platform is not Linux/Windows/Mac. + """ + return runspv.get_platform() + + +def get_bin_dir(): + """ + Helper function to get the directory of binaries (e.g. glslangvalidator) for the current + platform. Wraps around runspv.get_bin_dir(). + :return: the path of binaries for the platform in use. + """ + return runspv.get_bin_dir() + + +def tool_on_path(tool: str) -> str: + """ + Helper function to determine if a given tool is on the user's PATH variable. Wraps around + runspv.tool_on_path(). + :param tool: the tool's filename to look for. + :return: the path of the tool, else ToolNotOnPathError if the tool isn't on the PATH. + """ + return runspv.tool_on_path(tool) + + +ToolNotOnPathError = runspv.ToolNotOnPathError + + +def remove_end(str_in: str, str_end: str) -> str: + """ + Helper function to remove the end of a string. Useful for removing file extensions if you + know what extension your file should be. Wraps around runspv.remove_end(). + :param str_in: the string to remove the end from. + :param str_end: the suffix that you want to remove from str_in. + :return: str_in with str_end removed. + """ + return runspv.remove_end(str_in, str_end) + + +def filename_extension_suggests_glsl(file: str) -> bool: + """ + Helper function to determine if a file is a vertex, fragment, or compute shader. Wraps around + runspv.filename_extension_suggests_glsl(). + :param file: The filename/path to check. + :return: True if the file is a vertex/fragment/compute shader file, false otherwise. + """ + return runspv.filename_extension_suggests_glsl(file) + + +def write_to_file(content, filename) -> None: + """ + Helper function to write a string to a file opened in UTF-8 format. + :param content: The string to write to the file. + :param filename: The name/path of the file to write to. + """ + with open_helper(filename, 'w') as f: + f.write(content) + + +def remove(file) -> None: + """ + Helper function to delete a file. If the file is a directory, the directory will be recursively + deleted. + :param file: The file to be deleted. + """ + if os.path.isdir(file): + shutil.rmtree(file) + elif os.path.isfile(file): + os.remove(file) + + +def check_input_files_exist(filenames: List[str]) -> None: + """ + Helper function to determine if a list of files all exist. + :param filenames: The list of filenames to check. + :return: Nothing - throws FileNotFoundError if a file is not found. + """ + for filename in filenames: + if not os.path.isfile(filename): + raise FileNotFoundError('Input file "' + filename + '" not found') + + +def remove_start(s: str, start: str): + """ + Helper function to remove the beginning of a string. + :param s: the string to remove the prefix from. + :param start: the prefix of the string to remove. + :return: s with the start of the string removed. + """ + assert s.startswith(start) + return s[len(start):] + + +def subprocess_helper(cmd: List[str], check=True, timeout=None, verbose=False) \ + -> subprocess.CompletedProcess: + """ + Helper function to execute a command in the shell. Wraps around runspv.subprocess_helper(). + :param cmd: the command (and its arguments) to execute. + :param check: whether to throw CalledProcessError if a non-zero returncode is issued from the + process. Defaults to True. + :param timeout: time to wait in seconds before killing process and throwing TimeoutExpired. + Defaults to None. + :param verbose: whether to log process stdout/stderr more extensively. Defaults to False. + :return: a subprocess.CompletedProcess object. + """ + return runspv.subprocess_helper(cmd, check, timeout, verbose) + + +def run_catchsegv(cmd: List[str], timeout=None, verbose=False) -> str: + """ + Helper function similar to subprocess_helper, but is able to handle the killing of child + processes as well. This is most useful when trying to capture segmentation faults, as you can + run 'catchsegv (command)' in shell to ensure that the segfault is logged. Wraps around + runspv.run_catchsegv(). + :param cmd: the command (and its arguments) to execute. + :param timeout: time to wait in seconds before killing process and throwing TimeoutExpired. + Defaults to None. + :param verbose: whether to log process stdout/stderr more extensively. Defaults to False. + :return: the status of the process after running - 'SUCCESS', 'TIMEOUT', or 'CRASH'. + """ + return runspv.run_catchsegv(cmd, timeout, verbose) + + +def log(message: str): + """ + Helper function to output a message to stdout, and optionally log to a file if set_logfile() + was used previously to set a global logfile. Wraps around runspv.log(). + :param message: the message to log to stdout/file. + """ + runspv.log(message) + + +def set_logfile(file): + """ + Helper function to set the global logfile. Wraps around runspv.log_to_file. Workaround for + runspv.log() using a global logfile. + :param file: the file to log to - must be opened already. + """ + runspv.log_to_file = file + + +def unset_logfile(): + """ + Helper function to unset the global logfile. Wraps around runspv.log_to_file. Workaround for + runspv.log() using a global logfile. + """ + runspv.log_to_file = None diff --git a/python/src/main/python/drivers/glsl-generate.bat b/python/src/main/python/drivers/glsl-generate.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/glsl-generate.bat +++ b/python/src/main/python/drivers/glsl-generate.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/glsl-reduce.bat b/python/src/main/python/drivers/glsl-reduce.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/glsl-reduce.bat +++ b/python/src/main/python/drivers/glsl-reduce.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/glsl-server.bat b/python/src/main/python/drivers/glsl-server.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/glsl-server.bat +++ b/python/src/main/python/drivers/glsl-server.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/glsl-to-spv-worker.bat b/python/src/main/python/drivers/glsl-to-spv-worker.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/glsl-to-spv-worker.bat +++ b/python/src/main/python/drivers/glsl-to-spv-worker.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/glsl-to-spv-worker.py b/python/src/main/python/drivers/glsl-to-spv-worker.py index 4e93bce2a..d3a5c13a7 100755 --- a/python/src/main/python/drivers/glsl-to-spv-worker.py +++ b/python/src/main/python/drivers/glsl-to-spv-worker.py @@ -17,13 +17,13 @@ import argparse import os import random -import shutil import subprocess import sys import time from typing import Optional, List, Union import runspv +import gfuzz_common HERE = os.path.abspath(__file__) @@ -62,24 +62,6 @@ def print(s): ################################################################################ -def write_to_file(content, filename): - with runspv.open_helper(filename, 'w') as f: - f.write(content) - - -################################################################################ - - -def remove(f): - if os.path.isdir(f): - shutil.rmtree(f) - elif os.path.isfile(f): - os.remove(f) - - -################################################################################ - - def prepare_vert_file(output_dir: str) -> str: vert_file = os.path.join(output_dir, 'test.vert') if not os.path.isfile(vert_file): @@ -89,18 +71,10 @@ def prepare_vert_file(output_dir: str) -> str: gl_Position = a_position; } ''' - write_to_file(vert_file_default_content, vert_file) + gfuzz_common.write_to_file(vert_file_default_content, vert_file) return vert_file -################################################################################ - - -def remove_end(str_in: str, str_end: str): - assert str_in.endswith(str_end), 'Expected {} to end with {}'.format(str_in, str_end) - return str_in[:-len(str_end)] - - ################################################################################ OPT_OPTIONS = ['--ccp', @@ -168,12 +142,12 @@ def do_image_job( output_dir = os.path.join(work_dir, image_job.name) # Delete and create output directory. - remove(output_dir) + gfuzz_common.remove(output_dir) os.makedirs(output_dir, exist_ok=True) name = image_job.name if name.endswith('.frag'): - name = remove_end(name, '.frag') + name = gfuzz_common.remove_end(name, '.frag') # TODO(324): the worker currently assumes that no vertex shader is present in the image job. vert_file = prepare_vert_file(output_dir) if args.legacy_worker else None @@ -193,12 +167,12 @@ def do_image_job( res.passSanityCheck = True res.log = 'Start: ' + name + '\n' - write_to_file(image_job.fragmentSource, frag_file) - write_to_file(image_job.uniformsInfo, json_file) + gfuzz_common.write_to_file(image_job.fragmentSource, frag_file) + gfuzz_common.write_to_file(image_job.uniformsInfo, json_file) # Set runspv logger. Use try-finally to clean up. - with runspv.open_helper(log_file, 'w') as f: + with gfuzz_common.open_helper(log_file, 'w') as f: try: runspv.log_to_file = f @@ -234,26 +208,26 @@ def do_image_job( force=args.force, is_android=(args.target == 'android'), skip_render=skip_render, - spirv_opt_args=resolved_spirvopt_args, + spirv_opt_args=resolved_spirvopt_args ) except Exception as ex: runspv.log('Exception: ' + str(ex)) runspv.log('Removing STATUS file.') - remove(status_file) + gfuzz_common.remove(status_file) runspv.log('Continuing.') finally: runspv.log_to_file = None if os.path.isfile(log_file): - with runspv.open_helper(log_file, 'r') as f: + with gfuzz_common.open_helper(log_file, 'r') as f: res.log += f.read() if os.path.isfile(png_file): - with runspv.open_bin_helper(png_file, 'rb') as f: + with gfuzz_common.open_bin_helper(png_file, 'rb') as f: res.PNG = f.read() if os.path.isfile(status_file): - with runspv.open_helper(status_file, 'r') as f: + with gfuzz_common.open_helper(status_file, 'r') as f: status = f.read().rstrip() if status == 'SUCCESS': res.status = tt.JobStatus.SUCCESS @@ -267,9 +241,9 @@ def do_image_job( res.status = tt.JobStatus.UNEXPECTED_ERROR elif status == 'NONDET': res.status = tt.JobStatus.NONDET - with runspv.open_bin_helper(nondet_0, 'rb') as f: + with gfuzz_common.open_bin_helper(nondet_0, 'rb') as f: res.PNG = f.read() - with runspv.open_bin_helper(nondet_1, 'rb') as f: + with gfuzz_common.open_bin_helper(nondet_1, 'rb') as f: res.PNG2 = f.read() else: res.log += '\nUnknown status value: ' + status + '\n' @@ -296,7 +270,7 @@ def do_compute_job( output_dir = os.path.join(work_dir, comp_job.name) # Delete and create output directory. - remove(output_dir) + gfuzz_common.remove(output_dir) os.makedirs(output_dir, exist_ok=True) tmpcomp = os.path.join(output_dir, 'tmp.comp') @@ -307,9 +281,9 @@ def do_compute_job( # Output files from running the app. status_file = os.path.join(output_dir, 'STATUS') - write_to_file(comp_job.computeSource, tmpcomp) + gfuzz_common.write_to_file(comp_job.computeSource, tmpcomp) - write_to_file(comp_job.computeInfo, tmpjson) + gfuzz_common.write_to_file(comp_job.computeInfo, tmpjson) res = tt.ImageJobResult() res.log = '#### Start compute shader\n\n' @@ -318,7 +292,7 @@ def do_compute_job( # Set runspv logger. Use try-finally to clean up. - with runspv.open_helper(log_file, 'w') as f: + with gfuzz_common.open_helper(log_file, 'w') as f: try: runspv.log_to_file = f @@ -329,27 +303,27 @@ def do_compute_job( force=args.force, is_android=(args.target == 'android'), skip_render=comp_job.skipRender, - spirv_opt_args=spirv_opt_args, + spirv_opt_args=spirv_opt_args ) except Exception as ex: runspv.log('Exception: ' + str(ex)) runspv.log('Removing STATUS file.') - remove(status_file) + gfuzz_common.remove(status_file) runspv.log('Continuing.') finally: runspv.log_to_file = None if os.path.isfile(log_file): - with runspv.open_helper(log_file, 'r') as f: + with gfuzz_common.open_helper(log_file, 'r') as f: res.log += f.read() if os.path.isfile(status_file): - with runspv.open_helper(status_file, 'r') as f: + with gfuzz_common.open_helper(status_file, 'r') as f: status = f.read().rstrip() if status == 'SUCCESS': res.status = tt.JobStatus.SUCCESS assert (os.path.isfile(ssbo_json_file)) - with runspv.open_helper(ssbo_json_file, 'r') as f: + with gfuzz_common.open_helper(ssbo_json_file, 'r') as f: res.computeOutputs = f.read() elif status == 'CRASH': @@ -513,7 +487,7 @@ def main(): # Get worker info worker_info_file = 'worker_info.json' - remove(worker_info_file) + gfuzz_common.remove(worker_info_file) worker_info_json_string = '{}' @@ -529,7 +503,7 @@ def main(): 'the app permission to write to external storage is enabled.' ) - with runspv.open_helper(worker_info_file, 'r') as f: + with gfuzz_common.open_helper(worker_info_file, 'r') as f: worker_info_json_string = f.read() except Exception as ex: @@ -555,23 +529,23 @@ def main(): assert args.local_shader_job.endswith('.json'), \ 'Expected local shader job "{}" to end with .json' - shader_job_prefix = remove_end(args.local_shader_job, '.json') + shader_job_prefix = gfuzz_common.remove_end(args.local_shader_job, '.json') fake_job = tt.ImageJob() fake_job.name = os.path.basename(shader_job_prefix) assert os.path.isfile(args.local_shader_job), \ 'Shader job {} does not exist'.format(args.local_shader_job) - with runspv.open_helper(args.local_shader_job) as f: + with gfuzz_common.open_helper(args.local_shader_job, 'r') as f: fake_job.uniformsInfo = f.read() if os.path.isfile(shader_job_prefix + '.frag'): - with runspv.open_helper(shader_job_prefix + '.frag', 'r') as f: + with gfuzz_common.open_helper(shader_job_prefix + '.frag', 'r') as f: fake_job.fragmentSource = f.read() if os.path.isfile(shader_job_prefix + '.vert'): - with runspv.open_helper(shader_job_prefix + '.vert', 'r') as f: + with gfuzz_common.open_helper(shader_job_prefix + '.vert', 'r') as f: fake_job.vertexSource = f.read() if os.path.isfile(shader_job_prefix + '.comp'): - with runspv.open_helper(shader_job_prefix + '.comp', 'r') as f: + with gfuzz_common.open_helper(shader_job_prefix + '.comp', 'r') as f: fake_job.computeSource = f.read() fake_job.computeInfo = fake_job.uniformsInfo do_image_job(args, fake_job, spirvopt_args, work_dir='out') diff --git a/python/src/main/python/drivers/graphicsfuzz-piglit-converter b/python/src/main/python/drivers/graphicsfuzz-piglit-converter new file mode 100755 index 000000000..941317d4e --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz-piglit-converter @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if test -n "${PYTHON_GF}"; then + "${PYTHON_GF}" ${BASH_SOURCE}.py "$@" +elif type -P python3 >/dev/null; then + python3 ${BASH_SOURCE}.py "$@" +elif type -P py >/dev/null; then + py -3 ${BASH_SOURCE}.py "$@" +else + python ${BASH_SOURCE}.py "$@" +fi \ No newline at end of file diff --git a/python/src/main/python/drivers/graphicsfuzz-piglit-converter.bat b/python/src/main/python/drivers/graphicsfuzz-piglit-converter.bat new file mode 100644 index 000000000..4860a636c --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz-piglit-converter.bat @@ -0,0 +1,33 @@ +@echo off + +@REM +@REM Copyright 2019 The GraphicsFuzz Project Authors +@REM +@REM Licensed under the Apache License, Version 2.0 (the "License"); +@REM you may not use this file except in compliance with the License. +@REM You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. +@REM + +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* +) ELSE ( + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* + ) ELSE ( + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) + ) +) diff --git a/python/src/main/python/drivers/graphicsfuzz-piglit-converter.py b/python/src/main/python/drivers/graphicsfuzz-piglit-converter.py new file mode 100644 index 000000000..61b081486 --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz-piglit-converter.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import graphicsfuzz_piglit_converter +import sys + +try: + graphicsfuzz_piglit_converter.main_helper(sys.argv[1:]) +except BaseException as error: + sys.stderr.write('Error: ' + str(error) + '\n') + sys.exit(1) diff --git a/python/src/main/python/drivers/graphicsfuzz-tool b/python/src/main/python/drivers/graphicsfuzz-tool new file mode 100644 index 000000000..dbafbbaf2 --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz-tool @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2018 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if test -n "${PYTHON_GF}"; then + "${PYTHON_GF}" ${BASH_SOURCE}.py "$@" +elif type -P python3 >/dev/null; then + python3 ${BASH_SOURCE}.py "$@" +elif type -P py >/dev/null; then + py -3 ${BASH_SOURCE}.py "$@" +else + python ${BASH_SOURCE}.py "$@" +fi diff --git a/python/src/main/python/drivers/graphicsfuzz-tool.bat b/python/src/main/python/drivers/graphicsfuzz-tool.bat new file mode 100644 index 000000000..8d598fb54 --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz-tool.bat @@ -0,0 +1,33 @@ +@echo off + +@REM +@REM Copyright 2018 The GraphicsFuzz Project Authors +@REM +@REM Licensed under the Apache License, Version 2.0 (the "License"); +@REM you may not use this file except in compliance with the License. +@REM You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. +@REM + +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* +) ELSE ( + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* + ) ELSE ( + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) + ) +) diff --git a/python/src/main/python/drivers/graphicsfuzz-tool.py b/python/src/main/python/drivers/graphicsfuzz-tool.py new file mode 100644 index 000000000..0ea2c34ca --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz-tool.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import graphicsfuzz_tool +import sys + +try: + sys.exit(graphicsfuzz_tool.main_helper(sys.argv[1:])) +except ValueError as value_error: + sys.stderr.write(str(value_error)) + sys.exit(1) diff --git a/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py new file mode 100644 index 000000000..e10f2e16e --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz_piglit_converter.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +from typing import List + +import gfuzz_common + +# Note: We define a 'shader job' as the JSON file of a GraphicsFuzz shader. +# Each shader job has a corresponding shader file that has the same base name +# as the shader job - for example, a shader job file named 'variant_005.json' will +# have a corresponding shader file 'variant_005.frag' or 'variant_005.comp' in +# the same directory. + +# Piglit test section headers +REQUIRE_HEADER = '[require]' +VERTEX_HEADER = '[vertex shader passthrough]' +FRAGMENT_HEADER = '[fragment shader]' +TEST_HEADER = '[test]' + +# Draw command for piglit to draw the shader's output. Required in the test header. +DRAW_COMMAND = 'draw rect -1 -1 2 2' +# Command to clear the screen to black before drawing, for the test header. +CLEAR_COMMAND = 'clear color 0.0 0.0 0.0 1.0' + +# Strings used to specify the GL version to use in a piglit test's +# [require] header. +GLES_VERSION_STRING = 'GL ES >= ' +GLSL_VERSION_STRING = 'GLSL >= ' +GLSLES_VERSION_STRING = 'GLSL ES >= ' + +# GLSL preprocessor version flag. +SHADER_VERSION_FLAG = '#version' +# GLES specifier in version string. +ES_SPECIFIER = ' es' +# GLSL version headers specify a version without a decimal point, when piglit takes a version +# string with a decimal point. The easiest way of getting this is by dividing by 100. +SHADER_VERSION_FACTOR = 100 +# Uniform types to be found in the JSON. +UNIFORM_TYPES = { + 'glUniform1i': 'int', + 'glUniform2i': 'ivec2', + 'glUniform3i': 'ivec3', + 'glUniform4i': 'ivec4', + 'glUniform1ui': 'uint', + 'glUniform2ui': 'uvec2', + 'glUniform3ui': 'uvec3', + 'glUniform4ui': 'uvec4', + 'glUniform1f': 'float', + 'glUniform2f': 'vec2', + 'glUniform3f': 'vec3', + 'glUniform4f': 'vec4', + 'glUniformMatrix2fv': 'mat2', + 'glUniformMatrix3fv': 'mat3', + 'glUniformMatrix4fv': 'mat4', + 'glUniformMatrix2x3fv': 'mat2x3', + 'glUniformMatrix3x2fv': 'mat3x2', + 'glUniformMatrix2x4fv': 'mat2x4', + 'glUniformMatrix4x2fv': 'mat4x2', + 'glUniformMatrix3x4fv': 'mat3x4', + 'glUniformMatrix4x3fv': 'mat4x3' +} +UNIFORM_DEC = 'uniform' + + +def make_shader_test_string(shader_job: str, nodraw: bool) -> str: + """ + Makes a piglit shader_test from a shader job and shader. + :param shader_job: The path to the shader job file. + :param nodraw: determines if the draw command is added to draw the shader. + :return: the shader_test + """ + shader_job_json_parsed = get_json_properties(shader_job) + + with gfuzz_common.open_helper(get_shader_from_job(shader_job), 'r') as shader: + shader_file_string = shader.read() + + shader_lines = shader_file_string.split('\n') + shader_test_string = '' + # The version header always has to be on the first line of the shader. + shader_version_header = shader_lines[0] + + shader_test_string += make_require_header(shader_version_header) + '\n' + shader_test_string += make_vertex_shader_header() + '\n' + shader_test_string += make_fragment_shader_header(shader_file_string) + '\n' + shader_test_string += make_test_header(shader_job_json_parsed, nodraw) + + return shader_test_string + + +def make_require_header(shader_version_header: str) -> str: + """ + Creates the piglit [require] header as well as the required GL and GLSL version strings. + Note: GLSL version strings are formatted as '#version ### (es)', where ### is a + specific GLSL version multiplied by 100, and (es) is an optional specifier that + determines whether to use GLES or not. + :param shader_version_header: the version string in the GLSL file. + :return: the shader_test require header string. + """ + require_header = REQUIRE_HEADER + '\n' + # Piglit requires a version number with 1 digit of precision for the GL version, and + # 2 digits of precision for the GLSL version. + try: + shader_version = shader_version_header.split(' ')[1] + except IndexError: + raise IOError('Malformed shader - invalid GLSL version string.') + if not shader_version.isdigit(): + raise IOError('Malformed shader - invalid GLSL version string.') + # Piglit requires GL version to be specified explicitly if ES is in use. + if ES_SPECIFIER in shader_version_header: + require_header += GLES_VERSION_STRING + \ + format(float(shader_version) / SHADER_VERSION_FACTOR, '.1f') + '\n' + require_header += GLSLES_VERSION_STRING + else: + require_header += GLSL_VERSION_STRING + require_header += format(float(shader_version) / SHADER_VERSION_FACTOR, '.2f') + '\n' + return require_header + + +def make_vertex_shader_header() -> str: + """ + Creates the piglit [vertex shader] header. Currently uses passthrough, but this function can + be modified later if we have a need for an explicit vertex shader. + :return: the shader_test vertex header string. + """ + return VERTEX_HEADER + '\n' + + +def make_fragment_shader_header(fragment_shader: str) -> str: + """ + Creates the piglit [fragment shader] header. + :param fragment_shader: the fragment shader code. + :return: the shader_test fragment header string. + """ + frag_header = FRAGMENT_HEADER + '\n' + frag_header += fragment_shader + return frag_header + + +def make_test_header(shader_job_json_parsed: dict, nodraw: bool) -> str: + """ + Creates the [test] header. Loads uniforms based on the uniforms found in the JSON file. + :param shader_job_json_parsed: the parsed JSON properties. + :param nodraw: determines if the draw command is added to draw the shader. + :return: the shader_test test header string. + """ + test_header = TEST_HEADER + '\n' + for uniform_name, value in shader_job_json_parsed.items(): + test_header += UNIFORM_DEC + ' {type} {uniform_name} {args}\n'.format( + type=get_uniform_type_from_gl_func(value['func']), + uniform_name=uniform_name, + args=' '.join([str(arg) for arg in value['args']]) + ) + if not nodraw: + test_header += CLEAR_COMMAND + '\n' + test_header += DRAW_COMMAND + return test_header + + +def get_uniform_type_from_gl_func(func: str) -> str: + """ + Helper function to traverse the dict of JSON funcs to determine a uniform's type. + Throws AssertionError if the uniform type is not known. + :param func: the function to check. + :return: the GLSL type of the uniform. + """ + if func not in UNIFORM_TYPES.keys(): + raise AssertionError('Unknown uniform type: ' + func) + return UNIFORM_TYPES[func] + + +def is_version_header(line: str) -> bool: + """ + Helper function to see if a given string is a GLSL preprocessor version string. + :param line: the line of code to check. + :return: True if the string is a GLSL preprocessor version string, False otherwise. + """ + return SHADER_VERSION_FLAG in line + + +def get_json_properties(shader_job: str) -> dict: + """ + Helper function to parse a shader job JSON file into a dict of properties. + Throws IOError if the file can't be parsed. + :param shader_job: the path to the shader job file. + :return: a dict of JSON properties. + """ + with gfuzz_common.open_helper(shader_job, 'r') as job: + json_parsed = json.load(job) + return json_parsed + + +def get_shader_from_job(shader_job: str) -> str: + """ + Helper function to get the filename of a shader file from its corresponding shader job. + :param shader_job: the path of the shader job file. + :return: the path of the fragment shader file. + """ + return gfuzz_common.remove_end(shader_job, '.json') + '.frag' + + +def get_shader_test_from_job(shader_job: str) -> str: + """ + Helper function to get the filename of the new shader test from the given shader job. + :param shader_job: the path of the shader job file. + :return: the path of the shader_test file. + """ + return gfuzz_common.remove_end(shader_job, '.json') + '.shader_test' + + +def main_helper(args: List[str]) -> None: + """ + Main function. Parses arguments, delegates to other functions to write the shader_test string, + and writes the string to file. + :param args: the command line arguments. + """ + description = ( + 'Given a GraphicsFuzz shader job JSON file, produce a Mesa piglit shader_test file. ' + 'The shader test will be the same name as the shader job.') + + argparser = argparse.ArgumentParser(description=description) + + argparser.add_argument( + 'shader_job', + help='Path to the GraphicsFuzz shader job JSON file.') + + argparser.add_argument( + '--nodraw', + action='store_true', + help='Do not draw the shader output when running the test. Useful for crash testing.' + ) + + args = argparser.parse_args(args) + + test_string = make_shader_test_string(args.shader_job, args.nodraw) + + with gfuzz_common.open_helper(get_shader_test_from_job(args.shader_job), 'w') as shader_test: + shader_test.write(test_string) diff --git a/python/src/main/python/drivers/graphicsfuzz_tool.py b/python/src/main/python/drivers/graphicsfuzz_tool.py new file mode 100644 index 000000000..2bb81d6c2 --- /dev/null +++ b/python/src/main/python/drivers/graphicsfuzz_tool.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +# Copyright 2018 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +HERE = os.path.abspath(__file__) + +sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) + +import cmd_helpers + + +def main_helper(argv): + java_tool_path = cmd_helpers.get_tool_path() + print(java_tool_path) + + # Run the tool + + cmd = ["java", "-ea", "-cp", java_tool_path] + argv + print(cmd) + + generate_proc = subprocess.Popen(cmd) + generate_proc.communicate() + return generate_proc.returncode + + +if __name__ == "__main__": + sys.exit(main_helper(sys.argv[1:])) diff --git a/python/src/main/python/drivers/inspect-compute-results.bat b/python/src/main/python/drivers/inspect-compute-results.bat index aded83e44..4860a636c 100644 --- a/python/src/main/python/drivers/inspect-compute-results.bat +++ b/python/src/main/python/drivers/inspect-compute-results.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/inspect_compute_results.py b/python/src/main/python/drivers/inspect_compute_results.py index ba4238986..9781ffd73 100644 --- a/python/src/main/python/drivers/inspect_compute_results.py +++ b/python/src/main/python/drivers/inspect_compute_results.py @@ -21,13 +21,15 @@ import sys from typing import Callable, List, Optional, Tuple +import gfuzz_common + DEFAULT_REL_TOL = '1e-9' DEFAULT_ABS_TOL = '1e-20' def get_ssbo(result_json_filename: str) -> List: - with open(result_json_filename, 'r', encoding='utf-8', errors='ignore') as f: + with gfuzz_common.open_helper(result_json_filename, 'r') as f: parsed = json.load(f) if not parsed or 'outputs' not in parsed or 'ssbo' not in parsed['outputs']: raise ValueError('No SSBO data found') @@ -91,12 +93,6 @@ def fuzzydiff_ssbos(result_json_filename_1: str, lambda x, y: math.isclose(x, y, rel_tol=rel_tol, abs_tol=abs_tol)) -def check_input_files_exist(filenames: List[str]) -> None: - for filename in filenames: - if not os.path.isfile(filename): - raise FileNotFoundError('Input file "' + filename + '" not found') - - def main_helper(args: List[str]) -> int: description = ( 'Inspect and compare compute shader outputs.') @@ -154,7 +150,7 @@ def main_helper(args: List[str]) -> int: if len(args.inputs) != 1: raise ValueError( 'Command "show" requires exactly 1 input; ' + str(len(args.inputs)) + ' provided') - check_input_files_exist([args.inputs[0]]) + gfuzz_common.check_input_files_exist([args.inputs[0]]) show_ssbo(args.inputs[0]) return 0 @@ -162,7 +158,7 @@ def main_helper(args: List[str]) -> int: if len(args.inputs) != 2: raise ValueError('Command "exactdiff" requires exactly 2 inputs; ' + str( len(args.inputs)) + ' provided') - check_input_files_exist([args.inputs[0], args.inputs[1]]) + gfuzz_common.check_input_files_exist([args.inputs[0], args.inputs[1]]) result, msg = exactdiff_ssbos(args.inputs[0], args.inputs[1]) if result: return 0 @@ -173,7 +169,7 @@ def main_helper(args: List[str]) -> int: if len(args.inputs) != 2: raise ValueError('Command "fuzzydiff" requires exactly 2 inputs; ' + str( len(args.inputs)) + ' provided') - check_input_files_exist([args.inputs[0], args.inputs[1]]) + gfuzz_common.check_input_files_exist([args.inputs[0], args.inputs[1]]) result, msg = fuzzydiff_ssbos(args.inputs[0], args.inputs[1], abs_tol, rel_tol) if result: return 0 diff --git a/python/src/main/python/drivers/knownvalue-shader-generator b/python/src/main/python/drivers/knownvalue-shader-generator new file mode 100755 index 000000000..4ec96a5bb --- /dev/null +++ b/python/src/main/python/drivers/knownvalue-shader-generator @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if test -n "${PYTHON_GF}"; then + "${PYTHON_GF}" ${BASH_SOURCE}.py "$@" +elif type -P python3 >/dev/null; then + python3 ${BASH_SOURCE}.py "$@" +elif type -P py >/dev/null; then + py -3 ${BASH_SOURCE}.py "$@" +else + python ${BASH_SOURCE}.py "$@" +fi diff --git a/python/src/main/python/drivers/knownvalue-shader-generator.bat b/python/src/main/python/drivers/knownvalue-shader-generator.bat new file mode 100644 index 000000000..4860a636c --- /dev/null +++ b/python/src/main/python/drivers/knownvalue-shader-generator.bat @@ -0,0 +1,33 @@ +@echo off + +@REM +@REM Copyright 2019 The GraphicsFuzz Project Authors +@REM +@REM Licensed under the Apache License, Version 2.0 (the "License"); +@REM you may not use this file except in compliance with the License. +@REM You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. +@REM + +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* +) ELSE ( + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* + ) ELSE ( + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) + ) +) diff --git a/python/src/main/python/drivers/knownvalue-shader-generator.py b/python/src/main/python/drivers/knownvalue-shader-generator.py new file mode 100644 index 000000000..aa773ba57 --- /dev/null +++ b/python/src/main/python/drivers/knownvalue-shader-generator.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +HERE = os.path.abspath(__file__) + +sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) + +import cmd_helpers + + +def go(argv): + java_tool_path = cmd_helpers.get_tool_path() + print(java_tool_path) + + cmd = ["java", "-ea", "-cp", java_tool_path, "com.graphicsfuzz.generator.tool.KnownValueShaderGenerator"] \ + + argv + print(cmd) + + generate_proc = subprocess.Popen(cmd) + generate_proc.communicate() + return generate_proc.returncode + + +if __name__ == "__main__": + sys.exit(go(sys.argv[1:])) diff --git a/python/src/main/python/drivers/piglit-worker b/python/src/main/python/drivers/piglit-worker new file mode 100755 index 000000000..4ec96a5bb --- /dev/null +++ b/python/src/main/python/drivers/piglit-worker @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if test -n "${PYTHON_GF}"; then + "${PYTHON_GF}" ${BASH_SOURCE}.py "$@" +elif type -P python3 >/dev/null; then + python3 ${BASH_SOURCE}.py "$@" +elif type -P py >/dev/null; then + py -3 ${BASH_SOURCE}.py "$@" +else + python ${BASH_SOURCE}.py "$@" +fi diff --git a/python/src/main/python/drivers/piglit-worker.bat b/python/src/main/python/drivers/piglit-worker.bat new file mode 100644 index 000000000..4860a636c --- /dev/null +++ b/python/src/main/python/drivers/piglit-worker.bat @@ -0,0 +1,33 @@ +@echo off + +@REM +@REM Copyright 2019 The GraphicsFuzz Project Authors +@REM +@REM Licensed under the Apache License, Version 2.0 (the "License"); +@REM you may not use this file except in compliance with the License. +@REM You may obtain a copy of the License at +@REM +@REM https://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. +@REM + +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* +) ELSE ( + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* + ) ELSE ( + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) + ) +) diff --git a/python/src/main/python/drivers/piglit-worker.py b/python/src/main/python/drivers/piglit-worker.py new file mode 100644 index 000000000..20ab716a6 --- /dev/null +++ b/python/src/main/python/drivers/piglit-worker.py @@ -0,0 +1,413 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import os +import sys +import filecmp +import shutil +import time +from typing import List + +import gfuzz_common +import graphicsfuzz_piglit_converter + +HERE = os.path.abspath(__file__) + +# Add directory above to Python path for access to dependencies. +# Prepend it so we override any globally installed dependencies. +sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) + +# noinspection PyPep8 +from fuzzer_service import FuzzerService +# noinspection PyPep8 +import fuzzer_service.ttypes as tt +# noinspection PyPep8 +from thrift.transport import THttpClient, TTransport +# noinspection PyPep8 +from thrift.Thrift import TApplicationException +# noinspection PyPep8 +from thrift.protocol import TBinaryProtocol + +FRAG_SUFFIX = '.frag' +JSON_SUFFIX = '.json' +PNG_SUFFIX = '.png' +SHADER_TEST_SUFFIX = '.shader_test' + +SHADER_RUNNER_ARG_PNG = '-png' +SHADER_RUNNER_ARG_AUTO = '-auto' +SHADER_RUNNER_ARG_UNIFORMS = '-ignore-missing-uniforms' +SHADER_RUNNER_ARG_FBO = '-fbo' +SHADER_RUNNER_ARG_SUBTESTS = '-report-subtests' + +WORKER_INFO_FILE = 'worker_info.json' +PNG_FILENAME = 'shader_runner_gles3000.png' +COMPARE_PNG_FILENAME = 'shader_runner_gles3001.png' +NONDET0_PNG = 'nondet0.png' +NONDET1_PNG = 'nondet1.png' +LOGFILE_NAME = 'piglit_log.txt' +STATUS_FILENAME = 'STATUS' + +NO_DRAW_ARG = '--nodraw' + +STATUS_SUCCESS = 'SUCCESS' +STATUS_CRASH = 'CRASH' +STATUS_TIMEOUT = 'TIMEOUT' +STATUS_UNEXPECTED = 'UNEXPECTED_ERROR' +STATUS_NONDET = 'NONDET' + +TIMEOUT = 30 + + +def glxinfo_cmd() -> List[str]: + return [gfuzz_common.tool_on_path('glxinfo'), '-B'] + + +def shader_runner_cmd() -> List[str]: + return [gfuzz_common.tool_on_path('shader_runner_gles3')] + + +def catchsegv_cmd() -> str: + return gfuzz_common.tool_on_path('catchsegv') + + +def thrift_connect(server: str, worker_name: str, worker_info: str) -> (FuzzerService, str): + """ + Helper function to initiate a connection from this worker to a Thrift server. + Handles sending the worker name and info to the server. If there's a fatal problem with a worker + (such as a worker info mismatch between client/server), this function will terminate the + program. + :param server: The server request URL to connect to. + :param worker_name: The name of the worker to connect with. + :param worker_info: The worker info string. + :return: a FuzzerService object and the confirmed worker name, or None for both if the + connection failed without a fatal error. + """ + try: + http_client = THttpClient.THttpClient(server) + transport = TTransport.TBufferedTransport(http_client) + protocol = TBinaryProtocol.TBinaryProtocol(transport) + service = FuzzerService.Client(protocol) + transport.open() + + # Get worker name + gfuzz_common.log("Call getWorkerName()") + worker_res = service.getWorkerName(worker_info, worker_name) + assert type(worker_res) is not None + + if worker_res.workerName is None: + # noinspection PyProtectedMember + gfuzz_common.log('Worker error: ' + tt.WorkerNameError._VALUES_TO_NAMES[worker_res.error]) + exit(1) + + worker = worker_res.workerName + + gfuzz_common.log("Got worker: " + worker) + assert (worker == worker_name) + + return service, worker + + except (TApplicationException, ConnectionRefusedError, ConnectionResetError): + return None, None + + +def dump_glxinfo(filename: str) -> None: + """ + Helper function that dumps the stable results of 'glxinfo -B' to a JSON file. Removes any + file with the same name as filename before writing. Will throw an exception if 'glxinfo' + fails or the JSON file can't be written. + :param filename: the filename to write to. + """ + # There are some useless or unstable lines in glxinfo we need to remove before trying to parse + # into JSON. + glxinfo_lines = filter( + lambda glx_line: 'OpenGL' in glx_line, + gfuzz_common.subprocess_helper(glxinfo_cmd()).stdout.split('\n')) + # We form keys out of the OpenGL info descriptors and values out of the hardware dependent + # strings. For example, "OpenGL version string: 4.6.0 NVIDIA 430.14" would become + # { "OpenGL version string": "4.6.0 NVIDIA 430.14" }. + glx_dict = dict() + for line in glxinfo_lines: + prop = line.split(': ') + assert len(prop) is 2 + glx_dict.update({prop[0]: prop[1]}) + with gfuzz_common.open_helper(filename, 'w') as info_file: + info_file.write(json.JSONEncoder().encode(glx_dict)) + + +def do_image_job(image_job: tt.ImageJob, work_dir: str) -> tt.ImageJobResult: + """ + Does an image job. Sets up directories and some files, then delegates to run_image_job to + convert the job to a shader_test and run it. Sets a global logfile to log to for the lifetime + of the function. Gets the status of the shader job from a file that is written to by + run_image_job. + :param image_job: the image job containing the shader/uniforms. + :param work_dir: the directory to work in. + :return: the result of the image job, including the log, PNG and status. + """ + # Output directory is based on the name of job. + output_dir = os.path.join(work_dir, image_job.name) + + # Delete and create output directory. + gfuzz_common.remove(output_dir) + os.makedirs(output_dir, exist_ok=True) + + name = image_job.name + if name.endswith('.frag'): + name = gfuzz_common.remove_end(name, '.frag') + + frag_file = os.path.join(output_dir, name + FRAG_SUFFIX) + json_file = os.path.join(output_dir, name + JSON_SUFFIX) + log_file = os.path.join(output_dir, LOGFILE_NAME) + status_file = os.path.join(output_dir, STATUS_FILENAME) + png_file = os.path.join(output_dir, name + PNG_SUFFIX) + nondet_0 = os.path.join(output_dir, NONDET0_PNG) + nondet_1 = os.path.join(output_dir, NONDET1_PNG) + + gfuzz_common.write_to_file(image_job.fragmentSource, frag_file) + gfuzz_common.write_to_file(image_job.uniformsInfo, json_file) + + res = tt.ImageJobResult() + + # Set nice defaults to fields we will not update anyway + res.passSanityCheck = True + res.log = 'Start: ' + name + '\n' + + with gfuzz_common.open_helper(log_file, 'w') as f: + try: + gfuzz_common.set_logfile(f) + run_image_job(json_file, status_file, png_file, output_dir, image_job.skipRender) + except Exception as ex: + gfuzz_common.log(str(ex)) + gfuzz_common.log('Removing status file and continuing...') + gfuzz_common.remove(status_file) + finally: + gfuzz_common.unset_logfile() + + if os.path.isfile(log_file): + with gfuzz_common.open_helper(log_file, 'r') as f: + res.log += f.read() + + if os.path.isfile(png_file): + with gfuzz_common.open_bin_helper(png_file, 'rb') as f: + res.PNG = f.read() + + if os.path.isfile(status_file): + with gfuzz_common.open_helper(status_file, 'r') as f: + status = f.read().rstrip() + if status == STATUS_SUCCESS: + res.status = tt.JobStatus.SUCCESS + elif status == STATUS_CRASH: + res.status = tt.JobStatus.CRASH + elif status == STATUS_TIMEOUT: + res.status = tt.JobStatus.TIMEOUT + elif status == STATUS_UNEXPECTED: + res.status = tt.JobStatus.UNEXPECTED_ERROR + elif status == STATUS_NONDET: + res.status = tt.JobStatus.NONDET + with gfuzz_common.open_bin_helper(nondet_0, 'rb') as f: + res.PNG = f.read() + with gfuzz_common.open_bin_helper(nondet_1, 'rb') as f: + res.PNG2 = f.read() + else: + res.log += '\nUnknown status value: ' + status + '\n' + res.status = tt.JobStatus.UNEXPECTED_ERROR + else: + # Not even a status file? + res.log += '\nNo STATUS file\n' + res.status = tt.JobStatus.UNEXPECTED_ERROR + + return res + + +def run_image_job(json_file: str, status_file: str, + png_file: str, output_dir: str, skip_render: bool): + """ + Runs an image job. Converts the shader job to a piglit shader_test file, then delegates to + run_shader_test to render with shader_runner. Writes the status of the job to file. + :param json_file: The JSON uniforms to use with the shader. + :param status_file: The status file to write to. + :param png_file: The PNG file to write to. + :param output_dir: The directory to use for the job. + :param skip_render: whether to skip rendering or not. + """ + + use_catchsegv = True + + try: + gfuzz_common.tool_on_path('catchsegv') + except gfuzz_common.ToolNotOnPathError: + use_catchsegv = False + + assert os.path.isdir(output_dir) + assert os.path.isfile(json_file) + + arglist = [json_file] + if skip_render: + arglist.append(NO_DRAW_ARG) + + shader_test_file = graphicsfuzz_piglit_converter.get_shader_test_from_job(json_file) + + try: + gfuzz_common.log('Creating shader_test file...') + graphicsfuzz_piglit_converter.main_helper(arglist) + except Exception as ex: + gfuzz_common.log('Could not create shader_test from the given job.') + raise ex + shader_runner_cmd_list = shader_runner_cmd() + \ + [shader_test_file, SHADER_RUNNER_ARG_AUTO, + SHADER_RUNNER_ARG_UNIFORMS, SHADER_RUNNER_ARG_FBO, SHADER_RUNNER_ARG_SUBTESTS] + if use_catchsegv: + shader_runner_cmd_list.insert(0, catchsegv_cmd()) + if not skip_render: + shader_runner_cmd_list.append(SHADER_RUNNER_ARG_PNG) + + gfuzz_common.remove(PNG_FILENAME) + gfuzz_common.remove(COMPARE_PNG_FILENAME) + + status = \ + gfuzz_common.run_catchsegv(shader_runner_cmd_list, timeout=TIMEOUT, verbose=True) \ + if use_catchsegv else \ + gfuzz_common.subprocess_helper(shader_runner_cmd_list, timeout=TIMEOUT, verbose=True) + + # Piglit throws the output PNG render into whatever the current working directory is + # (and there's no way to specify a location to write to) - we need to move it to wherever our + # output is. + + if not skip_render and status == STATUS_SUCCESS: + try: + # An image was rendered, so we need to check for nondet. We do this by renaming the + # rendered image, rendering a second image, and using filecmp to compare the files. + assert os.path.isfile(PNG_FILENAME), \ + "Shader runner successfully rendered, but no image was dumped?" + gfuzz_common.log('An image was rendered - rendering again to check for nondet.') + os.rename(PNG_FILENAME, COMPARE_PNG_FILENAME) + status = \ + gfuzz_common.run_catchsegv(shader_runner_cmd_list, timeout=TIMEOUT, verbose=True) \ + if use_catchsegv else \ + gfuzz_common.subprocess_helper(shader_runner_cmd_list, timeout=TIMEOUT, verbose=True) + # Something is horribly wrong if shader crashes/timeouts are inconsistent per shader. + assert status == STATUS_SUCCESS, \ + "Shader inconsistently fails - check your graphics drivers?" + assert os.path.isfile(PNG_FILENAME), \ + "Shader runner successfully rendered, but no image was dumped?" + + gfuzz_common.log('Comparing dumped PNG images...') + if filecmp.cmp(PNG_FILENAME, COMPARE_PNG_FILENAME): + gfuzz_common.log('Images are identical.') + shutil.move(PNG_FILENAME, png_file) + else: + gfuzz_common.log('Images are different.') + status = STATUS_NONDET + shutil.move(COMPARE_PNG_FILENAME, os.path.join(output_dir, NONDET0_PNG)) + shutil.move(PNG_FILENAME, os.path.join(output_dir, NONDET1_PNG)) + finally: + gfuzz_common.log('Removing dumped images...') + gfuzz_common.remove(PNG_FILENAME) + gfuzz_common.remove(COMPARE_PNG_FILENAME) + + gfuzz_common.log('STATUS: ' + status) + + with gfuzz_common.open_helper(status_file, 'w') as f: + f.write(status) + + +def main(): + description = ( + 'Uses the piglit GLES3 shader runner to render shader jobs.' + ) + + parser = argparse.ArgumentParser(description=description) + + # Required + parser.add_argument( + 'worker_name', + help='The name that will refer to this worker.' + ) + + # Optional + parser.add_argument( + '--server', + default='http://localhost:8080', + help='Server URL to connect to (default: http://localhost:8080 )' + ) + + args = parser.parse_args() + + gfuzz_common.tool_on_path('shader_runner_gles3') + gfuzz_common.log('Worker: ' + args.worker_name) + server = args.server + '/request' + gfuzz_common.log('server: ' + server) + + # Get worker info + worker_info_json_string = '{}' + + gfuzz_common.log('Dumping glxinfo to file for worker info string...') + try: + dump_glxinfo(WORKER_INFO_FILE) + with gfuzz_common.open_helper(WORKER_INFO_FILE, 'r') as info_file: + worker_info_json_string = info_file.read() + except Exception as ex: + gfuzz_common.log(str(ex)) + gfuzz_common.log('Could not get worker info, continuing without it.') + + service = None + worker = None + + while True: + if not service: + gfuzz_common.log('Connecting to server...') + service, worker = thrift_connect(server, args.worker_name, worker_info_json_string) + if not service: + gfuzz_common.log('Failed to connect, retrying...') + time.sleep(1) + continue + + assert worker + + os.makedirs(worker, exist_ok=True) + + try: + job = service.getJob(worker) + if job.noJob is not None: + gfuzz_common.log("No job") + elif job.skipJob is not None: + gfuzz_common.log("Skip job") + service.jobDone(worker, job) + else: + assert job.imageJob + if job.imageJob.computeSource: + gfuzz_common.log("Got a compute job, but this worker " + "doesn't support compute shaders.") + job.imageJob.result = tt.ImageJobResult() + job.imageJob.result.status = tt.JobStatus.UNEXPECTED_ERROR + else: + gfuzz_common.log("#### Image job: " + job.imageJob.name) + job.imageJob.result = do_image_job(job.imageJob, work_dir=worker) + gfuzz_common.log("Sending back, results status: {}" + .format(job.imageJob.result.status)) + service.jobDone(worker, job) + gfuzz_common.remove(worker) + continue + except (TApplicationException, ConnectionError): + gfuzz_common.log("Connection to server lost. Re-initialising client.") + service = None + time.sleep(1) + + +if __name__ == '__main__': + main() diff --git a/python/src/main/python/drivers/report-compute-shader-family-results.bat b/python/src/main/python/drivers/report-compute-shader-family-results.bat index aded83e44..4860a636c 100644 --- a/python/src/main/python/drivers/report-compute-shader-family-results.bat +++ b/python/src/main/python/drivers/report-compute-shader-family-results.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/run-shader-family.bat b/python/src/main/python/drivers/run-shader-family.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/run-shader-family.bat +++ b/python/src/main/python/drivers/run-shader-family.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/runspv.bat b/python/src/main/python/drivers/runspv.bat index cafd86cce..8d598fb54 100644 --- a/python/src/main/python/drivers/runspv.bat +++ b/python/src/main/python/drivers/runspv.bat @@ -16,14 +16,18 @@ @REM limitations under the License. @REM -where /q py -IF ERRORLEVEL 0 ( - py -3 "%~dpn0.py" %* +IF DEFINED PYTHON_GF ( + "%PYTHON_GF%" "%~dpn0.py" %* ) ELSE ( - where /q python3 - IF ERRORLEVEL 0 ( - python3 "%~dpn0.py" %* + where /q py + IF %ERRORLEVEL% EQU 0 ( + py -3 "%~dpn0.py" %* ) ELSE ( - python "%~dpn0.py" %* + where /q python3 + IF %ERRORLEVEL% EQU 0 ( + python3 "%~dpn0.py" %* + ) ELSE ( + python "%~dpn0.py" %* + ) ) ) diff --git a/python/src/main/python/drivers/runspv.py b/python/src/main/python/drivers/runspv.py index 937e0d72b..fe1f82ca8 100755 --- a/python/src/main/python/drivers/runspv.py +++ b/python/src/main/python/drivers/runspv.py @@ -327,19 +327,47 @@ def remove_end(str_in: str, str_end: str): def run_spirv_opt( spv_file: str, - spirv_opt_args: List[str] -) -> str: + spirv_opt_args: List[str], + output: str +) -> None: + """ + Optimizes a SPIR-V file. - log('Running optimizer.') + :param spv_file: name of SPIR-V file to be optimized + :param spirv_opt_args: arguments to be passed to spirv-opt + :param output: name of file into which optimized SPIR-V will be written + :return: None + """ - result = spv_file + '.opt.spv' + log('Running optimizer.') - cmd = [spirvopt_path(), spv_file, '-o', result] + cmd = [spirvopt_path(), spv_file, '-o', output] cmd += spirv_opt_args subprocess_helper(cmd, timeout=TIMEOUT_SPIRV_OPT_SECONDS) - return result + +def convert_glsl_to_spv( + glsl_shader: str, + output: str +) -> None: + + """ + Runs glslangValidator on a GLSL shader to convert it to SPIR-V. + + :param glsl_shader: name of GLSL file on which to run glslangValidator + :param output: name of file into which generated SPIR-V will be written + :return: None + """ + + log('Running glslangValidator.') + + cmd = [ + glslang_path(), + '-V', glsl_shader, + '-o', output + ] + subprocess_helper(cmd, timeout=TIMEOUT_RUN) def filename_extension_suggests_glsl(file: str): @@ -406,7 +434,9 @@ def prepare_shader( assert len(result) > 0 if spirv_opt_args: - result = run_spirv_opt(result, spirv_opt_args) + optimized_spv = result + '.opt.spv' + run_spirv_opt(result, spirv_opt_args, optimized_spv) + result = optimized_spv return result @@ -428,7 +458,7 @@ def prepare_shader( def adb_helper( adb_args: List[str], check: bool, - verbose: bool=False + verbose: bool = False ) -> subprocess.CompletedProcess: adb_cmd = [adb_path()] + adb_args @@ -443,7 +473,7 @@ def adb_helper( def adb_check( adb_args: List[str], - verbose: bool=False + verbose: bool = False ) -> subprocess.CompletedProcess: return adb_helper(adb_args, check=True, verbose=verbose) @@ -451,7 +481,7 @@ def adb_check( def adb_can_fail( adb_args: List[str], - verbose: bool=False + verbose: bool = False ) -> subprocess.CompletedProcess: return adb_helper(adb_args, check=False, verbose=verbose) @@ -794,9 +824,75 @@ def spv_get_disassembly(shader_filename): return subprocess_helper(cmd).stdout -def uniform_json_to_amberscript(uniform_json): - """ - Returns the string representing VkScript version of uniform declarations. +def get_shader_as_comment(shader: str) -> str: + with open_helper(shader, 'r') as f: + lines = f.readlines() + + lines = [('# ' + line).rstrip() for line in lines] + return '\n'.join(lines) + + +def get_spirv_opt_args_comment(spirv_args: Optional[List[str]]) -> str: + if spirv_args: + result = '# spirv-opt was used with the following arguments:\n' + args = ['# \'{}\''.format(arg) for arg in spirv_args] + result += '\n'.join(args) + result += '\n\n' + return result + else: + return '' + +def get_header_comment_original_source( + vert_original: str, + frag_original: str, + comp_original: str, + spirv_args: str +): + has_frag_glsl = frag_original and filename_extension_suggests_glsl(frag_original) + has_vert_glsl = vert_original and filename_extension_suggests_glsl(vert_original) + has_comp_glsl = comp_original and filename_extension_suggests_glsl(comp_original) + + result = get_spirv_opt_args_comment(spirv_args) + + if has_frag_glsl or has_vert_glsl or has_comp_glsl: + result += '# Derived from the following GLSL.\n\n' + + if has_vert_glsl: + result += '# Vertex shader GLSL:\n' + result += get_shader_as_comment(vert_original) + result += '\n\n' + + if has_frag_glsl: + result += '# Fragment shader GLSL:\n' + result += get_shader_as_comment(frag_original) + result += '\n\n' + + if has_comp_glsl: + result += '# Compute shader GLSL:\n' + result += get_shader_as_comment(comp_original) + result += '\n\n' + + return result + + +def get_header_comment_original_source_image( + vert_original: str, + frag_original: str, + spirv_args: str +): + return get_header_comment_original_source(vert_original, frag_original, None, spirv_args) + + +def get_header_comment_original_source_comp( + comp_original: str, + spirv_args: str +): + return get_header_comment_original_source(None, None, comp_original, spirv_args) + + +def amberscript_uniform_buffer_decl(uniform_json): + ''' + Returns the string representing AmberScript version of uniform declarations. Skips the special '$compute' key, if present. { @@ -811,26 +907,34 @@ def uniform_json_to_amberscript(uniform_json): becomes: # myuniform - uniform ubo 0:3 float 0 42.0 + BUFFER myuniform DATA_TYPE float DATA + 42.0 + END - """ + ''' - uniform_types = { - 'glUniform1i': 'int', - 'glUniform2i': 'ivec2', - 'glUniform3i': 'ivec3', - 'glUniform4i': 'ivec4', + UNIFORM_TYPE = { + 'glUniform1i': 'int32', + 'glUniform2i': 'vec2', + 'glUniform3i': 'vec3', + 'glUniform4i': 'vec4', 'glUniform1f': 'float', - 'glUniform2f': 'vec2', - 'glUniform3f': 'vec3', - 'glUniform4f': 'vec4', + 'glUniform2f': 'vec2', + 'glUniform3f': 'vec3', + 'glUniform4f': 'vec4', + 'glUniformMatrix2fv': 'mat2x2', + 'glUniformMatrix2x3fv': 'mat2x3', + 'glUniformMatrix2x4fv': 'mat2x4', + 'glUniformMatrix3x2fv': 'mat3x2', + 'glUniformMatrix3fv': 'mat3x3', + 'glUniformMatrix3x4fv': 'mat3x4', + 'glUniformMatrix4x2fv': 'mat4x2', + 'glUniformMatrix4x3fv': 'mat4x3', + 'glUniformMatrix4fv': 'mat4x4', } - descriptor_set = 0 # always 0 in our tests - offset = 0 # We never have uniform offset in our tests - result = '' - with open_helper(uniform_json, 'r') as f: + with open(uniform_json, 'r') as f: j = json.load(f) for name, entry in j.items(): @@ -838,38 +942,51 @@ def uniform_json_to_amberscript(uniform_json): continue func = entry['func'] - if func not in uniform_types.keys(): - raise AssertionError('Error: unknown uniform type for function: ' + func) - uniform_type = uniform_types[func] + if func not in UNIFORM_TYPE.keys(): + raise ValueError('Error: unknown uniform type for function: ' + func) + uniform_type = UNIFORM_TYPE[func] result += '# ' + name + '\n' - result += 'uniform ubo {}:{}'.format(descriptor_set, entry['binding']) - result += ' ' + uniform_type - result += ' {}'.format(offset) + result += 'BUFFER ' + name + ' DATA_TYPE ' + uniform_type + ' DATA\n' for arg in entry['args']: result += ' {}'.format(arg) result += '\n' + result += 'END\n' return result -def get_shader_as_comment(shader: str) -> str: - with open_helper(shader, 'r') as f: - lines = f.readlines() +def amberscript_uniform_buffer_bind(uniform_json): + ''' + Returns the string representing AmberScript version of uniform binding. + Skips the special '$compute' key, if present. - lines = [('# ' + line).rstrip() for line in lines] - return '\n'.join(lines) + { + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, + "$compute": { ... will be ignored ... } + } + becomes: -def get_spirv_opt_args_comment(spirv_args: Optional[List[str]]) -> str: - if spirv_args: - result = '# spirv-opt was used with the following arguments:\n' - args = ['# \'{}\''.format(arg) for arg in spirv_args] - result += '\n'.join(args) - result += '\n\n' - return result - else: - return '' + BIND BUFFER myuniform AS uniform DESCRIPTOR_SET 0 BINDING 3 + ''' + + result = '' + with open(uniform_json, 'r') as f: + j = json.load(f) + for name, entry in j.items(): + + if name == '$compute': + continue + + result += 'BIND BUFFER ' + name + ' AS uniform ' + result += 'DESCRIPTOR_SET 0 BINDING {}\n'.format(entry['binding']) + + return result def amberscriptify_image( @@ -880,54 +997,49 @@ def amberscriptify_image( frag_original, spirv_args ): - """ - Generates Amberscript representation of an image test - """ - - result = '# Generated\n\n' - - result += '# A test for a bug found by GraphicsFuzz.\n\n' + ''' + Generates AmberScript representation of an image test + ''' has_frag_glsl = frag_original and filename_extension_suggests_glsl(frag_original) has_vert_glsl = vert_original and filename_extension_suggests_glsl(vert_original) - result += get_spirv_opt_args_comment(spirv_args) + result = '#!amber\n' + result += '# Generated AmberScript for a bug found by GraphicsFuzz\n\n' - if has_frag_glsl or has_vert_glsl: - result += '# Derived from the following GLSL.\n\n' + result += get_header_comment_original_source_image(vert_original, frag_original, spirv_args) - if has_vert_glsl: - result += '# Vertex shader GLSL:\n' - result += get_shader_as_comment(vert_original) - result += '\n\n' - - if has_frag_glsl: - result += '# Fragment shader GLSL:\n' - result += get_shader_as_comment(frag_original) - result += '\n\n' - - result += '[require]\n' - result += 'fbsize 256 256\n\n' - - result += '[require]\n' - result += 'fence_timeout ' + str(AMBER_FENCE_TIMEOUT_MS) + '\n\n' + result += 'SET ENGINE_DATA fence_timeout_ms ' + str(AMBER_FENCE_TIMEOUT_MS) + '\n\n' if vert: - result += '[vertex shader spirv]\n' + result += 'SHADER vertex gfz_vert SPIRV-ASM\n' result += spv_get_disassembly(vert) + result += 'END\n\n' else: - result += '[vertex shader passthrough]\n' - result += '\n\n' + result += 'SHADER vertex gfz_vert PASSTHROUGH\n\n' - result += '[fragment shader spirv]\n' + result += 'SHADER fragment gfz_frag SPIRV-ASM\n' result += spv_get_disassembly(frag) - result += '\n\n' + result += 'END\n\n' - result += '[test]\n' - result += '## Uniforms\n' - result += uniform_json_to_amberscript(uniform_json) + # This buffer MUST be named framebuffer to be able to retrieve the image + # Format MUST be B8G8R8A8_UNORM (other options may become available once + # Amber supports more formats for image extraction) + result += 'BUFFER framebuffer FORMAT B8G8R8A8_UNORM\n' + result += amberscript_uniform_buffer_decl(uniform_json) result += '\n' - result += 'draw rect -1 -1 2 2\n' + + result += 'PIPELINE graphics gfz_pipeline\n' + result += ' ATTACH gfz_vert\n' + result += ' ATTACH gfz_frag\n' + result += ' FRAMEBUFFER_SIZE 256 256\n' + result += ' BIND BUFFER framebuffer AS color LOCATION 0\n' + result += amberscript_uniform_buffer_bind(uniform_json) + result += 'END\n\n' + + result += 'CLEAR_COLOR gfz_pipeline 0 0 0 255\n' + result += 'CLEAR gfz_pipeline\n' + result += 'RUN gfz_pipeline DRAW_RECT POS 0 0 SIZE 256 256\n' return result @@ -940,7 +1052,7 @@ def run_image_amber( force: bool, is_android: bool, skip_render: bool, - spirv_opt_args: Optional[List[str]], + spirv_opt_args: Optional[List[str]] ): # The vertex shader is optional; passthrough will be used if it is not present assert not vert_original or os.path.isfile(vert_original) @@ -950,13 +1062,13 @@ def run_image_amber( frag = prepare_shader(output_dir, frag_original, spirv_opt_args) vert = prepare_shader(output_dir, vert_original, spirv_opt_args) if vert_original else None - amberscript_file = os.path.join(output_dir, 'tmpscript.shader_test') status_file = os.path.join(output_dir, 'STATUS') png_image = os.path.join(output_dir, 'image_0.png') device_image = ANDROID_DEVICE_GRAPHICSFUZZ_DIR + '/image_0.png' - with open_helper(amberscript_file, 'w') as f: + shader_test_file = os.path.join(output_dir, 'tmp_shader_test.amber') + with open_helper(shader_test_file, 'w') as f: f.write(amberscriptify_image( vert, frag, @@ -971,7 +1083,7 @@ def run_image_amber( adb_check([ 'push', - amberscript_file, + shader_test_file, ANDROID_DEVICE_GRAPHICSFUZZ_DIR ]) @@ -996,7 +1108,7 @@ def run_image_amber( 'cd ' + ANDROID_DEVICE_DIR + ' && ' './' + ANDROID_AMBER_NDK + flags - + ' -d ' + ANDROID_DEVICE_GRAPHICSFUZZ_DIR + '/' + os.path.basename(amberscript_file) + + ' -d ' + ANDROID_DEVICE_GRAPHICSFUZZ_DIR + '/' + os.path.basename(shader_test_file) ] adb_check(['logcat', '-c']) @@ -1041,7 +1153,7 @@ def run_image_amber( # -i tells amber to dump the framebuffer cmd.append('-i') cmd.append(png_image) - cmd.append(amberscript_file) + cmd.append(shader_test_file) status = 'SUCCESS' if added_catchsegv: @@ -1066,26 +1178,40 @@ def run_image_amber( ################################################################################ # Amber worker: compute test +def amber_check_buffer_single_type(json_filename): + with open_helper(json_filename, 'r') as f: + j = json.load(f) -def translate_type_for_amber(type_name: str) -> str: - if type_name == 'bool': - return 'uint' - return type_name + # Amber only supports one type per buffer, check this limitation. + field_type = None + for field_info in j['$compute']['buffer']['fields']: + if not field_type: + field_type = field_info['type'] + elif field_type != field_info['type']: + raise ValueError('Amber only supports one type per buffer') -def comp_json_to_amberscript(comp_json): +def amberscript_comp_buff_decl(comp_json): """ - Returns the string representing VkScript version of compute shader setup, - found under the special "$compute" key in JSON + Returns the string representing AmberScript declaration of buffers for a + compute shader test. { - "my_uniform_name": { ... ignored by this function ... }, + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, "$compute": { "num_groups": [12, 13, 14]; "buffer": { "binding": 123, - "input": [42, 43, 44, 45] + "fields": + [ + { "type": "int", "data": [ 0 ] }, + { "type": "int", "data": [ 1, 2 ] }, + ] } } @@ -1093,42 +1219,93 @@ def comp_json_to_amberscript(comp_json): becomes: - ssbo 123 subdata int 0 42 43 44 45 - - compute 12 13 14 + # myuniform + BUFFER myuniform DATA_TYPE float DATA + 42.0 + END + BUFFER gfz_ssbo DATA_TYPE int DATA + 0 1 2 + END """ + SSBO_TYPES = { + 'int': 'int32', + 'ivec2': 'vec2', + 'ivec3': 'vec3', + 'ivec4': 'vec4', + 'uint': 'uint32', + 'float': 'float', + 'vec2': 'vec2', + 'vec3': 'vec3', + 'vec4': 'vec4', + } + + # regular uniforms + result = amberscript_uniform_buffer_decl(comp_json) + with open_helper(comp_json, 'r') as f: j = json.load(f) assert '$compute' in j.keys(), 'Cannot find "$compute" key in JSON file' j = j['$compute'] - result = "" - binding = j['buffer']['binding'] offset = 0 + + # A single type for all fields is assumed here + assert len(j['buffer']['fields']) > 0, 'Compute shader test with empty SSBO' + json_datum_type = j['buffer']['fields'][0]['type'] + if json_datum_type not in SSBO_TYPES.keys(): + raise ValueError('Unsupported SSBO datum type: ' + json_datum_type) + datum_type = SSBO_TYPES[json_datum_type] + + result += 'BUFFER gfz_ssbo DATA_TYPE ' + datum_type + ' DATA\n' for field_info in j['buffer']['fields']: - result += ( - 'ssbo ' - + str(binding) - + ' subdata ' - + translate_type_for_amber(field_info['type']) - + ' ' - + str(offset) - ) for datum in field_info['data']: result += ' ' + str(datum) - offset += 4 - result += '\n' - result += '\n\n' - - result += 'compute' - result += ' ' + str(j['num_groups'][0]) - result += ' ' + str(j['num_groups'][1]) - result += ' ' + str(j['num_groups'][2]) result += '\n' + result += 'END\n\n' + + return result + + +def amberscript_comp_buff_bind(comp_json): + """ + Returns the string representing AmberScript binding of buffers for a + compute shader test. + + { + "myuniform": { + "func": "glUniform1f", + "args": [ 42.0 ], + "binding": 3 + }, + + "$compute": { + "num_groups": [12, 13, 14]; + "buffer": { + "binding": 123, + "fields": + [ + { "type": "int", "data": [ 0 ] }, + { "type": "int", "data": [ 1, 2 ] }, + ] + } + } + + } + + becomes: + + BIND BUFFER myuniform AS uniform DESCRIPTOR_SET 0 BINDING 3 + BIND BUFFER gfz_ssbo AS storage DESCRIPTOR_SET 0 BINDING 123 + """ + + # regular uniforms + result = amberscript_uniform_buffer_bind(comp_json) + + result += 'BIND BUFFER gfz_ssbo AS storage DESCRIPTOR_SET 0 BINDING ' + str(get_ssbo_binding(comp_json)) + '\n\n' return result @@ -1143,31 +1320,33 @@ def amberscriptify_comp( Generates an AmberScript representation of a compute test """ - result = '# Generated\n\n' + has_comp_glsl = comp_original and filename_extension_suggests_glsl(comp_original) - result += '# A test for a bug found by GraphicsFuzz.\n\n' + result = '#!amber\n' + result += '# Generated AmberScript for a bug found by GraphicsFuzz\n\n' - result += get_spirv_opt_args_comment(spirv_args) - - if comp_original and filename_extension_suggests_glsl(comp_original): - result += '# Derived from the following GLSL.\n\n' - result += '# Compute shader GLSL:\n' - result += get_shader_as_comment(comp_original) - result += '\n\n' + result += get_header_comment_original_source_comp(comp_original, spirv_args) - result += '[require]\n' - result += 'fence_timeout ' + str(AMBER_FENCE_TIMEOUT_MS) + '\n\n' + result += 'SET ENGINE_DATA fence_timeout_ms ' + str(AMBER_FENCE_TIMEOUT_MS) + '\n\n' - result += '[compute shader spirv]\n' + result += 'SHADER compute gfz_comp SPIRV-ASM\n' result += spv_get_disassembly(comp_spv) - result += '\n\n' + result += 'END\n\n' - result += '[test]\n' - result += '## Uniforms\n' - result += uniform_json_to_amberscript(comp_json) - result += '## SSBO\n' - result += comp_json_to_amberscript(comp_json) - result += '\n' + result += amberscript_comp_buff_decl(comp_json) + + result += 'PIPELINE compute gfz_pipeline\n' + result += ' ATTACH gfz_comp\n' + result += amberscript_comp_buff_bind(comp_json) + result += 'END\n\n' + + result += 'RUN gfz_pipeline' + with open_helper(comp_json, 'r') as f: + j = json.load(f) + num_groups = j['$compute']['num_groups'] + for dimension in num_groups: + result += ' ' + str(dimension) + result += '\n\n' return result @@ -1224,21 +1403,27 @@ def run_compute_amber( force: bool, is_android: bool, skip_render: bool, - spirv_opt_args: Optional[List[str]], + spirv_opt_args: Optional[List[str]] ) -> None: assert os.path.isfile(comp_original) assert os.path.isfile(json_file) + amber_check_buffer_single_type(json_file) - amberscript_file = os.path.join(output_dir, 'tmpscript.shader_test') ssbo_output = os.path.join(output_dir, 'ssbo') ssbo_json = os.path.join(output_dir, SSBO_JSON_FILENAME) status_file = os.path.join(output_dir, 'STATUS') comp = prepare_shader(output_dir, comp_original, spirv_opt_args) - with open_helper(amberscript_file, 'w') as f: - f.write(amberscriptify_comp(comp, json_file, comp_original, spirv_opt_args)) + shader_test_file = os.path.join(output_dir, 'tmp_shader_test.amber') + with open_helper(shader_test_file, 'w') as f: + f.write(amberscriptify_comp( + comp, + json_file, + comp_original, + spirv_opt_args, + )) # FIXME: in case of multiple SSBOs, we should pass the binding of the ones to be dumped ssbo_binding = str(get_ssbo_binding(json_file)) @@ -1249,7 +1434,7 @@ def run_compute_amber( # Prepare files on device. adb_check([ 'push', - amberscript_file, + shader_test_file, ANDROID_DEVICE_GRAPHICSFUZZ_DIR, ]) @@ -1275,7 +1460,7 @@ def run_compute_amber( 'cd ' + ANDROID_DEVICE_DIR + ' && ' './' + ANDROID_AMBER_NDK + flags - + ANDROID_DEVICE_GRAPHICSFUZZ_DIR + '/' + os.path.basename(amberscript_file) + + ANDROID_DEVICE_GRAPHICSFUZZ_DIR + '/' + os.path.basename(shader_test_file) ] adb_check(['logcat', '-c']) @@ -1320,7 +1505,7 @@ def run_compute_amber( cmd.append(ssbo_output) cmd.append('-B') cmd.append(ssbo_binding) - cmd.append(amberscript_file) + cmd.append(shader_test_file) status = 'SUCCESS' @@ -1519,7 +1704,7 @@ def main_helper(args): force=args.force, is_android=(args.target == 'android'), skip_render=args.skip_render, - spirv_opt_args=spirv_args, + spirv_opt_args=spirv_args ) finally: log_to_file = None diff --git a/python/src/main/python/drivers/shader_job_uniforms_to_spirv_fuzz_facts.py b/python/src/main/python/drivers/shader_job_uniforms_to_spirv_fuzz_facts.py new file mode 100644 index 000000000..67a43c49c --- /dev/null +++ b/python/src/main/python/drivers/shader_job_uniforms_to_spirv_fuzz_facts.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The GraphicsFuzz Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import struct +import sys +from typing import Any, Dict, List + +import runspv + + +def main_helper(args) -> None: + """ + Turn a GraphicsFuzz .json file into a spirv-fuzz .facts file, with one fact per word of uniform + data. + + :param args: command-line arguments + :return: None + """ + + parser = argparse.ArgumentParser() + + # Required arguments + parser.add_argument( + 'shader_job', + help='Shader job (.json) file.') + + parser.add_argument( + 'output_file', + help='Output file for facts.') + + args = parser.parse_args(args) + + # Generate uniform facts from .json file + fact_list = [] # type: List[dict] + + # Turn the information about uniform values into facts for spirv-fuzz. + with runspv.open_helper(args.shader_job, 'r') as f: + j = json.load(f) # type: Dict[str, Any] + for name, entry in j.items(): + + # Skip compute shader data + if name == "$compute": + continue + + scalar_uniform_functions = ['glUniform1i', 'glUniform1f'] + vector_uniform_functions = ['glUniform2i', 'glUniform3i', 'glUniform4i', 'glUniform2f', + 'glUniform3f', 'glUniform4f'] + + if not (entry["func"] in scalar_uniform_functions or entry["func"] in vector_uniform_functions): + print("Ignoring unsupported uniform function " + entry["func"]) + continue + + # Make a separate fact for every component value of each uniform. + for i in range(0, len(entry["args"])): + # Every uniform is in its own struct, so we index into the first element of that struct + # using index 0. If the uniform is a vector, we need to further index into the vector. + if entry["func"] in scalar_uniform_functions: + # The uniform is a scalar. + assert i == 0 + index_list = [0] + else: + assert entry["func"] in vector_uniform_functions + # The uniform is a vector, so we have two levels of indexing. + index_list = [0, i] + + # We need to pass the component value as an integer. If it is a float, we need to + # reinterpret its bits as an integer. + int_representation = entry["args"][i] + if isinstance(int_representation, float): + int_representation = struct.unpack(' {}: return json.load(open(str(ssbo_json), 'r')) -def check_images_match(tmp_path: pathlib2.Path, json_filename: str, is_android_1: bool, +def check_images_match(tmp_path: pathlib2.Path, + json_filename: str, + is_android_1: bool, is_android_2: bool, - is_amber_1: bool, is_amber_2: bool, fuzzy_image_comparison: bool, - spirvopt_options_1: str=None, spirvopt_options_2: str=None): + is_amber_1: bool, + is_amber_2: bool, + fuzzy_image_comparison: bool, + spirvopt_options_1: str=None, + spirvopt_options_2: str=None): out_dir_1 = tmp_path / 'out_1' out_dir_2 = tmp_path / 'out_2' @@ -494,6 +499,13 @@ def test_spirv_opt_fails_given_silly_argument_compute(tmp_path: pathlib2.Path): assert '-ODOESNOTEXIST is not a valid flag' in str(called_process_error) +def test_runspv_fails_if_comp_buffer_has_several_types(tmp_path: pathlib2.Path): + with pytest.raises(ValueError) as value_error: + runspv.main_helper(['host', get_compute_test('buffer-with-several-types.json'), + str(tmp_path / 'out')]) + assert 'Amber only supports one type per buffer' in str(value_error) + + def test_skip_render_image_amber_host(tmp_path: pathlib2.Path): check_no_image_skip_render(tmp_path, is_android=False, is_legacy_worker=False, json_filename='image_test_0007.json') @@ -636,7 +648,8 @@ def test_image_0002_host_vs_android_amber(tmp_path: pathlib2.Path): def test_image_0003_host_vs_android_amber(tmp_path: pathlib2.Path): check_images_match_host_vs_android_amber(tmp_path, 'image_test_0003.json') - +@pytest.mark.skip( + reason="Slight image difference probably due to floating point sensitivity.") def test_image_0004_host_vs_android_amber(tmp_path: pathlib2.Path): check_images_match_host_vs_android_amber(tmp_path, 'image_test_0004.json') @@ -645,6 +658,8 @@ def test_image_0005_host_vs_android_amber(tmp_path: pathlib2.Path): check_images_match_host_vs_android_amber(tmp_path, 'image_test_0005.json') +@pytest.mark.skip( + reason="Slight image difference probably due to floating point sensitivity.") def test_image_0006_host_vs_android_amber(tmp_path: pathlib2.Path): check_images_match_host_vs_android_amber(tmp_path, 'image_test_0006.json') @@ -709,6 +724,8 @@ def test_compute_0002_spirvopt_host_amber(tmp_path: pathlib2.Path): ################################# # spirv-opt vs. normal (android, amber) +@pytest.mark.skip( + reason="May trigger a bug on some device, see github issue #564.") def test_image_0003_spirvopt_android_amber(tmp_path: pathlib2.Path): check_images_match_spirvopt(tmp_path, 'image_test_0003.json', options='-O', @@ -716,6 +733,8 @@ def test_image_0003_spirvopt_android_amber(tmp_path: pathlib2.Path): is_amber=True) +@pytest.mark.skip( + reason="Slight image difference probably due to floating point sensitivity.") def test_image_0004_spirvopt_android_amber(tmp_path: pathlib2.Path): check_images_match_spirvopt(tmp_path, 'image_test_0004.json', options='-Os', diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/CheckAstFeatureVisitor.java b/reducer/src/main/java/com/graphicsfuzz/reducer/CheckAstFeatureVisitor.java index f5d70e432..201f8dc3a 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/CheckAstFeatureVisitor.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/CheckAstFeatureVisitor.java @@ -20,7 +20,7 @@ import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -33,7 +33,7 @@ * method to check for the feature of interest, and then invoke the trigger function * when the feature is found. */ -public abstract class CheckAstFeatureVisitor extends ScopeTreeBuilder { +public abstract class CheckAstFeatureVisitor extends ScopeTrackingVisitor { private Optional triggerFunction = Optional.empty(); private Map> callsIndirectly = new HashMap>(); @@ -49,7 +49,7 @@ public void visitFunctionPrototype(FunctionPrototype functionPrototype) { @Override public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { super.visitFunctionCallExpr(functionCallExpr); - final String enclosingFunctionName = enclosingFunction.getPrototype().getName(); + final String enclosingFunctionName = getEnclosingFunction().getPrototype().getName(); final String calledFunctionName = functionCallExpr.getCallee(); assert callsIndirectly.containsKey(enclosingFunctionName); if (!callsIndirectly.containsKey(calledFunctionName)) { @@ -72,7 +72,7 @@ boolean check(TranslationUnit tu) { /** * Use this method to register that the feature of interest has been found. */ - public void trigger() { - this.triggerFunction = Optional.of(enclosingFunction); + protected void trigger() { + this.triggerFunction = Optional.of(getEnclosingFunction()); } } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java index fb24f8b7a..4853dad44 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/ReductionDriver.java @@ -23,6 +23,7 @@ import com.graphicsfuzz.reducer.glslreducers.IReductionPassManager; import com.graphicsfuzz.reducer.glslreducers.SystematicReductionPass; import com.graphicsfuzz.reducer.glslreducers.SystematicReductionPassManager; +import com.graphicsfuzz.reducer.reductionopportunities.IReductionOpportunity; import com.graphicsfuzz.reducer.reductionopportunities.IReductionOpportunityFinder; import com.graphicsfuzz.reducer.reductionopportunities.ReducerContext; import com.graphicsfuzz.reducer.util.Simplify; @@ -31,6 +32,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Optional; @@ -84,20 +86,22 @@ public ReductionDriver(ReducerContext context, IReductionOpportunityFinder.largestFunctionsFinder(5), 1)); final List cleanupPasses = new ArrayList<>(); - for (IReductionOpportunityFinder finder : new IReductionOpportunityFinder[]{ + for (IReductionOpportunityFinder finder : Arrays.asList( IReductionOpportunityFinder.inlineUniformFinder(), IReductionOpportunityFinder.inlineInitializerFinder(), IReductionOpportunityFinder.inlineFunctionFinder(), IReductionOpportunityFinder.unusedParamFinder(), IReductionOpportunityFinder.foldConstantFinder(), - }) { + IReductionOpportunityFinder.redundantUniformMetadataFinder(), + IReductionOpportunityFinder.variableDeclToExprFinder(), + IReductionOpportunityFinder.globalVariableDeclToExprFinder())) { cleanupPasses.add(new SystematicReductionPass(context, verbose, finder)); } final List corePasses = new ArrayList<>(); - for (IReductionOpportunityFinder finder : new IReductionOpportunityFinder[]{ + for (IReductionOpportunityFinder finder : Arrays.asList( IReductionOpportunityFinder.vectorizationFinder(), IReductionOpportunityFinder.unswitchifyFinder(), IReductionOpportunityFinder.stmtFinder(), @@ -108,7 +112,8 @@ public ReductionDriver(ReducerContext context, IReductionOpportunityFinder.functionFinder(), IReductionOpportunityFinder.mutationFinder(), IReductionOpportunityFinder.loopMergeFinder(), - IReductionOpportunityFinder.compoundToBlockFinder(), + IReductionOpportunityFinder.flattenControlFlowFinder(), + IReductionOpportunityFinder.switchToLoopFinder(), IReductionOpportunityFinder.outlinedStatementFinder(), IReductionOpportunityFinder.unwrapFinder(), IReductionOpportunityFinder.removeStructFieldFinder(), @@ -117,8 +122,7 @@ public ReductionDriver(ReducerContext context, IReductionOpportunityFinder.liveFragColorWriteFinder(), IReductionOpportunityFinder.functionFinder(), IReductionOpportunityFinder.variableDeclFinder(), - IReductionOpportunityFinder.globalVariablesDeclarationFinder(), - }) { + IReductionOpportunityFinder.globalVariablesDeclarationFinder())) { final SystematicReductionPass pass = new SystematicReductionPass(context, verbose, finder); @@ -292,11 +296,10 @@ private void writeState(ShaderJob state, File shaderJobFileOutput, } fileOps.writeShaderJobFile( state, - shaderJobFileOutput, - context.getEmitGraphicsFuzzDefines() + shaderJobFileOutput ); if (requiresUniformBindings) { - assert state.hasUniformBindings(); + assert state.getPipelineInfo().getNumUniforms() == 0 || state.hasUniformBindings(); state.removeUniformBindings(); } } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/filejudge/CustomFileJudge.java b/reducer/src/main/java/com/graphicsfuzz/reducer/filejudge/CustomFileJudge.java index fb5a6a7f5..3e9efbf86 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/filejudge/CustomFileJudge.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/filejudge/CustomFileJudge.java @@ -22,6 +22,8 @@ import com.graphicsfuzz.util.ExecResult; import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,25 +31,24 @@ public class CustomFileJudge implements IFileJudge { private static final Logger LOGGER = LoggerFactory.getLogger(CustomFileJudge.class); - private final File judgeScript; - private final File directory; + private final List judgeScript; - public CustomFileJudge(File judgeScript, File directory) { + public CustomFileJudge(List judgeScript) { this.judgeScript = judgeScript; - this.directory = directory; } @Override public boolean isInteresting(File shaderJobFile, File shaderResultFileOutput) throws FileJudgeException { + List scriptPlusShaderArg = new ArrayList<>(judgeScript); + scriptPlusShaderArg.add(shaderJobFile.toString()); try { final ExecResult execResult = new ExecHelper().exec( ExecHelper.RedirectType.TO_LOG, - directory, + null, true, - judgeScript.getAbsolutePath(), - shaderJobFile.getAbsolutePath(), - shaderResultFileOutput.getAbsolutePath()); + scriptPlusShaderArg.toArray(new String[0]) + ); LOGGER.info("Custom file judge result: " + execResult.res); return execResult.res == 0; } catch (IOException | InterruptedException exception) { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateGraphicsFuzzDefines.java b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateGraphicsFuzzDefines.java new file mode 100644 index 000000000..1c91bbcd5 --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateGraphicsFuzzDefines.java @@ -0,0 +1,183 @@ +/* + * Copyright 2018 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.glslreducers; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; +import com.graphicsfuzz.common.ast.expr.BinOp; +import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.ConstantExpr; +import com.graphicsfuzz.common.ast.expr.Expr; +import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; +import com.graphicsfuzz.common.ast.expr.ParenExpr; +import com.graphicsfuzz.common.ast.expr.TernaryExpr; +import com.graphicsfuzz.common.ast.expr.UIntConstantExpr; +import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; +import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.MacroNames; + +public class EliminateGraphicsFuzzDefines extends StandardVisitor { + + private final ShadingLanguageVersion shadingLanguageVersion; + + private EliminateGraphicsFuzzDefines(ShadingLanguageVersion shadingLanguageVersion) { + this.shadingLanguageVersion = shadingLanguageVersion; + } + + /** + * Eliminates all GraphicsFuzz macros from the given translation unit, by inlining them. + * @param tu The translation unit to be processed. + * @return A clone of the given translation unit, with all GraphicsFuzz macros inlined. + */ + public static TranslationUnit transform(TranslationUnit tu) { + final TranslationUnit result = tu.clone(); + new EliminateGraphicsFuzzDefines(tu.getShadingLanguageVersion()).visit(result); + return result; + } + + @Override + protected void visitChildFromParent(IAstNode child, IAstNode parent) { + super.visitChildFromParent(child, parent); + if (child instanceof FunctionCallExpr) { + final FunctionCallExpr functionCallExpr = (FunctionCallExpr) child; + if (MacroNames.isIdentity(functionCallExpr) + || MacroNames.isZero(functionCallExpr) + || MacroNames.isOne(functionCallExpr) + || MacroNames.isFalse(functionCallExpr) + || MacroNames.isTrue(functionCallExpr)) { + parent.replaceChild(functionCallExpr, + addParenthesesIfNecessary(parent, functionCallExpr.getChild(1))); + } else if (MacroNames.isFuzzed(functionCallExpr) + || MacroNames.isDeadByConstruction(functionCallExpr) + || MacroNames.isSwitch(functionCallExpr) + || MacroNames.isLoopWrapper(functionCallExpr) + || MacroNames.isIfWrapperFalse(functionCallExpr) + || MacroNames.isIfWrapperTrue(functionCallExpr)) { + parent.replaceChild(functionCallExpr, + addParenthesesIfNecessary(parent, functionCallExpr.getChild(0))); + } else if (MacroNames.isMakeInBoundsInt(functionCallExpr)) { + expandMakeInBoundsInt(parent, functionCallExpr); + } else if (MacroNames.isMakeInBoundsUint(functionCallExpr)) { + expandMakeInBoundsUint(parent, functionCallExpr); + } + } + } + + private void expandMakeInBoundsInt(IAstNode parent, FunctionCallExpr functionCallExpr) { + + // This replaces a call to _GLF_MAKE_IN_BOUNDS_INT with either: + // - a call to clamp, if the shading language version supports integer clamping + // - a ternary expression otherwise + // The expression that is generated matches the definition of the macro, which can be found in + // PrettyPrinterVisitor.emitGraphicsFuzzDefines. + + final ConstantExpr one = new IntConstantExpr("1"); + final ConstantExpr zero = new IntConstantExpr("0"); + if (shadingLanguageVersion.supportedClampInt()) { + expandMakeInBoundsMacroToClamp(parent, functionCallExpr, one, + zero); + } else { + final Expr indexLessThanZero = + new BinaryExpr(new ParenExpr(functionCallExpr.getChild(0).clone()), + zero, BinOp.LT); + parent.replaceChild(functionCallExpr, + addParenthesesIfNecessary(parent, + new TernaryExpr(indexLessThanZero, zero.clone(), + new ParenExpr(getTernaryUpperBoundCheckForMakeInBounds(functionCallExpr, one))))); + } + } + + private void expandMakeInBoundsUint(IAstNode parent, FunctionCallExpr functionCallExpr) { + + // Similar to expandMakeInBoundsInt, but for the 'uint' case. + + final ConstantExpr one = new UIntConstantExpr("1u"); + if (shadingLanguageVersion.supportedClampUint()) { + expandMakeInBoundsMacroToClamp(parent, functionCallExpr, one, + new UIntConstantExpr("0u")); + } else { + parent.replaceChild(functionCallExpr, + addParenthesesIfNecessary(parent, + getTernaryUpperBoundCheckForMakeInBounds(functionCallExpr, + one))); + + } + } + + private Expr getTernaryUpperBoundCheckForMakeInBounds(FunctionCallExpr functionCallExpr, + ConstantExpr one) { + return new TernaryExpr(new BinaryExpr(new ParenExpr(functionCallExpr.getChild(0).clone()), + functionCallExpr.getChild(1).clone(), + BinOp.GE), + new BinaryExpr(functionCallExpr.getChild(1).clone(), one, BinOp.SUB), + new ParenExpr(functionCallExpr.getChild(0).clone())); + } + + private void expandMakeInBoundsMacroToClamp(IAstNode parent, FunctionCallExpr functionCallExpr, + ConstantExpr one, ConstantExpr zero) { + final Expr sizeMinusOne = new BinaryExpr(functionCallExpr.getChild(1), + one, + BinOp.SUB); + final Expr clamp = new FunctionCallExpr("clamp", functionCallExpr.getChild(0), + zero, + sizeMinusOne); + parent.replaceChild(functionCallExpr, + addParenthesesIfNecessary(parent, + clamp)); + } + + private IAstNode addParenthesesIfNecessary(IAstNode parent, Expr child) { + if (child instanceof ConstantExpr + || child instanceof ParenExpr + || child instanceof VariableIdentifierExpr + || child instanceof FunctionCallExpr) { + // Parentheses is unnecessary in cases such as _GLF_FUNCTION(1), + // _GLF_FUNCTION((1)), _GLF_FUNCTION(a), _GLF_FUNCTION(sin(a)). + return child; + } + + if (!(parent instanceof Expr)) { + // No parentheses needed if the parent is not an expression, + // for example, int x = _GLF_FUNCTION(a + b). + return child; + } + + if (parent instanceof ParenExpr) { + // If parent is a parentheses expression, adding more parentheses would be + // redundant, + // e.g. (_GLF_FUNCTION(a + b)). + return child; + } + + if (parent instanceof FunctionCallExpr) { + // Parentheses is unnecessary if parent is a function call expression. + // For example, foo(_GLF_FUNCTION(a)). + + // This asserts that the binary expression inside the function call is not a comma operator + // as it is invalid to have a comma appear directly here, e.g. _GLF_IDENTITY(expr, a, b) is + // not valid since a and b are treated as function arguments instead. + assert (!(child instanceof BinaryExpr) || ((BinaryExpr) child).getOp() != BinOp.COMMA); + return child; + } + + return new ParenExpr(child); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java deleted file mode 100644 index b027214b2..000000000 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/EliminateInjectionMacrosVisitor.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer.glslreducers; - -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; -import com.graphicsfuzz.common.ast.expr.ArrayIndexExpr; -import com.graphicsfuzz.common.ast.expr.BinaryExpr; -import com.graphicsfuzz.common.ast.expr.Expr; -import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; -import com.graphicsfuzz.common.ast.expr.MemberLookupExpr; -import com.graphicsfuzz.common.ast.expr.ParenExpr; -import com.graphicsfuzz.common.ast.expr.TernaryExpr; -import com.graphicsfuzz.common.ast.expr.TypeConstructorExpr; -import com.graphicsfuzz.common.ast.expr.UnaryExpr; -import com.graphicsfuzz.common.ast.stmt.DoStmt; -import com.graphicsfuzz.common.ast.stmt.ForStmt; -import com.graphicsfuzz.common.ast.stmt.IfStmt; -import com.graphicsfuzz.common.ast.stmt.LoopStmt; -import com.graphicsfuzz.common.ast.visitors.StandardVisitor; -import com.graphicsfuzz.reducer.reductionopportunities.MacroNames; - -public class EliminateInjectionMacrosVisitor extends StandardVisitor { - - @Override - public void visitBinaryExpr(BinaryExpr binaryExpr) { - super.visitBinaryExpr(binaryExpr); - cleanUpMacros(binaryExpr); - } - - @Override - public void visitUnaryExpr(UnaryExpr unaryExpr) { - super.visitUnaryExpr(unaryExpr); - cleanUpMacros(unaryExpr); - } - - @Override - public void visitParenExpr(ParenExpr parenExpr) { - super.visitParenExpr(parenExpr); - cleanUpMacros(parenExpr); - } - - @Override - public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { - super.visitFunctionCallExpr(functionCallExpr); - cleanUpMacros(functionCallExpr); - } - - public void visitArrayIndexExpr(ArrayIndexExpr arrayIndexExpr) { - super.visitArrayIndexExpr(arrayIndexExpr); - cleanUpMacros(arrayIndexExpr); - } - - public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { - super.visitMemberLookupExpr(memberLookupExpr); - cleanUpMacros(memberLookupExpr); - } - - public void visitTernaryExpr(TernaryExpr ternaryExpr) { - super.visitTernaryExpr(ternaryExpr); - cleanUpMacros(ternaryExpr); - } - - @Override - public void visitTypeConstructorExpr(TypeConstructorExpr typeConstructorExpr) { - super.visitTypeConstructorExpr(typeConstructorExpr); - cleanUpMacros(typeConstructorExpr); - } - - @Override - public void visitIfStmt(IfStmt ifStmt) { - super.visitIfStmt(ifStmt); - if (MacroNames.isDeadByConstruction(ifStmt.getCondition()) - || MacroNames.isIfWrapperFalse(ifStmt.getCondition()) - || MacroNames.isIfWrapperTrue(ifStmt.getCondition())) { - ifStmt.setCondition(ifStmt.getCondition().getChild(0)); - } - } - - private void cleanupLoop(LoopStmt loopStmt) { - if (MacroNames.isLoopWrapper(loopStmt.getCondition())) { - loopStmt.setCondition(loopStmt.getCondition().getChild(0)); - } - } - - @Override - public void visitDoStmt(DoStmt doStmt) { - super.visitDoStmt(doStmt); - cleanupLoop(doStmt); - } - - @Override - public void visitForStmt(ForStmt forStmt) { - super.visitForStmt(forStmt); - cleanupLoop(forStmt); - } - - private void cleanUpMacros(Expr parent) { - for (int i = 0; i < parent.getNumChildren(); i++) { - Expr child = parent.getChild(i); - // Note: it would be redundant to have a special function reduction opportunity - // be classed also as a fuzzed expression reduction opportunity - if (MacroNames.isIdentity(child) - || MacroNames.isZero(child) - || MacroNames.isOne(child) - || MacroNames.isFalse(child) - || MacroNames.isTrue(child)) { - replaceChildWithGrandchild(parent, i, 1); - } else if (MacroNames.isFuzzed(child) - || MacroNames.isDeadByConstruction(child)) { - replaceChildWithGrandchild(parent, i, 0); - } - } - } - - @Override - public void visitScalarInitializer(ScalarInitializer scalarInitializer) { - super.visitScalarInitializer(scalarInitializer); - if (MacroNames.isFuzzed(scalarInitializer.getExpr())) { - scalarInitializer.setExpr(scalarInitializer.getExpr().getChild(0)); - } - } - - private void replaceChildWithGrandchild(Expr parent, int childIndex, int grandchildIndex) { - assert parent.getChild(childIndex).getNumChildren() > grandchildIndex; - parent - .setChild(childIndex, new ParenExpr(parent.getChild(childIndex).getChild(grandchildIndex))); - } - -} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/SystematicReductionPass.java b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/SystematicReductionPass.java index 68bf10639..b71a5905e 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/SystematicReductionPass.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/glslreducers/SystematicReductionPass.java @@ -31,17 +31,22 @@ public class SystematicReductionPass extends AbstractReductionPass { private int granularity; private final int maximumGranularity; - public SystematicReductionPass(ReducerContext reducerContext, - boolean verbose, IReductionOpportunityFinder finder, - int maximumGranularity) { + public SystematicReductionPass( + ReducerContext reducerContext, + boolean verbose, + IReductionOpportunityFinder finder, + int maximumGranularity + ) { // Ignore verbose argument for now. super(reducerContext, finder); this.isInitialized = false; this.maximumGranularity = maximumGranularity; } - public SystematicReductionPass(ReducerContext reducerContext, - boolean verbose, IReductionOpportunityFinder finder) { + public SystematicReductionPass( + ReducerContext reducerContext, + boolean verbose, + IReductionOpportunityFinder finder) { this(reducerContext, verbose, finder, Integer.MAX_VALUE); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunity.java deleted file mode 100644 index 42b2cc534..000000000 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunity.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer.reductionopportunities; - -import com.graphicsfuzz.common.ast.IAstNode; -import com.graphicsfuzz.common.ast.stmt.BlockStmt; -import com.graphicsfuzz.common.ast.stmt.ForStmt; -import com.graphicsfuzz.common.ast.stmt.Stmt; -import com.graphicsfuzz.common.ast.visitors.VisitationDepth; -import java.util.ArrayList; -import java.util.List; - -public class CompoundToBlockReductionOpportunity extends AbstractReductionOpportunity { - - private final IAstNode parent; - private final Stmt compoundStmt; - private final Stmt childStmt; - - public CompoundToBlockReductionOpportunity(IAstNode parent, Stmt compoundStmt, Stmt childStmt, - VisitationDepth depth) { - super(depth); - this.parent = parent; - this.compoundStmt = compoundStmt; - this.childStmt = childStmt; - } - - @Override - public void applyReductionImpl() { - Stmt replacement; - if (compoundStmt instanceof ForStmt) { - final ForStmt forStmt = (ForStmt) compoundStmt; - assert childStmt == forStmt.getBody(); - List stmts = new ArrayList<>(); - stmts.add(forStmt.getInit()); - if (forStmt.getBody() instanceof BlockStmt) { - stmts.addAll(((BlockStmt) forStmt.getBody()).getStmts()); - } else { - stmts.add(forStmt.getBody()); - } - replacement = new BlockStmt(stmts, true); - } else { - replacement = childStmt; - } - parent.replaceChild(compoundStmt, replacement); - } - - @Override - public boolean preconditionHolds() { - return parent.hasChild(compoundStmt) - && compoundStmt.hasChild(childStmt); - } - -} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunities.java index 3d8d5fb85..41384cb3d 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunities.java @@ -19,13 +19,13 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import java.util.Arrays; import java.util.LinkedList; import java.util.List; -public class DestructifyReductionOpportunities extends ScopeTreeBuilder { +public class DestructifyReductionOpportunities extends ScopeTrackingVisitor { private final TranslationUnit tu; private final List opportunities; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunity.java index 773e7bee5..38bbad503 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunity.java @@ -19,7 +19,7 @@ import com.graphicsfuzz.common.ast.IParentMap; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.expr.MemberLookupExpr; @@ -33,7 +33,7 @@ import com.graphicsfuzz.common.ast.type.Type; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import com.graphicsfuzz.util.Constants; import java.util.Arrays; @@ -84,22 +84,22 @@ private StructifiedVariableInfo findOriginalVariableInfo() { return findOriginalVariableInfo( (StructNameType) declaration.getVariablesDeclaration().getBaseType() .getWithoutQualifiers(), - Optional.ofNullable((ScalarInitializer) declaration.getVariablesDeclaration() + Optional.ofNullable(declaration.getVariablesDeclaration() .getDeclInfo(0).getInitializer())) .get(); } private Optional findOriginalVariableInfo( StructNameType type, - Optional initializer) { + Optional initializer) { final StructDefinitionType structDefinitionType = tu.getStructDefinition(type); for (int i = 0; i < structDefinitionType.getNumFields(); i++) { final int currentIndex = i; - final Optional componentInitializer = initializer.map(item -> - new ScalarInitializer(((TypeConstructorExpr) item.getExpr()) + final Optional componentInitializer = initializer.map(item -> + new Initializer(((TypeConstructorExpr) item.getExpr()) .getArg(currentIndex))); if (structDefinitionType.getFieldName(i).startsWith(Constants.STRUCTIFICATION_FIELD_PREFIX)) { @@ -133,7 +133,7 @@ private void deStructify(StructifiedVariableInfo originalVariableInfo) { final IParentMap parentMap = IParentMap.createParentMap(tu); - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { @Override public void visitFunctionPrototype(FunctionPrototype functionPrototype) { @@ -170,7 +170,7 @@ public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { } VariableIdentifierExpr structVariable = ((VariableIdentifierExpr) memberLookupExpr .getStructure()); - ScopeEntry se = currentScope.lookupScopeEntry(structVariable.getName()); + ScopeEntry se = getCurrentScope().lookupScopeEntry(structVariable.getName()); if (se == null) { return; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenConditionalReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenConditionalReductionOpportunity.java new file mode 100644 index 000000000..a9d4808aa --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenConditionalReductionOpportunity.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.IfStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.ArrayList; +import java.util.List; + +public class FlattenConditionalReductionOpportunity extends AbstractReductionOpportunity { + + private final IAstNode parent; + private final IfStmt conditional; + private final boolean replaceWithThenBranch; + + public FlattenConditionalReductionOpportunity(IAstNode parent, IfStmt conditional, + boolean replaceWithThenBranch, + VisitationDepth depth) { + super(depth); + this.parent = parent; + this.conditional = conditional; + this.replaceWithThenBranch = replaceWithThenBranch; + } + + @Override + void applyReductionImpl() { + final List newStmts = new ArrayList<>(); + newStmts.add(new ExprStmt(conditional.getCondition())); + Stmt branchToAdd = replaceWithThenBranch ? conditional.getThenStmt() : + conditional.getElseStmt(); + if (branchToAdd instanceof BlockStmt) { + newStmts.addAll(((BlockStmt) branchToAdd).getStmts()); + } else { + newStmts.add(branchToAdd); + } + parent.replaceChild(conditional, new BlockStmt(newStmts, true)); + } + + @Override + public boolean preconditionHolds() { + return parent.hasChild(conditional) && (replaceWithThenBranch || conditional.hasElseStmt()); + } +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunities.java similarity index 62% rename from reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java rename to reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunities.java index 79a13854a..fd0c6de55 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunities.java @@ -31,16 +31,16 @@ import java.util.Arrays; import java.util.List; -public class CompoundToBlockReductionOpportunities - extends ReductionOpportunitiesBase { +public class FlattenControlFlowReductionOpportunities + extends ReductionOpportunitiesBase { - public CompoundToBlockReductionOpportunities( + private FlattenControlFlowReductionOpportunities( TranslationUnit tu, ReducerContext context) { super(tu, context); } - static List findOpportunities( + static List findOpportunities( ShaderJob shaderJob, ReducerContext context) { return shaderJob.getShaders() @@ -49,11 +49,11 @@ static List findOpportunities( .reduce(Arrays.asList(), ListConcat::concatenate); } - private static List findOpportunitiesForShader( + private static List findOpportunitiesForShader( TranslationUnit tu, ReducerContext context) { - CompoundToBlockReductionOpportunities finder = - new CompoundToBlockReductionOpportunities(tu, context); + FlattenControlFlowReductionOpportunities finder = + new FlattenControlFlowReductionOpportunities(tu, context); finder.visit(tu); return finder.getOpportunities(); } @@ -61,61 +61,70 @@ private static List findOpportunitiesForSha @Override public void visitDoStmt(DoStmt doStmt) { super.visitDoStmt(doStmt); - handleLoopStmt(doStmt); + if (allowedToReduce(doStmt)) { + addOpportunity(new FlattenDoWhileLoopReductionOpportunity( + parentMap.getParent(doStmt), + doStmt, + getVistitationDepth())); + } } @Override public void visitForStmt(ForStmt forStmt) { super.visitForStmt(forStmt); - handleLoopStmt(forStmt); + if (allowedToReduce(forStmt)) { + addOpportunity(new FlattenForLoopReductionOpportunity( + parentMap.getParent(forStmt), + forStmt, + getVistitationDepth())); + } } @Override public void visitWhileStmt(WhileStmt whileStmt) { super.visitWhileStmt(whileStmt); - handleLoopStmt(whileStmt); - } - - private void handleLoopStmt(LoopStmt loopStmt) { - if (ContainsTopLevelBreak.check(loopStmt.getBody()) - || ContainsTopLevelContinue.check(loopStmt.getBody())) { - return; - } - if (MacroNames.isDeadByConstruction(loopStmt.getCondition())) { - return; + if (allowedToReduce(whileStmt)) { + addOpportunity(new FlattenWhileLoopReductionOpportunity( + parentMap.getParent(whileStmt), + whileStmt, + getVistitationDepth())); } - addOpportunity(loopStmt, loopStmt.getBody()); } @Override public void visitIfStmt(IfStmt ifStmt) { super.visitIfStmt(ifStmt); - if (MacroNames.isDeadByConstruction(ifStmt.getCondition())) { - return; - } - addOpportunity(ifStmt, ifStmt.getThenStmt()); - if (ifStmt.hasElseStmt()) { - addOpportunity(ifStmt, ifStmt.getElseStmt()); - } - } - - private void addOpportunity(Stmt compoundStmt, - Stmt childStmt) { - if (allowedToReduce(compoundStmt)) { - addOpportunity(new CompoundToBlockReductionOpportunity( - parentMap.getParent(compoundStmt), compoundStmt, childStmt, + if (allowedToReduce(ifStmt)) { + addOpportunity(new FlattenConditionalReductionOpportunity( + parentMap.getParent(ifStmt), + ifStmt, + true, + getVistitationDepth())); + if (ifStmt.hasElseStmt()) { + addOpportunity(new FlattenConditionalReductionOpportunity( + parentMap.getParent(ifStmt), + ifStmt, + false, getVistitationDepth())); + } } } private boolean allowedToReduce(Stmt compoundStmt) { + if (compoundStmt instanceof LoopStmt) { + final LoopStmt loopStmt = (LoopStmt) compoundStmt; + if (ContainsTopLevelBreak.check(loopStmt.getBody()) + || ContainsTopLevelContinue.check(loopStmt.getBody())) { + return false; + } + } + return context.reduceEverywhere() - || injectionTracker.enclosedByDeadCodeInjection() - || injectionTracker.underUnreachableSwitchCase() + || currentProgramPointIsDeadCode() || (StmtReductionOpportunities.isLiveCodeInjection(compoundStmt) && !isLoopLimiterCheck(compoundStmt)) - || enclosingFunctionIsDead() - || SideEffectChecker.isSideEffectFree(compoundStmt, context.getShadingLanguageVersion()); + || SideEffectChecker.isSideEffectFree(compoundStmt, context.getShadingLanguageVersion(), + shaderKind); } private static boolean isLoopLimiterCheck(Stmt compoundStmt) { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenDoWhileLoopReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenDoWhileLoopReductionOpportunity.java new file mode 100644 index 000000000..a2aadf569 --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenDoWhileLoopReductionOpportunity.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.DoStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.ArrayList; +import java.util.List; + +public class FlattenDoWhileLoopReductionOpportunity extends FlattenLoopReductionOpportunity { + + public FlattenDoWhileLoopReductionOpportunity(IAstNode parent, DoStmt doStmt, + VisitationDepth depth) { + super(parent, doStmt, depth); + } + + @Override + void applyReductionImpl() { + final List newStmts = new ArrayList<>(); + addLoopBody(newStmts); + newStmts.add(new ExprStmt(getLoopStmt().getCondition())); + doReplacement(newStmts); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenForLoopReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenForLoopReductionOpportunity.java new file mode 100644 index 000000000..143764d0d --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenForLoopReductionOpportunity.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.ForStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.ArrayList; +import java.util.List; + +public class FlattenForLoopReductionOpportunity extends FlattenLoopReductionOpportunity { + + public FlattenForLoopReductionOpportunity(IAstNode parent, ForStmt forStmt, + VisitationDepth depth) { + super(parent, forStmt, depth); + } + + @Override + void applyReductionImpl() { + final List newStmts = new ArrayList<>(); + final ForStmt forStmt = (ForStmt) getLoopStmt(); + newStmts.add(forStmt.getInit()); + if (forStmt.hasCondition()) { + newStmts.add(new ExprStmt(forStmt.getCondition())); + } + addLoopBody(newStmts); + if (forStmt.hasIncrement()) { + newStmts.add(new ExprStmt(forStmt.getIncrement())); + } + doReplacement(newStmts); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenLoopReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenLoopReductionOpportunity.java new file mode 100644 index 000000000..2c6f7b3ed --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenLoopReductionOpportunity.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.LoopStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.List; + +public abstract class FlattenLoopReductionOpportunity extends AbstractReductionOpportunity { + + private final IAstNode parent; + private final LoopStmt loopStmt; + + FlattenLoopReductionOpportunity(IAstNode parent, LoopStmt loopStmt, + VisitationDepth depth) { + super(depth); + this.parent = parent; + this.loopStmt = loopStmt; + } + + LoopStmt getLoopStmt() { + return loopStmt; + } + + void doReplacement(List newStmts) { + parent.replaceChild(loopStmt, new BlockStmt(newStmts, true)); + } + + void addLoopBody(List newStmts) { + if (loopStmt.getBody() instanceof BlockStmt) { + newStmts.addAll(((BlockStmt) loopStmt.getBody()).getStmts()); + } else { + newStmts.add(loopStmt.getBody()); + } + } + + @Override + public final boolean preconditionHolds() { + return parent.hasChild(loopStmt); + } +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenWhileLoopReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenWhileLoopReductionOpportunity.java new file mode 100644 index 000000000..98fde4ca8 --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenWhileLoopReductionOpportunity.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.Stmt; +import com.graphicsfuzz.common.ast.stmt.WhileStmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.ArrayList; +import java.util.List; + +public class FlattenWhileLoopReductionOpportunity extends FlattenLoopReductionOpportunity { + + public FlattenWhileLoopReductionOpportunity(IAstNode parent, WhileStmt whileStmt, + VisitationDepth depth) { + super(parent, whileStmt, depth); + } + + @Override + void applyReductionImpl() { + final List newStmts = new ArrayList<>(); + newStmts.add(new ExprStmt(getLoopStmt().getCondition())); + addLoopBody(newStmts); + doReplacement(newStmts); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java index 392bcc679..167fe02e5 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunities.java @@ -52,6 +52,10 @@ private FoldConstantReductionOpportunities(TranslationUnit tu, @Override void identifyReductionOpportunitiesForChild(IAstNode parent, Expr child) { + if (!allowedToReduceExpr(parent, child)) { + return; + } + Optional maybeFce = asFunctionCallExpr(child); if (maybeFce.isPresent()) { switch (maybeFce.get().getCallee()) { @@ -223,14 +227,14 @@ private void findReplaceTypeConstructorWithElementOpportunities( return; } final BasicType basicType = (BasicType) structureType; - if (!BasicType.allVectorTypes().contains(basicType)) { + if (!basicType.isVector()) { return; } if (basicType.getNumElements() != tce.getNumArgs()) { // We could handle cases such as vec2(0.0).x resolving to 0.0; but for now we do not. return; } - if (!SideEffectChecker.isSideEffectFree(tce, context.getShadingLanguageVersion())) { + if (!SideEffectChecker.isSideEffectFree(tce, context.getShadingLanguageVersion(), shaderKind)) { // We mustn't eliminate side-effects from elements of the vector that we are not popping out. return; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunities.java index e3884b32c..266ed6a95 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunities.java @@ -46,7 +46,7 @@ public class FunctionReductionOpportunities extends StandardVisitor { private final List declaredFunctions; // All functions declared in the shader private FunctionReductionOpportunities(TranslationUnit tu, ReducerContext context) { - this.typer = new Typer(tu, context.getShadingLanguageVersion()); + this.typer = new Typer(tu); this.opportunities = new ArrayList<>(); this.calledFunctions = new HashSet<>(); this.declaredFunctions = Collections diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java new file mode 100644 index 000000000..2f2d785de --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunities.java @@ -0,0 +1,120 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.type.TypeQualifier; +import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.util.ListConcat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/* + * This class finds opportunities to remove initializers from global variable declarations and + * replace them with assignment statements which will be inserted as the first statement in main + * function. If main function is not found, do not consider finding this opportunities. + * For example, in: + * int a = 1; + * int b = foo(); + * void main(){ + * int c = 1; + * } + * + *

Because each of a and b has an initializer, we can transform the code fragment + * into the following: + * int a; + * int b; + * void main(){ + * a = 1; + * b = foo(); + * int c = 1; + * } + */ + +public class GlobalVariableDeclToExprReductionOpportunities + extends ReductionOpportunitiesBase { + private final List globalVariableDecl; + + private GlobalVariableDeclToExprReductionOpportunities(TranslationUnit tu, + ReducerContext context) { + super(tu, context); + this.globalVariableDecl = new ArrayList<>(); + } + + /** + * Find all initialized global variable declaration opportunities for the given translation unit. + * + * @param shaderJob The shader job to be searched. + * @param context Includes info such as whether we reduce everywhere or only reduce injections + * @return The opportunities that can be reduced + */ + static List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return shaderJob.getShaders() + .stream() + .map(item -> findOpportunitiesForShader(item, context)) + .reduce(Arrays.asList(), ListConcat::concatenate); + } + + private static List findOpportunitiesForShader( + TranslationUnit tu, + ReducerContext context) { + GlobalVariableDeclToExprReductionOpportunities finder = + new GlobalVariableDeclToExprReductionOpportunities(tu, context); + finder.visit(tu); + return finder.getOpportunities(); + } + + + @Override + public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) { + super.visitVariablesDeclaration(variablesDeclaration); + if (!context.reduceEverywhere()) { + // Replacing initializers with a new assignment statement might change semantics, + // do not consider these reduction opportunities if we are not reducing everywhere. + return; + } + + // As constant must always be initialized, we will not consider global variable declarations + // that have const qualifier (i.e., const int a = 1). + if (atGlobalScope() && !variablesDeclaration.getBaseType().hasQualifier(TypeQualifier.CONST)) { + globalVariableDecl.add(variablesDeclaration); + } + } + + @Override + public void visitFunctionDefinition(FunctionDefinition functionDefinition) { + super.visitFunctionDefinition(functionDefinition); + // We consider only the global variable declarations with the initializer that have been + // found before main function. + if (functionDefinition.getPrototype().getName().equals("main")) { + for (VariablesDeclaration variablesDeclaration : globalVariableDecl) { + for (VariableDeclInfo variableDeclInfo : variablesDeclaration.getDeclInfos()) { + if (variableDeclInfo.hasInitializer()) { + addOpportunity(new GlobalVariableDeclToExprReductionOpportunity( + getVistitationDepth(), variableDeclInfo, functionDefinition)); + } + } + } + } + } +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java new file mode 100644 index 000000000..b32c2ff91 --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunity.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.expr.BinOp; +import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; + +public class GlobalVariableDeclToExprReductionOpportunity extends AbstractReductionOpportunity { + + private final FunctionDefinition mainFunction; + // The initialized global variable declaration info. + private final VariableDeclInfo variableDeclInfo; + + GlobalVariableDeclToExprReductionOpportunity(VisitationDepth depth, + VariableDeclInfo variableDeclInfo, + FunctionDefinition mainFunction) { + super(depth); + this.variableDeclInfo = variableDeclInfo; + this.mainFunction = mainFunction; + } + + @Override + void applyReductionImpl() { + // Given the variable declaration info of global variable, we unset its initializer and + // derive a new assignment statement which will be inserted as the first statement in + // main function. + assert variableDeclInfo.hasInitializer(); + final BinaryExpr binaryExpr = new BinaryExpr( + new VariableIdentifierExpr(variableDeclInfo.getName()), + (variableDeclInfo.getInitializer()).getExpr(), + BinOp.ASSIGN + ); + mainFunction.getBody().insertStmt(0, new ExprStmt(binaryExpr)); + variableDeclInfo.setInitializer(null); + } + + @Override + public boolean preconditionHolds() { + return mainFunction.getPrototype().getName().equals("main") + && variableDeclInfo.hasInitializer(); + } +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java index c2ae7a351..47f06b667 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IReductionOpportunityFinder.java @@ -123,17 +123,17 @@ public String getName() { }; } - static IReductionOpportunityFinder compoundToBlockFinder() { - return new IReductionOpportunityFinder() { + static IReductionOpportunityFinder flattenControlFlowFinder() { + return new IReductionOpportunityFinder() { @Override - public List findOpportunities(ShaderJob shaderJob, - ReducerContext context) { - return CompoundToBlockReductionOpportunities.findOpportunities(shaderJob, context); + public List findOpportunities(ShaderJob shaderJob, + ReducerContext context) { + return FlattenControlFlowReductionOpportunities.findOpportunities(shaderJob, context); } @Override public String getName() { - return "compoundToBlock"; + return "flattenControlFlow"; } }; } @@ -462,4 +462,80 @@ public String getName() { }; } + static IReductionOpportunityFinder + redundantUniformMetadataFinder() { + return new IReductionOpportunityFinder() { + @Override + public List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return RemoveRedundantUniformMetadataReductionOpportunities.findOpportunities( + shaderJob, + context); + } + + @Override + public String getName() { + return "redundantUniformMetadata"; + } + }; + } + + static IReductionOpportunityFinder + variableDeclToExprFinder() { + return new IReductionOpportunityFinder() { + @Override + public List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return VariableDeclToExprReductionOpportunities.findOpportunities( + shaderJob, + context); + } + + @Override + public String getName() { + return "variableDeclToExpr"; + } + }; + } + + static IReductionOpportunityFinder + switchToLoopFinder() { + return new IReductionOpportunityFinder() { + @Override + public List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return SwitchToLoopReductionOpportunities.findOpportunities( + shaderJob, + context); + } + + @Override + public String getName() { + return "switchToLoop"; + } + }; + } + + static IReductionOpportunityFinder + globalVariableDeclToExprFinder() { + return new IReductionOpportunityFinder() { + @Override + public List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return GlobalVariableDeclToExprReductionOpportunities.findOpportunities( + shaderJob, + context); + } + + @Override + public String getName() { + return "globalVariableDeclToExpr"; + } + }; + } + } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunities.java index 28bb8578f..80d66b4c7 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunities.java @@ -21,6 +21,7 @@ import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.ListConcat; +import com.graphicsfuzz.common.util.MacroNames; import java.util.Arrays; import java.util.List; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunity.java index 1057d7899..2e8bb7fad 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunity.java @@ -20,6 +20,7 @@ import com.graphicsfuzz.common.ast.expr.Expr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import com.graphicsfuzz.common.util.MacroNames; public final class IdentityMutationReductionOpportunity extends AbstractReductionOpportunity { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InjectionTracker.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InjectionTracker.java index 34a7b60f3..5a6277674 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InjectionTracker.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InjectionTracker.java @@ -26,14 +26,12 @@ class InjectionTracker { private int numEnclosingFuzzedMacros; private int numEnclosingDeadCodeInjections; - private boolean underDeadMacro; private Deque> switchStmts; InjectionTracker() { this.numEnclosingFuzzedMacros = 0; this.numEnclosingDeadCodeInjections = 0; - this.underDeadMacro = false; this.switchStmts = new LinkedList<>(); } @@ -41,10 +39,6 @@ boolean underFuzzedMacro() { return numEnclosingFuzzedMacros > 0; } - boolean underDeadMacro() { - return underDeadMacro; - } - void enterFuzzedMacro() { numEnclosingFuzzedMacros++; } @@ -54,16 +48,6 @@ void exitFuzzedMacro() { numEnclosingFuzzedMacros--; } - void enterDeadMacro() { - assert !underDeadMacro; - underDeadMacro = true; - } - - void exitDeadMacro() { - assert underDeadMacro; - underDeadMacro = false; - } - void enterDeadCodeInjection() { numEnclosingDeadCodeInjections++; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java index 8be577160..6240237e3 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunities.java @@ -18,7 +18,6 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.Initializer; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.expr.ParenExpr; import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; @@ -79,7 +78,7 @@ private void addOpportunities() { if (!blackList.contains(pair.getLeft())) { addOpportunity(new SimplifyExprReductionOpportunity( parentMap.getParent(pair.getRight()), - new ParenExpr(((ScalarInitializer) pair.getLeft().getInitializer()).getExpr().clone()), + new ParenExpr((pair.getLeft().getInitializer()).getExpr().clone()), pair.getRight(), getVistitationDepth())); } @@ -89,7 +88,7 @@ private void addOpportunities() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - final ScopeEntry se = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + final ScopeEntry se = getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()); if (se == null || !se.hasVariableDeclInfo()) { return; } @@ -104,7 +103,7 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie if (new CheckPredicateVisitor() { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { - if (currentScope.isShadowed(variableIdentifierExpr.getName())) { + if (getCurrentScope().isShadowed(variableIdentifierExpr.getName())) { predicateHolds(); } } @@ -113,7 +112,7 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie } if (inLValueContext()) { - if (!context.reduceEverywhere() && !currentProgramPointHasNoEffect()) { + if (!context.reduceEverywhere() && !currentProgramPointIsDeadCode()) { // The declaration is used as an l-value. To preserve semantics we cannot inline its // initializer. For example, in: // int x = 2; @@ -138,39 +137,14 @@ private boolean allowedToReduce(VariableDeclInfo variableDeclInfo) { if (context.reduceEverywhere()) { return true; } - if (currentProgramPointHasNoEffect()) { + if (currentProgramPointIsDeadCode()) { return true; } if (StmtReductionOpportunities.isLooplimiter(variableDeclInfo.getName())) { // Do not mess with loop limiters. return false; } - if (initializerIsScalarAndSideEffectFree(variableDeclInfo) - && !referencesVariableIdentifier(variableDeclInfo.getInitializer())) { - // We need to be careful about inlining e.g.: "int x = y;", because if y is then modified, - // inlined uses of x would get the new value of y. - return true; - } - return false; - } - - private boolean referencesVariableIdentifier(Initializer initializer) { - return new CheckPredicateVisitor() { - @Override - public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { - predicateHolds(); - } - }.test(initializer); - } - - private boolean currentProgramPointHasNoEffect() { - if (injectionTracker.enclosedByDeadCodeInjection()) { - return true; - } - if (injectionTracker.underUnreachableSwitchCase()) { - return true; - } - if (enclosingFunctionIsDead()) { + if (isLiveInjectedVariableName(variableDeclInfo.getName())) { return true; } return false; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunities.java index 04e39cd53..4388e7edf 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunities.java @@ -21,14 +21,14 @@ import com.graphicsfuzz.common.ast.type.StructDefinitionType; import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import com.graphicsfuzz.util.Constants; import java.util.Arrays; import java.util.LinkedList; import java.util.List; -public class InlineStructifiedFieldReductionOpportunities extends ScopeTreeBuilder { +public class InlineStructifiedFieldReductionOpportunities extends ScopeTrackingVisitor { private final List opportunities; private final TranslationUnit tu; @@ -71,7 +71,7 @@ public void visitDeclarationStmt(DeclarationStmt declarationStmt) { public void findInliningOpportunities(StructNameType structType) { assert structType.getName().startsWith(Constants.STRUCTIFICATION_STRUCT_PREFIX); final StructDefinitionType structDefinitionType = - currentScope.lookupStructName(structType.getName()); + getCurrentScope().lookupStructName(structType.getName()); for (String f : structDefinitionType.getFieldNames()) { if (!f.startsWith(Constants.STRUCTIFICATION_FIELD_PREFIX)) { continue; @@ -81,7 +81,8 @@ public void findInliningOpportunities(StructNameType structType) { final StructNameType innerStructType = (StructNameType) structDefinitionType.getFieldType(f).getWithoutQualifiers(); opportunities.add(new InlineStructifiedFieldReductionOpportunity( - structDefinitionType, currentScope.lookupStructName(innerStructType.getName()), f, tu, + structDefinitionType, getCurrentScope().lookupStructName(innerStructType.getName()), f, + tu, getVistitationDepth())); findInliningOpportunities(innerStructType); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunity.java index 98ef0f319..5b0c6408f 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunity.java @@ -64,8 +64,7 @@ public InlineStructifiedFieldReductionOpportunity(StructDefinitionType outerStru @Override public void applyReductionImpl() { - // The GLSL version is irrelevant; really we want a Typer that doesn't require this. - final Typer typer = new Typer(tu, ShadingLanguageVersion.ESSL_100); + final Typer typer = new Typer(tu); final int indexOfInlinedField = outerStruct.getFieldIndex(fieldToInline); outerStruct.removeField(fieldToInline); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunities.java index e18c427da..0d71fd4d6 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunities.java @@ -59,8 +59,16 @@ private InlineUniformReductionOpportunities(TranslationUnit tu, @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); + + // We only inline uniforms if we are not preserving semantics, if the current program point is + // is dead code, or if the uniform is a live-injected variable. + if (!(context.reduceEverywhere() || currentProgramPointIsDeadCode() + || isLiveInjectedVariableName(variableIdentifierExpr.getName()))) { + return; + } + final String name = variableIdentifierExpr.getName(); - final ScopeEntry se = currentScope.lookupScopeEntry(name); + final ScopeEntry se = getCurrentScope().lookupScopeEntry(name); if (se == null) { return; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunity.java index eede0d738..d1885e63e 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunity.java @@ -28,7 +28,8 @@ import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; +import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.util.Constants; import java.util.ArrayList; import java.util.List; @@ -92,12 +93,12 @@ private boolean referencesBackup(Stmt stmt) { final VariableDeclInfo backupVdi = ((DeclarationStmt) block.getStmt(indexOfBackupDeclaration().get())) .getVariablesDeclaration().getDeclInfo(0); - return new ScopeTreeBuilder() { + return new ScopeTrackingVisitor() { private boolean found = false; @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - final ScopeEntry se = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + final ScopeEntry se = getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()); if (se != null && se.hasVariableDeclInfo() && se.getVariableDeclInfo() == backupVdi) { found = true; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunities.java index 11a02b377..513ce59e6 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunities.java @@ -17,7 +17,6 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; import com.graphicsfuzz.common.ast.expr.IntConstantExpr; @@ -27,7 +26,7 @@ import com.graphicsfuzz.common.ast.stmt.ForStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import com.graphicsfuzz.util.Constants; import java.util.Arrays; @@ -35,7 +34,7 @@ import java.util.List; import java.util.Optional; -public class LoopMergeReductionOpportunities extends ScopeTreeBuilder { +public class LoopMergeReductionOpportunities extends ScopeTrackingVisitor { private final List opportunities; @@ -106,9 +105,9 @@ private boolean canMergeLoops(Stmt first, Stmt second) { final BinOp firstLoopOp = ((BinaryExpr) firstLoop.getCondition()).getOp(); - final Integer secondLoopStart = new Integer(((IntConstantExpr) ((ScalarInitializer) - ((DeclarationStmt) secondLoop.getInit()).getVariablesDeclaration().getDeclInfo(0) - .getInitializer()).getExpr()).getValue()); + final Integer secondLoopStart = new Integer(((IntConstantExpr) ( + (DeclarationStmt) secondLoop.getInit()).getVariablesDeclaration().getDeclInfo(0) + .getInitializer().getExpr()).getValue()); assert firstLoopOp == BinOp.LT || firstLoopOp == BinOp.GT : "Unexpected operator in split loops."; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/NotReferencedFromLiveContext.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/NotReferencedFromLiveContext.java index 2929493a2..c29d3b4f0 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/NotReferencedFromLiveContext.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/NotReferencedFromLiveContext.java @@ -21,6 +21,7 @@ import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.util.MacroNames; import java.util.HashSet; import java.util.Set; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunities.java index 727340ba1..c4a3cd759 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunities.java @@ -29,6 +29,7 @@ import com.graphicsfuzz.util.Constants; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -48,8 +49,8 @@ static List findOpportunities( ReducerContext context) { return shaderJob.getShaders() .stream() - .map(item -> findOpportunitiesForShader(item)) - .reduce(Arrays.asList(), ListConcat::concatenate); + .map(OutlinedStatementReductionOpportunities::findOpportunitiesForShader) + .reduce(Collections.emptyList(), ListConcat::concatenate); } private static List findOpportunitiesForShader( diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReducerContext.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReducerContext.java index 8ff3e3bd9..85f07e106 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReducerContext.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReducerContext.java @@ -22,8 +22,8 @@ public class ReducerContext { - static final int DEFAULT_MAX_PERCENTAGE_TO_REDUCE = 50; - static final int DEFAULT_AGGRESSION_DECREASE_STEP = 5; + private static final int DEFAULT_MAX_PERCENTAGE_TO_REDUCE = 50; + private static final int DEFAULT_AGGRESSION_DECREASE_STEP = 5; private final boolean reduceEverywhere; private final ShadingLanguageVersion shadingLanguageVersion; @@ -31,27 +31,24 @@ public class ReducerContext { private final IdGenerator idGenerator; private final int maxPercentageToReduce; private final int aggressionDecreaseStep; - private final boolean emitGraphicsFuzzDefines; public ReducerContext(boolean reduceEverywhere, ShadingLanguageVersion shadingLanguageVersion, IRandom random, IdGenerator idGenerator, int maxPercentageToReduce, - int aggressionDecreaseStep, boolean emitGraphicsFuzzDefines) { + int aggressionDecreaseStep) { this.reduceEverywhere = reduceEverywhere; this.shadingLanguageVersion = shadingLanguageVersion; this.random = random; this.idGenerator = idGenerator; this.maxPercentageToReduce = maxPercentageToReduce; this.aggressionDecreaseStep = aggressionDecreaseStep; - this.emitGraphicsFuzzDefines = emitGraphicsFuzzDefines; } public ReducerContext(boolean reduceEverywhere, ShadingLanguageVersion shadingLanguageVersion, - IRandom random, IdGenerator idGenerator, boolean emitGraphicsFuzzDefines) { + IRandom random, IdGenerator idGenerator) { this(reduceEverywhere, shadingLanguageVersion, random, idGenerator, - DEFAULT_MAX_PERCENTAGE_TO_REDUCE, DEFAULT_AGGRESSION_DECREASE_STEP, - emitGraphicsFuzzDefines); + DEFAULT_MAX_PERCENTAGE_TO_REDUCE, DEFAULT_AGGRESSION_DECREASE_STEP); } public boolean reduceEverywhere() { @@ -81,8 +78,4 @@ public int getAggressionDecreaseStep() { return aggressionDecreaseStep; } - public boolean getEmitGraphicsFuzzDefines() { - return emitGraphicsFuzzDefines; - } - } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java index 669e5680e..bc16d6f4f 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunities.java @@ -59,13 +59,17 @@ public static List getReductionOpportunities( IReductionOpportunityFinder.exprToConstantFinder(), IReductionOpportunityFinder.compoundExprToSubExprFinder(), IReductionOpportunityFinder.mutationFinder(), - IReductionOpportunityFinder.compoundToBlockFinder(), + IReductionOpportunityFinder.flattenControlFlowFinder(), IReductionOpportunityFinder.inlineInitializerFinder(), IReductionOpportunityFinder.inlineFunctionFinder(), IReductionOpportunityFinder.liveFragColorWriteFinder(), IReductionOpportunityFinder.unusedParamFinder(), IReductionOpportunityFinder.foldConstantFinder(), - IReductionOpportunityFinder.inlineUniformFinder())) { + IReductionOpportunityFinder.inlineUniformFinder(), + IReductionOpportunityFinder.redundantUniformMetadataFinder(), + IReductionOpportunityFinder.variableDeclToExprFinder(), + IReductionOpportunityFinder.switchToLoopFinder(), + IReductionOpportunityFinder.globalVariableDeclToExprFinder())) { final List currentOpportunities = ros .findOpportunities(shaderJob, context); if (ReductionDriver.DEBUG_REDUCER) { diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java index 812c077a2..2a2f49561 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesBase.java @@ -20,7 +20,6 @@ import com.graphicsfuzz.common.ast.IParentMap; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.expr.BinaryExpr; import com.graphicsfuzz.common.ast.expr.ConstantExpr; @@ -35,7 +34,9 @@ import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.ast.type.Type; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; +import com.graphicsfuzz.common.util.MacroNames; +import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.common.util.SideEffectChecker; import com.graphicsfuzz.util.Constants; import java.util.ArrayList; @@ -43,16 +44,18 @@ public abstract class ReductionOpportunitiesBase - extends ScopeTreeBuilder { + extends ScopeTrackingVisitor { private final List opportunities; - protected final InjectionTracker injectionTracker; - protected final NotReferencedFromLiveContext notReferencedFromLiveContext; + final InjectionTracker injectionTracker; + final NotReferencedFromLiveContext notReferencedFromLiveContext; protected final IParentMap parentMap; protected final ReducerContext context; - protected String enclosingFunctionName; + final ShaderKind shaderKind; + + private String enclosingFunctionName; private int numEnclosingLValues; @@ -66,9 +69,10 @@ public ReductionOpportunitiesBase(TranslationUnit tu, ReducerContext context) { this.opportunities = new ArrayList<>(); this.injectionTracker = new InjectionTracker(); this.notReferencedFromLiveContext = new NotReferencedFromLiveContext(tu); + this.parentMap = IParentMap.createParentMap(tu); this.context = context; + this.shaderKind = tu.getShaderKind(); this.enclosingFunctionName = null; - this.parentMap = IParentMap.createParentMap(tu); this.numEnclosingLValues = 0; } @@ -77,7 +81,7 @@ public void visitFunctionDefinition(FunctionDefinition functionDefinition) { assert enclosingFunctionName == null; enclosingFunctionName = functionDefinition.getPrototype().getName(); super.visitFunctionDefinition(functionDefinition); - assert enclosingFunctionName == functionDefinition.getPrototype().getName(); + assert enclosingFunctionName.equals(functionDefinition.getPrototype().getName()); enclosingFunctionName = null; } @@ -90,8 +94,10 @@ public void visitSwitchStmt(SwitchStmt switchStmt) { @Override public void visitExprCaseLabel(ExprCaseLabel exprCaseLabel) { - super.visitExprCaseLabel(exprCaseLabel); - injectionTracker.notifySwitchCase(exprCaseLabel); + // Do *not* invoke super.visitExprCaseLabel(...), as we do not wish to look for opportunities + // to simplify the literal expression that is used as a case label. Literals are simple enough + // already, and attempting to change the value of a case label literal runs the risk of + // introducing duplicate case labels. } protected void visitChildOfBlock(BlockStmt block, int childIndex) { @@ -104,31 +110,33 @@ public void visitBlockStmt(BlockStmt block) { for (int i = 0; i < block.getNumStmts(); i++) { - visitChildOfBlock(block, i); - final Stmt child = block.getStmt(i); + if (child instanceof ExprCaseLabel) { + // It is important to notify the injection tracker of a switch case *before* we visit the + // switch case so that we can identify when we have entered the reachable part of an + // injected switch statement. + injectionTracker.notifySwitchCase((ExprCaseLabel) child); + } + + visitChildOfBlock(block, i); + if (child instanceof BreakStmt && parentMap.getParent(block) instanceof SwitchStmt) { injectionTracker.notifySwitchBreak(); } - if (isDeadCodeInjection(child)) { - injectionTracker.enterDeadCodeInjection(); - } visit(child); - if (isDeadCodeInjection(child)) { - injectionTracker.exitDeadCodeInjection(); - } + } leaveBlockStmt(block); } - public static boolean isDeadCodeInjection(Stmt stmt) { + static boolean isDeadCodeInjection(Stmt stmt) { return stmt instanceof IfStmt && MacroNames .isDeadByConstruction(((IfStmt) stmt).getCondition()); } - protected boolean enclosingFunctionIsDead() { + private boolean enclosingFunctionIsDead() { if (enclosingFunctionName == null) { // We are in global scope return false; @@ -137,6 +145,12 @@ protected boolean enclosingFunctionIsDead() { || notReferencedFromLiveContext.neverCalledFromLiveContext(enclosingFunctionName); } + boolean currentProgramPointIsDeadCode() { + return injectionTracker.enclosedByDeadCodeInjection() + || injectionTracker.underUnreachableSwitchCase() + || enclosingFunctionIsDead(); + } + @Override public void visitBinaryExpr(BinaryExpr binaryExpr) { boolean isSideEffecting = binaryExpr.getOp().isSideEffecting(); @@ -166,18 +180,36 @@ public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { if (MacroNames.isFuzzed(functionCallExpr)) { injectionTracker.enterFuzzedMacro(); } - if (MacroNames.isDeadByConstruction(functionCallExpr)) { - injectionTracker.enterDeadMacro(); - } super.visitFunctionCallExpr(functionCallExpr); - if (MacroNames.isDeadByConstruction(functionCallExpr)) { - injectionTracker.exitDeadMacro(); - } if (MacroNames.isFuzzed(functionCallExpr)) { injectionTracker.exitFuzzedMacro(); } } + @Override + public void visitIfStmt(IfStmt ifStmt) { + // This method is overridden in order to allow tracking of when we are inside a dead code + // injection. + + // Even for an "if(_GLF_DEAD(...))", the condition itself is not inside the dead code injection, + // so we visit it as usual first. + visitChildFromParent(ifStmt.getCondition(), ifStmt); + + if (isDeadCodeInjection(ifStmt)) { + // This is a dead code injection, so update the injection tracker appropriately before + // visiting the 'then' and (possibly) 'else' branches. + injectionTracker.enterDeadCodeInjection(); + } + visit(ifStmt.getThenStmt()); + if (ifStmt.hasElseStmt()) { + visit(ifStmt.getElseStmt()); + } + if (isDeadCodeInjection(ifStmt)) { + // Update the injection tracker to indicate that we have left this dead code injection. + injectionTracker.exitDeadCodeInjection(); + } + } + @Override protected void visitChildFromParent(IAstNode child, IAstNode parent) { super.visitChildFromParent(child, parent); @@ -207,12 +239,10 @@ boolean initializerIsScalarAndSideEffectFree(VariableDeclInfo variableDeclInfo) if (!variableDeclInfo.hasInitializer()) { return false; } - if (!(variableDeclInfo.getInitializer() instanceof ScalarInitializer)) { - return false; - } return SideEffectChecker.isSideEffectFree( - ((ScalarInitializer) variableDeclInfo.getInitializer()).getExpr(), - context.getShadingLanguageVersion()); + (variableDeclInfo.getInitializer()).getExpr(), + context.getShadingLanguageVersion(), + shaderKind); } boolean typeIsReducibleToConst(Type type) { @@ -248,4 +278,8 @@ final void addOpportunity( } } + static boolean isLiveInjectedVariableName(String name) { + return name.startsWith(Constants.LIVE_PREFIX); + } + } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunities.java new file mode 100644 index 000000000..8d9fddbc4 --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunities.java @@ -0,0 +1,71 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; +import com.graphicsfuzz.common.ast.type.TypeQualifier; +import com.graphicsfuzz.common.ast.visitors.StandardVisitor; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import com.graphicsfuzz.common.transformreduce.ShaderJob; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class RemoveRedundantUniformMetadataReductionOpportunities { + + private RemoveRedundantUniformMetadataReductionOpportunities() { + // This class just provides a static method; there is no cause to create an instance of the + // class. + } + + static List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + + // We initially grab names of all uniforms in the pipeline info. + final List canBeRemoved = + new ArrayList<>(shaderJob.getPipelineInfo().getUniformNames()); + + for (TranslationUnit tu : shaderJob.getShaders()) { + new StandardVisitor() { + @Override + public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) { + super.visitVariablesDeclaration(variablesDeclaration); + // A uniform in the pipeline info that its declaration is found in shaders + // will not be removed by the reducer. + if (variablesDeclaration.getBaseType().hasQualifier(TypeQualifier.UNIFORM)) { + for (VariableDeclInfo vdi : variablesDeclaration.getDeclInfos()) { + canBeRemoved.remove(vdi.getName()); + } + } + } + }.visit(tu); + } + + // Since we are going to remove redundant uniforms in the pipeline info that do not + // belong to any shader, the visitation depth should be zero. + return canBeRemoved + .stream() + .map(item -> new RemoveRedundantUniformMetadataReductionOpportunity(item, + shaderJob.getPipelineInfo(), + new VisitationDepth(0)) + ).collect(Collectors.toList()); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunity.java new file mode 100644 index 000000000..aaa98914e --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunity.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import com.graphicsfuzz.common.util.PipelineInfo; + +public class RemoveRedundantUniformMetadataReductionOpportunity + extends AbstractReductionOpportunity { + + private final PipelineInfo pipelineInfo; + private final String uniformName; + + RemoveRedundantUniformMetadataReductionOpportunity(String uniformName, + PipelineInfo pipelineInfo, + VisitationDepth depth) { + super(depth); + this.pipelineInfo = pipelineInfo; + this.uniformName = uniformName; + } + + @Override + void applyReductionImpl() { + pipelineInfo.removeUniform(uniformName); + } + + @Override + public boolean preconditionHolds() { + return pipelineInfo.getUniformNames().contains(uniformName); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunities.java index 11d2ee2cc..eaae73780 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunities.java @@ -22,14 +22,14 @@ import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import com.graphicsfuzz.util.Constants; import java.util.Arrays; import java.util.LinkedList; import java.util.List; -public class RemoveStructFieldReductionOpportunities extends ScopeTreeBuilder { +public class RemoveStructFieldReductionOpportunities extends ScopeTrackingVisitor { private final List opportunities; private final TranslationUnit translationUnit; @@ -76,7 +76,7 @@ private void getOpportunitiesForStruct(StructNameType structType, } final StructDefinitionType structDefinitionType = - currentScope.lookupStructName(structType.getName()); + getCurrentScope().lookupStructName(structType.getName()); for (String field : structDefinitionType.getFieldNames()) { if (!reachesOriginalVariable(structDefinitionType, field) @@ -109,7 +109,7 @@ private boolean reachesOriginalVariable(StructDefinitionType structDefinitionTyp final StructNameType fieldType = (StructNameType) structDefinitionType.getFieldType(field).getWithoutQualifiers(); final StructDefinitionType nestedStruct = - currentScope.lookupStructName(fieldType.getName()); + getCurrentScope().lookupStructName(fieldType.getName()); return nestedStruct.getFieldNames().stream() .anyMatch(item -> reachesOriginalVariable(nestedStruct, item)); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunities.java index 3ec88e2a2..d000fcb08 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunities.java @@ -125,7 +125,8 @@ public void visitFunctionDefinition(FunctionDefinition functionDefinition) { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - final ScopeEntry scopeEntry = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + final ScopeEntry scopeEntry = getCurrentScope().lookupScopeEntry(variableIdentifierExpr + .getName()); if (scopeEntry != null && scopeEntry.hasParameterDecl()) { unusedParametersForCurrentFunction.remove(scopeEntry.getParameterDecl()); } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ShadowChecker.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ShadowChecker.java index 76ff52684..5e8b27f06 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ShadowChecker.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/ShadowChecker.java @@ -20,9 +20,9 @@ import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; -class ShadowChecker extends ScopeTreeBuilder { +class ShadowChecker extends ScopeTrackingVisitor { private final BlockStmt blockOfInterest; private final String nameOfInterest; @@ -40,7 +40,7 @@ class ShadowChecker extends ScopeTreeBuilder { public void visitBlockStmt(BlockStmt stmt) { if (stmt == blockOfInterest) { inBlock = true; - possiblyShadowedScopeEntry = currentScope.lookupScopeEntry(nameOfInterest); + possiblyShadowedScopeEntry = getCurrentScope().lookupScopeEntry(nameOfInterest); } super.visitBlockStmt(stmt); if (stmt == blockOfInterest) { @@ -53,7 +53,7 @@ public void visitBlockStmt(BlockStmt stmt) { public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); if (inBlock && possiblyShadowedScopeEntry != null - && currentScope.lookupScopeEntry(variableIdentifierExpr.getName()) + && getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()) == possiblyShadowedScopeEntry) { ok = false; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java index 46e995a35..014f4260a 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SimplifyExprReductionOpportunities.java @@ -41,7 +41,7 @@ abstract class SimplifyExprReductionOpportunities TranslationUnit tu, ReducerContext context) { super(tu, context); - this.typer = new Typer(tu, context.getShadingLanguageVersion()); + this.typer = new Typer(tu); this.inLiveInjectedStmtOrDeclaration = false; } @@ -74,8 +74,8 @@ public void visitDeclarationStmt(DeclarationStmt declarationStmt) { boolean allowedToReduceExpr(IAstNode parent, Expr child) { if (child instanceof VariableIdentifierExpr) { - String name = ((VariableIdentifierExpr) child).getName(); - if (name.startsWith(Constants.LIVE_PREFIX) + final String name = ((VariableIdentifierExpr) child).getName(); + if (isLiveInjectedVariableName(name) && !StmtReductionOpportunities.isLooplimiter(name)) { return true; } @@ -102,7 +102,7 @@ boolean allowedToReduceExpr(IAstNode parent, Expr child) { return true; } - if (enclosingFunctionIsDead()) { + if (currentProgramPointIsDeadCode()) { return true; } @@ -110,11 +110,6 @@ boolean allowedToReduceExpr(IAstNode parent, Expr child) { return true; } - if (injectionTracker.enclosedByDeadCodeInjection() && !injectionTracker.underDeadMacro()) { - // We want to reduce expressions in dead code blocks, but not inside the dead macro itself - return true; - } - if (inLiveInjectedStmtOrDeclaration && !referencesLoopLimiter(child)) { return true; } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java index 6b9a3e265..493127e12 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunities.java @@ -34,12 +34,12 @@ import com.graphicsfuzz.common.ast.stmt.IfStmt; import com.graphicsfuzz.common.ast.stmt.LoopStmt; import com.graphicsfuzz.common.ast.stmt.NullStmt; -import com.graphicsfuzz.common.ast.stmt.ReturnStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.ListConcat; +import com.graphicsfuzz.common.util.MacroNames; import com.graphicsfuzz.common.util.SideEffectChecker; import com.graphicsfuzz.common.util.StructUtils; import com.graphicsfuzz.util.Constants; @@ -79,7 +79,8 @@ protected void visitChildOfBlock(BlockStmt block, int index) { final Stmt child = block.getStmt(index); if (isEmptyBlockStmt(child) || isDeadCodeInjection(child) || allowedToReduceStmt(block, index)) { - addOpportunity(new StmtReductionOpportunity(block, child, getVistitationDepth())); + addOpportunity(new StmtReductionOpportunity(getEnclosingFunction(), block, child, + getVistitationDepth())); } } @@ -110,15 +111,11 @@ private boolean allowedToReduceStmt(BlockStmt block, int childIndex) { return true; } - if (SideEffectChecker.isSideEffectFree(stmt, context.getShadingLanguageVersion())) { + if (SideEffectChecker.isSideEffectFree(stmt, context.getShadingLanguageVersion(), shaderKind)) { return true; } - if (injectionTracker.enclosedByDeadCodeInjection()) { - return true; - } - - if (injectionTracker.underUnreachableSwitchCase() && !isZeroSwitchCase(stmt)) { + if (currentProgramPointIsDeadCode()) { return true; } @@ -126,35 +123,16 @@ private boolean allowedToReduceStmt(BlockStmt block, int childIndex) { return true; } - // We cannot remove non-void return statements without special care, unless we are inside an - // injected block - if (containsNonVoidReturn(stmt)) { - if (!isReturnFollowedBySubsequentReturn(block, childIndex)) { - return false; - } + // Unless we are in an injected dead code block, we need to be careful about removing + // non-void return statements, so as to avoid making the shader invalid. + if (StmtReductionOpportunity + .removalCouldLeadToLackOfReturnFromNonVoidFunction(getEnclosingFunction(), block, stmt)) { + return false; } return context.reduceEverywhere() - || (isLiveCodeInjection(stmt) && !referencesLoopLimiter(stmt)) - || enclosingFunctionIsDead(); - - } + || (isLiveCodeInjection(stmt) && !referencesLoopLimiter(stmt)); - /** - * Returns true if and only if the statement at childIndex is a return statement, and the block - * has another return statement at a later index. - */ - private boolean isReturnFollowedBySubsequentReturn(BlockStmt block, int childIndex) { - if (!(block.getStmt(childIndex) instanceof ReturnStmt)) { - return false; - } - for (int subsequentChildIndex = childIndex + 1; subsequentChildIndex < block.getNumStmts(); - subsequentChildIndex++) { - if (block.getStmt(subsequentChildIndex) instanceof ReturnStmt) { - return true; - } - } - return false; } private boolean isLoopLimiterBlock(Stmt stmt) { @@ -192,11 +170,6 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie }.test(stmt); } - private boolean isZeroSwitchCase(Stmt stmt) { - return stmt instanceof ExprCaseLabel - && ((IntConstantExpr) ((ExprCaseLabel) stmt).getExpr()).getValue().equals("0"); - } - /** * Determines whether the given statement came from a live code injection. This is the case if * one of the following holds: @@ -276,10 +249,6 @@ private static boolean isLiveCodeVariableDeclaration(VariableDeclInfo vdi) { return isLiveInjectedVariableName(name); } - private static boolean isLiveInjectedVariableName(String name) { - return name.startsWith(Constants.LIVE_PREFIX); - } - /** *

* Determines whether the given statement came from a live code injection. @@ -323,17 +292,6 @@ private static boolean isLiveInjectionVariableReference(Expr lhs) { return isLiveInjectedVariableName(((VariableIdentifierExpr) lhs).getName()); } - private boolean containsNonVoidReturn(Stmt stmt) { - return new CheckPredicateVisitor() { - @Override - public void visitReturnStmt(ReturnStmt returnStmt) { - if (returnStmt.hasExpr()) { - predicateHolds(); - } - } - }.test(stmt); - } - private boolean isRedundantCopy(Stmt stmt) { if (!(stmt instanceof ExprStmt)) { return false; diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunity.java index 2c63c9fee..0a7c9fa2d 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunity.java @@ -16,15 +16,18 @@ package com.graphicsfuzz.reducer.reductionopportunities; +import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.stmt.BlockStmt; import com.graphicsfuzz.common.ast.stmt.CaseLabel; +import com.graphicsfuzz.common.ast.stmt.ReturnStmt; import com.graphicsfuzz.common.ast.stmt.Stmt; -import com.graphicsfuzz.common.ast.stmt.SwitchStmt; +import com.graphicsfuzz.common.ast.visitors.CheckPredicateVisitor; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; import com.graphicsfuzz.common.util.StatsVisitor; public final class StmtReductionOpportunity extends AbstractReductionOpportunity { + private final FunctionDefinition enclosingFunction; private final BlockStmt blockStmt; private final Stmt childOfBlockStmt; @@ -33,19 +36,24 @@ public final class StmtReductionOpportunity extends AbstractReductionOpportunity // due to the effects of other opportunities). private final int numRemovableNodes; - public StmtReductionOpportunity(BlockStmt blockStmt, + /** + * Creates an opportunity to remove childOfBlockStmt from blockStmt. + * @param enclosingFunction A function that encloses the statement to be removed + * @param blockStmt A block that directly encloses the statement to be removed + * @param childOfBlockStmt The statement to be removed + * @param depth The depth at which this opportunity was found + */ + public StmtReductionOpportunity(FunctionDefinition enclosingFunction, + BlockStmt blockStmt, Stmt childOfBlockStmt, VisitationDepth depth) { super(depth); + this.enclosingFunction = enclosingFunction; this.blockStmt = blockStmt; this.childOfBlockStmt = childOfBlockStmt; this.numRemovableNodes = new StatsVisitor(childOfBlockStmt).getNumNodes(); } - public Stmt getChild() { - return childOfBlockStmt; - } - @Override public void applyReductionImpl() { for (int i = 0; i < blockStmt.getNumStmts(); i++) { @@ -94,6 +102,11 @@ public boolean preconditionHolds() { return false; } + if (removalCouldLeadToLackOfReturnFromNonVoidFunction(enclosingFunction, blockStmt, + childOfBlockStmt)) { + return false; + } + return true; } @@ -101,4 +114,54 @@ public int getNumRemovableNodes() { return numRemovableNodes; } + static boolean removalCouldLeadToLackOfReturnFromNonVoidFunction( + FunctionDefinition enclosingFunction, BlockStmt block, Stmt childOfBlockStmt) { + if (!containsNonVoidReturn(childOfBlockStmt)) { + return false; + } + if (statementIsPrecededByReturn(block, childOfBlockStmt)) { + return false; + } + if (functionEndsWithReturn(enclosingFunction) && !isLastStatementInFunction(enclosingFunction, + childOfBlockStmt)) { + return false; + } + return true; + } + + private static boolean containsNonVoidReturn(Stmt stmt) { + return new CheckPredicateVisitor() { + @Override + public void visitReturnStmt(ReturnStmt returnStmt) { + if (returnStmt.hasExpr()) { + predicateHolds(); + } + } + }.test(stmt); + } + + private static boolean statementIsPrecededByReturn(BlockStmt block, Stmt childOfBlockStmt) { + for (Stmt stmt : block.getStmts()) { + if (stmt == childOfBlockStmt) { + return false; + } + if (stmt instanceof ReturnStmt) { + return true; + } + } + throw new RuntimeException("Should be unreachable."); + } + + private static boolean functionEndsWithReturn(FunctionDefinition functionDefinition) { + return functionDefinition.getBody().getNumStmts() > 0 + && functionDefinition.getBody().getLastStmt() + instanceof ReturnStmt; + } + + private static boolean isLastStatementInFunction(FunctionDefinition functionDefinition, + Stmt stmt) { + return functionDefinition.getBody().getNumStmts() > 0 + && functionDefinition.getBody().getLastStmt() == stmt; + } + } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StructifiedVariableInfo.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StructifiedVariableInfo.java index 57bfca543..2d13ff75f 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StructifiedVariableInfo.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/StructifiedVariableInfo.java @@ -16,7 +16,7 @@ package com.graphicsfuzz.reducer.reductionopportunities; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.type.Type; import java.util.Optional; @@ -24,9 +24,9 @@ class StructifiedVariableInfo { private final String name; private final Type type; - private final Optional initializer; + private final Optional initializer; - StructifiedVariableInfo(String name, Type type, Optional initializer) { + StructifiedVariableInfo(String name, Type type, Optional initializer) { this.name = name; this.type = type; this.initializer = initializer; @@ -40,7 +40,7 @@ public Type getType() { return type; } - public Optional getInitializer() { + public Optional getInitializer() { return initializer; } } diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunities.java new file mode 100644 index 000000000..06eca2a2a --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunities.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; +import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.util.ListConcat; +import java.util.Collections; +import java.util.List; + +public class SwitchToLoopReductionOpportunities extends + ReductionOpportunitiesBase { + + /** + * Find all switch-to-loop opportunities for the given translation unit. + * + * @param shaderJob The shader job to be searched. + * @param context Includes info such as whether we reduce everywhere + * @return The opportunities that can be reduced + */ + static List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return shaderJob.getShaders() + .stream() + .map(item -> findOpportunitiesForShader(item, context)) + .reduce(Collections.emptyList(), ListConcat::concatenate); + } + + private static List findOpportunitiesForShader( + TranslationUnit tu, + ReducerContext context) { + final SwitchToLoopReductionOpportunities finder = + new SwitchToLoopReductionOpportunities(tu, context); + finder.visit(tu); + return finder.getOpportunities(); + } + + private SwitchToLoopReductionOpportunities(TranslationUnit tu, ReducerContext context) { + super(tu, context); + } + + @Override + public void visitSwitchStmt(SwitchStmt switchStmt) { + super.visitSwitchStmt(switchStmt); + if (context.reduceEverywhere() || injectionTracker.enclosedByDeadCodeInjection()) { + addOpportunity(new SwitchToLoopReductionOpportunity(getVistitationDepth(), + parentMap.getParent(switchStmt), + switchStmt)); + } + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunity.java new file mode 100644 index 000000000..ba0c089a5 --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunity.java @@ -0,0 +1,63 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.IAstNode; +import com.graphicsfuzz.common.ast.expr.BoolConstantExpr; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.CaseLabel; +import com.graphicsfuzz.common.ast.stmt.DoStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.SwitchStmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; +import java.util.Collections; +import java.util.stream.Collectors; + +/** + * Turns a switch statement into a do...while(false) loop. The body of the loop comprises all + * the non-case/default statements from the switch. The reason a loop is used is to allow for + * break statements. The reduction opportunity is not semantics-preserving. + */ +public class SwitchToLoopReductionOpportunity extends AbstractReductionOpportunity { + + // The parent of the switch statement + private final IAstNode parent; + + // The switch statement to be replaced + private final SwitchStmt switchStmt; + + SwitchToLoopReductionOpportunity(VisitationDepth depth, IAstNode parent, SwitchStmt switchStmt) { + super(depth); + this.parent = parent; + this.switchStmt = switchStmt; + } + + @Override + void applyReductionImpl() { + final BlockStmt loopBody = new BlockStmt(Collections.emptyList(), true); + loopBody.addStmt(new ExprStmt(switchStmt.getExpr())); + switchStmt.getBody().getStmts().stream().filter(item -> !(item instanceof CaseLabel)) + .forEach(loopBody::addStmt); + parent.replaceChild(switchStmt, new DoStmt(loopBody, new BoolConstantExpr(false))); + } + + @Override + public boolean preconditionHolds() { + return parent.hasChild(switchStmt); + } + +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunities.java index a36337507..6a7c75363 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunities.java @@ -20,13 +20,14 @@ import com.graphicsfuzz.common.ast.stmt.SwitchStmt; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.ListConcat; +import com.graphicsfuzz.common.util.MacroNames; import java.util.Arrays; import java.util.List; public class UnswitchifyReductionOpportunities extends ReductionOpportunitiesBase { - public UnswitchifyReductionOpportunities( + private UnswitchifyReductionOpportunities( TranslationUnit tu, ReducerContext context) { super(tu, context); diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunities.java index 72299cbe7..1c0a7ec23 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunities.java @@ -17,9 +17,7 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.stmt.BlockStmt; -import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; import com.graphicsfuzz.common.ast.stmt.DoStmt; import com.graphicsfuzz.common.ast.stmt.ForStmt; import com.graphicsfuzz.common.ast.stmt.IfStmt; @@ -27,12 +25,12 @@ import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.ListConcat; +import com.graphicsfuzz.common.util.MacroNames; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; public class UnwrapReductionOpportunities extends ReductionOpportunitiesBase { @@ -120,7 +118,8 @@ protected void visitChildOfBlock(BlockStmt block, int childIndex) { final Set namesDeclaredDirectlyByParentOrInScopeAlready = new HashSet<>(); namesDeclaredDirectlyByParentOrInScopeAlready.addAll( UnwrapReductionOpportunity.getNamesDeclaredDirectlyByBlock(block)); - namesDeclaredDirectlyByParentOrInScopeAlready.addAll(currentScope.namesOfAllVariablesInScope()); + namesDeclaredDirectlyByParentOrInScopeAlready.addAll(getCurrentScope() + .namesOfAllVariablesInScope()); final Set namesDeclaredDirectlyByChild = UnwrapReductionOpportunity.getNamesDeclaredDirectlyByBlock(childAsBlock); if (Collections.disjoint(namesDeclaredDirectlyByParentOrInScopeAlready, diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunities.java index aeddad550..dcd5f4187 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunities.java @@ -46,8 +46,8 @@ private VariableDeclReductionOpportunities(TranslationUnit tu, } private void getReductionOpportunitiesForUnusedGlobals() { - for (String name : currentScope.keys()) { - ScopeEntry entry = currentScope.lookupScopeEntry(name); + for (String name : getCurrentScope().keys()) { + ScopeEntry entry = getCurrentScope().lookupScopeEntry(name); assert entry.hasVariableDeclInfo(); assert referencedScopeEntries.peek() != null; if (!referencedScopeEntries.peek().contains(entry)) { @@ -66,8 +66,8 @@ protected void pushScope() { @Override protected void popScope() { - for (String name : currentScope.keys()) { - ScopeEntry entry = currentScope.lookupScopeEntry(name); + for (String name : getCurrentScope().keys()) { + ScopeEntry entry = getCurrentScope().lookupScopeEntry(name); if (entry.hasVariableDeclInfo() && !referencedScopeEntries.peek().contains(entry)) { if (allowedToReduceLocalDecl(entry.getVariableDeclInfo())) { addOpportunity( @@ -92,9 +92,9 @@ protected void popScope() { // refer to a variable x in the parent scope, even though the current // scope also declares a variable x after the usage of x String name = entry.getVariableDeclInfo().getName(); - assert currentScope.lookupScopeEntry(name) == entry - || currentScope.getParent().lookupScopeEntry(name) == entry; - if (!currentScope.keys().contains(name)) { + assert getCurrentScope().lookupScopeEntry(name) == entry + || getCurrentScope().getParent().lookupScopeEntry(name) == entry; + if (!getCurrentScope().keys().contains(name)) { addReferencedScopeEntry(entry); } } @@ -110,7 +110,7 @@ private void addReferencedScopeEntry(ScopeEntry scopeEntry) { @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { super.visitVariableIdentifierExpr(variableIdentifierExpr); - ScopeEntry scopeEntry = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + ScopeEntry scopeEntry = getCurrentScope().lookupScopeEntry(variableIdentifierExpr.getName()); if (scopeEntry != null) { addReferencedScopeEntry(scopeEntry); } @@ -128,17 +128,12 @@ private boolean allowedToReduceLocalDecl(VariableDeclInfo variableDeclInfo) { } // Fine to remove if in a dead context, a live context, if no initializer, or if // initializer does not have side effects. - return context.reduceEverywhere() || enclosingFunctionIsDead() - || injectionTracker.enclosedByDeadCodeInjection() - || isLiveInjection(variableDeclInfo) + return context.reduceEverywhere() || currentProgramPointIsDeadCode() + || isLiveInjectedVariableName(variableDeclInfo.getName()) || !variableDeclInfo.hasInitializer() || initializerIsScalarAndSideEffectFree(variableDeclInfo); } - private boolean isLiveInjection(VariableDeclInfo variableDeclInfo) { - return variableDeclInfo.getName().startsWith(Constants.LIVE_PREFIX); - } - /** * Find all unused declaration opportunities for the given translation unit. * diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java new file mode 100644 index 000000000..055cad2df --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunities.java @@ -0,0 +1,105 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; +import com.graphicsfuzz.common.ast.type.TypeQualifier; +import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.util.ListConcat; +import java.util.Arrays; +import java.util.List; + +/* + * This class finds opportunities to remove initializers from variable declarations and + * replace them with assignment statements following the declaration. For example, in: + * int a = 1; + * int b = foo(); + * int c; + * + *

Because each of a and b has an initializer, we can transform the code fragment + * into the following: + * int a; + * a = 1; + * int b; + * b = foo(); + * int c; + */ + +public class VariableDeclToExprReductionOpportunities + extends ReductionOpportunitiesBase { + + private VariableDeclToExprReductionOpportunities(TranslationUnit tu, ReducerContext context) { + super(tu, context); + } + + /** + * Find all initialized variable declaration opportunities for the given translation unit. + * + * @param shaderJob The shader job to be searched. + * @param context Includes info such as whether we reduce everywhere or only reduce injections + * @return The opportunities that can be reduced + */ + static List findOpportunities( + ShaderJob shaderJob, + ReducerContext context) { + return shaderJob.getShaders() + .stream() + .map(item -> findOpportunitiesForShader(item, context)) + .reduce(Arrays.asList(), ListConcat::concatenate); + } + + private static List findOpportunitiesForShader( + TranslationUnit tu, + ReducerContext context) { + VariableDeclToExprReductionOpportunities finder = + new VariableDeclToExprReductionOpportunities(tu, context); + finder.visit(tu); + return finder.getOpportunities(); + } + + @Override + public void visitDeclarationStmt(DeclarationStmt declarationStmt) { + super.visitDeclarationStmt(declarationStmt); + if (!context.reduceEverywhere()) { + // Replacing initializers with a new assignment statement might have a side-effect, + // do not consider these reduction opportunities if we are not reducing everywhere. + return; + } + + if (declarationStmt.getVariablesDeclaration().getBaseType().hasQualifier(TypeQualifier.CONST)) { + // A constant must always be initialized, so do not consider variable declarations that + // have const qualifier (i.e., const int a = 1). + return; + } + final List declInfos = + declarationStmt.getVariablesDeclaration().getDeclInfos(); + // We iterate backwards so that when applying reduction the new expression is inserted at the + // correct order with respect to its original order in the variable declaration info list. + for (int i = declInfos.size() - 1; i >= 0; i--) { + final VariableDeclInfo variableDeclInfo = declInfos.get(i); + if (variableDeclInfo.hasInitializer()) { + addOpportunity(new VariableDeclToExprReductionOpportunity( + variableDeclInfo, + currentBlock(), + declarationStmt, + getVistitationDepth())); + } + } + } +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java new file mode 100644 index 000000000..2e832196a --- /dev/null +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunity.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; +import com.graphicsfuzz.common.ast.expr.BinOp; +import com.graphicsfuzz.common.ast.expr.BinaryExpr; +import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; +import com.graphicsfuzz.common.ast.stmt.BlockStmt; +import com.graphicsfuzz.common.ast.stmt.DeclarationStmt; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.visitors.VisitationDepth; + +public class VariableDeclToExprReductionOpportunity extends AbstractReductionOpportunity { + + // The initialized variable declaration info. + private final VariableDeclInfo variableDeclInfo; + // The parent of variableDeclInfo. + private final DeclarationStmt declarationStmt; + // The block in which the declaration statement resides. + private final BlockStmt enclosingBlock; + + VariableDeclToExprReductionOpportunity(VariableDeclInfo variableDeclInfo, + BlockStmt enclosingBlock, + DeclarationStmt declarationStmt, VisitationDepth depth) { + super(depth); + this.variableDeclInfo = variableDeclInfo; + this.enclosingBlock = enclosingBlock; + this.declarationStmt = declarationStmt; + } + + @Override + void applyReductionImpl() { + // Given the variable declaration info, we unset its initializer and derive a new assignment + // statement which will be inserted right after the declaration in the block statement. + assert variableDeclInfo.hasInitializer(); + final BinaryExpr binaryExpr = new BinaryExpr( + new VariableIdentifierExpr(variableDeclInfo.getName()), + (variableDeclInfo.getInitializer()).getExpr(), + BinOp.ASSIGN + ); + enclosingBlock.insertAfter(declarationStmt, new ExprStmt(binaryExpr)); + variableDeclInfo.setInitializer(null); + } + + @Override + public boolean preconditionHolds() { + return enclosingBlock.hasChild(declarationStmt) + && variableDeclInfo.hasInitializer(); + } +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunities.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunities.java index 892bc7c50..c53811e0d 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunities.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunities.java @@ -80,7 +80,7 @@ public void visitVariableDeclInfo(VariableDeclInfo variableDeclInfo) { String name = variableDeclInfo.getName(); if (MergeSet.isMergedVariable(name)) { List componentsData = MergeSet.getComponentData(name); - assert inBlock(); + assert inSomeBlock(); for (MergedVariablesComponentData data : componentsData) { assert enclosingVariablesDeclaration != null; final VectorizationReductionOpportunity potentialOpportunity diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunity.java b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunity.java index 6f6fdcb13..4626c2cce 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunity.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunity.java @@ -31,7 +31,7 @@ import com.graphicsfuzz.common.transformreduce.MergedVariablesComponentData; import com.graphicsfuzz.common.typing.Scope; import com.graphicsfuzz.common.typing.ScopeEntry; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; +import com.graphicsfuzz.common.typing.ScopeTrackingVisitor; import com.graphicsfuzz.common.util.ListConcat; import java.util.Arrays; import java.util.List; @@ -96,12 +96,12 @@ private void pullOutComponent() { // Look for cases where we are accessing the component we want to remove, and replace // those with the component variable. - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { super.visitMemberLookupExpr(memberLookupExpr); - if (isComponentAccess(memberLookupExpr, currentScope)) { + if (isComponentAccess(memberLookupExpr, getCurrentScope())) { try { parentMap.getParent(memberLookupExpr).replaceChild(memberLookupExpr, new VariableIdentifierExpr(getComponentName())); @@ -153,13 +153,13 @@ public boolean preconditionHolds() { } private boolean componentIsUsed() { - return new ScopeTreeBuilder() { + return new ScopeTrackingVisitor() { private boolean isUsed = false; @Override public void visitMemberLookupExpr(MemberLookupExpr memberLookupExpr) { super.visitMemberLookupExpr(memberLookupExpr); - isUsed |= isComponentAccess(memberLookupExpr, currentScope); + isUsed |= isComponentAccess(memberLookupExpr, getCurrentScope()); } public boolean componentIsUsed() { @@ -206,7 +206,7 @@ private boolean incompatibleComponentVariableIsDeclaredInBlock() { private boolean vectorIsUsedWithoutFieldLookup() { return - new ScopeTreeBuilder() { + new ScopeTrackingVisitor() { private boolean vectorUsedDirectly = false; @Override public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifierExpr) { @@ -217,7 +217,8 @@ public void visitVariableIdentifierExpr(VariableIdentifierExpr variableIdentifie if (!variableIdentifierExpr.getName().equals(vectorVariableDeclInfo.getName())) { return; } - final ScopeEntry se = currentScope.lookupScopeEntry(variableIdentifierExpr.getName()); + final ScopeEntry se = getCurrentScope().lookupScopeEntry(variableIdentifierExpr + .getName()); if (se != null && se.hasVariableDeclInfo() && se.getVariableDeclInfo() == vectorVariableDeclInfo) { vectorUsedDirectly = true; @@ -312,4 +313,4 @@ private boolean isComponentAccess(MemberLookupExpr memberLookupExpr, Scope curre return swizzleOffset == componentData.getOffset() && swizzleWidth == componentData.getWidth(); } -} \ No newline at end of file +} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java index 3c5a21b8d..88f71cc46 100755 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/GlslReduce.java @@ -46,6 +46,7 @@ import com.graphicsfuzz.util.Constants; import java.io.File; import java.io.IOException; +import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import net.sourceforge.argparse4j.ArgumentParsers; @@ -78,9 +79,9 @@ private static ArgumentParser getParser() { .description("glsl-reduce takes a shader job `SHADER_JOB.json` " + "(a .json file alongside shader files with the same name, such as SHADER_JOB.frag " + "and/or SHADER_JOB.vert or SHADER_JOB.comp), " - + "as well as further arguments or options to specify the interestingness test." - + "glsl-reduce will try to simplify the given shader to a smaller," - + "simpler shader that is still deemed \"interesting\"."); + + "as well as further arguments or options to specify the interestingness test. " + + "glsl-reduce will try to simplify the shaders to smaller, simpler shaders that are " + + "still deemed \"interesting\"."); // Required arguments parser.addArgument("shader-job") @@ -89,11 +90,17 @@ private static ArgumentParser getParser() { // Optional positional argument parser.addArgument("interestingness-test") - .help("Path to an executable shell script that should decide whether a shader job is " - + "interesting. Only allowed (and then also required) when performing a custom " - + "reduction, which is the default.") - .nargs("?") - .type(File.class); + .help("A command to execute (plus any fixed arguments) to determine " + + "whether a shader job is interesting. The command will typically compile and/or " + + "run the shader job and check some property. An exit status of 0 indicates that the " + + "shader job is interesting. The shader job will be passed as an argument " + + "(after any fixed arguments). Only allowed (and then also required) when performing " + + "a custom reduction, which is the default. Use -- to ensure all command line " + + "arguments that follow are parsed as positional arguments. " + + "E.g.\n" + + "glsl-reduce --preserve-semantics -- shader_job.json is-interesting --run-on-android") + .nargs("*") + .type(String.class); parser.addArgument("--reduction-kind") .help("Kind of reduction to be performed. Options are:\n" @@ -112,10 +119,10 @@ private static ArgumentParser getParser() { + " Reduces while histogram difference between produced image and " + "reference is above a threshold.\n" + " " + ReductionKind.VALIDATOR_ERROR - + " Reduces while validator gives a particular error\n" + + " Reduces while validator gives a particular error.\n" + " " + ReductionKind.ALWAYS_REDUCE - + " Always reduces (useful for testing)\n") - .setDefault("CUSTOM") + + " Always reduces (useful for testing).\n") + .setDefault(ReductionKind.CUSTOM.toString()) .type(String.class); parser.addArgument("--output") @@ -139,9 +146,9 @@ private static ArgumentParser getParser() { .action(Arguments.storeTrue()); parser.addArgument("--seed") - .help("Seed with which to initialize the random number generator that is used to control " - + "reduction decisions.") - .type(Integer.class); + .help("Seed (unsigned 64 bit long integer) with which to initialize the random number " + + "generator that is used to control reduction decisions.") + .type(String.class); parser.addArgument("--timeout") .help( @@ -275,12 +282,12 @@ public static void mainHelper( final double threshold = ns.get("threshold"); // TODO: integrate timeout into reducer - @SuppressWarnings("UnusedAssignment") Integer timeout = ns.get("timeout"); + @SuppressWarnings("unused") Integer timeout = ns.get("timeout"); final Integer maxSteps = ns.get("max_steps"); final Integer retryLimit = ns.get("retry_limit"); final Boolean verbose = ns.get("verbose"); final boolean skipRender = ns.get("skip_render"); - final int seed = ArgsUtil.getSeedArgument(ns); + final IRandom random = new RandomWrapper(ArgsUtil.getSeedArgument(ns)); final String errorString = ns.get("error_string"); final boolean reduceEverywhere = !ns.getBoolean("preserve_semantics"); final boolean stopOnError = ns.get("stop_on_error"); @@ -310,7 +317,7 @@ public static void mainHelper( final File referenceResultFile = ns.get("reference"); - final File customJudgeScript = ns.get("interestingness_test"); + final List customJudgeScript = ns.get("interestingness_test"); if (reductionKind == ReductionKind.CUSTOM) { if (server != null) { @@ -325,15 +332,21 @@ public static void mainHelper( if (referenceResultFile != null) { throwExceptionForCustomReduction("reference"); } - if (customJudgeScript == null) { + if (customJudgeScript.isEmpty()) { throw new RuntimeException("A custom reduction requires an interestingness test to be " + "specified."); } - if (!customJudgeScript.canExecute()) { + + // Sanity check that the custom judge script is executable. However, don't fail if the + // script can't be found, as the script/tool could be on the PATH. + File scriptFile = new File(customJudgeScript.get(0)); + if (scriptFile.exists() && !scriptFile.canExecute()) { throw new RuntimeException("Custom judge script must be executable."); } } else { - if (customJudgeScript != null) { + // Not a custom reduction. + + if (!customJudgeScript.isEmpty()) { throw new RuntimeException("An interestingness test is only supported when a custom " + "reduction is used."); } @@ -380,7 +393,7 @@ public static void mainHelper( switch (reductionKind) { case CUSTOM: fileJudge = - new CustomFileJudge(customJudgeScript, workDir); + new CustomFileJudge(customJudgeScript); break; case NO_IMAGE: fileJudge = @@ -440,7 +453,7 @@ public static void mainHelper( doReductionHelper( inputShaderJobFile, shaderJobShortName, - seed, + random, fileJudge, workDir, maxSteps, @@ -466,7 +479,7 @@ public static void mainHelper( public static void doReductionHelper( File initialShaderJobFile, String outputShortName, - int seed, + IRandom random, IFileJudge fileJudge, File workDir, int stepLimit, @@ -477,7 +490,6 @@ public static void doReductionHelper( throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { final ShadingLanguageVersion shadingLanguageVersion = getGlslVersionForShaderJob(initialShaderJobFile, fileOps); - final IRandom random = new RandomWrapper(seed); final IdGenerator idGenerator = new IdGenerator(); final int fileCountOffset = getFileCountOffset( @@ -502,16 +514,12 @@ public static void doReductionHelper( shaderJobFile ); - final boolean emitGraphicsFuzzDefines = - fileOps.doesShaderJobUseGraphicsFuzzDefines(shaderJobFile); - new ReductionDriver( new ReducerContext( reduceEverywhere, shadingLanguageVersion, random, - idGenerator, - emitGraphicsFuzzDefines), + idGenerator), verbose, fileOps, fileJudge, diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java deleted file mode 100755 index e2a6c5540..000000000 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPoint.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer.tool; - -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.util.GlslParserException; -import com.graphicsfuzz.common.util.IRandom; -import com.graphicsfuzz.common.util.ParseTimeoutException; -import com.graphicsfuzz.common.util.RandomWrapper; -import com.graphicsfuzz.common.util.ShaderJobFileOperations; -import com.graphicsfuzz.common.util.ShaderKind; -import com.graphicsfuzz.reducer.ReductionDriver; -import com.graphicsfuzz.reducer.reductionopportunities.ReducerContext; -import com.graphicsfuzz.util.ArgsUtil; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Comparator; -import net.sourceforge.argparse4j.ArgumentParsers; -import net.sourceforge.argparse4j.impl.Arguments; -import net.sourceforge.argparse4j.inf.ArgumentParser; -import net.sourceforge.argparse4j.inf.ArgumentParserException; -import net.sourceforge.argparse4j.inf.Namespace; -import org.apache.commons.io.FilenameUtils; - -public class ReducerBugPoint { - - private static final int STEP_LIMIT = 50; - - private static Namespace parse(String[] args) throws ArgumentParserException { - ArgumentParser parser = ArgumentParsers.newArgumentParser("ReducerBugPoint") - .defaultHelp(true) - .description("Find bugs in the reducer."); - - // Required arguments - parser.addArgument("shader") - .help("Path of shader job file (.json) to be analysed.") - .type(File.class); - - parser.addArgument("--output") - .help("Output directory.") - .setDefault(new File(".")) - .type(File.class); - - // Optional arguments - parser.addArgument("--max-iterations") - .help("Maximum number of times to iterate before giving up.") - .setDefault(30) - .type(Integer.class); - - parser.addArgument("--seed") - .help("Seed to initialize random number generator with.") - .type(Integer.class); - - parser.addArgument("--preserve-semantics") - .help("Only perform semantics-preserving reductions.") - .action(Arguments.storeTrue()); - - parser.addArgument("--verbose") - .help("Emit verbose info.") - .action(Arguments.storeTrue()); - - parser.addArgument("--exception-on-invalid") - .help("Throw exception when shader is invalid.") - .action(Arguments.storeTrue()); - - parser.addArgument("--expected-string") - .help("A string to look for in the exception message.") - .type(String.class); - - return parser.parseArgs(args); - } - - - public static void main(String[] args) - throws IOException, ParseTimeoutException, ArgumentParserException, InterruptedException, - GlslParserException { - - final Namespace ns = parse(args); - - final int seed = ArgsUtil.getSeedArgument(ns); - - final int maxIterations = ns.get("max_iterations"); - - final boolean reduceEverywhere = !ns.getBoolean("preserve_semantics"); - - final boolean verbose = ns.get("verbose"); - - final String expectedString = ns.getString("expected_string") == null - ? "" - : ns.getString("expected_string"); - - final IRandom generator = new RandomWrapper(seed); - - ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - - final File interestingShaderJobFile = new File("interesting.frag"); - - if (fileOps.doesShaderJobExist(interestingShaderJobFile)) { - fileOps.deleteShaderJobFile(interestingShaderJobFile); - } - - final File originalShaderJobFile = ns.get("shader"); - - fileOps.copyShaderJobFileTo(originalShaderJobFile, interestingShaderJobFile, false); - - final ShadingLanguageVersion shadingLanguageVersion = - ShadingLanguageVersion.getGlslVersionFromFirstTwoLines( - fileOps.getFirstTwoLinesOfShader(interestingShaderJobFile, ShaderKind.FRAGMENT)); - - ShaderJob initialState = fileOps.readShaderJobFile(interestingShaderJobFile); - - for (int i = 0; i < maxIterations; i++) { - - File workDir = Files.createDirectory(Paths.get("temp")).toFile(); - - System.err.println("Trying iteration " + i); - - try { - - new ReductionDriver( - new ReducerContext( - reduceEverywhere, - shadingLanguageVersion, - generator, - null, - 10, - 1, true), - verbose, - fileOps, - new RandomFileJudge( - generator, - 10, - ns.getBoolean("exception_on_invalid"), fileOps), - workDir - ).doReduction( - initialState, - FilenameUtils.removeExtension(interestingShaderJobFile.getAbsolutePath()), - 0, - STEP_LIMIT); - - } catch (Throwable throwable) { - if (!throwable.toString().contains(expectedString)) { - System.err.println("Exception does not contain required string:"); - System.err.println(throwable); - } else { - File[] files = - fileOps.listShaderJobFiles(workDir, (dir, name) -> name.endsWith("success.info")); - if (files.length == 0) { - continue; - } - final File maxSuccess = - Arrays.stream(files) - .max(Comparator.comparingInt(ReducerBugPoint::getStep)).get(); - - fileOps.deleteShaderJobFile(interestingShaderJobFile); - fileOps.copyShaderJobFileTo(maxSuccess, interestingShaderJobFile, true); - - initialState = fileOps.readShaderJobFile(interestingShaderJobFile); - - i = 0; - } - } finally { - fileOps.deleteDirectory(workDir); - } - } - } - - private static int getStep(File file) { - final String[] split = file.getName().split("_"); - return Integer.parseInt(split[split.length - 2]); - } - -} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java deleted file mode 100755 index 54a0d0889..000000000 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReducerBugPointBasic.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer.tool; - -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.util.GlslParserException; -import com.graphicsfuzz.common.util.IRandom; -import com.graphicsfuzz.common.util.IdGenerator; -import com.graphicsfuzz.common.util.ParseTimeoutException; -import com.graphicsfuzz.common.util.RandomWrapper; -import com.graphicsfuzz.common.util.ShaderJobFileOperations; -import com.graphicsfuzz.common.util.ShaderKind; -import com.graphicsfuzz.reducer.reductionopportunities.Compatibility; -import com.graphicsfuzz.reducer.reductionopportunities.IReductionOpportunity; -import com.graphicsfuzz.reducer.reductionopportunities.ReducerContext; -import com.graphicsfuzz.reducer.reductionopportunities.ReductionOpportunities; -import com.graphicsfuzz.util.ArgsUtil; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import net.sourceforge.argparse4j.ArgumentParsers; -import net.sourceforge.argparse4j.impl.Arguments; -import net.sourceforge.argparse4j.inf.ArgumentParser; -import net.sourceforge.argparse4j.inf.ArgumentParserException; -import net.sourceforge.argparse4j.inf.Namespace; -import org.apache.commons.lang3.exception.ExceptionUtils; - -public class ReducerBugPointBasic { - - private static Namespace parse(String[] args) throws ArgumentParserException { - ArgumentParser parser = ArgumentParsers.newArgumentParser("ReducerBugPoint") - .defaultHelp(true) - .description("Find bugs in the reducer."); - - // Required arguments - parser.addArgument("shader") - .help("Path of shader job file (.json) to be analysed.") - .type(File.class); - - // Optional arguments - parser.addArgument("--max-iterations") - .help("Maximum number of times to iterate before giving up.") - .setDefault(30) - .type(Integer.class); - - parser.addArgument("--seed") - .help("Seed to initialize random number generator with.") - .type(Integer.class); - - parser.addArgument("--preserve-semantics") - .help("Only perform semantics-preserving reductions.") - .action(Arguments.storeTrue()); - - parser.addArgument("--ignore-invalid") - .help("Do not log or fix on cases where the shader is invalid - " - + "ignore them and backtrack.") - .action(Arguments.storeTrue()); - - parser.addArgument("--expected-string") - .help("A string to look for in the exception message.") - .type(String.class); - - return parser.parseArgs(args); - } - - - public static void main(String[] args) - throws IOException, ParseTimeoutException, InterruptedException, ArgumentParserException, - GlslParserException { - - final Namespace ns = parse(args); - - ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - - final File shaderJobFile = ns.get("shader"); - - final ShadingLanguageVersion shadingLanguageVersion = - ShadingLanguageVersion - .getGlslVersionFromFirstTwoLines( - fileOps.getFirstTwoLinesOfShader(shaderJobFile, ShaderKind.FRAGMENT)); - - final int seed = ArgsUtil.getSeedArgument(ns); - - final IRandom generator = new RandomWrapper(seed); - - final ShaderJob originalShaderJob = fileOps.readShaderJobFile(shaderJobFile); - - final int maxIterations = ns.get("max_iterations"); - - final boolean reduceEverywhere = !ns.getBoolean("reduce_everywhere"); - - final boolean ignoreInvalid = ns.get("ignore_invalid"); - - final String expectedString = ns.getString("expected_string") == null - ? "" - : ns.getString("expected_string"); - - final IdGenerator idGenerator = new IdGenerator(); - - ShaderJob lastGoodButLeadingToBadShaderJob = originalShaderJob; - - int exceptionCount = 0; - int invalidCount = 0; - - int getOpsCounter = 0; - - for (int i = 0; i < maxIterations; i++) { - - ShaderJob current = lastGoodButLeadingToBadShaderJob.clone(); - - // TODO: this code was written pre vertex shader support, and does not take - // account of uniforms. - // If it still proves useful as a debugging tool, it could do with updating. - // Paul: partially updated during refactor, but not tested and still makes assumptions - // about frag files. - - while (true) { - final ShaderJob prev = current.clone(); - List ops; - try { - ops = ReductionOpportunities.getReductionOpportunities( - current, - new ReducerContext( - reduceEverywhere, - shadingLanguageVersion, - generator, - idGenerator, true), - fileOps); - - } catch (Exception exception) { - recordThrowsExceptionWhenGettingReductionOpportunities( - current, - exception, - fileOps); - break; - } - System.out.println(ops.size()); - if (ops.isEmpty()) { - break; - } - final List opsToApply = getOpsToApply(ops, generator); - //System.out.println("About to try applying " + op); - try { - for (IReductionOpportunity op : opsToApply) { - op.applyReduction(); - } - } catch (Exception exception) { - System.err.println("Exception occurred while applying a reduction opportunity."); - if (exception.toString().contains(expectedString)) { - lastGoodButLeadingToBadShaderJob = prev; - current = lastGoodButLeadingToBadShaderJob.clone(); - - fileOps.writeShaderJobFile( - lastGoodButLeadingToBadShaderJob, - new File("leads_to_exception_" + exceptionCount + ".json") - ); - - emitException(exception, "leads_to_exception_" + exceptionCount + ".txt", fileOps); - exceptionCount++; - } else { - System.err.println("The exception was not interesting as it did not contain \"" - + expectedString + "\""); - System.out.println(exception.toString()); - } - continue; - } - if (invalid(current, shadingLanguageVersion, fileOps)) { - System.err.println("Invalid shader after reduction step."); - if (ignoreInvalid) { - System.err.println("Ignoring it and backtracking."); - current = prev; - } else { - fileOps.writeShaderJobFile( - prev, - new File("leads_to_invalid_" + invalidCount + "_before.json")); - fileOps.writeShaderJobFile( - current, - new File("leads_to_invalid_" + invalidCount + "_after.json")); - invalidCount++; - lastGoodButLeadingToBadShaderJob = prev; - current = lastGoodButLeadingToBadShaderJob.clone(); - } - } - } - } - } - - private static List getOpsToApply(List ops, - IRandom generator) { - List result = new ArrayList<>(); - final IReductionOpportunity initialOp = ops.remove(generator.nextInt(ops.size())); - result.add(initialOp); - for (int i = 0; i < 10; i++) { - if (generator.nextInt(10) < 10 - i) { - break; - } - while (!ops.isEmpty()) { - final IReductionOpportunity op = ops.remove(generator.nextInt(ops.size())); - if (!Compatibility.compatible(initialOp.getClass(), op.getClass())) { - continue; - } - result.add(op); - break; - } - if (ops.isEmpty()) { - break; - } - } - return result; - } - - private static boolean invalid( - ShaderJob shaderJob, - ShadingLanguageVersion shadingLanguageVersion, - ShaderJobFileOperations fileOps) - throws IOException, InterruptedException { - File tempShaderJobFile = new File("temp_to_validate.json"); - fileOps.writeShaderJobFile( - shaderJob, - tempShaderJobFile); - return fileOps.areShadersValid(tempShaderJobFile, false); - } - - private static void emitException( - Exception exception, - String filename, - ShaderJobFileOperations fileOps) throws IOException { - String stacktrace = ExceptionUtils.getStackTrace(exception); - fileOps.writeStringToFile(new File(filename), stacktrace); - } - - private static void recordThrowsExceptionWhenGettingReductionOpportunities( - ShaderJob shaderJob, - Exception exception, - ShaderJobFileOperations fileOps) throws IOException, ParseTimeoutException, - InterruptedException, GlslParserException { - File tempShaderJobFile = new File("temp.json"); - - fileOps.writeShaderJobFile( - shaderJob, - tempShaderJobFile - ); - - ShaderJob reparsedShaderJob = fileOps.readShaderJobFile( - tempShaderJobFile - ); - - new ReportAstDifferences( - shaderJob.getShaders().get(0), - reparsedShaderJob.getShaders().get(0)); - - final File maybeExistingFile = new File("problem_getting_reduction_opportunities.json"); - - if (!fileOps.doesShaderJobExist(maybeExistingFile) - || (fileOps.getShaderLength(maybeExistingFile, ShaderKind.FRAGMENT) - > fileOps.getShaderLength(tempShaderJobFile, ShaderKind.FRAGMENT))) { - - if (fileOps.doesShaderJobExist(maybeExistingFile)) { - fileOps.deleteShaderJobFile(maybeExistingFile); - } - fileOps.moveShaderJobFileTo(tempShaderJobFile, maybeExistingFile, true); - final File maybeExistingExceptionFile = - new File("problem_getting_reduction_opportunities.txt"); - if (fileOps.isFile(maybeExistingExceptionFile)) { - fileOps.deleteFile(maybeExistingExceptionFile); - } - String stacktrace = ExceptionUtils.getStackTrace(exception); - fileOps.writeStringToFile(maybeExistingExceptionFile, stacktrace); - } - } - - -} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReportAstDifferences.java b/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReportAstDifferences.java deleted file mode 100644 index a13cbe591..000000000 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/tool/ReportAstDifferences.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer.tool; - -import com.graphicsfuzz.common.ast.IAstNode; -import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.ast.decl.FunctionDefinition; -import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; -import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; -import com.graphicsfuzz.common.ast.visitors.StandardVisitor; -import com.graphicsfuzz.common.typing.ScopeTreeBuilder; -import java.util.ArrayList; -import java.util.List; - -public class ReportAstDifferences { - - private class TraversalVisitor extends StandardVisitor { - - private List nodes = new ArrayList<>(); - - TraversalVisitor(TranslationUnit tu) { - visit(tu); - } - - @Override - public void visit(IAstNode node) { - nodes.add(node); - super.visit(node); - } - - List getNodes() { - return nodes; - } - - } - - private class InformativeScopeTreeBuilder extends ScopeTreeBuilder { - - InformativeScopeTreeBuilder(TranslationUnit tu) { - visit(tu); - } - - @Override - protected void pushScope() { - System.out.println("Entering a scope"); - super.pushScope(); - } - - @Override - protected void popScope() { - super.popScope(); - System.out.println("Leaving a scope"); - } - - @Override - public void visitFunctionDefinition(FunctionDefinition functionDefinition) { - System.out.println("Entering function " + functionDefinition.getPrototype().getName()); - super.visitFunctionDefinition(functionDefinition); - System.out.println("Leaving function " + functionDefinition.getPrototype().getName()); - } - - @Override - public void visitVariablesDeclaration(VariablesDeclaration variablesDeclaration) { - System.out.println("Entering variables declaration"); - for (VariableDeclInfo vdi : variablesDeclaration.getDeclInfos()) { - System.out.println(" " + vdi.getName()); - } - super.visitVariablesDeclaration(variablesDeclaration); - System.out.println("Leaving variables declaration"); - } - } - - public ReportAstDifferences(TranslationUnit tu1, TranslationUnit tu2) { - try { - new InformativeScopeTreeBuilder(tu1); - } catch (Exception exception) { - System.out.println("Exception was thrown:"); - exception.printStackTrace(System.out); - } - try { - new InformativeScopeTreeBuilder(tu2); - } catch (Exception exception) { - System.out.println("Exception was thrown:"); - exception.printStackTrace(System.out); - } - - /*List nodes1 = new TraversalVisitor(tu1).getNodes(); - List nodes2 = new TraversalVisitor(tu2).getNodes(); - Map oneToTwo = new HashMap<>(); - Map twoToOne = new HashMap<>(); - IParentMap parentMap1 = IParentMap.createParentMap(tu1); - IParentMap parentMap2 = IParentMap.createParentMap(tu2); - for (int i = 0; i < Math.min(nodes1.size(), nodes2.size()); i++) { - IAstNode one = nodes1.get(i); - IAstNode two = nodes1.get(i); - if (oneToTwo.containsKey(one) || twoToOne.containsKey(two)) { - if (oneToTwo.get(one) != two || twoToOne.get(two) != one) { - throw new RuntimeException(); - } - } else { - oneToTwo.put(one, two); - twoToOne.put(two, one); - } - } - if (nodes1.size() != nodes2.size()) { - throw new RuntimeException(); - }*/ - } - -} diff --git a/reducer/src/main/java/com/graphicsfuzz/reducer/util/Simplify.java b/reducer/src/main/java/com/graphicsfuzz/reducer/util/Simplify.java index 4cf7f3b3c..598d33435 100644 --- a/reducer/src/main/java/com/graphicsfuzz/reducer/util/Simplify.java +++ b/reducer/src/main/java/com/graphicsfuzz/reducer/util/Simplify.java @@ -18,16 +18,12 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.util.AddBraces; -import com.graphicsfuzz.reducer.glslreducers.EliminateInjectionMacrosVisitor; -import java.util.Optional; +import com.graphicsfuzz.reducer.glslreducers.EliminateGraphicsFuzzDefines; public class Simplify { public static TranslationUnit simplify(TranslationUnit tu) { - TranslationUnit temp = tu.clone(); - new EliminateInjectionMacrosVisitor().visit(temp); - temp = AddBraces.transform(temp); - return temp; + return AddBraces.transform(EliminateGraphicsFuzzDefines.transform(tu)); } } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java index 5a7b8b213..398906236 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/ReductionDriverTest.java @@ -23,7 +23,12 @@ import static org.junit.Assert.assertTrue; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.ast.expr.BinOp; +import com.graphicsfuzz.common.ast.expr.BinaryExpr; import com.graphicsfuzz.common.ast.expr.FunctionCallExpr; +import com.graphicsfuzz.common.ast.expr.IntConstantExpr; +import com.graphicsfuzz.common.ast.stmt.ExprStmt; +import com.graphicsfuzz.common.ast.stmt.ReturnStmt; import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; @@ -111,17 +116,16 @@ public boolean isInteresting( ReductionOpportunities. getReductionOpportunities( MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, version, generator, new IdGenerator(), true), + new ReducerContext(false, version, generator, new IdGenerator()), fileOps); - assertEquals(2, ops.size()); + assertEquals(3, ops.size()); new ReductionDriver( new ReducerContext( false, version, generator, - null, - true), + new IdGenerator()), false, fileOps, pessimist, testFolder.getRoot()) .doReduction(state, getPrefix(tempFile), 0, -1); @@ -231,7 +235,7 @@ private String reduce(IFileJudge judge, translationUnits); return new ReductionDriver(new ReducerContext(reduceEverywhere, version, - generator, new IdGenerator(), true), false, fileOps, + generator, new IdGenerator()), false, fileOps, judge, testFolder.getRoot()) .doReduction(state, getPrefix(tempFragmentShaderFile), 0, stepLimit); } @@ -278,8 +282,8 @@ public void testInitializersAreInlined() throws Exception { }; final String reducedFilesPrefix = new ReductionDriver( - new ReducerContext(false, version, generator, null, - true), false, fileOps, + new ReducerContext(false, version, generator, new IdGenerator()), + false, fileOps, referencesSinCosAnd3, testFolder.getRoot()) .doReduction(state, getPrefix(tempFile), 0,-1); @@ -553,7 +557,7 @@ public void testReductionWithUniformBindings() throws Exception { final String resultsPrefix = new ReductionDriver(new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), - new IdGenerator(), true), + new IdGenerator()), false, fileOps, (unused, item) -> true, workDir) @@ -563,8 +567,6 @@ public void testReductionWithUniformBindings() throws Exception { } - // TODO(478): Enable this test once the issue is fixed. - @Ignore @Test public void testReductionOfUnreferencedUniform() throws Exception { @@ -587,7 +589,7 @@ public void testReductionOfUnreferencedUniform() throws Exception { final String resultsPrefix = new ReductionDriver(new ReducerContext(true, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), - new IdGenerator(), true), + new IdGenerator()), false, fileOps, (unused, item) -> true, workDir) @@ -602,6 +604,257 @@ public void testReductionOfUnreferencedUniform() throws Exception { } + @Test + public void testSimplificationOfSwitch() throws Exception { + final String shader = "#version 310 es\n" + + "void main() {\n" + + " switch(0) {\n" + + " case 0:\n" + + " mix(0.0, 1.0, 0.0);\n" + + " break;\n" + + " default:\n" + + " 1;\n" + + " }\n" + + "}\n"; + + final String expected = "#version 310 es\n" + + "void main() {\n" + + " mix(0.0, 1.0, 0.0);\n" + + "}\n"; + + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + new PipelineInfo(), + ParseHelper.parse(shader)); + + final File workDir = testFolder.getRoot(); + final File tempShaderJobFile = new File(workDir, "temp.json"); + fileOps.writeShaderJobFile(shaderJob, tempShaderJobFile); + + final IFileJudge usesMixFileJudge = + new CheckAstFeaturesFileJudge(Collections.singletonList(() -> new CheckAstFeatureVisitor() { + @Override + public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { + super.visitFunctionCallExpr(functionCallExpr); + if (functionCallExpr.getCallee().equals("mix")) { + trigger(); + } + } + }), + ShaderKind.FRAGMENT, + fileOps); + + final String resultsPrefix = new ReductionDriver(new ReducerContext(true, + ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), + new IdGenerator()), + false, + fileOps, + usesMixFileJudge, + workDir) + .doReduction(shaderJob, "temp", 0, 100); + + CompareAsts.assertEqualAsts(expected, ParseHelper.parse(new File(testFolder.getRoot(), resultsPrefix + ".frag"))); + + } + + @Test + public void testEliminationOfReturns() throws Exception { + final String shader = "#version 310 es\n" + + "int foo() {\n" + + " if (false) {\n" + + " return 1;\n" + + " }\n" + + " return 0;\n" + + "}\n" + + "void main() {\n" + + " if (true) {\n" + + " foo();\n" + + " return;\n" + + " } else {\n" + + " return;\n" + + " }\n" + + " return;\n" + + "}\n"; + + final String expected = "#version 310 es\n" + + "int foo() {\n" + + " return 1;\n" + + "}\n" + + "void main() {\n" + + " foo();\n" + + "}\n"; + + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + new PipelineInfo(), + ParseHelper.parse(shader)); + + final File workDir = testFolder.getRoot(); + final File tempShaderJobFile = new File(workDir, "temp.json"); + fileOps.writeShaderJobFile(shaderJob, tempShaderJobFile); + + final IFileJudge usesFooFileJudge = + new CheckAstFeaturesFileJudge(Arrays.asList( + () -> new CheckAstFeatureVisitor() { + @Override + public void visitFunctionCallExpr(FunctionCallExpr functionCallExpr) { + super.visitFunctionCallExpr(functionCallExpr); + if (functionCallExpr.getCallee().equals("foo")) { + trigger(); + } + } + }, + () -> new CheckAstFeatureVisitor() { + @Override + public void visitReturnStmt(ReturnStmt returnStmt) { + super.visitReturnStmt(returnStmt); + if (returnStmt.hasExpr() && returnStmt.getExpr() instanceof IntConstantExpr && + ((IntConstantExpr) returnStmt.getExpr()).getNumericValue() == 1) { + trigger(); + } + } + }), + ShaderKind.FRAGMENT, + fileOps); + + final String resultsPrefix = new ReductionDriver(new ReducerContext(true, + ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), + new IdGenerator()), + false, + fileOps, + usesFooFileJudge, + workDir) + .doReduction(shaderJob, "temp", 0, 100); + + CompareAsts.assertEqualAsts(expected, ParseHelper.parse(new File(testFolder.getRoot(), resultsPrefix + ".frag"))); + + } + + @Test + public void testRemoveIfOrLoopButKeepGuard() throws Exception { + final String shaderIf = "#version 310 es\n" + + "void main() {\n" + + " if(1 > 0) {\n" + + " }\n" + + "}\n"; + + final String shaderWhile = "#version 310 es\n" + + "void main() {\n" + + " while(1 > 0) {\n" + + " }\n" + + "}\n"; + + final String shaderDoWhile = "#version 310 es\n" + + "void main() {\n" + + " do {\n" + + " } while(1 > 0);\n" + + "}\n"; + + final String shaderFor = "#version 310 es\n" + + "void main() {\n" + + " for (; 1 > 0; ) {\n" + + " }\n" + + "}\n"; + + final String expected = "#version 310 es\n" + + "void main() {\n" + + " 1 > 0;\n" + + "}\n"; + + final IFileJudge customFileJudge = + new CheckAstFeaturesFileJudge(Collections.singletonList( + () -> new CheckAstFeatureVisitor() { + @Override + public void visitBinaryExpr(BinaryExpr binaryExpr) { + super.visitBinaryExpr(binaryExpr); + if (binaryExpr.getOp() == BinOp.GT + && binaryExpr.getLhs() instanceof IntConstantExpr + && ((IntConstantExpr) binaryExpr.getLhs()).getNumericValue() == 1 + && binaryExpr.getRhs() instanceof IntConstantExpr + && ((IntConstantExpr) binaryExpr.getRhs()).getNumericValue() == 0) { + trigger(); + } + } + }), + ShaderKind.FRAGMENT, + fileOps); + + for (String shader : Arrays.asList(shaderIf, shaderWhile, shaderDoWhile, shaderFor)) { + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + new PipelineInfo(), + ParseHelper.parse(shader)); + + final File workDir = testFolder.getRoot(); + final File tempShaderJobFile = new File(workDir, "temp.json"); + fileOps.writeShaderJobFile(shaderJob, tempShaderJobFile); + + final String resultsPrefix = new ReductionDriver(new ReducerContext(true, + ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), + new IdGenerator()), + false, + fileOps, + customFileJudge, + workDir) + .doReduction(shaderJob, "temp", 0, 100); + + CompareAsts.assertEqualAsts(expected, ParseHelper.parse(new File(testFolder.getRoot(), resultsPrefix + ".frag"))); + } + + } + + @Test + public void testRemoveSwitchButKeepGuard() throws Exception { + + final String shaderSwitch = "#version 310 es\n" + + "void main() {\n" + + " switch(0) {\n" + + " case 1:\n" + + " break;\n" + + " }\n" + + "}\n"; + + final String expected = "#version 310 es\n" + + "void main() {\n" + + " 0;\n" + + "}\n"; + + final IFileJudge customFileJudge = + new CheckAstFeaturesFileJudge(Collections.singletonList( + () -> new CheckAstFeatureVisitor() { + @Override + public void visitIntConstantExpr(IntConstantExpr intConstantExpr) { + super.visitIntConstantExpr(intConstantExpr); + if (intConstantExpr.getNumericValue() == 0) { + trigger(); + } + } + }), + ShaderKind.FRAGMENT, + fileOps); + + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + new PipelineInfo(), + ParseHelper.parse(shaderSwitch)); + + final File workDir = testFolder.getRoot(); + final File tempShaderJobFile = new File(workDir, "temp.json"); + fileOps.writeShaderJobFile(shaderJob, tempShaderJobFile); + + final String resultsPrefix = new ReductionDriver(new ReducerContext(true, + ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), + new IdGenerator()), + false, + fileOps, + customFileJudge, + workDir) + .doReduction(shaderJob, "temp", 0, 100); + + CompareAsts.assertEqualAsts(expected, ParseHelper.parse(new File(testFolder.getRoot(), resultsPrefix + ".frag"))); + + } + private String getPrefix(File tempFile) { return FilenameUtils.removeExtension(tempFile.getName()); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/TestUtils.java b/reducer/src/test/java/com/graphicsfuzz/reducer/TestUtils.java deleted file mode 100755 index f6b1a3fe2..000000000 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/TestUtils.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer; - -import static org.junit.Assert.assertEquals; - -import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; -import com.graphicsfuzz.util.ExecHelper.RedirectType; -import com.graphicsfuzz.util.ExecResult; -import com.graphicsfuzz.util.ToolHelper; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.Optional; -import org.junit.rules.TemporaryFolder; - -public class TestUtils { - - public static void checkValid(TemporaryFolder testFolder, TranslationUnit tu) throws IOException, InterruptedException { - File tempFile = testFolder.newFile("temp.frag"); - new PrettyPrinterVisitor(System.out).visit(tu); - PrintStream ps = new PrintStream(new FileOutputStream(tempFile)); - PrettyPrinterVisitor ppv = new PrettyPrinterVisitor(ps, - PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, true, Optional.empty()); - ppv.visit(tu); - ps.close(); - ExecResult result = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, tempFile); - assertEquals(result.stderr.toString() + result.stdout.toString(), 0, result.res); - } - -} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundExprToSubExprReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundExprToSubExprReductionOpportunitiesTest.java index ce5d2dcdf..aa77c1b18 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundExprToSubExprReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundExprToSubExprReductionOpportunitiesTest.java @@ -20,6 +20,7 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; import com.graphicsfuzz.common.util.RandomWrapper; @@ -40,7 +41,7 @@ public void noEffectInRegularCode() throws Exception { final TranslationUnit tu = ParseHelper.parse(original); final List ops = CompoundExprToSubExprReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertTrue(ops.isEmpty()); } @@ -168,7 +169,24 @@ private List getOps(TranslationUnit tu, boolean reduceEverywhere) { return CompoundExprToSubExprReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(reduceEverywhere, ShadingLanguageVersion.GLSL_440, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); + } + + @Test + public void testSwitch() throws Exception { + final String program = "#version 310 es\n" + + "void main() {\n" + + " switch(0) {\n" + + " case - 1:\n" + + " 1;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + CompoundExprToSubExprReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(true, ShadingLanguageVersion.ESSL_310, new RandomWrapper(0), + new IdGenerator())); + assertEquals(0, ops.size()); } } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java deleted file mode 100755 index 7c7337a1e..000000000 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/CompoundToBlockReductionOpportunitiesTest.java +++ /dev/null @@ -1,514 +0,0 @@ -/* - * Copyright 2018 The GraphicsFuzz Project Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.graphicsfuzz.reducer.reductionopportunities; - -import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; -import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; -import com.graphicsfuzz.common.util.GlslParserException; -import com.graphicsfuzz.util.Constants; -import com.graphicsfuzz.common.util.IdGenerator; -import com.graphicsfuzz.common.util.ParseHelper; -import com.graphicsfuzz.common.util.ParseTimeoutException; -import com.graphicsfuzz.common.util.RandomWrapper; -import com.graphicsfuzz.common.util.ShaderJobFileOperations; -import java.io.IOException; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import org.junit.Ignore; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class CompoundToBlockReductionOpportunitiesTest { - - private final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); - - @Test - public void testUnderUnreachableSwitch() throws Exception { - final String original = "void main() {" - + " switch(" + Constants.GLF_SWITCH + "(0)) {" - + " case 1:" - + " if (true) {" - + " true;" - + " }" - + " case 2:" - + " if (true) {" - + " true;" - + " } else false;" - + " case 0:" - + " for (int i = 0; i < 100; i++) { }" - + " break;" - + " default:" - + " while(true) { 1; }" - + " }" - + "}"; - final String expected1 = "void main() {" - + " switch(" + Constants.GLF_SWITCH + "(0)) {" - + " case 1:" - + " { true; }" - + " case 2:" - + " if (true) {" - + " true;" - + " } else false;" - + " case 0:" - + " for (int i = 0; i < 100; i++) { }" - + " break;" - + " default:" - + " while(true) { 1; }" - + " }" - + "}"; - final String expected2 = "void main() {" - + " switch(" + Constants.GLF_SWITCH + "(0)) {" - + " case 1:" - + " if (true) {" - + " true;" - + " }" - + " case 2:" - + " { true; }" - + " case 0:" - + " for (int i = 0; i < 100; i++) { }" - + " break;" - + " default:" - + " while(true) { 1; }" - + " }" - + "}"; - final String expected3 = "void main() {" - + " switch(" + Constants.GLF_SWITCH + "(0)) {" - + " case 1:" - + " if (true) {" - + " true;" - + " }" - + " case 2:" - + " false;" - + " case 0:" - + " for (int i = 0; i < 100; i++) { }" - + " break;" - + " default:" - + " while(true) { 1; }" - + " }" - + "}"; - final String expected4 = "void main() {" - + " switch(" + Constants.GLF_SWITCH + "(0)) {" - + " case 1:" - + " if (true) {" - + " true;" - + " }" - + " case 2:" - + " if (true) {" - + " true;" - + " } else false;" - + " case 0:" - + " for (int i = 0; i < 100; i++) { }" - + " break;" - + " default:" - + " { 1; }" - + " }" - + "}"; - check(false, original, expected1, expected2, expected3, expected4); - } - - @Test - public void testDoNotRemoveDeadIf() throws Exception { - final String original = "" - + "void main() {" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + "}"; - final TranslationUnit tu = ParseHelper.parse(original); - assertTrue(CompoundToBlockReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new RandomWrapper(0), null, true)).isEmpty()); - } - - @Test - public void testInDeadCode() throws Exception { - final String original = "" - + "void main() {" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " if (false) {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + "}"; - final String expected = "" - + "void main() {" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + "}"; - check(false, original, expected); - } - - // TODO(482): Enable this test once the issue is fixed. - @Ignore - @Test - public void testDeadIfReduceEverywhere() throws Exception { - final String original = "" - + "void main() {" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + "}"; - final String expected1 = "" - + "void main() {" - + " {" // first if changed to block - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + "}"; - final String expected2 = "" - + "void main() {" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " {" // second if changed to block - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + "}"; - check(true, original, expected1, expected2); - } - - @Test - public void testInDeadCode2() throws Exception { - final String original = "" - + "void main() {" - + " int a = 4;" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " if (a) {" - + " if (a > 0) {" - + " int a = 2;" - + " a = a + 1;" - + " } else" - + " a++;" - + " }" - + " }" - + "}"; - final String expected1 = "" - + "void main() {" - + " int a = 4;" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " {" - + " if (a > 0) {" - + " int a = 2;" - + " a = a + 1;" - + " } else" - + " a++;" - + " }" - + " }" - + "}"; - final String expected2 = "" - + "void main() {" - + " int a = 4;" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " if (a) {" - + " {" - + " int a = 2;" - + " a = a + 1;" - + " }" - + " }" - + " }" - + "}"; - final String expected3 = "" - + "void main() {" - + " int a = 4;" - + " if (" + Constants.GLF_DEAD + "(false)) {" - + " if (a) {" - + " a++;" - + " }" - + " }" - + "}"; - check(false, original, expected1, expected2, expected3); - } - - @Test - public void testInLiveCode() throws Exception { - final String original = "void main() {" - + " int x;" - + " {" - + " int GLF_live1hello = 4;" - + " if (GLF_live1hello < 46)" - + " GLF_live1hello++;" - + " }" - + " if (true) {" - + " x++;" - + " }" - + "}"; - final String expected = "void main() {" - + " int x;" - + " {" - + " int GLF_live1hello = 4;" - + " GLF_live1hello++;" - + " }" - + " if (true) {" - + " x++;" - + " }" - + "}"; - check(false, original, expected); - } - - @Test - public void testInDeadFunction() throws Exception { - final String original = "" - + "int foo(int x) {" - + " do {" - + " x++;" - + " } while(x < 100);" - + "}" - + "int bar(int x) {" - + " do {" - + " x++;" - + " break;" - + " } while(x < 100);" - + "}" - + "int baz(int x) {" - + " do " - + " break;" - + " while(x < 100);" - + "}" - + "void main() {" - + "}"; - final String expected = "" - + "int foo(int x) {" - + " {" - + " x++;" - + " }" - + "}" - + "int bar(int x) {" - + " do {" - + " x++;" - + " break;" - + " } while(x < 100);" - + "}" - + "int baz(int x) {" - + " do " - + " break;" - + " while(x < 100);" - + "}" - + "void main() {" - + "}"; - check(false, original, expected); - } - - @Test - public void testReduceEverywhere() throws Exception { - final String original = "" - + "void main() {" - + " for(int i = 0; i < 100; i++) {" - + " for (int j = 0; j < 200; j++) {" - + " if (i > j) {" - + " } else if(j > 5) {" - + " while(j > 5) j--;" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - - final String expected1 = "" - + "void main() {" - + " {" - + " int i = 0;" - + " for (int j = 0; j < 200; j++) {" - + " if (i > j) {" - + " } else if(j > 5) {" - + " while(j > 5) j--;" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - - final String expected2 = "" - + "void main() {" - + " for(int i = 0; i < 100; i++) {" - + " {" - + " int j = 0;" - + " if (i > j) {" - + " } else if(j > 5) {" - + " while(j > 5) j--;" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - final String expected3 = "" - + "void main() {" - + " for(int i = 0; i < 100; i++) {" - + " for (int j = 0; j < 200; j++) {" - + " {" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - final String expected4 = "" - + "void main() {" - + " for(int i = 0; i < 100; i++) {" - + " for (int j = 0; j < 200; j++) {" - + " if(j > 5) {" - + " while(j > 5) j--;" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - final String expected5 = "" - + "void main() {" - + " for(int i = 0; i < 100; i++) {" - + " for (int j = 0; j < 200; j++) {" - + " if (i > j) {" - + " } else if(j > 5) {" - + " j--;" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - final String expected6 = "" - + "void main() {" - + " for(int i = 0; i < 100; i++) {" - + " for (int j = 0; j < 200; j++) {" - + " if (i > j) {" - + " } else {" - + " while(j > 5) j--;" - + " }" - + " for (int t = 1; t < 4; t++) {" - + " continue;" - + " }" - + " }" - + " }" - + "}"; - - check(true, original, expected1, expected2, expected3, expected4, expected5, expected6); - } - - @Test - public void reduceEverywhereSimpleFor() throws Exception { - final String original = "void main() {" - + " for (int i = 0; i < 10; i++) ;" - + "}"; - final String expected = "void main() {" - + " { int i = 0; ; }" - + "}"; - check(true, original, expected); - } - - @Test - public void reduceEverywhereSimpleFor2() throws Exception { - final String original = "void main() {" - + " for (int i = 0; i < 10; i++) { }" - + "}"; - final String expected = "void main() {" - + " { int i = 0; }" - + "}"; - check(true, original, expected); - } - - private void check(boolean reduceEverywhere, String original, String... expected) - throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { - final TranslationUnit tu = ParseHelper.parse(original); - List ops = - getOps(tu, reduceEverywhere); - final Set actualSet = new HashSet<>(); - for (int i = 0; i < ops.size(); i++) { - final TranslationUnit clonedTu = tu.clone(); - List clonedOps = - getOps(clonedTu, reduceEverywhere); - assertEquals(ops.size(), clonedOps.size()); - clonedOps.get(i).applyReduction(); - actualSet.add(PrettyPrinterVisitor.prettyPrintAsString(clonedTu)); - } - final Set expectedSet = new HashSet<>(); - for (String anExpected : expected) { - expectedSet.add(PrettyPrinterVisitor - .prettyPrintAsString(ParseHelper.parse(anExpected))); - } - assertEquals(expectedSet, actualSet); - } - - private List getOps(TranslationUnit tu, - boolean reduceEverywhere) { - return ReductionOpportunities. - getReductionOpportunities( - MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext( - reduceEverywhere, - ShadingLanguageVersion.GLSL_440, - new RandomWrapper(0), - new IdGenerator(), - true), - fileOps - ).stream() - .filter(item -> item instanceof CompoundToBlockReductionOpportunity) - .map(item -> (CompoundToBlockReductionOpportunity) item) - .collect(Collectors.toList()); - } - - @Test - public void testDoNotTurnForLoopIntoBlock() throws Exception { - final String shader = "#version 310 es\n" - + "void GLF_live4doConvert()\n" - + "{\n" - + "}\n" - + "void main()\n" - + "{\n" - + " vec4 c = vec4(0.0, 0.0, 0.0, 1.0);\n" - + " for(\n" - + " int i = 0;\n" - + " i < 3;\n" - + " i ++\n" - + " )\n" - + " {\n" - + " c[i] = c[i] * c[i];\n" - + " }\n" - + " " + Constants.LIVE_PREFIX + "4doConvert();\n" - + "}"; - final TranslationUnit tu = ParseHelper.parse(shader); - final List ops = - CompoundToBlockReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_310, - new RandomWrapper(0), new IdGenerator(), true)); - assertEquals(0, ops.size()); - } - -} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunitiesTest.java index 75f47604a..0b255d49c 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/DestructifyReductionOpportunitiesTest.java @@ -19,6 +19,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -40,7 +41,7 @@ public void checkForVariableInScope() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); final List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); // There should be no opportunities as there is already a variable called 'dist' in scope assertEquals(0, ops.size()); } @@ -70,7 +71,7 @@ public void checkForVariableInScope2() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); // There should be one opportunity as variable dist is in a different scope and not used in this scope. assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -94,7 +95,7 @@ public void checkForVariableInScope3() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); // There should be no opportunities as there is already a variable called 'dist' in scope, // and it is used. assertEquals(0, ops.size()); @@ -126,7 +127,7 @@ public void misc() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); final List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); // There should be no opportunities as there is already a variable called 'dist' in scope, // and it is used. assertEquals(1, ops.size()); @@ -135,4 +136,4 @@ public void misc() throws Exception { PrettyPrinterVisitor.prettyPrintAsString(tu)); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ExprToConstantReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ExprToConstantReductionOpportunitiesTest.java index 1305310c6..668820754 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ExprToConstantReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ExprToConstantReductionOpportunitiesTest.java @@ -20,6 +20,7 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -35,7 +36,7 @@ public void testOut() throws Exception { TranslationUnit tu = ParseHelper.parse(prog); List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); for (SimplifyExprReductionOpportunity op : ops) { op.applyReduction(); } @@ -48,7 +49,7 @@ public void testInOut() throws Exception { TranslationUnit tu = ParseHelper.parse(prog); List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); for (SimplifyExprReductionOpportunity op : ops) { op.applyReduction(); } @@ -62,7 +63,7 @@ public void testIn() throws Exception { TranslationUnit tu = ParseHelper.parse(prog); List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); for (SimplifyExprReductionOpportunity op : ops) { op.applyReduction(); } @@ -74,10 +75,27 @@ public void testSingleLiveVariable() throws Exception { final String program = "void main() { int GLF_live3_a; GLF_live3_a; }"; final TranslationUnit tu = ParseHelper.parse(program); final List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts("void main() { int GLF_live3_a; 1; }", tu); } -} \ No newline at end of file + @Test + public void testSwitch() throws Exception { + final String program = "#version 310 es\n" + + "void main() {\n" + + " switch(0) {\n" + + " case - 1:\n" + + " 1;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(true, ShadingLanguageVersion.ESSL_310, new RandomWrapper(0), + new IdGenerator())); + assertEquals(0, ops.size()); + } + +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunitiesTest.java new file mode 100755 index 000000000..a83b1e562 --- /dev/null +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FlattenControlFlowReductionOpportunitiesTest.java @@ -0,0 +1,986 @@ +/* + * Copyright 2018 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderJobFileOperations; +import com.graphicsfuzz.util.Constants; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class FlattenControlFlowReductionOpportunitiesTest { + + private final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + + @Test + public void testUnderUnreachableSwitch() throws Exception { + final String original = "void main() {" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " if (true) {" + + " true;" + + " }" + + " case 2:" + + " if (true) {" + + " true;" + + " } else false;" + + " case 0:" + + " for (int i = 0; i < 100; i++) { }" + + " break;" + + " default:" + + " while(true) { 1; }" + + " }" + + "}"; + final String expected1 = "void main() {" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " { true; true; }" + + " case 2:" + + " if (true) {" + + " true;" + + " } else false;" + + " case 0:" + + " for (int i = 0; i < 100; i++) { }" + + " break;" + + " default:" + + " while(true) { 1; }" + + " }" + + "}"; + final String expected2 = "void main() {" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " if (true) {" + + " true;" + + " }" + + " case 2:" + + " { true; true; }" + + " case 0:" + + " for (int i = 0; i < 100; i++) { }" + + " break;" + + " default:" + + " while(true) { 1; }" + + " }" + + "}"; + final String expected3 = "void main() {" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " if (true) {" + + " true;" + + " }" + + " case 2:" + + " { true; false; }" + + " case 0:" + + " for (int i = 0; i < 100; i++) { }" + + " break;" + + " default:" + + " while(true) { 1; }" + + " }" + + "}"; + final String expected4 = "void main() {" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " if (true) {" + + " true;" + + " }" + + " case 2:" + + " if (true) {" + + " true;" + + " } else false;" + + " case 0:" + + " for (int i = 0; i < 100; i++) { }" + + " break;" + + " default:" + + " { true; 1; }" + + " }" + + "}"; + check(false, original, expected1, expected2, expected3, expected4); + } + + @Test + public void testDoNotRemoveDeadIf() throws Exception { + final String original = "" + + "void main() {" + + " int a = 2;" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " a = a + 1;" + + " }" + + "}"; + final List opportunities = + getOps(ParseHelper.parse(original), false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testInDeadCode() throws Exception { + final String original = "" + + "void main() {" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " if (false) {" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + "}"; + final String expected = "" + + "void main() {" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " {" + + " false;" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + "}"; + check(false, original, expected); + } + + @Test + public void testDeadIfReduceEverywhere() throws Exception { + final String original = "" + + "void main() {" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + "}"; + final String expected1 = "" + + "void main() {" + + " {" // first if changed to block + + " " + Constants.GLF_DEAD + "(false);" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + "}"; + final String expected2 = "" + + "void main() {" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " {" // second if changed to block + + " " + Constants.GLF_DEAD + "(false);" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + "}"; + check(true, original, expected1, expected2); + } + + @Test + public void testInDeadCode2() throws Exception { + final String original = "" + + "void main() {" + + " int a = 4;" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " if (a) {" + + " if (a > 0) {" + + " int a = 2;" + + " a = a + 1;" + + " } else" + + " a++;" + + " }" + + " }" + + "}"; + final String expected1 = "" + + "void main() {" + + " int a = 4;" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " {" + + " a;" + + " if (a > 0) {" + + " int a = 2;" + + " a = a + 1;" + + " } else" + + " a++;" + + " }" + + " }" + + "}"; + final String expected2 = "" + + "void main() {" + + " int a = 4;" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " if (a) {" + + " {" + + " a > 0;" + + " int a = 2;" + + " a = a + 1;" + + " }" + + " }" + + " }" + + "}"; + final String expected3 = "" + + "void main() {" + + " int a = 4;" + + " if (" + Constants.GLF_DEAD + "(false)) {" + + " if (a) {" + + " {" + + " a > 0;" + + " a++;" + + " }" + + " }" + + " }" + + "}"; + check(false, original, expected1, expected2, expected3); + } + + @Test + public void testInLiveCode() throws Exception { + final String original = "void main() {" + + " int x;" + + " {" + + " int GLF_live1hello = 4;" + + " if (GLF_live1hello < 46)" + + " GLF_live1hello++;" + + " }" + + " if (true) {" + + " x++;" + + " }" + + "}"; + final String expected = "void main() {" + + " int x;" + + " {" + + " int GLF_live1hello = 4;" + + " {" + + " GLF_live1hello < 46;" + + " GLF_live1hello++;" + + " }" + + " }" + + " if (true) {" + + " x++;" + + " }" + + "}"; + check(false, original, expected); + } + + @Test + public void testInDeadFunction() throws Exception { + final String original = "" + + "int foo(int x) {" + + " do {" + + " x++;" + + " } while(x < 100);" + + "}" + + "int bar(int x) {" + + " do {" + + " x++;" + + " break;" + + " } while(x < 100);" + + "}" + + "int baz(int x) {" + + " do " + + " break;" + + " while(x < 100);" + + "}" + + "void main() {" + + "}"; + final String expected = "" + + "int foo(int x) {" + + " {" + + " x++;" + + " x < 100;" + + " }" + + "}" + + "int bar(int x) {" + + " do {" + + " x++;" + + " break;" + + " } while(x < 100);" + + "}" + + "int baz(int x) {" + + " do " + + " break;" + + " while(x < 100);" + + "}" + + "void main() {" + + "}"; + check(false, original, expected); + } + + @Test + public void testReduceEverywhere() throws Exception { + final String original = "" + + "void main() {" + + " for(int i = 0; i < 100; i++) {" + + " for (int j = 0; j < 200; j++) {" + + " if (i > j) {" + + " } else if(j > 5) {" + + " while(j > 5) j--;" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " }" + + " }" + + "}"; + + final String expected1 = "" + + "void main() {" + + " {" + + " int i = 0;" + + " i < 100;" + + " for (int j = 0; j < 200; j++) {" + + " if (i > j) {" + + " } else if(j > 5) {" + + " while(j > 5) j--;" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " }" + + " i++;" + + " }" + + "}"; + + final String expected2 = "" + + "void main() {" + + " for(int i = 0; i < 100; i++) {" + + " {" + + " int j = 0;" + + " j < 200;" + + " if (i > j) {" + + " } else if(j > 5) {" + + " while(j > 5) j--;" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " j++;" + + " }" + + " }" + + "}"; + + final String expected3 = "" + + "void main() {" + + " for(int i = 0; i < 100; i++) {" + + " for (int j = 0; j < 200; j++) {" + + " {" + + " i > j;" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " }" + + " }" + + "}"; + + final String expected4 = "" + + "void main() {" + + " for(int i = 0; i < 100; i++) {" + + " for (int j = 0; j < 200; j++) {" + + " {" + + " i > j;" + + " if(j > 5) {" + + " while(j > 5) j--;" + + " }" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " }" + + " }" + + "}"; + + final String expected5 = "" + + "void main() {" + + " for(int i = 0; i < 100; i++) {" + + " for (int j = 0; j < 200; j++) {" + + " if (i > j) {" + + " } else {" + + " j > 5;" + + " while(j > 5) j--;" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " }" + + " }" + + "}"; + + final String expected6 = "" + + "void main() {" + + " for(int i = 0; i < 100; i++) {" + + " for (int j = 0; j < 200; j++) {" + + " if (i > j) {" + + " } else if(j > 5) {" + + " {" + + " j > 5;" + + " j--;" + + " }" + + " }" + + " for (int t = 1; t < 4; t++) {" + + " continue;" + + " }" + + " }" + + " }" + + "}"; + + check(true, original, expected1, expected2, expected3, expected4, expected5, expected6); + } + + @Test + public void reduceEverywhereSimpleFor() throws Exception { + final String original = "void main() {" + + " for (int i = 0; i < 10; i++) ;" + + "}"; + final String expected = "void main() {" + + " { int i = 0; i < 10; ; i++; }" + + "}"; + check(true, original, expected); + } + + @Test + public void reduceEverywhereSimpleFor2() throws Exception { + final String original = "void main() {" + + " for (int i = 0; i < 10; i++) { }" + + "}"; + final String expected = "void main() {" + + " { int i = 0; i < 10; i++; }" + + "}"; + check(true, original, expected); + } + + private void check(boolean reduceEverywhere, String original, String... expected) + throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { + final TranslationUnit tu = ParseHelper.parse(original); + List ops = + getOps(tu, reduceEverywhere); + final Set actualSet = new HashSet<>(); + for (int i = 0; i < ops.size(); i++) { + final TranslationUnit clonedTu = tu.clone(); + List clonedOps = + getOps(clonedTu, reduceEverywhere); + assertEquals(ops.size(), clonedOps.size()); + clonedOps.get(i).applyReduction(); + actualSet.add(PrettyPrinterVisitor.prettyPrintAsString(clonedTu)); + } + final Set expectedSet = new HashSet<>(); + for (String anExpected : expected) { + expectedSet.add(PrettyPrinterVisitor + .prettyPrintAsString(ParseHelper.parse(anExpected))); + } + assertEquals(expectedSet, actualSet); + } + + @Test + public void testDoNotTurnForLoopIntoBlock() throws Exception { + final String shader = "#version 310 es\n" + + "void GLF_live4doConvert()\n" + + "{\n" + + "}\n" + + "void main()\n" + + "{\n" + + " vec4 c = vec4(0.0, 0.0, 0.0, 1.0);\n" + + " for(\n" + + " int i = 0;\n" + + " i < 3;\n" + + " i ++\n" + + " )\n" + + " {\n" + + " c[i] = c[i] * c[i];\n" + + " }\n" + + " " + Constants.LIVE_PREFIX + "4doConvert();\n" + + "}"; + final TranslationUnit tu = ParseHelper.parse(shader); + final List ops = + getOps(tu, false); + assertEquals(0, ops.size()); + } + + @Test + public void doNotReplaceDeadCodeInjectionWithBody() throws Exception { + final String original = "void main() {" + + " if (" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false))) {" + + " discard;" + + " }" + + "}"; + final List opportunities = + getOps(ParseHelper.parse(original), false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testIfWithReduceEverywhere() throws Exception { + // Fine to reduce this conditional to its guard as we are not preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " if (x > 0) {\n" + + " ++x;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " {\n" + + " x > 0;\n" + + " ++x;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, true); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testForWithReduceEverywhere() throws Exception { + // Fine to reduce this loop to its guard as we are not preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " for (i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " {\n" + + " i = 0;\n" + + " i < 100;\n" + + " i++;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, true); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testWhileWithReduceEverywhere() throws Exception { + // Fine to reduce this loop to its guard as we are not preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " while (x > 0) {\n" + + " x--;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " {\n" + + " x > 0;" + + " x--;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, true); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoWhileWithReduceEverywhere() throws Exception { + // Fine to reduce this loop to its guard as we are not preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " do {\n" + + " x++;\n" + + " } while (x > 0);\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " {\n" + + " x++;\n" + + " x > 0;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, true); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testNoIfWithPreserveSemantics() throws Exception { + // No reduction opportunities here if preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " if (x > 0) {\n" + + " ++x;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " x > 0;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testNoForWithPreserveSemantics() throws Exception { + // No reduction opportunities here if preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " for (i = 0; i < 100; i++) {\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " i < 100;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testNoWhileWithPreserveSemantics() throws Exception { + // No reduction opportunities here if preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " while (x > 0) {\n" + + " x--;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " x > 0;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testNoDoWhileWithPreserveSemantics() throws Exception { + // No reduction opportunities here if preserving semantics. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " do {\n" + + " x++;\n" + + " } while (x > 0);\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " x > 0;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testNoSwitchWithPreserveSemantics() throws Exception { + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " switch (x + 3) {\n" + + " default:\n" + + " x++;\n" + + " break;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " x + 3;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(0, opportunities.size()); + } + + @Test + public void testIfWithPreserveSemanticsSideEffectFree() throws Exception { + // Despite preserving semantics, fine to reduce this conditional to its guard as the + // conditional is side effect-free. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " if (x > 0) {\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 2;\n" + + " {\n" + + " x > 0;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testForWithPreserveSemanticsSideEffectFree() throws Exception { + // Despite preserving semantics, fine to reduce this loop to its guard as the loop is side + // effect-free. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " i = 0;\n" + + " for (; i < 100;) {\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " i = 0;\n" + + " {\n" + + " ;\n" + + " i < 100;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testWhileWithPreserveSemanticsSideEffectFree() throws Exception { + // Despite preserving semantics, fine to reduce this loop to its guard as the loop is side + // effect-free. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " while (x > 0) {\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " {\n" + + " x > 0;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoWhileWithPreserveSemanticsSideEffectFree() throws Exception { + // Despite preserving semantics, fine to reduce this loop to its guard as the loop is side + // effect-free. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " do {\n" + + " } while (x > 0);\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int x = 100;\n" + + " {\n" + + " x > 0;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testIfWithPreserveSemanticsUnreachableSwitch() throws Exception { + // Fine to reduce two of these loops to their guards despite preserving semantics as they are + // under unreachable switch cases. + final String original = "#version 310 es\n" + + "void main() {\n" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " {\n" + + " int x = 2;\n" + + " if (x > 0) {\n" + + " ++x;\n" + + " }\n" + + " }\n" + + " case 0:" + + " {\n" + + " int y = 2;\n" + + " if (y > 0) {\n" + + " ++y;\n" + + " }\n" + + " }\n" + + " break;\n" + + " default:\n" + + " {\n" + + " int z = 2;\n" + + " if (z > 0) {\n" + + " ++z;\n" + + " }\n" + + " }\n" + + " }" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " switch(" + Constants.GLF_SWITCH + "(0)) {" + + " case 1:" + + " {\n" + + " int x = 2;\n" + + " {\n" + + " x > 0;\n" + + " ++x;\n" + + " }\n" + + " }\n" + + " case 0:" + + " {\n" + + " int y = 2;\n" + + " if (y > 0) {\n" + + " ++y;\n" + + " }\n" + + " }\n" + + " break;\n" + + " default:\n" + + " {\n" + + " int z = 2;\n" + + " {\n" + + " z > 0;\n" + + " ++z;\n" + + " }\n" + + " }\n" + + " }" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(2, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + opportunities.get(1).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testForWithPreserveSemanticsDeadCode() throws Exception { + // Fine to reduce this loop to its guard despite preserving semantics as we are in dead code. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " if (" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false))) {" + + " for (i = 0; i < 100; i++) {\n" + + " }\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int i;\n" + + " if (" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false))) {" + + " {\n" + + " i = 0;\n" + + " i < 100;\n" + + " i++;\n" + + " }\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testWhileWithPreserveSemanticsLiveCode() throws Exception { + // Fine to reduce this loop to its guard despite preserving semantics as we are in injected + // live code. + final String original = "#version 310 es\n" + + "void main() {\n" + + " int " + Constants.LIVE_PREFIX + "x = 100;\n" + + " while (" + Constants.LIVE_PREFIX + "x > 0) {\n" + + " " + Constants.LIVE_PREFIX + "x--;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void main() {\n" + + " int " + Constants.LIVE_PREFIX + "x = 100;\n" + + " {\n" + + " " + Constants.LIVE_PREFIX + "x > 0;\n" + + " " + Constants.LIVE_PREFIX + "x--;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoWhileWithPreserveSemanticsDeadFunction() throws Exception { + // Fine to reduce this loop to its guard despite preserving semantics as we are in a dead + // function. + final String original = "#version 310 es\n" + + "void " + Constants.GLF_DEAD + "_foo() {\n" + + " int x = 100;\n" + + " do {\n" + + " x++;\n" + + " } while (x > 0);\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "void " + Constants.GLF_DEAD + "_foo() {\n" + + " int x = 100;\n" + + " {\n" + + " x++;\n" + + " x > 0;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List opportunities = getOps(tu, false); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReductionImpl(); + CompareAsts.assertEqualAsts(expected, tu); + } + + private List getOps(TranslationUnit tu, + boolean reduceEverywhere) { + return FlattenControlFlowReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(reduceEverywhere, + tu.getShadingLanguageVersion(), new RandomWrapper(0), + new IdGenerator())); + } + +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunitiesTest.java index 4a70a8833..f1deaa8da 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FoldConstantReductionOpportunitiesTest.java @@ -20,8 +20,10 @@ import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.CompareAsts; import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.RandomWrapper; import java.io.IOException; import java.util.List; import org.junit.Test; @@ -626,8 +628,8 @@ private void check(String before, int numOps, String after) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { final TranslationUnit tu = ParseHelper.parse(before); final List ops = FoldConstantReductionOpportunities - .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, null, null, true)); + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); ops.forEach(item -> item.applyReduction()); CompareAsts.assertEqualAsts(after, tu); assertEquals(numOps, ops.size()); diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunitiesTest.java index 9d74c87e0..6da1a7fbf 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/FunctionReductionOpportunitiesTest.java @@ -18,7 +18,9 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.RandomWrapper; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -31,8 +33,8 @@ public void testRemovable() throws Exception { + "void main() { }"; TranslationUnit tu = ParseHelper.parse(program); assertEquals(1, FunctionReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)) + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())) .size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java new file mode 100644 index 000000000..a55f53013 --- /dev/null +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariableDeclToExprReductionOpportunitiesTest.java @@ -0,0 +1,210 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.RandomWrapper; +import java.util.List; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class GlobalVariableDeclToExprReductionOpportunitiesTest { + + @Test + public void testDoNotReplace() throws Exception { + final String original = "int a = 1; void main() { }"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + // There should be no opportunities as the preserve semantics is enabled. + assertTrue(ops.isEmpty()); + } + + @Test + public void testZeroMethod() throws Exception { + final String original = "int a = 1;"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + // Since the new assignment statement must be only inserted as the first statement of + // the main function, thus we have to ensure that main function exists. + // In this case, there is no method at all. + assertTrue(ops.isEmpty()); + } + + @Test + public void testNoMainMethod() throws Exception { + final String original = "int a = 1; void foo() { }"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + // Since the new assignment statement must be only inserted as the first statement of + // the main function, thus we have to ensure that main function exists. + // In this case, there is one method but it is not main though so there should be no + // opportunities. + assertTrue(ops.isEmpty()); + } + + @Test + public void testInsertAsFirstStatement() throws Exception { + final String program = "" + + "int a = 1;" + + "void main() {" + + " int b = a; " + + "}"; + // We have to ensure that the new assignment statements is inserted + // as the first statement in main() function. + final String expected = "" + + "int a;" + + "void main() {" + + " a = 1;" + + " int b = a;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + assertEquals(1, ops.size()); + ops.forEach(GlobalVariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testMultipleDeclarations() throws Exception { + final String program = "" + + "int a = 1;" // Initialized variable declaration. + + "int b = foo();" // Initialized variable declaration. + + "int c;" // Uninitialized variable declaration. + + "void main() { }"; + final String expected = "" + + "int a;" + + "int b;" + + "int c;" // Uninitialized variable declaration. + + "void main() {" + + " b = foo();" + + " a = 1;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + // Only variable declarations a and b have the initializer. + // Thus, we expect the reducer to find only 2 opportunities. + assertEquals(2, ops.size()); + ops.forEach(GlobalVariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoNotReplaceConst() throws Exception { + final String original = "const int a = 1; void main() { }"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + // There should be no opportunities as it is invalid to declare constant variable + // without an initial value. + assertTrue(ops.isEmpty()); + } + + @Test + public void testMultipleLineDeclarationsOneLine() throws Exception { + final String program = "" + + "int b = 1, c, d = foo(), e, f = bar();" // This variable declaration has many + // declaration infos but we consider only + // the one that has initializer (b, d, and f). + + "void main() { }"; + final String expected = "" + + "int b, c, d, e, f;" + + "void main() {" + + " f = bar();" + + " d = foo();" + + " b = 1;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + assertEquals(3, ops.size()); + ops.forEach(GlobalVariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testAssignVariableIdentifier() throws Exception { + final String program = "" + + "int a = 1;" + + "int b = a, c = b;" // b depends on a and c depends on b. + + "int d = c;" // d depends on c. + + "void main() { }"; + // No need to retain the order of the assignment as we are running the semantics-changing mode. + final String expected = "" + + "int a;" + + "int b, c;" + + "int d;" + + "void main() {" + + " d = c;" + + " c = b;" + + " b = a;" + + " a = 1;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + assertEquals(4, ops.size()); + ops.forEach(GlobalVariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testGlobalVariableDeclAfterMain() throws Exception { + // The global variable declarations that have found after main method will not be considered. + final String original = "void main() { } int a = 1;"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + GlobalVariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + assertTrue(ops.isEmpty()); + } +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariablesDeclarationReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariablesDeclarationReductionOpportunitiesTest.java index 7b0bf47d2..afc67c5eb 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariablesDeclarationReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/GlobalVariablesDeclarationReductionOpportunitiesTest.java @@ -47,7 +47,7 @@ public void testDoNotRemoveLocalInitialization() throws Exception { List ops = ReductionOpportunities .getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); assertEquals(0, ops.size()); } @@ -60,7 +60,7 @@ public void testDoNotRemoveUsedLiveCodeDecl() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = GlobalVariablesDeclarationReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -73,7 +73,7 @@ public void testUsedStructIsNotRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = GlobalVariablesDeclarationReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -84,7 +84,7 @@ public void testUnusedStructIsRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse(original); final List ops = GlobalVariablesDeclarationReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); @@ -99,7 +99,7 @@ public void testUsedAnonymousStructIsNotRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = GlobalVariablesDeclarationReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -110,7 +110,7 @@ public void testUnusedAnonymousStructIsRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse(original); final List ops = GlobalVariablesDeclarationReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); @@ -125,15 +125,15 @@ public void testVarIsRemovedButStructLeftBehind() throws Exception { final TranslationUnit tu = ParseHelper.parse(original); final List ops = VariableDeclReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); final List moreOps = GlobalVariablesDeclarationReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(0, moreOps.size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunitiesTest.java index ed3da6a01..127201b2c 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/IdentityMutationReductionOpportunitiesTest.java @@ -162,9 +162,8 @@ public void testReductionOpportunityForIdentityAsExprStmt() throws Exception { private ReducerContext getReducerContext() { return new ReducerContext(false, ShadingLanguageVersion.ESSL_310, - new RandomWrapper(), - new IdGenerator(), - true); + new RandomWrapper(0), + new IdGenerator()); } private void checkOneReductionOpportunity(String program, String afterReduction) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineFunctionReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineFunctionReductionOpportunitiesTest.java index 7c9e4c2dc..2e4f04c21 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineFunctionReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineFunctionReductionOpportunitiesTest.java @@ -53,7 +53,7 @@ public void testBasicInline() throws Exception { final List ops = InlineFunctionReductionOpportunities.findOpportunities(shaderJob, new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator(), false)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -77,10 +77,10 @@ public void testTooLargeToInline() throws Exception { final List ops = InlineFunctionReductionOpportunities.findOpportunities(shaderJob, new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator(), false)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunitiesTest.java index ef56cdd91..745ce05a2 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineInitializerReductionOpportunitiesTest.java @@ -17,12 +17,15 @@ package com.graphicsfuzz.reducer.reductionopportunities; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.util.Constants; import java.util.List; import org.junit.Test; @@ -35,10 +38,14 @@ public void testGlobalScope() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); + + // If we are preserving semantics, we do not want to apply this transformation. + assertTrue(InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())).isEmpty()); } @Test @@ -53,12 +60,12 @@ public void testDoNotInlineLargeInitializer() throws Exception { List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(largeProgramTu), new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(smallProgramTu), new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(3, ops.size()); ops.get(0).applyReduction(); ops.get(1).applyReduction(); @@ -85,23 +92,20 @@ private String make1Plus1Plus1Expr(int n) { } @Test - public void testCanInlineAllExceptLValueIfReducingEverywhere() throws Exception { + public void testDoNotInlineWhenPreservingSemantics() throws Exception { final String program = "void main() { int i = 4 + 2; i += i + i + i; i -= i * i; }"; - final String expected = "void main() { int i = 4 + 2; i += (4 + 2) + (4 + 2) + (4 + 2); i -= (4 + 2) * (4 + 2); }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); - assertEquals(5, ops.size()); - ops.forEach(SimplifyExprReductionOpportunity::applyReduction); - CompareAsts.assertEqualAsts(expected, tu); + InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + assertTrue(ops.isEmpty()); } @Test - public void testNoInliningIfLValueUsageExistsWhenPreservingSemantics() throws Exception { + public void testDoNotInlineWhenPreservingSemantics2() throws Exception { final String program = "void main() { int i = 4 + 2; i += i + i + i; i -= i * i; }"; @@ -109,19 +113,19 @@ public void testNoInliningIfLValueUsageExistsWhenPreservingSemantics() throws Ex List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); - assertEquals(0, ops.size()); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + assertTrue(ops.isEmpty()); } @Test - public void testNoInliningIfPartsOfInitializerAreModifiedWhenPreservingSemantics() throws Exception { + public void testDoNotInlineWhenPreservingSemantics3() throws Exception { final String program = "void main() { int x = 2; int y = x; x = 3; y; }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); - assertEquals(0, ops.size()); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + assertTrue(ops.isEmpty()); } @Test @@ -131,7 +135,7 @@ public void testNoInliningWithNameShadowing1() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -143,8 +147,61 @@ public void testNoInliningWithNameShadowing2() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } -} \ No newline at end of file + @Test + public void testInliningInDeadCodeWithPreserveSemantics() throws Exception { + // We do want to be able to inline an initializer if we are in dead code, when preserving + // semantics + final String program = "void main() {\n" + + " if(" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false, false))) {\n" + + " int do_inline_me = 5 + 2;\n" + + " do_inline_me * do_inline_me;\n" + + " }\n" + + "}\n"; + final String expected = "void main() {\n" + + " if(" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false, false))) {\n" + + " int do_inline_me = 5 + 2;\n" + + " (5 + 2) * (5 + 2);\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + assertEquals(2, ops.size()); + ops.get(0).applyReduction(); + ops.get(1).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testInliningInLiveCodeWithPreserveSemantics() throws Exception { + // We do want to be able to inline an initializer if we are in live code, when preserving + // semantics + final String variableName = Constants.LIVE_PREFIX + "do_inline_me"; + final String program = "void main() {\n" + + " {\n" + + " int " + variableName + " = 5 + 2;\n" + + " " + variableName + " * " + variableName + ";\n" + + " }\n" + + "}\n"; + final String expected = "void main() {\n" + + " {\n" + + " int " + variableName + " = 5 + 2;\n" + + " (5 + 2) * (5 + 2);\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = + InlineInitializerReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + assertEquals(2, ops.size()); + ops.get(0).applyReduction(); + ops.get(1).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunitiesTest.java index f37c48c59..287708ef3 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunitiesTest.java @@ -17,10 +17,16 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.transformreduce.GlslShaderJob; +import com.graphicsfuzz.common.transformreduce.ShaderJob; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.PipelineInfo; +import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.util.Constants; import com.graphicsfuzz.common.util.ParseHelper; -import com.graphicsfuzz.reducer.TestUtils; import com.graphicsfuzz.util.ExecHelper.RedirectType; import com.graphicsfuzz.util.ExecResult; import com.graphicsfuzz.util.ToolHelper; @@ -34,9 +40,12 @@ import org.junit.rules.TemporaryFolder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class InlineStructifiedFieldReductionOpportunitiesTest { + private final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); + @Rule public TemporaryFolder testFolder = new TemporaryFolder(); @@ -72,9 +81,11 @@ public void checkInliningAppliedCorrectly() throws Exception { + "}\n"; TranslationUnit tu = ParseHelper.parse(program); - assertEquals(1, InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)).size()); + assertEquals(1, + InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())).size()); InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)).get(0).applyReduction(); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())).get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(programAfter)), PrettyPrinterVisitor.prettyPrintAsString(tu)); } @@ -108,7 +119,8 @@ public void inlineRegressionTest() throws Exception { TranslationUnit tu = ParseHelper.parse(program).clone(); - List ops = InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + List ops = + InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(3, ops.size()); File tempFile = testFolder.newFile("temp.frag"); @@ -118,16 +130,10 @@ public void inlineRegressionTest() throws Exception { } op.applyReduction(); - new PrettyPrinterVisitor(System.out).visit(tu); - - PrintStream ps = new PrintStream(new FileOutputStream(tempFile)); - PrettyPrinterVisitor ppv = new PrettyPrinterVisitor(ps, - PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, true, Optional.empty()); - ppv.visit(tu); - ps.close(); - ExecResult result = ToolHelper.runValidatorOnShader(RedirectType.TO_BUFFER, tempFile); - assertEquals(result.stderr.toString() + result.stdout.toString(), 0, result.res); + final File shaderJobFile = testFolder.newFile("temp.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), tu), + shaderJobFile); + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); break; } @@ -151,11 +157,16 @@ public void inlineRegressionTest2() throws Exception { TranslationUnit tu = ParseHelper.parse(program).clone(); List ops = InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); - TestUtils.checkValid(testFolder, tu); + + final File shaderJobFile = testFolder.newFile("temp.json"); + fileOps.writeShaderJobFile(new GlslShaderJob(Optional.empty(), new PipelineInfo(), tu), + shaderJobFile); + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunityTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunityTest.java index ac372c73e..e554b304c 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunityTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineStructifiedFieldReductionOpportunityTest.java @@ -21,7 +21,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.decl.FunctionDefinition; import com.graphicsfuzz.common.ast.decl.FunctionPrototype; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.type.StructDefinitionType; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; @@ -41,8 +41,6 @@ import com.graphicsfuzz.common.ast.visitors.VisitationDepth; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.util.Constants; -import com.graphicsfuzz.common.util.ParseTimeoutException; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Optional; @@ -86,7 +84,7 @@ public void applyReduction() throws Exception { Arrays.asList( new DeclarationStmt(new VariablesDeclaration(outerStructDefinitionType.getStructNameType(), new VariableDeclInfo("myOuter", null, - new ScalarInitializer( + new Initializer( new TypeConstructorExpr(outerStructTypeName, Arrays.asList( new TypeConstructorExpr(innerStructTypeName, diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java index 001bac84a..9f41b156e 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/InlineUniformReductionOpportunitiesTest.java @@ -23,12 +23,15 @@ import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.CompareAsts; import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.util.Constants; import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import org.junit.Test; @@ -197,13 +200,12 @@ public void inlineUniforms() throws Exception { private ShaderJob checkCanReduceToTarget(ShaderJob shaderJob, int expectedSize, String target) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { - boolean found = false; for (int i = 0; i < expectedSize; i++) { final ShaderJob temp = shaderJob.clone(); List ops = InlineUniformReductionOpportunities.findOpportunities(temp, - new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(), null, true)); + new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null)); assertEquals(expectedSize, ops.size()); ops.get(i).applyReduction(); if (CompareAsts.isEqualAsts(target, temp.getShaders().get(0))) { @@ -214,4 +216,164 @@ private ShaderJob checkCanReduceToTarget(ShaderJob shaderJob, int expectedSize, return null; } + @Test + public void testDoNotInlineWhenPreservingSemantics() throws Exception { + final String shader = "#version 310 es\n" + + "uniform int u;\n" + + "void main() {\n" + + " u;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("u", BasicType.INT, Optional.empty(), Collections.singletonList(2)); + final List opportunities = + InlineUniformReductionOpportunities.findOpportunities(new GlslShaderJob(Optional.empty(), + pipelineInfo, tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), new IdGenerator())); + assertTrue(opportunities.isEmpty()); + } + + @Test + public void testDoInlineWhenPreservingSemanticsDeadCode() throws Exception { + final String shader = "#version 310 es\n" + + "uniform int u;\n" + + "void main() {\n" + + " u;\n" + + " if(" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false, false))) {\n" + + " u;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "uniform int u;\n" + + "void main() {\n" + + " u;\n" + + " if(" + Constants.GLF_DEAD + "(" + Constants.GLF_FALSE + "(false, false))) {\n" + + " 2;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("u", BasicType.INT, Optional.empty(), Collections.singletonList(2)); + final List opportunities = + InlineUniformReductionOpportunities.findOpportunities(new GlslShaderJob(Optional.empty(), + pipelineInfo, tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), new IdGenerator())); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoInlineWhenPreservingSemanticsDeadFunction() throws Exception { + final String shader = "#version 310 es\n" + + "uniform int u;\n" + + "void " + Constants.GLF_DEAD + "_foo() {\n" + + " u;\n" + + "}\n" + + "void main() {\n" + + " u;\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "uniform int u;\n" + + "void " + Constants.GLF_DEAD + "_foo() {\n" + + " 2;\n" + + "}\n" + + "void main() {\n" + + " u;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("u", BasicType.INT, Optional.empty(), Collections.singletonList(2)); + final List opportunities = + InlineUniformReductionOpportunities.findOpportunities(new GlslShaderJob(Optional.empty(), + pipelineInfo, tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), new IdGenerator())); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoInlineWhenPreservingSemanticsUnreachableSwitchCase() throws Exception { + final String shader = "#version 310 es\n" + + "uniform int u;\n" + + "void main() {\n" + + " switch(" + Constants.GLF_SWITCH + "(0)) {\n" + + " case 2:\n" + + " u;\n" + + " case 3:\n" + + " u;\n" + + " case 0:\n" + + " u;\n" + + " case 5:\n" + + " u;\n" + + " break;\n" + + " case 6:\n" + + " u;\n" + + " default:\n" + + " u;\n" + + " }\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "uniform int u;\n" + + "void main() {\n" + + " switch(" + Constants.GLF_SWITCH + "(0)) {\n" + + " case 2:\n" + + " 2;\n" + + " case 3:\n" + + " 2;\n" + + " case 0:\n" + + " u;\n" + + " case 5:\n" + + " u;\n" + + " break;\n" + + " case 6:\n" + + " 2;\n" + + " default:\n" + + " 2;\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("u", BasicType.INT, Optional.empty(), Collections.singletonList(2)); + final List opportunities = + InlineUniformReductionOpportunities.findOpportunities(new GlslShaderJob(Optional.empty(), + pipelineInfo, tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), new IdGenerator())); + assertEquals(4, opportunities.size()); + for (SimplifyExprReductionOpportunity op : opportunities) { + op.applyReduction(); + } + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testDoInlineWhenPreservingSemanticsLiveCode() throws Exception { + final String shader = "#version 310 es\n" + + "uniform int u;\n" + + "uniform int " + Constants.LIVE_PREFIX + "v;\n" + + "void main() {\n" + + " u;\n" + + " " + Constants.LIVE_PREFIX + "v;\n" + + "}\n"; + final String expected = "#version 310 es\n" + + "uniform int u;\n" + + "uniform int " + Constants.LIVE_PREFIX + "v;\n" + + "void main() {\n" + + " u;\n" + + " 3;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(shader); + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("u", BasicType.INT, Optional.empty(), Collections.singletonList(2)); + pipelineInfo.addUniform(Constants.LIVE_PREFIX + "v", BasicType.INT, Optional.empty(), Collections.singletonList(3)); + final List opportunities = + InlineUniformReductionOpportunities.findOpportunities(new GlslShaderJob(Optional.empty(), + pipelineInfo, tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_310, + new RandomWrapper(0), new IdGenerator())); + assertEquals(1, opportunities.size()); + opportunities.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunitiesTest.java index 7783f6119..e8a99c2f3 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LiveOutputVariableWriteReductionOpportunitiesTest.java @@ -17,7 +17,10 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.util.Constants; import com.graphicsfuzz.common.util.CompareAsts; import com.graphicsfuzz.common.util.OpenGlConstants; @@ -48,7 +51,8 @@ public void testLiveGLFragColorWriteOpportunity() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = LiveOutputVariableWriteReductionOpportunities - .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(reducedProgram, tu); @@ -69,7 +73,8 @@ public void testLiveGLFragColorWriteOpportunityAtRootOfMain() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = LiveOutputVariableWriteReductionOpportunities - .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext( + false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(reducedProgram, tu); @@ -94,7 +99,8 @@ public void testOpportunityIsPresent1() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = LiveOutputVariableWriteReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); @@ -115,7 +121,8 @@ public void testOpportunityIsPresent2() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = LiveOutputVariableWriteReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); @@ -137,7 +144,7 @@ public void testOpportunityIsPresent3() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = LiveOutputVariableWriteReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunitiesTest.java index fd188f586..cffe03673 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/LoopMergeReductionOpportunitiesTest.java @@ -19,7 +19,10 @@ import static org.junit.Assert.assertEquals; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.util.Constants; import com.graphicsfuzz.common.util.ParseHelper; import java.util.List; @@ -62,7 +65,8 @@ public void findOpportunities() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List opportunities = - LoopMergeReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + LoopMergeReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, opportunities.size()); diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunitiesTest.java index f482dfeb7..bc1648d2a 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/OutlinedStatementReductionOpportunitiesTest.java @@ -17,8 +17,11 @@ package com.graphicsfuzz.reducer.reductionopportunities; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; import org.junit.Test; @@ -36,7 +39,8 @@ public void testOutline() throws Exception { TranslationUnit tu = ParseHelper.parse(program); - List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -56,7 +60,8 @@ public void testOutline2() throws Exception { TranslationUnit tu = ParseHelper.parse(program); - List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -78,7 +83,9 @@ public void testOutline3() throws Exception { TranslationUnit tu = ParseHelper.parse(program); - List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -95,9 +102,11 @@ public void testOutline4() throws Exception { TranslationUnit tu = ParseHelper.parse(program); - List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + List ops = OutlinedStatementReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + assertEquals(0, ops.size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesTest.java index e7a8907e9..af4e8e3f6 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/ReductionOpportunitiesTest.java @@ -60,7 +60,7 @@ public void testDeadConditionalNotReplaced() throws Exception { List opportunities = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); // There should be no ExprToConstant reduction opportunity, because the expressions do not occur // under dead code, and the fuzzed expression is too simple to be reduced. assertFalse(opportunities.stream().anyMatch(item -> item instanceof SimplifyExprReductionOpportunity)); @@ -74,17 +74,17 @@ public void testPopScope() throws Exception { ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); } private void stressTestStructification(String variantProgram, String reducedProgram) throws Exception{ TranslationUnit tu = ParseHelper.parse(variantProgram); - IRandom generator = new RandomWrapper(); + IRandom generator = new RandomWrapper(0); while (true) { List ops = InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); if (ops.isEmpty()) { break; } @@ -93,7 +93,7 @@ private void stressTestStructification(String variantProgram, String reducedProg while (true) { List ops - = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); if (ops.isEmpty()) { break; } @@ -104,7 +104,7 @@ private void stressTestStructification(String variantProgram, String reducedProg List ops = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); if (ops.isEmpty()) { break; } @@ -333,8 +333,8 @@ public void testStructInlineWithParam() throws Exception { TranslationUnit tu = ParseHelper.parse(program); List ops = - InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); - InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); + InlineStructifiedFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); @@ -380,7 +380,7 @@ public void testDestructifyWithParam() throws Exception { List ops = DestructifyReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); @@ -425,7 +425,7 @@ public void exprSimplificationAndLoopSplitInteraction() throws Exception { { List ops = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); for (int i = 0; i < ops.size(); i++) { if (ops.get(i) instanceof SimplifyExprReductionOpportunity) { indicesOfExprToConstOps.add(i); @@ -441,13 +441,13 @@ public void exprSimplificationAndLoopSplitInteraction() throws Exception { List ops = ReductionOpportunities.getReductionOpportunities( MakeShaderJobFromFragmentShader.make(aClone), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator(), true), fileOps); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator()), fileOps); ((SimplifyExprReductionOpportunity) ops.get(index)).applyReduction(); } for (IReductionOpportunity op : ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(aClone), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator(), true), fileOps)) { + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator()), fileOps)) { if (op instanceof LoopMergeReductionOpportunity) { op.applyReduction(); } @@ -508,7 +508,7 @@ private void tryAllCompatibleOpportunities(String program) TranslationUnit tu = ParseHelper.parse(program); int numOps = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator(), true), fileOps).size(); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator()), fileOps).size(); for (int i = 0; i < numOps; i++) { for (int j = 0; j < numOps; j++) { @@ -519,7 +519,7 @@ private void tryAllCompatibleOpportunities(String program) List ops = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(aClone), new ReducerContext(false, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator(), true), fileOps); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator()), fileOps); if (Compatibility.compatible(ops.get(i).getClass(), ops.get(j).getClass())) { ops.get(i).applyReduction(); @@ -537,7 +537,7 @@ public void reduceRedundantStuff() throws Exception { while (true) { List ops = ReductionOpportunities.getReductionOpportunities( MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new RandomWrapper(0), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); if (ops.isEmpty()) { break; } @@ -554,7 +554,7 @@ public void testReduceToConstantInLiveCode() throws Exception { + " float GLF_live3x = sin(4.0);" + " float GLF_live3y = GLF_live3x + GLF_live3x;" + "}"); - List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + List ops = ExprToConstantReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(4, ops.size()); } @@ -579,7 +579,7 @@ public void testLeaveLoopLimiter() throws Exception { while (true) { List ops = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new RandomWrapper(), new IdGenerator(), true), fileOps); + ShadingLanguageVersion.GLSL_440, new RandomWrapper(0), new IdGenerator()), fileOps); if (ops.isEmpty()) { break; } @@ -618,7 +618,7 @@ public void testTernary() throws Exception { List ops = ExprToConstantReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); if (i >= ops.size()) { break; } @@ -652,7 +652,7 @@ public void testRemoveLoopLimiter() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = StmtReductionOpportunities.findOpportunities( MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), @@ -670,13 +670,13 @@ public void testInterplayBetweenVectorizationAndIdentity() throws Exception { + "}"); { List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } { List ops = IdentityMutationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts("void main() {" @@ -689,7 +689,7 @@ public void testInterplayBetweenVectorizationAndIdentity() throws Exception { } { List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(2, ops.size()); ops.get(0).applyReduction(); ops.get(1).applyReduction(); @@ -704,7 +704,7 @@ public void testInterplayBetweenVectorizationAndIdentity() throws Exception { { List ops = VariableDeclReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts("void main() {" @@ -718,7 +718,7 @@ public void testInterplayBetweenVectorizationAndIdentity() throws Exception { { List ops = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts("void main() {" @@ -757,7 +757,7 @@ public void reduceNestedVectors() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(4, ops.size()); // (1) remove b from GLF_merged2_0_1_1_1_1_1bc @@ -879,7 +879,7 @@ public void reduceNestedVectors() throws Exception { List stmtOps = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(7, stmtOps.size()); for (StmtReductionOpportunity op : stmtOps) { op.applyReduction(); @@ -887,7 +887,7 @@ public void reduceNestedVectors() throws Exception { CompareAsts.assertEqualAsts(expected5, tu); ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(3, ops.size()); // (6) remove a from GLF_merged3_0_1_1_1_1_1_2_1_1abc @@ -959,7 +959,7 @@ public void reduceNestedVectors() throws Exception { stmtOps = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(3, stmtOps.size()); for (StmtReductionOpportunity op : stmtOps) { op.applyReduction(); @@ -982,7 +982,7 @@ public void reduceNestedVectors() throws Exception { VariableDeclReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(5, varDeclOps.size()); for (VariableDeclReductionOpportunity op : varDeclOps) { op.applyReduction(); @@ -1000,7 +1000,7 @@ public void reduceNestedVectors() throws Exception { StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(5, finalStmtOps.size()); for (StmtReductionOpportunity op : finalStmtOps) { op.applyReduction(); @@ -1022,14 +1022,14 @@ public void testRemoveDeclarationsInUnreachableFunction() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = VariableDeclReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(4, ops.size()); for (VariableDeclReductionOpportunity op : ops) { op.applyReduction(); } final List moreOps = StmtReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(4, moreOps.size()); } @@ -1076,7 +1076,7 @@ public void testUniformsGetRemoved() throws Exception { List ops = ReductionOpportunities.getReductionOpportunities( MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new RandomWrapper(0), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); assertEquals(1, ops.size()); assertTrue(ops.get(0) instanceof VariableDeclReductionOpportunity); ops.get(0).applyReduction(); @@ -1113,7 +1113,7 @@ public void testReductionOpportunitiesOnEmptyForLoops() throws Exception { List ops = ReductionOpportunities.getReductionOpportunities( MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.GLSL_440, - new RandomWrapper(0), new IdGenerator(), true), fileOps); + new RandomWrapper(0), new IdGenerator()), fileOps); ops.forEach(IReductionOpportunity::applyReduction); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java index cabfda1d4..b6dc44313 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveRedundantUniformMetadataReductionOpportunitiesTest.java @@ -21,22 +21,20 @@ import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderKind; import java.util.Collections; import java.util.List; import java.util.Optional; -import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import static org.junit.Assert.assertEquals; public class RemoveRedundantUniformMetadataReductionOpportunitiesTest { - // TODO(478): enable this test once the issue is addressed. - @Ignore @Test public void testRemoveUnused() throws Exception { // Make a shader job with an empty fragment shader, and declare one uniform in the pipeline @@ -50,15 +48,13 @@ public void testRemoveUnused() throws Exception { assertEquals(1, shaderJob.getPipelineInfo().getNumUniforms()); // There should be exactly one opportunity to remove a piece of unused pipeline state. - // TODO(478): remove the Assert.fail(), and un-comment the lines that follow it. - Assert.fail(); - //List ops = - // RemoveRedundantUniformMetadatReductionOpportunities - // .findOpportunities(shaderJob, - // new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, - // true)); - //assertEquals(1, ops.size()); - //ops.get(0).applyReduction(); + List ops = + RemoveRedundantUniformMetadataReductionOpportunities + .findOpportunities(shaderJob, + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); + assertEquals(1, ops.size()); + ops.get(0).applyReduction(); // Check that after applying the reduction opportunity there are no uniforms in the pipeline // state and that the shader has not changed. @@ -66,8 +62,36 @@ public void testRemoveUnused() throws Exception { assertEquals(0, shaderJob.getPipelineInfo().getNumUniforms()); } - // TODO(478): enable this test once the issue is addressed. - @Ignore + @Test + public void testRemoveUnusedNameShadowing() throws Exception { + // Checks for the case where a uniform declared in the pipeline state is not used, but another + // variable shadows its name. + final String minimalShader = "void main() { int shadow; }"; + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("shadow", BasicType.FLOAT, Optional.empty(), + Collections.singletonList(10.0)); + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + pipelineInfo, ParseHelper.parse(minimalShader)); + // The pipeline info has an unused uniform named 'shadow', and the shader declares an unrelated + // variable called 'shadow'. We would like the uniform to be removed from the pipeline info, + // as it is not used. + assertEquals(1, shaderJob.getPipelineInfo().getNumUniforms()); + + // There should be exactly one opportunity to remove a piece of unused pipeline state. + List ops = + RemoveRedundantUniformMetadataReductionOpportunities + .findOpportunities(shaderJob, + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); + assertEquals(1, ops.size()); + ops.get(0).applyReduction(); + + // Check that after applying the reduction opportunity there are no uniforms in the pipeline + // state and that the shader has not changed. + CompareAsts.assertEqualAsts(minimalShader, shaderJob.getFragmentShader().get()); + assertEquals(0, shaderJob.getPipelineInfo().getNumUniforms()); + } + @Test public void testDoNotRemoveUsed() throws Exception { // Make a shader job with a simple fragment shader that declares (but does not use) @@ -81,15 +105,43 @@ public void testDoNotRemoveUsed() throws Exception { // Check that initially there is indeed one uniform in the pipeline state. assertEquals(1, shaderJob.getPipelineInfo().getNumUniforms()); - // There should be exactly one opportunity to remove a piece of unused pipeline state. - // TODO(478): remove the Assert.fail(), and un-comment the lines that follow it. - Assert.fail(); - //List ops = - // RemoveRedundantUniformMetadatReductionOpportunities - // .findOpportunities(shaderJob, - // new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, - // true)); - //assertEquals(0, ops.size()); + // There should be no opportunities to remove a piece of unused pipeline state. + List ops = + RemoveRedundantUniformMetadataReductionOpportunities + .findOpportunities(shaderJob, + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); + assertEquals(0, ops.size()); + } + + @Test + public void testDoNotRemoveUsedMultipleShaders() throws Exception { + // A shader job with a vertex shader and fragment shader that each use a different + // uniform (but that do not use a uniform in common). Neither uniform should be removed + // from the pipeline state. + final String minimalVertexShader = "uniform float used_in_vertex_only; void main() { " + + "used_in_vertex_only; }"; + final String minimalFragmentShader = "uniform float used_in_fragment_only; void main() { " + + "used_in_fragment_only; }"; + final PipelineInfo pipelineInfo = new PipelineInfo(); + pipelineInfo.addUniform("used_in_vertex_only", BasicType.FLOAT, Optional.empty(), + Collections.singletonList(10.0)); + pipelineInfo.addUniform("used_in_fragment_only", BasicType.FLOAT, Optional.empty(), + Collections.singletonList(20.0)); + final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), + pipelineInfo, ParseHelper.parse(minimalVertexShader, ShaderKind.VERTEX), + ParseHelper.parse(minimalFragmentShader, ShaderKind.FRAGMENT)); + // Check that initially there are indeed two uniforms in the pipeline state. + assertEquals(2, shaderJob.getPipelineInfo().getNumUniforms()); + + // There should be no opportunities to remove a piece of unused pipeline state, since both + // uniforms are referenced. + List ops = + RemoveRedundantUniformMetadataReductionOpportunities + .findOpportunities(shaderJob, + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); + assertEquals(0, ops.size()); } } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunitiesTest.java index 35493427e..52b70788c 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunitiesTest.java @@ -19,6 +19,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -47,7 +48,7 @@ public void nestedStruct() throws Exception { TranslationUnit tu = ParseHelper.parse(program); assertEquals(2, RemoveStructFieldReductionOpportunities - .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)).size()); + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())).size()); } @@ -67,7 +68,7 @@ public void singleField() throws Exception { TranslationUnit tu = ParseHelper.parse(program); assertEquals(0, RemoveStructFieldReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)).size()); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())).size()); } @@ -99,7 +100,7 @@ public void testNonLocalInitialization() throws Exception { TranslationUnit tu = ParseHelper.parse(shader); List ops = RemoveStructFieldReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(true, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + new ReducerContext(true, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(tu), PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected))); @@ -132,7 +133,7 @@ public void twoFields() throws Exception { TranslationUnit tu = ParseHelper.parse(program); final List ops = RemoveStructFieldReductionOpportunities - .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(2, ops .size()); ops.stream().filter(item -> item.getFieldToRemove().equals("_f0")) @@ -149,4 +150,4 @@ public void twoFields() throws Exception { PrettyPrinterVisitor.prettyPrintAsString(tu)); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunityTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunityTest.java index 022ed11e4..5da758de8 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunityTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveStructFieldReductionOpportunityTest.java @@ -18,7 +18,7 @@ import static org.junit.Assert.assertEquals; -import com.graphicsfuzz.common.ast.decl.ScalarInitializer; +import com.graphicsfuzz.common.ast.decl.Initializer; import com.graphicsfuzz.common.ast.type.StructDefinitionType; import com.graphicsfuzz.common.ast.decl.VariableDeclInfo; import com.graphicsfuzz.common.ast.decl.VariablesDeclaration; @@ -30,10 +30,6 @@ import com.graphicsfuzz.common.ast.type.BasicType; import com.graphicsfuzz.common.ast.type.StructNameType; import com.graphicsfuzz.common.ast.visitors.VisitationDepth; -import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import org.junit.Test; @@ -61,9 +57,9 @@ public void applyReduction() throws Exception { barConstructor.clone(), barConstructor.clone())); VariableDeclInfo v1 = new VariableDeclInfo("v1", null, - new ScalarInitializer(fooConstructor.clone())); + new Initializer(fooConstructor.clone())); VariableDeclInfo v2 = new VariableDeclInfo("v2", null, - new ScalarInitializer(fooConstructor.clone())); + new Initializer(fooConstructor.clone())); DeclarationStmt declarationStmt = new DeclarationStmt(new VariablesDeclaration(foo.getStructNameType(), Arrays.asList(v1, v2))); @@ -73,7 +69,7 @@ public void applyReduction() throws Exception { declarationStmt), false); assertEquals("foo v1 = foo(1.0, bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0)), bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0))), v2 = foo(1.0, bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0)), bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0)));\n", - getString(declarationStmt)); + declarationStmt.getText()); IReductionOpportunity ro1 = new RemoveStructFieldReductionOpportunity(foo, "a", b, new VisitationDepth(0)); @@ -87,31 +83,25 @@ public void applyReduction() throws Exception { ro1.applyReduction(); assertEquals("foo v1 = foo(bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0)), bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0))), v2 = foo(bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0)), bar(vec2(0.0, 0.0), vec3(0.0, 0.0, 0.0)));\n", - getString(declarationStmt)); + declarationStmt.getText()); assertEquals(2, bar.getNumFields()); assertEquals(2, foo.getNumFields()); ro2.applyReduction(); assertEquals("foo v1 = foo(bar(vec3(0.0, 0.0, 0.0)), bar(vec3(0.0, 0.0, 0.0))), v2 = foo(bar(vec3(0.0, 0.0, 0.0)), bar(vec3(0.0, 0.0, 0.0)));\n", - getString(declarationStmt)); + declarationStmt.getText()); assertEquals(1, bar.getNumFields()); assertEquals(2, foo.getNumFields()); ro3.applyReduction(); assertEquals("foo v1 = foo(bar(vec3(0.0, 0.0, 0.0))), v2 = foo(bar(vec3(0.0, 0.0, 0.0)));\n", - getString(declarationStmt)); + declarationStmt.getText()); assertEquals(1, bar.getNumFields()); assertEquals(1, foo.getNumFields()); } - private String getString(DeclarationStmt declarationStmt) { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - new PrettyPrinterVisitor(new PrintStream(output)).visit(declarationStmt); - return new String(output.toByteArray(), StandardCharsets.UTF_8); - } - -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunitiesTest.java index 6d5172e98..b1aed8b22 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/RemoveUnusedParameterReductionOpportunitiesTest.java @@ -19,6 +19,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -139,7 +140,7 @@ private List findOpportunities(Transl boolean reduceEverywhere) { return RemoveUnusedParameterReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(reduceEverywhere, - ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), null, true)); + ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); } } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java index 7d7a25ae3..855ade1ea 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/StmtReductionOpportunitiesTest.java @@ -25,7 +25,9 @@ import com.graphicsfuzz.common.ast.stmt.Stmt; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; +import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.util.Constants; @@ -76,7 +78,7 @@ public void testSwitch1() throws Exception { final TranslationUnit tu = ParseHelper.parse(prog); List ops = StmtReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_130, - new RandomWrapper(), null, true)); + new RandomWrapper(0), new IdGenerator())); for (StmtReductionOpportunity op : ops) { op.applyReduction(); @@ -102,7 +104,7 @@ public void testDoNotLeaveDefaultEmpty() throws Exception { List ops = StmtReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, ShadingLanguageVersion.ESSL_310, - new RandomWrapper(), null, false)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); @@ -122,7 +124,7 @@ public void testEmptyBlock() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, null, null, null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); assertTrue(ops.get(0) instanceof StmtReductionOpportunity); ops.get(0).applyReduction(); @@ -136,7 +138,7 @@ public void testNullStmtRemoved() throws Exception { final String reducedProgram = "void main() { }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, null, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); assertTrue(ops.get(0) instanceof StmtReductionOpportunity); ops.get(0).applyReduction(); @@ -149,7 +151,9 @@ public void testNullStmtsInForNotTouched() throws Exception { final String program = "void main() { for(int i = 0; i < 100; i++) ; }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(0, ops.size()); } @@ -158,7 +162,9 @@ public void testNullStmtsInForNotTouched2() throws Exception { final String program = "int x; void foo() { x = 42; } void main() { if (foo()) ; else ; }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), + new IdGenerator())); assertEquals(0, ops.size()); } @@ -173,7 +179,7 @@ public void testSimpleLiveCodeRemoval() throws Exception { + "}"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); assertTrue(ops.get(0) instanceof StmtReductionOpportunity); ops.get(0).applyReduction(); @@ -195,7 +201,7 @@ public void testUnaryIncDecLiveCodeRemoval() throws Exception { + "}"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(4, ops.size()); for (int i = 0; i < ops.size(); i++) { assertTrue(ops.get(i) instanceof StmtReductionOpportunity); @@ -211,7 +217,7 @@ public void testSideEffectFreeTypeInitializerRemoved() throws Exception { final String reducedProgram = "void main() { }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); assertTrue(ops.get(0) instanceof StmtReductionOpportunity); ops.get(0).applyReduction(); @@ -224,7 +230,7 @@ public void testSideEffectingTypeInitializerNotRemoved() throws Exception { final String program = "void main() { float x; vec4(0.0, x++, 0.0, 0.0); }"; final TranslationUnit tu = ParseHelper.parse(program); List ops = - StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, null, null, true)); + StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -250,8 +256,8 @@ public void testDeadStatementsPickedUp() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = StmtReductionOpportunities.findOpportunities( MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), null, - true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), + new IdGenerator())); assertEquals(4, ops.size()); } @@ -271,42 +277,61 @@ public void testDeadBasicReturnsPickedUp() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = StmtReductionOpportunities.findOpportunities( MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), null, true)); + new ReducerContext(false, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), new IdGenerator())); assertEquals(3, ops.size()); } @Test public void testRemoveDuplicateReturn1() throws Exception { final String program = "int foo() { return 0; return 1; return 2; return 3; }"; - final String expected = "int foo() { return 3; }"; + final String expected = "int foo() { return 1; }"; final TranslationUnit tu = ParseHelper.parse(program); - final List ops = StmtReductionOpportunities.findOpportunities( - MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), null, - true)); - assertEquals(3, ops.size()); + final ReducerContext context = new ReducerContext(true, ShadingLanguageVersion.ESSL_300, + new RandomWrapper(0), + new IdGenerator()); + final ShaderJob shaderJob = MakeShaderJobFromFragmentShader.make(tu); + List ops = StmtReductionOpportunities.findOpportunities( + shaderJob, + context); + assertEquals(4, ops.size()); ops.get(0).applyReduction(); - ops.get(1).applyReduction(); + ops.get(3).applyReduction(); ops.get(2).applyReduction(); + assertFalse(ops.get(1).preconditionHolds()); CompareAsts.assertEqualAsts(expected, tu); + ops = StmtReductionOpportunities.findOpportunities( + shaderJob, + context); + assertTrue(ops.isEmpty()); } @Test public void testRemoveDuplicateReturn2() throws Exception { - final String program = "int foo() { int x; return 0; x = 1; return 1; x = 2; return 2; return " - + "3; }"; - final String expected = "int foo() { int x; return 3; }"; + final String program = "int foo() {\n" + + " int x;\n" + + " return 0;\n" + + " x = 1;\n" + + " return 1;\n" + + " x = 2;\n" + + " return 2;\n" + + " return 3;\n" + + "}\n"; + final String expected = "int foo() {\n" + + " int x;\n" + + " return 3;\n" + + "}\n"; final TranslationUnit tu = ParseHelper.parse(program); final List ops = StmtReductionOpportunities.findOpportunities( MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), null, - true)); - assertEquals(5, ops.size()); + new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), + new IdGenerator())); + assertEquals(6, ops.size()); ops.get(0).applyReduction(); ops.get(1).applyReduction(); ops.get(2).applyReduction(); ops.get(3).applyReduction(); ops.get(4).applyReduction(); + assertFalse(ops.get(5).preconditionHolds()); CompareAsts.assertEqualAsts(expected, tu); } @@ -317,11 +342,70 @@ public void testRemoveArbitraryVoidReturn() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); final List ops = StmtReductionOpportunities.findOpportunities( MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), null, - true)); + new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), new IdGenerator())); assertEquals(5, ops.size()); } + @Test + public void testRemoveNonVoidReturn() throws Exception { + // Both return statements are candidates for removal, but removing one disables removing the + // other. + final String program = "int foo() { return 1; return 0; }"; + final String expected = "int foo() { return 0; }"; + final TranslationUnit tu = ParseHelper.parse(program); + final ReducerContext context = new ReducerContext(true, ShadingLanguageVersion.ESSL_300, + new RandomWrapper(0), new IdGenerator()); + final ShaderJob shaderJob = MakeShaderJobFromFragmentShader.make(tu); + List ops = StmtReductionOpportunities.findOpportunities( + shaderJob, + context); + assertEquals(2, ops.size()); + ops.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + assertFalse(ops.get(1).preconditionHolds()); + ops = StmtReductionOpportunities.findOpportunities( + shaderJob, + context); + assertTrue(ops.isEmpty()); + } + + @Test + public void testDoNotRemoveReturnThatWouldChangeSemantics() throws Exception { + final String program = "layout(location = 0) out vec4 color;\n" + + "void main() {\n" + + " return;\n" + + " color = vec4(1.0);\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = StmtReductionOpportunities.findOpportunities( + MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(false, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), + new IdGenerator())); + // We cannot remove the return, as this would make the write to 'color' reachable. + // We could in principle remove the color write itself, since it is unreachable. Right now we + // do not, so this test would have to be re-thought if we decided to add that facility. + assertEquals(0, ops.size()); + } + + @Test + public void testRemoveReturnsInIf() throws Exception { + final String program = "layout(location = 0) out vec4 color;\n" + + "int foo() {\n" + + " if (true) {\n" + + " return 1;\n" + + " } else {\n" + + " return 2;\n" + + " }\n" + + " return 0;\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(program); + final List ops = StmtReductionOpportunities.findOpportunities( + MakeShaderJobFromFragmentShader.make(tu), + new ReducerContext(true, ShadingLanguageVersion.ESSL_300, new RandomWrapper(0), + new IdGenerator())); + assertTrue(ops.size() > 0); + } + @Test public void testIdentifyLiveCodeProperly() throws Exception { final String liveInjectedVariableName = Constants.LIVE_PREFIX + "somevar"; diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunitiesTest.java new file mode 100644 index 000000000..592c4c36b --- /dev/null +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/SwitchToLoopReductionOpportunitiesTest.java @@ -0,0 +1,210 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.util.Constants; +import java.util.List; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class SwitchToLoopReductionOpportunitiesTest { + + @Test + public void testDoNotReduceIfPreservingSemantics() throws Exception { + final String original = "void main() {\n" + + " if (" + Constants.GLF_DEAD + "(false)) {\n" + + " switch (0) {\n" + + " case 0:\n" + + " return;\n" + + " }\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + VariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_320, + new RandomWrapper(0), new IdGenerator())); + // There should be no opportunities as the preserve semantics is enabled. + assertTrue(ops.isEmpty()); + } + + @Test + public void testDoReduceWhenPreservingSemanticsIfInDeadCode() throws Exception { + final String original = "void main() {\n" + + " if (" + Constants.GLF_DEAD + "(false)) {\n" + + " switch (0) {\n" + + " case 0:\n" + + " return;\n" + + " }\n" + + " }\n" + + "}\n"; + final String expected = "void main() {\n" + + " if (" + Constants.GLF_DEAD + "(false)) {\n" + + " do {\n" + + " 0;" + + " return;\n" + + " } while (false);\n" + + " }\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + SwitchToLoopReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_320, + new RandomWrapper(0), new IdGenerator())); + // There should be an opportunity, as the switch statement is in a dead code block. + assertEquals(1, ops.size()); + ops.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testNoBreaks() throws Exception { + final String original = "void main() {\n" + + " int a;\n" + + " int x = 3;\n" + + " switch (x) {\n" + + " case 0:\n" + + " {\n" + + " x = 1;\n" + + " }\n" + + " case 1:\n" + + " a = 4;\n" + + " x = 2;\n" + + " case 3:\n" + + " default:\n" + + " x = 1;\n" + + " }\n" + + "}\n"; + final String expected = "void main() {\n" + + " int a;\n" + + " int x = 3;\n" + + " do {\n" + + " x;\n" + + " {\n" + + " x = 1;\n" + + " }\n" + + " a = 4;\n" + + " x = 2;\n" + + " x = 1;\n" + + " } while (false);\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + SwitchToLoopReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_320, + new RandomWrapper(0), new IdGenerator())); + assertEquals(1, ops.size()); + ops.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testWithBreaks() throws Exception { + final String original = "void main() {\n" + + " int a;\n" + + " int x = 3;\n" + + " switch (x) {\n" + + " case 0:\n" + + " {\n" + + " x = 1;\n" + + " }\n" + + " break;\n" + + " case 1:\n" + + " a = 4;\n" + + " x = 2;\n" + + " break;\n" + + " case 3:\n" + + " default:\n" + + " x = 1;\n" + + " break;\n" + + " }\n" + + "}\n"; + final String expected = "void main() {\n" + + " int a;\n" + + " int x = 3;\n" + + " do {\n" + + " x;\n" + + " {\n" + + " x = 1;\n" + + " }\n" + + " break;\n" + + " a = 4;\n" + + " x = 2;\n" + + " break;\n" + + " x = 1;\n" + + " break;\n" + + " } while (false);\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + SwitchToLoopReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_320, + new RandomWrapper(0), new IdGenerator())); + assertEquals(1, ops.size()); + ops.get(0).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testTwoSwitches() throws Exception { + final String original = "void main() {\n" + + " int x;\n" + + " switch (0) {\n" + + " case 0:\n" + + " x = 1;\n" + + " }\n" + + " switch (1) {\n" + + " case 1:\n" + + " x = 2;\n" + + " }\n" + + "}\n"; + final String expected = "void main() {\n" + + " int x;\n" + + " do {\n" + + " 0;\n" + + " x = 1;\n" + + " } while (false);\n" + + " do {\n" + + " 1;\n" + + " x = 2;\n" + + " } while (false);\n" + + "}\n"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + SwitchToLoopReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_320, + new RandomWrapper(0), new IdGenerator())); + assertEquals(2, ops.size()); + ops.get(0).applyReduction(); + ops.get(1).applyReduction(); + CompareAsts.assertEqualAsts(expected, tu); + } + +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunitiesTest.java index 4112d4778..6fa98e910 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnswitchifyReductionOpportunitiesTest.java @@ -18,6 +18,7 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import java.util.List; @@ -36,8 +37,8 @@ public void testNotInjected() throws Exception { new ReducerContext(false, ShadingLanguageVersion.GLSL_130, new RandomWrapper(0), - null, true)); + new IdGenerator())); assertEquals(0, ops.size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunitiesTest.java index a1236d0f9..94ed4406d 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/UnwrapReductionOpportunitiesTest.java @@ -45,7 +45,7 @@ public void testBlock1() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); assertEquals(1, ops.size()); } @@ -55,7 +55,7 @@ public void testEmptyBlock() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); // No opportunities because the inner block is empty. Another reduction pass may be able to // delete it, but it cannot be unwrapped. assertEquals(0, ops.size()); @@ -67,7 +67,7 @@ public void testNestedEmptyBlocks() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); assertEquals(1, ops.size()); } @@ -78,7 +78,7 @@ public void testUnwrapWithDeclNoClash() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(expected, tu); @@ -91,7 +91,7 @@ public void testNoUnwrapWithDeclClash() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); // The inner block cannot be unwrapped as it would change which variable 'x = 2' refers to. assertEquals(1, ops.size()); } @@ -102,7 +102,7 @@ public void testNoUnwrapWithDeclClash2() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); assertEquals(0, ops.size()); } @@ -112,7 +112,7 @@ public void testOneUnwrapDisablesAnother() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, - ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), null, true)); + ShadingLanguageVersion.GLSL_440, new SameValueRandom(false, 0), new IdGenerator())); assertEquals(2, ops.size()); assertTrue(ops.get(0).preconditionHolds()); assertTrue(ops.get(1).preconditionHolds()); @@ -144,7 +144,7 @@ public void misc() throws Exception { assertTrue(PrettyPrinterVisitor.prettyPrintAsString(tu).contains(expectedStmt)); IRandom generator = new RandomWrapper(1); - List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, generator, null, true)); + List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, generator, new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); assertTrue(PrettyPrinterVisitor.prettyPrintAsString(tu).contains(expectedStmt)); @@ -156,8 +156,8 @@ public void testDoubleRemoval() throws Exception { final String expected = "void main() { }"; final TranslationUnit tu = ParseHelper.parse(shader); final IRandom generator = new RandomWrapper(0); - List stmtOps = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, generator, null, true)); - List unwrapOps = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, generator, null, true)); + List stmtOps = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, generator, new IdGenerator())); + List unwrapOps = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, generator, new IdGenerator())); assertTrue(!stmtOps.isEmpty()); assertTrue(!unwrapOps.isEmpty()); stmtOps.forEach(StmtReductionOpportunity::applyReduction); @@ -190,12 +190,12 @@ public void unwrapFor() throws Exception { final RandomWrapper generator = new RandomWrapper(0); final IdGenerator idGenerator = new IdGenerator(); List ops = UnwrapReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, version, - generator, idGenerator, true)); + generator, idGenerator)); assertEquals(1, ops.size()); ops.get(0).applyReduction(); List remainingOps = VariableDeclReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, version, generator, idGenerator, true)); + new ReducerContext(false, version, generator, idGenerator)); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(tu), PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected))); assertEquals(2, remainingOps.size()); remainingOps.get(0).applyReduction(); @@ -209,7 +209,7 @@ public void unwrapFor() throws Exception { assertEquals(PrettyPrinterVisitor.prettyPrintAsString(tu), PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected2))); remainingOps = StmtReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, version, generator, idGenerator, true)); + new ReducerContext(false, version, generator, idGenerator)); assertEquals(3, remainingOps.size()); remainingOps.get(0).applyReduction(); remainingOps.get(1).applyReduction(); @@ -219,7 +219,7 @@ public void unwrapFor() throws Exception { assertEquals(PrettyPrinterVisitor.prettyPrintAsString(tu), PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected3))); remainingOps = ReductionOpportunities.getReductionOpportunities(MakeShaderJobFromFragmentShader.make(tu), - new ReducerContext(false, version, generator, idGenerator, true), fileOps); + new ReducerContext(false, version, generator, idGenerator), fileOps); assertEquals(0, remainingOps.size()); } diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunitiesTest.java index ac25d5b25..735bca51d 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclReductionOpportunitiesTest.java @@ -40,7 +40,7 @@ public void testRemoveUnusedLiveCodeDecl() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = VariableDeclReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(reducedProgram, tu); @@ -53,7 +53,7 @@ public void testAnonymousStruct() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = VariableDeclReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(reducedProgram, tu); @@ -66,7 +66,7 @@ public void testNamedStruct() throws Exception { final TranslationUnit tu = ParseHelper.parse(program); List ops = VariableDeclReductionOpportunities .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.ESSL_100, - new RandomWrapper(0), null, true)); + new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); CompareAsts.assertEqualAsts(reducedProgram, tu); @@ -86,10 +86,9 @@ public void testNothingToRemove() throws Exception { false, ShadingLanguageVersion.ESSL_100, new RandomWrapper(0), - new IdGenerator(), - true) + new IdGenerator()) ); assertEquals(0, ops.size()); } -} \ No newline at end of file +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java new file mode 100644 index 000000000..5a7a5e270 --- /dev/null +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VariableDeclToExprReductionOpportunitiesTest.java @@ -0,0 +1,142 @@ +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphicsfuzz.reducer.reductionopportunities; + +import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; +import com.graphicsfuzz.common.util.CompareAsts; +import com.graphicsfuzz.common.util.IdGenerator; +import com.graphicsfuzz.common.util.ParseHelper; +import com.graphicsfuzz.common.util.RandomWrapper; +import java.util.List; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class VariableDeclToExprReductionOpportunitiesTest { + + @Test + public void testDoNotReplace() throws Exception { + final String original = "void main() { int a = 1, b = 2; int c = 3; }"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + VariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + // There should be no opportunities as the preserve semantics is enabled. + assertTrue(ops.isEmpty()); + } + + @Test + public void testDoNotReplaceConst() throws Exception { + final String original = "void main() { const int a = 1;}"; + final TranslationUnit tu = ParseHelper.parse(original); + final List ops = + VariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + // There should be no opportunities as it is invalid to declare constant variable + // without an initial value. + assertTrue(ops.isEmpty()); + } + + @Test + public void testMultipleDeclarations() throws Exception { + final String program = "void main() {" + + "int a = 1;" // Initialized variable declaration. + + "int b = foo();" // Initialized variable declaration. + + "int c;" // Uninitialized variable declaration. + + "}"; + final String expected = "void main() {" + + " int a;" + + " a = 1;" + + " int b;" + + " b = foo();" + + " int c;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + List ops = VariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + // Only variable declarations a and b have the initializer. + // Thus, we expect the reducer to find only 2 opportunities. + assertEquals(2, ops.size()); + ops.forEach(VariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testMultipleLineDeclarationsOneLine() throws Exception { + final String program = "void main() {" + + "int a;" + + "int b = 1, c, d = foo(), e, f = bar(); " // This variable declaration has many + // declaration infos but we consider only + // the one that has initializer (b, d, and f). + + "int g;" + + "}"; + final String expected = "void main() {" + + " int a;" + + " int b, c, d, e, f;" + + " b = 1;" + + " d = foo();" + + " f = bar();" + + " int g;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + List ops = VariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + assertEquals(3, ops.size()); + ops.forEach(VariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + + @Test + public void testAssignVariableIdentifier() throws Exception { + final String program = "void main() {" + + "int a = 1;" + + "int b = a, c = b;" // b depends on a and c depends on b. + + "int d = c;" // d depends on c. + + "}"; + // As here we have the variable identifier as the initializer, we need to + // make sure that new expressions generated by the reducer are added + // in the correct order. + final String expected = "void main() {" + + " int a;" + + " a = 1;" + + " int b, c;" + + " b = a;" + + " c = b;" + + " int d;" + + " d = c;" + + "}"; + final TranslationUnit tu = ParseHelper.parse(program); + List ops = VariableDeclToExprReductionOpportunities + .findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(true, + ShadingLanguageVersion.ESSL_100, + new RandomWrapper(0), new IdGenerator())); + assertEquals(4, ops.size()); + ops.forEach(VariableDeclToExprReductionOpportunity::applyReductionImpl); + CompareAsts.assertEqualAsts(expected, tu); + } + +} diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunitiesTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunitiesTest.java index d9528928b..25d686064 100755 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunitiesTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/reductionopportunities/VectorizationReductionOpportunitiesTest.java @@ -22,6 +22,7 @@ import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; import com.graphicsfuzz.common.util.CannedRandom; import com.graphicsfuzz.common.util.IRandom; +import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.common.util.ZeroCannedRandom; @@ -57,7 +58,8 @@ public void testInDead() throws Exception { + " }\n" + "}\n"; final TranslationUnit tu = ParseHelper.parse(shader); - final List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, new RandomWrapper(0), null, true)); + final List ops = + VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, new RandomWrapper(0), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); @@ -75,7 +77,7 @@ public void testIncompatibleDeclaration() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(0, ops.size()); } @@ -94,7 +96,7 @@ public void testProblematicDeclarationPreviousScope() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(0, ops.size()); } @@ -127,7 +129,7 @@ public void testOneProblematicDeclarationPreviousScope() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected)), PrettyPrinterVisitor.prettyPrintAsString(tu)); @@ -176,7 +178,7 @@ public void misc1() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(2, ops.size()); ops = ops.stream().filter(item -> item.getComponentType() == BasicType.FLOAT).collect(Collectors.toList()); assertEquals(2, ops.size()); @@ -218,7 +220,7 @@ public void misc2() throws Exception { final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(2, ops.size()); ops.stream().filter(item -> item.getVectorName().equals("GLF_merged2_0_1_1_1_1_1bc") && item.getComponentName().equals("b")).findAny().get().applyReduction(); @@ -228,7 +230,7 @@ public void misc2() throws Exception { assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected2)), PrettyPrinterVisitor.prettyPrintAsString(tu)); ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(0, ops.size()); } @@ -270,7 +272,7 @@ public void misc3() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(shader); final IRandom cannedRandom = new ZeroCannedRandom(); - List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, cannedRandom, null, true)); + List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, cannedRandom, new IdGenerator())); assertEquals(3, ops.size()); ops.stream().filter(item -> item.getComponentName().equals("P")).findAny().get().applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected1)), PrettyPrinterVisitor.prettyPrintAsString(tu)); @@ -278,7 +280,7 @@ public void misc3() throws Exception { assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected2)), PrettyPrinterVisitor.prettyPrintAsString(tu)); ops.stream().filter(item -> item.getComponentName().equals("GLF_merged2_0_1_1_1_1_1PQ")).findAny().get().applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected3)), PrettyPrinterVisitor.prettyPrintAsString(tu)); - ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, cannedRandom, null, true)); + ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, cannedRandom, new IdGenerator())); assertEquals(0, ops.size()); } @@ -320,7 +322,7 @@ public void misc4() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(shader); final IRandom cannedRandom = new CannedRandom(2); - final List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, cannedRandom, null, true)); + final List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, cannedRandom, new IdGenerator())); ops.stream().filter(item -> item.getComponentName().equals("a")).findAny().get().applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected1)), PrettyPrinterVisitor.prettyPrintAsString(tu)); ops.stream().filter(item -> item.getComponentName().equals("c")).findAny().get().applyReduction(); @@ -367,7 +369,7 @@ public void misc5() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(shader); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(3, ops.size()); ops.stream().filter(item -> item.getComponentName().equals("GLF_merged2_0_1_1_1_1_1PQ")).findAny().get().applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected1)), PrettyPrinterVisitor.prettyPrintAsString(tu)); @@ -406,12 +408,12 @@ public void testCompatibleDeclaration() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(2, ops.size()); ops.get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected1)), PrettyPrinterVisitor.prettyPrintAsString(tu)); ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(1, ops.size()); ops.get(0).applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected2)), PrettyPrinterVisitor.prettyPrintAsString(tu)); @@ -452,12 +454,12 @@ public void testBasicBehaviour() throws Exception { + "}\n"; final TranslationUnit tu = ParseHelper.parse(original); List ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(2, ops.size()); ops.stream().filter(item -> item.getComponentName().equals("b")).findAny().get().applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected1)), PrettyPrinterVisitor.prettyPrintAsString(tu)); ops = VectorizationReductionOpportunities.findOpportunities(MakeShaderJobFromFragmentShader.make(tu), new ReducerContext(false, ShadingLanguageVersion.GLSL_440, - new ZeroCannedRandom(), null, true)); + new ZeroCannedRandom(), new IdGenerator())); assertEquals(1, ops.size()); ops.stream().filter(item -> item.getComponentName().equals("c")).findAny().get().applyReduction(); assertEquals(PrettyPrinterVisitor.prettyPrintAsString(ParseHelper.parse(expected2)), PrettyPrinterVisitor.prettyPrintAsString(tu)); diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/tool/GlslReduceTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/tool/GlslReduceTest.java index 2d7003321..da9dca7bc 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/tool/GlslReduceTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/tool/GlslReduceTest.java @@ -145,7 +145,10 @@ public void checkCustomJudgeIsExecutable() throws Exception { @Test public void testAlwaysReduceJudgeMaximallyReduces() throws Exception { final File jsonFile = getShaderJobReady(); - final File emptyFile = temporaryFolder.newFile(); + // We make this a .bat file to avoid a "what application would you like to use to open this + // file?" pop-up on Windows. (On other platforms the fact that it has the .bat extension does + // not matter.) + final File emptyFile = temporaryFolder.newFile("judge.bat"); emptyFile.setExecutable(true); GlslReduce.mainHelper(new String[]{ jsonFile.getAbsolutePath(), diff --git a/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java b/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java index 8289411d7..96636f151 100644 --- a/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java +++ b/reducer/src/test/java/com/graphicsfuzz/reducer/util/SimplifyTest.java @@ -19,13 +19,10 @@ import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.util.CompareAsts; import com.graphicsfuzz.common.util.ParseHelper; -import org.junit.Ignore; import org.junit.Test; public class SimplifyTest { - // TODO(110) - this test fails due to issue 110, and should be enabled once that issue is fixed. - @Ignore @Test public void testIfParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" @@ -40,11 +37,10 @@ public void testIfParenthesesRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } - @Ignore @Test public void testWhileParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" - + " while (_GLF_DEAD(_GLF_FALSE(false, false))) {" + + " while(_GLF_WRAPPED_LOOP(_GLF_FALSE(false ,_GLF_DEAD(_GLF_FALSE(false, false))))) {" + " }" + "}"); final String expected = "void main() {" @@ -55,30 +51,28 @@ public void testWhileParenthesesRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } - @Ignore @Test public void testForParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" + " for (" - + " int i = _GLF_ONE(1, _GLF_IDENTITY(1, 1));" + + " int i = int(_GLF_ONE(1.0, _GLF_IDENTITY(1.0, 1.0)));" + " i > _GLF_IDENTITY(1, _GLF_FUZZED(1));" + " i++)" + " { }" + " }" + "}"); final String expected = "void main() {" - + " for (int i = 1; i > 1; i++)" + + " for (int i = int(1.0); i > 1; i++)" + " { }" + "}"; final TranslationUnit simplifiedTu = Simplify.simplify(tu); CompareAsts.assertEqualAsts(expected, simplifiedTu); } - @Ignore @Test public void testSwitchParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" - + " switch(_GLF_ZERO(0, _GLF_IDENTITY(0, 0)))" + + " switch(_GLF_SWITCH(_GLF_ZERO(0, _GLF_IDENTITY(0, 0))))" + " {" + " }" + "}"); @@ -91,11 +85,10 @@ public void testSwitchParenthesesRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } - @Ignore @Test public void testDoWhileParenthesesRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" - + " do { } while (_GLF_DEAD(_GLF_FALSE(false, false)));" + + " do { } while (_GLF_WRAPPED_LOOP(_GLF_DEAD(_GLF_FALSE(false, false))));" + "}"); final String expected = "void main() {" + " do { } while (false);" @@ -104,7 +97,6 @@ public void testDoWhileParenthesesRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } - @Ignore @Test public void testMacroBlockRemoved() throws Exception { final TranslationUnit tu = ParseHelper.parse("void main() {" @@ -113,7 +105,7 @@ public void testMacroBlockRemoved() throws Exception { + " int a = _GLF_IDENTITY(_GLF_FUZZED(1), 1);" + "}" ); - final String expected = "void() {" + final String expected = "void main() {" + " 1;" + " 1;" + " int a = 1;" @@ -122,4 +114,111 @@ public void testMacroBlockRemoved() throws Exception { CompareAsts.assertEqualAsts(expected, simplifiedTu); } + @Test + public void testIdentityNotNestedRemoved() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() {" + + " if(_GLF_IDENTITY(true, true)) {}" + + " int x = _GLF_IDENTITY(1, 1);" + + " x = _GLF_IDENTITY(1, 1);" + + " _GLF_IDENTITY(1, 1);" + + "}" + ); + final String expected = "void main() {" + + " if(true) {}" + + " int x = 1;" + + " x = 1;" + + " 1;" + + "}"; + final TranslationUnit simplifiedTu = Simplify.simplify(tu); + CompareAsts.assertEqualAsts(expected, simplifiedTu); + } + + @Test + public void testFunctionCallParenthesesRemoved() throws Exception { + final TranslationUnit tu = ParseHelper.parse("void main() {" + + "foo(_GLF_IDENTITY(1 + 2, 1 + 2));" + + "}" + ); + final String expected = "void main() {" + + " foo(1 + 2);" + + "}"; + final TranslationUnit simplifiedTu = Simplify.simplify(tu); + CompareAsts.assertEqualAsts(expected, simplifiedTu); + } + + @Test + public void testMakeInBoundsIntClampRemoved() throws Exception { + final TranslationUnit tu = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " float A[4];\n" + + " A[_GLF_MAKE_IN_BOUNDS_INT(A[_GLF_MAKE_IN_BOUNDS_INT(3, 4)], 4)] =\n" + + " A[_GLF_MAKE_IN_BOUNDS_INT(1 + 2, 4)];\n" + + "}\n" + ); + final String expected = "#version 310 es\n" + + "void main() {\n" + + " float A[4];\n" + + " A[clamp(A[clamp(3, 0, 4 - 1)], 0, 4 - 1)] =\n" + + " A[clamp(1 + 2, 0, 4 - 1)];\n" + + "}\n"; + final TranslationUnit simplifiedTu = Simplify.simplify(tu); + CompareAsts.assertEqualAsts(expected, simplifiedTu); + } + + @Test + public void testMakeInBoundsUintClampRemoved() throws Exception { + final TranslationUnit tu = ParseHelper.parse("#version 310 es\n" + + "void main() {\n" + + " float A[4];\n" + + " A[_GLF_MAKE_IN_BOUNDS_UINT(A[_GLF_MAKE_IN_BOUNDS_UINT(3u, 4u)], 4u)] =\n" + + " A[_GLF_MAKE_IN_BOUNDS_UINT(uint(1 + 2), 4u)];\n" + + "}\n" + ); + final String expected = "#version 310 es\n" + + "void main() {\n" + + " float A[4];\n" + + " A[clamp(A[clamp(3u, 0u, 4u - 1u)], 0u, 4u - 1u)] =\n" + + " A[clamp(uint(1 + 2), 0u, 4u - 1u)];\n" + + "}\n"; + final TranslationUnit simplifiedTu = Simplify.simplify(tu); + CompareAsts.assertEqualAsts(expected, simplifiedTu); + } + + @Test + public void testMakeInBoundsIntTernaryRemoved() throws Exception { + final TranslationUnit tu = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " float A[4];\n" + + " A[_GLF_MAKE_IN_BOUNDS_INT(A[_GLF_MAKE_IN_BOUNDS_INT(3, 4)], 4)] = 1.0;\n" + + "}\n" + ); + final String inner = "A[((3) < 0 ? 0 : ((3) >= 4 ? 4 - 1 : (3)))]"; + final String expected = "#version 100\n" + + "void main() {\n" + + " float A[4];\n" + + " A[((" + inner + ") < 0 ? 0 : ((" + inner + ") >= 4 ? 4 - 1 : (" + inner + ")))] = 1" + + ".0;\n" + + "}\n"; + final TranslationUnit simplifiedTu = Simplify.simplify(tu); + CompareAsts.assertEqualAsts(expected, simplifiedTu); + } + + @Test + public void testMakeInBoundsUintTernaryRemoved() throws Exception { + final TranslationUnit tu = ParseHelper.parse("#version 100\n" + + "void main() {\n" + + " float A[4];\n" + + " A[_GLF_MAKE_IN_BOUNDS_UINT(A[_GLF_MAKE_IN_BOUNDS_UINT(3u, 4u)], 4u)] = 1.0;\n" + + "}\n" + ); + final String inner = "A[((3u) >= 4u ? 4u - 1u : (3u))]"; + final String expected = "#version 100\n" + + "void main() {\n" + + " float A[4];\n" + + " A[((" + inner + ") >= 4u ? 4u - 1u : (" + inner + "))] = 1.0;\n" + + "}\n"; + final TranslationUnit simplifiedTu = Simplify.simplify(tu); + CompareAsts.assertEqualAsts(expected, simplifiedTu); + } + } diff --git a/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java b/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java index 30d61d26f..d00d8f081 100755 --- a/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java +++ b/server/src/main/java/com/graphicsfuzz/server/webui/WebUi.java @@ -29,7 +29,6 @@ import com.graphicsfuzz.server.thrift.CommandResult; import com.graphicsfuzz.server.thrift.FuzzerServiceManager; import com.graphicsfuzz.server.thrift.WorkerInfo; -import com.graphicsfuzz.server.thrift.WorkerNameNotFoundException; import com.graphicsfuzz.util.Constants; import java.io.File; import java.io.FileNotFoundException; @@ -39,6 +38,7 @@ import java.io.PrintWriter; import java.nio.charset.Charset; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -72,7 +72,7 @@ * / : homepage: list workers, and options to start experiments. * Should also display the server queue state. * worker/"workername" : worker page, with overview of results - * worker/"workername"/"shadersetname" : result of this worker on this shaderset. + * worker/"workername"/"shadersetname" : result of this worker on this shader family. * Can start reduction on a single variant * startExperiment/ * @@ -85,8 +85,9 @@ public class WebUi extends HttpServlet { private long startTime; private final AccessFileInfo accessFileInfo; - private final FilenameFilter variantFragFilter = - (dir, name) -> name.startsWith("variant_") && name.endsWith(".frag"); + private static final String WARNING_CLASS_WRONG_RESULT = "wrongresult"; + private static final String WARNING_CLASS_WARN_RESULT = "warnresult"; + private static final String WARNING_CLASS_METRICS_DISAGREE = "metricsdisimg"; private enum ImageDifferenceResult { IDENTICAL, @@ -102,6 +103,12 @@ private static final class ImageDifferenceResultSet { public double histogramDistance = 0.0; } + private enum ComputeDifferenceResult { + IDENTICAL, + SIMILAR, + DIFFERENT + } + private final ShaderJobFileOperations fileOps; private final FuzzerServiceManager.Iface fuzzerServiceManagerProxy; @@ -112,24 +119,24 @@ public WebUi(FuzzerServiceManager.Iface fuzzerServiceManager, ShaderJobFileOpera this.fuzzerServiceManagerProxy = fuzzerServiceManager; } - private static final class Shaderset { + private static final class ShaderFamily { final String name; final File dir; final File preview; final int nbVariants; - final boolean isComp; + final boolean isCompute; - public Shaderset(String name) { + public ShaderFamily(String name) { this.name = name; - this.dir = new File(WebUiConstants.SHADERSET_DIR, name); + this.dir = new File(WebUiConstants.SHADER_FAMILIES_DIR, name); this.preview = new File(dir, "thumb.png"); - this.isComp = new File(dir, "reference.comp").isFile(); + this.isCompute = new File(dir, "reference.comp").isFile(); this.nbVariants = getNbVariants(); } private int getNbVariants() { final File[] variants; - if (this.isComp) { + if (this.isCompute) { variants = dir.listFiles(item -> item.getName().startsWith("variant") && item.getName().endsWith(".comp")); } else { @@ -140,26 +147,33 @@ private int getNbVariants() { } } - private static final class ShadersetExp { - final Shaderset shaderset; + private static final class ShaderFamilyResult { + final ShaderFamily shaderFamily; final String name; final String worker; final File dir; int nbVariants; int nbVariantDone; int nbErrors; + + // For image shader families: int nbSameImage; int nbSlightlyDifferentImage; int nbMetricsDisagree; int nbWrongImage; - public ShadersetExp(String name, String worker, AccessFileInfo accessFileInfo) + // For compute shader families: + int nbSameComputeResult; + int nbSlightlyDifferentComputeResult; + int nbWrongComputeResult; + + public ShaderFamilyResult(String name, String worker, AccessFileInfo accessFileInfo) throws FileNotFoundException { this.name = name; this.worker = worker; this.dir = new File(WebUiConstants.WORKER_DIR + "/" + worker + "/" + name); - this.shaderset = new Shaderset(name); - this.nbVariants = shaderset.nbVariants; + this.shaderFamily = new ShaderFamily(name); + this.nbVariants = shaderFamily.nbVariants; // Set variant counters for (File file : dir.listFiles()) { @@ -168,23 +182,40 @@ public ShadersetExp(String name, String worker, AccessFileInfo accessFileInfo) JsonObject info = accessFileInfo.getResultInfo(file); String status = info.get("status").getAsString(); if (status.contentEquals("SUCCESS")) { - ImageDifferenceResult result = getImageDiffResult(info).summary; - switch (result) { - case IDENTICAL: - ++nbSameImage; - break; - case SIMILAR: - ++nbSlightlyDifferentImage; - break; - case DIFFERENT: - ++nbWrongImage; - break; - case METRICS_DISAGREE: - ++nbWrongImage; - ++nbMetricsDisagree; - break; - default: - LOGGER.error("Unrecognized image difference result: " + result); + if (shaderFamily.isCompute) { + ComputeDifferenceResult result = getComputeDiffResult(info); + switch (result) { + case IDENTICAL: + ++nbSameComputeResult; + break; + case SIMILAR: + ++nbSlightlyDifferentComputeResult; + break; + case DIFFERENT: + ++nbWrongComputeResult; + break; + default: + LOGGER.error("Unrecognized compute difference result: " + result); + } + } else { + ImageDifferenceResult result = getImageDiffResult(info).summary; + switch (result) { + case IDENTICAL: + ++nbSameImage; + break; + case SIMILAR: + ++nbSlightlyDifferentImage; + break; + case DIFFERENT: + ++nbWrongImage; + break; + case METRICS_DISAGREE: + ++nbWrongImage; + ++nbMetricsDisagree; + break; + default: + LOGGER.error("Unrecognized image difference result: " + result); + } } } else { nbErrors++; @@ -209,10 +240,10 @@ private static ImageDifferenceResultSet getImageDiffResult(JsonObject info) { return result; } - if (!info.has("metrics")) { + if (!info.has(metricsKey)) { return result; } - final JsonObject metricsJson = info.get("metrics").getAsJsonObject(); + final JsonObject metricsJson = info.get(metricsKey).getAsJsonObject(); // Check if fuzzy diff metric thinks the images are different. @@ -256,6 +287,28 @@ private static ImageDifferenceResultSet getImageDiffResult(JsonObject info) { return result; } + private static ComputeDifferenceResult getComputeDiffResult(JsonObject info) { + + final String comparisonWithReferenceKey = "comparison_with_reference"; + + if (info == null || !info.has(comparisonWithReferenceKey)) { + return ComputeDifferenceResult.DIFFERENT; + } + + final JsonObject comparisonJson = info.get(comparisonWithReferenceKey).getAsJsonObject(); + + if (comparisonJson.has("exact_match") && comparisonJson.get("exact_match").getAsBoolean()) { + return ComputeDifferenceResult.IDENTICAL; + } + + if (comparisonJson.has("fuzzy_match") && comparisonJson.get("fuzzy_match").getAsBoolean()) { + return ComputeDifferenceResult.SIMILAR; + } + + return ComputeDifferenceResult.DIFFERENT; + + } + private enum ReductionStatus { NOREDUCTION, ONGOING, FINISHED, NOTINTERESTING, EXCEPTION, INCOMPLETE } @@ -325,23 +378,23 @@ private List getLiveWorkers(boolean includeInactive) throws TExcepti return workers; } - // Get list of all shaderset directories - private List getAllShadersets(HttpServletRequest request, HttpServletResponse response) + // Get list of all shader family directories + private List getAllShaderFamilies(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - List shadersets = new ArrayList<>(); - File shadersetDir = new File(WebUiConstants.SHADERSET_DIR); - if (!shadersetDir.isDirectory()) { + List shaderFamilies = new ArrayList<>(); + File shaderFamiliesDir = new File(WebUiConstants.SHADER_FAMILIES_DIR); + if (!shaderFamiliesDir.isDirectory()) { err404(request, response); - return shadersets; + return shaderFamilies; } - File[] shadersetFiles = shadersetDir.listFiles(); - for (File shaderset : shadersetFiles) { - if (shaderset.isDirectory()) { - shadersets.add(shaderset); + File[] shaderFamilyFiles = shaderFamiliesDir.listFiles(); + for (File shaderFamily : shaderFamilyFiles) { + if (shaderFamily.isDirectory()) { + shaderFamilies.add(shaderFamily); } } - shadersets.sort(Comparator.naturalOrder()); - return shadersets; + shaderFamilies.sort(Comparator.naturalOrder()); + return shaderFamilies; } // Homepage: /webui @@ -409,21 +462,20 @@ private void homepage(HttpServletRequest request, HttpServletResponse response) } htmlAppendLn(""); - //List of shadersets + // List of shader families htmlAppendLn( "

\n", "

Shader Families

\n", "
\n"); - //StringBuilder shadersetListHTML = new StringBuilder(); - List shadersets = getAllShadersets(request, response); - if (shadersets.size() > 0) { - for (File file : shadersets) { - Shaderset shaderset = new Shaderset(file.getName()); - htmlAppendLn("", + List shaderFamilies = getAllShaderFamilies(request, response); + if (shaderFamilies.size() > 0) { + for (File file : shaderFamilies) { + ShaderFamily shaderFamily = new ShaderFamily(file.getName()); + htmlAppendLn("", "Reference image preview", - "
", shaderset.name, - "
#variants: ", Integer.toString(shaderset.nbVariants), + shaderFamily.preview.getPath(), "' onerror=\"this.style.display='none'\">", + "
", shaderFamily.name, + "
#variants: ", Integer.toString(shaderFamily.nbVariants), "
"); } } @@ -564,34 +616,36 @@ private void worker(HttpServletRequest request, HttpServletResponse response) for (File shaderFamilyFile : shaderFamilies) { final String shaderFamily = shaderFamilyFile.getName(); - ShadersetExp shadersetExp = new ShadersetExp(shaderFamily, workerName, accessFileInfo); + ShaderFamilyResult shaderFamilyResult = new ShaderFamilyResult(shaderFamily, workerName, + accessFileInfo); - // TODO(360): Show results for compute shaders. For now, just indicate that some results - // exists, and point to documentation. - if (shadersetExp.shaderset.isComp) { + if (shaderFamilyResult.shaderFamily.isCompute) { htmlAppendLn( - "", - "
", - "
Compute shader family: ", shaderFamily, "
", - "The Web interface does not support showing results for compute shaders", - " yet, click to open documentation.", + "
", + "COMPUTE", + "
", shaderFamily, "
", + "Variant done: ", Integer.toString(shaderFamilyResult.nbVariantDone), + " / ", Integer.toString(shaderFamilyResult.nbVariants), + " | Wrong results: ", Integer.toString(shaderFamilyResult.nbWrongComputeResult), + " | Slightly different results: ", + Integer.toString(shaderFamilyResult.nbSlightlyDifferentComputeResult), + " | Errors: ", Integer.toString(shaderFamilyResult.nbErrors), + "
"); + } else { + htmlAppendLn( + "", + "", + "
", shaderFamily, "
", + "Variant done: ", Integer.toString(shaderFamilyResult.nbVariantDone), + " / ", Integer.toString(shaderFamilyResult.nbVariants), + " | Wrong images: ", Integer.toString(shaderFamilyResult.nbWrongImage), + " | Slightly different images: ", Integer.toString( + shaderFamilyResult.nbSlightlyDifferentImage), + " | Errors: ", Integer.toString(shaderFamilyResult.nbErrors), + " | Metrics disagree: ", Integer.toString(shaderFamilyResult.nbMetricsDisagree), "
"); - continue; } - - htmlAppendLn( - "", - "", - "
", shaderFamily, "
", - "Variant done: ", Integer.toString(shadersetExp.nbVariantDone), - " / ", Integer.toString(shadersetExp.nbVariants), - " | Wrong images: ", Integer.toString(shadersetExp.nbWrongImage), - " | Slightly different images: ", Integer.toString(shadersetExp.nbSlightlyDifferentImage), - " | Errors: ", Integer.toString(shadersetExp.nbErrors), - " | Metrics disagree: ", Integer.toString(shadersetExp.nbMetricsDisagree), - "
"); } htmlAppendLn("
"); @@ -725,10 +779,10 @@ private void experimentSetup(HttpServletRequest request, HttpServletResponse res } } - List shadersets = getAllShadersets(request, response); + List shaderFamilies = getAllShaderFamilies(request, response); htmlAppendLn("

Shader families

"); - if (shadersets.size() == 0) { + if (shaderFamilies.size() == 0) { htmlAppendLn("

No shader families detected

"); } else { htmlAppendLn("
"); int dataNum = 0; - for (File f : shadersets) { + for (File f : shaderFamilies) { htmlAppendLn("
", "
", " - private void shadersetResults(HttpServletRequest request, HttpServletResponse response) + // Results page for a shader family showing results by all workers - + // /webui/shaderset/ + private void shaderFamilyResults(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { response.setContentType("text/html"); @@ -810,7 +865,7 @@ private void viewShader(HttpServletRequest request, HttpServletResponse response shaderPath.append("/").append(path[i]); } File shader = new File(shaderPath.toString()); - String shaderset = shader.getParentFile().getName(); + String shaderFamily = shader.getParentFile().getName(); String shaderName = FilenameUtils.removeExtension(shader.getName()); htmlHeader(shaderName); @@ -850,6 +905,19 @@ private void viewShader(HttpServletRequest request, HttpServletResponse response response.getWriter().println(html); } + private static String posixPath(String path, String... otherParts) { + StringBuilder result = new StringBuilder(path); + for (String part : otherParts) { + result.append("/"); + result.append(part); + } + return result.toString(); + } + + private static File posixPathToFile(String path, String... otherParts) { + return new File(FilenameUtils.separatorsToSystem(posixPath(path, otherParts))); + } + // Page to view the result from a single shader by a worker - /webui/result/ private void viewResult(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException, TException { @@ -865,18 +933,22 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response final String shaderFamily = path[4]; final String variant = path[5]; final String variantDir = - WebUiConstants.WORKER_DIR + "/" + worker + "/" + shaderFamily + "/"; - final String variantFullPathNoExtension = variantDir + variant; - - File infoFile = new File(variantDir, variant + ".info.json"); + posixPath(WebUiConstants.WORKER_DIR, worker, shaderFamily); + final String variantFullPathNoExtension = posixPath(variantDir, variant); + File infoFile = posixPathToFile(variantDir, variant + ".info.json"); if (!infoFile.isFile()) { err404(request, response, "Invalid result path: cannot find corresponding info file"); return; } + final boolean isCompute = + posixPathToFile("shaderfamilies", shaderFamily, variant + ".comp") + .isFile(); + JsonObject info = accessFileInfo.getResultInfo(infoFile); final String status = info.get("status").getAsString(); - String shaderPath = "shaderfamilies/" + shaderFamily + "/" + variant + ".frag"; + final String shaderPath = "shaderfamilies/" + shaderFamily + "/" + variant + "." + + (isCompute ? "comp" : "frag"); htmlHeader("Single result"); htmlAppendLn("

Single result

", @@ -898,76 +970,132 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response "", "
"); - String referencePngPath = - variantFullPathNoExtension.replace(variant, "reference.png"); + if (isCompute) { - htmlAppendLn("

Reference image:

", - ""); + ComputeDifferenceResult computeDiffResult = getComputeDiffResult(info); - String pngPath = variantDir + variant + ".png"; - File pngFile = new File(pngPath); + if (computeDiffResult == ComputeDifferenceResult.IDENTICAL + && status.equals("SUCCESS") + && !variant.equals("reference")) { + htmlAppendLn("

Compute results are identical.

"); + } + + if (computeDiffResult != ComputeDifferenceResult.IDENTICAL + && status.equals("SUCCESS") + && !variant.equals("reference")) { + + String exactDiffOutput = null; + String fuzzyDiffOutput = null; + + final JsonElement comparisonWithReference = + info.get("comparison_with_reference"); + if (comparisonWithReference != null && comparisonWithReference.isJsonObject()) { + { + JsonElement exactDiffElement = comparisonWithReference.getAsJsonObject().get( + "exactdiff_output"); + if (exactDiffElement != null && exactDiffElement.isJsonPrimitive() + && exactDiffElement.getAsJsonPrimitive().isString()) { + exactDiffOutput = exactDiffElement.getAsJsonPrimitive().getAsString(); + } + } + { + JsonElement fuzzyDiffElement = comparisonWithReference.getAsJsonObject().get( + "fuzzydiff_output"); + if (fuzzyDiffElement != null && fuzzyDiffElement.isJsonPrimitive() + && fuzzyDiffElement.getAsJsonPrimitive().isString()) { + fuzzyDiffOutput = fuzzyDiffElement.getAsJsonPrimitive().getAsString(); + } + } + } + + htmlAppendLn( + "Compute output comparison results:", + "
    ", + "
  • ", + "Exact comparison: ", + exactDiffOutput != null ? exactDiffOutput : "No results", + "
  • ", + "
  • ", + "Fuzzy comparison: ", + fuzzyDiffOutput != null ? fuzzyDiffOutput : "No results", + "
  • ", + "
" + ); - if (!variant.equals("reference")) { - if (pngFile.exists()) { - htmlAppendLn("

Result image:

", - ""); } - } - String gifPath = variantDir + variant + ".gif"; - File gifFile = new File(gifPath); - if (gifFile.exists()) { - htmlAppendLn("

Results non-deterministic animation:

", - "", - "

Here are the second-to-last and last renderings:

\n", - " ", - " "); - } - - if (!(pngFile.exists()) && !(gifFile.exists())) { - htmlAppendLn("

No image to display for this result status

"); - } - - htmlAppendLn("
"); - - ImageDifferenceResultSet metricResults = getImageDiffResult(info); - - if (metricResults.summary == ImageDifferenceResult.IDENTICAL - && status.equals("SUCCESS") - && !variant.equals("reference")) { - htmlAppendLn("

Images are identical.

"); - } - - if (metricResults.summary != ImageDifferenceResult.IDENTICAL - && status.equals("SUCCESS") - && !variant.equals("reference")) { - - htmlAppendLn( - "Image comparison metrics:", - "
    ", - "
  • ", - "Summary: ", - metricResults.summary.toString(), - "
  • ", - "
  • ", - "Fuzzy comparison: ", - metricResults.fuzzy != null ? metricResults.fuzzy.toString() : "No results", - "
  • ", - "
  • ", - "Histogram comparison: ", - metricResults.histogram != null - ? ( - metricResults.histogram.toString() - + " (distance: " + metricResults.histogramDistance + ")" - ) - : "No results", - "
  • ", - "
" - ); + } else { + final String referencePngPath = posixPath(variantDir, "reference.png"); + + htmlAppendLn("

Reference image:

", + ""); + + String pngPath = posixPath(variantDir, variant + ".png"); + File pngFile = posixPathToFile(pngPath); + + if (!variant.equals("reference")) { + if (pngFile.exists()) { + htmlAppendLn("

Result image:

", + ""); + } + } + + String gifPath = posixPath(variantDir, variant + ".gif"); + File gifFile = posixPathToFile(gifPath); + if (gifFile.exists()) { + htmlAppendLn("

Results non-deterministic animation:

", + "", + "

Here are the second-to-last and last renderings:

\n", + " ", + " "); + } + + if (!pngFile.exists() && !gifFile.exists()) { + htmlAppendLn("

No image to display for this result status

"); + } + + htmlAppendLn("
"); + + ImageDifferenceResultSet metricResults = getImageDiffResult(info); + + if (metricResults.summary == ImageDifferenceResult.IDENTICAL + && status.equals("SUCCESS") + && !variant.equals("reference")) { + htmlAppendLn("

Images are identical.

"); + } + + if (metricResults.summary != ImageDifferenceResult.IDENTICAL + && status.equals("SUCCESS") + && !variant.equals("reference")) { + + htmlAppendLn( + "Image comparison metrics:", + "
    ", + "
  • ", + "Summary: ", + metricResults.summary.toString(), + "
  • ", + "
  • ", + "Fuzzy comparison: ", + metricResults.fuzzy != null ? metricResults.fuzzy.toString() : "No results", + "
  • ", + "
  • ", + "Histogram comparison: ", + metricResults.histogram != null + ? ( + metricResults.histogram.toString() + + " (distance: " + metricResults.histogramDistance + ")" + ) + : "No results", + "
  • ", + "
" + ); + } + } @@ -981,12 +1109,12 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response "
\n", "

Run log

\n", "\n", "
"); // Get result file - File result = new File(variantFullPathNoExtension); + File result = posixPathToFile(variantFullPathNoExtension); // Information/links for result File referenceRes = new File(result.getParentFile(), "reference.info.json"); @@ -1001,7 +1129,7 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response final ReductionStatus referenceReductionStatus = getReductionStatus(worker, shaderFamily, "reference"); - File referenceShader = new File(WebUiConstants.SHADERSET_DIR + "/" + File referenceShader = new File(WebUiConstants.SHADER_FAMILIES_DIR + "/" + shaderFamily, "reference.frag"); if (referenceReductionStatus == ReductionStatus.FINISHED) { referenceShader = new File( @@ -1010,24 +1138,28 @@ private void viewResult(HttpServletRequest request, HttpServletResponse response ); } - - String reductionHtml = ""; final ReductionStatus reductionStatus = getReductionStatus(worker, shaderFamily, variant); htmlAppendLn("

Reduction status: ", reductionStatus.toString(), "

"); if (reductionStatus == ReductionStatus.NOREDUCTION) { - htmlAppendLn("", - ""); + + if (isCompute) { + htmlAppendLn("

Reductions for compute shaders are not currently supported via the" + + " web UI.

"); + } else { + htmlAppendLn("", + ""); + } } else { htmlAppendLn("

\n", "\n", @@ -1147,8 +1279,8 @@ private void produceDiff(String shader, File reductionDir, File referenceShader) } } - private ReductionStatus getReductionStatus(String worker, String shaderSet, String shader) { - final File reductionDir = ReductionFilesHelper.getReductionDir(worker, shaderSet, shader); + private ReductionStatus getReductionStatus(String worker, String shaderFamily, String shader) { + final File reductionDir = ReductionFilesHelper.getReductionDir(worker, shaderFamily, shader); if (! reductionDir.exists()) { return ReductionStatus.NOREDUCTION; } @@ -1241,17 +1373,17 @@ private void startExperiment(HttpServletRequest request, HttpServletResponse res StringBuilder msg = new StringBuilder(); String[] workers = request.getParameterValues("workercheck"); - String[] shadersets = request.getParameterValues("shadersetcheck"); + String[] shaderFamilies = request.getParameterValues("shadersetcheck"); if (workers == null || workers.length == 0) { msg.append("Select at least one worker"); - } else if (shadersets == null || shadersets.length == 0) { - msg.append("Select at least one shaderset"); + } else if (shaderFamilies == null || shaderFamilies.length == 0) { + msg.append("Select at least one shader family"); } else { //Start experiments for each worker/shader combination, display result in alert for (String worker : workers) { - for (String shaderset : shadersets) { - msg.append("Experiment ").append(shaderset).append(" on worker ").append(worker); + for (String shaderFamily : shaderFamilies) { + msg.append("Experiment ").append(shaderFamily).append(" on worker ").append(worker); List commands = new ArrayList<>(); commands.add("run_shader_family"); commands.add("--server"); @@ -1259,10 +1391,10 @@ private void startExperiment(HttpServletRequest request, HttpServletResponse res commands.add("--worker"); commands.add(worker); commands.add("--output"); - commands.add("processing/" + worker + "/" + shaderset); - commands.add(WebUiConstants.SHADERSET_DIR + "/" + shaderset); - fuzzerServiceManagerProxy.queueCommand("run_shader_family: " + shaderset, commands, - worker,"processing/" + worker + "/" + shaderset + "/command.log"); + commands.add("processing/" + worker + "/" + shaderFamily); + commands.add(WebUiConstants.SHADER_FAMILIES_DIR + "/" + shaderFamily); + fuzzerServiceManagerProxy.queueCommand("run_shader_family: " + shaderFamily, commands, + worker,"processing/" + worker + "/" + shaderFamily + "/command.log"); msg.append(" started successfully!\\n"); } } @@ -1277,7 +1409,7 @@ private void startExperiment(HttpServletRequest request, HttpServletResponse res response.getWriter().println(html); } - // Page for selecting workers/shadersets to compare results - /webui/compareResults + // Page for selecting workers/shader families to compare results - /webui/compareResults private void compareResults(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { @@ -1302,7 +1434,7 @@ private void compareResults(HttpServletRequest request, HttpServletResponse resp response.getWriter().println(html); } - // Page for selecting workers/shadersets to compare results - /webui/compare + // Page for selecting workers/shader families to compare results - /webui/compare private void compareWorkers(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { @@ -1358,11 +1490,11 @@ private void compareWorkers(HttpServletRequest request, HttpServletResponse resp ""); dataNum = 0; - for (File shaderFamily: getAllShadersets(request, response)) { + for (File shaderFamilyFile: getAllShaderFamilies(request, response)) { // TODO(360): Handle compute shaders - Shaderset shaderset = new Shaderset(shaderFamily.getName()); - if (shaderset.isComp) { + ShaderFamily shaderFamily = new ShaderFamily(shaderFamilyFile.getName()); + if (shaderFamily.isCompute) { continue; } @@ -1370,9 +1502,9 @@ private void compareWorkers(HttpServletRequest request, HttpServletResponse resp "
", "", + " value='", shaderFamilyFile.getName(), "'>", "", + shaderFamilyFile.getName(), "", "
"); dataNum += 1; } @@ -1487,7 +1619,7 @@ private void reduce(HttpServletRequest request, HttpServletResponse response) args.add(retryLimit); } final String seed = request.getParameter("seed"); - if (seed != null) { + if (seed != null && seed.length() > 0) { args.add("--seed"); args.add(seed); } @@ -1520,15 +1652,15 @@ private void reduceReference(String shaderJobFilePath, String worker) throws TEx } else { referenceShaderJobFile = shaderJobFile; } - String shaderset = referenceShaderJobFile.getParentFile().getName(); + String shaderFamily = referenceShaderJobFile.getParentFile().getName(); File reductionDir = - new File(WebUiConstants.WORKER_DIR + "/" + worker + "/" + shaderset + "/reductions", + new File(WebUiConstants.WORKER_DIR + "/" + worker + "/" + shaderFamily + "/reductions", "reference"); if (reductionDir.isDirectory()) { return; } File referenceResult = - new File(WebUiConstants.WORKER_DIR + "/" + worker + "/" + shaderset, + new File(WebUiConstants.WORKER_DIR + "/" + worker + "/" + shaderFamily, "reference.info.json"); List args = new ArrayList<>(); args.add("glsl-reduce"); @@ -1545,7 +1677,7 @@ private void reduceReference(String shaderJobFilePath, String worker) throws TEx args.add("--server"); args.add("http://localhost:8080"); fuzzerServiceManagerProxy.queueCommand( - "Reference Reduction: " + shaderset, + "Reference Reduction: " + shaderFamily, args, worker, new File(reductionDir, "command.log").toString()); @@ -1659,7 +1791,7 @@ private void runShader(HttpServletRequest request, HttpServletResponse response) // e.g. shaderfamilies/family1/variant2.json File shader = new File(shaderPath); // e.g. family1 - String shaderset = shader.getParentFile().getName(); + String shaderFamily = shader.getParentFile().getName(); String[] workers = request.getParameterValues("workercheck"); String javascript = getResourceContent("goBack.js"); @@ -1674,11 +1806,11 @@ private void runShader(HttpServletRequest request, HttpServletResponse response) commands.add("--worker"); commands.add(worker); commands.add("--output"); - commands.add("processing/" + worker + "/" + shaderset + "/"); + commands.add("processing/" + worker + "/" + shaderFamily + "/"); try { fuzzerServiceManagerProxy .queueCommand("run_shader_family: " + shaderPath, commands, worker, - "processing/" + worker + "/" + shaderset + "/command.log"); + "processing/" + worker + "/" + shaderFamily + "/command.log"); } catch (TException exception) { err404(request, response, exception.getMessage()); return; @@ -1782,7 +1914,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) } else if (actions[1].equals("experiment")) { experimentSetup(request, response); } else if (actions[1].equals("shaderset")) { - shadersetResults(request, response); + shaderFamilyResults(request, response); } else if (actions[1].equals("shader")) { viewShader(request, response); } else if (actions[1].equals("result")) { @@ -1920,7 +2052,7 @@ private String reductionLabelColor(ReductionStatus reductionStatus) { } private void htmlVariantResultTableCell(File variantInfoFile, String referencePngPath, - ReductionStatus reductionStatus) throws FileNotFoundException { + ReductionStatus reductionStatus, boolean isCompute) throws FileNotFoundException { JsonObject info = accessFileInfo.getResultInfo(variantInfoFile); String status = info.get("status").getAsString(); @@ -1928,44 +2060,87 @@ private void htmlVariantResultTableCell(File variantInfoFile, String referencePn if (status.contentEquals("SUCCESS")) { - ImageDifferenceResult result = getImageDiffResult(info).summary; + if (isCompute) { - if (result == ImageDifferenceResult.IDENTICAL) { - htmlAppendLn("", - ""); + final ComputeDifferenceResult result = getComputeDiffResult(info); + + if (result == ComputeDifferenceResult.IDENTICAL) { + htmlAppendLn("", + "MATCH", + ""); + } else { + String warningClass = WARNING_CLASS_WRONG_RESULT; + switch (result) { + case SIMILAR: + warningClass = WARNING_CLASS_WARN_RESULT; + break; + case DIFFERENT: + warningClass = WARNING_CLASS_WRONG_RESULT; + break; + default: + LOGGER.error("Unrecognized compute difference result: " + result); + } + htmlAppendLn("", + "", + "DIFFERENCE", + "\n", + "
", + reductionStatus.toString(), + "
"); + } } else { - String warningClass = "wrongimg"; - switch (result) { - case SIMILAR: - warningClass = "warnimg"; - break; - case DIFFERENT: - warningClass = "wrongimg"; - break; - case METRICS_DISAGREE: - warningClass = "metricsdisimg"; - break; - default: - LOGGER.error("Unrecognized image difference result: " + result); + + final ImageDifferenceResult result = getImageDiffResult(info).summary; + + if (result == ImageDifferenceResult.IDENTICAL) { + htmlAppendLn("", + ""); + } else { + String warningClass = WARNING_CLASS_WRONG_RESULT; + switch (result) { + case SIMILAR: + warningClass = WARNING_CLASS_WARN_RESULT; + break; + case DIFFERENT: + warningClass = WARNING_CLASS_WRONG_RESULT; + break; + case METRICS_DISAGREE: + warningClass = WARNING_CLASS_METRICS_DISAGREE; + break; + default: + LOGGER.error("Unrecognized image difference result: " + result); + } + htmlAppendLn("", + "", + "\n", + "
", + reductionStatus.toString(), + "
"); } - htmlAppendLn("", - "", - "\n", - "
", - reductionStatus.toString(), - "
"); + } } else if (status.contentEquals("NONDET")) { + // This code applies to image results only. That is OK at present, as we do not check + // nondeterminism for compute results. Should that change, this could will have to be + // re-worked. + htmlAppendLn("", @@ -2117,7 +2292,7 @@ private void htmlReductionForm( "

Random Seed:

", "", "", - "", + "", "", "", "", @@ -2139,48 +2314,49 @@ private void htmlReductionForm( ""); } - private void htmlComparativeTable(String shaderFamily, String[] workers) + private void htmlComparativeTable(String shaderFamilyFilename, String[] workers) throws FileNotFoundException { - Shaderset shaderset = new Shaderset(shaderFamily); - if (shaderset.isComp) { - // TODO(360): Handle compute shaders - htmlAppendLn("

Compute shader family: ", - shaderFamily, ". ", - "The UI does not display results for compute shaders yet.", - " ", - "See the compute shader documentation for more information.", - "

"); - return; - } + final ShaderFamily shaderFamily = new ShaderFamily(shaderFamilyFilename); + + // A filter for all variant shader jobs + final FilenameFilter variantShaderJobFilter = + (dir, name) -> name.startsWith("variant_") && name.endsWith(".json"); htmlAppendLn("\n", ""); - File variantsDir = new File(WebUiConstants.SHADERSET_DIR + "/" + shaderFamily); - File[] variantFragFiles = variantsDir.listFiles(variantFragFilter); - Arrays.sort(variantFragFiles, (f1, f2) -> + final File variantsDir = new File(WebUiConstants.SHADER_FAMILIES_DIR, shaderFamilyFilename); + File[] variantShaderJobFiles = variantsDir.listFiles(variantShaderJobFilter); + if (variantShaderJobFiles == null) { + // If no variant shader job files are found, the array reference will be null; set it to an + // empty array so that we can render a table with references only. + variantShaderJobFiles = new File[0]; + } + + Arrays.sort(variantShaderJobFiles, (f1, f2) -> new AlphanumComparator().compare(f1.getName(), f2.getName())); boolean showWorkerNames = workers.length > 1; + final String extension = shaderFamily.isCompute ? "comp" : "frag"; + // First row: variant names if (showWorkerNames) { htmlAppendLn(""); } htmlAppendLn(""); - for (File f: variantFragFiles) { + for (File f: variantShaderJobFiles) { htmlAppendLn(""); + FilenameUtils.removeExtension(f.getName()), ""); } htmlAppendLn("\n", ""); @@ -2192,12 +2368,10 @@ private void htmlComparativeTable(String shaderFamily, String[] workers) htmlAppendLn(""); } - // Reference result is separate as it doesn't contain a "identical" field, etc - // FIXME: make sure reference result has same format as variants to be able to refactor - String refHref = WebUiConstants.WORKER_DIR + "/" + worker + "/" - + shaderFamily + "/reference"; - File refInfoFile = new File(refHref + ".info.json"); - String refPngPath = refHref + ".png"; + final String refHref = WebUiConstants.WORKER_DIR + "/" + worker + "/" + + shaderFamilyFilename + "/reference"; + final File refInfoFile = new File(refHref + ".info.json"); + final String refPngPath = refHref + ".png"; htmlAppendLn(""); } @@ -2246,15 +2428,15 @@ private void htmlResultColorLegendTable() { "", "", "", - "", + "", "", - "", + "", "", "", "", "", "", - "", + "", "", "", diff --git a/server/src/main/java/com/graphicsfuzz/server/webui/WebUiConstants.java b/server/src/main/java/com/graphicsfuzz/server/webui/WebUiConstants.java index fc9d796ca..d5fe97504 100644 --- a/server/src/main/java/com/graphicsfuzz/server/webui/WebUiConstants.java +++ b/server/src/main/java/com/graphicsfuzz/server/webui/WebUiConstants.java @@ -24,7 +24,7 @@ private WebUiConstants() { static final String FILE_ROUTE = "file"; static final String WORKER_DIR = "processing"; - static final String SHADERSET_DIR = "shaderfamilies"; + static final String SHADER_FAMILIES_DIR = "shaderfamilies"; static final String WORKER_INFO_FILE = "client.json"; static final String COMPUTE_SHADER_DOC_URL = "https://github.com/google/graphicsfuzz/blob/master/docs/glsl-fuzz-walkthrough" diff --git a/server/src/main/resources/public/graphicsfuzz.css b/server/src/main/resources/public/graphicsfuzz.css index a2a129496..8cba5d483 100644 --- a/server/src/main/resources/public/graphicsfuzz.css +++ b/server/src/main/resources/public/graphicsfuzz.css @@ -34,11 +34,11 @@ td.gfz-error { background-color: #ffaaaa; } -td.wrongimg { +td.wrongresult { background-color: #ffff44; } -td.warnimg { +td.warnresult { background-color: #ccccbb; } diff --git a/shaders/src/main/glsl/samples/100/colorgrid_modulo.frag b/shaders/src/main/glsl/samples/100/colorgrid_modulo.frag index 642e99d5e..10978c024 100644 --- a/shaders/src/main/glsl/samples/100/colorgrid_modulo.frag +++ b/shaders/src/main/glsl/samples/100/colorgrid_modulo.frag @@ -18,7 +18,6 @@ precision mediump float; -uniform vec2 injectionSwitch; uniform vec2 resolution; float nb_mod(float limit, float ref) { diff --git a/shaders/src/main/glsl/samples/100/mandelbrot_blurry.frag b/shaders/src/main/glsl/samples/100/mandelbrot_zoom.frag similarity index 81% rename from shaders/src/main/glsl/samples/100/mandelbrot_blurry.frag rename to shaders/src/main/glsl/samples/100/mandelbrot_zoom.frag index 72f105246..4023d98a2 100644 --- a/shaders/src/main/glsl/samples/100/mandelbrot_blurry.frag +++ b/shaders/src/main/glsl/samples/100/mandelbrot_zoom.frag @@ -28,8 +28,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; int iteration = 0; for (int k = 0; k < 1000; k++) { @@ -44,7 +47,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < 1000) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/100/mandelbrot_blurry.json b/shaders/src/main/glsl/samples/100/mandelbrot_zoom.json similarity index 100% rename from shaders/src/main/glsl/samples/100/mandelbrot_blurry.json rename to shaders/src/main/glsl/samples/100/mandelbrot_zoom.json diff --git a/shaders/src/main/glsl/samples/100/bubblesort_flag.frag b/shaders/src/main/glsl/samples/100/stable_bubblesort_flag.frag similarity index 100% rename from shaders/src/main/glsl/samples/100/bubblesort_flag.frag rename to shaders/src/main/glsl/samples/100/stable_bubblesort_flag.frag diff --git a/shaders/src/main/glsl/samples/100/bubblesort_flag.json b/shaders/src/main/glsl/samples/100/stable_bubblesort_flag.json similarity index 100% rename from shaders/src/main/glsl/samples/100/bubblesort_flag.json rename to shaders/src/main/glsl/samples/100/stable_bubblesort_flag.json diff --git a/shaders/src/main/glsl/samples/300es/binarysearch_bw.frag b/shaders/src/main/glsl/samples/300es/binarysearch_bw.frag new file mode 100644 index 000000000..1cef54511 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/binarysearch_bw.frag @@ -0,0 +1,164 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct BinarySearchObject{ + int prime_numbers[10]; +}; + +int FINDMSB(int a) { + int result = -1; + int temp = a; + for (int i = 0; i < 31; i++) { + bool bitIsZero = (temp % 2) == 0; + if (a < 0 && bitIsZero) { + result = i; + } else if (a > 0 && !bitIsZero) { + result = i; + } + temp = temp >> 1; + } + return result; +} + +#define LDEXP(significand, exponent) ((significand)*pow(2.0, exponent)) + +vec2 brick(vec2 uv) { + int a = 4; + do { + uv.y -= step(injectionSwitch.y, uv.x); + uv.x -= fract(tanh(uv.x)) / LDEXP(injectionSwitch.y, float(FINDMSB(a))); + a--; + } while (a > int(injectionSwitch.x)); + int b = 3; + do { + uv.y -= step(injectionSwitch.y, uv.x) + float(a); + uv.x *= (isnan(uv.y) ? cosh(gl_FragCoord.y) : tanh(gl_FragCoord.x)); + b--; + } while (b > int(injectionSwitch.x)); + int c = 2; + do { + uv.y -= step(injectionSwitch.y, uv.x) + float(a) + float(b); + uv.x += LDEXP(injectionSwitch.y, isinf(uv.y + uv.x) ? float(FINDMSB(b)) : float(FINDMSB(a))); + c--; + } while (c > int(injectionSwitch.x)); + int d = 1; + do { + uv.y -= step(injectionSwitch.y, uv.x) + float(a) + float(b) + float(c); + d--; + } while (d > int(injectionSwitch.x)); + return fract(uv); +} + +float patternize(vec2 uv) { + vec2 size = vec2(0.45); + vec2 st = smoothstep(size, size, uv); + switch (int(mod(gl_FragCoord.y, 5.0))) { + case 0: + return mix(pow(st.x, injectionSwitch.y), st.x, size.y); + break; + case 1: + return mix(pow(uv.y, injectionSwitch.y), st.y, size.x); + break; + case 2: + discard; + break; + case 3: + return mix(pow(uv.y, injectionSwitch.y), uv.y, size.y); + break; + case 4: + return mix(pow(st.y, injectionSwitch.y), st.x, size.x); + break; + } +} + +int binarySearch(BinarySearchObject obj, int x) { + int l = 0, r = 9; + while (l <= r) { + int m = (l + r) / 2; + if (obj.prime_numbers[m] == x) { + return m; + } + + if (obj.prime_numbers[m] < x) { + l = m + 1; + } else { + r = m - 1; + } + } + // If an element is not present in the array we return -1. + return -1; +} + +void main() { + BinarySearchObject obj; + // Initialize first 10 prime numbers to the array. + for (int i = 0; i < 10; i++) { + if (i == 0) { + obj.prime_numbers[i] = 2; + } else if (i == 1) { + obj.prime_numbers[i] = 3; + } else if (i == 2) { + obj.prime_numbers[i] = 5; + } else if (i == 3) { + obj.prime_numbers[i] = 7; + } else if (i == 4) { + obj.prime_numbers[i] = 11; + } else if (i == 5) { + obj.prime_numbers[i] = 13; + } else if (i == 6) { + obj.prime_numbers[i] = 17; + } else if (i == 7) { + obj.prime_numbers[i] = 19; + } else if (i == 8) { + obj.prime_numbers[i] = 23; + } else if (i == 9) { + obj.prime_numbers[i] = 29; + } + } + + vec2 uv = (gl_FragCoord.xy / resolution.x) * vec2(resolution.x / resolution.y, 1.0); + vec2 b = brick(uv * 7.0); + vec3 color = vec3(patternize(b)); + + if (gl_FragCoord.y < resolution.y / 1.1) { + // We are going to search the item in array by giving the value of the item index 4 and 0 in an array. + if (binarySearch(obj, obj.prime_numbers[4]) != -(int(resolution.y)) && binarySearch(obj, obj.prime_numbers[0]) >= -(int(resolution.x))) { + color.yz -= dot(float(binarySearch(obj, obj.prime_numbers[4])), float(binarySearch(obj, obj.prime_numbers[0]))); + } else { + discard; + } + } else { + // The following condition is true as there is no value 1 in the array. + if (binarySearch(obj, 1) == -1) { + discard; + } else { + color.yz += color.yz; + } + } + + _GLF_color = vec4(color, injectionSwitch.y); + +} diff --git a/shaders/src/main/glsl/samples/300es/bubblesort_flag.json b/shaders/src/main/glsl/samples/300es/binarysearch_bw.json similarity index 100% rename from shaders/src/main/glsl/samples/300es/bubblesort_flag.json rename to shaders/src/main/glsl/samples/300es/binarysearch_bw.json diff --git a/shaders/src/main/glsl/samples/300es/binarysearch_tree.frag b/shaders/src/main/glsl/samples/300es/binarysearch_tree.frag new file mode 100644 index 000000000..944fb705c --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/binarysearch_tree.frag @@ -0,0 +1,176 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct BST{ + int data; + int leftIndex; + int rightIndex; +}; + +BST tree[10]; + +void makeTreeNode(inout BST tree, int data) +{ + tree.data = data; + tree.leftIndex = -1; + tree.rightIndex = -1; +} + +void insert(int treeIndex, int data) +{ + int baseIndex = 0; + while (baseIndex <= treeIndex) { + // If new value is smaller thatn the current node, we known that we will have + // add this element in the left side. + if (data <= tree[baseIndex].data) { + // If a left subtree of the current node is empty, the new node is added as + // a left subtree of the current node. + if (tree[baseIndex].leftIndex == -1) { + tree[baseIndex].leftIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].leftIndex; + continue; + } + } else { + // If a right subtree of the current node is empty, the new node is added as + // a right subtree of the current node. + if (tree[baseIndex].rightIndex == -1) { + tree[baseIndex].rightIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].rightIndex; + continue; + } + } + } +} + +// Return element data if the given target exists in a tree. Otherwise, we simply return -1. +int search(int target){ + BST currentNode; + int index = 0; + while (index != -1) { + currentNode = tree[index]; + if (currentNode.data == target) { + return target; + } + index = target > currentNode.data ? currentNode.rightIndex : currentNode.leftIndex; + } + return -1; +} + +vec3 hueColor(float angle) { + float nodeData = float(search(15)); + vec3 color; + color = clamp(fract(angle * vec3(1.0, 5.0, nodeData)), 0.0, 1.0); + color.x *= cosh(isnan(float(search(30))) ? injectionSwitch.x : injectionSwitch.y); + return color; +} + +float makeFrame(float v) { + v *= 6.5; + if (v < 1.5) { + return float(search(100)); + } + if (v < 4.0) { + return injectionSwitch.x; + } + if (v < float(search(6))) { + return 1.0; + } + return 10.0 + float(search(30)); +} + +/* +* This shader implements binary search tree using an array data structure. The elements of +* tree are kept in the array that contains a list of BST object holding indices of left and +* right subtree in the array. +* +* - Tree representation of the number used in this shader: +* 9 +* / \ +* 5 12 +* / \ \ +* 2 7 15 +* / \ / \ +* 6 8 13 17 +* +* - Array representation: +* [9, 5, 12, 15, 7, 8, 2, 6, 17, 13] +* +*/ + +void main() { + int treeIndex = int(injectionSwitch.x); + // Initialize root node. + makeTreeNode(tree[int(injectionSwitch.x)], 9); + // Each time we insert a new node into the tree, we increment one. + treeIndex++; + + insert(treeIndex, 5); + treeIndex++; + insert(treeIndex, 12); + treeIndex++; + insert(treeIndex, 15); + treeIndex++; + insert(treeIndex, 7); + treeIndex++; + insert(treeIndex, 8); + treeIndex++; + insert(treeIndex, 2); + treeIndex++; + insert(treeIndex, 6); + treeIndex++; + insert(treeIndex, 17); + treeIndex++; + insert(treeIndex, 13); + + vec2 z = (gl_FragCoord.yx / resolution); + float x = makeFrame(z.x); + float y = makeFrame(z.y); + + int sum = -100; + for (int target = 0; target < 20; target ++) { + int result = search(target); + if (result > 0) { + sum += result; + } else { + switch (result) { + case -1: + sum += int(injectionSwitch.y); + break; + case 0: + return; + } + } + } + float a = tan(x + y * float(sum)); + _GLF_color = vec4(hueColor(a), 1.); + +} diff --git a/shaders/src/main/glsl/samples/300es/binarysearch_tree.json b/shaders/src/main/glsl/samples/300es/binarysearch_tree.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/binarysearch_tree.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/colorgrid_modulo.frag b/shaders/src/main/glsl/samples/300es/colorgrid_modulo.frag index 5567f1af9..f2487d549 100644 --- a/shaders/src/main/glsl/samples/300es/colorgrid_modulo.frag +++ b/shaders/src/main/glsl/samples/300es/colorgrid_modulo.frag @@ -20,7 +20,6 @@ precision highp float; layout(location = 0) out vec4 _GLF_color; -uniform vec2 injectionSwitch; uniform vec2 resolution; float nb_mod(float limit, float ref) { diff --git a/shaders/src/main/glsl/samples/300es/householder_lattice.frag b/shaders/src/main/glsl/samples/300es/householder_lattice.frag new file mode 100644 index 000000000..e061b06c1 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/householder_lattice.frag @@ -0,0 +1,119 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +int MATRIX_N = 4; + +uniform mat4 matrix_a_uni; + +void main() +{ + // We need to modify A in place, so we have to copy the uniform + mat4 matrix_a = mat4(matrix_a_uni); + vec4 matrix_b = gl_FragCoord.wxyz; + + vec4 matrix_u = vec4(0.0); + + float magnitudeX = 0.0; + float alpha1 = 0.0; + float alpha2 = 0.0; + float alpha3 = 0.0; + float beta = 0.0; + // QR factorization + for(int k = 0; k < MATRIX_N - 1; k++) + { + // define vector x as the column A(k) only with entries k - n + // creating our U vector at the same time we create X magnitude. + for(int x = MATRIX_N - 1; x >= k; x--) // start from the bottom and go up. + { + magnitudeX += pow(matrix_a[x][k], 2.0); + matrix_u[x] = matrix_a[x][k]; + } + magnitudeX = sqrt(magnitudeX); + // entry k is the top of our U vector for this iteration + // define U as x - sign(x1) (magnitude X) (elementary matrix 1) + matrix_u[k] -= (sign(matrix_u[k]) * magnitudeX); + + // alpha1 = U(T) * U gives us the dot product + for(int u = MATRIX_N - 1; u >= k; u--) + { + alpha1 += pow(matrix_u[u], 2.0); + } + + // alpha2 = 2.0 / (U(T) * U) = 2.0 / alpha1 + alpha2 = 2.0 / alpha1; + + for(int j = k; j < MATRIX_N; j++) + { + // evaluate alpha3 = U(T) * a(j) updated k - 1 times + for(int a = MATRIX_N - 1; a >= k; a--) + { + alpha3 += matrix_u[a] * matrix_a[a][j]; + } + + beta = alpha2 * alpha3; + + // evaluate a(j) updated k times = a(j) updated k - 1 times - (beta * U) + for(int a = MATRIX_N - 1; a >= k; a--) + { + matrix_a[a][j] = matrix_a[a][j] - (beta * matrix_u[a]); + } + alpha3 = 0.0; + beta = 0.0; + } + + // evaluate alpha3 = U(T) * a(j) updated k - 1 times + for(int b = MATRIX_N - 1; b >= k; b--) + { + alpha3 += matrix_u[b] * matrix_b[b]; + } + + // evaluate beta = alpha2 * alpha3 + beta = alpha2 * alpha3; + + // evaluate b(j) updated k times = b(j) updated k - 1 times - (beta * U) + for(int b = MATRIX_N - 1; b >= k; b--) + { + matrix_b[b] = matrix_b[b] - (beta * matrix_u[b]); + } + + magnitudeX = 0.0; + alpha1 = 0.0; + alpha2 = 0.0; + alpha3 = 0.0; + beta = 0.0; + } + + // back substitution algorithm + for(int i = (MATRIX_N - 1); i >= 0; i--) + { + for(int j = (MATRIX_N - 1); j >= (i + 1); j--) + { + matrix_b[i] -= (matrix_a[i][j] * matrix_b[j]); + } + + // final operation, using the diagonal; matrix_b becomes the solution matrix x + matrix_b[i] /= matrix_a[i][i]; + } + _GLF_color = tan(matrix_b); + _GLF_color.a = 1.0; +} + diff --git a/shaders/src/main/glsl/samples/300es/householder_lattice.json b/shaders/src/main/glsl/samples/300es/householder_lattice.json new file mode 100644 index 000000000..51d6c7a30 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/householder_lattice.json @@ -0,0 +1,11 @@ +{ + "matrix_a_uni": { + "func": "glUniformMatrix4fv", + "args": [ + 1.0, 5.0, 3.0, 7.0, + 9.0, 6.0, 7.0, 8.0, + 1.0, 2.0, 3.0, 4.0, + 5.0, 6.0, 7.0, 8.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.frag b/shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.frag new file mode 100644 index 000000000..ee2237e0d --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.frag @@ -0,0 +1,76 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +vec3 pickColor(int i) { + return vec3(float(i) / 50.0, float(i) / 120.0, float(i) / 140.0); +} + +vec3 mand(float xCoord, float yCoord) +{ + int xpos = int(xCoord) * 256; + int ypos = int(yCoord) * 256; + int height = int(resolution.y) * 256; + int width = int(resolution.x) * 256; + + int c_re = ((xpos - width / 2) * 819) / width - 102; + int c_im = ((ypos - height / 2) * 819) / width; + + int x = 0, y = 0; + int iteration = 0; + for (int k = 0; k < 1000; k++) + { + if (x * x + y * y > 262144) + { + break; + } + int x_new = ((x * x - y * y) / 256 + c_re); + y = (2 * x * y / 256 + c_im); + x = x_new; + iteration++; + } + if (iteration < 1000) + { + return pickColor(iteration); + } + else + { + return vec3(0.0, 0.0, 0.5); + } +} + +void main() { + vec3 data[16]; + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + data[4*j + i] = mand(gl_FragCoord.x + float(i - 1), gl_FragCoord.y + float(j - 1)); + } + } + vec3 sum = vec3(0.0); + for (int i = 0; i < 16; i++) { + sum += data[i]; + } + sum /= vec3(16.0); + _GLF_color = vec4(sum, 1.0); +} + diff --git a/shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.json b/shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.json new file mode 100644 index 000000000..28c3c3e61 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/mandelbrot_fixed_point.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/mandelbrot_blurry.frag b/shaders/src/main/glsl/samples/300es/mandelbrot_zoom.frag similarity index 81% rename from shaders/src/main/glsl/samples/300es/mandelbrot_blurry.frag rename to shaders/src/main/glsl/samples/300es/mandelbrot_zoom.frag index 99cc3c753..477a673c3 100644 --- a/shaders/src/main/glsl/samples/300es/mandelbrot_blurry.frag +++ b/shaders/src/main/glsl/samples/300es/mandelbrot_zoom.frag @@ -30,8 +30,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; int iteration = 0; for (int k = 0; k < 1000; k++) { @@ -46,7 +49,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < 1000) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/300es/mandelbrot_blurry.json b/shaders/src/main/glsl/samples/300es/mandelbrot_zoom.json similarity index 100% rename from shaders/src/main/glsl/samples/300es/mandelbrot_blurry.json rename to shaders/src/main/glsl/samples/300es/mandelbrot_zoom.json diff --git a/shaders/src/main/glsl/samples/300es/mergesort_mosaic.frag b/shaders/src/main/glsl/samples/300es/mergesort_mosaic.frag new file mode 100644 index 000000000..8afcebd6c --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/mergesort_mosaic.frag @@ -0,0 +1,169 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +// Size of an array. +const int N = 10; +// An array and its temperary array whose elements will be sorted. +int data[10], temp[10]; + +// Merge two sorted subarrays data[from ... mid] and data[mid+1 ... to]. +void merge(int from, int mid, int to) { + int k = from, i = from, j = mid +1; + + while (i <= mid && j <= to) { + if (data[i] < data[j]) { + temp[k++] = data[i++]; + } else { + temp[k++] = data[j++]; + } + } + // Copy remaining elements. + while (i < N && i <= mid) { + temp[k++] = data[i++]; + } + // Copy back to the original array. + for (int i = from; i <= to; i++) { + data[i] = temp[i]; + } +} + +// Sort array using the iterative approach. +void mergeSort() { + int low = 0; + int high = N - 1; + + // Devide the array into blocks of size m. + // m = [1, 2, 4 ,8, 16...]. + for (int m = 1; m <= high; m = 2 * m) { + + // For m = 1, i = [0, 2, 4, 6, 8]. + // For m = 2, i = [0, 4, 8]. + // For m = 4, i = [0, 8]. + for (int i = low; i< high; i += 2 * m) { + int from = i; + int mid = i + m - 1; + int to = min (i + 2 * m - 1, high); + merge(from, mid, to); + } + } +} + +#define LDEXP(significand, exponent) ((significand)*pow(2.0, exponent)) + +void main() { + int i = int(injectionSwitch.x); + do { + switch(i) { + case 0: + data[i] = 4; + break; + case 1: + data[i] = 3 ; + break; + case 2: + data[i] = 2 ; + break; + case 3: + data[i] = 1 ; + break; + case 4: + data[i] = 0 ; + break; + case 5: + data[i] = -1; + break; + case 6: + data[i] = -2; + break; + case 7: + data[i] = -3; + break; + case 8: + data[i] = -4; + break; + case 9: + data[i] = -5; + break; + } + i++; + } while (i < 10); + + for (int j = 0; j < 10; j++) { + temp[j] = data[j]; + } + mergeSort(); + + mat3 pos = mat3( + vec3(4, 0, int(injectionSwitch.y)) * LDEXP(0.5, 2.0), + vec3(0, -5, int(injectionSwitch.y)) * LDEXP(0.2, 5.0), + vec3(1, 8, int(injectionSwitch.y)) * LDEXP(injectionSwitch.y, 0.0) + ); + vec3 vecCoor = roundEven(pos * vec3(gl_FragCoord.xx / resolution.yx, 1)); + vec2 color; + + do { + if (int(gl_FragCoord[1]) < 30) { + color = fract(sin(vecCoor.yx - trunc(float(data[0])))); + color[0] = dFdy(gl_FragCoord.y); + break; + } else if (int(gl_FragCoord[1]) < 60) { + color = fract(sin(vecCoor.yx - trunc(float(data[1])))); + color[1] *= atan(faceforward(injectionSwitch, color.xx, vecCoor.yx).y); + break; + } else if (int(gl_FragCoord[1]) < 90) { + color = fract(sin(vecCoor.yx - trunc(float(data[2])))); + color.x += atanh(color.x) * cosh(injectionSwitch.y) * smoothstep(color, injectionSwitch, gl_FragCoord.yy).x; + break; + } else if (int(gl_FragCoord[1]) < 120) { + color = fract(acosh(clamp(vecCoor.yx - trunc(float(data[3])), 1.0, 1000.0))); + color.x += (isnan(gl_FragCoord.x) ? log2(gl_FragCoord.x) : log2(gl_FragCoord.y)); + break; + } else if (int(gl_FragCoord[1]) < 150) { + discard; + } else if (int(gl_FragCoord[1]) < 180) { + color = fract(sin(vecCoor.yx - trunc(float(data[4])))); + color[1] += asinh(gl_FragCoord.y * LDEXP(color.y, float(-i))); + break; + } else if (int(gl_FragCoord[1]) < 210) { + color = fract(sin(vecCoor.yx - trunc(float(data[5])))); + color.y -= tanh(color.x) / cosh(color.y); + break; + } else if (int(gl_FragCoord[1]) < 240) { + color = fract(asinh(vecCoor.yx - trunc(float(data[6])))); + color.y -= isnan(float(i)) ? tanh(gl_FragCoord.x): atanh(gl_FragCoord.y); + break; + } else if (int(gl_FragCoord[1]) < 270) { + color = fract(sin(vecCoor.yx - trunc(float(data[7])))); + color.y *= mix(normalize(vecCoor), normalize(vec3(color, degrees(color.x))), injectionSwitch.x).y; + break; + } else { + discard; + } + } while(0 == int(injectionSwitch.x)); + + _GLF_color = vec4(vec3(color, trunc(injectionSwitch.y)), injectionSwitch.y); + +} diff --git a/shaders/src/main/glsl/samples/310es/bubblesort_flag.json b/shaders/src/main/glsl/samples/300es/mergesort_mosaic.json similarity index 100% rename from shaders/src/main/glsl/samples/310es/bubblesort_flag.json rename to shaders/src/main/glsl/samples/300es/mergesort_mosaic.json diff --git a/shaders/src/main/glsl/samples/300es/prefix_sum_checkers.frag b/shaders/src/main/glsl/samples/300es/prefix_sum_checkers.frag new file mode 100644 index 000000000..36043ed57 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/prefix_sum_checkers.frag @@ -0,0 +1,95 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +vec2 pattern(in vec2 x) { + vec2 n = floor(x); + vec3 m = vec3(1.0); + + for (int j = -1; j <= int(injectionSwitch.y); j++) { + for (int i = -1; i <= int(injectionSwitch.y); i++) { + vec2 g = vec2(float(j), float(i)); + vec2 o = mix(n, g, 0.2); + if (injectionSwitch.x < (m.x)) { + int k = 1; + while (k >= 0) { + o = o + o; + k--; + } + m = vec3(injectionSwitch.x, cos(o)); + } + } + } + return vec2(m.x, m.y - m.z); +} + +void main() { + vec2 uv = gl_FragCoord.xy / resolution.y; + float A[50]; + for (int i = 0; i < 200; i++) { + if (i >= int(resolution.x)) { + break; + } + if ((4 * (i/4)) == i) { + A[i/4] = float(i); + } + } + for (int i = 0; i < 50; i++) { + if (i < int(gl_FragCoord.x)) { + break; + } + if (i > 0) { + A[i] += A[i - 1]; + } + } + + vec2 c = pattern((15.0 + tan(0.2)) * uv); + vec3 col; + if (int(gl_FragCoord.y) < 20) { + col = .5 + cos(c.y + vec3(resolution.x, A[4]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 40) { + col = .5 + cos(c.y + vec3(resolution.x, A[9]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 60) { + col = .5 + cos(c.y + vec3(resolution.x, A[14]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 80) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 100) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 120) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 140) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 160) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 180) { + col = .5 + cos(c.y + vec3(resolution.x, A[44]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 200) { + col = .5 + cos(c.y + vec3(resolution.x, A[49]/resolution.x + 50.0, 22.0)); + } else { + discard; + } + + _GLF_color = vec4(col, 1.0); +} diff --git a/shaders/src/main/glsl/samples/300es/prefix_sum_checkers.json b/shaders/src/main/glsl/samples/300es/prefix_sum_checkers.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/prefix_sum_checkers.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/quicksort_palette.frag b/shaders/src/main/glsl/samples/300es/quicksort_palette.frag new file mode 100644 index 000000000..a33448293 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/quicksort_palette.frag @@ -0,0 +1,131 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct QuicksortObject{ + int numbers[10]; +}; + +QuicksortObject obj; + + void swap(int i, int j) { + int temp = obj.numbers[i]; + obj.numbers[i] = obj.numbers[j]; + obj.numbers[j] = temp; + } + + // Since "partition" is the preserved word, we add prefix to this function name to prevent an error. + int performPartition(int l, int h) { + // The rightmost element is chosen as a pivot. + int pivot = obj.numbers[h]; + int i = (l - 1); + + for (int j = l; j <= h - 1; j++) { + if (obj.numbers[j] <= pivot) { + i++; + swap(i, j); + } + } + swap(i + 1, h); + return (i + 1); + } + + void quicksort() { + int l = 0, h = 9; + int stack[10]; + int top = -1; + + stack[++top] = l; + stack[++top] = h; + + while (top >= 0) { + h = stack[top--]; + l = stack[top--]; + + int p = performPartition(l, h); + if (p - 1 > l) { + stack[++top] = l; + stack[++top] = p - 1; + } + if (p + 1 < h) { + stack[++top] = p + 1; + stack[++top] = h; + } + } + } + +vec3 palette(vec3 a, vec3 b, vec3 c, vec3 d) { + return fract(mix(d * a, a * c, b + d - c)); +} + +float randomize(vec2 co) +{ + return float(floor(fract(sin(dot(co.xy ,vec2(12.5, 3.))) * 4250. + 0.02) + 0.5)); +} + +bool puzzlelize(vec2 pos) +{ + return randomize(pos.xy) < .12; +} + +void main() { + // Initialize decreasing values to an array starting from 10. + for (int i = int(injectionSwitch.x); i < 10; i ++) { + obj.numbers[i] = (10 - i) * int(injectionSwitch.y); + } + quicksort(); + vec2 grid = vec2(20, 20); + vec2 uv = gl_FragCoord.xy / resolution; + vec3 color = palette(vec3(float(obj.numbers[4]) * 0.1), vec3(0.9, float(obj.numbers[8]) * 0.1, 0.8), trunc(vec3(injectionSwitch.y)), vec3(injectionSwitch.x, 0.3, 0.7)); + if (uv.x > (1.0 / 4.0)) { + int count = int(injectionSwitch.x); + do { + color += palette(vec3(0.5, float(obj.numbers[8]) * 0.1, 0.2), vec3(0.5), trunc(vec3(injectionSwitch.y)), vec3(float(obj.numbers[4]) * 0.1, injectionSwitch.x, 0.6)); + count++; + } while (count != obj.numbers[int(injectionSwitch.x)]); + grid = vec2(count + obj.numbers[8], count + obj.numbers[6]); + } + if (uv.x > (2.0 / 4.0)) { + int count = int(injectionSwitch.x); + do { + color -= palette(vec3(float(obj.numbers[4]) * 0.1), trunc(vec3(0.1)), vec3(float(obj.numbers[int(injectionSwitch.y)]) * 0.1), vec3(injectionSwitch.x, float(obj.numbers[2]) * 0.1 , float(obj.numbers[8]) * 0.1)); + count++; + } while (count != obj.numbers[1]); + grid += vec2(count + int(injectionSwitch.y), count + int(injectionSwitch.x)); + } + if(uv.x > (3.0 / 4.0)) { + int count = int(injectionSwitch.x); + do { + color -= palette(vec3(float(obj.numbers[int(injectionSwitch.y)]) * 0.1), vec3(0.8), trunc(vec3(0.1)), vec3(injectionSwitch.x, 0.6, float(obj.numbers[int(injectionSwitch.x)]) * 0.1)); + count++; + } while (count != obj.numbers[2]); + grid += vec2(count + obj.numbers[3], count + obj.numbers[3]); + } + + vec2 position = vec2(gl_FragCoord.x, resolution.x - gl_FragCoord.y); + position = floor(position / grid); + _GLF_color = vec4(color, injectionSwitch.y) + vec4(!puzzlelize(position)); + +} diff --git a/shaders/src/main/glsl/samples/donors/bubblesort_flag.json b/shaders/src/main/glsl/samples/300es/quicksort_palette.json similarity index 100% rename from shaders/src/main/glsl/samples/donors/bubblesort_flag.json rename to shaders/src/main/glsl/samples/300es/quicksort_palette.json diff --git a/shaders/src/main/glsl/samples/300es/selection_sort_struct.frag b/shaders/src/main/glsl/samples/300es/selection_sort_struct.frag new file mode 100644 index 000000000..b833868b1 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/selection_sort_struct.frag @@ -0,0 +1,73 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +struct Obj { + float odd_numbers[10]; + float even_numbers[10]; +}; + +void main() { + Obj obj; + + // Initialize first 10 odd numbers to the array. + int odd_index = 0; + float odd_number = 1.0; + while (odd_index <= 9) { + obj.odd_numbers[odd_index] = odd_number; + odd_number += 2.0; + odd_index++; + } + // Similarly, initialize even numbers and iterate backward. + int even_index = 9; + float even_number = 0.0; + while (even_index >= 0) { + obj.even_numbers[even_index] = even_number; + even_number += 2.; + even_index--; + } + + // Perform the selection sort. + for (int i = 0; i < 9; i++) { + int index = i; + for (int j = i + 1; j < 10; j++) { + if (obj.even_numbers[j] < obj.even_numbers[index]) { + index = j; + } + } + float smaller_number = obj.even_numbers[index]; + obj.even_numbers[index] = obj.even_numbers[i]; + obj.even_numbers[i] = smaller_number; + } + + vec2 uv = gl_FragCoord.xy/resolution.xy; + vec3 col = tan(pow(uv.xxx, uv.yyy) + + vec3( + obj.odd_numbers[int(floor(gl_FragCoord.x/1000.0))], + obj.even_numbers[int(floor(gl_FragCoord.y/1000.0))], + sinh(uv.x) + )); + + _GLF_color.rgb = col; + _GLF_color.a = 1.0; +} diff --git a/shaders/src/main/glsl/samples/300es/selection_sort_struct.json b/shaders/src/main/glsl/samples/300es/selection_sort_struct.json new file mode 100644 index 000000000..269b946d9 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/selection_sort_struct.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/300es/squares.json b/shaders/src/main/glsl/samples/300es/squares.json index bc2d1fb1b..fdc0723fa 100644 --- a/shaders/src/main/glsl/samples/300es/squares.json +++ b/shaders/src/main/glsl/samples/300es/squares.json @@ -5,13 +5,6 @@ 0.0 ] }, - "mouse": { - "func": "glUniform2f", - "args": [ - 0.0, - 0.0 - ] - }, "resolution": { "func": "glUniform2f", "args": [ diff --git a/shaders/src/main/glsl/samples/300es/bubblesort_flag.frag b/shaders/src/main/glsl/samples/300es/stable_bubblesort_flag.frag similarity index 100% rename from shaders/src/main/glsl/samples/300es/bubblesort_flag.frag rename to shaders/src/main/glsl/samples/300es/stable_bubblesort_flag.frag diff --git a/shaders/src/main/glsl/samples/webgl1/bubblesort_flag.json b/shaders/src/main/glsl/samples/300es/stable_bubblesort_flag.json similarity index 100% rename from shaders/src/main/glsl/samples/webgl1/bubblesort_flag.json rename to shaders/src/main/glsl/samples/300es/stable_bubblesort_flag.json diff --git a/shaders/src/main/glsl/samples/300es/trigonometric_strip.frag b/shaders/src/main/glsl/samples/300es/trigonometric_strip.frag new file mode 100644 index 000000000..53d7e47f9 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/trigonometric_strip.frag @@ -0,0 +1,74 @@ +#version 300 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +float compute_derivative_x(float v){ + return dFdx(v) * injectionSwitch.y; +} + +float compute_derivative_y(float v){ + return dFdy(v) * injectionSwitch.y; +} + +float compute_stripe(float v) { + return smoothstep(-.9, 1., (v)/ ((injectionSwitch.y > injectionSwitch.x) ? compute_derivative_x(v): compute_derivative_y(v))); +} + +void main() { + vec2 uv = gl_FragCoord.xy / resolution.x; + vec3 col = vec3(0, 0, 0); + + bool c1 = uv.y < 0.25; + if (c1) { + float stripe = compute_stripe(cos((uv.x + uv.y ) * 20.0 )); + col = mix(vec3(uv.x, 0, 0.75), vec3(.8, .7, uv.x), stripe); + _GLF_color = vec4(col, 1.0); + return; + } + + bool c2 = uv.y < 0.5; + if (!c1 && c2) { + float stripe = compute_stripe(tan((uv.x + uv.y) * 20.0 )); + col = mix(vec3(0.5, uv.x, 0.1), vec3(.4, 0, .5), stripe); + _GLF_color = vec4(col, 1.0); + return; + } + + bool c3 = uv.y < 0.75; + if (!c1 && !c2 && c3) { + float stripe = compute_stripe(cos((uv.x + uv.y) * 20.0 )); + col = mix(vec3(.7, sinh(uv.x), uv.x ), vec3(.3, .5, uv.x), stripe); + _GLF_color = vec4(col, 1.0); + return; + } + + bool c4 = uv.y >= 0.75; + if (!c1 && !c2 && !c3 && c4) { + float stripe = compute_stripe(tan((uv.x + uv.y) * 20.0 )); + col = mix(vec3(uv.x, .8, 0), vec3(1, uv.y, 0), stripe); + _GLF_color = vec4(col, 1.0); + return; + } +} diff --git a/shaders/src/main/glsl/samples/300es/trigonometric_strip.json b/shaders/src/main/glsl/samples/300es/trigonometric_strip.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/300es/trigonometric_strip.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_bw.frag b/shaders/src/main/glsl/samples/310es/binarysearch_bw.frag new file mode 100644 index 000000000..1368e7a5b --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/binarysearch_bw.frag @@ -0,0 +1,147 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct BinarySearchObject{ + int prime_numbers[10]; +}; + +vec2 brick(vec2 uv) { + int a = 4; + do { + uv.y -= step(injectionSwitch.y, uv.x); + uv.x -= fract(tanh(uv.x)) / ldexp(injectionSwitch.y, findMSB(a)); + a--; + } while (a > int(injectionSwitch.x)); + int b = 3; + do { + uv.y -= step(injectionSwitch.y, uv.x) + float(a); + uv.x *= (isnan(uv.y) ? cosh(gl_FragCoord.y) : tanh(gl_FragCoord.x)); + b--; + } while (b > int(injectionSwitch.x)); + int c = 2; + do { + uv.y -= step(injectionSwitch.y, uv.x) + float(a) + float(b); + uv.x += ldexp(injectionSwitch.y, isinf(uv.y + uv.x) ? findMSB(b) : findMSB(a)); + c--; + } while (c > int(injectionSwitch.x)); + int d = 1; + do { + uv.y -= step(injectionSwitch.y, uv.x) + float(a) + float(b) + float(c); + d--; + } while (d > int(injectionSwitch.x)); + return fract(uv); +} + +float patternize(vec2 uv) { + vec2 size = vec2(0.45); + vec2 st = smoothstep(size, size, uv); + switch (int(mod(gl_FragCoord.y, 5.0))) { + case 0: + return mix(pow(st.x, injectionSwitch.y), st.x, size.y); + break; + case 1: + return mix(pow(uv.y, injectionSwitch.y), st.y, size.x); + break; + case 2: + discard; + break; + case 3: + return mix(pow(uv.y, injectionSwitch.y), uv.y, size.y); + break; + case 4: + return mix(pow(st.y, injectionSwitch.y), st.x, size.x); + break; + } +} + +int binarySearch(BinarySearchObject obj, int x) { + int l = 0, r = 9; + while (l <= r) { + int m = (l + r) / 2; + if (obj.prime_numbers[m] == x) { + return m; + } + + if (obj.prime_numbers[m] < x) { + l = m + 1; + } else { + r = m - 1; + } + } + // If an element is not present in the array we return -1. + return -1; +} + +void main() { + BinarySearchObject obj; + // Initialize first 10 prime numbers to the array. + for (int i = 0; i < 10; i++) { + if (i == 0) { + obj.prime_numbers[i] = 2; + } else if (i == 1) { + obj.prime_numbers[i] = 3; + } else if (i == 2) { + obj.prime_numbers[i] = 5; + } else if (i == 3) { + obj.prime_numbers[i] = 7; + } else if (i == 4) { + obj.prime_numbers[i] = 11; + } else if (i == 5) { + obj.prime_numbers[i] = 13; + } else if (i == 6) { + obj.prime_numbers[i] = 17; + } else if (i == 7) { + obj.prime_numbers[i] = 19; + } else if (i == 8) { + obj.prime_numbers[i] = 23; + } else if (i == 9) { + obj.prime_numbers[i] = 29; + } + } + + vec2 uv = (gl_FragCoord.xy / resolution.x) * vec2(resolution.x / resolution.y, 1.0); + vec2 b = brick(uv * 7.0); + vec3 color = vec3(patternize(b)); + + if (gl_FragCoord.y < resolution.y / 1.1) { + // We are going to search the item in array by giving the value of the item index 4 and 0 in an array. + if (binarySearch(obj, obj.prime_numbers[4]) != -(int(resolution.y)) && binarySearch(obj, obj.prime_numbers[0]) >= -(int(resolution.x))) { + color.yz -= dot(float(binarySearch(obj, obj.prime_numbers[4])), float(binarySearch(obj, obj.prime_numbers[0]))); + } else { + discard; + } + } else { + // The following condition is true as there is no value 1 in the array. + if (binarySearch(obj, 1) == -1) { + discard; + } else { + color.yz += color.yz; + } + } + + _GLF_color = vec4(color, injectionSwitch.y); + +} diff --git a/shaders/src/main/glsl/samples/webgl2/bubblesort_flag.json b/shaders/src/main/glsl/samples/310es/binarysearch_bw.json similarity index 100% rename from shaders/src/main/glsl/samples/webgl2/bubblesort_flag.json rename to shaders/src/main/glsl/samples/310es/binarysearch_bw.json diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag b/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag new file mode 100644 index 000000000..47a58490c --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/binarysearch_tree.frag @@ -0,0 +1,174 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +struct BST{ + int data; + int leftIndex; + int rightIndex; +}; + +BST tree[10]; + +void makeTreeNode(inout BST tree, int data) +{ + tree.data = data; + tree.leftIndex = -1; + tree.rightIndex = -1; +} + +void insert(int treeIndex, int data) +{ + int baseIndex = 0; + while (baseIndex <= treeIndex) { + // If new value is smaller thatn the current node, we known that we will have + // add this element in the left side. + if (data <= tree[baseIndex].data) { + // If a left subtree of the current node is empty, the new node is added as + // a left subtree of the current node. + if (tree[baseIndex].leftIndex == -1) { + tree[baseIndex].leftIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].leftIndex; + continue; + } + } else { + // If a right subtree of the current node is empty, the new node is added as + // a right subtree of the current node. + if (tree[baseIndex].rightIndex == -1) { + tree[baseIndex].rightIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].rightIndex; + continue; + } + } + } +} + +// Return element data if the given target exists in a tree. Otherwise, we simply return -1. +int search(int target){ + BST currentNode; + int index = 0; + while (index != -1) { + currentNode = tree[index]; + if (currentNode.data == target) { + return target; + } + index = target > currentNode.data ? currentNode.rightIndex : currentNode.leftIndex; + } + return -1; +} + +vec3 hueColor(float angle) { + float nodeData = float(search(15)); + vec3 color; + color = clamp(fract(angle * vec3(1.0, 5.0, nodeData)), 0.0, 1.0); + color.x *= cosh(isnan(float(search(30))) ? 0.0 : 1.0); + return color; +} + +float makeFrame(float v) { + v *= 6.5; + if (v < 1.5) { + return float(search(100)); + } + if (v < 4.0) { + return 0.0; + } + if (v < float(search(6))) { + return 1.0; + } + return 10.0 + float(search(30)); +} + +/* +* This shader implements binary search tree using an array data structure. The elements of +* tree are kept in the array that contains a list of BST object holding indices of left and +* right subtree in the array. +* +* - Tree representation of the number used in this shader: +* 9 +* / \ +* 5 12 +* / \ \ +* 2 7 15 +* / \ / \ +* 6 8 13 17 +* +* - Array representation: +* [9, 5, 12, 15, 7, 8, 2, 6, 17, 13] +* +*/ + +void main() { + int treeIndex = 0; + // Initialize root node. + makeTreeNode(tree[0], 9); + // Each time we insert a new node into the tree, we increment one. + treeIndex++; + + insert(treeIndex, 5); + treeIndex++; + insert(treeIndex, 12); + treeIndex++; + insert(treeIndex, 15); + treeIndex++; + insert(treeIndex, 7); + treeIndex++; + insert(treeIndex, 8); + treeIndex++; + insert(treeIndex, 2); + treeIndex++; + insert(treeIndex, 6); + treeIndex++; + insert(treeIndex, 17); + treeIndex++; + insert(treeIndex, 13); + + vec2 z = (gl_FragCoord.yx / resolution); + float x = makeFrame(z.x); + float y = makeFrame(z.y); + + int sum = -100; + for (int target = 0; target < 20; target ++) { + int result = search(target); + if (result > 0) { + sum += result; + } else { + switch (result) { + case -1: + sum += 1; + break; + case 0: + return; + } + } + } + float a = tan(x + y * float(sum)); + _GLF_color = vec4(hueColor(a), 1.); + +} diff --git a/shaders/src/main/glsl/samples/310es/binarysearch_tree.json b/shaders/src/main/glsl/samples/310es/binarysearch_tree.json new file mode 100644 index 000000000..269b946d9 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/binarysearch_tree.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.frag b/shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.frag new file mode 100644 index 000000000..b6af4ef77 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.frag @@ -0,0 +1,70 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; +precision highp int; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +bool checkSwap(float a, float b) +{ + return gl_FragCoord.y < resolution.y / 2.0 ? a > b : a < b; +} +void main() +{ + int msb10 = 1024; + int msb9 = 512; + uint uselessOutVariable; + float data[10]; + for(int i = bitfieldReverse(int(injectionSwitch.x)); i < findMSB(msb10); i++) + { + data[i] = float(usubBorrow(uint(10), uint(i), uselessOutVariable)) * injectionSwitch.y; + } + int i = bitfieldExtract(int(injectionSwitch.x), bitCount(0), int(injectionSwitch.x)); + do + { + for(int j = bitfieldExtract(int(injectionSwitch.x), 0, 0); j < findLSB(msb10); j++) + { + if(uint(j) < uaddCarry(uint(i), 1u, uselessOutVariable)) + { + continue; + } + bool doSwap = checkSwap(data[i], data[j]); + if(doSwap) + { + float temp = data[i]; + data[i] = data[j]; + data[j] = temp; + } + } + i++; + } while(i < findMSB(msb9)); + if(gl_FragCoord.x < resolution.x / 2.0) + { + _GLF_color = vec4(data[findMSB(1)] / 10.0, data[findLSB(32)] / 10.0, data[findMSB(msb9)] / 10.0, 1.0); + } + else + { + _GLF_color = vec4(data[findLSB(32)] / 10.0, data[findMSB(msb9)] / 10.0, data[findMSB(1)] / 10.0, 1.0); + } +} + diff --git a/shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.json b/shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/bubblesort_flag_intfunctions.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/colorgrid_modulo.frag b/shaders/src/main/glsl/samples/310es/colorgrid_modulo.frag index 8416a648b..4961cb25c 100644 --- a/shaders/src/main/glsl/samples/310es/colorgrid_modulo.frag +++ b/shaders/src/main/glsl/samples/310es/colorgrid_modulo.frag @@ -20,7 +20,6 @@ precision highp float; layout(location = 0) out vec4 _GLF_color; -uniform vec2 injectionSwitch; uniform vec2 resolution; float nb_mod(float limit, float ref) { diff --git a/shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.frag b/shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.frag new file mode 100644 index 000000000..e16d678ff --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.frag @@ -0,0 +1,65 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; +precision highp int; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +int msb8 = 256; + +float nb_mod(float limit, float ref) { + int msb8 = 256; + float s = float(bitfieldExtract(int(resolution.x), 0, 0)); + int i = bitCount(msb8); + while(i < bitfieldInsert(800, i, 0, 0)) { + if (mod(float(i), ref) <= 0.01) { + s += 0.2; + } + if (float(i) >= limit) { + return s; + } + i++; + } + return s; +} + +void main() +{ + int msb8 = 256; + vec4 c = vec4(bitfieldExtract(0, 0, 0), 0.0, 0.0, bitCount(msb8)); + float ref = floor(resolution.x / float(findLSB(msb8))); + + c.x = nb_mod(gl_FragCoord.x, ref); + c.y = nb_mod(gl_FragCoord.y, ref); + c.z = c.x + c.y; + int i = bitfieldReverse(bitfieldExtract(0, 0, 0)); + do { + if (c[i] >= 1.0) { + c[i] = c[i] * c[i]; + } + i++; + } while (i < findMSB(findLSB(msb8))); + c.x = mod(c.x, 1.0); + c.y = mod(c.y, 1.0); + c.z = mod(c.z, 1.0); + _GLF_color = c; +} + diff --git a/shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.json b/shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.json new file mode 100644 index 000000000..269b946d9 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/colorgrid_modulo_intfunctions.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/householder_lattice.frag b/shaders/src/main/glsl/samples/310es/householder_lattice.frag new file mode 100644 index 000000000..b7c36184f --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/householder_lattice.frag @@ -0,0 +1,119 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +int MATRIX_N = 4; + +uniform mat4 matrix_a_uni; + +void main() +{ + // We need to modify A in place, so we have to copy the uniform + mat4 matrix_a = mat4(matrix_a_uni); + vec4 matrix_b = gl_FragCoord.wxyz; + + vec4 matrix_u = vec4(0.0); + + float magnitudeX = 0.0; + float alpha1 = 0.0; + float alpha2 = 0.0; + float alpha3 = 0.0; + float beta = 0.0; + // QR factorization + for(int k = 0; k < MATRIX_N - 1; k++) + { + // define vector x as the column A(k) only with entries k - n + // creating our U vector at the same time we create X magnitude. + for(int x = MATRIX_N - 1; x >= k; x--) // start from the bottom and go up. + { + magnitudeX += pow(matrix_a[x][k], 2.0); + matrix_u[x] = matrix_a[x][k]; + } + magnitudeX = sqrt(magnitudeX); + // entry k is the top of our U vector for this iteration + // define U as x - sign(x1) (magnitude X) (elementary matrix 1) + matrix_u[k] -= (sign(matrix_u[k]) * magnitudeX); + + // alpha1 = U(T) * U gives us the dot product + for(int u = MATRIX_N - 1; u >= k; u--) + { + alpha1 += pow(matrix_u[u], 2.0); + } + + // alpha2 = 2.0 / (U(T) * U) = 2.0 / alpha1 + alpha2 = 2.0 / alpha1; + + for(int j = k; j < MATRIX_N; j++) + { + // evaluate alpha3 = U(T) * a(j) updated k - 1 times + for(int a = MATRIX_N - 1; a >= k; a--) + { + alpha3 += matrix_u[a] * matrix_a[a][j]; + } + + beta = alpha2 * alpha3; + + // evaluate a(j) updated k times = a(j) updated k - 1 times - (beta * U) + for(int a = MATRIX_N - 1; a >= k; a--) + { + matrix_a[a][j] = matrix_a[a][j] - (beta * matrix_u[a]); + } + alpha3 = 0.0; + beta = 0.0; + } + + // evaluate alpha3 = U(T) * a(j) updated k - 1 times + for(int b = MATRIX_N - 1; b >= k; b--) + { + alpha3 += matrix_u[b] * matrix_b[b]; + } + + // evaluate beta = alpha2 * alpha3 + beta = alpha2 * alpha3; + + // evaluate b(j) updated k times = b(j) updated k - 1 times - (beta * U) + for(int b = MATRIX_N - 1; b >= k; b--) + { + matrix_b[b] = matrix_b[b] - (beta * matrix_u[b]); + } + + magnitudeX = 0.0; + alpha1 = 0.0; + alpha2 = 0.0; + alpha3 = 0.0; + beta = 0.0; + } + + // back substitution algorithm + for(int i = (MATRIX_N - 1); i >= 0; i--) + { + for(int j = (MATRIX_N - 1); j >= (i + 1); j--) + { + matrix_b[i] -= (matrix_a[i][j] * matrix_b[j]); + } + + // final operation, using the diagonal; matrix_b becomes the solution matrix x + matrix_b[i] /= matrix_a[i][i]; + } + _GLF_color = tan(matrix_b); + _GLF_color.a = 1.0; +} + diff --git a/shaders/src/main/glsl/samples/310es/householder_lattice.json b/shaders/src/main/glsl/samples/310es/householder_lattice.json new file mode 100644 index 000000000..51d6c7a30 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/householder_lattice.json @@ -0,0 +1,11 @@ +{ + "matrix_a_uni": { + "func": "glUniformMatrix4fv", + "args": [ + 1.0, 5.0, 3.0, 7.0, + 9.0, 6.0, 7.0, 8.0, + 1.0, 2.0, 3.0, 4.0, + 5.0, 6.0, 7.0, 8.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.frag b/shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.frag new file mode 100644 index 000000000..636df8f2e --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.frag @@ -0,0 +1,76 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +vec3 pickColor(int i) { + return vec3(float(i) / 50.0, float(i) / 120.0, float(i) / 140.0); +} + +vec3 mand(float xCoord, float yCoord) +{ + int xpos = int(xCoord) * 256; + int ypos = int(yCoord) * 256; + int height = int(resolution.y) * 256; + int width = int(resolution.x) * 256; + + int c_re = ((xpos - width / 2) * 819) / width - 102; + int c_im = ((ypos - height / 2) * 819) / width; + + int x = 0, y = 0; + int iteration = 0; + for (int k = 0; k < 1000; k++) + { + if (x * x + y * y > 262144) + { + break; + } + int x_new = ((x * x - y * y) / 256 + c_re); + y = (2 * x * y / 256 + c_im); + x = x_new; + iteration++; + } + if (iteration < 1000) + { + return pickColor(iteration); + } + else + { + return vec3(0.0, 0.0, 0.5); + } +} + +void main() { + vec3 data[16]; + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + data[4*j + i] = mand(gl_FragCoord.x + float(i - 1), gl_FragCoord.y + float(j - 1)); + } + } + vec3 sum = vec3(0.0); + for (int i = 0; i < 16; i++) { + sum += data[i]; + } + sum /= vec3(16.0); + _GLF_color = vec4(sum, 1.0); +} + diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.json b/shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.json new file mode 100644 index 000000000..28c3c3e61 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/mandelbrot_fixed_point.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_blurry.frag b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom.frag similarity index 81% rename from shaders/src/main/glsl/samples/310es/mandelbrot_blurry.frag rename to shaders/src/main/glsl/samples/310es/mandelbrot_zoom.frag index 00329aeb0..e1e10ab7a 100644 --- a/shaders/src/main/glsl/samples/310es/mandelbrot_blurry.frag +++ b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom.frag @@ -30,8 +30,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; int iteration = 0; for (int k = 0; k < 1000; k++) { @@ -46,7 +49,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < 1000) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_blurry.json b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom.json similarity index 100% rename from shaders/src/main/glsl/samples/310es/mandelbrot_blurry.json rename to shaders/src/main/glsl/samples/310es/mandelbrot_zoom.json diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_zoom_intfunctions.frag b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom_intfunctions.frag new file mode 100644 index 000000000..2f4464d08 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom_intfunctions.frag @@ -0,0 +1,85 @@ +#version 310 es + +/* + * Copyright 2018 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; +precision highp int; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +vec3 pickColor(int i) { + return vec3(float(i) / 50.0, float(i) / 120.0, float(i) / 140.0); +} + +vec3 mand(float xCoord, float yCoord) { + float height = resolution.y; + float width = resolution.x; + + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; + float x = 0.0, y = 0.0; + if (0.0 > resolution.x) + { + x = 1.0; + y = 1.0; + } + int iteration = bitfieldReverse(int(x)); + int k = bitfieldExtract(int(y), bitCount(int(x)), int(y)); + int iterationCap = 1000; + do { + if (x*x+y*y > 4.0) { + break; + } + float x_new = x*x - y*y + c_re; + y = 2.0*x*y + c_im; + x = x_new; + iteration++; + k++; + } while(k < bitfieldInsert(iterationCap + (257.0 > resolution.y ? 1 : 0), 0, 0, 0)); + if (0.0 > resolution.y) + { + iterationCap += 1; + } + if (iteration < bitfieldInsert(iterationCap, 0, 0, 0)) { + return pickColor(iteration); + } else { + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); + } +} + +void main() { + int msb16 = 65536; + uint uselessOutVariable; + vec3 data[16]; + for (int i = 0; i < findMSB(16); i++) { + for (int j = 0; j < findLSB(16); j++) { + data[uaddCarry(uint(4*j), uint(i), uselessOutVariable)] = mand(gl_FragCoord.x + float(i - bitCount(1)), gl_FragCoord.y + float(j - bitCount(1))); + } + } + vec3 sum = vec3(0.0); + for (int i = bitfieldReverse(0); i < findMSB(msb16); i++) { + sum += data[i]; + } + sum /= vec3(16.0); + _GLF_color = vec4(sum, 1.0); +} + diff --git a/shaders/src/main/glsl/samples/310es/mandelbrot_zoom_intfunctions.json b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom_intfunctions.json new file mode 100644 index 000000000..28c3c3e61 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/mandelbrot_zoom_intfunctions.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/matrices_many_loops.frag b/shaders/src/main/glsl/samples/310es/matrices_many_loops.frag new file mode 100644 index 000000000..8dfda73f2 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/matrices_many_loops.frag @@ -0,0 +1,99 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform float one; + +uniform vec2 resolution; + +#define POPULATE_MAT(MAT, COLS, ROWS) \ + for (int c = 0; c < COLS; c++) { \ + for (int r = 0; r < ROWS; r++) { \ + MAT[c][r] = one; \ + } \ + } + +// The following macro populates sums[INDEX] to be the sum of all the +// entries in MAT (which must be a COLS x ROWS matrix) divided by +// 16x.0. + +#define POPULATE_SUMS(MAT, COLS, ROWS, INDEX) \ + sums[INDEX] = 0.0; \ + for (int c = 0; c < COLS; c++) { \ + for (int r = 0; r < ROWS; r++) { \ + sums[INDEX] += MAT[c][r]; \ + } \ + } \ + sums[INDEX] /= 16.0; + +mat2x2 m22; + +mat2x3 m23; + +mat2x4 m24; + +mat3x2 m32; + +mat3x3 m33; + +mat3x4 m34; + +mat4x2 m42; + +mat4x3 m43; + +mat4x4 m44; + +void main() +{ + + POPULATE_MAT(m22, 2, 2); + POPULATE_MAT(m23, 2, 3); + POPULATE_MAT(m24, 2, 4); + POPULATE_MAT(m32, 3, 2); + POPULATE_MAT(m33, 3, 3); + POPULATE_MAT(m34, 3, 4); + POPULATE_MAT(m42, 4, 2); + POPULATE_MAT(m43, 4, 3); + POPULATE_MAT(m44, 4, 4); + + float sums[9]; + + POPULATE_SUMS(m22, 2, 2, 0); + POPULATE_SUMS(m23, 2, 3, 1); + POPULATE_SUMS(m24, 2, 4, 2); + POPULATE_SUMS(m32, 3, 2, 3); + POPULATE_SUMS(m33, 3, 3, 4); + POPULATE_SUMS(m34, 3, 4, 5); + POPULATE_SUMS(m42, 4, 2, 6); + POPULATE_SUMS(m43, 4, 3, 7); + POPULATE_SUMS(m44, 4, 4, 8); + + int region_x = int(gl_FragCoord.x / (resolution.x / 3.0)); + int region_y = int(gl_FragCoord.y / (resolution.x / 3.0)); + int overall_region = region_y * 3 + region_x; + + if (overall_region > 0 && overall_region < 9) { + _GLF_color = vec4(vec3(sums[overall_region]), 1.0); + } else { + _GLF_color = vec4(vec3(0.0), 1.0); + } +} diff --git a/shaders/src/main/glsl/samples/310es/matrices_many_loops.json b/shaders/src/main/glsl/samples/310es/matrices_many_loops.json new file mode 100644 index 000000000..0ad587e72 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/matrices_many_loops.json @@ -0,0 +1,14 @@ +{ + "one": { + "func": "glUniform1f", + "args": [ + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/matrices_smart_loops.frag b/shaders/src/main/glsl/samples/310es/matrices_smart_loops.frag new file mode 100644 index 000000000..cf6dcbb91 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/matrices_smart_loops.frag @@ -0,0 +1,150 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform float one; + +uniform vec2 resolution; + +mat2x2 m22; + +mat2x3 m23; + +mat2x4 m24; + +mat3x2 m32; + +mat3x3 m33; + +mat3x4 m34; + +mat4x2 m42; + +mat4x3 m43; + +mat4x4 m44; + +void main() +{ + uint matrix_number = 0u; + for(int cols = 2; cols <= 4; cols++) + { + for(int rows = 2; rows <= 4; rows++) + { + for(int c = 0; c < cols; c++) + { + for(int r = 0; r < rows; r++) + { + switch(matrix_number) + { + case 0u: + m22[c][r] = one; + break; + case 1u: + m23[c][r] = one; + break; + case 2u: + m24[c][r] = one; + break; + case 3u: + m32[c][r] = one; + break; + case 4u: + m33[c][r] = one; + break; + case 5u: + m34[c][r] = one; + break; + case 6u: + m42[c][r] = one; + break; + case 7u: + m43[c][r] = one; + break; + case 8u: + m44[c][r] = one; + break; + } + } + } + matrix_number = matrix_number + 1u; + } + } + + float sums[9]; + + int sum_index = 0; + for(int cols = 2; cols <= 4; cols++) + { + for(int rows = 2; rows <= 4; rows++) + { + sums[sum_index] = 0.0; + for(int c = 0; c < cols; c++) + { + for(int r = 0; r < rows; r++) + { + switch(sum_index) + { + case 0: + sums[sum_index] += m22[c][r]; + break; + case 1: + sums[sum_index] += m23[c][r]; + break; + case 2: + sums[sum_index] += m24[c][r]; + break; + case 3: + sums[sum_index] += m32[c][r]; + break; + case 4: + sums[sum_index] += m33[c][r]; + break; + case 5: + sums[sum_index] += m34[c][r]; + break; + case 6: + sums[sum_index] += m42[c][r]; + break; + case 7: + sums[sum_index] += m43[c][r]; + break; + case 8: + sums[sum_index] += m44[c][r]; + break; + } + } + } + sums[sum_index] /= 16.0; + sum_index ++; + } + } + + int region_x = int(gl_FragCoord.x / (resolution.x / 3.0)); + int region_y = int(gl_FragCoord.y / (resolution.x / 3.0)); + int overall_region = region_y * 3 + region_x; + + if (overall_region > 0 && overall_region < 9) { + _GLF_color = vec4(vec3(sums[overall_region]), 1.0); + } else { + _GLF_color = vec4(vec3(0.0), 1.0); + } +} diff --git a/shaders/src/main/glsl/samples/310es/matrices_smart_loops.json b/shaders/src/main/glsl/samples/310es/matrices_smart_loops.json new file mode 100644 index 000000000..0ad587e72 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/matrices_smart_loops.json @@ -0,0 +1,14 @@ +{ + "one": { + "func": "glUniform1f", + "args": [ + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag b/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag new file mode 100644 index 000000000..2ce17a2ee --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/mergesort_mosaic.frag @@ -0,0 +1,167 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +// Size of an array. +const int N = 10; +// An array and its temperary array whose elements will be sorted. +int data[10], temp[10]; + +// Merge two sorted subarrays data[from ... mid] and data[mid+1 ... to]. +void merge(int from, int mid, int to) { + int k = from, i = from, j = mid +1; + + while (i <= mid && j <= to) { + if (data[i] < data[j]) { + temp[k++] = data[i++]; + } else { + temp[k++] = data[j++]; + } + } + // Copy remaining elements. + while (i < N && i <= mid) { + temp[k++] = data[i++]; + } + // Copy back to the original array. + for (int i = from; i <= to; i++) { + data[i] = temp[i]; + } +} + +// Sort array using the iterative approach. +void mergeSort() { + int low = 0; + int high = N - 1; + + // Devide the array into blocks of size m. + // m = [1, 2, 4 ,8, 16...]. + for (int m = 1; m <= high; m = 2 * m) { + + // For m = 1, i = [0, 2, 4, 6, 8]. + // For m = 2, i = [0, 4, 8]. + // For m = 4, i = [0, 8]. + for (int i = low; i< high; i += 2 * m) { + int from = i; + int mid = i + m - 1; + int to = min (i + 2 * m - 1, high); + merge(from, mid, to); + } + } +} + +void main() { + int i = int(injectionSwitch.x); + do { + switch(i) { + case 0: + data[i] = 4; + break; + case 1: + data[i] = 3; + break; + case 2: + data[i] = 2; + break; + case 3: + data[i] = 1; + break; + case 4: + data[i] = 0; + break; + case 5: + data[i] = -1; + break; + case 6: + data[i] = -2; + break; + case 7: + data[i] = -3; + break; + case 8: + data[i] = -4; + break; + case 9: + data[i] = -5; + break; + } + i++; + } while (i < 10); + + for (int j = 0; j < 10; j++) { + temp[j] = data[j]; + } + mergeSort(); + + mat3 pos = mat3( + vec3(4, 0, int(injectionSwitch.y)) * ldexp(0.5, 2), + vec3(0, -5, int(injectionSwitch.y)) * ldexp(0.2, 5), + vec3(1, 8, int(injectionSwitch.y)) * ldexp(injectionSwitch.y, 0) + ); + vec3 vecCoor = roundEven(pos * vec3(gl_FragCoord.xx / resolution.yx, 1)); + vec2 color; + + do { + if (int(gl_FragCoord[1]) < 30) { + color = fract(sin(vecCoor.yx - trunc(float(data[0])))); + color[0] = dFdy(gl_FragCoord.y); + break; + } else if (int(gl_FragCoord[1]) < 60) { + color = fract(sin(vecCoor.yx - trunc(float(data[1])))); + color[1] *= atan(faceforward(injectionSwitch, color.xx, vecCoor.yx).y); + break; + } else if (int(gl_FragCoord[1]) < 90) { + color = fract(sin(vecCoor.yx - trunc(float(data[2])))); + color.x += atanh(color.x) * cosh(injectionSwitch.y) * smoothstep(color, injectionSwitch, gl_FragCoord.yy).x; + break; + } else if (int(gl_FragCoord[1]) < 120) { + color = fract(acosh(clamp(vecCoor.yx - trunc(float(data[3])), 1.0, 1000.0))); + color.x += (isnan(gl_FragCoord.x) ? log2(gl_FragCoord.x) : log2(gl_FragCoord.y)); + break; + } else if (int(gl_FragCoord[1]) < 150) { + discard; + } else if (int(gl_FragCoord[1]) < 180) { + color = fract(sin(vecCoor.yx - trunc(float(data[4])))); + color[1] += asinh(gl_FragCoord.y * ldexp(color.y, -i)); + break; + } else if (int(gl_FragCoord[1]) < 210) { + color = fract(sin(vecCoor.yx - trunc(float(data[5])))); + color.y -= tanh(color.x) / cosh(color.y); + break; + } else if (int(gl_FragCoord[1]) < 240) { + color = fract(asinh(vecCoor.yx - trunc(float(data[6])))); + color.y -= isnan(float(i)) ? tanh(gl_FragCoord.x): atanh(gl_FragCoord.y); + break; + } else if (int(gl_FragCoord[1]) < 270) { + color = fract(sin(vecCoor.yx - trunc(float(data[7])))); + color.y *= mix(normalize(vecCoor), normalize(vec3(color, degrees(color.x))), injectionSwitch.x).y; + break; + } else { + discard; + } + } while(0 == int(injectionSwitch.x)); + + _GLF_color = vec4(vec3(color, trunc(injectionSwitch.y)), injectionSwitch.y); + +} diff --git a/shaders/src/main/glsl/samples/310es/mergesort_mosaic.json b/shaders/src/main/glsl/samples/310es/mergesort_mosaic.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/mergesort_mosaic.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/muller.frag b/shaders/src/main/glsl/samples/310es/muller.frag new file mode 100644 index 000000000..72c48900c --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/muller.frag @@ -0,0 +1,76 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; +precision highp int; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec3 polynomial; + +uniform vec3 initial_xvalues; + +float fx(float x) +{ + return polynomial.x * pow(x, 2.0) + polynomial.y * x + polynomial.z; +} + +void main() +{ + // n + float x2 = initial_xvalues.x; + // n - 1 + float x1 = initial_xvalues.y; + // n - 2 + float x0 = initial_xvalues.z; + // temp storage + float temp = 0.0; + // constants in the quadratic formula + // calculated with derived formulas for Muller's method + float A = 0.0; + float B = 0.0; + float C = 0.0; + + while(abs(x2 - x1) >= .000000000000001) + { + float h0 = x0 - x2; + float h1 = x1 - x2; + float k0 = fx(x0) - fx(x2); + float k1 = fx(x1) - fx(x2); + temp = x2; + A = (((h1) * (k0)) - ((h0) * (k1))) / + ((pow((h0), 2.0) * (h1)) - (pow((h1), 2.0) * (h0))); + B = (((k0) - (A * (pow((h0), 2.0)))) / (h0)); + C = fx(x2); + + // get the new root + x2 = x2 - ((2.0 * C) / (B + sign(B) * sqrt(pow(B, 2.0) - (4.0 * A * C)))); + + // move up in the sequence. + x0 = x1; + x1 = temp; // x1 = original x2; + } + if(x2 <= -0.9 && x2 >= -1.1) + { + _GLF_color = vec4(1.0, 0.0, 0.0, 1.0); + } + else + { + _GLF_color = vec4(0.0, 1.0, 0.0, 1.0); + } +} diff --git a/shaders/src/main/glsl/samples/310es/muller.json b/shaders/src/main/glsl/samples/310es/muller.json new file mode 100644 index 000000000..1bf27217a --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/muller.json @@ -0,0 +1,18 @@ +{ + "polynomial": { + "func": "glUniform3f", + "args": [ + 1.0, + 3.0, + 2.0 + ] + }, + "initial_xvalues": { + "func": "glUniform3f", + "args": [ + 5.0, + 3.0, + 1.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/prefix_sum_checkers.frag b/shaders/src/main/glsl/samples/310es/prefix_sum_checkers.frag new file mode 100644 index 000000000..05858c552 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/prefix_sum_checkers.frag @@ -0,0 +1,95 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +vec2 pattern(in vec2 x) { + vec2 n = floor(x); + vec3 m = vec3(1.0); + + for (int j = -1; j <= int(injectionSwitch.y); j++) { + for (int i = -1; i <= int(injectionSwitch.y); i++) { + vec2 g = vec2(float(j), float(i)); + vec2 o = mix(n, g, 0.2); + if (injectionSwitch.x < (m.x)) { + int k = 1; + while (k >= 0) { + o = o + o; + k--; + } + m = vec3(injectionSwitch.x, cos(o)); + } + } + } + return vec2(m.x, m.y - m.z); +} + +void main() { + vec2 uv = gl_FragCoord.xy / resolution.y; + float A[50]; + for (int i = 0; i < 200; i++) { + if (i >= int(resolution.x)) { + break; + } + if ((4 * (i/4)) == i) { + A[i/4] = float(i); + } + } + for (int i = 0; i < 50; i++) { + if (i < int(gl_FragCoord.x)) { + break; + } + if (i > 0) { + A[i] += A[i - 1]; + } + } + + vec2 c = pattern((15.0 + tan(0.2)) * uv); + vec3 col; + if (int(gl_FragCoord.y) < 20) { + col = .5 + cos(c.y + vec3(resolution.x, A[4]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 40) { + col = .5 + cos(c.y + vec3(resolution.x, A[9]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 60) { + col = .5 + cos(c.y + vec3(resolution.x, A[14]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 80) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 100) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 120) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 140) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 160) { + col = .5 + cos(c.y + vec3(resolution.x, A[39]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 180) { + col = .5 + cos(c.y + vec3(resolution.x, A[44]/resolution.x + 50.0, 22.0)); + } else if (int(gl_FragCoord.y) < 200) { + col = .5 + cos(c.y + vec3(resolution.x, A[49]/resolution.x + 50.0, 22.0)); + } else { + discard; + } + + _GLF_color = vec4(col, 1.0); +} diff --git a/shaders/src/main/glsl/samples/310es/prefix_sum_checkers.json b/shaders/src/main/glsl/samples/310es/prefix_sum_checkers.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/prefix_sum_checkers.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.frag b/shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.frag new file mode 100644 index 000000000..0eeca69ef --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.frag @@ -0,0 +1,84 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; +precision highp int; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +void main(void) { + int msb9 = 512; + int msb10 = 1024; + int msb14 = 16384; + int msb15 = 32768; + int msb19 = 524288; + int msb20 = 1048576; + int msb24 = 16777216; + int msb25 = 33554432; + int msb29 = 536870912; + int msb30 = 1073741824; + uint uselessOutVariable; + float A[50]; + int i = bitfieldExtract(0, 0, 0); + do { + if (i >= int(resolution.x)) { + break; + } + if (findLSB(16) * (i/findMSB(16)) == i) { + A[i/findLSB(16)] = float(i); + } + i++; + } while(i < bitfieldInsert(200, 0, 0, 0)); + i = findLSB(0); + do { + if (i < int(gl_FragCoord.x)) { + break; + } + if (i > findMSB(0)) { + A[i] += A[int(usubBorrow(uint(i), 1u, uselessOutVariable))]; + } + i++; + } while(i < bitfieldInsert(50, 0, 0, 0)); + if (int(gl_FragCoord.x) < findLSB(msb20)) { + _GLF_color = vec4(A[bitfieldReverse(0)]/resolution.x, A[findMSB(16)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(40, 0, 0, 0)) { + _GLF_color = vec4(A[findLSB(32)]/resolution.x, A[findMSB(msb9)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(60, 0, 0, 0)) { + _GLF_color = vec4(A[findMSB(msb10)]/resolution.x, A[findLSB(msb14)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(80, 0, 0, 0)) { + _GLF_color = vec4(A[findLSB(msb15)]/resolution.x, A[findMSB(msb19)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(100, 0, 0, 0)) { + _GLF_color = vec4(A[findMSB(msb20)]/resolution.x, A[findLSB(msb24)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(120, 0, 0, 0)) { + _GLF_color = vec4(A[findLSB(msb25)]/resolution.x, A[findMSB(msb29)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(140, 0, 0, 0)) { + _GLF_color = vec4(A[findMSB(msb30)]/resolution.x, A[bitfieldInsert(34, 0, 0, 0)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(160, 0, 0, 0)) { + _GLF_color = vec4(A[bitfieldInsert(35, 0, 0, 0)]/resolution.x, A[bitfieldInsert(39, 0, 0, 0)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(180, 0, 0, 0)) { + _GLF_color = vec4(A[bitfieldInsert(40, 0, 0, 0)]/resolution.x, A[bitfieldInsert(44, 0, 0, 0)]/resolution.y, 1.0, 1.0); + } else if (int(gl_FragCoord.x) < bitfieldInsert(180, 0, 0, 0)) { + _GLF_color = vec4(A[bitfieldInsert(45, 0, 0, 0)]/resolution.x, A[bitfieldInsert(49, 0, 0, 0)]/resolution.y, 1.0, 1.0); + } else { + discard; + } + +} + diff --git a/shaders/src/main/glsl/samples/donors/mandelbrot_blurry.json b/shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.json similarity index 100% rename from shaders/src/main/glsl/samples/donors/mandelbrot_blurry.json rename to shaders/src/main/glsl/samples/310es/prefix_sum_intfunctions.json diff --git a/shaders/src/main/glsl/samples/310es/quicksort_palette.frag b/shaders/src/main/glsl/samples/310es/quicksort_palette.frag new file mode 100644 index 000000000..95ebbad29 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/quicksort_palette.frag @@ -0,0 +1,131 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +struct QuicksortObject{ + int numbers[10]; +}; + +QuicksortObject obj; + + void swap(int i, int j) { + int temp = obj.numbers[i]; + obj.numbers[i] = obj.numbers[j]; + obj.numbers[j] = temp; + } + + // Since "partition" is the preserved word, we add prefix to this function name to prevent an error. + int performPartition(int l, int h) { + // The rightmost element is chosen as a pivot. + int pivot = obj.numbers[h]; + int i = (l - 1); + + for (int j = l; j <= h - 1; j++) { + if (obj.numbers[j] <= pivot) { + i++; + swap(i, j); + } + } + swap(i + 1, h); + return (i + 1); + } + + void quicksort() { + int l = 0, h = 9; + int stack[10]; + int top = -1; + + stack[++top] = l; + stack[++top] = h; + + while (top >= 0) { + h = stack[top--]; + l = stack[top--]; + + int p = performPartition(l, h); + if (p - 1 > l) { + stack[++top] = l; + stack[++top] = p - 1; + } + if (p + 1 < h) { + stack[++top] = p + 1; + stack[++top] = h; + } + } + } + +vec3 palette(vec3 a, vec3 b, vec3 c, vec3 d) { + return fract(mix(d * a, a * c, b + d - c)); +} + +float randomize(vec2 co) +{ + return float(floor(fract(sin(dot(co.xy ,vec2(12.5, 3.))) * 4250. + 0.02) + 0.5)); +} + +bool puzzlelize(vec2 pos) +{ + return randomize(pos.xy) < .12; +} + +void main() { + // Initialize decreasing values to an array starting from 10. + for (int i = int(injectionSwitch.x); i < 10; i ++) { + obj.numbers[i] = (10 - i) * int(injectionSwitch.y); + } + quicksort(); + vec2 grid = vec2(20, 20); + vec2 uv = gl_FragCoord.xy / resolution; + vec3 color = palette(vec3(float(obj.numbers[4]) * 0.1), vec3(0.9, float(obj.numbers[8]) * 0.1, 0.8), trunc(vec3(injectionSwitch.y)), vec3(injectionSwitch.x, 0.3, 0.7)); + if (uv.x > (1.0 / 4.0)) { + int count = int(injectionSwitch.x); + do { + color += palette(vec3(0.5, float(obj.numbers[8]) * 0.1, 0.2), vec3(0.5), trunc(vec3(injectionSwitch.y)), vec3(float(obj.numbers[4]) * 0.1, injectionSwitch.x, 0.6)); + count++; + } while (count != obj.numbers[int(injectionSwitch.x)]); + grid = vec2(count + obj.numbers[8], count + obj.numbers[6]); + } + if (uv.x > (2.0 / 4.0)) { + int count = int(injectionSwitch.x); + do { + color -= palette(vec3(float(obj.numbers[4]) * 0.1), trunc(vec3(0.1)), vec3(float(obj.numbers[int(injectionSwitch.y)]) * 0.1), vec3(injectionSwitch.x, float(obj.numbers[2]) * 0.1 , float(obj.numbers[8]) * 0.1)); + count++; + } while (count != obj.numbers[1]); + grid += vec2(count + int(injectionSwitch.y), count + int(injectionSwitch.x)); + } + if(uv.x > (3.0 / 4.0)) { + int count = int(injectionSwitch.x); + do { + color -= palette(vec3(float(obj.numbers[int(injectionSwitch.y)]) * 0.1), vec3(0.8), trunc(vec3(0.1)), vec3(injectionSwitch.x, 0.6, float(obj.numbers[int(injectionSwitch.x)]) * 0.1)); + count++; + } while (count != obj.numbers[2]); + grid += vec2(count + obj.numbers[3], count + obj.numbers[3]); + } + + vec2 position = vec2(gl_FragCoord.x, resolution.x - gl_FragCoord.y); + position = floor(position / grid); + _GLF_color = vec4(color, injectionSwitch.y) + vec4(!puzzlelize(position)); + +} diff --git a/shaders/src/main/glsl/samples/310es/quicksort_palette.json b/shaders/src/main/glsl/samples/310es/quicksort_palette.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/quicksort_palette.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/selection_sort_struct.frag b/shaders/src/main/glsl/samples/310es/selection_sort_struct.frag new file mode 100644 index 000000000..827b02271 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/selection_sort_struct.frag @@ -0,0 +1,73 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +struct Obj { + float odd_numbers[10]; + float even_numbers[10]; +}; + +void main() { + Obj obj; + + // Initialize first 10 odd numbers to the array. + int odd_index = 0; + float odd_number = 1.0; + while (odd_index <= 9) { + obj.odd_numbers[odd_index] = odd_number; + odd_number += 2.0; + odd_index++; + } + // Similarly, initialize even numbers and iterate backward. + int even_index = 9; + float even_number = 0.0; + while (even_index >= 0) { + obj.even_numbers[even_index] = even_number; + even_number += 2.; + even_index--; + } + + // Perform the selection sort. + for (int i = 0; i < 9; i++) { + int index = i; + for (int j = i + 1; j < 10; j++) { + if (obj.even_numbers[j] < obj.even_numbers[index]) { + index = j; + } + } + float smaller_number = obj.even_numbers[index]; + obj.even_numbers[index] = obj.even_numbers[i]; + obj.even_numbers[i] = smaller_number; + } + + vec2 uv = gl_FragCoord.xy/resolution.xy; + vec3 col = tan(pow(uv.xxx, uv.yyy) + + vec3( + obj.odd_numbers[int(floor(gl_FragCoord.x/1000.0))], + obj.even_numbers[int(floor(gl_FragCoord.y/1000.0))], + sinh(uv.x) + )); + + _GLF_color.rgb = col; + _GLF_color.a = 1.0; +} diff --git a/shaders/src/main/glsl/samples/310es/selection_sort_struct.json b/shaders/src/main/glsl/samples/310es/selection_sort_struct.json new file mode 100644 index 000000000..269b946d9 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/selection_sort_struct.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/squares.json b/shaders/src/main/glsl/samples/310es/squares.json index bc2d1fb1b..fdc0723fa 100644 --- a/shaders/src/main/glsl/samples/310es/squares.json +++ b/shaders/src/main/glsl/samples/310es/squares.json @@ -5,13 +5,6 @@ 0.0 ] }, - "mouse": { - "func": "glUniform2f", - "args": [ - 0.0, - 0.0 - ] - }, "resolution": { "func": "glUniform2f", "args": [ diff --git a/shaders/src/main/glsl/samples/310es/squares_intfunctions.frag b/shaders/src/main/glsl/samples/310es/squares_intfunctions.frag new file mode 100644 index 000000000..2f92a875d --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/squares_intfunctions.frag @@ -0,0 +1,144 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform float time; + +uniform vec2 resolution; + +float h_r; +float s_g; +float b_b; + +void doConvert() +{ + int msb8 = 256; + vec3 temp; + temp = b_b * (float(bitCount(msb8)) - s_g) + (b_b - b_b * (float(bitCount(msb8)) - s_g)) * clamp(abs(abs(6.0 * (h_r - vec3(bitfieldReverse(0), bitCount(msb8), 2) / 3.0)) - 3.0) - float(bitCount(msb8)), float(bitfieldExtract(int(resolution.x), 0, 0)), float(bitCount(int(resolution.x)))); + h_r = temp.x; + s_g = temp.y; + b_b = temp.z; +} + +vec3 computeColor(float c, vec2 position) +{ + int msb8 = 256; + h_r = fract(c); + s_g = float(bitCount(msb8)); + b_b = (0.5 + (sin(time) * 0.5 + 0.5)); + doConvert(); + s_g *= float(bitCount(msb8)) / position.y; + h_r *= float(bitCount(msb8)) / position.x; + if (abs(position.y - position.x) < 0.5) { + b_b = clamp(float(bitfieldExtract(int(resolution.x), 0, 0)), float(bitCount(msb8)), b_b * float(bitCount(msb8)) * 3.0); + } + return vec3(h_r, s_g, b_b); +} + +vec3 defaultColor() { + return vec3(float(bitfieldExtract(int(resolution.y), 0, 0))); +} + +vec3 drawShape(vec2 pos, vec2 square, vec3 setting) +{ + int msb8 = 256; + bool c1 = pos.x - setting.x < square.x; + if (!c1) { + return defaultColor(); + } + bool c2 = pos.x + setting.x > square.x; + if (!c2) { + return defaultColor(); + } + bool c3 = pos.y - setting.x < square.y; + if (!c3) { + return defaultColor(); + } + bool c4 = pos.y + setting.x > square.y; + if (!c4) { + return defaultColor(); + } + + bool c5 = pos.x - (setting.x - setting.y) < square.x; + if (!c5) { + return computeColor(setting.z / (5.0 * float(findMSB(msb8))), pos); + } + bool c6 = pos.x + (setting.x - setting.y) > square.x; + if (!c6) { + return computeColor(setting.z / (5.0 * float(findMSB(msb8))), pos); + } + bool c7 = pos.y - (setting.x - setting.y) < square.y; + if (!c7) { + return computeColor(setting.z / (5.0 * float(findMSB(msb8))), pos); + } + bool c8 = pos.y + (setting.x - setting.y) > square.y; + if (!c8) { + return computeColor(setting.z / (5.0 * float(findMSB(msb8))), pos); + } + return defaultColor(); +} + +vec3 computePoint(mat2 rotationMatrix) +{ + int msb8 = 256; + vec2 aspect; + aspect = resolution.xy / min(resolution.x, resolution.y); + vec2 position; + position = (gl_FragCoord.xy / resolution.xy) * aspect; + vec2 center; + center = vec2(0.5) * aspect; + position *= rotationMatrix; + center *= rotationMatrix; + vec3 result = vec3(0.0); + for(int i = bitfieldInsert(35, 0, 0, bitfieldExtract(int(resolution.x), 0, 0)); i >= bitfieldReverse(bitfieldExtract(int(resolution.x), 0, 0)); i --) + { + vec3 d; + d = drawShape(position, center + vec2(sin(float(i) / (float(bitCount(msb8)) * 10.0) + time) / 4.0 * float(bitCount(msb8)), float(bitfieldExtract(int(resolution.x), 0, 0))), vec3(0.01 + sin(float(i) / 100.0 * float(bitCount(msb8))), 0.01, float(i))); + if(length(d) <= float(bitfieldExtract(int(resolution.x), 0, 0))) { + continue; + } + result = vec3(d); + } + return result; +} + +void main() { + int msb8 = 256; + float angle; + angle = sin(time) * 0.1; + mat2 rotationMatrix; + rotationMatrix = mat2(sin(angle), - cos(angle), cos(angle), sin(angle)); + vec3 point1; + point1 = computePoint(rotationMatrix); + mat2 rotationMatrix2; + rotationMatrix2 = rotationMatrix * rotationMatrix; + vec3 point2; + point2 = computePoint(rotationMatrix2); + mat2 rotationMatrix3; + rotationMatrix3 = rotationMatrix * rotationMatrix * rotationMatrix; + vec3 point3; + point3 = computePoint(rotationMatrix3); + vec3 mixed; + mixed = mix(point1, point2, vec3(0.3)); + mixed = mix(mixed, point3, vec3(0.3)); + _GLF_color = vec4(mixed, float(bitCount(msb8))); +} + diff --git a/shaders/src/main/glsl/samples/310es/squares_intfunctions.json b/shaders/src/main/glsl/samples/310es/squares_intfunctions.json new file mode 100644 index 000000000..a12f190d9 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/squares_intfunctions.json @@ -0,0 +1,15 @@ +{ + "time": { + "func": "glUniform1f", + "args": [ + 0.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.frag b/shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.frag new file mode 100644 index 000000000..5829c1b26 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.frag @@ -0,0 +1,159 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +struct BST{ + int data; + int leftIndex; + int rightIndex; +}; + +BST tree[10]; + +void makeTreeNode(inout BST tree, int data) +{ + tree.data = data; + tree.leftIndex = -1; + tree.rightIndex = -1; +} + +void insert(int treeIndex, int data) +{ + int baseIndex = 0; + while (baseIndex <= treeIndex) { + // If new value is smaller thatn the current node, we known that we will have + // add this element in the left side. + if (data <= tree[baseIndex].data) { + // If a left subtree of the current node is empty, the new node is added as + // a left subtree of the current node. + if (tree[baseIndex].leftIndex == -1) { + tree[baseIndex].leftIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].leftIndex; + continue; + } + } else { + // If a right subtree of the current node is empty, the new node is added as + // a right subtree of the current node. + if (tree[baseIndex].rightIndex == -1) { + tree[baseIndex].rightIndex = treeIndex; + makeTreeNode(tree[treeIndex], data); + return; + } else { + baseIndex = tree[baseIndex].rightIndex; + continue; + } + } + } +} + +// Return element data if the given target exists in a tree. Otherwise, we simply return -1. +int search(int target){ + BST currentNode; + int index = 0; + while (index != -1) { + currentNode = tree[index]; + if (currentNode.data == target) { + return target; + } + index = target > currentNode.data ? currentNode.rightIndex : currentNode.leftIndex; + } + return -1; +} + +/* +* This shader implements binary search tree using an array data structure. The elements of +* tree are kept in the array that contains a list of BST object holding indices of left and +* right subtree in the array. +* +* - Tree representation of the number used in this shader: +* 9 +* / \ +* 5 12 +* / \ \ +* 2 7 15 +* / \ / \ +* 6 8 13 17 +* +* - Array representation: +* [9, 5, 12, 15, 7, 8, 2, 6, 17, 13] +* +*/ + +void main() { + int treeIndex = 0; + // Initialize root node. + makeTreeNode(tree[0], 9); + // Each time we insert a new node into the tree, we increment one. + treeIndex++; + + insert(treeIndex, 5); + treeIndex++; + insert(treeIndex, 12); + treeIndex++; + insert(treeIndex, 15); + treeIndex++; + insert(treeIndex, 7); + treeIndex++; + insert(treeIndex, 8); + treeIndex++; + insert(treeIndex, 2); + treeIndex++; + insert(treeIndex, 6); + treeIndex++; + insert(treeIndex, 17); + treeIndex++; + insert(treeIndex, 13); + + int count = 0; + for (int i = 0; i < 20; i++) { + int result = search(i); + switch (i) { + case 9: + case 5: + case 12: + case 15: + case 7: + case 8: + case 2: + case 6: + case 17: + case 13: + if (result == i) { + count++; + } + break; + default: + if (result == -1) { + count++; + } + break; + } + } + if (count == 20) { + _GLF_color = vec4(1.0, 0.0, 0.0, 1.0); + } else { + _GLF_color = vec4(0.0, 0.0, 1.0, 1.0); + } +} + diff --git a/shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.json b/shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_binarysearch_tree.json @@ -0,0 +1 @@ +{} diff --git a/shaders/src/main/glsl/samples/310es/bubblesort_flag.frag b/shaders/src/main/glsl/samples/310es/stable_bubblesort_flag.frag similarity index 100% rename from shaders/src/main/glsl/samples/310es/bubblesort_flag.frag rename to shaders/src/main/glsl/samples/310es/stable_bubblesort_flag.frag diff --git a/shaders/src/main/glsl/samples/310es/stable_bubblesort_flag.json b/shaders/src/main/glsl/samples/310es/stable_bubblesort_flag.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_bubblesort_flag.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.frag b/shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.frag new file mode 100644 index 000000000..86d654bd6 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.frag @@ -0,0 +1,61 @@ +#version 310 es + +/* + * Copyright 2018 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +float compute_value(float limit, float thirty_two) { + float result = -0.5; + + // The loop will not really execute 800 times, as 'limit' is bounded by gl_FragCoord, and there is an early exit based on 'limit'. + for (int i = 1; i < 800; i++) { + if ((i % 32) == 0) { + // Avoid computing e.g. mod(float(32), round(32.0)), which could be sensitive to round-off if mutated. + result += 0.4; + } else if (mod(float(i), round(thirty_two)) <= 0.01) { + // This should never get executed, because the previous if condition would get triggered. + result += 100.0; + } + if (float(i) >= limit) { + return result; + } + } + return result; +} + +void main() +{ + vec3 c = vec3(7.0, 8.0, 9.0); + + // This is guaranteed to be 32.0, as resolution.x is 256.0. + float thirty_two = round(resolution.x / 8.0); + + c.x = compute_value(gl_FragCoord.x, thirty_two); + c.y = compute_value(gl_FragCoord.y, thirty_two); + c.z = c.x + c.y; + + for (int i = 0; i < 3; i++) { + if (c[i] >= 1.0) { + c[i] = c[i] * c[i]; + } + } + _GLF_color = vec4(normalize(abs(c)), 1.0); +} diff --git a/shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.json b/shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.json new file mode 100644 index 000000000..269b946d9 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_colorgrid_modulo.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/stable_mergesort.frag b/shaders/src/main/glsl/samples/310es/stable_mergesort.frag new file mode 100644 index 000000000..54e8ffe82 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_mergesort.frag @@ -0,0 +1,140 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +// Size of an array. +const int N = 10; +// An array and its temperary array whose elements will be sorted. +int data[10], temp[10]; + +// Merge two sorted subarrays data[from ... mid] and data[mid+1 ... to]. +void merge(int from, int mid, int to) { + int k = from, i = from, j = mid +1; + + while (i <= mid && j <= to) { + if (data[i] < data[j]) { + temp[k++] = data[i++]; + } else { + temp[k++] = data[j++]; + } + } + // Copy remaining elements. + while (i < N && i <= mid) { + temp[k++] = data[i++]; + } + // Copy back to the original array. + for (int i = from; i <= to; i++) { + data[i] = temp[i]; + } +} + +// Sort array using the iterative approach. +void mergeSort() { + int low = 0; + int high = N - 1; + + // Devide the array into blocks of size m. + // m = [1, 2, 4 ,8, 16...]. + for (int m = 1; m <= high; m = 2 * m) { + + // For m = 1, i = [0, 2, 4, 6, 8]. + // For m = 2, i = [0, 4, 8]. + // For m = 4, i = [0, 8]. + for (int i = low; i< high; i += 2 * m) { + int from = i; + int mid = i + m - 1; + int to = min (i + 2 * m - 1, high); + merge(from, mid, to); + } + } +} + +void main() { + int i = int(injectionSwitch.x); + do { + switch(i) { + case 0: + data[i] = 4; + break; + case 1: + data[i] = 3; + break; + case 2: + data[i] = 2; + break; + case 3: + data[i] = 1; + break; + case 4: + data[i] = 0; + break; + case 5: + data[i] = -1; + break; + case 6: + data[i] = -2; + break; + case 7: + data[i] = -3; + break; + case 8: + data[i] = -4; + break; + case 9: + data[i] = -5; + break; + } + i++; + } while (i < 10); + + for (int j = 0; j < 10; j++) { + temp[j] = data[j]; + } + mergeSort(); + + float grey; + if(int(gl_FragCoord[1]) < 30) { + grey = 0.5 + float(data[0]) / 10.0; // data[0] should be -5, this should end up being 0.0 + } else if(int(gl_FragCoord[1]) < 60) { + grey = 0.5 + float(data[1]) / 10.0; // data[1] should be -4, this should end up being 0.1 + } else if(int(gl_FragCoord[1]) < 90) { + grey = 0.5 + float(data[2]) / 10.0; // similar + } else if(int(gl_FragCoord[1]) < 120) { + grey = 0.5 + float(data[3]) / 10.0; // similar + } else if(int(gl_FragCoord[1]) < 150) { + discard; // Deliberately discard this one + } else if(int(gl_FragCoord[1]) < 180) { + grey = 0.5 + float(data[5]) / 10.0; + } else if(int(gl_FragCoord[1]) < 210) { + grey = 0.5 + float(data[6]) / 10.0; + } else if(int(gl_FragCoord[1]) < 240) { + grey = 0.5 + float(data[7]) / 10.0; + } else if(int(gl_FragCoord[1]) < 270) { + grey = 0.5 + float(data[8]) / 10.0; + } else { + discard; + } + _GLF_color = vec4(vec3(grey), 1.0); +} diff --git a/shaders/src/main/glsl/samples/310es/stable_mergesort.json b/shaders/src/main/glsl/samples/310es/stable_mergesort.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_mergesort.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/stable_quicksort.frag b/shaders/src/main/glsl/samples/310es/stable_quicksort.frag new file mode 100644 index 000000000..0078f1984 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_quicksort.frag @@ -0,0 +1,112 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 resolution; + +struct QuicksortObject{ + int numbers[10]; +}; + +QuicksortObject obj; + + void swap(int i, int j) { + int temp = obj.numbers[i]; + obj.numbers[i] = obj.numbers[j]; + obj.numbers[j] = temp; + } + + // Since "partition" is the preserved word, we add prefix to this function name to prevent an error. + int performPartition(int l, int h) { + // The rightmost element is chosen as a pivot. + int pivot = obj.numbers[h]; + int i = (l - 1); + + for (int j = l; j <= h - 1; j++) { + if (obj.numbers[j] <= pivot) { + i++; + swap(i, j); + } + } + swap(i + 1, h); + return (i + 1); + } + + void quicksort() { + int l = 0, h = 9; + int stack[10]; + int top = -1; + + stack[++top] = l; + stack[++top] = h; + + while (top >= 0) { + h = stack[top--]; + l = stack[top--]; + + int p = performPartition(l, h); + if (p - 1 > l) { + stack[++top] = l; + stack[++top] = p - 1; + } + if (p + 1 < h) { + stack[++top] = p + 1; + stack[++top] = h; + } + } + } + +void main() { + // Initialize decreasing values to an array starting from 10. + for (int i = 0; i < 10; i ++) { + obj.numbers[i] = (10 - i); + obj.numbers[i] = obj.numbers[i] * obj.numbers[i]; + } + quicksort(); + vec2 uv = gl_FragCoord.xy / resolution; + vec3 color = vec3(1.0, 2.0, 3.0); + color.x += float(obj.numbers[0]); + if (uv.x > (1.0 / 4.0)) { + color.x += float(obj.numbers[1]); + } + if (uv.x > (2.0 / 4.0)) { + color.y += float(obj.numbers[2]); + } + if(uv.x > (3.0 / 4.0)) { + color.z += float(obj.numbers[3]); + } + color.y += float(obj.numbers[4]); + if (uv.y > (1.0 / 4.0)) { + color.x += float(obj.numbers[5]); + } + if (uv.y > (2.0 / 4.0)) { + color.y += float(obj.numbers[6]); + } + if(uv.y > (3.0 / 4.0)) { + color.z += float(obj.numbers[7]); + } + color.z += float(obj.numbers[8]); + if (abs(uv.x - uv.y) < 0.25) { + color.x += float(obj.numbers[9]); + } + _GLF_color = vec4(normalize(color), 1.0); + +} diff --git a/shaders/src/main/glsl/samples/310es/stable_quicksort.json b/shaders/src/main/glsl/samples/310es/stable_quicksort.json new file mode 100644 index 000000000..28c3c3e61 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/stable_quicksort.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/310es/trigonometric_strip.frag b/shaders/src/main/glsl/samples/310es/trigonometric_strip.frag new file mode 100644 index 000000000..4c05e0d3c --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/trigonometric_strip.frag @@ -0,0 +1,74 @@ +#version 310 es + +/* + * Copyright 2019 The GraphicsFuzz Project Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +precision highp float; + +layout(location = 0) out vec4 _GLF_color; + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +float compute_derivative_x(float v){ + return dFdx(v) * injectionSwitch.y; +} + +float compute_derivative_y(float v){ + return dFdy(v) * injectionSwitch.y; +} + +float compute_stripe(float v) { + return smoothstep(-.9, 1., (v)/ ((injectionSwitch.y > injectionSwitch.x) ? compute_derivative_x(v): compute_derivative_y(v))); +} + +void main() { + vec2 uv = gl_FragCoord.xy / resolution.x; + vec3 col = vec3(0, 0, 0); + + bool c1 = uv.y < 0.25; + if (c1) { + float stripe = compute_stripe(cos((uv.x + uv.y ) * 20.0 )); + col = mix(vec3(uv.x, 0, 0.75), vec3(.8, .7, uv.x), stripe); + _GLF_color = vec4(col, 1.0); + return; + } + + bool c2 = uv.y < 0.5; + if (!c1 && c2) { + float stripe = compute_stripe(tan((uv.x + uv.y) * 20.0 )); + col = mix(vec3(0.5, uv.x, 0.1), vec3(.4, 0, .5), stripe); + _GLF_color = vec4(col, 1.0); + return; + } + + bool c3 = uv.y < 0.75; + if (!c1 && !c2 && c3) { + float stripe = compute_stripe(cos((uv.x + uv.y) * 20.0 )); + col = mix(vec3(.7, sinh(uv.x), uv.x ), vec3(.3, .5, uv.x), stripe); + _GLF_color = vec4(col, 1.0); + return; + } + + bool c4 = uv.y >= 0.75; + if (!c1 && !c2 && !c3 && c4) { + float stripe = compute_stripe(tan((uv.x + uv.y) * 20.0 )); + col = mix(vec3(uv.x, .8, 0), vec3(1, uv.y, 0), stripe); + _GLF_color = vec4(col, 1.0); + return; + } +} diff --git a/shaders/src/main/glsl/samples/310es/trigonometric_strip.json b/shaders/src/main/glsl/samples/310es/trigonometric_strip.json new file mode 100644 index 000000000..d7b58d068 --- /dev/null +++ b/shaders/src/main/glsl/samples/310es/trigonometric_strip.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/compute/310es/comp-0001-findmax.json b/shaders/src/main/glsl/samples/compute/310es/comp-0001-findmax.json index 3c4e1dcc0..af1fb5c35 100644 --- a/shaders/src/main/glsl/samples/compute/310es/comp-0001-findmax.json +++ b/shaders/src/main/glsl/samples/compute/310es/comp-0001-findmax.json @@ -6,8 +6,8 @@ "fields": [ { "type": "int", "data": [ 0 ] }, - { "type": "int", "data": - [ 11, 12, 13, 14, 15, 16, 17, 18, + { "type": "int", "data": + [ 11, 12, 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 41, 42, 43, 44, 45, 46, 47, 48, diff --git a/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.comp b/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.comp index 611b2b632..31e2b52d2 100644 --- a/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.comp +++ b/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.comp @@ -23,7 +23,9 @@ layout(std430, binding = 0) buffer theSSBO { float data_in[N]; float data_out[N]; - bool is_exclusive; + // Amber only supports one type in a given buffer, hence the 'is_exclusive' + // bool field is declared as a float + float is_exclusive; }; layout(local_size_x=N, local_size_y=1, local_size_z=1) in; @@ -38,7 +40,7 @@ void main() { barrier(); - for (uint offset = 1u; offset < uint(N); offset <<= 1u) + for (uint offset = 1u; offset < uint(N); offset <<= 1u) { if (tid >= offset) { @@ -55,7 +57,7 @@ void main() { barrier(); } - if (is_exclusive) { + if (bool(is_exclusive)) { data_out[tid] = (tid == 0u) ? 0.0 : result[tid - 1u]; } else { data_out[tid] = result[tid]; diff --git a/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.json b/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.json index 99c8752ab..f8fb16916 100644 --- a/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.json +++ b/shaders/src/main/glsl/samples/compute/310es/comp-0004-koggestone.json @@ -40,7 +40,7 @@ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ] }, - { "type": "bool", "data": [ 0 ] } + { "type": "float", "data": [ 0 ] } ] } } diff --git a/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.comp b/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.comp index 27474d827..81e304338 100644 --- a/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.comp +++ b/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.comp @@ -23,7 +23,9 @@ #define INVOCATIONS 100 layout(std430, binding = 0) buffer theSSBO { - bool is_exclusive; + // Amber only supports one type in a given buffer, hence the 'is_exclusive' + // bool field is declared as a uint + uint is_exclusive; uint data_out[DATA_SIZE]; uint data_in[DATA_SIZE]; }; @@ -36,12 +38,12 @@ void main() { uint tid = gl_LocalInvocationIndex; uint gid = gl_GlobalInvocationID.x; - + if (tid != gid) { // They should be the same. Use them both and do this check just to shake things up a bit. return; } - + result[2u * tid] = data_in[2u * gid]; result[2u * tid + 1u] = data_in[2u * gid + 1u]; @@ -65,10 +67,10 @@ void main() { result[me] = max(result[spine], result[me]); d = d * 2u; } - + barrier(); - if (is_exclusive) { + if (bool(is_exclusive)) { data_out[2u * gid] = (tid == 0u) ? 0u : result[2u * tid - 1u]; data_out[2u * gid + 1u] = result[2u * tid]; } else { diff --git a/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.json b/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.json index 3dcec9fd4..78bf8e149 100644 --- a/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.json +++ b/shaders/src/main/glsl/samples/compute/310es/comp-0005-sklansky.json @@ -5,7 +5,7 @@ "binding": 0, "fields": [ - { "type": "bool", "data": [ 1 ] }, + { "type": "uint", "data": [ 1 ] }, { "type": "uint", "data": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, diff --git a/shaders/src/main/glsl/samples/donors/mandelbrot_blurry.frag b/shaders/src/main/glsl/samples/donors/mandelbrot_zoom.frag similarity index 81% rename from shaders/src/main/glsl/samples/donors/mandelbrot_blurry.frag rename to shaders/src/main/glsl/samples/donors/mandelbrot_zoom.frag index b52569b8d..846ba7cf4 100644 --- a/shaders/src/main/glsl/samples/donors/mandelbrot_blurry.frag +++ b/shaders/src/main/glsl/samples/donors/mandelbrot_zoom.frag @@ -24,8 +24,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; int iteration = 0; for (int k = 0; k < 1000; k++) { @@ -40,7 +43,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < 1000) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/webgl1/mandelbrot_blurry.json b/shaders/src/main/glsl/samples/donors/mandelbrot_zoom.json similarity index 100% rename from shaders/src/main/glsl/samples/webgl1/mandelbrot_blurry.json rename to shaders/src/main/glsl/samples/donors/mandelbrot_zoom.json diff --git a/shaders/src/main/glsl/samples/donors/bubblesort_flag.frag b/shaders/src/main/glsl/samples/donors/stable_bubblesort_flag.frag similarity index 100% rename from shaders/src/main/glsl/samples/donors/bubblesort_flag.frag rename to shaders/src/main/glsl/samples/donors/stable_bubblesort_flag.frag diff --git a/shaders/src/main/glsl/samples/donors/stable_bubblesort_flag.json b/shaders/src/main/glsl/samples/donors/stable_bubblesort_flag.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/donors/stable_bubblesort_flag.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/webgl1/colorgrid_modulo.frag b/shaders/src/main/glsl/samples/webgl1/colorgrid_modulo.frag index 093653d47..4f0c4a2e4 100644 --- a/shaders/src/main/glsl/samples/webgl1/colorgrid_modulo.frag +++ b/shaders/src/main/glsl/samples/webgl1/colorgrid_modulo.frag @@ -19,7 +19,6 @@ precision mediump float; -uniform vec2 injectionSwitch; uniform vec2 resolution; float nb_mod(float limit, float ref) { diff --git a/shaders/src/main/glsl/samples/webgl1/mandelbrot_blurry.frag b/shaders/src/main/glsl/samples/webgl1/mandelbrot_zoom.frag similarity index 81% rename from shaders/src/main/glsl/samples/webgl1/mandelbrot_blurry.frag rename to shaders/src/main/glsl/samples/webgl1/mandelbrot_zoom.frag index 6a65e258c..1f1b8926d 100644 --- a/shaders/src/main/glsl/samples/webgl1/mandelbrot_blurry.frag +++ b/shaders/src/main/glsl/samples/webgl1/mandelbrot_zoom.frag @@ -29,8 +29,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; int iteration = 0; for (int k = 0; k < 1000; k++) { @@ -45,7 +48,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < 1000) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/webgl2/mandelbrot_blurry.json b/shaders/src/main/glsl/samples/webgl1/mandelbrot_zoom.json similarity index 100% rename from shaders/src/main/glsl/samples/webgl2/mandelbrot_blurry.json rename to shaders/src/main/glsl/samples/webgl1/mandelbrot_zoom.json diff --git a/shaders/src/main/glsl/samples/webgl1/bubblesort_flag.frag b/shaders/src/main/glsl/samples/webgl1/stable_bubblesort_flag.frag similarity index 100% rename from shaders/src/main/glsl/samples/webgl1/bubblesort_flag.frag rename to shaders/src/main/glsl/samples/webgl1/stable_bubblesort_flag.frag diff --git a/shaders/src/main/glsl/samples/webgl1/stable_bubblesort_flag.json b/shaders/src/main/glsl/samples/webgl1/stable_bubblesort_flag.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/webgl1/stable_bubblesort_flag.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/webgl2/colorgrid_modulo.frag b/shaders/src/main/glsl/samples/webgl2/colorgrid_modulo.frag index 4aa24d598..71033c147 100644 --- a/shaders/src/main/glsl/samples/webgl2/colorgrid_modulo.frag +++ b/shaders/src/main/glsl/samples/webgl2/colorgrid_modulo.frag @@ -21,7 +21,6 @@ precision highp float; layout(location = 0) out vec4 _GLF_color; -uniform vec2 injectionSwitch; uniform vec2 resolution; float nb_mod(float limit, float ref) { diff --git a/shaders/src/main/glsl/samples/webgl2/mandelbrot_blurry.frag b/shaders/src/main/glsl/samples/webgl2/mandelbrot_zoom.frag similarity index 81% rename from shaders/src/main/glsl/samples/webgl2/mandelbrot_blurry.frag rename to shaders/src/main/glsl/samples/webgl2/mandelbrot_zoom.frag index 90303fb31..cfb7f97e4 100644 --- a/shaders/src/main/glsl/samples/webgl2/mandelbrot_blurry.frag +++ b/shaders/src/main/glsl/samples/webgl2/mandelbrot_zoom.frag @@ -31,8 +31,11 @@ vec3 mand(float xCoord, float yCoord) { float height = resolution.y; float width = resolution.x; - float c_re = 0.8*(xCoord - width/2.0)*4.0/width - 0.4; - float c_im = 0.8*(yCoord - height/2.0)*4.0/width; + float xpos = xCoord * 0.1 + (resolution.x * 0.6); + float ypos = yCoord * 0.1 + (resolution.y * 0.4); + + float c_re = 0.8*(xpos - width/2.0)*4.0/width - 0.4; + float c_im = 0.8*(ypos - height/2.0)*4.0/width; float x = 0.0, y = 0.0; int iteration = 0; for (int k = 0; k < 1000; k++) { @@ -47,7 +50,7 @@ vec3 mand(float xCoord, float yCoord) { if (iteration < 1000) { return pickColor(iteration); } else { - return vec3(0.0); + return vec3(xCoord / resolution.x, 0.0, yCoord / resolution.y); } } diff --git a/shaders/src/main/glsl/samples/webgl2/mandelbrot_zoom.json b/shaders/src/main/glsl/samples/webgl2/mandelbrot_zoom.json new file mode 100644 index 000000000..000b854cf --- /dev/null +++ b/shaders/src/main/glsl/samples/webgl2/mandelbrot_zoom.json @@ -0,0 +1,9 @@ +{ + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shaders/src/main/glsl/samples/webgl2/bubblesort_flag.frag b/shaders/src/main/glsl/samples/webgl2/stable_bubblesort_flag.frag similarity index 100% rename from shaders/src/main/glsl/samples/webgl2/bubblesort_flag.frag rename to shaders/src/main/glsl/samples/webgl2/stable_bubblesort_flag.frag diff --git a/shaders/src/main/glsl/samples/webgl2/stable_bubblesort_flag.json b/shaders/src/main/glsl/samples/webgl2/stable_bubblesort_flag.json new file mode 100644 index 000000000..6ab644a33 --- /dev/null +++ b/shaders/src/main/glsl/samples/webgl2/stable_bubblesort_flag.json @@ -0,0 +1,16 @@ +{ + "injectionSwitch": { + "func": "glUniform2f", + "args": [ + 0.0, + 1.0 + ] + }, + "resolution": { + "func": "glUniform2f", + "args": [ + 256.0, + 256.0 + ] + } +} diff --git a/shadersets-util/pom.xml b/shadersets-util/pom.xml index e8a79bcb5..4dac88efa 100644 --- a/shadersets-util/pom.xml +++ b/shadersets-util/pom.xml @@ -47,7 +47,7 @@ limitations under the License. com.graphicsfuzzserver-thrift-gen - + com.google.code.gson gson diff --git a/test-util/src/main/java/com/graphicsfuzz/common/util/CheckUtilityClass.java b/test-util/src/main/java/com/graphicsfuzz/common/util/CheckUtilityClass.java index 7e7272a72..38b8fbe82 100644 --- a/test-util/src/main/java/com/graphicsfuzz/common/util/CheckUtilityClass.java +++ b/test-util/src/main/java/com/graphicsfuzz/common/util/CheckUtilityClass.java @@ -41,8 +41,7 @@ public static void assertUtilityClassWellDefined(final Class utilityClass) assertEquals("Utility class can only have one constructor", 1, utilityClass.getDeclaredConstructors().length); final Constructor constructor = utilityClass.getDeclaredConstructor(); - if (constructor.isAccessible() - || !Modifier.isPrivate(constructor.getModifiers())) { + if (!Modifier.isPrivate(constructor.getModifiers())) { fail("Utility class constructor must be private"); } constructor.setAccessible(true); diff --git a/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java b/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java index 618f7811e..17b982e61 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/GeneratorUnitTest.java @@ -17,9 +17,12 @@ package com.graphicsfuzz.tester; import com.graphicsfuzz.common.ast.TranslationUnit; +import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.transformreduce.ShaderJob; import com.graphicsfuzz.common.util.GlslParserException; +import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.common.util.ShaderKind; @@ -38,9 +41,7 @@ import com.graphicsfuzz.generator.transformation.VectorizeTransformation; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.TransformationProbabilities; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; @@ -48,17 +49,17 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; import org.bytedeco.javacpp.indexer.UByteIndexer; import org.bytedeco.javacpp.opencv_core.Mat; import org.bytedeco.javacpp.opencv_imgcodecs; -import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class GeneratorUnitTest { @@ -77,18 +78,18 @@ public class GeneratorUnitTest { @Test public void testRenderBlackImage() throws Exception { - File shaderFile = temporaryFolder.newFile("shader.frag"); - + final File shaderJobFile = temporaryFolder.newFile("shader.json"); + final ShaderJobFileOperations fileOps = new ShaderJobFileOperations(); final String shader = "#version 100\n" + "precision mediump float;\n" + "void main() {\n" + " gl_FragColor = vec4(1.0, 0.0, 0.0, 1.0);\n" + "}\n"; - BufferedWriter bw = new BufferedWriter(new FileWriter(shaderFile)); - bw.write(shader); - bw.close(); + fileOps.writeShaderJobFile( + new GlslShaderJob(Optional.empty(), new PipelineInfo(), ParseHelper.parse(shader)), + shaderJobFile); - final File image = Util.getImageUsingSwiftshader(shaderFile, temporaryFolder); + final File image = Util.getImage(shaderJobFile, temporaryFolder, fileOps); final Mat mat = opencv_imgcodecs.imread(image.getAbsolutePath()); final UByteIndexer sI = mat.createIndexer(); @@ -106,32 +107,32 @@ public void testRenderBlackImage() throws Exception { @Test public void testStructify() throws Exception { testTransformationMultiVersions(() -> new StructificationTransformation(), TransformationProbabilities.DEFAULT_PROBABILITIES, - "structs.frag"); + "structs"); } @Test public void testDeadJumps() throws Exception { testTransformationMultiVersions(() -> new AddJumpTransformation(), TransformationProbabilities.onlyAddJumps(), - "jumps.frag"); + "jumps"); } @Test public void testIdentity() throws Exception { testTransformationMultiVersions(() -> new IdentityTransformation(), TransformationProbabilities - .onlyMutateExpressions(), "mutate.frag"); + .onlyMutateExpressions(), "mutate"); } @Test public void testOutlineStatements() throws Exception { testTransformationMultiVersions(() -> new OutlineStatementTransformation(), TransformationProbabilities.onlyOutlineStatements(), - "outline.frag"); + "outline"); } @Test public void testSplitForLoops() throws Exception { testTransformationMultiVersions(() -> new SplitForLoopTransformation(), TransformationProbabilities - .onlySplitLoops(), "split.frag"); + .onlySplitLoops(), "split"); } @Test @@ -140,9 +141,9 @@ public void testDonateDeadCode() throws Exception { TransformationProbabilities.likelyDonateDeadCode()::donateDeadCodeAtStmt, Util.getDonorsFolder(), GenerationParams.normal(ShaderKind.FRAGMENT, true)), TransformationProbabilities.likelyDonateDeadCode(), - "donatedead.frag", - Arrays.asList("bubblesort_flag.json", "squares.json"), - Arrays.asList("bubblesort_flag.json", "squares.json")); + "donatedead", + Arrays.asList("bubblesort_flag.json", "squares.json", "mandelbrot_zoom.json"), + Arrays.asList("bubblesort_flag.json", "squares.json", "mandelbrot_zoom.json")); // Reason for blacklisting^: slow. } @@ -153,7 +154,7 @@ public void testDonateLiveCode() throws Exception { Util.getDonorsFolder(), GenerationParams.normal(ShaderKind.FRAGMENT, true), false), TransformationProbabilities.likelyDonateLiveCode(), - "donatelive.frag", + "donatelive", Arrays.asList("squares.json"), Arrays.asList("squares.json")); // Reason for blacklisting^: slow. @@ -162,27 +163,27 @@ public void testDonateLiveCode() throws Exception { @Test public void testAddDeadFragColorWrites() throws Exception { testTransformationMultiVersions(() -> new AddDeadOutputWriteTransformation(), TransformationProbabilities - .onlyAddDeadFragColorWrites(), "deadfragcolor.frag", Arrays.asList(), Arrays.asList()); + .onlyAddDeadFragColorWrites(), "deadfragcolor", Arrays.asList(), Arrays.asList()); } @Test public void testAddLiveOutputVariableWrites() throws Exception { testTransformationMultiVersions(() -> new AddLiveOutputWriteTransformation(), TransformationProbabilities - .onlyAddLiveFragColorWrites(), "liveoutvar.frag", Arrays.asList(), Arrays.asList()); + .onlyAddLiveFragColorWrites(), "liveoutvar", Arrays.asList(), Arrays.asList()); } @Test public void testVectorize() throws Exception { testTransformationMultiVersions(() -> new VectorizeTransformation(), TransformationProbabilities.onlyVectorize(), - "vectorize.frag", Arrays.asList(), Arrays.asList()); + "vectorize", Arrays.asList(), Arrays.asList()); } @Test public void mutateAndVectorize() throws Exception { testTransformationMultiVersions(Arrays.asList(() -> new IdentityTransformation(), () -> new VectorizeTransformation()), TransformationProbabilities.onlyVectorizeAndMutate(), - "mutate_and_vectorize.frag", + "mutate_and_vectorize", Arrays.asList(), Arrays.asList()); } @@ -190,16 +191,16 @@ public void mutateAndVectorize() throws Exception { public void testStructification() throws Exception { testTransformationMultiVersions(() -> new StructificationTransformation(), TransformationProbabilities.onlyStructify(), - "structify.frag", Arrays.asList(), Arrays.asList()); + "structify", Arrays.asList(), Arrays.asList()); } @Test public void testWrap() throws Exception { testTransformationMultiVersions(() -> new AddWrappingConditionalTransformation(), TransformationProbabilities.onlyWrap(), - "wrap.frag", - Arrays.asList("bubblesort_flag.json"), - Arrays.asList("bubblesort_flag.json")); + "wrap", + Arrays.asList("bubblesort_flag.json", "colorgrid_modulo.json"), + Arrays.asList("bubblesort_flag.json", "colorgrid_modulo.json")); // Reason for blacklisting^: slow. } @@ -242,8 +243,7 @@ private void testTransformation(List transformations, } File referenceImage = null; if (!skipRender) { - referenceImage = Util.renderShader( - originalShaderJobFile, + referenceImage = Util.validateAndGetImage(originalShaderJobFile, temporaryFolder, fileOps); } @@ -331,29 +331,21 @@ private void generateAndCheckVariant( Generate.setInjectionSwitch(shaderJob.getPipelineInfo()); Generate.randomiseUnsetUniforms(tu, shaderJob.getPipelineInfo(), generator); - // Using fileOps, even though the rest of the code here does not use it yet. // Write shaders to shader job file and validate. - - Assert.assertTrue(suffix.endsWith(".frag")); - // e.g. "_matrix_mult" - final String suffixNoExtension = FilenameUtils.removeExtension(suffix); - // e.g. "temp/orig_matrix_mult.json" final File shaderJobFile = Paths.get( temporaryFolder.getRoot().getAbsolutePath(), - originalShaderJobFile.getName() + suffixNoExtension + ".json" + originalShaderJobFile.getName() + suffix + ".json" ).toFile(); fileOps.writeShaderJobFile( shaderJob, shaderJobFile ); - fileOps.areShadersValid(shaderJobFile, true); + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); if (!skipRender) { - File underlyingFragFile = fileOps.getUnderlyingShaderFile(shaderJobFile, ShaderKind.FRAGMENT); - // TODO: Use fileOps. - final File variantImage = Util.getImage(underlyingFragFile, temporaryFolder, fileOps); + final File variantImage = Util.getImage(shaderJobFile, temporaryFolder, fileOps); Util.assertImagesSimilar(referenceImage, variantImage); } @@ -372,5 +364,36 @@ private void testTransformationForSpecificShader(List t new File[] { referenceJson }); } + @Test + public void testDonateDeadCodeNoCompatibleDonors() throws Exception { + // By design, we set up a situation where the reference and donors are all identical + // and all declare a struct, which makes them incompatible (because at the moment we + // conservatively say that a donor and reference are incompatible if they declare structs + // with the same name). + // + // Dead code donation will not be able to succeed, but should have no effect rather than + // aborting. + final File donors = temporaryFolder.newFolder(); + final ShaderJob declaresSingleStruct = new GlslShaderJob(Optional.empty(), + new PipelineInfo(), ParseHelper.parse("#version 300 es\n" + + "precision highp float;\n" + + "struct S {\n" + + " int x;\n" + + "};\n" + + "void main() {\n" + + "}\n")); + for (int i = 0; i < 3; i++) { + fileOps.writeShaderJobFile(declaresSingleStruct, new File(donors, "donor_" + i + ".json")); + } + final File reference = temporaryFolder.newFile("reference.json"); + fileOps.writeShaderJobFile(declaresSingleStruct, reference); + testTransformation(Collections.singletonList(() -> new DonateDeadCodeTransformation( + TransformationProbabilities.likelyDonateDeadCode()::donateDeadCodeAtStmt, + donors, + GenerationParams.normal(ShaderKind.FRAGMENT, true))), + TransformationProbabilities.likelyDonateDeadCode(), + "donatedead", Collections.emptyList(), + new File[] { reference }); + } } diff --git a/tester/src/test/java/com/graphicsfuzz/tester/MiscellaneousGenerateThenReduceTest.java b/tester/src/test/java/com/graphicsfuzz/tester/MiscellaneousGenerateThenReduceTest.java index af3d1cead..bc5dba37c 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/MiscellaneousGenerateThenReduceTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/MiscellaneousGenerateThenReduceTest.java @@ -78,7 +78,7 @@ private void checkControlFlowWrapElimination(String program) .getReductionOpportunities(new GlslShaderJob(Optional.empty(), new PipelineInfo(), tu), new ReducerContext(false, shadingLanguageVersion, - new SameValueRandom(false, 0), new IdGenerator(), true), fileOps); + new SameValueRandom(false, 0), new IdGenerator()), fileOps); if (ops.isEmpty()) { break; } diff --git a/tester/src/test/java/com/graphicsfuzz/tester/ObfuscatorUnitTest.java b/tester/src/test/java/com/graphicsfuzz/tester/ObfuscatorUnitTest.java index 92ea26d4d..de0b6033a 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/ObfuscatorUnitTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/ObfuscatorUnitTest.java @@ -42,14 +42,13 @@ public void testObfuscate() throws Exception { final IRandom generator = new RandomWrapper(0); for (File originalShaderJobFile : Util.getReferenceShaderJobFiles100es(fileOps)) { final File originalImage = - Util.renderShader( - originalShaderJobFile, temporaryFolder, fileOps); + Util.validateAndGetImage(originalShaderJobFile, temporaryFolder, fileOps); final ShaderJob shaderJob = fileOps.readShaderJobFile(originalShaderJobFile); final ShaderJob obfuscated = Obfuscator.obfuscate(shaderJob, generator); final File obfuscatedImage = Util.validateAndGetImage( obfuscated, - originalShaderJobFile.getName() + ".obfuscated.frag", + originalShaderJobFile.getName() + ".obfuscated.json", temporaryFolder, fileOps); assertTrue(FileUtils.contentEquals(originalImage, obfuscatedImage)); diff --git a/tester/src/test/java/com/graphicsfuzz/tester/ReducerUnitTest.java b/tester/src/test/java/com/graphicsfuzz/tester/ReducerUnitTest.java index e079529ce..0cb603c17 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/ReducerUnitTest.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/ReducerUnitTest.java @@ -16,8 +16,6 @@ package com.graphicsfuzz.tester; -import static org.junit.Assert.assertEquals; - import com.graphicsfuzz.common.ast.TranslationUnit; import com.graphicsfuzz.common.ast.expr.BinOp; import com.graphicsfuzz.common.ast.expr.BinaryExpr; @@ -26,32 +24,28 @@ import com.graphicsfuzz.common.ast.expr.VariableIdentifierExpr; import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.tool.PrettyPrinterVisitor; -import com.graphicsfuzz.common.util.GlslParserException; -import com.graphicsfuzz.common.util.PipelineInfo; -import com.graphicsfuzz.reducer.tool.GlslReduce; -import com.graphicsfuzz.util.Constants; import com.graphicsfuzz.common.transformreduce.GlslShaderJob; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.util.ShaderJobFileOperations; -import com.graphicsfuzz.util.ExecHelper.RedirectType; -import com.graphicsfuzz.util.ExecResult; +import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.IRandom; import com.graphicsfuzz.common.util.IdGenerator; import com.graphicsfuzz.common.util.ParseHelper; import com.graphicsfuzz.common.util.ParseTimeoutException; +import com.graphicsfuzz.common.util.PipelineInfo; import com.graphicsfuzz.common.util.RandomWrapper; +import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.common.util.ShaderKind; -import com.graphicsfuzz.util.ToolHelper; +import com.graphicsfuzz.common.util.StatsVisitor; import com.graphicsfuzz.generator.tool.Generate; -import com.graphicsfuzz.generator.transformation.ITransformation; import com.graphicsfuzz.generator.transformation.AddDeadOutputWriteTransformation; import com.graphicsfuzz.generator.transformation.AddJumpTransformation; import com.graphicsfuzz.generator.transformation.AddLiveOutputWriteTransformation; -import com.graphicsfuzz.generator.transformation.SplitForLoopTransformation; import com.graphicsfuzz.generator.transformation.DonateDeadCodeTransformation; import com.graphicsfuzz.generator.transformation.DonateLiveCodeTransformation; +import com.graphicsfuzz.generator.transformation.ITransformation; import com.graphicsfuzz.generator.transformation.IdentityTransformation; import com.graphicsfuzz.generator.transformation.OutlineStatementTransformation; +import com.graphicsfuzz.generator.transformation.SplitForLoopTransformation; import com.graphicsfuzz.generator.util.GenerationParams; import com.graphicsfuzz.generator.util.TransformationProbabilities; import com.graphicsfuzz.reducer.CheckAstFeatureVisitor; @@ -59,9 +53,14 @@ import com.graphicsfuzz.reducer.IFileJudge; import com.graphicsfuzz.reducer.ReductionDriver; import com.graphicsfuzz.reducer.reductionopportunities.IReductionOpportunity; -import com.graphicsfuzz.reducer.reductionopportunities.ReductionOpportunities; import com.graphicsfuzz.reducer.reductionopportunities.ReducerContext; +import com.graphicsfuzz.reducer.reductionopportunities.ReductionOpportunities; +import com.graphicsfuzz.reducer.tool.GlslReduce; import com.graphicsfuzz.reducer.tool.RandomFileJudge; +import com.graphicsfuzz.util.Constants; +import com.graphicsfuzz.util.ExecHelper.RedirectType; +import com.graphicsfuzz.util.ExecResult; +import com.graphicsfuzz.util.ToolHelper; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -82,6 +81,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.junit.Assert.assertEquals; + public class ReducerUnitTest { private static final Logger LOGGER = LoggerFactory.getLogger(ReducerUnitTest.class); @@ -106,29 +107,29 @@ private void testGenerateAndReduce(File originalShaderJobFile, List transformations, RandomWrapper generator) throws Exception { - // TODO: We reduced this due to slow shader run time. Consider reverting to 100000 when run - // time is improved. - final int maxBytes = 10000; + final int maxAstNodes = 2000; final File referenceImage = - Util.renderShader( - originalShaderJobFile, + Util.validateAndGetImage(originalShaderJobFile, temporaryFolder, fileOps); - final PipelineInfo pipelineInfo = new PipelineInfo(originalShaderJobFile); - final TranslationUnit tu = generateSizeLimitedShader( - fileOps.getUnderlyingShaderFile(originalShaderJobFile, ShaderKind.FRAGMENT), - transformations, generator, maxBytes); - Generate.addInjectionSwitchIfNotPresent(tu); + final ShaderJob shaderJob = generateSizeLimitedShader(originalShaderJobFile, + transformations, generator, maxAstNodes); + final PipelineInfo pipelineInfo = shaderJob.getPipelineInfo(); + final TranslationUnit fragmentShader = shaderJob.getFragmentShader().get(); + Generate.addInjectionSwitchIfNotPresent(fragmentShader); Generate.setInjectionSwitch(pipelineInfo); - Generate.randomiseUnsetUniforms(tu, pipelineInfo, generator); + Generate.randomiseUnsetUniforms(fragmentShader, + pipelineInfo, + generator); final IdGenerator idGenerator = new IdGenerator(); for (int step = 0; step < 10; step++) { List ops = ReductionOpportunities.getReductionOpportunities( - new GlslShaderJob(Optional.empty(), new PipelineInfo(), tu), - new ReducerContext(false, tu.getShadingLanguageVersion(), generator, idGenerator, true), + new GlslShaderJob(Optional.empty(), pipelineInfo, + fragmentShader), + new ReducerContext(false, fragmentShader.getShadingLanguageVersion(), generator, idGenerator), fileOps); if (ops.isEmpty()) { break; @@ -136,12 +137,10 @@ private void testGenerateAndReduce(File originalShaderJobFile, LOGGER.info("Step: {}; ops: {}", step, ops.size()); ops.get(generator.nextInt(ops.size())).applyReduction(); - final ShaderJob shaderJob = new GlslShaderJob(Optional.empty(), pipelineInfo, tu); - final File variantImage = Util.validateAndGetImage( shaderJob, - originalShaderJobFile.getName() + "_reduced_" + step + ".frag", + originalShaderJobFile.getName() + "_reduced_" + step + ".json", temporaryFolder, fileOps); Util.assertImagesSimilar(referenceImage, variantImage); @@ -149,31 +148,24 @@ private void testGenerateAndReduce(File originalShaderJobFile, } - private TranslationUnit generateSizeLimitedShader( - File fragmentShader, + private ShaderJob generateSizeLimitedShader( + File shaderJobFile, List transformations, IRandom generator, - final int maxBytes + final int maxAstNodes ) throws IOException, ParseTimeoutException, InterruptedException, GlslParserException { while (true) { List transformationsCopy = new ArrayList<>(); transformationsCopy.addAll(transformations); - final TranslationUnit tu = ParseHelper.parse(fragmentShader); + final ShaderJob shaderJob = fileOps.readShaderJobFile(shaderJobFile); + final TranslationUnit fragmentShader = shaderJob.getFragmentShader().get(); for (int i = 0; i < 4; i++) { getTransformation(transformationsCopy, generator).apply( - tu, TransformationProbabilities.DEFAULT_PROBABILITIES, + fragmentShader, TransformationProbabilities.DEFAULT_PROBABILITIES, generator, GenerationParams.normal(ShaderKind.FRAGMENT, true)); } - File tempFile = temporaryFolder.newFile(); - PrettyPrinterVisitor.emitShader(tu, Optional.empty(), - new PrintStream( - new FileOutputStream(tempFile)), - PrettyPrinterVisitor.DEFAULT_INDENTATION_WIDTH, - PrettyPrinterVisitor.DEFAULT_NEWLINE_SUPPLIER, - true - ); - if (tempFile.length() <= maxBytes) { - return tu; + if (new StatsVisitor(fragmentShader).getNumNodes() <= maxAstNodes) { + return shaderJob; } } } @@ -228,7 +220,7 @@ public void reduceRepeatedly(String shaderJobFileName, int numIterations, new ReductionDriver(new ReducerContext(false, shadingLanguageVersion, generator, - new IdGenerator(), true), false, fileOps, + new IdGenerator()), false, fileOps, new RandomFileJudge(generator, threshold, throwExceptionOnInvalid, fileOps), workDir) .doReduction(initialState, shaderJobShortName, 0, @@ -392,7 +384,7 @@ private String runReductionOnShader(File shaderJobFile, IFileJudge fileJudge) fileOps.copyShaderJobFileTo(shaderJobFile, new File(temporaryFolder.getRoot(), shaderJobFile.getName()), false); return new ReductionDriver(new ReducerContext(false, version, generator, - new IdGenerator(), true), false, fileOps, + new IdGenerator()), false, fileOps, fileJudge, temporaryFolder.getRoot()) .doReduction(state, shaderJobShortName, 0, -1); } diff --git a/tester/src/test/java/com/graphicsfuzz/tester/Util.java b/tester/src/test/java/com/graphicsfuzz/tester/Util.java index b382541f8..2ce2b9566 100755 --- a/tester/src/test/java/com/graphicsfuzz/tester/Util.java +++ b/tester/src/test/java/com/graphicsfuzz/tester/Util.java @@ -16,13 +16,8 @@ package com.graphicsfuzz.tester; -import com.graphicsfuzz.common.ast.TranslationUnit; -import com.graphicsfuzz.common.glslversion.ShadingLanguageVersion; import com.graphicsfuzz.common.transformreduce.ShaderJob; -import com.graphicsfuzz.common.util.FileHelper; -import com.graphicsfuzz.common.util.GlslParserException; import com.graphicsfuzz.common.util.ImageUtil; -import com.graphicsfuzz.common.util.ParseTimeoutException; import com.graphicsfuzz.common.util.ShaderJobFileOperations; import com.graphicsfuzz.common.util.ShaderKind; import com.graphicsfuzz.util.ExecHelper.RedirectType; @@ -33,10 +28,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Paths; -import java.util.Optional; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.junit.Assert; import org.junit.rules.TemporaryFolder; import static org.junit.Assert.assertEquals; @@ -48,99 +39,37 @@ private Util() { // Utility class } - static File renderShader(File originalShader, - TemporaryFolder temporaryFolder, ShaderJobFileOperations fileOps) - throws IOException, InterruptedException, ParseTimeoutException, GlslParserException { - final ShaderJob shaderJob = fileOps.readShaderJobFile(originalShader); - - return validateAndGetImage( - shaderJob, - originalShader.getName() + ".reference.frag", - temporaryFolder, - fileOps); - } - static File validateAndGetImage( - File shaderFile, + File shaderJobFile, TemporaryFolder temporaryFolder, ShaderJobFileOperations fileOps) throws IOException, InterruptedException { - - validate(shaderFile); - return getImage(shaderFile, temporaryFolder, fileOps); - } - - static File getImage( - File shaderFile, - TemporaryFolder temporaryFolder, - ShaderJobFileOperations fileOps) throws IOException, InterruptedException { - - final Optional shaderTranslatorArg = getShaderTranslatorArg(shaderFile, fileOps); - if (shaderTranslatorArg.isPresent()) { - final ExecResult shaderTranslatorResult = ToolHelper - .runShaderTranslatorOnShader(RedirectType.TO_LOG, shaderFile, - shaderTranslatorArg.get()); - assertEquals(0, shaderTranslatorResult.res); - } - return getImageUsingSwiftshader(shaderFile, temporaryFolder); - } - - static void validate(File shaderFile) throws IOException, InterruptedException { - final ExecResult validatorResult = ToolHelper - .runValidatorOnShader(RedirectType.TO_LOG, shaderFile); - assertEquals(validatorResult.res, 0); - } - - private static Optional getShaderTranslatorArg( - File shaderFile, - ShaderJobFileOperations fileOps) throws IOException { - Optional shaderTranslatorArgs; - - // Using fileOps here, even though the rest of the code does not use it yet. - Assert.assertTrue(shaderFile.getName().endsWith(".frag")); - final File shaderJobFile = FileHelper.replaceExtension(shaderFile, ".json"); - - final ShadingLanguageVersion shadingLanguageVersionFromShader = - ShadingLanguageVersion.getGlslVersionFromFirstTwoLines( - fileOps.getFirstTwoLinesOfShader(shaderJobFile, ShaderKind.FRAGMENT)); - // TODO: Use fileOps. - - if (shadingLanguageVersionFromShader == ShadingLanguageVersion.ESSL_300 - || shadingLanguageVersionFromShader == ShadingLanguageVersion.WEBGL2_SL) { - shaderTranslatorArgs = Optional.of("-s=w2"); - } else if(shadingLanguageVersionFromShader == ShadingLanguageVersion.WEBGL_SL) { - shaderTranslatorArgs = Optional.of("-s=w"); - } else { - shaderTranslatorArgs = Optional.empty(); - } - return shaderTranslatorArgs; + assertTrue(fileOps.areShadersValid(shaderJobFile, false)); + assertTrue(fileOps.areShadersValidShaderTranslator(shaderJobFile, false)); + return getImage(shaderJobFile, temporaryFolder, fileOps); } static File validateAndGetImage( ShaderJob shaderJob, - String fileName, + String shaderJobFilename, TemporaryFolder temporaryFolder, ShaderJobFileOperations fileOps) throws IOException, InterruptedException { - - final File shaderJobFileOutput = new File( + final File shaderJobFile = new File( temporaryFolder.getRoot(), - FilenameUtils.removeExtension(fileName) + ".json"); - - fileOps.writeShaderJobFile(shaderJob, shaderJobFileOutput); - - // TODO: Use fileOps more. - - final File tempFile = FileHelper.replaceExtension(shaderJobFileOutput, ".frag"); - - return validateAndGetImage(tempFile, temporaryFolder, fileOps); + shaderJobFilename); + fileOps.writeShaderJobFile(shaderJob, shaderJobFile); + return validateAndGetImage(shaderJobFile, temporaryFolder, fileOps); } - static File getImageUsingSwiftshader(File shaderFile, TemporaryFolder temporaryFolder) throws IOException, InterruptedException { + static File getImage( + File shaderJobFile, + TemporaryFolder temporaryFolder, + ShaderJobFileOperations fileOps) throws IOException, InterruptedException { File imageFile = temporaryFolder.newFile(); ExecResult res = ToolHelper.runSwiftshaderOnShader(RedirectType.TO_LOG, - shaderFile, + fileOps.getUnderlyingShaderFile(shaderJobFile, ShaderKind.FRAGMENT), imageFile, false, 32, diff --git a/build/licenses/android-support-libraries.txt b/third_party/licenses/android-support-libraries.txt similarity index 100% rename from build/licenses/android-support-libraries.txt rename to third_party/licenses/android-support-libraries.txt diff --git a/build/licenses/elg-headers.txt b/third_party/licenses/elg-headers.txt similarity index 100% rename from build/licenses/elg-headers.txt rename to third_party/licenses/elg-headers.txt diff --git a/build/licenses/javacpp.txt b/third_party/licenses/javacpp.txt similarity index 100% rename from build/licenses/javacpp.txt rename to third_party/licenses/javacpp.txt diff --git a/build/licenses/libpng-LICENSE.txt b/third_party/licenses/libpng-LICENSE.txt similarity index 100% rename from build/licenses/libpng-LICENSE.txt rename to third_party/licenses/libpng-LICENSE.txt diff --git a/build/licenses/max-sills-shaders.txt b/third_party/licenses/max-sills-shaders.txt similarity index 100% rename from build/licenses/max-sills-shaders.txt rename to third_party/licenses/max-sills-shaders.txt diff --git a/build/licenses/minigbm.txt b/third_party/licenses/minigbm.txt similarity index 100% rename from build/licenses/minigbm.txt rename to third_party/licenses/minigbm.txt diff --git a/build/licenses/opengl-headers.txt b/third_party/licenses/opengl-headers.txt similarity index 100% rename from build/licenses/opengl-headers.txt rename to third_party/licenses/opengl-headers.txt diff --git a/build/licenses/valters-mednis-shaders.txt b/third_party/licenses/valters-mednis-shaders.txt similarity index 100% rename from build/licenses/valters-mednis-shaders.txt rename to third_party/licenses/valters-mednis-shaders.txt diff --git a/build/licenses/zt-process-killer.txt b/third_party/licenses/zt-process-killer.txt similarity index 100% rename from build/licenses/zt-process-killer.txt rename to third_party/licenses/zt-process-killer.txt diff --git a/util/pom.xml b/util/pom.xml index 30d88e8de..f1d84c2c8 100644 --- a/util/pom.xml +++ b/util/pom.xml @@ -43,6 +43,10 @@ limitations under the License. commons-io commons-io + + org.apache.commons + commons-rng-simple + diff --git a/util/src/main/java/com/graphicsfuzz/util/ArgsUtil.java b/util/src/main/java/com/graphicsfuzz/util/ArgsUtil.java index 13a6ea726..a3616b84e 100644 --- a/util/src/main/java/com/graphicsfuzz/util/ArgsUtil.java +++ b/util/src/main/java/com/graphicsfuzz/util/ArgsUtil.java @@ -16,15 +16,15 @@ package com.graphicsfuzz.util; -import java.util.Random; import net.sourceforge.argparse4j.inf.Namespace; +import org.apache.commons.rng.simple.internal.SeedFactory; public final class ArgsUtil { - public static int getSeedArgument(Namespace ns) { - Integer seed = ns.get("seed"); + public static long getSeedArgument(Namespace ns) { + String seed = ns.getString("seed"); if (seed == null) { - seed = new Random().nextInt(); + return SeedFactory.createLong(); } - return seed; + return Long.parseUnsignedLong(seed); } } diff --git a/util/src/main/java/com/graphicsfuzz/util/Constants.java b/util/src/main/java/com/graphicsfuzz/util/Constants.java index 2336a9679..e8987d4bc 100755 --- a/util/src/main/java/com/graphicsfuzz/util/Constants.java +++ b/util/src/main/java/com/graphicsfuzz/util/Constants.java @@ -45,6 +45,19 @@ private Constants() { public static final String GLF_IDENTITY = "_GLF_IDENTITY"; public static final String GLF_SWITCH = "_GLF_SWITCH"; + // Macro names used to make array accesses in bounds + public static final String GLF_MAKE_IN_BOUNDS_INT = "_GLF_MAKE_IN_BOUNDS_INT"; + public static final String GLF_MAKE_IN_BOUNDS_UINT = "_GLF_MAKE_IN_BOUNDS_UINT"; + + public static final String GLF_PRIMITIVE = "_GLF_PRIMITIVE"; + public static final String GLF_PRIMITIVE_GLOBAL = "_GLF_PRIMITIVE_GLOBAL"; + public static final String GLF_COMPUTE = "_GLF_COMPUTE"; + public static final String GLF_PARAM = "_GLF_PARAM"; + public static final String GLF_UNKNOWN_PARAM = "GLF_UNKNOWN_PARAM"; + public static final String GLF_INIT_GLOBALS = "_GLF_init_globals"; + public static final String GLF_UNIFORM = "_GLF_UNIFORM"; + + public static final String INJECTED_LOOP_COUNTER = "_injected_loop_counter"; public static final String INJECTION_SWITCH = "injectionSwitch"; diff --git a/vulkan-worker/samples/shader.vert b/vulkan-worker/samples/shader.vert index 28cbd521e..143937fbf 100644 --- a/vulkan-worker/samples/shader.vert +++ b/vulkan-worker/samples/shader.vert @@ -20,7 +20,6 @@ layout (std140, binding = 0) uniform bufferVals { float f; } myBufferVals; layout (location = 0) in vec4 pos; -layout (location = 1) in vec4 inColor; void main() { gl_Position = pos; } diff --git a/vulkan-worker/samples/shader.vert.spv b/vulkan-worker/samples/shader.vert.spv index 3ce34d2ee..298b0718b 100644 Binary files a/vulkan-worker/samples/shader.vert.spv and b/vulkan-worker/samples/shader.vert.spv differ
Worker", "", "reference", "", "", - f.getName().replace(".frag", ""), "
", worker, "", - ""); + "'>"); + if (shaderFamily.isCompute) { + htmlAppendLn("COMPUTE"); + + } else { + htmlAppendLn(""); + } + htmlAppendLn(""); } else { htmlAppendLn("", "No result yet
Variant is identical to reference
Variant is significantly different from reference
Variant is similar but not identical to reference
Rendering the variant led to an error
Variant leads to non-deterministic rendering
The image comparison metrics used to compare variant and reference ", "disagree on whether they are different or not